XQuery syntax highlighting improvements.

2013-01-25 12:10:34 +01:00 · 2013-01-25 12:10:34 +01:00 · 751f45d2db
commit 751f45d2db
parent c1f7c51137
8 changed files with 237 additions and 119 deletions
--- a/lib/ace/mode/xquery.js
+++ b/lib/ace/mode/xquery.js
@ -131,12 +131,14 @@ oop.inherits(Mode, TextMode);
        });
        
        worker.on("ok", function(e) {
-            session.clearAnnotations();
+          session.clearAnnotations();
        });
        
        worker.on("highlight", function(tokens) {
-          if(that.$deltas.length > 0) return;
-          that.$tokenizer.tokens = tokens.data;
+          //console.log(that.$deltas);
+          //if(that.$deltas.length > 0) return;
+          that.$tokenizer.tokens = tokens.data.tokens;
+          that.$tokenizer.lines  = tokens.data.lines;
          session.bgTokenizer.lines = [];
          session.bgTokenizer.states = [];
          session.bgTokenizer.fireUpdateEvent(0, session.getLength() - 1);
--- a/lib/ace/mode/xquery/JSONParseTreeHandler.js
+++ b/lib/ace/mode/xquery/JSONParseTreeHandler.js
@ -32,8 +32,14 @@ define(function(require, exports, module){

  var JSONParseTreeHandler = exports.JSONParseTreeHandler = function(code) {

-    //List of nodes that are left untouched in the parse tree size optimization.
-    var blacklist = ["VarDeclStatement"];
+    //List of nodes that are not targeted by the parse tree size optimization.
+    var list = [
+        "OrExpr", "AndExpr", "ComparisonExpr", "StringConcatExpr", "RangeExpr"
+        //, "AdditiveExpr", "MultiplicativeExpr"
+        , "UnionExpr", "IntersectExceptExpr", "InstanceofExpr", "TreatExpr", "CastableExpr"
+        , "CastExpr", "UnaryExpr", "ValueExpr", "FTContainsExpr", "SimpleMapExpr", "PathExpr", "RelativePathExpr"
+        , "PostfixExpr", "StepExpr"
+    ];
    
    var ast = null;
    var ptr = null;
@ -63,13 +69,31 @@ define(function(require, exports, module){
     
      if(ptr.children.length > 0) {
        var s = ptr.children[0];
-        var e = ptr.children[ptr.children.length - 1];
+        var e = null;
+        //We want to skip empty non terminals. For instance PredicateList:
+        // [108] AxisStep ::= (ReverseStep | ForwardStep) PredicateList
+        // [120] PredicateList ::= Predicate*
+        for(var i= ptr.children.length - 1; i >= 0;i--) {
+          e = ptr.children[i];
+          if(e.pos.el !== 0 || e.pos.ec !== 0) {
+            break;
+          }
+        }
        ptr.pos.sl = s.pos.sl;
        ptr.pos.sc = s.pos.sc;
        ptr.pos.el = e.pos.el;
        ptr.pos.ec = e.pos.ec;
      }
      
+      //Normalize EQName && FunctionName
+      if(ptr.name === "FunctionName") {
+        ptr.name = "EQName";  
+      }
+      if(ptr.name === "EQName" && ptr.value === undefined) {
+        ptr.value = ptr.children[0].value;
+        ptr.children.pop();
+      }
+      
      if(ptr.getParent !== null) {
        ptr = ptr.getParent;
        //for(var i in ptr.children) {
@ -80,12 +104,12 @@ define(function(require, exports, module){
      }

      //Parse tree size optimization
-      //if(ptr.children.length > 0) {
-      //  var lastChild = ptr.children[ptr.children.length - 1];
-      //  if(lastChild.children.length === 1 && blacklist.indexOf(lastChild.name) !== -1) {
-      //    ptr.children[ptr.children.length - 1] = lastChild.children[0];
-      //  }
-      //}
+      if(ptr.children.length > 0) {
+        var lastChild = ptr.children[ptr.children.length - 1];
+        if(lastChild.children.length === 1 && list.indexOf(lastChild.name) !== -1) {
+          ptr.children[ptr.children.length - 1] = lastChild.children[0];
+        }
+      }
    }
 
    this.peek = function() {
--- a/lib/ace/mode/xquery/XQueryLexer.js
+++ b/lib/ace/mode/xquery/XQueryLexer.js
@ -242,7 +242,7 @@ exports.XQueryLexer = function() {
      
      } catch(e) {
        if(e instanceof tokenizer.ParseException) {
-          var message = tokenizer.getErrorMessage(e);
+          //var message = tokenizer.getErrorMessage(e);
          //console.log(stack[stack.length - 1]);
          //console.log(line);
          //console.log(line.substring(e.getBegin(), e.getEnd()));
@ -264,33 +264,34 @@ exports.XQueryLexer = function() {
      }
    }
   
+    
    if(this.tokens[row] !== undefined) {
+      var cachedLine = this.lines[row];
+      var begin = sharedStart([line, cachedLine]);
+      var diff = cachedLine.length - line.length;
      var idx = 0;
      var col = 0;
      for(var i in tokens) {
        var token = tokens[i];
-        if(token.type === "keyword") {
          for(var j in this.tokens[row]) {
            var semanticToken = this.tokens[row][j];
-            if(semanticToken.sc === col) {
+            if(
+               ((col + token.value.length) <= begin.length && semanticToken.sc === col && semanticToken.ec === (col + token.value.length)) ||
+               (semanticToken.sc === (col + diff) && semanticToken.ec === (col + token.value.length + diff))
+              ) {
              idx = i;
-              tokens[i].type = "support.function";
+              tokens[i].type = semanticToken.type;
            }
          }
-        }
        col += token.value.length;
      }
-      //for(var i = tokens.length - 1; i > idx; i--) {
-      //  var token = tokens[i];
-      //  if(token.type === "keyword") {
-      //    for(var j in this.tokens[row]) {
-      //      var semanticToken = this.tokens[row][j];
-      //      if(semanticToken.ec === endCol) {
-      //        tokens[i].type = "support.function";
-      //      }
-      //    }
-      //  }
-      //}
+      console.log("======");
+      console.log(sharedStart([line, this.lines[row]]));
+      console.log(this.lines[row]);
+      console.log(line);
+      console.log(line.length - this.lines[row].length);
+      console.log("======");
+      
    }

    return {
@ -298,5 +299,16 @@ exports.XQueryLexer = function() {
      state: JSON.stringify(stack)
    };
  };
+  
+  function sharedStart(A) {
+    var tem1, tem2, s, A = A.slice(0).sort();
+    tem1 = A[0];
+    s = tem1.length;
+    tem2 = A.pop();
+    while(s && tem2.indexOf(tem1) == -1) {
+        tem1 = tem1.substring(0, --s);
+    }
+    return tem1;
+  }
 };
 });
--- a/lib/ace/mode/xquery/XQueryParser.js
+++ b/lib/ace/mode/xquery/XQueryParser.js
@ -1,4 +1,4 @@
-// This file was generated on Wed Dec 12, 2012 12:21 (UTC+01) by REx v5.20 which is Copyright (c) 1979-2012 by Gunther Rademacher <grd@gmx.net>
+// This file was generated on Sat Jan 5, 2013 17:36 (UTC+01) by REx v5.21 which is Copyright (c) 1979-2012 by Gunther Rademacher <grd@gmx.net>
 // REx command line: XQueryParser.ebnf -ll 2 -backtrack -tree -javascript -a xqlint

                                                            // line 2 "XQueryParser.ebnf"
@ -84,6 +84,8 @@
    eventHandler.reset(input);
  }

+  this.reset = function(l, b, e) {reset(l, b, e);};
+
  this.getOffendingToken = function(e)
  {
    var o = e.getOffending();
@ -7896,16 +7898,17 @@
                                    // 'union' | 'unordered' | 'updating' | 'validate' | 'value' | 'variable' |
                                    // 'version' | 'where' | 'while' | 'with' | 'xquery' | '{' | '{|'
            try_StepExpr();
-            lk = -1;
+            memoize(2, e0A, -1);
+            continue;
          }
          catch (p1A)
          {
-            lk = -2;
+            b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+            b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+            b2 = b2A; e2 = e2A; end = e2A; }}
+            memoize(2, e0A, -2);
+            break;
          }
-          b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-          b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-          b2 = b2A; e2 = e2A; end = e2A; }}
-          memoize(2, e0, lk);
        }
      }
      if (lk != -1
@ -9466,16 +9469,17 @@
        try
        {
          try_PostfixExpr();
-          lk = -1;
+          memoize(3, e0A, -1);
+          lk = -3;
        }
        catch (p1A)
        {
          lk = -2;
+          b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+          b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+          b2 = b2A; e2 = e2A; end = e2A; }}
+          memoize(3, e0A, -2);
        }
-        b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-        b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-        b2 = b2A; e2 = e2A; end = e2A; }}
-        memoize(3, e0, lk);
      }
    }
    switch (lk)
@ -10081,6 +10085,8 @@
    case 141568:                    // 'unordered' '{'
      try_PostfixExpr();
      break;
+    case -3:
+      break;
    default:
      try_AxisStep();
    }
@ -12937,16 +12943,17 @@
        try
        {
          try_KindTest();
-          lk = -1;
+          memoize(4, e0A, -1);
+          lk = -8;
        }
        catch (p1A)
        {
          lk = -6;
+          b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+          b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+          b2 = b2A; e2 = e2A; end = e2A; }}
+          memoize(4, e0A, -6);
        }
-        b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-        b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-        b2 = b2A; e2 = e2A; end = e2A; }}
-        memoize(4, e0, lk);
      }
    }
    switch (lk)
@ -12984,6 +12991,8 @@
    case 242:                       // 'structured-item'
      try_StructuredItemTest();
      break;
+    case -8:
+      break;
    default:
      try_AtomicOrUnionType();
    }
@ -14004,22 +14013,25 @@
      try
      {
        try_AnyFunctionTest();
-        lk = -1;
+        memoize(5, e0A, -1);
+        lk = -3;
      }
      catch (p1A)
      {
        lk = -2;
+        b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+        b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+        b2 = b2A; e2 = e2A; end = e2A; }}
+        memoize(5, e0A, -2);
      }
-      b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-      b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-      b2 = b2A; e2 = e2A; end = e2A; }}
-      memoize(5, e0, lk);
    }
    switch (lk)
    {
    case -1:
      try_AnyFunctionTest();
      break;
+    case -3:
+      break;
    default:
      try_TypedFunctionTest();
    }
@ -22135,16 +22147,17 @@
          try
          {
            try_Statement();
-            lk = -1;
+            memoize(6, e0A, -1);
+            continue;
          }
          catch (p1A)
          {
-            lk = -2;
+            b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+            b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+            b2 = b2A; e2 = e2A; end = e2A; }}
+            memoize(6, e0A, -2);
+            break;
          }
-          b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-          b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-          b2 = b2A; e2 = e2A; end = e2A; }}
-          memoize(6, e0, lk);
        }
      }
      if (lk != -1
@ -23499,7 +23512,8 @@
        try
        {
          try_ApplyStatement();
-          lk = -1;
+          memoize(7, e0A, -1);
+          lk = -14;
        }
        catch (p1A)
        {
@ -23509,7 +23523,8 @@
            b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
            b2 = b2A; e2 = e2A; end = e2A; }}
            try_AssignStatement();
-            lk = -2;
+            memoize(7, e0A, -2);
+            lk = -14;
          }
          catch (p2A)
          {
@ -23519,7 +23534,8 @@
              b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
              b2 = b2A; e2 = e2A; end = e2A; }}
              try_BlockStatement();
-              lk = -3;
+              memoize(7, e0A, -3);
+              lk = -14;
            }
            catch (p3A)
            {
@ -23529,19 +23545,20 @@
                b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
                b2 = b2A; e2 = e2A; end = e2A; }}
                try_VarDeclStatement();
-                lk = -12;
+                memoize(7, e0A, -12);
+                lk = -14;
              }
              catch (p12A)
              {
                lk = -13;
+                b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+                b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+                b2 = b2A; e2 = e2A; end = e2A; }}
+                memoize(7, e0A, -13);
              }
            }
          }
        }
-        b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-        b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-        b2 = b2A; e2 = e2A; end = e2A; }}
-        memoize(7, e0, lk);
      }
    }
    switch (lk)
@ -23587,6 +23604,8 @@
    case -13:
      try_WhileStatement();
      break;
+    case -14:
+      break;
    default:
      try_ApplyStatement();
    }
@ -24728,11 +24747,9 @@
    shiftT(250);                    // 'try'
    lookahead1W(87);                // S^WS | '(:' | '{'
    try_BlockStatement();
-    for (;;)
-    {
-      lookahead1W(36);              // S^WS | '(:' | 'catch'
-      shiftT(91);                   // 'catch'
-      lookahead1W(251);             // Wildcard | EQName^Token | S^WS | '(:' | 'after' | 'allowing' | 'ancestor' |
+    lookahead1W(36);                // S^WS | '(:' | 'catch'
+    shiftT(91);                     // 'catch'
+    lookahead1W(251);               // Wildcard | EQName^Token | S^WS | '(:' | 'after' | 'allowing' | 'ancestor' |
                                    // 'ancestor-or-self' | 'and' | 'as' | 'ascending' | 'at' | 'attribute' |
                                    // 'base-uri' | 'before' | 'boundary-space' | 'break' | 'case' | 'cast' |
                                    // 'castable' | 'catch' | 'child' | 'collation' | 'comment' | 'constraint' |
@ -24753,8 +24770,10 @@
                                    // 'try' | 'tumbling' | 'type' | 'typeswitch' | 'union' | 'unordered' | 'updating' |
                                    // 'validate' | 'value' | 'variable' | 'version' | 'where' | 'while' | 'with' |
                                    // 'xquery'
-      try_CatchErrorList();
-      try_BlockStatement();
+    try_CatchErrorList();
+    try_BlockStatement();
+    for (;;)
+    {
      lookahead1W(278);             // Wildcard | EQName^Token | IntegerLiteral | DecimalLiteral | DoubleLiteral |
                                    // StringLiteral | S^WS | EOF | '$' | '%' | '(' | '(#' | '(:' | '+' | '-' | '.' |
                                    // '..' | '/' | '//' | '<' | '<!--' | '<?' | '@' | '[' | 'after' | 'allowing' |
@ -24863,16 +24882,17 @@
                                    // 'xquery'
            try_CatchErrorList();
            try_BlockStatement();
-            lk = -1;
+            memoize(8, e0A, -1);
+            continue;
          }
          catch (p1A)
          {
-            lk = -2;
+            b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+            b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+            b2 = b2A; e2 = e2A; end = e2A; }}
+            memoize(8, e0A, -2);
+            break;
          }
-          b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-          b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-          b2 = b2A; e2 = e2A; end = e2A; }}
-          memoize(8, e0, lk);
        }
      }
      if (lk != -1
@ -24989,6 +25009,31 @@
      {
        break;
      }
+      lookahead1W(36);              // S^WS | '(:' | 'catch'
+      shiftT(91);                   // 'catch'
+      lookahead1W(251);             // Wildcard | EQName^Token | S^WS | '(:' | 'after' | 'allowing' | 'ancestor' |
+                                    // 'ancestor-or-self' | 'and' | 'as' | 'ascending' | 'at' | 'attribute' |
+                                    // 'base-uri' | 'before' | 'boundary-space' | 'break' | 'case' | 'cast' |
+                                    // 'castable' | 'catch' | 'child' | 'collation' | 'comment' | 'constraint' |
+                                    // 'construction' | 'context' | 'continue' | 'copy' | 'copy-namespaces' | 'count' |
+                                    // 'decimal-format' | 'declare' | 'default' | 'delete' | 'descendant' |
+                                    // 'descendant-or-self' | 'descending' | 'div' | 'document' | 'document-node' |
+                                    // 'element' | 'else' | 'empty' | 'empty-sequence' | 'encoding' | 'end' | 'eq' |
+                                    // 'every' | 'except' | 'exit' | 'external' | 'first' | 'following' |
+                                    // 'following-sibling' | 'for' | 'ft-option' | 'function' | 'ge' | 'group' | 'gt' |
+                                    // 'idiv' | 'if' | 'import' | 'in' | 'index' | 'insert' | 'instance' | 'integrity' |
+                                    // 'intersect' | 'into' | 'is' | 'item' | 'last' | 'lax' | 'le' | 'let' | 'loop' |
+                                    // 'lt' | 'mod' | 'modify' | 'module' | 'namespace' | 'namespace-node' | 'ne' |
+                                    // 'node' | 'nodes' | 'only' | 'option' | 'or' | 'order' | 'ordered' | 'ordering' |
+                                    // 'parent' | 'preceding' | 'preceding-sibling' | 'processing-instruction' |
+                                    // 'rename' | 'replace' | 'return' | 'returning' | 'revalidation' | 'satisfies' |
+                                    // 'schema' | 'schema-attribute' | 'schema-element' | 'score' | 'self' | 'sliding' |
+                                    // 'some' | 'stable' | 'start' | 'strict' | 'switch' | 'text' | 'to' | 'treat' |
+                                    // 'try' | 'tumbling' | 'type' | 'typeswitch' | 'union' | 'unordered' | 'updating' |
+                                    // 'validate' | 'value' | 'variable' | 'version' | 'where' | 'while' | 'with' |
+                                    // 'xquery'
+      try_CatchErrorList();
+      try_BlockStatement();
    }
  }

@ -26497,16 +26542,16 @@
                                    // 'validate' | 'value' | 'variable' | 'version' | 'where' | 'while' | 'with' |
                                    // 'xquery' | '{' | '{|'
          try_ExprSingle();
-          lk = -1;
+          memoize(9, e0A, -1);
        }
        catch (p1A)
        {
-          lk = -2;
+          b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+          b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+          b2 = b2A; e2 = e2A; end = e2A; }}
+          memoize(9, e0A, -2);
        }
-        b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-        b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-        b2 = b2A; e2 = e2A; end = e2A; }}
-        memoize(9, e0, lk);
+        lk = -2;
      }
    }
    if (lk == -1)
@ -27214,16 +27259,17 @@
          shiftT(276);              // '{'
          lookahead1W(88);          // S^WS | '(:' | '}'
          shiftT(282);              // '}'
-          lk = -1;
+          memoize(10, e0A, -1);
+          lk = -3;
        }
        catch (p1A)
        {
          lk = -2;
+          b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+          b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+          b2 = b2A; e2 = e2A; end = e2A; }}
+          memoize(10, e0A, -2);
        }
-        b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-        b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-        b2 = b2A; e2 = e2A; end = e2A; }}
-        memoize(10, e0, lk);
      }
    }
    switch (lk)
@ -27233,6 +27279,8 @@
      lookahead1W(88);              // S^WS | '(:' | '}'
      shiftT(282);                  // '}'
      break;
+    case -3:
+      break;
    default:
      try_BlockExpr();
    }
@ -27473,16 +27521,17 @@
          shiftT(276);              // '{'
          lookahead1W(88);          // S^WS | '(:' | '}'
          shiftT(282);              // '}'
-          lk = -1;
+          memoize(11, e0A, -1);
+          lk = -3;
        }
        catch (p1A)
        {
          lk = -2;
+          b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+          b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+          b2 = b2A; e2 = e2A; end = e2A; }}
+          memoize(11, e0A, -2);
        }
-        b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-        b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-        b2 = b2A; e2 = e2A; end = e2A; }}
-        memoize(11, e0, lk);
      }
    }
    switch (lk)
@ -27492,6 +27541,8 @@
      lookahead1W(88);              // S^WS | '(:' | '}'
      shiftT(282);                  // '}'
      break;
+    case -3:
+      break;
    default:
      try_BlockExpr();
    }
@ -28610,16 +28661,17 @@
        try
        {
          try_BlockExpr();
-          lk = -10;
+          memoize(12, e0A, -10);
+          lk = -14;
        }
        catch (p10A)
        {
          lk = -11;
+          b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
+          b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
+          b2 = b2A; e2 = e2A; end = e2A; }}
+          memoize(12, e0A, -11);
        }
-        b0 = b0A; e0 = e0A; l1 = l1A; if (l1 == 0) {end = e0A;} else {
-        b1 = b1A; e1 = e1A; l2 = l2A; if (l2 == 0) {end = e1A;} else {
-        b2 = b2A; e2 = e2A; end = e2A; }}
-        memoize(12, e0, lk);
      }
    }
    switch (lk)
@ -28904,6 +28956,8 @@
    case 278:                       // '{|'
      try_JSONSimpleObjectUnion();
      break;
+    case -14:
+      break;
    default:
      try_Constructor();
    }
@ -29427,8 +29481,8 @@
  var l1, b1, e1;
  var l2, b2, e2;
  var bx, ex, sx, lx, tx;
-  var memo;
  var eventHandler;
+  var memo;

  function memoize(i, e, v)
  {
@ -29633,7 +29687,7 @@
      end = current - 1;
      var c1 = end < size ? input.charCodeAt(end) : 0;
      if (c1 >= 0xdc00 && c1 < 0xe000) --end;
-      error(begin, end, state, -1, -1);
+      return error(begin, end, state, -1, -1);
    }

    if (nonbmp)
@ -33235,5 +33289,5 @@ XQueryParser.TOKEN =

                                                            // line 1174 "XQueryParser.ebnf"
                                                            });
-                                                            // line 33239 "XQueryParser.js"
+                                                            // line 33293 "XQueryParser.js"
 // End
--- a/lib/ace/mode/xquery/XQueryTokenizer.ebnf
+++ b/lib/ace/mode/xquery/XQueryTokenizer.ebnf
@ -47,6 +47,15 @@ start   ::= '<![CDATA['
          | "{"
          | "("
          | ")"
+          | "/"
+          | "["
+          | "]"
+          | ","
+          | "."
+          | Annotation
+          | ModuleDecl
+          | OptionDecl
+          | AttrTest
          | Wildcard
          | IntegerLiteral
          | DecimalLiteral
@ -59,6 +68,7 @@ start   ::= '<![CDATA['

 StartTag  ::= '>' | '/>' | QName | "=" | '"' | "'" | EOF

+
 TagContent
         ::= ElementContentChar | Tag | EndTag | '<![CDATA[' | '<!--' | PredefinedEntityRef | CharRef | '{{' | '}}' | '{' | EOF
          /* ws: explicit */
@ -84,7 +94,8 @@ PI       ::= DirPIContents | '?>' | EOF
 Pragma   ::= PragmaContents | '#)' | EOF
          /* ws: explicit */

-Comment  ::= CommentContents | ':)' | '(:' | EOF
+Comment  ::= ':)' | '(:' | CommentContents | EOF
+          /* ws: explicit */

 CommentDoc
         ::= DocTag | DocCommentContents | ':)' | '(:' | EOF
@ -98,6 +109,10 @@ AposString
         ::= PredefinedEntityRef | CharRef | EscapeApos | AposChar | "'" | EOF
          /* ws: explicit */

+Prefix  ::= NCName
+
+_EQName ::= EQName
+
 Whitespace
         ::= S^WS
          /* ws: definition */
@ -364,7 +379,17 @@ NCName   ::= NCName^Token
           | 'returning'
 <?TOKENS?>

-Operator ::= ':=' | '=' | '<' | '>' | '-' | '+' | 'div'
+ModuleDecl
+         ::= ('import' S)? ('module' | 'schema') S 'namespace'
+
+Annotation
+         ::= '%' EQName ?
+
+OptionDecl
+         ::= 'declare' S ( ( 'decimal-format' | 'option' ) 
+           |  ('default' S 'decimal-format') ) 
+
+Operator ::= '!=' | ':=' | '>=' | '<=' | '=' | '<' | '>' | '-' | '+' | 'div' | '||' | '?'

 Variable ::= '$' EQName

@ -378,12 +403,10 @@ DirCommentContents
         ::= ( ( Char - '-' ) | '-' ( Char - '-' ) )+
 DirPIContents
         ::= ( Char* - ( Char* '?' Char* ) ) 
-/*
-CDataSection
-         ::= '<![CDATA[' CDataSectionContents ']]>'
-*/
 CDataSectionContents
-         ::= Char* - ( Char* ']]>' Char* )
+         ::= ( Char+ - ( Char* ']]>' Char* ) ) & ']]'
+           | ( Char+ - ( Char* ']]>' Char* ) ) & $ 
+AttrTest  ::= "@" ( Wildcard | QName )
 Wildcard ::= "*"
         | (NCName ":" "*")
         | ("*" ":" NCName)
@ -442,14 +465,15 @@ CharRef  ::= '&#' [0-9]+ ';'
           | '&#x' [0-9A-Fa-f]+ ';'
 Digits   ::= [0-9]+
 CommentContents
-         ::= ( ( Char+ - ( Char* ( '(:' | ':)' ) Char* ) ) - ( Char* '(' ) ) &':'
-           |   ( Char+ - ( Char* ( '(:' | ':)' ) Char* ) )                   &'('
-DocTag   ::= ' @' NCName
+         ::= ( Char+ - ( Char* ( '(:' | ':)' ) Char* ) ) & '('
+           | ( Char+ - ( Char* ( '(:' | ':)' ) Char* ) ) & $
+           | ( ( Char+ - ( Char* ( '(:' | ':)' | ' @' ) Char* ) ) - ( Char* '(' ) ) & ':'
+DocTag   ::= ' @' NCName?
 DocCommentContents
-         ::= ( ( Char+ - ( Char* ( '(:' | ':)' | ' @' ) Char* ) ) - ( Char* '(' ) ) &':'
-           |   ( Char+ - ( Char* ( '(:' | ':)' | ' @' ) Char* ) )                   &'('
+         ::= ( ( Char+ - ( Char* ( '(:' | ':)' | ' @' ) Char* ) ) - ( Char* '(' ) ) & ':'
+           |   ( Char+ - ( Char* ( '(:' | ':)' | ' @' ) Char* ) )                   & '('
           |   ( Char+ - ( Char* ( '(:' | ':)' | ' @' ) Char* ) )                   & ' @'
-           |   ( Char+ - ( Char* ( '(:' | ':)' | ' @' ) Char* ) ) 
+           | ( Char+ - ( Char* ( '(:' | ':)' | ' @') Char* ) ) & $
 EOF      ::= $
 NonNCNameChar
          ::= $
--- a/lib/ace/mode/xquery/visitors/SemanticHighlighter.js
+++ b/lib/ace/mode/xquery/visitors/SemanticHighlighter.js
@ -30,8 +30,9 @@

 define(function(require, exports, module) {

-  var SemanticHighlighter = exports.SemanticHighlighter = function(ast) {
+  var SemanticHighlighter = exports.SemanticHighlighter = function(ast, code) {
    
+    this.lines = code.split("\n");
    this.tokens = new Array(ast.pos.el + 1);

    this.getTokens = function() {
@ -42,7 +43,8 @@ define(function(require, exports, module) {
   this.EQName = function(node)
   {
     var row = node.pos.sl;
-     this.tokens[row] = this.tokens[row] === undefined ? [] : this.tokens[row] ;
+     this.tokens[row] = this.tokens[row] === undefined ? [] : this.tokens[row];
+     node.pos.type = "support.function";
     this.tokens[row].push(node.pos);
     return true;
   };
--- a/lib/ace/mode/xquery/visitors/SyntaxHighlighter.js
+++ b/lib/ace/mode/xquery/visitors/SyntaxHighlighter.js
@ -33,7 +33,7 @@ define(function(require, exports, module){
 var CommentParser  = require("../CommentParser").CommentParser;
 var CommentHandler = require("../CommentHandler").CommentHandler; 
 
-var SyntaxHighlighter = exports.SyntaxHighlighter = function(tree)
+var SyntaxHighlighter = exports.SyntaxHighlighter = function(tree, source)
 {
   var keywords = ['after', 'ancestor', 'ancestor-or-self', 'and', 'as', 'ascending', 'attribute', 'before', 'case', 'cast', 'castable', 'child', 'collation', 'comment', 'copy', 'count', 'declare', 'default', 'delete', 'descendant', 'descendant-or-self', 'descending', 'div', 'document', 'document-node', 'element', 'else', 'empty', 'empty-sequence', 'end', 'eq', 'every', 'except', 'first', 'following', 'following-sibling', 'for', 'function', 'ge', 'group', 'gt', 'idiv', 'if', 'then', 'import', 'insert', 'instance', 'intersect', 'into', 'is', 'item', 'last', 'le', 'let', 'lt', 'mod', 'modify', 'module', 'namespace', 'namespace-node', 'ne', 'node', 'only', 'or', 'order', 'ordered', 'parent', 'preceding', 'preceding-sibling', 'processing-instruction', 'rename', 'replace', 'return', 'satisfies', 'schema-attribute', 'schema-element', 'self', 'some', 'stable', 'start', 'switch', 'text', 'to', 'treat', 'try', 'typeswitch', 'union', 'unordered', 'validate', 'where', 'with', 'xquery', 'contains', 'paragraphs', 'sentences', 'times', 'words', 'by', 'collection', 'allowing', 'at', 'base-uri', 'boundary-space', 'break', 'catch', 'construction', 'context', 'continue', 'copy-namespaces', 'decimal-format', 'encoding', 'exit', 'external', 'ft-option', 'in', 'index', 'integrity', 'lax', 'nodes', 'option', 'ordering', 'revalidation', 'schema', 'score', 'sliding', 'strict', 'tumbling', 'type', 'updating', 'value', 'variable', 'version', 'while', 'constraint', 'loop', 'returning', 'append', 'array', 'json-item', 'object', 'structured-item', 'when', 'next', 'previous', 'window'];

--- a/lib/ace/mode/xquery_worker.js
+++ b/lib/ace/mode/xquery_worker.js
@ -55,9 +55,9 @@ oop.inherits(XQueryWorker, Mirror);
      parser.parse_XQuery();
      this.sender.emit("ok");
      var ast = h.getParseTree();
-      var highlighter = new SemanticHighlighter(ast);
+      var highlighter = new SemanticHighlighter(ast, value);
      var tokens = highlighter.getTokens();
-      this.sender.emit("highlight", tokens);
+      this.sender.emit("highlight", { tokens: tokens, lines: highlighter.lines });
    } catch(e) {
      if(e instanceof parser.ParseException) {
        var prefix = value.substring(0, e.getBegin());