Syntax highlighter improvements.

lib/ace/mode/xquery/CommentHandler.js

@@ -0,0 +1,129 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2010, Ajax.org B.V.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+ 
+define(function(require, exports, module){
+  var CommentHandler = exports.CommentHandler = function(code) {
+
+    var ast = null;
+    var ptr = null;
+    var remains = code;
+    var cursor = 0;
+    var lineCursor = 0;
+    var line = 0;
+    var col = 0;
+
+    function createNode(name){
+      return { name: name, children: [], getParent: null, pos: { sl: 0, sc: 0, el: 0, ec: 0 } };
+    }
+  
+    function pushNode(name, begin){
+      var node = createNode(name);
+      if(ast === null) {
+        ast = node;
+        ptr = node;
+      } else {
+        node.getParent = ptr;
+        ptr.children.push(node);
+        ptr = ptr.children[ptr.children.length - 1];
+      }
+    }
+    
+    function popNode(name, end){
+     
+      if(ptr.children.length > 0) {
+        var s = ptr.children[0];
+        var e = ptr.children[ptr.children.length - 1];
+        ptr.pos.sl = s.pos.sl;
+        ptr.pos.sc = s.pos.sc;
+        ptr.pos.el = e.pos.el;
+        ptr.pos.ec = e.pos.ec;
+      }
+      
+      if(ptr.getParent !== null) {
+        ptr = ptr.getParent;
+        for(var i in ptr.children) {
+          delete ptr.children[i].getParent;
+        }
+      } else {
+        delete ptr.getParent;
+      }
+    }
+ 
+    this.peek = function() {
+      return ptr;    
+    };
+    
+    this.getParseTree = function() {
+      return ast;
+    };
+ 
+    this.reset = function(input) {};
+
+    this.startNonterminal = function(name, begin) {
+      pushNode(name, begin);
+    };
+
+    this.endNonterminal = function(name, end) {
+      popNode(name, end);
+    };
+
+    this.terminal = function(name, begin, end) {
+      name = (name.substring(0, 1) === "'" && name.substring(name.length - 1) === "'") ? "TOKEN" : name;
+      pushNode(name, begin); 
+      setValue(ptr, begin, end);
+      popNode(name, end);
+    };
+
+    this.whitespace = function(begin, end) {
+      var name = "WS";
+      pushNode(name, begin);
+      setValue(ptr, begin, end);
+      popNode(name, end);
+    };
+
+    function setValue(node, begin, end) {
+      var e = end - cursor;
+      ptr.value = remains.substring(0, e); 
+      var sl = line;
+      var sc = line === 0 ? lineCursor : lineCursor - 1;
+      var el = sl + ptr.value.split("\n").length - 1;
+      var lastIdx = ptr.value.lastIndexOf("\n");
+      var ec = lastIdx === -1 ? sc + ptr.value.length : ptr.value.substring(lastIdx).length;
+      remains = remains.substring(e);
+      cursor = end;
+      lineCursor = lastIdx === -1 ? lineCursor + (ptr.value.length) : ec; 
+      line = el; 
+      ptr.pos.sl = sl; 
+      ptr.pos.sc = sc; 
+      ptr.pos.el = el; 
+      ptr.pos.ec = ec; 
+    } 
+  };
+});

lib/ace/mode/xquery/CommentParser.ebnf

@@ -0,0 +1,55 @@
+<?xqlint
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2010, Ajax.org B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module){
+var CommentParser = exports.CommentParser = function CommentParser(string, parsingEventHandler)
+{
+  init(string, parsingEventHandler);
+?>
+
+
+Comments ::= (S^WS | Comment)* EOF
+
+Comment  ::= '(:' ( CommentContents | Comment )* ':)'
+
+<?TOKENS?>
+S        ::= [#x0009#x000A#x000D#x0020]+
+Char     ::= [#x0009#x000A#x000D#x0020-#xD7FF#xE000-#xFFFD#x10000-#x10FFFF]
+CommentContents
+         ::= ( ( Char+ - ( Char* ( '(:' | ':)' ) Char* ) ) - ( Char* '(' ) ) &':'
+           |   ( Char+ - ( Char* ( '(:' | ':)' ) Char* ) )                   &'('
+
+EOF      ::= $
+<?ENCORE?>
+
+<?xqlint
+});
+?>

lib/ace/mode/xquery/CommentParser.js

@@ -0,0 +1,371 @@
+// This file was generated on Wed Dec 12, 2012 20:06 (UTC+01) by REx v5.20 which is Copyright (c) 1979-2012 by Gunther Rademacher <grd@gmx.net>
+// REx command line: CommentParser.ebnf -tree -javascript -a xqlint
+
+                                                            // line 2 "CommentParser.ebnf"
+                                                            /* ***** BEGIN LICENSE BLOCK *****
+                                                             * Distributed under the BSD license:
+                                                             *
+                                                             * Copyright (c) 2010, Ajax.org B.V.
+                                                             * All rights reserved.
+                                                             *
+                                                             * Redistribution and use in source and binary forms, with or without
+                                                             * modification, are permitted provided that the following conditions are met:
+                                                             *     * Redistributions of source code must retain the above copyright
+                                                             *       notice, this list of conditions and the following disclaimer.
+                                                             *     * Redistributions in binary form must reproduce the above copyright
+                                                             *       notice, this list of conditions and the following disclaimer in the
+                                                             *       documentation and/or other materials provided with the distribution.
+                                                             *     * Neither the name of Ajax.org B.V. nor the
+                                                             *       names of its contributors may be used to endorse or promote products
+                                                             *       derived from this software without specific prior written permission.
+                                                             *
+                                                             * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+                                                             * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+                                                             * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+                                                             * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+                                                             * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+                                                             * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+                                                             * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+                                                             * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+                                                             * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+                                                             * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+                                                             *
+                                                             * ***** END LICENSE BLOCK ***** */
+
+                                                            define(function(require, exports, module){
+                                                            var CommentParser = exports.CommentParser = function CommentParser(string, parsingEventHandler)
+                                                            {
+                                                              init(string, parsingEventHandler);
+                                                            // line 40 "CommentParser.js"
+  var self = this;
+
+  this.ParseException = function(b, e, s, o, x)
+  {
+    var
+      begin = b,
+      end = e,
+      state = s,
+      offending = o,
+      expected = x;
+
+    this.getBegin = function() {return begin;};
+    this.getEnd = function() {return end;};
+    this.getState = function() {return state;};
+    this.getExpected = function() {return expected;};
+    this.getOffending = function() {return offending;};
+
+    this.getMessage = function()
+    {
+      return offending < 0 ? "lexical analysis failed" : "syntax error";
+    };
+  };
+
+  function init(string, parsingEventHandler)
+  {
+    eventHandler = parsingEventHandler;
+    input = string;
+    size = string.length;
+    reset(0, 0, 0);
+  }
+
+  this.getInput = function()
+  {
+    return input;
+  };
+
+  function reset(l, b, e)
+  {
+                 b0 = b; e0 = b;
+    l1 = l; b1 = b; e1 = e;
+    end = e;
+    eventHandler.reset(input);
+  }
+
+  this.getOffendingToken = function(e)
+  {
+    var o = e.getOffending();
+    return o >= 0 ? CommentParser.TOKEN[o] : null;
+  };
+
+  this.getExpectedTokenSet = function(e)
+  {
+    var expected;
+    if (e.getExpected() < 0)
+    {
+      expected = getExpectedTokenSet(e.getState());
+    }
+    else
+    {
+      expected = [CommentParser.TOKEN[e.getExpected()]];
+    }
+    return expected;
+  };
+
+  this.getErrorMessage = function(e)
+  {
+    var tokenSet = this.getExpectedTokenSet(e);
+    var found = this.getOffendingToken(e);
+    var prefix = input.substring(0, e.getBegin());
+    var lines = prefix.split("\n");
+    var line = lines.length;
+    var column = lines[line - 1].length + 1;
+    var size = e.getEnd() - e.getBegin();
+    return e.getMessage()
+         + (found == null ? "" : ", found " + found)
+         + "\nwhile expecting "
+         + (tokenSet.length == 1 ? tokenSet[0] : ("[" + tokenSet.join(", ") + "]"))
+         + "\n"
+         + (size == 0 ? "" : "after successfully scanning " + size + " characters beginning ")
+         + "at line " + line + ", column " + column + ":\n..."
+         + input.substring(e.getBegin(), Math.min(input.length, e.getBegin() + 64))
+         + "...";
+  };
+
+  this.parse_Comments = function()
+  {
+    eventHandler.startNonterminal("Comments", e0);
+    for (;;)
+    {
+      lookahead1(0);                // S^WS | EOF | '(:'
+      if (l1 == 3)                  // EOF
+      {
+        break;
+      }
+      switch (l1)
+      {
+      case 1:                       // S^WS
+        shift(1);                   // S^WS
+        break;
+      default:
+        parse_Comment();
+      }
+    }
+    shift(3);                       // EOF
+    eventHandler.endNonterminal("Comments", e0);
+  };
+
+  function parse_Comment()
+  {
+    eventHandler.startNonterminal("Comment", e0);
+    shift(4);                       // '(:'
+    for (;;)
+    {
+      lookahead1(1);                // CommentContents | '(:' | ':)'
+      if (l1 == 5)                  // ':)'
+      {
+        break;
+      }
+      switch (l1)
+      {
+      case 2:                       // CommentContents
+        shift(2);                   // CommentContents
+        break;
+      default:
+        parse_Comment();
+      }
+    }
+    shift(5);                       // ':)'
+    eventHandler.endNonterminal("Comment", e0);
+  }
+
+  var lk, b0, e0;
+  var l1, b1, e1;
+  var eventHandler;
+
+  function error(b, e, s, l, t)
+  {
+    throw new self.ParseException(b, e, s, l, t);
+  }
+
+  function shift(t)
+  {
+    if (l1 == t)
+    {
+      eventHandler.terminal(CommentParser.TOKEN[l1], b1, e1 > size ? size : e1);
+      b0 = b1; e0 = e1; l1 = 0;
+    }
+    else
+    {
+      error(b1, e1, 0, l1, t);
+    }
+  }
+
+  function lookahead1(set)
+  {
+    if (l1 == 0)
+    {
+      l1 = match(set);
+      b1 = begin;
+      e1 = end;
+    }
+  }
+
+  var input;
+  var size;
+  var begin;
+  var end;
+  var state;
+
+  function match(tokenset)
+  {
+    var nonbmp = false;
+    begin = end;
+    var current = end;
+    var result = CommentParser.INITIAL[tokenset];
+
+    for (var code = result & 15; code != 0; )
+    {
+      var charclass;
+      var c0 = current < size ? input.charCodeAt(current) : 0;
+      ++current;
+      if (c0 < 0x80)
+      {
+        charclass = CommentParser.MAP0[c0];
+      }
+      else if (c0 < 0xd800)
+      {
+        var c1 = c0 >> 5;
+        charclass = CommentParser.MAP1[(c0 & 31) + CommentParser.MAP1[(c1 & 31) + CommentParser.MAP1[c1 >> 5]]];
+      }
+      else
+      {
+        if (c0 < 0xdc00)
+        {
+          var c1 = current < size ? input.charCodeAt(current) : 0;
+          if (c1 >= 0xdc00 && c1 < 0xe000)
+          {
+            ++current;
+            c0 = ((c0 & 0x3ff) << 10) + (c1 & 0x3ff) + 0x10000;
+            nonbmp = true;
+          }
+        }
+        var lo = 0, hi = 1;
+        for (var m = 1; ; m = (hi + lo) >> 1)
+        {
+          if (CommentParser.MAP2[m] > c0) hi = m - 1;
+          else if (CommentParser.MAP2[2 + m] < c0) lo = m + 1;
+          else {charclass = CommentParser.MAP2[4 + m]; break;}
+          if (lo > hi) {charclass = 0; break;}
+        }
+      }
+
+      state = code;
+      var i0 = (charclass << 4) + code - 1;
+      code = CommentParser.TRANSITION[(i0 & 3) + CommentParser.TRANSITION[i0 >> 2]];
+
+      if (code > 15)
+      {
+        result = code;
+        code &= 15;
+        end = current;
+      }
+    }
+
+    result >>= 4;
+    if (result == 0)
+    {
+      end = current - 1;
+      var c1 = end < size ? input.charCodeAt(end) : 0;
+      if (c1 >= 0xdc00 && c1 < 0xe000) --end;
+      error(begin, end, state, -1, -1);
+    }
+
+    if (nonbmp)
+    {
+      for (var i = result >> 3; i > 0; --i)
+      {
+        --end;
+        var c1 = end < size ? input.charCodeAt(end) : 0;
+        if (c1 >= 0xdc00 && c1 < 0xe000) --end;
+      }
+    }
+    else
+    {
+      end -= result >> 3;
+    }
+
+    return (result & 7) - 1;
+  }
+
+  function getExpectedTokenSet(s)
+  {
+    var set = [];
+    if (s > 0)
+    {
+      for (var i = 0; i < 6; i += 32)
+      {
+        var j = i;
+        for (var f = ec(i >>> 5, s); f != 0; f >>>= 1, ++j)
+        {
+          if ((f & 1) != 0)
+          {
+            set[set.length] = CommentParser.TOKEN[j];
+          }
+        }
+      }
+    }
+    return set;
+  }
+
+  function ec(t, s)
+  {
+    var i0 = t * 9 + s - 1;
+    return CommentParser.EXPECTED[i0];
+  }
+}
+
+CommentParser.MAP0 =
+[
+  /*   0 */ 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2,
+  /*  36 */ 2, 2, 2, 2, 3, 4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  /*  72 */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  /* 108 */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
+];
+
+CommentParser.MAP1 =
+[
+  /*   0 */ 54, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
+  /*  27 */ 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
+  /*  54 */ 88, 120, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147,
+  /*  76 */ 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0,
+  /* 104 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 2, 2, 2, 3, 4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  /* 140 */ 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  /* 176 */ 2, 2, 2
+];
+
+CommentParser.MAP2 =
+[
+  /* 0 */ 57344, 65536, 65533, 1114111, 2, 2
+];
+
+CommentParser.INITIAL =
+[
+  /* 0 */ 1, 2
+];
+
+CommentParser.TRANSITION =
+[
+  /*  0 */ 33, 33, 33, 33, 28, 37, 32, 33, 31, 37, 32, 33, 43, 50, 53, 33, 31, 39, 33, 33, 46, 57, 59, 33, 63, 33, 33,
+  /* 27 */ 33, 35, 5, 35, 0, 5, 0, 0, 0, 0, 5, 5, 5, 5, 96, 5, 4, 6, 0, 0, 7, 0, 80, 184, 184, 184, 184, 0, 0, 0, 185,
+  /* 58 */ 80, 185, 0, 0, 0, 64, 0, 0, 0
+];
+
+CommentParser.EXPECTED =
+[
+  /* 0 */ 26, 52, 2, 16, 4, 20, 36, 4, 4
+];
+
+CommentParser.TOKEN =
+[
+  "(0)",
+  "S",
+  "CommentContents",
+  "EOF",
+  "'(:'",
+  "':)'"
+];
+
+                                                            // line 54 "CommentParser.ebnf"
+                                                            });
+                                                            // line 371 "CommentParser.js"
+// End

lib/ace/mode/xquery/JSONParseTreeHandler.js

@@ -29,18 +29,26 @@
  * ***** END LICENSE BLOCK ***** */
  
 define(function(require, exports, module){
-  var JSONParseTreeHandler = exports.JSONParseTreeHandler = function() {
+
+  var JSONParseTreeHandler = exports.JSONParseTreeHandler = function(code) {
+
+    //List of nodes that are left untouched in the parse tree size optimization.
+    var blacklist = ["VarDeclStatement"];
     
     var ast = null;
     var ptr = null;
-    
+    var remains = code;
+    var cursor = 0;
+    var lineCursor = 0;
+    var line = 0;
+    var col = 0;
+
     function createNode(name){
-      return { name: name, children: [], getParent: null };
+      return { name: name, children: [], getParent: null, pos: { sl: 0, sc: 0, el: 0, ec: 0 } };
     }
   
     function pushNode(name, begin){
       var node = createNode(name);
-      node.begin = begin;
       if(ast === null) {
         ast = node;
         ptr = node;
@@ -52,17 +60,38 @@ define(function(require, exports, module){
     }
     
     function popNode(name, end){
-      ptr.end = end;
+     
+      if(ptr.children.length > 0) {
+        var s = ptr.children[0];
+        var e = ptr.children[ptr.children.length - 1];
+        ptr.pos.sl = s.pos.sl;
+        ptr.pos.sc = s.pos.sc;
+        ptr.pos.el = e.pos.el;
+        ptr.pos.ec = e.pos.ec;
+      }
+      
       if(ptr.getParent !== null) {
         ptr = ptr.getParent;
-        for(var i in ptr.children) {
+        //for(var i in ptr.children) {
-          delete ptr.children[i].getParent;
+          //delete ptr.children[i].getParent;
-        }
+        //}
       } else {
-        delete ptr.getParent;
+        //delete ptr.getParent;
       }
+
+      //Parse tree size optimization
+      //if(ptr.children.length > 0) {
+      //  var lastChild = ptr.children[ptr.children.length - 1];
+      //  if(lastChild.children.length === 1 && blacklist.indexOf(lastChild.name) !== -1) {
+      //    ptr.children[ptr.children.length - 1] = lastChild.children[0];
+      //  }
+      //}
     }
  
+    this.peek = function() {
+      return ptr;    
+    };
+    
     this.getParseTree = function() {
       return ast;
     };
@@ -78,15 +107,35 @@ define(function(require, exports, module){
     };
 
     this.terminal = function(name, begin, end) {
-      var name = (name.substring(0, 1) === "'" && name.substring(name.length - 1) === "'") ? "TOKEN" : name;
+      name = (name.substring(0, 1) === "'" && name.substring(name.length - 1) === "'") ? "TOKEN" : name;
-      pushNode(name, begin);
+      pushNode(name, begin); 
+      setValue(ptr, begin, end);
       popNode(name, end);
     };
 
     this.whitespace = function(begin, end) {
       var name = "WS";
       pushNode(name, begin);
+      setValue(ptr, begin, end);
       popNode(name, end);
-    }; 
+    };
+
+    function setValue(node, begin, end) {
+      var e = end - cursor;
+      ptr.value = remains.substring(0, e); 
+      var sl = line;
+      var sc = line === 0 ? lineCursor : lineCursor - 1;
+      var el = sl + ptr.value.split("\n").length - 1;
+      var lastIdx = ptr.value.lastIndexOf("\n");
+      var ec = lastIdx === -1 ? sc + ptr.value.length : ptr.value.substring(lastIdx).length;
+      remains = remains.substring(e);
+      cursor = end;
+      lineCursor = lastIdx === -1 ? lineCursor + (ptr.value.length) : ec; 
+      line = el; 
+      ptr.pos.sl = sl; 
+      ptr.pos.sc = sc; 
+      ptr.pos.el = el; 
+      ptr.pos.ec = ec; 
+    } 
   };
 });

lib/ace/mode/xquery/visitors/SyntaxHighlighter.js

@@ -30,10 +30,16 @@
 
 define(function(require, exports, module){
 
-var SyntaxHighlighter = exports.SyntaxHighlighter = function(source, tree)
+var CommentParser  = require("../CommentParser").CommentParser;
+var CommentHandler = require("../CommentHandler").CommentHandler; 
+ 
+var SyntaxHighlighter = exports.SyntaxHighlighter = function(tree)
 {
    var keywords = ['after', 'ancestor', 'ancestor-or-self', 'and', 'as', 'ascending', 'attribute', 'before', 'case', 'cast', 'castable', 'child', 'collation', 'comment', 'copy', 'count', 'declare', 'default', 'delete', 'descendant', 'descendant-or-self', 'descending', 'div', 'document', 'document-node', 'element', 'else', 'empty', 'empty-sequence', 'end', 'eq', 'every', 'except', 'first', 'following', 'following-sibling', 'for', 'function', 'ge', 'group', 'gt', 'idiv', 'if', 'then', 'import', 'insert', 'instance', 'intersect', 'into', 'is', 'item', 'last', 'le', 'let', 'lt', 'mod', 'modify', 'module', 'namespace', 'namespace-node', 'ne', 'node', 'only', 'or', 'order', 'ordered', 'parent', 'preceding', 'preceding-sibling', 'processing-instruction', 'rename', 'replace', 'return', 'satisfies', 'schema-attribute', 'schema-element', 'self', 'some', 'stable', 'start', 'switch', 'text', 'to', 'treat', 'try', 'typeswitch', 'union', 'unordered', 'validate', 'where', 'with', 'xquery', 'contains', 'paragraphs', 'sentences', 'times', 'words', 'by', 'collection', 'allowing', 'at', 'base-uri', 'boundary-space', 'break', 'catch', 'construction', 'context', 'continue', 'copy-namespaces', 'decimal-format', 'encoding', 'exit', 'external', 'ft-option', 'in', 'index', 'integrity', 'lax', 'nodes', 'option', 'ordering', 'revalidation', 'schema', 'score', 'sliding', 'strict', 'tumbling', 'type', 'updating', 'value', 'variable', 'version', 'while', 'constraint', 'loop', 'returning', 'append', 'array', 'json-item', 'object', 'structured-item', 'when', 'next', 'previous', 'window'];
-   var states  = ["cdata", "comment", "tag"];
+
+   var docTags = "([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[a-zA-Z]{2,6})|(@[\\w\\d_]+)|(TODO)";
+   
+   var states  = ["cdata", "comment", "tag", "comment.doc"];
    var info = { lines: [ [] ], states: [] };
 
    this.getTokens = function(recover) {
@@ -45,8 +51,10 @@ var SyntaxHighlighter = exports.SyntaxHighlighter = function(source, tree)
            var token = info.lines[i][j];
            computed += token.value;
          }
+         if(i < info.lines.length - 1)
+           computed += "\n";
        }
-       this.addTokens(source.substring(computed.length + 1), "text");
+       this.addTokens(source.substring(computed.length), "text");
      }
      return info;
    };
@@ -69,10 +77,20 @@ var SyntaxHighlighter = exports.SyntaxHighlighter = function(source, tree)
      }
    };
 
-   this.getNodeValue = function(node)
+   this.getNodeValue = function(node) {
-   {
+     var value = "";
-     return source.substring(node.begin, node.end);
+     if(node.value === undefined) {
+       for(var i in node.children)
+       {
+         var child = node.children[i];
+         value += this.getNodeValue(child);
+       }
+     } else {
+       value += node.value;
+     }
+     return value;
    };
+ 
    
    this.DirPIConstructor = function(node)
    {
@@ -315,30 +333,43 @@ var SyntaxHighlighter = exports.SyntaxHighlighter = function(source, tree)
    };
   
    this.WS = function(node) {
-     var value = this.getNodeValue(node);
+     var value = node.value;
-     var remains = new String(value);
+     var h = new CommentHandler(value);
-     var state = 1;
+     var parser = new CommentParser(value, h);
-     var j = 0;
+     parser.parse_Comments();
-     for(var i = 0; i < value.length; i++) {
+     var ast = h.getParseTree();
-       var isOpen = value.substring(i, i + 2) == "(:";
+     var children = ast.children;
-       var isClose = value.substring(i, i + 2) == ":)";
+     for(var i in children)
-       
+     {
-       if(isOpen) {
+       var child = children[i];
-         state++;
+       if(child.name === "Comment" && child.children[1] && child.children[1].value.substring(0, 1) === "~")
-       } else if(isClose) {
+       {
-         state--;
+         var remains = this.getNodeValue(child);
+         while(remains.length > 0) {
+           var match = remains.match(docTags);
+           if(match !== null) {
+             var str = match[0];
+             var index = match.index;
+             if(index > 0) {
+               this.addTokens(remains.substring(0, index), "comment.doc");
+               remains = remains.substring(index);
+             }
+             this.addTokens(remains.substring(0, str.length), "comment.doc.tag");
+             remains = remains.substring(str.length);
+           } else {
+             this.addTokens(remains, "comment.doc");
+             break;
+           }
+         }
        }
-
+       else if(child.name === "Comment")
-       if(isOpen && !(state % 2)) {
+       {
-         this.addTokens(remains.substring(0, i), "text");
+         this.addTokens(this.getNodeValue(child), "comment");
-         remains = remains.substring(i);
+       } else if(child.name === "S") {
-         j = i;
+         this.addTokens(child.value, "text");
-       } else if(isClose && (state % 2)) {
+       }
-         this.addTokens(remains.substring(0, i - j + 2), "comment");
+     }
-         remains = remains.substring(i - j + 2); 
+     return true;
-       }   
-     }   
-     this.addTokens(remains, "text");
    };
    
    this.EverythingElse = function(node)

lib/ace/mode/xquery_highlight_rules.js

@@ -54,10 +54,16 @@ var XQueryHighlightRules = function() {
             regex: "<\\!--",
             next: "comment"
         }, {
+            token: "comment.doc",
+            regex: "\\(:~",
+            next: "comment.doc"
+        },
+        {
             token: "comment",
             regex: "\\(:",
             next: "comment"
-        }, {
+        },
+        {
             token: "text", // opening tag
             regex: "<\\/?",
             next: "tag"
@@ -73,6 +79,14 @@ var XQueryHighlightRules = function() {
         }, {
             token: "string",
             regex: "'.*?'"
+        }, {
+            token: "string",
+            regex: "'.*?",
+            next:  "apos-string"
+        }, {
+            token: "string",
+            regex: '".*?',
+            next:  "quot-string"
         }, {
             token: "text",
             regex: "\\s+"
@@ -122,7 +136,33 @@ var XQueryHighlightRules = function() {
             token: "comment",
             regex: "(?:[^\\]]|\\](?!\\]>))+"
         }],
-    
+        
+        "comment.doc": [
+          {
+            token: "comment.doc",
+            regex: ".*:\\)",
+            next:  "start"
+          }, {
+            token: "comment.doc.tag",
+            regex: "[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[a-zA-Z]{2,6}"
+          }, {
+            token : "comment.doc.tag",
+            regex : "@[\\w\\d_]+"
+          }, {
+              token : "comment.doc",
+              regex : "\\s+"
+          }, {
+              token : "comment.doc.tag",
+              regex : "TODO"
+          }, {
+              token : "comment.doc",
+              regex : "[^@:]+"
+          }, {
+              token : "comment.doc",
+              regex : "."
+          }
+        ], 
+
         comment: [{
             token: "comment",
             regex: ".*?-->",
@@ -134,6 +174,24 @@ var XQueryHighlightRules = function() {
         }, {
             token: "comment",
             regex: ".+"
+        }],
+
+        "apos-string": [{
+          token: "string",
+          regex: ".*?'",
+          next: "start"
+        }, {
+          token: "string",
+          regex: ".*"
+        }],
+
+        "quot-string": [{
+          token: "string",
+          regex: '.*?"',
+          next: "start"
+        }, {
+          token: "string",
+          regex: ".*"
         }]
     };
 };

lib/ace/mode/xquery_worker.js

@@ -35,7 +35,7 @@ var oop = require("../lib/oop");
 var Mirror = require("../worker/mirror").Mirror;
 var JSONParseTreeHandler  = require("./xquery/JSONParseTreeHandler").JSONParseTreeHandler;
 var XQueryParser  = require("./xquery/XQueryParser").XQueryParser;
-var SyntaxHighlighter = require("../mode/xquery/visitors/SyntaxHighlighter").SyntaxHighlighter;
+var SyntaxHighlighter = require("./xquery/visitors/SyntaxHighlighter").SyntaxHighlighter;
 
 var XQueryWorker = exports.XQueryWorker = function(sender) {
     Mirror.call(this, sender);
@@ -49,26 +49,30 @@ oop.inherits(XQueryWorker, Mirror);
   this.onUpdate = function() {
     this.sender.emit("start");
     var value = this.doc.getValue();    
-    var h = new JSONParseTreeHandler();
+    var h = new JSONParseTreeHandler(value);
     var parser = new XQueryParser(value, h);
     try {
       parser.parse_XQuery();
       this.sender.emit("ok");
       var ast = h.getParseTree();
-      var highlighter = new SyntaxHighlighter(value, ast);
+      var highlighter = new SyntaxHighlighter(ast);
       var tokens = highlighter.getTokens();
       this.sender.emit("highlight", tokens);
     } catch(e) {
-      var prefix = value.substring(0, e.getBegin());
+      if(e instanceof parser.ParseException) {
-      var line = prefix.split("\n").length;
+        var prefix = value.substring(0, e.getBegin());
-      var column = e.getBegin() - prefix.lastIndexOf("\n");
+        var line = prefix.split("\n").length;
-      var message = parser.getErrorMessage(e);
+        var column = e.getBegin() - prefix.lastIndexOf("\n");
-      this.sender.emit("error", {
+        var message = parser.getErrorMessage(e);
-        row: line - 1,
+        this.sender.emit("error", {
-        column: column,
+          row: line - 1,
-        text: message,
+          column: column,
-        type: "error"
+          text: message,
-      });
+          type: "error"
+        });
+      } else {
+        throw e;
+      }
     }
  };