Merge /Users/wcandillon/Dropbox/ace

lib/ace/mode/xquery/antlr3-all.js

@@ -16,6 +16,220 @@ http://developer.yahoo.net/yui/license.txt
 version: 2.5.1
 */
 define(function(require, exports, module){
+var NewLazyTokenStream = exports.NewLazyTokenStream = function(tokenSource) {
+
+  this.tokenSource = tokenSource;
+  this.tokens = [];
+  this.isWsExplicit = false;
+  this.p = 0;
+  this.channel = org.antlr.runtime.Token.DEFAULT_CHANNEL;
+  this.lastMarker = null;
+
+  this.LT = function(k) {
+    if (k == 0)
+      return null;
+    if (k < 0)
+      return this.readReverseNthGoodToken(-k);
+
+    return this.readNthGoodToken(k);
+  };
+
+  this.get = function(i) {
+    if (i >= this.tokens.length)
+      return org.antlr.runtime.Token.EOF_TOKEN;
+    return this.tokens[i];
+  };
+
+  this.getTokenSource = function() {
+    return this.tokenSource;
+  };
+
+  this.toString = function(start, stop) {
+     if(start == undefined) {
+       return this.toString(0, this.tokens.length - 1);
+     } else
+     if(start instanceof org.antlr.runtime.Token) {
+       return this.toString(start.getTokenIndex(), stop.getTokenIndex());
+     } else {
+      if (start < 0)
+        start = 0;
+      if (this.p <= stop) {
+        this.readNTokens(stop - this.p + 1);
+      }
+
+          var sb = "";
+          for (var i = start; i <= stop && i < this.tokens.length; i++) {
+              sb += this.tokens[i].getText();
+          }
+          return sb;
+      }
+    };
+
+    this.LA = function(i) {
+        return this.LT(i).getType();
+    };
+
+    this.done = false;
+
+    // public void consume() {
+    // if (done && p >= tokens.size())
+    // return;
+    //
+    // Token t = null;
+    // do {
+    // p++;
+    // t = LT(1);
+    // if (t == Token.EOF_TOKEN) {
+    // done = true;
+    // return;
+    // }
+    // p = t.getTokenIndex();
+    //
+    // } while (!isWsExplicit && t.getChannel() != channel);
+    //    
+    // if (LT(1) == Token.EOF_TOKEN) {
+    // done = true;
+    // }
+    // }
+
+    this.consume = function() {
+        if (this.done)
+            return;
+        this.p++;
+        if (!this.isWsExplicit) {
+            this.jumpToFirstValidToken();
+        }
+    };
+
+    this.getSourceName = function() {
+      return this.getTokenSource().getSourceName();
+    };
+
+    this.index = function() {
+        return this.p;
+    };
+
+    this.mark = function() {
+        this.lastMarker = this.index();
+        return this.lastMarker;
+    };
+
+    this.release = function(marker) {
+    };
+
+    this.rewind = function(marker) {
+      if(marker == undefined)
+        this.seek(this.lastMarker);
+      else
+        this.seek(marker);
+    };
+
+    this.seek = function(index) {
+        this.p = index;
+        this.done = false;
+    };
+
+    this.size = function() {
+        return this.tokens.length;
+    };
+
+    this.setTokenSource = function(source) {
+        this.tokenSource = source;
+        this.setWsExplicit(source.isWsExplicit);
+
+        // un-read the unused tokens
+        // they are different for the new source
+        if (this.p < this.tokens.length) {
+            var rIndex = this.p > 0 ? this.tokens[this.p - 1].getStopIndex() : 0;
+            this.tokenSource.rewindToIndex(rIndex + 1);
+            for (var i = this.tokens.length - 1; i >= this.p; i--) {
+                this.tokens.splice(i, 1);
+            }
+        }
+
+        // if we ignore WS, jump to next token
+        if (!this.isWsExplicit) {
+            this.jumpToFirstValidToken();
+        }
+    };
+
+    this.setWsExplicit = function(explicit) {
+        this.isWsExplicit = explicit;
+        if (!explicit) {
+            this.jumpToFirstValidToken();
+        }
+    };
+
+    this.readNthGoodToken = function(n) {
+        var count = this.tokens.length;
+        // number of buffered tokens available
+        var avt = count - this.p;
+        // i counts good tokens, j counts all tokens
+        var i = 1, j = 0;
+        var t = null;
+        while (i <= n) {
+            if (j < avt) // read from buffer
+                t = this.tokens[this.p + j];
+            else { // read from source
+                t = this.tokenSource.nextToken();
+                if (t == org.antlr.runtime.Token.EOF_TOKEN) {
+                    return t;
+                }
+                t.setTokenIndex(count++);
+                this.tokens.push(t);
+            }
+
+            if (this.isWsExplicit || t.getChannel() == this.channel) {
+                i++;
+            }
+            j++;
+        }
+        return t;
+    };
+
+    this.readReverseNthGoodToken = function(n) {
+        if (n == 0 || (this.p - n) < 0)
+            return null;
+
+        // i counts good tokens, j counts all tokens
+        var i = 1, j = 0;
+        var t = null;
+        while (this.p - 1 - j >= 0) {
+            t = this.get(this.p - 1 - j);
+
+            if (this.isWsExplicit || t.getChannel() == this.channel) {
+                if (i++ == n)
+                    return t;
+            }
+            j++;
+        }
+        return null;
+    };
+
+    this.readNTokens = function(n) {
+        var t = null;
+        for (var i = 0; i < n; i++) {
+            if (this.tokens.length > this.p + i)
+                continue;
+
+            t = this.tokenSource.nextToken();
+            if (t == org.antlr.runtime.Token.EOF_TOKEN)
+                return;
+
+            t.setTokenIndex(this.p + i);
+            this.tokens.push(t);
+        }
+    };
+
+    this.jumpToFirstValidToken = function() {
+        var t = this.LT(1);
+        if (t != org.antlr.runtime.Token.EOF_TOKEN) {
+            this.done = false;
+            this.p = t.getTokenIndex();
+        }
+    }; 
+};
+
 // create org.antlr module
 if (typeof org == "undefined" || !org) {
     var org = {};
@@ -474,7 +688,7 @@ org.antlr.runtime.RecognitionException = function(input) {
     org.antlr.runtime.RecognitionException.superclass.constructor.call(this);
     this.input = input;
     this.index = input.index();
-    if ( input instanceof org.antlr.runtime.CommonTokenStream ) {
+    if ( input instanceof NewLazyTokenStream ) {//org.antlr.runtime.CommonTokenStream ) {
         this.token = input.LT(1);
         this.line = this.token.getLine();
         this.charPositionInLine = this.token.getCharPositionInLine();
@@ -608,7 +822,7 @@ org.antlr.lang.extend(org.antlr.runtime.RecognitionException, Error,
      *  @return {Number} type of the unexpected input element.
      */
     getUnexpectedType: function() {
-		if ( this.input instanceof org.antlr.runtime.CommonTokenStream ) {
+		if ( this.input instanceof NewLazyTokenStream) {//org.antlr.runtime.CommonTokenStream ) {
 			return this.token.getType();
 		}
 		else if ( this.input instanceof org.antlr.runtime.tree.TreeNodeStream ) {

lib/ace/mode/xquery/xquery.js

@@ -37,15 +37,20 @@
  
 define(function(require, exports, module) {
 
-  var org =  require("./antlr3-all").org;
+  var antlr = require("./antlr3-all");
+  var org =  antlr.org;
+  var NewLazyTokenStream = antlr.NewLazyTokenStream;
   var XQueryLexer   = require("./XQueryLexer").XQueryLexer;
   var XQueryParser  = require("./XQueryParser").XQueryParser;
 
   exports.parse = function(code) {
       var cstream = new org.antlr.runtime.ANTLRStringStream(code);
       var lexer = new XQueryLexer(cstream);
-      var tstream = new org.antlr.runtime.CommonTokenStream(lexer);
+      var tstream = new NewLazyTokenStream(lexer);
+      tstream.jumpToFirstValidToken();
       var parser = new XQueryParser(tstream);
+      parser.source = cstream;
+      parser.stream = tstream;
       parser.p_Module();
   };
 });