Merge remote-tracking branch 'remotes/upstream/master'

Conflicts:
	lib/ace/commands/command_manager.js
	lib/ace/test/mockrenderer.js
	lib/ace/test/all_browser.js

lib/ace/ace.js

@@ -48,7 +48,12 @@ var EditSession = require("./edit_session").EditSession;
 var UndoManager = require("./undomanager").UndoManager;
 var Renderer = require("./virtual_renderer").VirtualRenderer;
 
+// The following require()s are for inclusion in the built ace file
 require("./worker/worker_client");
+require("./keyboard/hash_handler");
+require("./keyboard/state_handler");
+require("./lib/net");
+require("./placeholder");
 require("./config").init();
 
 exports.edit = function(el) {

lib/ace/commands/command_manager.js

@@ -67,7 +67,7 @@ oop.inherits(CommandManager, HashHandler);
                     this.exec(x, editor);
                 else
                     this.exec(x[0], editor, x[1]);
-            }, this)
+            }, this);
         } finally {
             this.$inReplay = false;
         }
@@ -79,9 +79,9 @@ oop.inherits(CommandManager, HashHandler);
                 x[0] = x[0].name;
             if (!x[1])
                 x = x[0];
-            return x
-        })
-    }
+            return x;
+        });
+    };
 
 }).call(CommandManager.prototype);
 

lib/ace/commands/default_commands.js

@@ -334,13 +334,13 @@ exports.commands = [{
 }, {
     name: "backspace",
     bindKey: bindKey(
-        "Ctrl-Backspace|Command-Backspace|Option-Backspace|Shift-Backspace|Backspace",
+        "Command-Backspace|Option-Backspace|Shift-Backspace|Backspace",
         "Ctrl-Backspace|Command-Backspace|Shift-Backspace|Backspace|Ctrl-H"
     ),
     exec: function(editor) { editor.remove("left"); }
 }, {
     name: "removetolinestart",
-    bindKey: bindKey("Alt-Backspace", "Option-Backspace"),
+    bindKey: bindKey("Alt-Backspace", "Command-Backspace"),
     exec: function(editor) { editor.removeToLineStart(); }
 }, {
     name: "removetolineend",

lib/ace/document.js

@@ -178,6 +178,7 @@ var Document = function(text) {
 
         position = this.$clipPosition(position);
 
+        // only detect new lines if the document has no line break yet
         if (this.getLength() <= 1)
             this.$detectNewLine(text);
 

lib/ace/edit_session.js

@@ -508,6 +508,10 @@ var EditSession = function(text, mode) {
                 return callback(_self.$modes[mode]);
             
             _self.$modes[mode] = new module.Mode();
+            _self._emit("loadmode", {
+                name: mode,
+                mode: _self.$modes[mode]
+            });
             callback(_self.$modes[mode]);
         }
 

lib/ace/editor.js

@@ -62,7 +62,7 @@ var Editor = function(renderer, session) {
     var container = renderer.getContainerElement();
     this.container = container;
     this.renderer = renderer;
-    
+
     this.textInput  = new TextInput(renderer.getTextAreaContainer(), this);
     this.keyBinding = new KeyBinding(this);
 
@@ -316,7 +316,11 @@ var Editor = function(renderer, session) {
         this.renderer.updateCursor();
 
         if (!this.$blockScrolling) {
-            this.renderer.scrollCursorIntoView();
+            var selection = this.getSelection();
+            if (selection.isEmpty())
+                this.renderer.scrollCursorIntoView(selection.getCursor());
+            else
+                this.renderer.scrollSelectionIntoView(selection.getSelectionLead(), selection.getSelectionAnchor());
         }
 
         // move text input over the cursor
@@ -652,7 +656,7 @@ var Editor = function(renderer, session) {
         this.$showFoldWidgets = show;
         this.renderer.updateFull();
     };
-    
+
     this.getShowFoldWidgets = function() {
         return this.renderer.$gutterLayer.getShowFoldWidgets();
     };
@@ -869,7 +873,7 @@ var Editor = function(renderer, session) {
             range.start.row += linesMoved;
             range.end.row += linesMoved;
             selection.setSelectionRange(range, reverse);
-        } 
+        }
         else {
             selection.setSelectionAnchor(rows.last+linesMoved+1, 0);
             selection.$moveSelection(function() {
@@ -1033,13 +1037,13 @@ var Editor = function(renderer, session) {
             cursor.column -= 2;
             pos = this.session.findMatchingBracket(cursor);
         }
-        
+
         if (pos) {
             this.clearSelection();
             this.moveCursorTo(pos.row, pos.column);
         }
     };
-    
+
     this.gotoLine = function(lineNumber, column) {
         this.selection.clearSelection();
         this.session.unfold({row: lineNumber - 1, column: column || 0});
@@ -1229,4 +1233,4 @@ var Editor = function(renderer, session) {
 
 
 exports.Editor = Editor;
-});
+});

lib/ace/ext/textarea.js

@@ -395,6 +395,7 @@ function setupSettingPanel(settingDiv, settingOpener, api, options) {
             svg:        "SVG",
             textile:    "Textile",
             groovy:     "Groovy",
+            liquid:     "Liquid",
             Scala:      "Scala"
         },
         theme: {

lib/ace/keyboard/hash_handler.js

@@ -125,7 +125,7 @@ function HashHandler(config, platform) {
 
         var key = typeof binding == "string" ? binding: binding[this.platform];
         this.bindKey(key, command);
-    }
+    };
 
     function parseKeys(keys, val, ret) {
         var key;
@@ -142,10 +142,10 @@ function HashHandler(config, platform) {
         return {
             key: key,
             hashId: hashId
-        }
+        };
     }
 
-    function splitSafe(s, separator) {
+    function splitSafe(s) {
         return (s.trim()
             .split(new RegExp("[\\s ]*\\-[\\s ]*", "g"), 999));
     }

lib/ace/mode/behaviour/cstyle.js

@@ -52,12 +52,12 @@ var CstyleBehaviour = function () {
                 return {
                     text: '{' + selected + '}',
                     selection: false
-                }
+                };
             } else {
                 return {
                     text: '{}',
                     selection: [1, 1]
-                }
+                };
             }
         } else if (text == '}') {
             var cursor = editor.getCursorPosition();
@@ -69,7 +69,7 @@ var CstyleBehaviour = function () {
                     return {
                         text: '',
                         selection: [1, 1]
-                    }
+                    };
                 }
             }
         } else if (text == "\n") {
@@ -87,7 +87,7 @@ var CstyleBehaviour = function () {
                 return {
                     text: '\n' + indent + '\n' + next_indent,
                     selection: [1, indent.length, 1, indent.length]
-                }
+                };
             }
         }
     });
@@ -112,12 +112,12 @@ var CstyleBehaviour = function () {
                 return {
                     text: '(' + selected + ')',
                     selection: false
-                }
+                };
             } else {
                 return {
                     text: '()',
                     selection: [1, 1]
-                }
+                };
             }
         } else if (text == ')') {
             var cursor = editor.getCursorPosition();
@@ -129,7 +129,7 @@ var CstyleBehaviour = function () {
                     return {
                         text: '',
                         selection: [1, 1]
-                    }
+                    };
                 }
             }
         }
@@ -148,14 +148,15 @@ var CstyleBehaviour = function () {
     });
 
     this.add("string_dquotes", "insertion", function (state, action, editor, session, text) {
-        if (text == '"') {
+        if (text == '"' || text == "'") {
+            var quote = text;
             var selection = editor.getSelectionRange();
             var selected = session.doc.getTextRange(selection);
             if (selected !== "") {
                 return {
-                    text: '"' + selected + '"',
+                    text: quote + selected + quote,
                     selection: false
-                }
+                };
             } else {
                 var cursor = editor.getCursorPosition();
                 var line = session.doc.getLine(cursor.row);
@@ -176,7 +177,7 @@ var CstyleBehaviour = function () {
                     if (token.type == "string") {
                       quotepos = -1;
                     } else if (quotepos < 0) {
-                      quotepos = token.value.indexOf('"');
+                      quotepos = token.value.indexOf(quote);
                     }
                     if ((token.value.length + col) > selection.start.column) {
                         break;
@@ -185,19 +186,19 @@ var CstyleBehaviour = function () {
                 }
 
                 // Try and be smart about when we auto insert.
-                if (!token || (quotepos < 0 && token.type !== "comment" && (token.type !== "string" || ((selection.start.column !== token.value.length+col-1) && token.value.lastIndexOf('"') === token.value.length-1)))) {
+                if (!token || (quotepos < 0 && token.type !== "comment" && (token.type !== "string" || ((selection.start.column !== token.value.length+col-1) && token.value.lastIndexOf(quote) === token.value.length-1)))) {
                     return {
-                        text: '""',
+                        text: quote + quote,
                         selection: [1,1]
-                    }
+                    };
                 } else if (token && token.type === "string") {
                     // Ignore input and move right one if we're typing over the closing quote.
                     var rightChar = line.substring(cursor.column, cursor.column + 1);
-                    if (rightChar == '"') {
+                    if (rightChar == quote) {
                         return {
                             text: '',
                             selection: [1, 1]
-                        }
+                        };
                     }
                 }
             }
@@ -206,7 +207,7 @@ var CstyleBehaviour = function () {
 
     this.add("string_dquotes", "deletion", function (state, action, editor, session, range) {
         var selected = session.doc.getTextRange(range);
-        if (!range.isMultiLine() && selected == '"') {
+        if (!range.isMultiLine() && (selected == '"' || selected == "'")) {
             var line = session.doc.getLine(range.start.row);
             var rightChar = line.substring(range.start.column + 1, range.start.column + 2);
             if (rightChar == '"') {
@@ -216,7 +217,8 @@ var CstyleBehaviour = function () {
         }
     });
 
-}
+};
+
 oop.inherits(CstyleBehaviour, Behaviour);
 
 exports.CstyleBehaviour = CstyleBehaviour;

lib/ace/mode/behaviour/xquery.js

@@ -0,0 +1,68 @@
+/*
+*  eXide - web-based XQuery IDE
+*  
+*  Copyright (C) 2011 Wolfgang Meier
+*
+*  This program is free software: you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License as published by
+*  the Free Software Foundation, either version 3 of the License, or
+*  (at your option) any later version.
+*
+*  This program is distributed in the hope that it will be useful,
+*  but WITHOUT ANY WARRANTY; without even the implied warranty of
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*  GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU General Public License
+*  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+define(function(require, exports, module) {
+"use strict";
+
+  var oop = require("../../lib/oop");
+  var Behaviour = require('../behaviour').Behaviour;
+  var CstyleBehaviour = require('./cstyle').CstyleBehaviour;
+
+  var XQueryBehaviour = function (parent) {
+      
+      this.inherit(CstyleBehaviour, ["braces", "parens", "string_dquotes"]); // Get string behaviour
+      this.parent = parent;
+      
+      this.add("brackets", "insertion", function (state, action, editor, session, text) {
+          if (text == "\n") {
+              var cursor = editor.getCursorPosition();
+              var line = session.doc.getLine(cursor.row);
+              var rightChars = line.substring(cursor.column, cursor.column + 2);
+              if (rightChars == '</') {
+                  var indent = this.$getIndent(session.doc.getLine(cursor.row)) + session.getTabString();
+                  var next_indent = this.$getIndent(session.doc.getLine(cursor.row));
+
+                  return {
+                      text: '\n' + indent + '\n' + next_indent,
+                      selection: [1, indent.length, 1, indent.length]
+                  }
+              }
+          }
+          return false;
+      });
+
+      // Check for open tag if user enters / and auto-close it.
+      this.add("slash", "insertion", function (state, action, editor, session, text) {
+        if (text == "/") {
+          var cursor = editor.getCursorPosition();
+        var line = session.doc.getLine(cursor.row);
+        if (cursor.column > 0 && line.charAt(cursor.column - 1) == "<") {
+          line = line.substring(0, cursor.column) + "/" + line.substring(cursor.column);
+          var lines = session.doc.getAllLines();
+          lines[cursor.row] = line;
+          // call mode helper to close the tag if possible
+          parent.exec("closeTag", lines.join(session.doc.getNewLineCharacter()), cursor.row);
+        }
+        }
+      return false;
+      });
+  }
+  oop.inherits(XQueryBehaviour, Behaviour);
+
+  exports.XQueryBehaviour = XQueryBehaviour;
+});

lib/ace/mode/clojure_highlight_rules.js

@@ -138,9 +138,9 @@ var ClojureHighlightRules = function() {
                 token : "comment",
                 regex : ";.*$"
             }, {
-                    token : "comment", // multi line comment
-                    regex : "^\=begin$",
-                    next : "comment"
+                token : "comment", // multi line comment
+                regex : "^=begin$",
+                next : "comment"
             }, {
                 token : "keyword", //parens
                 regex : "[\\(|\\)]"
@@ -193,17 +193,17 @@ var ClojureHighlightRules = function() {
                 regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
             }, {
                 token : "string", // symbol
-                regex : "[:](?:[a-zA-Z]|\d)+"
+                regex : "[:](?:[a-zA-Z]|\\d)+"
             }, {
-            token : "string.regexp", //Regular Expressions
-            regex : '/#"(?:\.|(\\\")|[^\""\n])*"/g'
+                token : "string.regexp", //Regular Expressions
+                regex : '/#"(?:\\.|(?:\\\")|[^\""\n])*"/g'
             }
-              
+
         ],
         "comment" : [
             {
                 token : "comment", // closing comment
-                regex : "^\=end$",
+                regex : "^=end$",
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line

lib/ace/mode/css_highlight_rules.js

@@ -105,11 +105,17 @@ var CssHighlightRules = function() {
         "purple|red|silver|teal|white|yellow").split("|")
     );
 
+    var fonts = lang.arrayToMap(
+        ("arial|century|comic|courier|garamond|georgia|helvetica|impact|lucida|" +
+        "symbol|system|tahoma|times|trebuchet|utopia|verdana|webdings|sans-serif|" +
+        "serif|monospace").split("|")
+    );
+    
     // regexp must not have capturing parentheses. Use (?:) instead.
     // regexps are ordered -> the first match is used
 
     var numRe = "\\-?(?:(?:[0-9]+)|(?:[0-9]*\\.[0-9]+))";
-
+    
     var base_ruleset = [
         {
             token : "comment", // multi line comment
@@ -123,8 +129,11 @@ var CssHighlightRules = function() {
             token : "string", // single line
             regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
         }, {
-            token : "constant.numeric",
-            regex : numRe + "(?:em|ex|px|cm|mm|in|pt|pc|deg|rad|grad|ms|s|hz|khz|%)"
+            token : ["constant.numeric", "keyword"],
+            regex : "(" + numRe + ")(em|ex|px|ch|cm|mm|in|pt|pc|deg|rad|dpi|grad|ms|s|hz|khz|%)"
+        }, {
+            token : ["constant.numeric"],
+            regex : "([0-9]+)"
         }, {
             token : "constant.numeric",  // hex6 color
             regex : "#[a-f0-9]{6}"
@@ -145,12 +154,15 @@ var CssHighlightRules = function() {
                 else if (colors.hasOwnProperty(value.toLowerCase())) {
                     return "support.constant.color";
                 }
+                else if (fonts.hasOwnProperty(value.toLowerCase())) {
+                    return "support.constant.fonts";
+                }
                 else {
                     return "text";
                 }
             },
             regex : "\\-?[a-zA-Z_][a-zA-Z0-9_\\-]*"
-        }
+        } 
       ];
 
     var ruleset = lang.copyArray(base_ruleset);

lib/ace/mode/css_highlight_rules_test.js

@@ -57,7 +57,7 @@ module.exports = {
         var line = "-12px";
         var tokens = this.tokenizer.getLineTokens(line, "ruleset").tokens;
 
-        assert.equal(1, tokens.length);
+        assert.equal(2, tokens.length);
         assert.equal("constant.numeric", tokens[0].type);
     },
 

lib/ace/mode/javascript_highlight_rules.js

@@ -93,20 +93,20 @@ var JavaScriptHighlightRules = function() {
     );
 
     // TODO: Unicode escape sequences
-    var identifierRe = "[" + unicode.packages.L + "\\$_][" 
+    var identifierRe = "[" + unicode.packages.L + "\\$_]["
         + unicode.packages.L
         + unicode.packages.Mn + unicode.packages.Mc
         + unicode.packages.Nd
         + unicode.packages.Pc + "\\$_]*\\b";
-    
+
     var escapedRe = "\\\\(?:x[0-9a-fA-F]{2}|" + // hex
         "u[0-9a-fA-F]{4}|" + // unicode
         "[0-2][0-7]{0,2}|" + // oct
         "3[0-6][0-7]?|" + // oct
-        "37[0-7]?|" + // oct 
+        "37[0-7]?|" + // oct
         "[4-7][0-7]?|" + //oct
         ".)";
-        
+
     // regexp must not have capturing parentheses. Use (?:) instead.
     // regexps are ordered -> the first match is used
 
@@ -114,13 +114,13 @@ var JavaScriptHighlightRules = function() {
         "start" : [
             {
                 token : "comment",
-                regex : "\\/\\/.*$"
+                regex : /\/\/.*$/
             },
             new DocCommentHighlightRules().getStartRule("doc-start"),
             {
                 token : "comment", // multi line comment
                 merge : true,
-                regex : "\\/\\*",
+                regex : /\/\*/,
                 next : "comment"
             }, {
                 token : "string",
@@ -132,33 +132,121 @@ var JavaScriptHighlightRules = function() {
                 next  : "qqstring"
             }, {
                 token : "constant.numeric", // hex
-                regex : "0[xX][0-9a-fA-F]+\\b"
+                regex : /0[xX][0-9a-fA-F]+\b/
             }, {
                 token : "constant.numeric", // float
-                regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
-            }, {
-                token : ["storage.type", "text", "entity.name.function", "text", "paren.lparen", "variable.parameter", "paren.rparen"],
+                regex : /[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?\b/
+            }, { // match stuff like: Sound.prototype.play = function() { }
+                token : [
+                    "storage.type",
+                    "punctuation.operator",
+                    "support.function",
+                    "punctuation.operator",
+                    "entity.name.function",
+                    "text",
+                    "keyword.operator",
+                    "text",
+                    "storage.type",
+                    "text",
+                    "paren.lparen",
+                    "variable.parameter",
+                    "paren.rparen"
+                ],
+                regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))"
+            }, { // match stuff like: Sound.prototype.play = myfunc
+                token : [
+                    "storage.type",
+                    "punctuation.operator",
+                    "support.function",
+                    "punctuation.operator",
+                    "entity.name.function",
+                    "text",
+                    "keyword.operator",
+                    "text"
+                ],
+                regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)"
+            }, { // match stuff like: Sound.play = function() {  }
+                token : [
+                    "storage.type",
+                    "punctuation.operator",
+                    "entity.name.function",
+                    "text",
+                    "keyword.operator",
+                    "text",
+                    "storage.type",
+                    "text",
+                    "paren.lparen",
+                    "variable.parameter",
+                    "paren.rparen"
+                ],
+                regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))"
+            }, { // match stuff like: play = function() {  }
+                token : [
+                    "entity.name.function",
+                    "text",
+                    "keyword.operator",
+                    "text",
+                    "storage.type",
+                    "text",
+                    "paren.lparen",
+                    "variable.parameter",
+                    "paren.rparen"
+                ],
+                regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))"
+            }, { // match regular function like: function myFunc(arg) { }
+                token : [
+                    "storage.type",
+                    "text",
+                    "entity.name.function",
+                    "text",
+                    "paren.lparen",
+                    "variable.parameter",
+                    "paren.rparen"
+                ],
                 regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()(.*?)(\\))"
+            }, { // match stuff like: foobar: function() { }
+                token : [
+                    "entity.name.function",
+                    "text",
+                    "punctuation.operator",
+                    "text",
+                    "storage.type",
+                    "text",
+                    "paren.lparen",
+                    "variable.parameter",
+                    "paren.rparen"
+                ],
+                regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()(.*?)(\\))"
+            }, { // Attempt to match : function() { } (this is for issues with 'foo': function() { })
+                token : [
+                    "text",
+                    "text",
+                    "storage.type",
+                    "text",
+                    "paren.lparen",
+                    "variable.parameter",
+                    "paren.rparen"
+                ],
+                regex : "(:)(\\s*)(function)?(\\s*)(\\()([^)]*)(\\))"
             }, {
                 token : "constant.language.boolean",
-                regex : "(?:true|false)\\b"
+                regex : /(?:true|false)\b/
             }, {
                 token : "keyword",
                 regex : "(?:" + kwBeforeRe + ")\\b",
                 next : "regex_allowed"
             }, {
                 token : "support.function",
-                regex : "\\b(?:s(?:h(?:ift|ow(?:Mod(?:elessDialog|alDialog)|Help))|croll(?:X|By(?:Pages|Lines)?|Y|To)?|t(?:op|rike)|i(?:n|zeToContent|debar|gnText)|ort|u(?:p|b(?:str(?:ing)?)?)|pli(?:ce|t)|e(?:nd|t(?:Re(?:sizable|questHeader)|M(?:i(?:nutes|lliseconds)|onth)|Seconds|Ho(?:tKeys|urs)|Year|Cursor|Time(?:out)?|Interval|ZOptions|Date|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Date|FullYear)|FullYear|Active)|arch)|qrt|lice|avePreferences|mall)|h(?:ome|andleEvent)|navigate|c(?:har(?:CodeAt|At)|o(?:s|n(?:cat|textual|firm)|mpile)|eil|lear(?:Timeout|Interval)?|a(?:ptureEvents|ll)|reate(?:StyleSheet|Popup|EventObject))|t(?:o(?:GMTString|S(?:tring|ource)|U(?:TCString|pperCase)|Lo(?:caleString|werCase))|est|a(?:n|int(?:Enabled)?))|i(?:s(?:NaN|Finite)|ndexOf|talics)|d(?:isableExternalCapture|ump|etachEvent)|u(?:n(?:shift|taint|escape|watch)|pdateCommands)|j(?:oin|avaEnabled)|p(?:o(?:p|w)|ush|lugins.refresh|a(?:ddings|rse(?:Int|Float)?)|r(?:int|ompt|eference))|e(?:scape|nableExternalCapture|val|lementFromPoint|x(?:p|ec(?:Script|Command)?))|valueOf|UTC|queryCommand(?:State|Indeterm|Enabled|Value)|f(?:i(?:nd|le(?:ModifiedDate|Size|CreatedDate|UpdatedDate)|xed)|o(?:nt(?:size|color)|rward)|loor|romCharCode)|watch|l(?:ink|o(?:ad|g)|astIndexOf)|a(?:sin|nchor|cos|t(?:tachEvent|ob|an(?:2)?)|pply|lert|b(?:s|ort))|r(?:ou(?:nd|teEvents)|e(?:size(?:By|To)|calc|turnValue|place|verse|l(?:oad|ease(?:Capture|Events)))|andom)|g(?:o|et(?:ResponseHeader|M(?:i(?:nutes|lliseconds)|onth)|Se(?:conds|lection)|Hours|Year|Time(?:zoneOffset)?|Da(?:y|te)|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Da(?:y|te)|FullYear)|FullYear|A(?:ttention|llResponseHeaders)))|m(?:in|ove(?:B(?:y|elow)|To(?:Absolute)?|Above)|ergeAttributes|a(?:tch|rgins|x))|b(?:toa|ig|o(?:ld|rderWidths)|link|ack))\\b(?=\\()"     
+                regex : /\b(?:s(?:h(?:ift|ow(?:Mod(?:elessDialog|alDialog)|Help))|croll(?:X|By(?:Pages|Lines)?|Y|To)?|t(?:op|rike)|i(?:n|zeToContent|debar|gnText)|ort|u(?:p|b(?:str(?:ing)?)?)|pli(?:ce|t)|e(?:nd|t(?:Re(?:sizable|questHeader)|M(?:i(?:nutes|lliseconds)|onth)|Seconds|Ho(?:tKeys|urs)|Year|Cursor|Time(?:out)?|Interval|ZOptions|Date|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Date|FullYear)|FullYear|Active)|arch)|qrt|lice|avePreferences|mall)|h(?:ome|andleEvent)|navigate|c(?:har(?:CodeAt|At)|o(?:s|n(?:cat|textual|firm)|mpile)|eil|lear(?:Timeout|Interval)?|a(?:ptureEvents|ll)|reate(?:StyleSheet|Popup|EventObject))|t(?:o(?:GMTString|S(?:tring|ource)|U(?:TCString|pperCase)|Lo(?:caleString|werCase))|est|a(?:n|int(?:Enabled)?))|i(?:s(?:NaN|Finite)|ndexOf|talics)|d(?:isableExternalCapture|ump|etachEvent)|u(?:n(?:shift|taint|escape|watch)|pdateCommands)|j(?:oin|avaEnabled)|p(?:o(?:p|w)|ush|lugins.refresh|a(?:ddings|rse(?:Int|Float)?)|r(?:int|ompt|eference))|e(?:scape|nableExternalCapture|val|lementFromPoint|x(?:p|ec(?:Script|Command)?))|valueOf|UTC|queryCommand(?:State|Indeterm|Enabled|Value)|f(?:i(?:nd|le(?:ModifiedDate|Size|CreatedDate|UpdatedDate)|xed)|o(?:nt(?:size|color)|rward)|loor|romCharCode)|watch|l(?:ink|o(?:ad|g)|astIndexOf)|a(?:sin|nchor|cos|t(?:tachEvent|ob|an(?:2)?)|pply|lert|b(?:s|ort))|r(?:ou(?:nd|teEvents)|e(?:size(?:By|To)|calc|turnValue|place|verse|l(?:oad|ease(?:Capture|Events)))|andom)|g(?:o|et(?:ResponseHeader|M(?:i(?:nutes|lliseconds)|onth)|Se(?:conds|lection)|Hours|Year|Time(?:zoneOffset)?|Da(?:y|te)|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Da(?:y|te)|FullYear)|FullYear|A(?:ttention|llResponseHeaders)))|m(?:in|ove(?:B(?:y|elow)|To(?:Absolute)?|Above)|ergeAttributes|a(?:tch|rgins|x))|b(?:toa|ig|o(?:ld|rderWidths)|link|ack))\b(?=\()/
             }, {
                 token : "support.function.dom",
-                regex : "\\b(?:s(?:ub(?:stringData|mit)|plitText|e(?:t(?:NamedItem|Attribute(?:Node)?)|lect))|has(?:ChildNodes|Feature)|namedItem|c(?:l(?:ick|o(?:se|neNode))|reate(?:C(?:omment|DATASection|aption)|T(?:Head|extNode|Foot)|DocumentFragment|ProcessingInstruction|E(?:ntityReference|lement)|Attribute))|tabIndex|i(?:nsert(?:Row|Before|Cell|Data)|tem)|open|delete(?:Row|C(?:ell|aption)|T(?:Head|Foot)|Data)|focus|write(?:ln)?|a(?:dd|ppend(?:Child|Data))|re(?:set|place(?:Child|Data)|move(?:NamedItem|Child|Attribute(?:Node)?)?)|get(?:NamedItem|Element(?:sBy(?:Name|TagName)|ById)|Attribute(?:Node)?)|blur)\\b(?=\\()"                
+                regex : /\b(?:s(?:ub(?:stringData|mit)|plitText|e(?:t(?:NamedItem|Attribute(?:Node)?)|lect))|has(?:ChildNodes|Feature)|namedItem|c(?:l(?:ick|o(?:se|neNode))|reate(?:C(?:omment|DATASection|aption)|T(?:Head|extNode|Foot)|DocumentFragment|ProcessingInstruction|E(?:ntityReference|lement)|Attribute))|tabIndex|i(?:nsert(?:Row|Before|Cell|Data)|tem)|open|delete(?:Row|C(?:ell|aption)|T(?:Head|Foot)|Data)|focus|write(?:ln)?|a(?:dd|ppend(?:Child|Data))|re(?:set|place(?:Child|Data)|move(?:NamedItem|Child|Attribute(?:Node)?)?)|get(?:NamedItem|Element(?:sBy(?:Name|TagName)|ById)|Attribute(?:Node)?)|blur)\b(?=\()/
             }, {
-                token : "support.function.constant",
-                regex : "\\b(?:s(?:ystemLanguage|cr(?:ipts|ollbars|een(?:X|Y|Top|Left))|t(?:yle(?:Sheets)?|atus(?:Text|bar)?)|ibling(?:Below|Above)|ource|uffixes|e(?:curity(?:Policy)?|l(?:ection|f)))|h(?:istory|ost(?:name)?|as(?:h|Focus))|y|X(?:MLDocument|SLDocument)|n(?:ext|ame(?:space(?:s|URI)|Prop))|M(?:IN_VALUE|AX_VALUE)|c(?:haracterSet|o(?:n(?:structor|trollers)|okieEnabled|lorDepth|mp(?:onents|lete))|urrent|puClass|l(?:i(?:p(?:boardData)?|entInformation)|osed|asses)|alle(?:e|r)|rypto)|t(?:o(?:olbar|p)|ext(?:Transform|Indent|Decoration|Align)|ags)|SQRT(?:1_2|2)|i(?:n(?:ner(?:Height|Width)|put)|ds|gnoreCase)|zIndex|o(?:scpu|n(?:readystatechange|Line)|uter(?:Height|Width)|p(?:sProfile|ener)|ffscreenBuffering)|NEGATIVE_INFINITY|d(?:i(?:splay|alog(?:Height|Top|Width|Left|Arguments)|rectories)|e(?:scription|fault(?:Status|Ch(?:ecked|arset)|View)))|u(?:ser(?:Profile|Language|Agent)|n(?:iqueID|defined)|pdateInterval)|_content|p(?:ixelDepth|ort|ersonalbar|kcs11|l(?:ugins|atform)|a(?:thname|dding(?:Right|Bottom|Top|Left)|rent(?:Window|Layer)?|ge(?:X(?:Offset)?|Y(?:Offset)?))|r(?:o(?:to(?:col|type)|duct(?:Sub)?|mpter)|e(?:vious|fix)))|e(?:n(?:coding|abledPlugin)|x(?:ternal|pando)|mbeds)|v(?:isibility|endor(?:Sub)?|Linkcolor)|URLUnencoded|P(?:I|OSITIVE_INFINITY)|f(?:ilename|o(?:nt(?:Size|Family|Weight)|rmName)|rame(?:s|Element)|gColor)|E|whiteSpace|l(?:i(?:stStyleType|n(?:eHeight|kColor))|o(?:ca(?:tion(?:bar)?|lName)|wsrc)|e(?:ngth|ft(?:Context)?)|a(?:st(?:M(?:odified|atch)|Index|Paren)|yer(?:s|X)|nguage))|a(?:pp(?:MinorVersion|Name|Co(?:deName|re)|Version)|vail(?:Height|Top|Width|Left)|ll|r(?:ity|guments)|Linkcolor|bove)|r(?:ight(?:Context)?|e(?:sponse(?:XML|Text)|adyState))|global|x|m(?:imeTypes|ultiline|enubar|argin(?:Right|Bottom|Top|Left))|L(?:N(?:10|2)|OG(?:10E|2E))|b(?:o(?:ttom|rder(?:Width|RightWidth|BottomWidth|Style|Color|TopWidth|LeftWidth))|ufferDepth|elow|ackground(?:Color|Image)))\\b"                
+                token : "support.constant",
+                regex : /\b(?:s(?:ystemLanguage|cr(?:ipts|ollbars|een(?:X|Y|Top|Left))|t(?:yle(?:Sheets)?|atus(?:Text|bar)?)|ibling(?:Below|Above)|ource|uffixes|e(?:curity(?:Policy)?|l(?:ection|f)))|h(?:istory|ost(?:name)?|as(?:h|Focus))|y|X(?:MLDocument|SLDocument)|n(?:ext|ame(?:space(?:s|URI)|Prop))|M(?:IN_VALUE|AX_VALUE)|c(?:haracterSet|o(?:n(?:structor|trollers)|okieEnabled|lorDepth|mp(?:onents|lete))|urrent|puClass|l(?:i(?:p(?:boardData)?|entInformation)|osed|asses)|alle(?:e|r)|rypto)|t(?:o(?:olbar|p)|ext(?:Transform|Indent|Decoration|Align)|ags)|SQRT(?:1_2|2)|i(?:n(?:ner(?:Height|Width)|put)|ds|gnoreCase)|zIndex|o(?:scpu|n(?:readystatechange|Line)|uter(?:Height|Width)|p(?:sProfile|ener)|ffscreenBuffering)|NEGATIVE_INFINITY|d(?:i(?:splay|alog(?:Height|Top|Width|Left|Arguments)|rectories)|e(?:scription|fault(?:Status|Ch(?:ecked|arset)|View)))|u(?:ser(?:Profile|Language|Agent)|n(?:iqueID|defined)|pdateInterval)|_content|p(?:ixelDepth|ort|ersonalbar|kcs11|l(?:ugins|atform)|a(?:thname|dding(?:Right|Bottom|Top|Left)|rent(?:Window|Layer)?|ge(?:X(?:Offset)?|Y(?:Offset)?))|r(?:o(?:to(?:col|type)|duct(?:Sub)?|mpter)|e(?:vious|fix)))|e(?:n(?:coding|abledPlugin)|x(?:ternal|pando)|mbeds)|v(?:isibility|endor(?:Sub)?|Linkcolor)|URLUnencoded|P(?:I|OSITIVE_INFINITY)|f(?:ilename|o(?:nt(?:Size|Family|Weight)|rmName)|rame(?:s|Element)|gColor)|E|whiteSpace|l(?:i(?:stStyleType|n(?:eHeight|kColor))|o(?:ca(?:tion(?:bar)?|lName)|wsrc)|e(?:ngth|ft(?:Context)?)|a(?:st(?:M(?:odified|atch)|Index|Paren)|yer(?:s|X)|nguage))|a(?:pp(?:MinorVersion|Name|Co(?:deName|re)|Version)|vail(?:Height|Top|Width|Left)|ll|r(?:ity|guments)|Linkcolor|bove)|r(?:ight(?:Context)?|e(?:sponse(?:XML|Text)|adyState))|global|x|m(?:imeTypes|ultiline|enubar|argin(?:Right|Bottom|Top|Left))|L(?:N(?:10|2)|OG(?:10E|2E))|b(?:o(?:ttom|rder(?:Width|RightWidth|BottomWidth|Style|Color|TopWidth|LeftWidth))|ufferDepth|elow|ackground(?:Color|Image)))\b/
             }, {
-                token : ["punctuation.operator", "support.function.firebug"],
-                regex : "(\\.)(warn|info|log|error|time|timeEnd|assert)\\b"
-                
+                token : ["storage.type", "punctuation.operator", "support.function.firebug"],
+                regex : /\b(console)(\.)(warn|info|log|error|time|timeEnd|assert)\b/
             }, {
                 token : function(value) {
                     if (globals.hasOwnProperty(value))
@@ -181,29 +269,29 @@ var JavaScriptHighlightRules = function() {
                 regex : identifierRe
             }, {
                 token : "keyword.operator",
-                regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)",
+                regex : /!|\$|%|&|\*|\-\-|\-|\+\+|\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\|\||\?\:|\*=|%=|\+=|\-=|&=|\^=|\b(?:in|instanceof|new|delete|typeof|void)/,
                 next  : "regex_allowed"
             }, {
                 token : "punctuation.operator",
-                regex : "\\?|\\:|\\,|\\;|\\.",
+                regex : /\?|\:|\,|\;|\./,
                 next  : "regex_allowed"
             }, {
                 token : "paren.lparen",
-                regex : "[[({]",
+                regex : /[\[({]/,
                 next  : "regex_allowed"
             }, {
                 token : "paren.rparen",
-                regex : "[\\])}]"
+                regex : /[\])}]/
             }, {
                 token : "keyword.operator",
-                regex : "\\/=?",
+                regex : /\/=?/,
                 next  : "regex_allowed"
             }, {
                 token: "comment",
-                regex: "^#!.*$" 
+                regex: /^#!.*$/
             }, {
                 token : "text",
-                regex : "\\s+"
+                regex : /\s+/
             }
         ],
         // regular expressions are only allowed after certain tokens. This
@@ -228,7 +316,7 @@ var JavaScriptHighlightRules = function() {
             }, {
                 // immediately return to the start mode without matching
                 // anything
-                token: "empty", 
+                token: "empty",
                 regex: "",
                 next: "start"
             }
@@ -240,10 +328,10 @@ var JavaScriptHighlightRules = function() {
                 next: "regex"
             }, {
 				// flag
-                token: "string.regexp", 
+                token: "string.regexp",
                 regex: "/\\w*",
                 next: "start",
-                merge: true 
+                merge: true
             }, {
                 token: "string.regexp",
                 regex: "[^\\\\/\\[]+",
@@ -255,9 +343,9 @@ var JavaScriptHighlightRules = function() {
                 next: "regex_character_class",
                 merge: true
             }, {
-                token: "empty", 
+                token: "empty",
                 regex: "",
-                next: "start" 
+                next: "start"
             }
         ],
         "regex_character_class": [
@@ -276,9 +364,9 @@ var JavaScriptHighlightRules = function() {
                 next: "regex_character_class",
                 merge: true
             }, {
-                token: "empty", 
+                token: "empty",
                 regex: "",
-                next: "start" 
+                next: "start"
             }
         ],
         "comment_regex_allowed" : [
@@ -332,7 +420,7 @@ var JavaScriptHighlightRules = function() {
             }
         ]
     };
-    
+
     this.embedRules(DocCommentHighlightRules, "doc-",
         [ new DocCommentHighlightRules().getEndRule("start") ]);
 };

lib/ace/mode/javascript_worker.js

@@ -41,6 +41,7 @@ define(function(require, exports, module) {
 var oop = require("../lib/oop");
 var Mirror = require("../worker/mirror").Mirror;
 var lint = require("../worker/jshint").JSHINT;
+var parser = require("../narcissus/parser");
     
 var JavaScriptWorker = exports.JavaScriptWorker = function(sender) {
     Mirror.call(this, sender);
@@ -56,7 +57,6 @@ oop.inherits(JavaScriptWorker, Mirror);
         value = value.replace(/^#!.*\n/, "\n");
         
 //        var start = new Date();
-        var parser = require("../narcissus/jsparse");
         try {
             parser.parse(value);
         } catch(e) {

lib/ace/mode/javascript_worker_test.js

@@ -59,19 +59,31 @@ module.exports = {
             }
         };
     },
-    
+
     "test check for syntax error": function() {
         var worker = new JavaScriptWorker(this.sender);
         worker.setValue("Juhu Kinners");
         worker.deferredUpdate.call();
-        
+
         var error = this.sender.events[0][1];
         assert.equal(error.text, "missing ; before statement");
         assert.equal(error.type, "error");
         assert.equal(error.row, 0);
         assert.equal(error.column, null);
     },
-    
+
+    "test invalid multi line string": function() {
+        var worker = new JavaScriptWorker(this.sender);
+        worker.setValue('"a\n\\nn"');
+        worker.deferredUpdate.call();
+
+        var error = this.sender.events[0][1];
+        assert.equal(error.text, "Unterminated string literal");
+        assert.equal(error.type, "error");
+        assert.equal(error.row, 0);
+        assert.equal(error.column, null);
+    },
+
     "test check for narcissus bug": function() {
         var worker = new JavaScriptWorker(this.sender);
         worker.setValue("if('");

lib/ace/mode/liquid.js

@@ -0,0 +1,116 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Ajax.org Code Editor (ACE).
+ *
+ * The Initial Developer of the Original Code is
+ * Ajax.org B.V.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *      Fabian Jakobs <fabian AT ajax DOT org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+
+var oop = require("../lib/oop");
+var TextMode = require("./text").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var LiquidHighlightRules = require("./liquid_highlight_rules").LiquidHighlightRules;
+var MatchingBraceOutdent = require("./matching_brace_outdent").MatchingBraceOutdent;
+var Range = require("../range").Range;
+
+var Mode = function() {
+    this.$tokenizer = new Tokenizer(new LiquidHighlightRules().getRules());
+    this.$outdent = new MatchingBraceOutdent();
+};
+oop.inherits(Mode, TextMode);
+
+(function() {
+
+    this.toggleCommentLines = function(state, doc, startRow, endRow) {
+        var outdent = true;
+        var outentedRows = [];
+        var re = /^(\s*)#/;
+
+        for (var i=startRow; i<= endRow; i++) {
+            if (!re.test(doc.getLine(i))) {
+                outdent = false;
+                break;
+            }
+        }
+
+        if (outdent) {
+            var deleteRange = new Range(0, 0, 0, 0);
+            for (var i=startRow; i<= endRow; i++)
+            {
+                var line = doc.getLine(i);
+                var m = line.match(re);
+                deleteRange.start.row = i;
+                deleteRange.end.row = i;
+                deleteRange.end.column = m[0].length;
+                doc.replace(deleteRange, m[1]);
+            }
+        }
+        else {
+            doc.indentRows(startRow, endRow, "#");
+        }
+    };
+
+    this.getNextLineIndent = function(state, line, tab) {
+        var indent = this.$getIndent(line);
+
+        var tokenizedLine = this.$tokenizer.getLineTokens(line, state);
+        var tokens = tokenizedLine.tokens;
+        var endState = tokenizedLine.state;
+
+        if (tokens.length && tokens[tokens.length-1].type == "comment") {
+            return indent;
+        }
+
+        if (state == "start") {
+            var match = line.match(/^.*[\{\(\[]\s*$/);
+            if (match) {
+                indent += tab;
+            }
+        }
+
+        return indent;
+    };
+
+    this.checkOutdent = function(state, line, input) {
+        return this.$outdent.checkOutdent(line, input);
+    };
+
+    this.autoOutdent = function(state, doc, row) {
+        this.$outdent.autoOutdent(doc, row);
+    };
+
+}).call(Mode.prototype);
+
+exports.Mode = Mode;
+});

lib/ace/mode/liquid_highlight_rules.js

@@ -0,0 +1,220 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Ajax.org Code Editor (ACE).
+ *
+ * The Initial Developer of the Original Code is
+ * Ajax.org B.V.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *      Fabian Jakobs <fabian AT ajax DOT org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var CssHighlightRules = require("./css_highlight_rules").CssHighlightRules;
+var JavaScriptHighlightRules = require("./javascript_highlight_rules").JavaScriptHighlightRules;
+var lang = require("../lib/lang");
+var xmlUtil = require("./xml_util");
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var LiquidHighlightRules = function() {
+
+    // see: https://developer.mozilla.org/en/Liquid/Reference/Global_Objects
+    var functions = lang.arrayToMap(
+      // Standard Filters
+        ("date|capitalize|downcase|upcase|first|last|join|sort|map|size|escape|" +
+         "escape_once|strip_html|strip_newlines|newline_to_br|replace|replace_first|" +
+         "truncate|truncatewords|prepend|append|minus|plus|times|divided_by|split"
+      ).split("|")
+    );
+
+    var keywords = lang.arrayToMap(
+      // Standard Tags
+        ("capture|endcapture|case|endcase|when|comment|endcomment|" +
+        "cycle|for|endfor|in|reversed|if|endif|else|elsif|include|endinclude|unless|endunless|" +
+     // Commonly used tags
+        "style|text|image|widget|plugin|marker|endmarker|tablerow|endtablerow").split("|")
+    );
+
+    var builtinVariables = lang.arrayToMap(
+        ['forloop']
+        // ("forloop\\.(length|index|index0|rindex|rindex0|first|last)|limit|offset|range" +
+        // "tablerowloop\\.(length|index|index0|rindex|rindex0|first|last|col|col0|"+
+        // "col_first|col_last)").split("|")
+    );
+
+    var definitions = lang.arrayToMap(("assign").split("|"));
+
+    // regexp must not have capturing parentheses. Use (?:) instead.
+    // regexps are ordered -> the first match is used
+
+    this.$rules = {
+            start : [{
+                token : "variable",
+                regex : "{%",
+                next : "liquid_start"
+            }, {
+                token : "variable",
+                regex : "{{",
+                next : "liquid_start"
+            }, {
+                token : "meta.tag",
+                merge : true,
+                regex : "<\\!\\[CDATA\\[",
+                next : "cdata"
+            }, {
+                token : "xml_pe",
+                regex : "<\\?.*?\\?>"
+            }, {
+                token : "comment",
+                merge : true,
+                regex : "<\\!--",
+                next : "comment"
+            }, {
+                token : "meta.tag",
+                regex : "<(?=\\s*script\\b)",
+                next : "script"
+            }, {
+                token : "meta.tag",
+                regex : "<(?=\\s*style\\b)",
+                next : "style"
+            }, {
+                token : "meta.tag", // opening tag
+                regex : "<\\/?",
+                next : "tag"
+            }, {
+                token : "text",
+                regex : "\\s+"
+            }, {
+                token : "text",
+                regex : "[^<]+"
+            } ],
+
+            cdata : [ {
+                token : "text",
+                regex : "\\]\\]>",
+                next : "start"
+            }, {
+                token : "text",
+                merge : true,
+                regex : "\\s+"
+            }, {
+                token : "text",
+                merge : true,
+                regex : ".+"
+            } ],
+
+            comment : [ {
+                token : "comment",
+                regex : ".*?-->",
+                next : "start"
+            }, {
+                token : "comment",
+                merge : true,
+                regex : ".+"
+            } ] ,
+
+            liquid_start : [{
+                token: "variable",
+                regex: "}}",
+                next: "start"
+            }, {
+                token: "variable",
+                regex: "%}",
+                next: "start"
+            }, {
+                token : "string", // single line
+                regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
+            }, {
+                token : "string", // single line
+                regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
+            }, {
+                token : "constant.numeric", // hex
+                regex : "0[xX][0-9a-fA-F]+\\b"
+            }, {
+                token : "constant.numeric", // float
+                regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
+            }, {
+                token : "constant.language.boolean",
+                regex : "(?:true|false)\\b"
+            }, {
+                token : function(value) {
+                    if (functions.hasOwnProperty(value))
+                        return "support.function";
+                    else if (keywords.hasOwnProperty(value))
+                        return "keyword";
+                    else if (builtinVariables.hasOwnProperty(value))
+                        return "variable.language";
+                    else if (definitions.hasOwnProperty(value))
+                        return "keyword.definition";
+                    else
+                        return "identifier";
+                },
+                regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            }, {
+                token : "keyword.operator",
+                regex : "\/|\\*|\\-|\\+|=|!=|\\?\\:"
+            }, {
+                token : "paren.lparen",
+                regex : /[\[\({]/
+            }, {
+                token : "paren.rparen",
+                regex : /[\])}]/
+            }, {
+                token : "text",
+                regex : "\\s+"
+            }]
+    };
+
+    xmlUtil.tag(this.$rules, "tag", "start");
+    xmlUtil.tag(this.$rules, "style", "css-start");
+    xmlUtil.tag(this.$rules, "script", "js-start");
+
+    this.embedRules(JavaScriptHighlightRules, "js-", [{
+        token: "comment",
+        regex: "\\/\\/.*(?=<\\/script>)",
+        next: "tag"
+    }, {
+        token: "meta.tag",
+        regex: "<\\/(?=script)",
+        next: "tag"
+    }]);
+
+    this.embedRules(CssHighlightRules, "css-", [{
+        token: "meta.tag",
+        regex: "<\\/(?=style)",
+        next: "tag"
+    }]);
+};
+oop.inherits(LiquidHighlightRules, TextHighlightRules);
+
+exports.LiquidHighlightRules = LiquidHighlightRules;
+});

lib/ace/mode/liquid_highlight_rules_test.js

@@ -0,0 +1,91 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Ajax.org Code Editor (ACE).
+ *
+ * The Initial Developer of the Original Code is
+ * Ajax.org B.V.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *      Fabian Jakobs <fabian AT ajax DOT org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+if (typeof process !== "undefined") {
+    require("amd-loader");
+}
+
+define(function(require, exports, module) {
+
+var LiquidMode = require("./liquid").Mode;
+var assert = require("../test/assertions");
+
+module.exports = {
+
+    name: "Liquid Tokenizer",
+
+    setUp : function() {
+        this.tokenizer = new LiquidMode().getTokenizer();
+    },
+
+    "test: tokenize tags" : function() {
+        var line = "for one in many";
+
+        var tokens = this.tokenizer.getLineTokens(line, "liquid_start").tokens;
+
+        assert.equal(7, tokens.length);
+        assert.equal("keyword", tokens[0].type);
+        assert.equal("text", tokens[1].type);
+        assert.equal("identifier", tokens[2].type);
+        assert.equal("text", tokens[3].type);
+        assert.equal("keyword", tokens[4].type);
+        assert.equal("text", tokens[5].type);
+        assert.equal("identifier", tokens[6].type);
+    },
+
+    "test: tokenize parens" : function() {
+        var line = "[{( )}]";
+
+        var tokens = this.tokenizer.getLineTokens(line, "liquid_start").tokens;
+
+        assert.equal(7, tokens.length);
+        assert.equal("paren.lparen", tokens[0].type);
+        assert.equal("paren.lparen", tokens[1].type);
+        assert.equal("paren.lparen", tokens[2].type);
+        assert.equal("text", tokens[3].type);
+        assert.equal("paren.rparen", tokens[4].type);
+        assert.equal("paren.rparen", tokens[5].type);
+        assert.equal("paren.rparen", tokens[6].type);
+    }
+
+};
+
+});
+
+if (typeof module !== "undefined" && module === require.main) {
+    require("asyncjs").test.testcase(module.exports).exec()
+}

lib/ace/mode/markdown_highlight_rules.js

@@ -64,7 +64,7 @@ var MarkdownHighlightRules = function() {
             token : "empty_line",
             regex : '^$'
         }, { // code span `
-            token : "support.function",
+            token : ["support.function", "support.function", "support.function"],
             regex : "(`+)([^\\r]*?[^`])(\\1)"
         }, { // code block
             token : "support.function",
@@ -94,7 +94,7 @@ var MarkdownHighlightRules = function() {
             next  : "blockquote"
         }, { // reference
             token : ["text", "constant", "text", "url", "string", "text"],
-            regex : "^([ ]{0,3}\\[)([^\\]]+)(\\]:\\s*)([^ ]+)(\\s*(?:[\"][^\"]+[\"])?\\s*)$"
+            regex : "^([ ]{0,3}\\[)([^\\]]+)(\\]:\\s*)([^ ]+)(\\s*(?:[\"][^\"]+[\"])?(\\s*))$"
         }, { // link by reference
             token : ["text", "string", "text", "constant", "text"],
             regex : "(\\[)((?:[[^\\]]*\\]|[^\\[\\]])*)(\\][ ]?(?:\\n[ ]*)?\\[)(.*?)(\\])"
@@ -120,10 +120,10 @@ var MarkdownHighlightRules = function() {
             regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
             next  : "listblock"
         }, { // strong ** __
-            token : "string",
+            token : ["string", "string", "string"],
             regex : "([*]{2}|[_]{2}(?=\\S))([^\\r]*?\\S[*_]*)(\\1)"
         }, { // emphasis * _
-            token : "string",
+            token : ["string", "string", "string"],
             regex : "([*]|[_](?=\\S))([^\\r]*?\\S[*_]*)(\\1)"
         }, { // 
             token : ["text", "url", "text"],

lib/ace/mode/php_highlight_rules.js

@@ -952,14 +952,14 @@ var PhpHighlightRules = function() {
                 regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
             }, {
                 token : "string", // multi line string start
-                regex : '["].*\\\\$',
+                regex : '["][\\s\\S]*',
                 next : "qqstring"
             }, {
                 token : "string", // single line
                 regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
             }, {
                 token : "string", // multi line string start
-                regex : "['].*\\\\$",
+                regex : "['][\\s\\S]+",
                 next : "qstring"
             }, {
                 token : "constant.numeric", // hex
@@ -1041,21 +1041,21 @@ var PhpHighlightRules = function() {
         "qqstring" : [
             {
                 token : "string",
-                regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
+                regex : '"',
                 next : "start"
             }, {
                 token : "string",
-                regex : '.+'
+                regex : '[^"]+'
             }
         ],
         "qstring" : [
             {
                 token : "string",
-                regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
+                regex : "'",
                 next : "start"
             }, {
                 token : "string",
-                regex : '.+'
+                regex : "[^']+"
             }
         ],
         "htmlcomment" : [
@@ -1094,7 +1094,7 @@ var PhpHighlightRules = function() {
                  next : "htmltag"
              }, {
                  token : "meta.tag",
-                 regex : ">",
+                 regex : ">"
              }, {
                  token : 'text',
                  regex : "(?:media|type|href)"
@@ -1104,7 +1104,7 @@ var PhpHighlightRules = function() {
              }, {
                  token : "paren.lparen",
                  regex : "\{",
-                 next : "cssdeclaration",
+                 next : "cssdeclaration"
              }, {
                  token : "keyword",
                  regex : "#[A-Za-z0-9\-\_\.]+"
@@ -1146,7 +1146,7 @@ var PhpHighlightRules = function() {
                    regex : ";",
                    next : "cssdeclaration"
                }
-         ],
+         ]
     };
 
     this.embedRules(DocCommentHighlightRules, "doc-",

lib/ace/mode/sh_highlight_rules.js

@@ -58,11 +58,6 @@ var ShHighlightRules = function() {
          ).split('|')
     );
 
-    var builtinVariables = lang.arrayToMap(
-            // TODO
-        ('$?|$$|$!|$SHLVL').split('|')
-    );
-
     var integer = "(?:(?:[1-9]\\d*)|(?:0))";
     // var integer = "(?:" + decimalInteger + ")";
 
@@ -71,6 +66,14 @@ var ShHighlightRules = function() {
     var pointFloat = "(?:(?:" + intPart + "?" + fraction + ")|(?:" + intPart + "\\.))";
     var exponentFloat = "(?:(?:" + pointFloat + "|" +  intPart + ")" + ")";
     var floatNumber = "(?:" + exponentFloat + "|" + pointFloat + ")";
+    var fileDescriptor = "(?:&" + intPart + ")";
+
+    var variableName = "[a-zA-Z][a-zA-Z0-9_]*";
+    var variable = "(?:(?:\\$" + variableName + ")|(?:" + variableName + "=))";
+
+    var builtinVariable = "(?:\\$(?:SHLVL|\\$|\\!|\\?))";
+
+    var func = "(?:" + variableName + "\\s*\\(\\))";
 
     this.$rules = {
         "start" : [ {
@@ -79,6 +82,18 @@ var ShHighlightRules = function() {
         }, {
             token : "string",           // " string
             regex : '"(?:[^\\\\]|\\\\.)*?"'
+        }, {
+            token : "variable.language",
+            regex : builtinVariable
+        }, {
+            token : "variable",
+            regex : variable
+        }, {
+            token : "support.function",
+            regex : func,
+        }, {
+            token : "support.function",
+            regex : fileDescriptor
         }, {
             token : "string",           // ' string
             regex : "'(?:[^\\\\]|\\\\.)*?'"
@@ -94,8 +109,6 @@ var ShHighlightRules = function() {
                     return "keyword";
                 else if (languageConstructs.hasOwnProperty(value))
                     return "constant.language";
-                else if (builtinVariables.hasOwnProperty(value))
-                    return "support.function";
                 else if (value == "debugger")
                     return "invalid.deprecated";
                 else

lib/ace/mode/xquery.js

@@ -0,0 +1,102 @@
+/*
+ *  eXide - web-based XQuery IDE
+ *  
+ *  Copyright (C) 2011 Wolfgang Meier
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextMode = require("./text").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var XQueryHighlightRules = require("./xquery_highlight_rules").XQueryHighlightRules;
+var XQueryBehaviour = require("./behaviour/xquery").XQueryBehaviour;
+var Range = require("../range").Range;
+
+var Mode = function(parent) {
+    this.$tokenizer = new Tokenizer(new XQueryHighlightRules().getRules());
+    this.$behaviour = new XQueryBehaviour(parent);
+};
+
+oop.inherits(Mode, TextMode);
+
+(function() {
+
+    this.getNextLineIndent = function(state, line, tab) {
+      var indent = this.$getIndent(line);
+      var match = line.match(/\s*(?:then|else|return|[{\(]|<\w+>)\s*$/);
+      if (match)
+        indent += tab;
+        return indent;
+    };
+    
+    this.checkOutdent = function(state, line, input) {
+      if (! /^\s+$/.test(line))
+            return false;
+
+        return /^\s*[\}\)]/.test(input);
+    };
+    
+    this.autoOutdent = function(state, doc, row) {
+      var line = doc.getLine(row);
+        var match = line.match(/^(\s*[\}\)])/);
+
+        if (!match) return 0;
+
+        var column = match[1].length;
+        var openBracePos = doc.findMatchingBracket({row: row, column: column});
+
+        if (!openBracePos || openBracePos.row == row) return 0;
+
+        var indent = this.$getIndent(doc.getLine(openBracePos.row));
+        doc.replace(new Range(row, 0, row, column-1), indent);
+    };
+
+    this.$getIndent = function(line) {
+        var match = line.match(/^(\s+)/);
+        if (match) {
+            return match[1];
+        }
+
+        return "";
+    };
+    
+    this.toggleCommentLines = function(state, doc, startRow, endRow) {
+        var i, line;
+        var outdent = true;
+        var re = /^\s*\(:(.*):\)/;
+
+        for (i=startRow; i<= endRow; i++) {
+            if (!re.test(doc.getLine(i))) {
+                outdent = false;
+                break;
+            }
+        }
+
+        var range = new Range(0, 0, 0, 0);
+        for (i=startRow; i<= endRow; i++) {
+            line = doc.getLine(i);
+            range.start.row  = i;
+            range.end.row    = i;
+            range.end.column = line.length;
+
+            doc.replace(range, outdent ? line.match(re)[1] : "(:" + line + ":)");
+        }
+    };
+}).call(Mode.prototype);
+
+exports.Mode = Mode;
+});

lib/ace/mode/xquery_highlight_rules.js

@@ -0,0 +1,145 @@
+/*
+ *  eXide - web-based XQuery IDE
+ *  
+ *  Copyright (C) 2011 Wolfgang Meier
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var lang = require("../lib/lang");
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var XQueryHighlightRules = function() {
+
+  var keywords = lang.arrayToMap(
+    ("return|for|let|where|order|by|declare|function|variable|xquery|version|option|namespace|import|module|when|encoding|" +
+     "switch|default|try|catch|group|tumbling|sliding|window|start|end|at|only|" +
+     "using|stemming|" +
+     "while|" + 
+     "external|" +
+     "if|then|else|as|and|or|typeswitch|case|ascending|descending|empty|in|count|updating|insert|delete|replace|value|node|attribute|text|element|into|of|with|contains").split("|")
+    );
+
+    // regexp must not have capturing parentheses
+    // regexps are ordered -> the first match is used
+
+    this.$rules = {
+        start : [ {
+            token : "text",
+            regex : "<\\!\\[CDATA\\[",
+            next : "cdata"
+        }, {
+            token : "xml_pe",
+            regex : "<\\?.*?\\?>"
+        }, {
+            token : "comment",
+            regex : "<\\!--",
+            next : "comment"
+    }, {
+      token : "comment",
+      regex : "\\(:",
+      next : "comment"
+        }, {
+            token : "text", // opening tag
+            regex : "<\\/?",
+            next : "tag"
+        }, {
+            token : "constant", // number
+            regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
+    }, {
+            token : "variable", // variable
+            regex : "\\$[a-zA-Z_][a-zA-Z0-9_\\-:]*\\b"
+    }, {
+      token: "string",
+      regex : '".*?"'
+    }, {
+      token: "string",
+      regex : "'.*?'"
+        }, {
+            token : "text",
+            regex : "\\s+"
+        }, {
+            token: "support.function",
+            regex: "\\w[\\w+_\\-:]+(?=\\()"
+        }, {
+      token : function(value) {
+            if (keywords[value])
+                return "keyword";
+            else
+                return "identifier";
+      },
+      regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+    }, {
+            token: "keyword.operator",
+            regex: "\\*|=|<|>|\\-|\\+|and|or|eq|ne|lt|gt"
+        }, {
+            token: "lparen",
+            regex: "[[({]"
+        }, {
+            token: "rparen",
+            regex: "[\\])}]"
+        } ],
+
+        tag : [ {
+            token : "text",
+            regex : ">",
+            next : "start"
+        }, {
+            token : "meta.tag",
+            regex : "[-_a-zA-Z0-9:]+"
+        }, {
+            token : "text",
+            regex : "\\s+"
+        }, {
+            token : "string",
+            regex : '".*?"'
+        }, {
+            token : "string",
+            regex : "'.*?'"
+        } ],
+
+        cdata : [ {
+            token : "text",
+            regex : "\\]\\]>",
+            next : "start"
+        }, {
+            token : "text",
+            regex : "\\s+"
+        }, {
+            token : "text",
+            regex : "(?:[^\\]]|\\](?!\\]>))+"
+        } ],
+
+        comment : [ {
+            token : "comment",
+            regex : ".*?-->",
+            next : "start"
+        }, {
+      token: "comment",
+      regex : ".*:\\)",
+      next : "start"
+        }, {
+            token : "comment",
+            regex : ".+"
+    } ]
+    };
+};
+
+oop.inherits(XQueryHighlightRules, TextHighlightRules);
+
+exports.XQueryHighlightRules = XQueryHighlightRules;
+});

lib/ace/narcissus/definitions.js

@@ -0,0 +1,696 @@
+/* vim: set sw=4 ts=4 et tw=78: */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is the Narcissus JavaScript engine.
+ *
+ * The Initial Developer of the Original Code is
+ * Brendan Eich <brendan@mozilla.org>.
+ * Portions created by the Initial Developer are Copyright (C) 2004
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Tom Austin <taustin@ucsc.edu>
+ *   Brendan Eich <brendan@mozilla.org>
+ *   Shu-Yu Guo <shu@rfrn.org>
+ *   Dave Herman <dherman@mozilla.com>
+ *   Dimitris Vardoulakis <dimvar@ccs.neu.edu>
+ *   Patrick Walton <pcwalton@mozilla.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/*
+ * Narcissus - JS implemented in JS.
+ *
+ * Well-known constants and lookup tables.  Many consts are generated from the
+ * tokens table via eval to minimize redundancy, so consumers must be compiled
+ * separately to take advantage of the simple switch-case constant propagation
+ * done by SpiderMonkey.
+ */
+
+define(function(require, exports, module) {
+
+var tokens = [
+    // End of source.
+    "END",
+
+    // Operators and punctuators.  Some pair-wise order matters, e.g. (+, -)
+    // and (UNARY_PLUS, UNARY_MINUS).
+    "\n", ";",
+    ",",
+    "=",
+    "?", ":", "CONDITIONAL",
+    "||",
+    "&&",
+    "|",
+    "^",
+    "&",
+    "==", "!=", "===", "!==",
+    "<", "<=", ">=", ">",
+    "<<", ">>", ">>>",
+    "+", "-",
+    "*", "/", "%",
+    "!", "~", "UNARY_PLUS", "UNARY_MINUS",
+    "++", "--",
+    ".",
+    "[", "]",
+    "{", "}",
+    "(", ")",
+
+    // Nonterminal tree node type codes.
+    "SCRIPT", "BLOCK", "LABEL", "FOR_IN", "CALL", "NEW_WITH_ARGS", "INDEX",
+    "ARRAY_INIT", "OBJECT_INIT", "PROPERTY_INIT", "GETTER", "SETTER",
+    "GROUP", "LIST", "LET_BLOCK", "ARRAY_COMP", "GENERATOR", "COMP_TAIL",
+
+    // Contextual keywords.
+    "IMPLEMENTS", "INTERFACE", "LET", "MODULE", "PACKAGE", "PRIVATE",
+    "PROTECTED", "PUBLIC", "STATIC", "USE", "YIELD",
+
+    // Terminals.
+    "IDENTIFIER", "NUMBER", "STRING", "REGEXP",
+
+    // Keywords.
+    "break",
+    "case", "catch", "const", "continue",
+    "debugger", "default", "delete", "do",
+    "else", "export",
+    "false", "finally", "for", "function",
+    "if", "import", "in", "instanceof",
+    "new", "null",
+    "return",
+    "switch",
+    "this", "throw", "true", "try", "typeof",
+    "var", "void",
+    "while", "with",
+];
+
+var strictKeywords = {
+    __proto__: null,
+    "implements": true,
+    "interface": true,
+    "let": true,
+    "module": true,
+    "package": true,
+    "private": true,
+    "protected": true,
+    "public": true,
+    "static": true,
+    "use": true,
+    "yield": true
+};
+
+var statementStartTokens = [
+    "break",
+    "const", "continue",
+    "debugger", "do",
+    "for",
+    "if",
+    "let",
+    "return",
+    "switch",
+    "throw", "try",
+    "var",
+    "yield",
+    "while", "with",
+];
+
+// Whitespace characters (see ECMA-262 7.2)
+var whitespaceChars = [
+    // normal whitespace:
+    "\u0009", "\u000B", "\u000C", "\u0020", "\u00A0", "\uFEFF",
+
+    // high-Unicode whitespace:
+    "\u1680", "\u180E",
+    "\u2000", "\u2001", "\u2002", "\u2003", "\u2004", "\u2005", "\u2006",
+    "\u2007", "\u2008", "\u2009", "\u200A",
+    "\u202F", "\u205F", "\u3000"
+];
+
+var whitespace = {};
+for (var i = 0; i < whitespaceChars.length; i++) {
+    whitespace[whitespaceChars[i]] = true;
+}
+
+// Operator and punctuator mapping from token to tree node type name.
+// NB: because the lexer doesn't backtrack, all token prefixes must themselves
+// be valid tokens (e.g. !== is acceptable because its prefixes are the valid
+// tokens != and !).
+var opTypeNames = {
+    '\n':   "NEWLINE",
+    ';':    "SEMICOLON",
+    ',':    "COMMA",
+    '?':    "HOOK",
+    ':':    "COLON",
+    '||':   "OR",
+    '&&':   "AND",
+    '|':    "BITWISE_OR",
+    '^':    "BITWISE_XOR",
+    '&':    "BITWISE_AND",
+    '===':  "STRICT_EQ",
+    '==':   "EQ",
+    '=':    "ASSIGN",
+    '!==':  "STRICT_NE",
+    '!=':   "NE",
+    '<<':   "LSH",
+    '<=':   "LE",
+    '<':    "LT",
+    '>>>':  "URSH",
+    '>>':   "RSH",
+    '>=':   "GE",
+    '>':    "GT",
+    '++':   "INCREMENT",
+    '--':   "DECREMENT",
+    '+':    "PLUS",
+    '-':    "MINUS",
+    '*':    "MUL",
+    '/':    "DIV",
+    '%':    "MOD",
+    '!':    "NOT",
+    '~':    "BITWISE_NOT",
+    '.':    "DOT",
+    '[':    "LEFT_BRACKET",
+    ']':    "RIGHT_BRACKET",
+    '{':    "LEFT_CURLY",
+    '}':    "RIGHT_CURLY",
+    '(':    "LEFT_PAREN",
+    ')':    "RIGHT_PAREN"
+};
+
+// Hash of keyword identifier to tokens index.  NB: we must null __proto__ to
+// avoid toString, etc. namespace pollution.
+var keywords = {__proto__: null};
+var mozillaKeywords = {__proto__: null};
+
+// Define const END, etc., based on the token names.  Also map name to index.
+var tokenIds = {};
+
+var hostSupportsEvalConst = (function() {
+    try {
+        return eval("(function(s) { eval(s); return x })('const x = true;')");
+    } catch (e) {
+        return false;
+    }
+})();
+
+// Building up a string to be eval'd in different contexts.
+var consts = hostSupportsEvalConst ? "const " : "var ";
+for (var i = 0, j = tokens.length; i < j; i++) {
+    if (i > 0)
+        consts += ", ";
+    var t = tokens[i];
+    var name;
+    if (/^[a-z]/.test(t)) {
+        name = t.toUpperCase();
+        if (name === "LET" || name === "YIELD")
+            mozillaKeywords[name] = i;
+        if (strictKeywords[name])
+            strictKeywords[name] = i;
+        keywords[t] = i;
+    } else {
+        name = (/^\W/.test(t) ? opTypeNames[t] : t);
+    }
+    consts += name + " = " + i;
+    tokenIds[name] = i;
+    tokens[t] = i;
+}
+consts += ";";
+
+var isStatementStartCode = {__proto__: null};
+for (i = 0, j = statementStartTokens.length; i < j; i++)
+    isStatementStartCode[keywords[statementStartTokens[i]]] = true;
+
+// Map assignment operators to their indexes in the tokens array.
+var assignOps = ['|', '^', '&', '<<', '>>', '>>>', '+', '-', '*', '/', '%'];
+
+for (i = 0, j = assignOps.length; i < j; i++) {
+    t = assignOps[i];
+    assignOps[t] = tokens[t];
+}
+
+function defineGetter(obj, prop, fn, dontDelete, dontEnum) {
+    Object.defineProperty(obj, prop,
+                          { get: fn, configurable: !dontDelete, enumerable: !dontEnum });
+}
+
+function defineGetterSetter(obj, prop, getter, setter, dontDelete, dontEnum) {
+    Object.defineProperty(obj, prop, {
+        get: getter,
+        set: setter,
+        configurable: !dontDelete,
+        enumerable: !dontEnum
+    });
+}
+
+function defineMemoGetter(obj, prop, fn, dontDelete, dontEnum) {
+    Object.defineProperty(obj, prop, {
+        get: function() {
+            var val = fn();
+            defineProperty(obj, prop, val, dontDelete, true, dontEnum);
+            return val;
+        },
+        configurable: true,
+        enumerable: !dontEnum
+    });
+}
+
+function defineProperty(obj, prop, val, dontDelete, readOnly, dontEnum) {
+    Object.defineProperty(obj, prop,
+                          { value: val, writable: !readOnly, configurable: !dontDelete,
+                            enumerable: !dontEnum });
+}
+
+// Returns true if fn is a native function.  (Note: SpiderMonkey specific.)
+function isNativeCode(fn) {
+    // Relies on the toString method to identify native code.
+    return ((typeof fn) === "function") && fn.toString().match(/\[native code\]/);
+}
+
+var Fpapply = Function.prototype.apply;
+
+function apply(f, o, a) {
+    return Fpapply.call(f, [o].concat(a));
+}
+
+var applyNew;
+
+// ES5's bind is a simpler way to implement applyNew
+if (Function.prototype.bind) {
+    applyNew = function applyNew(f, a) {
+        return new (f.bind.apply(f, [,].concat(Array.prototype.slice.call(a))))();
+    };
+} else {
+    applyNew = function applyNew(f, a) {
+        switch (a.length) {
+          case 0:
+            return new f();
+          case 1:
+            return new f(a[0]);
+          case 2:
+            return new f(a[0], a[1]);
+          case 3:
+            return new f(a[0], a[1], a[2]);
+          default:
+            var argStr = "a[0]";
+            for (var i = 1, n = a.length; i < n; i++)
+                argStr += ",a[" + i + "]";
+            return eval("new f(" + argStr + ")");
+        }
+    };
+}
+
+function getPropertyDescriptor(obj, name) {
+    while (obj) {
+        if (({}).hasOwnProperty.call(obj, name))
+            return Object.getOwnPropertyDescriptor(obj, name);
+        obj = Object.getPrototypeOf(obj);
+    }
+}
+
+function getPropertyNames(obj) {
+    var table = Object.create(null, {});
+    while (obj) {
+        var names = Object.getOwnPropertyNames(obj);
+        for (var i = 0, n = names.length; i < n; i++)
+            table[names[i]] = true;
+        obj = Object.getPrototypeOf(obj);
+    }
+    return Object.keys(table);
+}
+
+function getOwnProperties(obj) {
+    var map = {};
+    for (var name in Object.getOwnPropertyNames(obj))
+        map[name] = Object.getOwnPropertyDescriptor(obj, name);
+    return map;
+}
+
+function blacklistHandler(target, blacklist) {
+    var mask = Object.create(null, {});
+    var redirect = Dict.create(blacklist).mapObject(function(name) { return mask; });
+    return mixinHandler(redirect, target);
+}
+
+function whitelistHandler(target, whitelist) {
+    var catchall = Object.create(null, {});
+    var redirect = Dict.create(whitelist).mapObject(function(name) { return target; });
+    return mixinHandler(redirect, catchall);
+}
+
+/*
+ * Mixin proxies break the single-inheritance model of prototypes, so
+ * the handler treats all properties as own-properties:
+ *
+ *                  X
+ *                  |
+ *     +------------+------------+
+ *     |                 O       |
+ *     |                 |       |
+ *     |  O         O    O       |
+ *     |  |         |    |       |
+ *     |  O    O    O    O       |
+ *     |  |    |    |    |       |
+ *     |  O    O    O    O    O  |
+ *     |  |    |    |    |    |  |
+ *     +-(*)--(w)--(x)--(y)--(z)-+
+ */
+
+function mixinHandler(redirect, catchall) {
+    function targetFor(name) {
+        return hasOwn(redirect, name) ? redirect[name] : catchall;
+    }
+
+    function getMuxPropertyDescriptor(name) {
+        var desc = getPropertyDescriptor(targetFor(name), name);
+        if (desc)
+            desc.configurable = true;
+        return desc;
+    }
+
+    function getMuxPropertyNames() {
+        var names1 = Object.getOwnPropertyNames(redirect).filter(function(name) {
+            return name in redirect[name];
+        });
+        var names2 = getPropertyNames(catchall).filter(function(name) {
+            return !hasOwn(redirect, name);
+        });
+        return names1.concat(names2);
+    }
+
+    function enumerateMux() {
+        var result = Object.getOwnPropertyNames(redirect).filter(function(name) {
+            return name in redirect[name];
+        });
+        for (name in catchall) {
+            if (!hasOwn(redirect, name))
+                result.push(name);
+        };
+        return result;
+    }
+
+    function hasMux(name) {
+        return name in targetFor(name);
+    }
+
+    return {
+        getOwnPropertyDescriptor: getMuxPropertyDescriptor,
+        getPropertyDescriptor: getMuxPropertyDescriptor,
+        getOwnPropertyNames: getMuxPropertyNames,
+        defineProperty: function(name, desc) {
+            Object.defineProperty(targetFor(name), name, desc);
+        },
+        "delete": function(name) {
+            var target = targetFor(name);
+            return delete target[name];
+        },
+        // FIXME: ha ha ha
+        fix: function() { },
+        has: hasMux,
+        hasOwn: hasMux,
+        get: function(receiver, name) {
+            var target = targetFor(name);
+            return target[name];
+        },
+        set: function(receiver, name, val) {
+            var target = targetFor(name);
+            target[name] = val;
+            return true;
+        },
+        enumerate: enumerateMux,
+        keys: enumerateMux
+    };
+}
+
+function makePassthruHandler(obj) {
+    // Handler copied from
+    // http://wiki.ecmascript.org/doku.php?id=harmony:proxies&s=proxy%20object#examplea_no-op_forwarding_proxy
+    return {
+        getOwnPropertyDescriptor: function(name) {
+            var desc = Object.getOwnPropertyDescriptor(obj, name);
+
+            // a trapping proxy's properties must always be configurable
+            desc.configurable = true;
+            return desc;
+        },
+        getPropertyDescriptor: function(name) {
+            var desc = getPropertyDescriptor(obj, name);
+
+            // a trapping proxy's properties must always be configurable
+            desc.configurable = true;
+            return desc;
+        },
+        getOwnPropertyNames: function() {
+            return Object.getOwnPropertyNames(obj);
+        },
+        defineProperty: function(name, desc) {
+            Object.defineProperty(obj, name, desc);
+        },
+        "delete": function(name) { return delete obj[name]; },
+        fix: function() {
+            if (Object.isFrozen(obj)) {
+                return getOwnProperties(obj);
+            }
+
+            // As long as obj is not frozen, the proxy won't allow itself to be fixed.
+            return undefined; // will cause a TypeError to be thrown
+        },
+
+        has: function(name) { return name in obj; },
+        hasOwn: function(name) { return ({}).hasOwnProperty.call(obj, name); },
+        get: function(receiver, name) { return obj[name]; },
+
+        // bad behavior when set fails in non-strict mode
+        set: function(receiver, name, val) { obj[name] = val; return true; },
+        enumerate: function() {
+            var result = [];
+            for (name in obj) { result.push(name); };
+            return result;
+        },
+        keys: function() { return Object.keys(obj); }
+    };
+}
+
+var hasOwnProperty = ({}).hasOwnProperty;
+
+function hasOwn(obj, name) {
+    return hasOwnProperty.call(obj, name);
+}
+
+function Dict(table, size) {
+    this.table = table || Object.create(null, {});
+    this.size = size || 0;
+}
+
+Dict.create = function(table) {
+    var init = Object.create(null, {});
+    var size = 0;
+    var names = Object.getOwnPropertyNames(table);
+    for (var i = 0, n = names.length; i < n; i++) {
+        var name = names[i];
+        init[name] = table[name];
+        size++;
+    }
+    return new Dict(init, size);
+};
+
+Dict.prototype = {
+    has: function(x) { return hasOwnProperty.call(this.table, x); },
+    set: function(x, v) {
+        if (!hasOwnProperty.call(this.table, x))
+            this.size++;
+        this.table[x] = v;
+    },
+    get: function(x) { return this.table[x]; },
+    getDef: function(x, thunk) {
+        if (!hasOwnProperty.call(this.table, x)) {
+            this.size++;
+            this.table[x] = thunk();
+        }
+        return this.table[x];
+    },
+    forEach: function(f) {
+        var table = this.table;
+        for (var key in table)
+            f.call(this, key, table[key]);
+    },
+    map: function(f) {
+        var table1 = this.table;
+        var table2 = Object.create(null, {});
+        this.forEach(function(key, val) {
+            table2[key] = f.call(this, val, key);
+        });
+        return new Dict(table2, this.size);
+    },
+    mapObject: function(f) {
+        var table1 = this.table;
+        var table2 = Object.create(null, {});
+        this.forEach(function(key, val) {
+            table2[key] = f.call(this, val, key);
+        });
+        return table2;
+    },
+    toObject: function() {
+        return this.mapObject(function(val) { return val; });
+    },
+    choose: function() {
+        return Object.getOwnPropertyNames(this.table)[0];
+    },
+    remove: function(x) {
+        if (hasOwnProperty.call(this.table, x)) {
+            this.size--;
+            delete this.table[x];
+        }
+    },
+    copy: function() {
+        var table = Object.create(null, {});
+        for (var key in this.table)
+            table[key] = this.table[key];
+        return new Dict(table, this.size);
+    },
+    keys: function() {
+        return Object.keys(this.table);
+    },
+    toString: function() { return "[object Dict]" }
+};
+
+var _WeakMap = typeof WeakMap === "function" ? WeakMap : (function() {
+    // shim for ES6 WeakMap with poor asymptotics
+    function WeakMap(array) {
+        this.array = array || [];
+    }
+
+    function searchMap(map, key, found, notFound) {
+        var a = map.array;
+        for (var i = 0, n = a.length; i < n; i++) {
+            var pair = a[i];
+            if (pair.key === key)
+                return found(pair, i);
+        }
+        return notFound();
+    }
+
+    WeakMap.prototype = {
+        has: function(x) {
+            return searchMap(this, x, function() { return true }, function() { return false });
+        },
+        set: function(x, v) {
+            var a = this.array;
+            searchMap(this, x,
+                      function(pair) { pair.value = v },
+                      function() { a.push({ key: x, value: v }) });
+        },
+        get: function(x) {
+            return searchMap(this, x,
+                             function(pair) { return pair.value },
+                             function() { return null });
+        },
+        "delete": function(x) {
+            var a = this.array;
+            searchMap(this, x,
+                      function(pair, i) { a.splice(i, 1) },
+                      function() { });
+        },
+        toString: function() { return "[object WeakMap]" }
+    };
+
+    return WeakMap;
+})();
+
+// non-destructive stack
+function Stack(elts) {
+    this.elts = elts || null;
+}
+
+Stack.prototype = {
+    push: function(x) {
+        return new Stack({ top: x, rest: this.elts });
+    },
+    top: function() {
+        if (!this.elts)
+            throw new Error("empty stack");
+        return this.elts.top;
+    },
+    isEmpty: function() {
+        return this.top === null;
+    },
+    find: function(test) {
+        for (var elts = this.elts; elts; elts = elts.rest) {
+            if (test(elts.top))
+                return elts.top;
+        }
+        return null;
+    },
+    has: function(x) {
+        return Boolean(this.find(function(elt) { return elt === x }));
+    },
+    forEach: function(f) {
+        for (var elts = this.elts; elts; elts = elts.rest) {
+            f(elts.top);
+        }
+    }
+};
+
+if (!Array.prototype.copy) {
+    defineProperty(Array.prototype, "copy",
+                   function() {
+                       var result = [];
+                       for (var i = 0, n = this.length; i < n; i++)
+                           result[i] = this[i];
+                       return result;
+                   }, false, false, true);
+}
+
+if (!Array.prototype.top) {
+    defineProperty(Array.prototype, "top",
+                   function() {
+                       return this.length && this[this.length-1];
+                   }, false, false, true);
+}
+
+exports.tokens = tokens;
+exports.whitespace = whitespace;
+exports.opTypeNames = opTypeNames;
+exports.keywords = keywords;
+exports.mozillaKeywords = mozillaKeywords;
+exports.strictKeywords = strictKeywords;
+exports.isStatementStartCode = isStatementStartCode;
+exports.tokenIds = tokenIds;
+exports.consts = consts;
+exports.assignOps = assignOps;
+exports.defineGetter = defineGetter;
+exports.defineGetterSetter = defineGetterSetter;
+exports.defineMemoGetter = defineMemoGetter;
+exports.defineProperty = defineProperty;
+exports.isNativeCode = isNativeCode;
+exports.apply = apply;
+exports.applyNew = applyNew;
+exports.mixinHandler = mixinHandler;
+exports.whitelistHandler = whitelistHandler;
+exports.blacklistHandler = blacklistHandler;
+exports.makePassthruHandler = makePassthruHandler;
+exports.Dict = Dict;
+exports.WeakMap = _WeakMap;
+exports.Stack = Stack;
+
+});

lib/ace/narcissus/jsdefs.js

@@ -1,679 +0,0 @@
-/* vim: set sw=4 ts=4 et tw=78: */
-/* ***** BEGIN LICENSE BLOCK *****
- * Version: MPL 1.1/GPL 2.0/LGPL 2.1
- *
- * The contents of this file are subject to the Mozilla Public License Version
- * 1.1 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS" basis,
- * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
- * for the specific language governing rights and limitations under the
- * License.
- *
- * The Original Code is the Narcissus JavaScript engine.
- *
- * The Initial Developer of the Original Code is
- * Brendan Eich <brendan@mozilla.org>.
- * Portions created by the Initial Developer are Copyright (C) 2004
- * the Initial Developer. All Rights Reserved.
- *
- * Contributor(s):
- *   Tom Austin <taustin@ucsc.edu>
- *   Brendan Eich <brendan@mozilla.org>
- *   Shu-Yu Guo <shu@rfrn.org>
- *   Dave Herman <dherman@mozilla.com>
- *   Dimitris Vardoulakis <dimvar@ccs.neu.edu>
- *   Patrick Walton <pcwalton@mozilla.com>
- *
- * Alternatively, the contents of this file may be used under the terms of
- * either the GNU General Public License Version 2 or later (the "GPL"), or
- * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
- * in which case the provisions of the GPL or the LGPL are applicable instead
- * of those above. If you wish to allow use of your version of this file only
- * under the terms of either the GPL or the LGPL, and not to allow others to
- * use your version of this file under the terms of the MPL, indicate your
- * decision by deleting the provisions above and replace them with the notice
- * and other provisions required by the GPL or the LGPL. If you do not delete
- * the provisions above, a recipient may use your version of this file under
- * the terms of any one of the MPL, the GPL or the LGPL.
- *
- * ***** END LICENSE BLOCK ***** */
-
-/*
- * Narcissus - JS implemented in JS.
- *
- * Well-known constants and lookup tables.  Many consts are generated from the
- * tokens table via eval to minimize redundancy, so consumers must be compiled
- * separately to take advantage of the simple switch-case constant propagation
- * done by SpiderMonkey.
- */
-
-define(function(require, exports, module) {
-
-    var narcissus = {
-        options: {
-            version: 185,
-            // Global variables to hide from the interpreter
-            hiddenHostGlobals: { Narcissus: true },
-            // Desugar SpiderMonkey language extensions?
-            desugarExtensions: false
-        },
-        hostSupportsEvalConst: (function() {
-            try {
-                return eval("(function(s) { eval(s); return x })('const x = true;')");
-            } catch (e) {
-                return false;
-            }
-        })(),
-        hostGlobal: this
-    };
-    Narcissus = narcissus;
-
-    var tokens = [
-        // End of source.
-        "END",
-
-        // Operators and punctuators.  Some pair-wise order matters, e.g. (+, -)
-        // and (UNARY_PLUS, UNARY_MINUS).
-        "\n", ";",
-        ",",
-        "=",
-        "?", ":", "CONDITIONAL",
-        "||",
-        "&&",
-        "|",
-        "^",
-        "&",
-        "==", "!=", "===", "!==",
-        "<", "<=", ">=", ">",
-        "<<", ">>", ">>>",
-        "+", "-",
-        "*", "/", "%",
-        "!", "~", "UNARY_PLUS", "UNARY_MINUS",
-        "++", "--",
-        ".",
-        "[", "]",
-        "{", "}",
-        "(", ")",
-
-        // Nonterminal tree node type codes.
-        "SCRIPT", "BLOCK", "LABEL", "FOR_IN", "CALL", "NEW_WITH_ARGS", "INDEX",
-        "ARRAY_INIT", "OBJECT_INIT", "PROPERTY_INIT", "GETTER", "SETTER",
-        "GROUP", "LIST", "LET_BLOCK", "ARRAY_COMP", "GENERATOR", "COMP_TAIL",
-
-        // Terminals.
-        "IDENTIFIER", "NUMBER", "STRING", "REGEXP",
-
-        // Keywords.
-        "break",
-        "case", "catch", "const", "continue",
-        "debugger", "default", "delete", "do",
-        "else", "export",
-        "false", "finally", "for", "function",
-        "if", "import", "in", "instanceof",
-        "let", "module",
-        "new", "null",
-        "return",
-        "switch",
-        "this", "throw", "true", "try", "typeof",
-        "var", "void",
-        "yield",
-        "while", "with",
-    ];
-
-    var statementStartTokens = [
-        "break",
-        "const", "continue",
-        "debugger", "do",
-        "for",
-        "if",
-        "return",
-        "switch",
-        "throw", "try",
-        "var",
-        "yield",
-        "while", "with",
-    ];
-
-    // Whitespace characters (see ECMA-262 7.2)
-    var whitespaceChars = [
-        // normal whitespace:
-        "\u0009", "\u000B", "\u000C", "\u0020", "\u00A0", "\uFEFF", 
-
-        // high-Unicode whitespace:
-        "\u1680", "\u180E",
-        "\u2000", "\u2001", "\u2002", "\u2003", "\u2004", "\u2005", "\u2006",
-        "\u2007", "\u2008", "\u2009", "\u200A",
-        "\u202F", "\u205F", "\u3000"
-    ];
-
-    var whitespace = {};
-    for (var i = 0; i < whitespaceChars.length; i++) {
-        whitespace[whitespaceChars[i]] = true;
-    }
-
-    // Operator and punctuator mapping from token to tree node type name.
-    // NB: because the lexer doesn't backtrack, all token prefixes must themselves
-    // be valid tokens (e.g. !== is acceptable because its prefixes are the valid
-    // tokens != and !).
-    var opTypeNames = {
-        '\n':   "NEWLINE",
-        ';':    "SEMICOLON",
-        ',':    "COMMA",
-        '?':    "HOOK",
-        ':':    "COLON",
-        '||':   "OR",
-        '&&':   "AND",
-        '|':    "BITWISE_OR",
-        '^':    "BITWISE_XOR",
-        '&':    "BITWISE_AND",
-        '===':  "STRICT_EQ",
-        '==':   "EQ",
-        '=':    "ASSIGN",
-        '!==':  "STRICT_NE",
-        '!=':   "NE",
-        '<<':   "LSH",
-        '<=':   "LE",
-        '<':    "LT",
-        '>>>':  "URSH",
-        '>>':   "RSH",
-        '>=':   "GE",
-        '>':    "GT",
-        '++':   "INCREMENT",
-        '--':   "DECREMENT",
-        '+':    "PLUS",
-        '-':    "MINUS",
-        '*':    "MUL",
-        '/':    "DIV",
-        '%':    "MOD",
-        '!':    "NOT",
-        '~':    "BITWISE_NOT",
-        '.':    "DOT",
-        '[':    "LEFT_BRACKET",
-        ']':    "RIGHT_BRACKET",
-        '{':    "LEFT_CURLY",
-        '}':    "RIGHT_CURLY",
-        '(':    "LEFT_PAREN",
-        ')':    "RIGHT_PAREN"
-    };
-
-    // Hash of keyword identifier to tokens index.  NB: we must null __proto__ to
-    // avoid toString, etc. namespace pollution.
-    var keywords = {__proto__: null};
-
-    // Define const END, etc., based on the token names.  Also map name to index.
-    var tokenIds = {};
-
-    // Building up a string to be eval'd in different contexts.
-    var consts = Narcissus.hostSupportsEvalConst ? "const " : "var ";
-    for (var i = 0, j = tokens.length; i < j; i++) {
-        if (i > 0)
-            consts += ", ";
-        var t = tokens[i];
-        var name;
-        if (/^[a-z]/.test(t)) {
-            name = t.toUpperCase();
-            keywords[t] = i;
-        } else {
-            name = (/^\W/.test(t) ? opTypeNames[t] : t);
-        }
-        consts += name + " = " + i;
-        tokenIds[name] = i;
-        tokens[t] = i;
-    }
-    consts += ";";
-
-    var isStatementStartCode = {__proto__: null};
-    for (i = 0, j = statementStartTokens.length; i < j; i++)
-        isStatementStartCode[keywords[statementStartTokens[i]]] = true;
-
-    // Map assignment operators to their indexes in the tokens array.
-    var assignOps = ['|', '^', '&', '<<', '>>', '>>>', '+', '-', '*', '/', '%'];
-
-    for (i = 0, j = assignOps.length; i < j; i++) {
-        t = assignOps[i];
-        assignOps[t] = tokens[t];
-    }
-
-    function defineGetter(obj, prop, fn, dontDelete, dontEnum) {
-        Object.defineProperty(obj, prop,
-                              { get: fn, configurable: !dontDelete, enumerable: !dontEnum });
-    }
-
-    function defineGetterSetter(obj, prop, getter, setter, dontDelete, dontEnum) {
-        Object.defineProperty(obj, prop, {
-            get: getter,
-            set: setter,
-            configurable: !dontDelete,
-            enumerable: !dontEnum
-        });
-    }
-
-    function defineMemoGetter(obj, prop, fn, dontDelete, dontEnum) {
-        Object.defineProperty(obj, prop, {
-            get: function() {
-                var val = fn();
-                defineProperty(obj, prop, val, dontDelete, true, dontEnum);
-                return val;
-            },
-            configurable: true,
-            enumerable: !dontEnum
-        });
-    }
-
-    function defineProperty(obj, prop, val, dontDelete, readOnly, dontEnum) {
-        Object.defineProperty(obj, prop,
-                              { value: val, writable: !readOnly, configurable: !dontDelete,
-                                enumerable: !dontEnum });
-    }
-
-    // Returns true if fn is a native function.  (Note: SpiderMonkey specific.)
-    function isNativeCode(fn) {
-        // Relies on the toString method to identify native code.
-        return ((typeof fn) === "function") && fn.toString().match(/\[native code\]/);
-    }
-
-    function getPropertyDescriptor(obj, name) {
-        while (obj) {
-            if (({}).hasOwnProperty.call(obj, name))
-                return Object.getOwnPropertyDescriptor(obj, name);
-            obj = Object.getPrototypeOf(obj);
-        }
-    }
-
-    function getPropertyNames(obj) {
-        var table = Object.create(null, {});
-        while (obj) {
-            var names = Object.getOwnPropertyNames(obj);
-            for (var i = 0, n = names.length; i < n; i++)
-                table[names[i]] = true;
-            obj = Object.getPrototypeOf(obj);
-        }
-        return Object.keys(table);
-    }
-
-    function getOwnProperties(obj) {
-        var map = {};
-        for (var name in Object.getOwnPropertyNames(obj))
-            map[name] = Object.getOwnPropertyDescriptor(obj, name);
-        return map;
-    }
-
-    function blacklistHandler(target, blacklist) {
-        var mask = Object.create(null, {});
-        var redirect = StringMap.create(blacklist).mapObject(function(name) { return mask; });
-        return mixinHandler(redirect, target);
-    }
-
-    function whitelistHandler(target, whitelist) {
-        var catchall = Object.create(null, {});
-        var redirect = StringMap.create(whitelist).mapObject(function(name) { return target; });
-        return mixinHandler(redirect, catchall);
-    }
-
-    function mirrorHandler(target, writable) {
-        var handler = makePassthruHandler(target);
-
-        var defineProperty = handler.defineProperty;
-        handler.defineProperty = function(name, desc) {
-            if (!desc.enumerable)
-                throw new Error("mirror property must be enumerable");
-            if (!desc.configurable)
-                throw new Error("mirror property must be configurable");
-            if (desc.writable !== writable)
-                throw new Error("mirror property must " + (writable ? "" : "not ") + "be writable");
-            defineProperty(name, desc);
-        };
-
-        handler.fix = function() { };
-        handler.getOwnPropertyDescriptor = handler.getPropertyDescriptor;
-        handler.getOwnPropertyNames = getPropertyNames.bind(handler, target);
-        handler.keys = handler.enumerate;
-        handler["delete"] = function() { return false; };
-        handler.hasOwn = handler.has;
-        return handler;
-    }
-
-    /*
-     * Mixin proxies break the single-inheritance model of prototypes, so
-     * the handler treats all properties as own-properties:
-     *
-     *                  X
-     *                  |
-     *     +------------+------------+
-     *     |                 O       |
-     *     |                 |       |
-     *     |  O         O    O       |
-     *     |  |         |    |       |
-     *     |  O    O    O    O       |
-     *     |  |    |    |    |       |
-     *     |  O    O    O    O    O  |
-     *     |  |    |    |    |    |  |
-     *     +-(*)--(w)--(x)--(y)--(z)-+
-     */
-
-    function mixinHandler(redirect, catchall) {
-        function targetFor(name) {
-            return hasOwn(redirect, name) ? redirect[name] : catchall;
-        }
-
-        function getMuxPropertyDescriptor(name) {
-            var desc = getPropertyDescriptor(targetFor(name), name);
-            if (desc)
-                desc.configurable = true;
-            return desc;
-        }
-
-        function getMuxPropertyNames() {
-            var names1 = Object.getOwnPropertyNames(redirect).filter(function(name) {
-                return name in redirect[name];
-            });
-            var names2 = getPropertyNames(catchall).filter(function(name) {
-                return !hasOwn(redirect, name);
-            });
-            return names1.concat(names2);
-        }
-
-        function enumerateMux() {
-            var result = Object.getOwnPropertyNames(redirect).filter(function(name) {
-                return name in redirect[name];
-            });
-            for (name in catchall) {
-                if (!hasOwn(redirect, name))
-                    result.push(name);
-            };
-            return result;
-        }
-
-        function hasMux(name) {
-            return name in targetFor(name);
-        }
-
-        return {
-            getOwnPropertyDescriptor: getMuxPropertyDescriptor,
-            getPropertyDescriptor: getMuxPropertyDescriptor,
-            getOwnPropertyNames: getMuxPropertyNames,
-            defineProperty: function(name, desc) {
-                Object.defineProperty(targetFor(name), name, desc);
-            },
-            "delete": function(name) {
-                var target = targetFor(name);
-                return delete target[name];
-            },
-            // FIXME: ha ha ha
-            fix: function() { },
-            has: hasMux,
-            hasOwn: hasMux,
-            get: function(receiver, name) {
-                var target = targetFor(name);
-                return target[name];
-            },
-            set: function(receiver, name, val) {
-                var target = targetFor(name);
-                target[name] = val;
-                return true;
-            },
-            enumerate: enumerateMux,
-            keys: enumerateMux
-        };
-    }
-
-    function makePassthruHandler(obj) {
-        // Handler copied from
-        // http://wiki.ecmascript.org/doku.php?id=harmony:proxies&s=proxy%20object#examplea_no-op_forwarding_proxy
-        return {
-            getOwnPropertyDescriptor: function(name) {
-                var desc = Object.getOwnPropertyDescriptor(obj, name);
-
-                // a trapping proxy's properties must always be configurable
-                desc.configurable = true;
-                return desc;
-            },
-            getPropertyDescriptor: function(name) {
-                var desc = getPropertyDescriptor(obj, name);
-
-                // a trapping proxy's properties must always be configurable
-                desc.configurable = true;
-                return desc;
-            },
-            getOwnPropertyNames: function() {
-                return Object.getOwnPropertyNames(obj);
-            },
-            defineProperty: function(name, desc) {
-                Object.defineProperty(obj, name, desc);
-            },
-            "delete": function(name) { return delete obj[name]; },
-            fix: function() {
-                if (Object.isFrozen(obj)) {
-                    return getOwnProperties(obj);
-                }
-
-                // As long as obj is not frozen, the proxy won't allow itself to be fixed.
-                return undefined; // will cause a TypeError to be thrown
-            },
-
-            has: function(name) { return name in obj; },
-            hasOwn: function(name) { return ({}).hasOwnProperty.call(obj, name); },
-            get: function(receiver, name) { return obj[name]; },
-
-            // bad behavior when set fails in non-strict mode
-            set: function(receiver, name, val) { obj[name] = val; return true; },
-            enumerate: function() {
-                var result = [];
-                for (name in obj) { result.push(name); };
-                return result;
-            },
-            keys: function() { return Object.keys(obj); }
-        };
-    }
-
-    var hasOwnProperty = ({}).hasOwnProperty;
-
-    function hasOwn(obj, name) {
-        return hasOwnProperty.call(obj, name);
-    }
-
-    function StringMap(table, size) {
-        this.table = table || Object.create(null, {});
-        this.size = size || 0;
-    }
-
-    StringMap.create = function(table) {
-        var init = Object.create(null, {});
-        var size = 0;
-        var names = Object.getOwnPropertyNames(table);
-        for (var i = 0, n = names.length; i < n; i++) {
-            var name = names[i];
-            init[name] = table[name];
-            size++;
-        }
-        return new StringMap(init, size);
-    };
-
-    StringMap.prototype = {
-        has: function(x) { return hasOwnProperty.call(this.table, x); },
-        set: function(x, v) {
-            if (!hasOwnProperty.call(this.table, x))
-                this.size++;
-            this.table[x] = v;
-        },
-        get: function(x) { return this.table[x]; },
-        getDef: function(x, thunk) {
-            if (!hasOwnProperty.call(this.table, x)) {
-                this.size++;
-                this.table[x] = thunk();
-            }
-            return this.table[x];
-        },
-        forEach: function(f) {
-            var table = this.table;
-            for (var key in table)
-                f.call(this, key, table[key]);
-        },
-        map: function(f) {
-            var table1 = this.table;
-            var table2 = Object.create(null, {});
-            this.forEach(function(key, val) {
-                table2[key] = f.call(this, val, key);
-            });
-            return new StringMap(table2, this.size);
-        },
-        mapObject: function(f) {
-            var table1 = this.table;
-            var table2 = Object.create(null, {});
-            this.forEach(function(key, val) {
-                table2[key] = f.call(this, val, key);
-            });
-            return table2;
-        },
-        toObject: function() {
-            return this.mapObject(function(val) { return val; });
-        },
-        choose: function() {
-            return Object.getOwnPropertyNames(this.table)[0];
-        },
-        remove: function(x) {
-            if (hasOwnProperty.call(this.table, x)) {
-                this.size--;
-                delete this.table[x];
-            }
-        },
-        copy: function() {
-            var table = Object.create(null, {});
-            for (var key in this.table)
-                table[key] = this.table[key];
-            return new StringMap(table, this.size);
-        },
-        toString: function() { return "[object StringMap]" }
-    };
-
-    // an object-key table with poor asymptotics (replace with WeakMap when possible)
-    function ObjectMap(array) {
-        this.array = array || [];
-    }
-
-    function searchMap(map, key, found, notFound) {
-        var a = map.array;
-        for (var i = 0, n = a.length; i < n; i++) {
-            var pair = a[i];
-            if (pair.key === key)
-                return found(pair, i);
-        }
-        return notFound();
-    }
-
-    ObjectMap.prototype = {
-        has: function(x) {
-            return searchMap(this, x, function() { return true }, function() { return false });
-        },
-        set: function(x, v) {
-            var a = this.array;
-            searchMap(this, x,
-                      function(pair) { pair.value = v },
-                      function() { a.push({ key: x, value: v }) });
-        },
-        get: function(x) {
-            return searchMap(this, x,
-                             function(pair) { return pair.value },
-                             function() { return null });
-        },
-        getDef: function(x, thunk) {
-            var a = this.array;
-            return searchMap(this, x,
-                             function(pair) { return pair.value },
-                             function() {
-                                 var v = thunk();
-                                 a.push({ key: x, value: v });
-                                 return v;
-                             });
-        },
-        forEach: function(f) {
-            var a = this.array;
-            for (var i = 0, n = a.length; i < n; i++) {
-                var pair = a[i];
-                f.call(this, pair.key, pair.value);
-            }
-        },
-        choose: function() {
-            return this.array[0].key;
-        },
-        get size() {
-            return this.array.length;
-        },
-        remove: function(x) {
-            var a = this.array;
-            searchMap(this, x,
-                      function(pair, i) { a.splice(i, 1) },
-                      function() { });
-        },
-        copy: function() {
-            return new ObjectMap(this.array.map(function(pair) {
-                return { key: pair.key, value: pair.value }
-            }));
-        },
-        clear: function() {
-            this.array = [];
-        },
-        toString: function() { return "[object ObjectMap]" }
-    };
-
-    // non-destructive stack
-    function Stack(elts) {
-        this.elts = elts || null;
-    }
-
-    Stack.prototype = {
-        push: function(x) {
-            return new Stack({ top: x, rest: this.elts });
-        },
-        top: function() {
-            if (!this.elts)
-                throw new Error("empty stack");
-            return this.elts.top;
-        },
-        isEmpty: function() {
-            return this.top === null;
-        },
-        find: function(test) {
-            for (var elts = this.elts; elts; elts = elts.rest) {
-                if (test(elts.top))
-                    return elts.top;
-            }
-            return null;
-        },
-        has: function(x) {
-            return Boolean(this.find(function(elt) { return elt === x }));
-        },
-        forEach: function(f) {
-            for (var elts = this.elts; elts; elts = elts.rest) {
-                f(elts.top);
-            }
-        }
-    };
-
-    module.exports = {
-        tokens: tokens,
-        whitespace: whitespace,
-        opTypeNames: opTypeNames,
-        keywords: keywords,
-        isStatementStartCode: isStatementStartCode,
-        tokenIds: tokenIds,
-        consts: consts,
-        assignOps: assignOps,
-        defineGetter: defineGetter,
-        defineGetterSetter: defineGetterSetter,
-        defineMemoGetter: defineMemoGetter,
-        defineProperty: defineProperty,
-        isNativeCode: isNativeCode,
-        mirrorHandler: mirrorHandler,
-        mixinHandler: mixinHandler,
-        whitelistHandler: whitelistHandler,
-        blacklistHandler: blacklistHandler,
-        makePassthruHandler: makePassthruHandler,
-        StringMap: StringMap,
-        ObjectMap: ObjectMap,
-        Stack: Stack
-    };
-});

lib/ace/narcissus/jslex.js

@@ -1,549 +0,0 @@
-/* vim: set sw=4 ts=4 et tw=78: */
-/* ***** BEGIN LICENSE BLOCK *****
- * Version: MPL 1.1/GPL 2.0/LGPL 2.1
- *
- * The contents of this file are subject to the Mozilla Public License Version
- * 1.1 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS" basis,
- * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
- * for the specific language governing rights and limitations under the
- * License.
- *
- * The Original Code is the Narcissus JavaScript engine.
- *
- * The Initial Developer of the Original Code is
- * Brendan Eich <brendan@mozilla.org>.
- * Portions created by the Initial Developer are Copyright (C) 2004
- * the Initial Developer. All Rights Reserved.
- *
- * Contributor(s):
- *   Tom Austin <taustin@ucsc.edu>
- *   Brendan Eich <brendan@mozilla.org>
- *   Shu-Yu Guo <shu@rfrn.org>
- *   Dave Herman <dherman@mozilla.com>
- *   Dimitris Vardoulakis <dimvar@ccs.neu.edu>
- *   Patrick Walton <pcwalton@mozilla.com>
- *
- * Alternatively, the contents of this file may be used under the terms of
- * either the GNU General Public License Version 2 or later (the "GPL"), or
- * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
- * in which case the provisions of the GPL or the LGPL are applicable instead
- * of those above. If you wish to allow use of your version of this file only
- * under the terms of either the GPL or the LGPL, and not to allow others to
- * use your version of this file under the terms of the MPL, indicate your
- * decision by deleting the provisions above and replace them with the notice
- * and other provisions required by the GPL or the LGPL. If you do not delete
- * the provisions above, a recipient may use your version of this file under
- * the terms of any one of the MPL, the GPL or the LGPL.
- *
- * ***** END LICENSE BLOCK ***** */
-
-/*
- * Narcissus - JS implemented in JS.
- *
- * Lexical scanner.
- */
-
- define(function(require, exports, module) {
-
-    var definitions = require("./jsdefs");
-
-    // Set constants in the local scope.
-    eval(definitions.consts);
-
-    // Banned keywords by language version
-    const blackLists = { 160: {}, 185: {}, harmony: {} };
-    blackLists[160][LET] = true;
-    blackLists[160][MODULE] = true;
-    blackLists[160][YIELD] = true;
-    blackLists[185][MODULE] = true;
-
-    // Build up a trie of operator tokens.
-    var opTokens = {};
-    for (var op in definitions.opTypeNames) {
-        if (op === '\n' || op === '.')
-            continue;
-
-        var node = opTokens;
-        for (var i = 0; i < op.length; i++) {
-            var ch = op[i];
-            if (!(ch in node))
-                node[ch] = {};
-            node = node[ch];
-            node.op = op;
-        }
-    }
-
-    /*
-     * Tokenizer :: (source, filename, line number) -> Tokenizer
-     */
-    function Tokenizer(s, f, l) {
-        this.cursor = 0;
-        this.source = String(s);
-        this.tokens = [];
-        this.tokenIndex = 0;
-        this.lookahead = 0;
-        this.scanNewlines = false;
-        this.unexpectedEOF = false;
-        this.filename = f || "";
-        this.lineno = l || 1;
-        this.blackList = blackLists[Narcissus.options.version];
-        this.blockComments = null;
-    }
-
-    Tokenizer.prototype = {
-        get done() {
-            // We need to set scanOperand to true here because the first thing
-            // might be a regexp.
-            return this.peek(true) === END;
-        },
-
-        get token() {
-            return this.tokens[this.tokenIndex];
-        },
-
-        match: function (tt, scanOperand) {
-            return this.get(scanOperand) === tt || this.unget();
-        },
-
-        mustMatch: function (tt) {
-            if (!this.match(tt)) {
-                throw this.newSyntaxError("Missing " +
-                                          definitions.tokens[tt].toLowerCase());
-            }
-            return this.token;
-        },
-
-        peek: function (scanOperand) {
-            var tt, next;
-            if (this.lookahead) {
-                next = this.tokens[(this.tokenIndex + this.lookahead) & 3];
-                tt = (this.scanNewlines && next.lineno !== this.lineno)
-                     ? NEWLINE
-                     : next.type;
-            } else {
-                tt = this.get(scanOperand);
-                this.unget();
-            }
-            return tt;
-        },
-
-        peekOnSameLine: function (scanOperand) {
-            this.scanNewlines = true;
-            var tt = this.peek(scanOperand);
-            this.scanNewlines = false;
-            return tt;
-        },
-
-        lastBlockComment: function() {
-            var length = this.blockComments.length;
-            return length ? this.blockComments[length - 1] : null;
-        },
-
-        // Eat comments and whitespace.
-        skip: function () {
-            var input = this.source;
-            this.blockComments = [];
-            for (;;) {
-                var ch = input[this.cursor++];
-                var next = input[this.cursor];
-                // handle \r, \r\n and (always preferable) \n
-                if (ch === '\r') {
-                    // if the next character is \n, we don't care about this at all
-                    if (next === '\n') continue;
-
-                    // otherwise, we want to consider this as a newline
-                    ch = '\n';
-                }
-
-                if (ch === '\n' && !this.scanNewlines) {
-                    this.lineno++;
-                } else if (ch === '/' && next === '*') {
-                    var commentStart = ++this.cursor;
-                    for (;;) {
-                        ch = input[this.cursor++];
-                        if (ch === undefined)
-                            throw this.newSyntaxError("Unterminated comment");
-
-                        if (ch === '*') {
-                            next = input[this.cursor];
-                            if (next === '/') {
-                                var commentEnd = this.cursor - 1;
-                                this.cursor++;
-                                break;
-                            }
-                        } else if (ch === '\n') {
-                            this.lineno++;
-                        }
-                    }
-                    this.blockComments.push(input.substring(commentStart, commentEnd));
-                } else if (ch === '/' && next === '/') {
-                    this.cursor++;
-                    for (;;) {
-                        ch = input[this.cursor++];
-                        next = input[this.cursor];
-                        if (ch === undefined)
-                            return;
-
-                        if (ch === '\r') {
-                            // check for \r\n
-                            if (next !== '\n') ch = '\n';
-                        }
-
-                        if (ch === '\n') {
-                            if (this.scanNewlines) {
-                                this.cursor--;
-                            } else {
-                                this.lineno++;
-                            }
-                            break;
-                        }
-                    }
-                } else if (!(ch in definitions.whitespace)) {
-                    this.cursor--;
-                    return;
-                }
-            }
-        },
-
-        // Lex the exponential part of a number, if present. Return true iff an
-        // exponential part was found.
-        lexExponent: function() {
-            var input = this.source;
-            var next = input[this.cursor];
-            if (next === 'e' || next === 'E') {
-                this.cursor++;
-                ch = input[this.cursor++];
-                if (ch === '+' || ch === '-')
-                    ch = input[this.cursor++];
-
-                if (ch < '0' || ch > '9')
-                    throw this.newSyntaxError("Missing exponent");
-
-                do {
-                    ch = input[this.cursor++];
-                } while (ch >= '0' && ch <= '9');
-                this.cursor--;
-
-                return true;
-            }
-
-            return false;
-        },
-
-        lexZeroNumber: function (ch) {
-            var token = this.token, input = this.source;
-            token.type = NUMBER;
-
-            ch = input[this.cursor++];
-            if (ch === '.') {
-                do {
-                    ch = input[this.cursor++];
-                } while (ch >= '0' && ch <= '9');
-                this.cursor--;
-
-                this.lexExponent();
-                token.value = parseFloat(token.start, this.cursor);
-            } else if (ch === 'x' || ch === 'X') {
-                do {
-                    ch = input[this.cursor++];
-                } while ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') ||
-                         (ch >= 'A' && ch <= 'F'));
-                this.cursor--;
-
-                token.value = parseInt(input.substring(token.start, this.cursor));
-            } else if (ch >= '0' && ch <= '7') {
-                do {
-                    ch = input[this.cursor++];
-                } while (ch >= '0' && ch <= '7');
-                this.cursor--;
-
-                token.value = parseInt(input.substring(token.start, this.cursor));
-            } else {
-                this.cursor--;
-                this.lexExponent();     // 0E1, &c.
-                token.value = 0;
-            }
-        },
-
-        lexNumber: function (ch) {
-            var token = this.token, input = this.source;
-            token.type = NUMBER;
-
-            var floating = false;
-            do {
-                ch = input[this.cursor++];
-                if (ch === '.' && !floating) {
-                    floating = true;
-                    ch = input[this.cursor++];
-                }
-            } while (ch >= '0' && ch <= '9');
-
-            this.cursor--;
-
-            var exponent = this.lexExponent();
-            floating = floating || exponent;
-
-            var str = input.substring(token.start, this.cursor);
-            token.value = floating ? parseFloat(str) : parseInt(str);
-        },
-
-        lexDot: function (ch) {
-            var token = this.token, input = this.source;
-            var next = input[this.cursor];
-            if (next >= '0' && next <= '9') {
-                do {
-                    ch = input[this.cursor++];
-                } while (ch >= '0' && ch <= '9');
-                this.cursor--;
-
-                this.lexExponent();
-
-                token.type = NUMBER;
-                token.value = parseFloat(token.start, this.cursor);
-            } else {
-                token.type = DOT;
-                token.assignOp = null;
-                token.value = '.';
-            }
-        },
-
-        lexString: function (ch) {
-            var token = this.token, input = this.source;
-            token.type = STRING;
-
-            var hasEscapes = false;
-            var delim = ch;
-            if (input.length <= this.cursor)
-                throw this.newSyntaxError("Unterminated string literal");
-            while ((ch = input[this.cursor++]) !== delim) {
-                if (this.cursor == input.length)
-                    throw this.newSyntaxError("Unterminated string literal");
-                if (ch === '\\') {
-                    hasEscapes = true;
-                    if (++this.cursor == input.length)
-                        throw this.newSyntaxError("Unterminated string literal");
-                }
-            }
-
-            token.value = hasEscapes
-                          ? eval(input.substring(token.start, this.cursor))
-                          : input.substring(token.start + 1, this.cursor - 1);
-        },
-
-        lexRegExp: function (ch) {
-            var token = this.token, input = this.source;
-            token.type = REGEXP;
-
-            do {
-                ch = input[this.cursor++];
-                if (ch === '\\') {
-                    this.cursor++;
-                } else if (ch === '[') {
-                    do {
-                        if (ch === undefined)
-                            throw this.newSyntaxError("Unterminated character class");
-
-                        if (ch === '\\')
-                            this.cursor++;
-
-                        ch = input[this.cursor++];
-                    } while (ch !== ']');
-                } else if (ch === undefined) {
-                    throw this.newSyntaxError("Unterminated regex");
-                }
-            } while (ch !== '/');
-
-            do {
-                ch = input[this.cursor++];
-            } while (ch >= 'a' && ch <= 'z');
-
-            this.cursor--;
-
-            token.value = eval(input.substring(token.start, this.cursor));
-        },
-
-        lexOp: function (ch) {
-            var token = this.token, input = this.source;
-
-            // A bit ugly, but it seems wasteful to write a trie lookup routine
-            // for only 3 characters...
-            var node = opTokens[ch];
-            var next = input[this.cursor];
-            if (next in node) {
-                node = node[next];
-                this.cursor++;
-                next = input[this.cursor];
-                if (next in node) {
-                    node = node[next];
-                    this.cursor++;
-                    next = input[this.cursor];
-                }
-            }
-
-            var op = node.op;
-            if (definitions.assignOps[op] && input[this.cursor] === '=') {
-                this.cursor++;
-                token.type = ASSIGN;
-                token.assignOp = definitions.tokenIds[definitions.opTypeNames[op]];
-                op += '=';
-            } else {
-                token.type = definitions.tokenIds[definitions.opTypeNames[op]];
-                token.assignOp = null;
-            }
-
-            token.value = op;
-        },
-
-        // FIXME: Unicode escape sequences
-        lexIdent: function (ch) {
-            var token = this.token;
-            var id = ch;
-
-            while ((ch = this.getValidIdentifierChar(false)) !== null) {
-                id += ch;
-            }
-
-            token.type = definitions.keywords[id] || IDENTIFIER;
-            if (token.type in this.blackList) {
-                // banned keyword, this is an identifier
-                token.type = IDENTIFIER;
-            }
-            token.value = id;
-        },
-
-        /*
-         * Tokenizer.get :: void -> token type
-         *
-         * Consume input *only* if there is no lookahead.
-         * Dispatch to the appropriate lexing function depending on the input.
-         */
-        get: function (scanOperand) {
-            var token;
-            while (this.lookahead) {
-                --this.lookahead;
-                this.tokenIndex = (this.tokenIndex + 1) & 3;
-                token = this.tokens[this.tokenIndex];
-                if (token.type !== NEWLINE || this.scanNewlines)
-                    return token.type;
-            }
-
-            this.skip();
-
-            this.tokenIndex = (this.tokenIndex + 1) & 3;
-            token = this.tokens[this.tokenIndex];
-            if (!token)
-                this.tokens[this.tokenIndex] = token = {};
-
-            var input = this.source;
-            if (this.cursor >= input.length)
-                return token.type = END;
-
-            token.start = this.cursor;
-            token.lineno = this.lineno;
-
-            var ich = this.getValidIdentifierChar(true);
-            var ch = (ich === null) ? input[this.cursor++] : null;
-            if (ich !== null) {
-                this.lexIdent(ich);
-            } else if (scanOperand && ch === '/') {
-                this.lexRegExp(ch);
-            } else if (ch in opTokens) {
-                this.lexOp(ch);
-            } else if (ch === '.') {
-                this.lexDot(ch);
-            } else if (ch >= '1' && ch <= '9') {
-                this.lexNumber(ch);
-            } else if (ch === '0') {
-                this.lexZeroNumber(ch);
-            } else if (ch === '"' || ch === "'") {
-                this.lexString(ch);
-            } else if (this.scanNewlines && (ch === '\n' || ch === '\r')) {
-                // if this was a \r, look for \r\n
-                if (ch === '\r' && input[this.cursor] === '\n') this.cursor++;
-                token.type = NEWLINE;
-                token.value = '\n';
-                this.lineno++;
-            } else {
-                throw this.newSyntaxError("Illegal token");
-            }
-
-            token.end = this.cursor;
-            return token.type;
-        },
-
-        /*
-         * Tokenizer.unget :: void -> undefined
-         *
-         * Match depends on unget returning undefined.
-         */
-        unget: function () {
-            if (++this.lookahead === 4) throw "PANIC: too much lookahead!";
-            this.tokenIndex = (this.tokenIndex - 1) & 3;
-        },
-
-        newSyntaxError: function (m) {
-            m = (this.filename ? this.filename + ":" : "") + this.lineno + ": " + m;
-            var e = new SyntaxError(m, this.filename, this.lineno);
-            e.source = this.source;
-            e.cursor = this.lookahead
-                       ? this.tokens[(this.tokenIndex + this.lookahead) & 3].start
-                       : this.cursor;
-            return e;
-        },
-
-        /* Gets a single valid identifier char from the input stream, or null
-         * if there is none.
-         * Since JavaScript provides no convenient way to determine if a
-         * character is in a particular Unicode category, we use
-         * metacircularity to accomplish this (oh yeaaaah!) */
-        getValidIdentifierChar: function(first) {
-            var input = this.source;
-            if (this.cursor >= input.length) return null;
-            var ch = input[this.cursor];
-
-            // first check for \u escapes
-            if (ch === '\\' && input[this.cursor+1] === 'u') {
-                // get the character value
-                try {
-                    ch = String.fromCharCode(parseInt(
-                        input.substring(this.cursor + 2, this.cursor + 6),
-                        16));
-                } catch (ex) {
-                    return null;
-                }
-                this.cursor += 5;
-            }
-
-            // check directly for ASCII
-            if (ch <= "\u007F") {
-                if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '$' || ch === '_' ||
-                    (!first && (ch >= '0' && ch <= '9'))) {
-                    this.cursor++;
-                    return ch;
-                }
-                return null;
-            }
-    
-            // create an object to test this in
-            var x = {};
-            x["x"+ch] = true;
-            x[ch] = true;
-    
-            // then use eval to determine if it's a valid character
-            var valid = false;
-            try {
-                valid = (Function("x", "return (x." + (first?"":"x") + ch + ");")(x) === true);
-            } catch (ex) {}
-            if (valid) this.cursor++;
-            return (valid ? ch : null);
-        },
-    };
-
-
-    module.exports = { Tokenizer: Tokenizer };
-
-});

lib/ace/narcissus/lexer.js

@@ -0,0 +1,598 @@
+/* vim: set sw=4 ts=4 et tw=78: */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is the Narcissus JavaScript engine.
+ *
+ * The Initial Developer of the Original Code is
+ * Brendan Eich <brendan@mozilla.org>.
+ * Portions created by the Initial Developer are Copyright (C) 2004
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Tom Austin <taustin@ucsc.edu>
+ *   Brendan Eich <brendan@mozilla.org>
+ *   Shu-Yu Guo <shu@rfrn.org>
+ *   Stephan Herhut <stephan.a.herhut@intel.com>
+ *   Dave Herman <dherman@mozilla.com>
+ *   Dimitris Vardoulakis <dimvar@ccs.neu.edu>
+ *   Patrick Walton <pcwalton@mozilla.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/*
+ * Narcissus - JS implemented in JS.
+ *
+ * Lexical scanner.
+ */
+
+ define(function(require, exports, module) {
+
+var definitions = require('./definitions');
+
+// Set constants in the local scope.
+eval(definitions.consts);
+
+// Build up a trie of operator tokens.
+var opTokens = {};
+for (var op in definitions.opTypeNames) {
+    if (op === '\n' || op === '.')
+        continue;
+
+    var node = opTokens;
+    for (var i = 0; i < op.length; i++) {
+        var ch = op[i];
+        if (!(ch in node))
+            node[ch] = {};
+        node = node[ch];
+        node.op = op;
+    }
+}
+
+/*
+ * Since JavaScript provides no convenient way to determine if a
+ * character is in a particular Unicode category, we use
+ * metacircularity to accomplish this (oh yeaaaah!)
+ */
+function isValidIdentifierChar(ch, first) {
+    // check directly for ASCII
+    if (ch <= "\u007F") {
+        if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '$' || ch === '_' ||
+            (!first && (ch >= '0' && ch <= '9'))) {
+            return true;
+        }
+        return false;
+    }
+
+    // create an object to test this in
+    var x = {};
+    x["x"+ch] = true;
+    x[ch] = true;
+
+    // then use eval to determine if it's a valid character
+    var valid = false;
+    try {
+        valid = (Function("x", "return (x." + (first?"":"x") + ch + ");")(x) === true);
+    } catch (ex) {}
+
+    return valid;
+}
+
+function isIdentifier(str) {
+    if (typeof str !== "string")
+        return false;
+
+    if (str.length === 0)
+        return false;
+
+    if (!isValidIdentifierChar(str[0], true))
+        return false;
+
+    for (var i = 1; i < str.length; i++) {
+        if (!isValidIdentifierChar(str[i], false))
+            return false;
+    }
+
+    return true;
+}
+
+/*
+ * Tokenizer :: (source, filename, line number, boolean) -> Tokenizer
+ */
+function Tokenizer(s, f, l, allowHTMLComments) {
+    this.cursor = 0;
+    this.source = String(s);
+    this.tokens = [];
+    this.tokenIndex = 0;
+    this.lookahead = 0;
+    this.scanNewlines = false;
+    this.filename = f || "";
+    this.lineno = l || 1;
+    this.allowHTMLComments = allowHTMLComments;
+    this.blockComments = null;
+}
+
+Tokenizer.prototype = {
+    get done() {
+        // We need to set scanOperand to true here because the first thing
+        // might be a regexp.
+        return this.peek(true) === END;
+    },
+
+    get token() {
+        return this.tokens[this.tokenIndex];
+    },
+
+    match: function (tt, scanOperand, keywordIsName) {
+        return this.get(scanOperand, keywordIsName) === tt || this.unget();
+    },
+
+    mustMatch: function (tt, keywordIsName) {
+        if (!this.match(tt, false, keywordIsName)) {
+            throw this.newSyntaxError("Missing " +
+                                      definitions.tokens[tt].toLowerCase());
+        }
+        return this.token;
+    },
+
+    peek: function (scanOperand) {
+        var tt, next;
+        if (this.lookahead) {
+            next = this.tokens[(this.tokenIndex + this.lookahead) & 3];
+            tt = (this.scanNewlines && next.lineno !== this.lineno)
+                ? NEWLINE
+                : next.type;
+        } else {
+            tt = this.get(scanOperand);
+            this.unget();
+        }
+        return tt;
+    },
+
+    peekOnSameLine: function (scanOperand) {
+        this.scanNewlines = true;
+        var tt = this.peek(scanOperand);
+        this.scanNewlines = false;
+        return tt;
+    },
+
+    lastBlockComment: function() {
+        var length = this.blockComments.length;
+        return length ? this.blockComments[length - 1] : null;
+    },
+
+    // Eat comments and whitespace.
+    skip: function () {
+        var input = this.source;
+        this.blockComments = [];
+        for (;;) {
+            var ch = input[this.cursor++];
+            var next = input[this.cursor];
+            // handle \r, \r\n and (always preferable) \n
+            if (ch === '\r') {
+                // if the next character is \n, we don't care about this at all
+                if (next === '\n') continue;
+
+                // otherwise, we want to consider this as a newline
+                ch = '\n';
+            }
+
+            if (ch === '\n' && !this.scanNewlines) {
+                this.lineno++;
+            } else if (ch === '/' && next === '*') {
+                var commentStart = ++this.cursor;
+                for (;;) {
+                    ch = input[this.cursor++];
+                    if (ch === undefined)
+                        throw this.newSyntaxError("Unterminated comment");
+
+                    if (ch === '*') {
+                        next = input[this.cursor];
+                        if (next === '/') {
+                            var commentEnd = this.cursor - 1;
+                            this.cursor++;
+                            break;
+                        }
+                    } else if (ch === '\n') {
+                        this.lineno++;
+                    }
+                }
+                this.blockComments.push(input.substring(commentStart, commentEnd));
+            } else if ((ch === '/' && next === '/') ||
+                       (this.allowHTMLComments && ch === '<' && next === '!' &&
+                        input[this.cursor + 1] === '-' && input[this.cursor + 2] === '-' &&
+                        (this.cursor += 2))) {
+                this.cursor++;
+                for (;;) {
+                    ch = input[this.cursor++];
+                    next = input[this.cursor];
+                    if (ch === undefined)
+                        return;
+
+                    if (ch === '\r') {
+                        // check for \r\n
+                        if (next !== '\n') ch = '\n';
+                    }
+
+                    if (ch === '\n') {
+                        if (this.scanNewlines) {
+                            this.cursor--;
+                        } else {
+                            this.lineno++;
+                        }
+                        break;
+                    }
+                }
+            } else if (!(ch in definitions.whitespace)) {
+                this.cursor--;
+                return;
+            }
+        }
+    },
+
+    // Lex the exponential part of a number, if present. Return true iff an
+    // exponential part was found.
+    lexExponent: function() {
+        var input = this.source;
+        var next = input[this.cursor];
+        if (next === 'e' || next === 'E') {
+            this.cursor++;
+            ch = input[this.cursor++];
+            if (ch === '+' || ch === '-')
+                ch = input[this.cursor++];
+
+            if (ch < '0' || ch > '9')
+                throw this.newSyntaxError("Missing exponent");
+
+            do {
+                ch = input[this.cursor++];
+            } while (ch >= '0' && ch <= '9');
+            this.cursor--;
+
+            return true;
+        }
+
+        return false;
+    },
+
+    lexZeroNumber: function (ch) {
+        var token = this.token, input = this.source;
+        token.type = NUMBER;
+
+        ch = input[this.cursor++];
+        if (ch === '.') {
+            do {
+                ch = input[this.cursor++];
+            } while (ch >= '0' && ch <= '9');
+            this.cursor--;
+
+            this.lexExponent();
+            token.value = parseFloat(
+                input.substring(token.start, this.cursor));
+        } else if (ch === 'x' || ch === 'X') {
+            do {
+                ch = input[this.cursor++];
+            } while ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') ||
+                     (ch >= 'A' && ch <= 'F'));
+            this.cursor--;
+
+            token.value = parseInt(input.substring(token.start, this.cursor));
+        } else if (ch >= '0' && ch <= '7') {
+            do {
+                ch = input[this.cursor++];
+            } while (ch >= '0' && ch <= '7');
+            this.cursor--;
+
+            token.value = parseInt(input.substring(token.start, this.cursor));
+        } else {
+            this.cursor--;
+            this.lexExponent();     // 0E1, &c.
+            token.value = 0;
+        }
+    },
+
+    lexNumber: function (ch) {
+        var token = this.token, input = this.source;
+        token.type = NUMBER;
+
+        var floating = false;
+        do {
+            ch = input[this.cursor++];
+            if (ch === '.' && !floating) {
+                floating = true;
+                ch = input[this.cursor++];
+            }
+        } while (ch >= '0' && ch <= '9');
+
+        this.cursor--;
+
+        var exponent = this.lexExponent();
+        floating = floating || exponent;
+
+        var str = input.substring(token.start, this.cursor);
+        token.value = floating ? parseFloat(str) : parseInt(str);
+    },
+
+    lexDot: function (ch) {
+        var token = this.token, input = this.source;
+        var next = input[this.cursor];
+        if (next >= '0' && next <= '9') {
+            do {
+                ch = input[this.cursor++];
+            } while (ch >= '0' && ch <= '9');
+            this.cursor--;
+
+            this.lexExponent();
+
+            token.type = NUMBER;
+            token.value = parseFloat(
+                input.substring(token.start, this.cursor));
+        } else {
+            token.type = DOT;
+            token.assignOp = null;
+            token.value = '.';
+        }
+    },
+
+    lexString: function (ch) {
+        var token = this.token, input = this.source;
+        token.type = STRING;
+
+        var hasEscapes = false;
+        var delim = ch;
+        if (input.length <= this.cursor)
+            throw this.newSyntaxError("Unterminated string literal");
+        while ((ch = input[this.cursor++]) !== delim) {
+            if (ch == '\n' || ch == '\r')
+                throw this.newSyntaxError("Unterminated string literal");
+            if (this.cursor == input.length)
+                throw this.newSyntaxError("Unterminated string literal");
+            if (ch === '\\') {
+                hasEscapes = true;
+                if (++this.cursor == input.length)
+                    throw this.newSyntaxError("Unterminated string literal");
+            }
+        }
+
+        token.value = hasEscapes
+            ? eval(input.substring(token.start, this.cursor))
+            : input.substring(token.start + 1, this.cursor - 1);
+    },
+
+    lexRegExp: function (ch) {
+        var token = this.token, input = this.source;
+        token.type = REGEXP;
+
+        do {
+            ch = input[this.cursor++];
+            if (ch === '\\') {
+                this.cursor++;
+            } else if (ch === '[') {
+                do {
+                    if (ch === undefined)
+                        throw this.newSyntaxError("Unterminated character class");
+
+                    if (ch === '\\')
+                        this.cursor++;
+
+                    ch = input[this.cursor++];
+                } while (ch !== ']');
+            } else if (ch === undefined) {
+                throw this.newSyntaxError("Unterminated regex");
+            }
+        } while (ch !== '/');
+
+        do {
+            ch = input[this.cursor++];
+        } while (ch >= 'a' && ch <= 'z');
+
+        this.cursor--;
+
+        token.value = eval(input.substring(token.start, this.cursor));
+    },
+
+    lexOp: function (ch) {
+        var token = this.token, input = this.source;
+
+        // A bit ugly, but it seems wasteful to write a trie lookup routine
+        // for only 3 characters...
+        var node = opTokens[ch];
+        var next = input[this.cursor];
+        if (next in node) {
+            node = node[next];
+            this.cursor++;
+            next = input[this.cursor];
+            if (next in node) {
+                node = node[next];
+                this.cursor++;
+                next = input[this.cursor];
+            }
+        }
+
+        var op = node.op;
+        if (definitions.assignOps[op] && input[this.cursor] === '=') {
+            this.cursor++;
+            token.type = ASSIGN;
+            token.assignOp = definitions.tokenIds[definitions.opTypeNames[op]];
+            op += '=';
+        } else {
+            token.type = definitions.tokenIds[definitions.opTypeNames[op]];
+            token.assignOp = null;
+        }
+
+        token.value = op;
+    },
+
+    // FIXME: Unicode escape sequences
+    lexIdent: function (ch, keywordIsName) {
+        var token = this.token;
+        var id = ch;
+
+        while ((ch = this.getValidIdentifierChar(false)) !== null) {
+            id += ch;
+        }
+
+        token.type = IDENTIFIER;
+        token.value = id;
+
+        if (keywordIsName)
+            return;
+
+        var kw;
+
+        if (this.parser.mozillaMode) {
+            kw = definitions.mozillaKeywords[id];
+            if (kw) {
+                token.type = kw;
+                return;
+            }
+        }
+
+        if (this.parser.x.strictMode) {
+            kw = definitions.strictKeywords[id];
+            if (kw) {
+                token.type = kw;
+                return;
+            }
+        }
+
+        kw = definitions.keywords[id];
+        if (kw)
+            token.type = kw;
+    },
+
+    /*
+     * Tokenizer.get :: ([boolean[, boolean]]) -> token type
+     *
+     * Consume input *only* if there is no lookahead.
+     * Dispatch to the appropriate lexing function depending on the input.
+     */
+    get: function (scanOperand, keywordIsName) {
+        var token;
+        while (this.lookahead) {
+            --this.lookahead;
+            this.tokenIndex = (this.tokenIndex + 1) & 3;
+            token = this.tokens[this.tokenIndex];
+            if (token.type !== NEWLINE || this.scanNewlines)
+                return token.type;
+        }
+
+        this.skip();
+
+        this.tokenIndex = (this.tokenIndex + 1) & 3;
+        token = this.tokens[this.tokenIndex];
+        if (!token)
+            this.tokens[this.tokenIndex] = token = {};
+
+        var input = this.source;
+        if (this.cursor >= input.length)
+            return token.type = END;
+
+        token.start = this.cursor;
+        token.lineno = this.lineno;
+
+        var ich = this.getValidIdentifierChar(true);
+        var ch = (ich === null) ? input[this.cursor++] : null;
+        if (ich !== null) {
+            this.lexIdent(ich, keywordIsName);
+        } else if (scanOperand && ch === '/') {
+            this.lexRegExp(ch);
+        } else if (ch in opTokens) {
+            this.lexOp(ch);
+        } else if (ch === '.') {
+            this.lexDot(ch);
+        } else if (ch >= '1' && ch <= '9') {
+            this.lexNumber(ch);
+        } else if (ch === '0') {
+            this.lexZeroNumber(ch);
+        } else if (ch === '"' || ch === "'") {
+            this.lexString(ch);
+        } else if (this.scanNewlines && (ch === '\n' || ch === '\r')) {
+            // if this was a \r, look for \r\n
+            if (ch === '\r' && input[this.cursor] === '\n') this.cursor++;
+            token.type = NEWLINE;
+            token.value = '\n';
+            this.lineno++;
+        } else {
+            throw this.newSyntaxError("Illegal token");
+        }
+
+        token.end = this.cursor;
+        return token.type;
+    },
+
+    /*
+     * Tokenizer.unget :: void -> undefined
+     *
+     * Match depends on unget returning undefined.
+     */
+    unget: function () {
+        if (++this.lookahead === 4) throw "PANIC: too much lookahead!";
+        this.tokenIndex = (this.tokenIndex - 1) & 3;
+    },
+
+    newSyntaxError: function (m) {
+        m = (this.filename ? this.filename + ":" : "") + this.lineno + ": " + m;
+        var e = new SyntaxError(m, this.filename, this.lineno);
+        e.source = this.source;
+        e.cursor = this.lookahead
+            ? this.tokens[(this.tokenIndex + this.lookahead) & 3].start
+            : this.cursor;
+        return e;
+    },
+
+
+    /* Gets a single valid identifier char from the input stream, or null
+     * if there is none.
+     */
+    getValidIdentifierChar: function(first) {
+        var input = this.source;
+        if (this.cursor >= input.length) return null;
+        var ch = input[this.cursor];
+
+        // first check for \u escapes
+        if (ch === '\\' && input[this.cursor+1] === 'u') {
+            // get the character value
+            try {
+                ch = String.fromCharCode(parseInt(
+                    input.substring(this.cursor + 2, this.cursor + 6),
+                    16));
+            } catch (ex) {
+                return null;
+            }
+            this.cursor += 5;
+        }
+
+        var valid = isValidIdentifierChar(ch, first);
+        if (valid) this.cursor++;
+        return (valid ? ch : null);
+    },
+};
+
+
+exports.isIdentifier = isIdentifier;
+exports.Tokenizer = Tokenizer;
+
+});

lib/ace/narcissus/options.js

@@ -0,0 +1,61 @@
+/* vim: set sw=4 ts=4 et tw=78: */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is the Narcissus JavaScript engine.
+ *
+ * The Initial Developer of the Original Code is
+ * Brendan Eich <brendan@mozilla.org>.
+ * Portions created by the Initial Developer are Copyright (C) 2004
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Tom Austin <taustin@ucsc.edu>
+ *   Brendan Eich <brendan@mozilla.org>
+ *   Shu-Yu Guo <shu@rfrn.org>
+ *   Dave Herman <dherman@mozilla.com>
+ *   Dimitris Vardoulakis <dimvar@ccs.neu.edu>
+ *   Patrick Walton <pcwalton@mozilla.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+
+// Global variables to hide from the interpreter
+exports.hiddenHostGlobals = { Narcissus: true };
+
+// Desugar SpiderMonkey language extensions?
+exports.desugarExtensions = false;
+
+// Allow HTML comments?
+exports.allowHTMLComments = false;
+
+// Allow non-standard Mozilla extensions?
+exports.mozillaMode = true;
+
+// Allow experimental paren-free mode?
+exports.parenFreeMode = false;
+
+});

lib/ace/test/all_browser.js

@@ -22,21 +22,21 @@ var testNames = [
     "ace/layer/text_test",
     "ace/lib/event_emitter_test",
     "ace/mode/coffee/parser_test",
-    "ace/mode/coffee_tokenizer_test",
+    "ace/mode/coffee_highlight_rules_test",
     "ace/mode/coldfusion_test",
     "ace/mode/css_test",
-    "ace/mode/css_tokenizer_test",
+    "ace/mode/css_highlight_rules_test",
     "ace/mode/css_worker",
     "ace/mode/html_test",
-    "ace/mode/html_tokenizer_test",
+    "ace/mode/html_highlight_rules_test",
     "ace/mode/javascript_test",
-    "ace/mode/javascript_tokenizer_test",
+    "ace/mode/javascript_highlight_rules_test",
     "ace/mode/javascript_worker_test",
     "ace/mode/python_test",
-    "ace/mode/ruby_tokenizer_test",
+    "ace/mode/ruby_highlight_rules_test",
     "ace/mode/text_test",
     "ace/mode/xml_test",
-    "ace/mode/xml_tokenizer_test",
+    "ace/mode/xml_highlight_rules_test",
     "ace/mode/folding/cstyle_test",
     "ace/mode/folding/html_test",
     "ace/mode/folding/pythonic_test",

lib/ace/test/mockrenderer.js

@@ -1,191 +1,194 @@
-/* ***** BEGIN LICENSE BLOCK *****
- * Version: MPL 1.1/GPL 2.0/LGPL 2.1
- *
- * The contents of this file are subject to the Mozilla Public License Version
- * 1.1 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS" basis,
- * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
- * for the specific language governing rights and limitations under the
- * License.
- *
- * The Original Code is Ajax.org Code Editor (ACE).
- *
- * The Initial Developer of the Original Code is
- * Ajax.org B.V.
- * Portions created by the Initial Developer are Copyright (C) 2010
- * the Initial Developer. All Rights Reserved.
- *
- * Contributor(s):
- *      Fabian Jakobs <fabian AT ajax DOT org>
- *
- * Alternatively, the contents of this file may be used under the terms of
- * either the GNU General Public License Version 2 or later (the "GPL"), or
- * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
- * in which case the provisions of the GPL or the LGPL are applicable instead
- * of those above. If you wish to allow use of your version of this file only
- * under the terms of either the GPL or the LGPL, and not to allow others to
- * use your version of this file under the terms of the MPL, indicate your
- * decision by deleting the provisions above and replace them with the notice
- * and other provisions required by the GPL or the LGPL. If you do not delete
- * the provisions above, a recipient may use your version of this file under
- * the terms of any one of the MPL, the GPL or the LGPL.
- *
- * ***** END LICENSE BLOCK ***** */
-
-define(function(require, exports, module) {
-"use strict";
-
-var MockRenderer = exports.MockRenderer = function(visibleRowCount) {
-    this.container = document.createElement("div");
-    this.visibleRowCount = visibleRowCount || 20;
-
-    this.layerConfig = {
-        firstVisibleRow : 0,
-        lastVisibleRow : this.visibleRowCount
-    };
-
-    this.isMockRenderer = true;
-    
-    this.$gutter = {};
-};
-
-
-MockRenderer.prototype.getFirstVisibleRow = function() {
-    return this.layerConfig.firstVisibleRow;
-};
-
-MockRenderer.prototype.getLastVisibleRow = function() {
-    return this.layerConfig.lastVisibleRow;
-};
-
-MockRenderer.prototype.getFirstFullyVisibleRow = function() {
-    return this.layerConfig.firstVisibleRow;
-};
-
-MockRenderer.prototype.getLastFullyVisibleRow = function() {
-    return this.layerConfig.lastVisibleRow;
-};
-
-MockRenderer.prototype.getContainerElement = function() {
-    return this.container;
-};
-
-MockRenderer.prototype.getMouseEventTarget = function() {
-    return this.container;
-};
-
-MockRenderer.prototype.getTextAreaContainer = function() {
-    return this.container;
-};
-
-MockRenderer.prototype.moveTextAreaToCursor = function() {
-};
-
-MockRenderer.prototype.setSession = function(session) {
-    this.session = session;
-};
-
-MockRenderer.prototype.getSession = function(session) {
-    return this.session;
-};
-
-MockRenderer.prototype.setTokenizer = function() {
-};
-
-MockRenderer.prototype.on = function() {
-};
-
-MockRenderer.prototype.updateCursor = function() {
-};
-
-MockRenderer.prototype.scrollToX = function(scrollTop) {};
-MockRenderer.prototype.scrollToY = function(scrollLeft) {};
-    
-MockRenderer.prototype.scrollToLine = function(line, center) {
-    var lineHeight = { lineHeight: 16 };
-    var row = 0;
-    for (var l = 1; l < line; l++) {
-        row += this.session.getRowHeight(lineHeight, l-1) / lineHeight.lineHeight;
-    }
-
-    if (center) {
-        row -= this.visibleRowCount / 2;
-    }
-    this.scrollToRow(row);
-};
-
-MockRenderer.prototype.scrollCursorIntoView = function() {
-    var cursor = this.session.getSelection().getCursor();
-    if (cursor.row < this.layerConfig.firstVisibleRow) {
-        this.scrollToRow(cursor.row);
-    }
-    else if (cursor.row > this.layerConfig.lastVisibleRow) {
-        this.scrollToRow(cursor.row);
-    }
-};
-
-MockRenderer.prototype.scrollToRow = function(row) {
-    var row = Math.min(this.session.getLength() - this.visibleRowCount, Math.max(0,
-                                                                          row));
-    this.layerConfig.firstVisibleRow = row;
-    this.layerConfig.lastVisibleRow = row + this.visibleRowCount;
-};
-
-MockRenderer.prototype.getScrollTopRow = function() {
-  return this.layerConfig.firstVisibleRow;
-};
-
-MockRenderer.prototype.draw = function() {
-};
-
-MockRenderer.prototype.updateLines = function(startRow, endRow) {
-};
-
-MockRenderer.prototype.updateBackMarkers = function() {
-};
-
-MockRenderer.prototype.updateFrontMarkers = function() {
-};
-
-MockRenderer.prototype.setBreakpoints = function() {
-};
-
-MockRenderer.prototype.onResize = function() {
-};
-
-MockRenderer.prototype.updateFull = function() {
-};
-
-MockRenderer.prototype.updateText = function() {
-};
-
-MockRenderer.prototype.showCursor = function() {
-};
-
-MockRenderer.prototype.visualizeFocus = function() {
-};
-
-MockRenderer.prototype.setAnnotations = function() {
-};
-
-MockRenderer.prototype.setStyle = function() {
-};
-
-MockRenderer.prototype.unsetStyle = function() {
-};
-
-MockRenderer.prototype.textToScreenCoordinates = function() {
-    return {
-        pageX: 0,
-        pageY: 0
-    }
-};
-
-MockRenderer.prototype.adjustWrapLimit = function () {
-    
-};
-
-});
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Ajax.org Code Editor (ACE).
+ *
+ * The Initial Developer of the Original Code is
+ * Ajax.org B.V.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *      Fabian Jakobs <fabian AT ajax DOT org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+"use strict";
+
+var MockRenderer = exports.MockRenderer = function(visibleRowCount) {
+    this.container = document.createElement("div");
+    this.visibleRowCount = visibleRowCount || 20;
+
+    this.layerConfig = {
+        firstVisibleRow : 0,
+        lastVisibleRow : this.visibleRowCount
+    };
+
+    this.isMockRenderer = true;
+
+    this.$gutter = {};
+};
+
+
+MockRenderer.prototype.getFirstVisibleRow = function() {
+    return this.layerConfig.firstVisibleRow;
+};
+
+MockRenderer.prototype.getLastVisibleRow = function() {
+    return this.layerConfig.lastVisibleRow;
+};
+
+MockRenderer.prototype.getFirstFullyVisibleRow = function() {
+    return this.layerConfig.firstVisibleRow;
+};
+
+MockRenderer.prototype.getLastFullyVisibleRow = function() {
+    return this.layerConfig.lastVisibleRow;
+};
+
+MockRenderer.prototype.getContainerElement = function() {
+    return this.container;
+};
+
+MockRenderer.prototype.getMouseEventTarget = function() {
+    return this.container;
+};
+
+MockRenderer.prototype.getTextAreaContainer = function() {
+    return this.container;
+};
+
+MockRenderer.prototype.moveTextAreaToCursor = function() {
+};
+
+MockRenderer.prototype.setSession = function(session) {
+    this.session = session;
+};
+
+MockRenderer.prototype.getSession = function(session) {
+    return this.session;
+};
+
+MockRenderer.prototype.setTokenizer = function() {
+};
+
+MockRenderer.prototype.on = function() {
+};
+
+MockRenderer.prototype.updateCursor = function() {
+};
+
+MockRenderer.prototype.scrollToX = function(scrollTop) {};
+MockRenderer.prototype.scrollToY = function(scrollLeft) {};
+
+MockRenderer.prototype.scrollToLine = function(line, center) {
+    var lineHeight = { lineHeight: 16 };
+    var row = 0;
+    for (var l = 1; l < line; l++) {
+        row += this.session.getRowHeight(lineHeight, l-1) / lineHeight.lineHeight;
+    }
+
+    if (center) {
+        row -= this.visibleRowCount / 2;
+    }
+    this.scrollToRow(row);
+};
+
+MockRenderer.prototype.scrollSelectionIntoView = function() {
+};
+
+MockRenderer.prototype.scrollCursorIntoView = function() {
+    var cursor = this.session.getSelection().getCursor();
+    if (cursor.row < this.layerConfig.firstVisibleRow) {
+        this.scrollToRow(cursor.row);
+    }
+    else if (cursor.row > this.layerConfig.lastVisibleRow) {
+        this.scrollToRow(cursor.row);
+    }
+};
+
+MockRenderer.prototype.scrollToRow = function(row) {
+    var row = Math.min(this.session.getLength() - this.visibleRowCount, Math.max(0,
+                                                                          row));
+    this.layerConfig.firstVisibleRow = row;
+    this.layerConfig.lastVisibleRow = row + this.visibleRowCount;
+};
+
+MockRenderer.prototype.getScrollTopRow = function() {
+  return this.layerConfig.firstVisibleRow;
+};
+
+MockRenderer.prototype.draw = function() {
+};
+
+MockRenderer.prototype.updateLines = function(startRow, endRow) {
+};
+
+MockRenderer.prototype.updateBackMarkers = function() {
+};
+
+MockRenderer.prototype.updateFrontMarkers = function() {
+};
+
+MockRenderer.prototype.setBreakpoints = function() {
+};
+
+MockRenderer.prototype.onResize = function() {
+};
+
+MockRenderer.prototype.updateFull = function() {
+};
+
+MockRenderer.prototype.updateText = function() {
+};
+
+MockRenderer.prototype.showCursor = function() {
+};
+
+MockRenderer.prototype.visualizeFocus = function() {
+};
+
+MockRenderer.prototype.setAnnotations = function() {
+};
+
+MockRenderer.prototype.setStyle = function() {
+};
+
+MockRenderer.prototype.unsetStyle = function() {
+};
+
+MockRenderer.prototype.textToScreenCoordinates = function() {
+    return {
+        pageX: 0,
+        pageY: 0
+    }
+};
+
+MockRenderer.prototype.adjustWrapLimit = function () {
+
+};
+
+});

lib/ace/theme/chrome.js

@@ -47,7 +47,6 @@ exports.cssText = ".ace-chrome .ace_editor {\
 }\
 \
 .ace-chrome .ace_gutter {\
-  width: 50px;\
   background: #e8e8e8;\
   color: #333;\
   overflow : hidden;\
@@ -58,10 +57,6 @@ exports.cssText = ".ace-chrome .ace_editor {\
   text-align: right;\
 }\
 \
-.ace-chrome .ace_gutter-layer .ace_gutter-cell {\
-  padding-right: 6px;\
-}\
-\
 .ace-chrome .ace_print_margin {\
   width: 1px;\
   background: #e8e8e8;\

lib/ace/theme/clouds.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: #BFBFBF;\
 }\
 \
-.ace-clouds .ace_keyword {\
+.ace-clouds .ace_keyword, .ace-clouds .ace_meta {\
   color:#AF956F;\
 }\
 \

lib/ace/theme/clouds_midnight.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: #BFBFBF;\
 }\
 \
-.ace-clouds-midnight .ace_keyword {\
+.ace-clouds-midnight .ace_keyword, .ace-clouds-midnight .ace_meta {\
   color:#927C5D;\
 }\
 \

lib/ace/theme/cobalt.js

@@ -101,11 +101,11 @@ exports.cssText = "\
   color: rgba(255, 255, 255, 0.15);\
 }\
 \
-.ace-cobalt .ace_keyword {\
+.ace-cobalt .ace_keyword, .ace-cobalt .ace_meta {\
   color:#FF9D00;\
 }\
 \
-.ace-cobalt .ace_constant {\
+.ace-cobalt .ace_constant, .ace-cobalt .ace_constant.ace_other {\
   color:#FF628C;\
 }\
 \
@@ -118,6 +118,10 @@ background-color:#800F00;\
   color:#80FFBB;\
 }\
 \
+.ace-cobalt .ace_support.ace_constant {\
+  color:#EB939A;\
+}\
+\
 .ace-cobalt .ace_fold {\
     background-color: #FF9D00;\
     border-color: #FFFFFF;\

lib/ace/theme/dawn.js

@@ -101,11 +101,11 @@ exports.cssText = "\
   color: rgba(75, 75, 126, 0.50);\
 }\
 \
-.ace-dawn .ace_keyword {\
+.ace-dawn .ace_keyword, .ace-dawn .ace_meta {\
   color:#794938;\
 }\
 \
-.ace-dawn .ace_constant {\
+.ace-dawn .ace_constant, .ace-dawn .ace_constant.ace_other {\
   color:#811F24;\
 }\
 \
@@ -126,6 +126,10 @@ color:#B52A1D;\
   color:#691C97;\
 }\
 \
+.ace-dawn .ace_support.ace_constant {\
+  color:#B4371F;\
+}\
+\
 .ace-dawn .ace_fold {\
     background-color: #794938;\
     border-color: #080808;\

lib/ace/theme/idle_fingers.js

@@ -101,11 +101,11 @@ exports.cssText = "\
   color: #404040;\
 }\
 \
-.ace-idle-fingers .ace_keyword {\
+.ace-idle-fingers .ace_keyword, .ace-idle-fingers .ace_meta {\
   color:#CC7833;\
 }\
 \
-.ace-idle-fingers .ace_constant {\
+.ace-idle-fingers .ace_constant, .ace-idle-fingers .ace_constant.ace_other {\
   color:#6C99BB;\
 }\
 \
@@ -114,6 +114,10 @@ exports.cssText = "\
 background-color:#FF0000;\
 }\
 \
+.ace-idle-fingers .ace_support.ace_constant {\
+  color:#6C99BB;\
+}\
+\
 .ace-idle-fingers .ace_fold {\
     background-color: #CC7833;\
     border-color: #FFFFFF;\

lib/ace/theme/kr_theme.js

@@ -101,11 +101,11 @@ exports.cssText = "\
   color: rgba(255, 177, 111, 0.32);\
 }\
 \
-.ace-kr-theme .ace_keyword {\
+.ace-kr-theme .ace_keyword, .ace-kr-theme .ace_meta {\
   color:#949C8B;\
 }\
 \
-.ace-kr-theme .ace_constant {\
+.ace-kr-theme .ace_constant, .ace-kr-theme .ace_constant.ace_other {\
   color:rgba(210, 117, 24, 0.76);\
 }\
 \
@@ -118,6 +118,10 @@ background-color:#A41300;\
   color:#9FC28A;\
 }\
 \
+.ace-kr-theme .ace_support.ace_constant {\
+  color:#C27E66;\
+}\
+\
 .ace-kr-theme .ace_fold {\
     background-color: #949C8B;\
     border-color: #FCFFE0;\

lib/ace/theme/merbivore.js

@@ -101,11 +101,11 @@ exports.cssText = "\
   color: #404040;\
 }\
 \
-.ace-merbivore .ace_keyword {\
+.ace-merbivore .ace_keyword, .ace-merbivore .ace_meta {\
   color:#FC6F09;\
 }\
 \
-.ace-merbivore .ace_constant {\
+.ace-merbivore .ace_constant, .ace-merbivore .ace_constant.ace_other {\
   color:#1EDAFB;\
 }\
 \
@@ -126,6 +126,10 @@ exports.cssText = "\
 background-color:#990000;\
 }\
 \
+.ace-merbivore .ace_support.ace_constant {\
+  color:#8DFF0A;\
+}\
+\
 .ace-merbivore .ace_fold {\
     background-color: #FC6F09;\
     border-color: #E6E1DC;\

lib/ace/theme/merbivore_soft.js

@@ -101,11 +101,11 @@ exports.cssText = "\
   color: #404040;\
 }\
 \
-.ace-merbivore-soft .ace_keyword {\
+.ace-merbivore-soft .ace_keyword, .ace-merbivore-soft .ace_meta {\
   color:#FC803A;\
 }\
 \
-.ace-merbivore-soft .ace_constant {\
+.ace-merbivore-soft .ace_constant, .ace-merbivore-soft .ace_constant.ace_other {\
   color:#68C1D8;\
 }\
 \
@@ -131,6 +131,10 @@ background-color:#FE3838;\
 background-color:#FE3838;\
 }\
 \
+.ace-merbivore-soft .ace_support.ace_constant {\
+  color:#8EC65F;\
+}\
+\
 .ace-merbivore-soft .ace_fold {\
     background-color: #FC803A;\
     border-color: #E6E1DC;\

lib/ace/theme/mono_industrial.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: rgba(102, 108, 104, 0.50);\
 }\
 \
-.ace-mono-industrial .ace_keyword {\
+.ace-mono-industrial .ace_keyword, .ace-mono-industrial .ace_meta {\
   color:#A39E64;\
 }\
 \
@@ -109,7 +109,7 @@ exports.cssText = "\
   color:#A8B3AB;\
 }\
 \
-.ace-mono-industrial .ace_constant {\
+.ace-mono-industrial .ace_constant, .ace-mono-industrial .ace_constant.ace_other {\
   color:#E98800;\
 }\
 \
@@ -122,6 +122,10 @@ exports.cssText = "\
 background-color:rgba(153, 0, 0, 0.68);\
 }\
 \
+.ace-mono-industrial .ace_support.ace_constant {\
+  color:#C87500;\
+}\
+\
 .ace-mono-industrial .ace_fold {\
     background-color: #A8B3AB;\
     border-color: #FFFFFF;\
@@ -135,6 +139,10 @@ background-color:rgba(153, 0, 0, 0.68);\
   color:#C23B00;\
 }\
 \
+.ace-mono-industrial .ace_variable {\
+  color:#A8B3AB;\
+}\
+\
 .ace-mono-industrial .ace_variable.ace_parameter {\
   color:#648BD2;\
 }\

lib/ace/theme/monokai.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: #49483E;\
 }\
 \
-.ace-monokai .ace_keyword {\
+.ace-monokai .ace_keyword, .ace-monokai .ace_meta {\
   color:#F92672;\
 }\
 \
@@ -113,6 +113,10 @@ exports.cssText = "\
   color:#AE81FF;\
 }\
 \
+.ace-monokai .ace_constant.ace_other {\
+  color:#AE81FF;\
+}\
+\
 .ace-monokai .ace_invalid {\
   color:#F8F8F0;\
 background-color:#F92672;\
@@ -123,6 +127,10 @@ background-color:#F92672;\
 background-color:#AE81FF;\
 }\
 \
+.ace-monokai .ace_support.ace_constant {\
+  color:#66D9EF;\
+}\
+\
 .ace-monokai .ace_fold {\
     background-color: #A6E22E;\
     border-color: #F8F8F2;\
@@ -136,11 +144,15 @@ background-color:#AE81FF;\
   color:#F92672;\
 }\
 \
-.ace-monokai .ace_storage.ace_type {\
+.ace-monokai .ace_storage.ace_type,  .ace-monokai .ace_support.ace_type{\
   font-style:italic;\
 color:#66D9EF;\
 }\
 \
+.ace-monokai .ace_variable {\
+  color:#A6E22E;\
+}\
+\
 .ace-monokai .ace_variable.ace_parameter {\
   font-style:italic;\
 color:#FD971F;\

lib/ace/theme/pastel_on_dark.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: rgba(255, 255, 255, 0.25);\
 }\
 \
-.ace-pastel-on-dark .ace_keyword {\
+.ace-pastel-on-dark .ace_keyword, .ace-pastel-on-dark .ace_meta {\
   color:#757aD8;\
 }\
 \
@@ -109,7 +109,7 @@ exports.cssText = "\
   color:#797878;\
 }\
 \
-.ace-pastel-on-dark .ace_constant {\
+.ace-pastel-on-dark .ace_constant, .ace-pastel-on-dark .ace_constant.ace_other {\
   color:#4FB7C5;\
 }\
 \

lib/ace/theme/solarized_dark.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: rgba(147, 161, 161, 0.50);\
 }\
 \
-.ace-solarized-dark .ace_keyword {\
+.ace-solarized-dark .ace_keyword, .ace-solarized-dark .ace_meta {\
   color:#859900;\
 }\
 \
@@ -113,6 +113,10 @@ exports.cssText = "\
   color:#D33682;\
 }\
 \
+.ace-solarized-dark .ace_constant.ace_other {\
+  color:#CB4B16;\
+}\
+\
 .ace-solarized-dark .ace_fold {\
     background-color: #268BD2;\
     border-color: #93A1A1;\
@@ -126,6 +130,10 @@ exports.cssText = "\
   color:#93A1A1;\
 }\
 \
+.ace-solarized-dark .ace_variable {\
+  color:#268BD2;\
+}\
+\
 .ace-solarized-dark .ace_string {\
   color:#2AA198;\
 }\

lib/ace/theme/solarized_light.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: rgba(147, 161, 161, 0.50);\
 }\
 \
-.ace-solarized-light .ace_keyword {\
+.ace-solarized-light .ace_keyword, .ace-solarized-light .ace_meta {\
   color:#859900;\
 }\
 \
@@ -113,6 +113,10 @@ exports.cssText = "\
   color:#D33682;\
 }\
 \
+.ace-solarized-light .ace_constant.ace_other {\
+  color:#CB4B16;\
+}\
+\
 .ace-solarized-light .ace_fold {\
     background-color: #268BD2;\
     border-color: #586E75;\
@@ -126,6 +130,10 @@ exports.cssText = "\
   color:#073642;\
 }\
 \
+.ace-solarized-light .ace_variable {\
+  color:#268BD2;\
+}\
+\
 .ace-solarized-light .ace_string {\
   color:#2AA198;\
 }\

lib/ace/theme/tomorrow.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: #D1D1D1;\
 }\
 \
-.ace-tomorrow .ace_keyword {\
+.ace-tomorrow .ace_keyword, .ace-tomorrow .ace_meta {\
   color:#8959A8;\
 }\
 \
@@ -117,6 +117,10 @@ exports.cssText = "\
   color:#F5871F;\
 }\
 \
+.ace-tomorrow .ace_constant.ace_other {\
+  color:#666969;\
+}\
+\
 .ace-tomorrow .ace_invalid {\
   color:#FFFFFF;\
 background-color:#C82829;\
@@ -127,6 +131,10 @@ background-color:#C82829;\
 background-color:#8959A8;\
 }\
 \
+.ace-tomorrow .ace_support.ace_constant {\
+  color:#F5871F;\
+}\
+\
 .ace-tomorrow .ace_fold {\
     background-color: #4271AE;\
     border-color: #4D4D4C;\
@@ -140,10 +148,14 @@ background-color:#8959A8;\
   color:#8959A8;\
 }\
 \
-.ace-tomorrow .ace_storage.ace_type {\
+.ace-tomorrow .ace_storage.ace_type,  .ace-tomorrow .ace_support.ace_type{\
   color:#8959A8;\
 }\
 \
+.ace-tomorrow .ace_variable {\
+  color:#4271AE;\
+}\
+\
 .ace-tomorrow .ace_variable.ace_parameter {\
   color:#F5871F;\
 }\

lib/ace/theme/tomorrow_night.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: #4B4E55;\
 }\
 \
-.ace-tomorrow-night .ace_keyword {\
+.ace-tomorrow-night .ace_keyword, .ace-tomorrow-night .ace_meta {\
   color:#B294BB;\
 }\
 \
@@ -117,6 +117,10 @@ exports.cssText = "\
   color:#DE935F;\
 }\
 \
+.ace-tomorrow-night .ace_constant.ace_other {\
+  color:#CED1CF;\
+}\
+\
 .ace-tomorrow-night .ace_invalid {\
   color:#CED2CF;\
 background-color:#DF5F5F;\
@@ -127,6 +131,10 @@ background-color:#DF5F5F;\
 background-color:#B798BF;\
 }\
 \
+.ace-tomorrow-night .ace_support.ace_constant {\
+  color:#DE935F;\
+}\
+\
 .ace-tomorrow-night .ace_fold {\
     background-color: #81A2BE;\
     border-color: #C5C8C6;\
@@ -140,10 +148,14 @@ background-color:#B798BF;\
   color:#B294BB;\
 }\
 \
-.ace-tomorrow-night .ace_storage.ace_type {\
+.ace-tomorrow-night .ace_storage.ace_type,  .ace-tomorrow-night .ace_support.ace_type{\
   color:#B294BB;\
 }\
 \
+.ace-tomorrow-night .ace_variable {\
+  color:#81A2BE;\
+}\
+\
 .ace-tomorrow-night .ace_variable.ace_parameter {\
   color:#DE935F;\
 }\

lib/ace/theme/tomorrow_night_blue.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: #404F7D;\
 }\
 \
-.ace-tomorrow-night-blue .ace_keyword {\
+.ace-tomorrow-night-blue .ace_keyword, .ace-tomorrow-night-blue .ace_meta {\
   color:#EBBBFF;\
 }\
 \
@@ -117,6 +117,10 @@ exports.cssText = "\
   color:#FFC58F;\
 }\
 \
+.ace-tomorrow-night-blue .ace_constant.ace_other {\
+  color:#FFFFFF;\
+}\
+\
 .ace-tomorrow-night-blue .ace_invalid {\
   color:#FFFFFF;\
 background-color:#F99DA5;\
@@ -127,6 +131,10 @@ background-color:#F99DA5;\
 background-color:#EBBBFF;\
 }\
 \
+.ace-tomorrow-night-blue .ace_support.ace_constant {\
+  color:#FFC58F;\
+}\
+\
 .ace-tomorrow-night-blue .ace_fold {\
     background-color: #BBDAFF;\
     border-color: #FFFFFF;\
@@ -140,10 +148,14 @@ background-color:#EBBBFF;\
   color:#EBBBFF;\
 }\
 \
-.ace-tomorrow-night-blue .ace_storage.ace_type {\
+.ace-tomorrow-night-blue .ace_storage.ace_type,  .ace-tomorrow-night-blue .ace_support.ace_type{\
   color:#EBBBFF;\
 }\
 \
+.ace-tomorrow-night-blue .ace_variable {\
+  color:#BBDAFF;\
+}\
+\
 .ace-tomorrow-night-blue .ace_variable.ace_parameter {\
   color:#FFC58F;\
 }\

lib/ace/theme/tomorrow_night_bright.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: #343434;\
 }\
 \
-.ace-tomorrow-night-bright .ace_keyword {\
+.ace-tomorrow-night-bright .ace_keyword, .ace-tomorrow-night-bright .ace_meta {\
   color:#C397D8;\
 }\
 \
@@ -117,6 +117,10 @@ exports.cssText = "\
   color:#E78C45;\
 }\
 \
+.ace-tomorrow-night-bright .ace_constant.ace_other {\
+  color:#EEEEEE;\
+}\
+\
 .ace-tomorrow-night-bright .ace_invalid {\
   color:#CED2CF;\
 background-color:#DF5F5F;\
@@ -127,6 +131,10 @@ background-color:#DF5F5F;\
 background-color:#B798BF;\
 }\
 \
+.ace-tomorrow-night-bright .ace_support.ace_constant {\
+  color:#E78C45;\
+}\
+\
 .ace-tomorrow-night-bright .ace_fold {\
     background-color: #7AA6DA;\
     border-color: #DEDEDE;\
@@ -140,10 +148,14 @@ background-color:#B798BF;\
   color:#C397D8;\
 }\
 \
-.ace-tomorrow-night-bright .ace_storage.ace_type {\
+.ace-tomorrow-night-bright .ace_storage.ace_type,  .ace-tomorrow-night-bright .ace_support.ace_type{\
   color:#C397D8;\
 }\
 \
+.ace-tomorrow-night-bright .ace_variable {\
+  color:#7AA6DA;\
+}\
+\
 .ace-tomorrow-night-bright .ace_variable.ace_parameter {\
   color:#E78C45;\
 }\

lib/ace/theme/tomorrow_night_eighties.js

@@ -101,7 +101,7 @@ exports.cssText = "\
   color: #6A6A6A;\
 }\
 \
-.ace-tomorrow-night-eighties .ace_keyword {\
+.ace-tomorrow-night-eighties .ace_keyword, .ace-tomorrow-night-eighties .ace_meta {\
   color:#CC99CC;\
 }\
 \
@@ -117,6 +117,10 @@ exports.cssText = "\
   color:#F99157;\
 }\
 \
+.ace-tomorrow-night-eighties .ace_constant.ace_other {\
+  color:#CCCCCC;\
+}\
+\
 .ace-tomorrow-night-eighties .ace_invalid {\
   color:#CDCDCD;\
 background-color:#F2777A;\
@@ -127,6 +131,10 @@ background-color:#F2777A;\
 background-color:#CC99CC;\
 }\
 \
+.ace-tomorrow-night-eighties .ace_support.ace_constant {\
+  color:#F99157;\
+}\
+\
 .ace-tomorrow-night-eighties .ace_fold {\
     background-color: #6699CC;\
     border-color: #CCCCCC;\
@@ -140,10 +148,14 @@ background-color:#CC99CC;\
   color:#CC99CC;\
 }\
 \
-.ace-tomorrow-night-eighties .ace_storage.ace_type {\
+.ace-tomorrow-night-eighties .ace_storage.ace_type,  .ace-tomorrow-night-eighties .ace_support.ace_type{\
   color:#CC99CC;\
 }\
 \
+.ace-tomorrow-night-eighties .ace_variable {\
+  color:#6699CC;\
+}\
+\
 .ace-tomorrow-night-eighties .ace_variable.ace_parameter {\
   color:#F99157;\
 }\

lib/ace/theme/twilight.js

@@ -101,11 +101,11 @@ exports.cssText = "\
   color: rgba(255, 255, 255, 0.25);\
 }\
 \
-.ace-twilight .ace_keyword {\
+.ace-twilight .ace_keyword, .ace-twilight .ace_meta {\
   color:#CDA869;\
 }\
 \
-.ace-twilight .ace_constant {\
+.ace-twilight .ace_constant, .ace-twilight .ace_constant.ace_other {\
   color:#CF6A4C;\
 }\
 \
@@ -124,6 +124,10 @@ color:#D2A8A1;\
   color:#9B859D;\
 }\
 \
+.ace-twilight .ace_support.ace_constant {\
+  color:#CF6A4C;\
+}\
+\
 .ace-twilight .ace_fold {\
     background-color: #AC885B;\
     border-color: #F8F8F8;\
@@ -137,6 +141,10 @@ color:#D2A8A1;\
   color:#F9EE98;\
 }\
 \
+.ace-twilight .ace_variable {\
+  color:#AC885B;\
+}\
+\
 .ace-twilight .ace_string {\
   color:#8F9D6A;\
 }\

lib/ace/theme/vibrant_ink.js

@@ -101,11 +101,11 @@ exports.cssText = "\
   color: #404040;\
 }\
 \
-.ace-vibrant-ink .ace_keyword {\
+.ace-vibrant-ink .ace_keyword, .ace-vibrant-ink .ace_meta {\
   color:#FF6600;\
 }\
 \
-.ace-vibrant-ink .ace_constant {\
+.ace-vibrant-ink .ace_constant, .ace-vibrant-ink .ace_constant.ace_other {\
   color:#339999;\
 }\
 \
@@ -132,6 +132,10 @@ background-color:#000000;\
   color:#FFCC00;\
 }\
 \
+.ace-vibrant-ink .ace_variable {\
+  color:#FFCC00;\
+}\
+\
 .ace-vibrant-ink .ace_variable.ace_parameter {\
   font-style:italic;\
 }\

lib/ace/token_iterator_test.js

@@ -56,17 +56,17 @@ module.exports = {
             "    } // Real Tab.",
             "}"
         ];
-        var session = new EditSession(lines.join("\n"), new JavaScriptMode());      
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());
 
         var iterator = new TokenIterator(session, 0, 0);
         assert.equal(iterator.getCurrentToken().value, "function");
         assert.equal(iterator.getCurrentTokenRow(), 0);
         assert.equal(iterator.getCurrentTokenColumn(), 0);
-        
+
         iterator.stepForward();
         assert.equal(iterator.getCurrentToken().value, " ");
         assert.equal(iterator.getCurrentTokenRow(), 0);
-        assert.equal(iterator.getCurrentTokenColumn(), 8);      
+        assert.equal(iterator.getCurrentTokenColumn(), 8);
 
         var iterator = new TokenIterator(session, 0, 4);
         assert.equal(iterator.getCurrentToken().value, "function");
@@ -76,7 +76,7 @@ module.exports = {
         iterator.stepForward();
         assert.equal(iterator.getCurrentToken().value, " ");
         assert.equal(iterator.getCurrentTokenRow(), 0);
-        assert.equal(iterator.getCurrentTokenColumn(), 8);      
+        assert.equal(iterator.getCurrentTokenColumn(), 8);
 
         var iterator = new TokenIterator(session, 2, 18);
         assert.equal(iterator.getCurrentToken().value, "items");
@@ -86,8 +86,8 @@ module.exports = {
         iterator.stepForward();
         assert.equal(iterator.getCurrentToken().value, "[");
         assert.equal(iterator.getCurrentTokenRow(), 2);
-        assert.equal(iterator.getCurrentTokenColumn(), 19);     
-        
+        assert.equal(iterator.getCurrentTokenColumn(), 19);
+
         var iterator = new TokenIterator(session, 4, 0);
         assert.equal(iterator.getCurrentToken().value, "}");
         assert.equal(iterator.getCurrentTokenRow(), 4);
@@ -96,21 +96,21 @@ module.exports = {
         iterator.stepBackward();
         assert.equal(iterator.getCurrentToken().value, "// Real Tab.");
         assert.equal(iterator.getCurrentTokenRow(), 3);
-        assert.equal(iterator.getCurrentTokenColumn(), 6);      
-       
+        assert.equal(iterator.getCurrentTokenColumn(), 6);
+
         var iterator = new TokenIterator(session, 5, 0);
         assert.equal(iterator.getCurrentToken(), null);
     },
- 
+
     "test: token iterator initialization in text document" : function() {
         var lines = [
-            "Lorem ipsum dolor sit amet, consectetur adipisicing elit,", 
+            "Lorem ipsum dolor sit amet, consectetur adipisicing elit,",
             "sed do eiusmod tempor incididunt ut labore et dolore magna",
-            "aliqua. Ut enim ad minim veniam, quis nostrud exercitation", 
+            "aliqua. Ut enim ad minim veniam, quis nostrud exercitation",
             "ullamco laboris nisi ut aliquip ex ea commodo consequat."
         ];
         var session = new EditSession(lines.join("\n"));
-        
+
         var iterator = new TokenIterator(session, 0, 0);
         assert.equal(iterator.getCurrentToken().value, lines[0]);
         assert.equal(iterator.getCurrentTokenRow(), 0);
@@ -125,16 +125,16 @@ module.exports = {
         assert.equal(iterator.getCurrentToken().value, lines[2]);
         assert.equal(iterator.getCurrentTokenRow(), 2);
         assert.equal(iterator.getCurrentTokenColumn(), 0);
-        
+
         var iterator = new TokenIterator(session, 3, lines[3].length-1);
         assert.equal(iterator.getCurrentToken().value, lines[3]);
         assert.equal(iterator.getCurrentTokenRow(), 3);
         assert.equal(iterator.getCurrentTokenColumn(), 0);
-       
+
         var iterator = new TokenIterator(session, 4, 0);
         assert.equal(iterator.getCurrentToken(), null);
-    }, 
-    
+    },
+
     "test: token iterator step forward in JavaScript document" : function() {
         var lines = [
             "function foo(items) {",
@@ -143,10 +143,10 @@ module.exports = {
             "    } // Real Tab.",
             "}"
         ];
-        var session = new EditSession(lines.join("\n"), new JavaScriptMode());      
-    
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());
+
         var rows = session.getTokens(0, lines.length-1);
-        var tokens = [];    
+        var tokens = [];
         for (var i = 0; i < rows.length; i++)
             tokens = tokens.concat(rows[i].tokens);
 
@@ -156,7 +156,7 @@ module.exports = {
         assert.equal(iterator.stepForward(), null);
         assert.equal(iterator.getCurrentToken(), null);
     },
-    
+
     "test: token iterator step backward in JavaScript document" : function() {
         var lines = [
             "function foo(items) {",
@@ -165,13 +165,13 @@ module.exports = {
             "     } // Real Tab.",
             "}"
         ];
-        var session = new EditSession(lines.join("\n"), new JavaScriptMode());      
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());
 
         var rows = session.getTokens(0, lines.length-1);
-        var tokens = [];    
+        var tokens = [];
         for (var i = 0; i < rows.length; i++)
             tokens = tokens.concat(rows[i].tokens);
-    
+
         var iterator = new TokenIterator(session, 4, 0);
         for (var i = tokens.length-2; i >= 0; i--)
             assert.equal(iterator.stepBackward(), tokens[i]);
@@ -187,13 +187,13 @@ module.exports = {
             "    } // Real Tab.",
             "}"
         ];
-        var session = new EditSession(lines.join("\n"), new JavaScriptMode());      
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());
 
         var iterator = new TokenIterator(session, 0, 0);
-        
+
         iterator.stepForward();
         iterator.stepForward();
-        
+
         assert.equal(iterator.getCurrentToken().value, "foo");
         assert.equal(iterator.getCurrentTokenRow(), 0);
         assert.equal(iterator.getCurrentTokenColumn(), 9);
@@ -205,8 +205,7 @@ module.exports = {
         iterator.stepForward();
         iterator.stepForward();
         iterator.stepForward();
-        iterator.stepForward();
-        
+
         assert.equal(iterator.getCurrentToken().value, "for");
         assert.equal(iterator.getCurrentTokenRow(), 1);
         assert.equal(iterator.getCurrentTokenColumn(), 4);

lib/ace/tokenizer.js

@@ -50,23 +50,30 @@ var Tokenizer = function(rules, flag) {
         var ruleRegExps = [];
         var matchTotal = 0;
         var mapping = this.matchMappings[key] = {};
-        
+
         for ( var i = 0; i < state.length; i++) {
+
+            if (state[i].regex instanceof RegExp)
+                state[i].regex = state[i].regex.toString().slice(1, -1);
+
             // Count number of matching groups. 2 extra groups from the full match
             // And the catch-all on the end (used to force a match);
             var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
-        
+
             // Replace any backreferences and offset appropriately.
             var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
                 return "\\" + (parseInt(digit, 10) + matchTotal + 1);
             });
-            
+
+            if (matchcount > 1 && state[i].token.length !== matchcount-1)
+                throw new Error("Matching groups and length of the token array don't match in rule #" + i + " of state " + key);
+
             mapping[matchTotal] = {
                 rule: i,
                 len: matchcount
             };
             matchTotal += matchcount;
-            
+
             ruleRegExps.push(adjustedregex);
         }
 
@@ -82,47 +89,47 @@ var Tokenizer = function(rules, flag) {
         var mapping = this.matchMappings[currentState];
         var re = this.regExps[currentState];
         re.lastIndex = 0;
-        
+
         var match, tokens = [];
-        
+
         var lastIndex = 0;
-        
+
         var token = {
             type: null,
             value: ""
         };
-        
+
         while (match = re.exec(line)) {
             var type = "text";
             var rule = null;
             var value = [match[0]];
 
             for (var i = 0; i < match.length-2; i++) {
-                if (match[i + 1] !== undefined) {
-                    rule = state[mapping[i].rule];
-                    
-                    if (mapping[i].len > 1) {
-                        value = match.slice(i+2, i+1+mapping[i].len);
-                    }
-                    
-                    // compute token type
-                    if (typeof rule.token == "function")
-                        type = rule.token.apply(this, value);
-                    else
-                        type = rule.token;
+                if (match[i + 1] === undefined)
+                    continue;
 
-                    var next = rule.next;                    
-                    if (next && next !== currentState) {
-                        currentState = next;
-                        state = this.rules[currentState];
-                        mapping = this.matchMappings[currentState];
-                        lastIndex = re.lastIndex;
+                rule = state[mapping[i].rule];
 
-                        re = this.regExps[currentState];
-                        re.lastIndex = lastIndex;
-                    }
-                    break;
+                if (mapping[i].len > 1)
+                    value = match.slice(i+2, i+1+mapping[i].len);
+
+                // compute token type
+                if (typeof rule.token == "function")
+                    type = rule.token.apply(this, value);
+                else
+                    type = rule.token;
+
+                var next = rule.next;
+                if (next && next !== currentState) {
+                    currentState = next;
+                    state = this.rules[currentState];
+                    mapping = this.matchMappings[currentState];
+                    lastIndex = re.lastIndex;
+
+                    re = this.regExps[currentState];
+                    re.lastIndex = lastIndex;
                 }
+                break;
             }
 
             if (value[0]) {
@@ -131,13 +138,15 @@ var Tokenizer = function(rules, flag) {
                     type = [type];
                 }
                 for (var i = 0; i < value.length; i++) {
+                    if (!value[i])
+                        continue;
+
                     if ((!rule || rule.merge || type[i] === "text") && token.type === type[i]) {
                         token.value += value[i];
                     } else {
-                        if (token.type) {
+                        if (token.type)
                             tokens.push(token);
-                        }
-                    
+
                         token = {
                             type: type[i],
                             value: value[i]
@@ -145,10 +154,10 @@ var Tokenizer = function(rules, flag) {
                     }
                 }
             }
-            
+
             if (lastIndex == line.length)
                 break;
-            
+
             lastIndex = re.lastIndex;
         }
 

lib/ace/virtual_renderer.js

@@ -60,13 +60,13 @@ dom.importCssString(editorCss, "ace_editor");
 
 var VirtualRenderer = function(container, theme) {
     var _self = this;
-    
+
     this.container = container;
 
     // TODO: this breaks rendering in Cloud9 with multiple ace instances
 //    // Imports CSS once per DOM document ('ace_editor' serves as an identifier).
 //    dom.importCssString(editorCss, "ace_editor", container.ownerDocument);
-    
+
     dom.addCssClass(container, "ace_editor");
 
     this.setTheme(theme);
@@ -84,8 +84,8 @@ var VirtualRenderer = function(container, theme) {
     this.scroller.appendChild(this.content);
 
     this.$gutterLayer = new GutterLayer(this.$gutter);
-    this.$gutterLayer.on("changeGutterWidth", this.onResize.bind(this, true));    
-    
+    this.$gutterLayer.on("changeGutterWidth", this.onResize.bind(this, true));
+
     this.$markerBack = new MarkerLayer(this.content);
 
     var textLayer = this.$textLayer = new TextLayer(this.content);
@@ -110,7 +110,7 @@ var VirtualRenderer = function(container, theme) {
 
     this.scrollTop = 0;
     this.scrollLeft = 0;
-    
+
     event.addListener(this.scroller, "scroll", function() {
         var scrollLeft = _self.scroller.scrollLeft;
         _self.scrollLeft = scrollLeft;
@@ -352,7 +352,7 @@ var VirtualRenderer = function(container, theme) {
         // this persists in IE9
         if (useragent.isIE)
             return;
-        
+
         if (this.layerConfig.lastRow === 0)
             return;
 
@@ -428,13 +428,13 @@ var VirtualRenderer = function(container, theme) {
         // horizontal scrolling
         if (changes & this.CHANGE_H_SCROLL) {
             this.scroller.scrollLeft = this.scrollLeft;
-            
+
             // read the value after writing it since the value might get clipped
             var scrollLeft = this.scroller.scrollLeft;
             this.scrollLeft = scrollLeft;
             this.session.setScrollLeft(scrollLeft);
         }
-        
+
         // full
         if (changes & this.CHANGE_FULL) {
             this.$textLayer.checkForSizeChanges();
@@ -647,12 +647,18 @@ var VirtualRenderer = function(container, theme) {
         this.$cursorLayer.showCursor();
     };
 
-    this.scrollCursorIntoView = function() {
+    this.scrollSelectionIntoView = function(anchor, lead) {
+        // first scroll anchor into view then scroll lead into view
+        this.scrollCursorIntoView(anchor);
+        this.scrollCursorIntoView(lead);
+    };
+
+    this.scrollCursorIntoView = function(cursor) {
         // the editor is not visible
         if (this.$size.scrollerHeight === 0)
             return;
 
-        var pos = this.$cursorLayer.getPixelPosition();
+        var pos = this.$cursorLayer.getPixelPosition(cursor);
 
         var left = pos.left;
         var top = pos.top;
@@ -821,7 +827,7 @@ var VirtualRenderer = function(container, theme) {
     this._loadTheme = function(name, callback) {
         if (!config.get("packaged"))
             return callback();
-            
+
         var base = name.split("/").pop();
         var filename = config.get("themePath") + "/theme-" + base + config.get("suffix");
         net.loadScript(filename, callback);
@@ -833,14 +839,14 @@ var VirtualRenderer = function(container, theme) {
         this.$themeValue = theme;
         if (!theme || typeof theme == "string") {
             var moduleName = theme || "ace/theme/textmate";
-            
+
             var module;
             try {
                 module = require(moduleName);
             } catch (e) {};
             if (module)
                 return afterLoad(module);
-            
+
             _self._loadTheme(moduleName, function() {
                 require([moduleName], function(module) {
                     if (_self.$themeValue !== theme)