Merge master

demo/kitchen-sink/doclist.js

@@ -90,6 +90,7 @@ var docs = {
     "docs/latex.tex": {name: "LaTeX", wrapped: true},
     "docs/less.less": "LESS",
     "docs/lisp.lisp": "Lisp",
+    "docs/scheme.scm": "Scheme",
     "docs/liquid.liquid": "Liquid",
     "docs/lua.lua": "Lua",
     "docs/lucene.lucene": "Lucene",
@@ -119,6 +120,7 @@ var docs = {
     "docs/tcl.tcl": "Tcl",
     "docs/tex.tex": "Tex",
     "docs/textile.textile": {name: "Textile", wrapped: true},
+    "docs/tmSnippet.tmSnippet": "tmSnippet",
     "docs/typescript.ts": "Typescript",
     "docs/vbscript.vbs": "VBScript",
     "docs/xml.xml": "XML",

demo/kitchen-sink/docs/ruby.rb

@@ -20,4 +20,16 @@ class Range
   end
 end
 
-{:id => 34, :sdfasdfasdf => "asdasdads"}
+{:id => 34, :key => "value"}
+
+
+    herDocs = [<<'FOO', <<BAR, <<-BAZ, <<-`EXEC`] #comment
+  FOO #{literal}
+FOO
+  BAR #{fact(10)}
+BAR
+  BAZ indented
+    BAZ
+        echo hi
+    EXEC
+puts herDocs

demo/kitchen-sink/docs/scheme.scm

@@ -0,0 +1,21 @@
+(define (prompt-for-cd)
+   "Prompts
+    for CD"
+   (prompt-read "Title" 1.53 1 2/4 1.7 1.7e0 2.9E-4 +42 -7 #b001 #b001/100 #o777 #O777 #xabc55 #c(0 -5.6))
+   (prompt-read "Artist")
+   (or (parse-integer (prompt-read "Rating") #:junk-allowed #t) 0)
+  (if x (format #t "yes") (format #f "no") ;and here comment
+  ) 
+  ;; second line comment
+  '(+ 1 2)
+  (position-if-not char-set:whitespace line #:start beg))
+  (quote (privet 1 2 3))
+  '(hello world)
+  (* 5 7)
+  (1 2 34 5)
+  (#:use "aaaa")
+  (let ((x 10) (y 20))
+    (display (+ x y))
+  ) 
+
+  "asdad\0eqweqe"

demo/kitchen-sink/docs/tmSnippet.tmSnippet

@@ -0,0 +1,19 @@
+$$------------------------------------
+tabTrigger: t
+name: Heading 3
+scope: language
+content: -----------------------------
+\begin{${1:documnet}}
+    ${2:$TM_SELECTED_TEXT:some latex}
+    ${3:$TM_SELECTED_TEXT/a/b/c}
+    ${4:${TM_SELECTED_TEXT/(.)/\\u$1/g:7}}
+\end{$1}
+$0\\$$
+$$------------------------------------
+tabTrigger: ^3
+name: Heading 3
+scope: language
+content: -----------------------------
+
+${TM_CURRENT_LINE/./^/g}
+$$------------------------------------

demo/kitchen-sink/modelist.js

@@ -62,7 +62,8 @@ var modesByName = {
     jsx:        ["JSX"          , "jsx"],
     latex:      ["LaTeX"        , "latex|tex|ltx|bib"],
     less:       ["LESS"         , "less"],
-    lisp:       ["Lisp"         , "lisp|scm|rkt"],
+    lisp:       ["Lisp"         , "lisp"],
+    scheme:     ["Scheme"       , "scm|rkt"],
     liquid:     ["Liquid"       , "liquid"],
     lua:        ["Lua"          , "lua"],
     luapage:    ["LuaPage"      , "lp"], // http://keplerproject.github.com/cgilua/manual.html#templates
@@ -91,6 +92,7 @@ var modesByName = {
     tex:        ["Tex"          , "tex"],
     text:       ["Text"         , "txt"],
     textile:    ["Textile"      , "textile"],
+    tm_snippet: ["tmSnippet"    , "tmSnippet"],
     typescript: ["Typescript"   , "typescript|ts|str"],
     vbscript:   ["VBScript"     , "vbs"],
     xml:        ["XML"          , "xml|rdf|rss|wsdl|xslt|atom|mathml|mml|xul|xbl"],

index.html

@@ -906,6 +906,11 @@ oop.inherits(FoldMode, BaseFoldMode);
                                         style="position: relative; left: 20px; top: 5px;">
                                     <a href="https://addons.mozilla.org/en-US/firefox/addon/acebug/">Acebug</a>
                                 </li>
+                                <li>
+                                    <img src="http://cibox.org/images/logo.png"
+                                        style="position: relative; left: -16px; width: 134px;top: 2px;">
+                                    <a href="http://cibox.org/slivu/stackoverflow?lang=ruby&file=splat-params-with-espresso.rb">CIBox</a>
+                                </li>
                                 <li>
                                     <a href="https://github.com/Gozala/sky-edit">Sky Edit</a>
                                 </li>

lib/ace/background_tokenizer.js

@@ -236,7 +236,7 @@ var BackgroundTokenizer = function(tokenizer, editor) {
             data.state = "start";
         }
 
-        if (this.states[row] !== data.state) {
+        if (this.states[row] + "" !== data.state + "") {
             this.states[row] = data.state;
             this.lines[row + 1] = null;
             if (this.currentLine > row + 1)

lib/ace/ext/textarea.js

@@ -73,7 +73,7 @@ function applyStyles(elm, styles) {
 }
 
 function setupContainer(element, getValue) {
-        if (element.type != 'textarea') {
+    if (element.type != 'textarea') {
         throw "Textarea required!";
     }
 
@@ -132,11 +132,7 @@ function setupContainer(element, getValue) {
     resizeEvent();
 
     // Insert the div container after the element.
-    if (element.nextSibling) {
+    parentNode.insertBefore(container, element.nextSibling);
-        parentNode.insertBefore(container, element.nextSibling);
-    } else {
-        parentNode.appendChild(container);
-    }
 
     // Override the forms onsubmit function. Set the innerHTML and value
     // of the textarea before submitting.
@@ -145,7 +141,6 @@ function setupContainer(element, getValue) {
             var oldSumit = parentNode.onsubmit;
             // Override the onsubmit function of the form.
             parentNode.onsubmit = function(evt) {
-                element.innerHTML = getValue();
                 element.value = getValue();
                 // If there is a onsubmit function already, then call
                 // it with the current context and pass the event.

lib/ace/lib/es5-shim.js

@@ -1,28 +1,5 @@
-// vim: ts=4 sts=4 sw=4 expandtab
+// https://github.com/kriskowal/es5-shim
-// -- kriskowal Kris Kowal Copyright (C) 2009-2011 MIT License
+// Copyright 2009-2012 by contributors, MIT License
-// -- tlrobinson Tom Robinson Copyright (C) 2009-2010 MIT License (Narwhal Project)
-// -- dantman Daniel Friesen Copyright (C) 2010 XXX TODO License or CLA
-// -- fschaefer Florian Schäfer Copyright (C) 2010 MIT License
-// -- Gozala Irakli Gozalishvili Copyright (C) 2010 MIT License
-// -- kitcambridge Kit Cambridge Copyright (C) 2011 MIT License
-// -- kossnocorp Sasha Koss XXX TODO License or CLA
-// -- bryanforbes Bryan Forbes XXX TODO License or CLA
-// -- killdream Quildreen Motta Copyright (C) 2011 MIT Licence
-// -- michaelficarra Michael Ficarra Copyright (C) 2011 3-clause BSD License
-// -- sharkbrainguy Gerard Paapu Copyright (C) 2011 MIT License
-// -- bbqsrc Brendan Molloy (C) 2011 Creative Commons Zero (public domain)
-// -- iwyg XXX TODO License or CLA
-// -- DomenicDenicola Domenic Denicola Copyright (C) 2011 MIT License
-// -- xavierm02 Montillet Xavier XXX TODO License or CLA
-// -- Raynos Raynos XXX TODO License or CLA
-// -- samsonjs Sami Samhuri Copyright (C) 2010 MIT License
-// -- rwldrn Rick Waldron Copyright (C) 2011 MIT License
-// -- lexer Alexey Zakharov XXX TODO License or CLA
-
-/*!
-    Copyright (c) 2009, 280 North Inc. http://280north.com/
-    MIT License. http://github.com/280north/narwhal/blob/master/README.md
-*/
 
 define(function(require, exports, module) {
 
@@ -32,12 +9,9 @@ define(function(require, exports, module) {
  *
  * Annotated ES5: http://es5.github.com/ (specific links below)
  * ES5 Spec: http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-262.pdf
- *
+ * Required reading: http://javascriptweblog.wordpress.com/2011/12/05/extending-javascript-natives/
- * @module
  */
 
-/*whatsupdoc*/
-
 //
 // Function
 // ========
@@ -46,13 +20,16 @@ define(function(require, exports, module) {
 // ES-5 15.3.4.5
 // http://es5.github.com/#x15.3.4.5
 
+function Empty() {}
+
 if (!Function.prototype.bind) {
     Function.prototype.bind = function bind(that) { // .length is 1
         // 1. Let Target be the this value.
         var target = this;
         // 2. If IsCallable(Target) is false, throw a TypeError exception.
-        if (typeof target != "function")
+        if (typeof target != "function") {
-            throw new TypeError(); // TODO message
+            throw new TypeError("Function.prototype.bind called on incompatible " + target);
+        }
         // 3. Let A be a new (possibly empty) internal list of all of the
         //   argument values provided after thisArg (arg1, arg2 etc), in order.
         // XXX slicedArgs will stand in for "A" if used
@@ -85,17 +62,14 @@ if (!Function.prototype.bind) {
                 // 5. Return the result of calling the [[Construct]] internal
                 //   method of target providing args as the arguments.
 
-                var F = function(){};
-                F.prototype = target.prototype;
-                var self = new F;
-
                 var result = target.apply(
-                    self,
+                    this,
                     args.concat(slice.call(arguments))
                 );
-                if (result !== null && Object(result) === result)
+                if (Object(result) === result) {
                     return result;
-                return self;
+                }
+                return this;
 
             } else {
                 // 15.3.4.5.1 [[Call]]
@@ -125,6 +99,12 @@ if (!Function.prototype.bind) {
             }
 
         };
+        if(target.prototype) {
+            Empty.prototype = target.prototype;
+            bound.prototype = new Empty();
+            // Clean up dangling references.
+            Empty.prototype = null;
+        }
         // XXX bound.length is never writable, so don't even try
         //
         // 15. If the [[Class]] internal property of Target is "Function", then
@@ -168,7 +148,8 @@ var call = Function.prototype.call;
 var prototypeOfArray = Array.prototype;
 var prototypeOfObject = Object.prototype;
 var slice = prototypeOfArray.slice;
-var toString = call.bind(prototypeOfObject.toString);
+// Having a toString local variable name breaks in Opera so use _toString.
+var _toString = call.bind(prototypeOfObject.toString);
 var owns = call.bind(prototypeOfObject.hasOwnProperty);
 
 // If JS engine supports accessors creating shortcuts.
@@ -189,12 +170,109 @@ if ((supportsAccessors = owns(prototypeOfObject, "__defineGetter__"))) {
 // =====
 //
 
+// ES5 15.4.4.12
+// http://es5.github.com/#x15.4.4.12
+// Default value for second param
+// [bugfix, ielt9, old browsers]
+// IE < 9 bug: [1,2].splice(0).join("") == "" but should be "12"
+if ([1,2].splice(0).length != 2) {
+    if(function() { // test IE < 9 to splice bug - see issue #138
+        function makeArray(l) {
+            var a = new Array(l+2);
+            a[0] = a[1] = 0;
+            return a;
+        }
+        var array = [], lengthBefore;
+        
+        array.splice.apply(array, makeArray(20));
+        array.splice.apply(array, makeArray(26));
+
+        lengthBefore = array.length; //46
+        array.splice(5, 0, "XXX"); // add one element
+
+        lengthBefore + 1 == array.length
+
+        if (lengthBefore + 1 == array.length) {
+            return true;// has right splice implementation without bugs
+        }
+        // else {
+        // IE8 bug
+        // }
+    }()) {//IE 6/7
+        var array_splice = Array.prototype.splice;
+        Array.prototype.splice = function(start, deleteCount) {
+            if (!arguments.length) {
+                return [];
+            } else {
+                return array_splice.apply(this, [
+                    start === void 0 ? 0 : start,
+                    deleteCount === void 0 ? (this.length - start) : deleteCount
+                ].concat(slice.call(arguments, 2)))
+            }
+        };
+    } else {//IE8
+        // taken from http://docs.sencha.com/ext-js/4-1/source/Array2.html
+        Array.prototype.splice = function(pos, removeCount){
+            var length = this.length;
+            if (pos > 0) {
+                if (pos > length)
+                    pos = length;
+            } else if (pos == void 0) {
+                pos = 0;
+            } else if (pos < 0) {
+                pos = Math.max(length + pos, 0);
+            }
+
+            if (!(pos+removeCount < length))
+                removeCount = length - pos;
+
+            var removed = this.slice(pos, pos+removeCount);
+            var insert = slice.call(arguments, 2);
+            var add = insert.length;            
+
+            // we try to use Array.push when we can for efficiency...
+            if (pos === length) {
+                if (add) {
+                    this.push.apply(this, insert);
+                }
+            } else {
+                var remove = Math.min(removeCount, length - pos);
+                var tailOldPos = pos + remove;
+                var tailNewPos = tailOldPos + add - remove;
+                var tailCount = length - tailOldPos;
+                var lengthAfterRemove = length - remove;
+
+                if (tailNewPos < tailOldPos) { // case A
+                    for (var i = 0; i < tailCount; ++i) {
+                        this[tailNewPos+i] = this[tailOldPos+i];
+                    }
+                } else if (tailNewPos > tailOldPos) { // case B
+                    for (i = tailCount; i--; ) {
+                        this[tailNewPos+i] = this[tailOldPos+i];
+                    }
+                } // else, add == remove (nothing to do)
+
+                if (add && pos === lengthAfterRemove) {
+                    this.length = lengthAfterRemove; // truncate array
+                    this.push.apply(this, insert);
+                } else {
+                    this.length = lengthAfterRemove + add; // reserves space
+                    for (i = 0; i < add; ++i) {
+                        this[pos+i] = insert[i];
+                    }
+                }
+            }
+            return removed;
+        };
+    }
+}
+
 // ES5 15.4.3.2
 // http://es5.github.com/#x15.4.3.2
 // https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/Array/isArray
 if (!Array.isArray) {
     Array.isArray = function isArray(obj) {
-        return toString(obj) == "[object Array]";
+        return _toString(obj) == "[object Array]";
     };
 }
 
@@ -213,25 +291,34 @@ if (!Array.isArray) {
 // ES5 15.4.4.18
 // http://es5.github.com/#x15.4.4.18
 // https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/array/forEach
+
+// Check failure of by-index access of string characters (IE < 9)
+// and failure of `0 in boxedString` (Rhino)
+var boxedString = Object("a"),
+    splitString = boxedString[0] != "a" || !(0 in boxedString);
+
 if (!Array.prototype.forEach) {
     Array.prototype.forEach = function forEach(fun /*, thisp*/) {
-        var self = toObject(this),
+        var object = toObject(this),
+            self = splitString && _toString(this) == "[object String]" ?
+                this.split("") :
+                object,
             thisp = arguments[1],
-            i = 0,
+            i = -1,
             length = self.length >>> 0;
 
         // If no callback function or if callback is not a callable function
-        if (toString(fun) != "[object Function]") {
+        if (_toString(fun) != "[object Function]") {
             throw new TypeError(); // TODO message
         }
 
-        while (i < length) {
+        while (++i < length) {
             if (i in self) {
                 // Invoke the callback function with call, passing arguments:
-                // context, property value, property key, thisArg object context
+                // context, property value, property key, thisArg object
-                fun.call(thisp, self[i], i, self);
+                // context
+                fun.call(thisp, self[i], i, object);
             }
-            i++;
         }
     };
 }
@@ -241,19 +328,22 @@ if (!Array.prototype.forEach) {
 // https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Objects/Array/map
 if (!Array.prototype.map) {
     Array.prototype.map = function map(fun /*, thisp*/) {
-        var self = toObject(this),
+        var object = toObject(this),
+            self = splitString && _toString(this) == "[object String]" ?
+                this.split("") :
+                object,
             length = self.length >>> 0,
             result = Array(length),
             thisp = arguments[1];
 
         // If no callback function or if callback is not a callable function
-        if (toString(fun) != "[object Function]") {
+        if (_toString(fun) != "[object Function]") {
-            throw new TypeError(); // TODO message
+            throw new TypeError(fun + " is not a function");
         }
 
         for (var i = 0; i < length; i++) {
             if (i in self)
-                result[i] = fun.call(thisp, self[i], i, self);
+                result[i] = fun.call(thisp, self[i], i, object);
         }
         return result;
     };
@@ -264,19 +354,27 @@ if (!Array.prototype.map) {
 // https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Objects/Array/filter
 if (!Array.prototype.filter) {
     Array.prototype.filter = function filter(fun /*, thisp */) {
-        var self = toObject(this),
+        var object = toObject(this),
+            self = splitString && _toString(this) == "[object String]" ?
+                this.split("") :
+                    object,
             length = self.length >>> 0,
             result = [],
+            value,
             thisp = arguments[1];
 
         // If no callback function or if callback is not a callable function
-        if (toString(fun) != "[object Function]") {
+        if (_toString(fun) != "[object Function]") {
-            throw new TypeError(); // TODO message
+            throw new TypeError(fun + " is not a function");
         }
 
         for (var i = 0; i < length; i++) {
-            if (i in self && fun.call(thisp, self[i], i, self))
+            if (i in self) {
-                result.push(self[i]);
+                value = self[i];
+                if (fun.call(thisp, value, i, object)) {
+                    result.push(value);
+                }
+            }
         }
         return result;
     };
@@ -287,18 +385,22 @@ if (!Array.prototype.filter) {
 // https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/Array/every
 if (!Array.prototype.every) {
     Array.prototype.every = function every(fun /*, thisp */) {
-        var self = toObject(this),
+        var object = toObject(this),
+            self = splitString && _toString(this) == "[object String]" ?
+                this.split("") :
+                object,
             length = self.length >>> 0,
             thisp = arguments[1];
 
         // If no callback function or if callback is not a callable function
-        if (toString(fun) != "[object Function]") {
+        if (_toString(fun) != "[object Function]") {
-            throw new TypeError(); // TODO message
+            throw new TypeError(fun + " is not a function");
         }
 
         for (var i = 0; i < length; i++) {
-            if (i in self && !fun.call(thisp, self[i], i, self))
+            if (i in self && !fun.call(thisp, self[i], i, object)) {
                 return false;
+            }
         }
         return true;
     };
@@ -309,18 +411,22 @@ if (!Array.prototype.every) {
 // https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/Array/some
 if (!Array.prototype.some) {
     Array.prototype.some = function some(fun /*, thisp */) {
-        var self = toObject(this),
+        var object = toObject(this),
+            self = splitString && _toString(this) == "[object String]" ?
+                this.split("") :
+                object,
             length = self.length >>> 0,
             thisp = arguments[1];
 
         // If no callback function or if callback is not a callable function
-        if (toString(fun) != "[object Function]") {
+        if (_toString(fun) != "[object Function]") {
-            throw new TypeError(); // TODO message
+            throw new TypeError(fun + " is not a function");
         }
 
         for (var i = 0; i < length; i++) {
-            if (i in self && fun.call(thisp, self[i], i, self))
+            if (i in self && fun.call(thisp, self[i], i, object)) {
                 return true;
+            }
         }
         return false;
     };
@@ -331,17 +437,21 @@ if (!Array.prototype.some) {
 // https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Objects/Array/reduce
 if (!Array.prototype.reduce) {
     Array.prototype.reduce = function reduce(fun /*, initial*/) {
-        var self = toObject(this),
+        var object = toObject(this),
+            self = splitString && _toString(this) == "[object String]" ?
+                this.split("") :
+                object,
             length = self.length >>> 0;
 
         // If no callback function or if callback is not a callable function
-        if (toString(fun) != "[object Function]") {
+        if (_toString(fun) != "[object Function]") {
-            throw new TypeError(); // TODO message
+            throw new TypeError(fun + " is not a function");
         }
 
         // no value to return if no initial value and an empty array
-        if (!length && arguments.length == 1)
+        if (!length && arguments.length == 1) {
-            throw new TypeError(); // TODO message
+            throw new TypeError("reduce of empty array with no initial value");
+        }
 
         var i = 0;
         var result;
@@ -355,14 +465,16 @@ if (!Array.prototype.reduce) {
                 }
 
                 // if array contains no values, no initial value to return
-                if (++i >= length)
+                if (++i >= length) {
-                    throw new TypeError(); // TODO message
+                    throw new TypeError("reduce of empty array with no initial value");
+                }
             } while (true);
         }
 
         for (; i < length; i++) {
-            if (i in self)
+            if (i in self) {
-                result = fun.call(void 0, result, self[i], i, self);
+                result = fun.call(void 0, result, self[i], i, object);
+            }
         }
 
         return result;
@@ -374,17 +486,21 @@ if (!Array.prototype.reduce) {
 // https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Objects/Array/reduceRight
 if (!Array.prototype.reduceRight) {
     Array.prototype.reduceRight = function reduceRight(fun /*, initial*/) {
-        var self = toObject(this),
+        var object = toObject(this),
+            self = splitString && _toString(this) == "[object String]" ?
+                this.split("") :
+                object,
             length = self.length >>> 0;
 
         // If no callback function or if callback is not a callable function
-        if (toString(fun) != "[object Function]") {
+        if (_toString(fun) != "[object Function]") {
-            throw new TypeError(); // TODO message
+            throw new TypeError(fun + " is not a function");
         }
 
         // no value to return if no initial value, empty array
-        if (!length && arguments.length == 1)
+        if (!length && arguments.length == 1) {
-            throw new TypeError(); // TODO message
+            throw new TypeError("reduceRight of empty array with no initial value");
+        }
 
         var result, i = length - 1;
         if (arguments.length >= 2) {
@@ -397,14 +513,16 @@ if (!Array.prototype.reduceRight) {
                 }
 
                 // if array contains no values, no initial value to return
-                if (--i < 0)
+                if (--i < 0) {
-                    throw new TypeError(); // TODO message
+                    throw new TypeError("reduceRight of empty array with no initial value");
+                }
             } while (true);
         }
 
         do {
-            if (i in this)
+            if (i in this) {
-                result = fun.call(void 0, result, self[i], i, self);
+                result = fun.call(void 0, result, self[i], i, object);
+            }
         } while (i--);
 
         return result;
@@ -414,17 +532,21 @@ if (!Array.prototype.reduceRight) {
 // ES5 15.4.4.14
 // http://es5.github.com/#x15.4.4.14
 // https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/Array/indexOf
-if (!Array.prototype.indexOf) {
+if (!Array.prototype.indexOf || ([0, 1].indexOf(1, 2) != -1)) {
     Array.prototype.indexOf = function indexOf(sought /*, fromIndex */ ) {
-        var self = toObject(this),
+        var self = splitString && _toString(this) == "[object String]" ?
+                this.split("") :
+                toObject(this),
             length = self.length >>> 0;
 
-        if (!length)
+        if (!length) {
             return -1;
+        }
 
         var i = 0;
-        if (arguments.length > 1)
+        if (arguments.length > 1) {
             i = toInteger(arguments[1]);
+        }
 
         // handle negative indices
         i = i >= 0 ? i : Math.max(0, length + i);
@@ -440,21 +562,26 @@ if (!Array.prototype.indexOf) {
 // ES5 15.4.4.15
 // http://es5.github.com/#x15.4.4.15
 // https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/Array/lastIndexOf
-if (!Array.prototype.lastIndexOf) {
+if (!Array.prototype.lastIndexOf || ([0, 1].lastIndexOf(0, -3) != -1)) {
     Array.prototype.lastIndexOf = function lastIndexOf(sought /*, fromIndex */) {
-        var self = toObject(this),
+        var self = splitString && _toString(this) == "[object String]" ?
+                this.split("") :
+                toObject(this),
             length = self.length >>> 0;
 
-        if (!length)
+        if (!length) {
             return -1;
+        }
         var i = length - 1;
-        if (arguments.length > 1)
+        if (arguments.length > 1) {
             i = Math.min(i, toInteger(arguments[1]));
+        }
         // handle negative indices
         i = i >= 0 ? i : length - Math.abs(i);
         for (; i >= 0; i--) {
-            if (i in self && sought === self[i])
+            if (i in self && sought === self[i]) {
                 return i;
+            }
         }
         return -1;
     };
@@ -803,13 +930,18 @@ if (!Object.keys) {
         ],
         dontEnumsLength = dontEnums.length;
 
-    for (var key in {"toString": null})
+    for (var key in {"toString": null}) {
         hasDontEnumBug = false;
+    }
 
     Object.keys = function keys(object) {
 
-        if ((typeof object != "object" && typeof object != "function") || object === null)
+        if (
+            (typeof object != "object" && typeof object != "function") ||
+            object === null
+        ) {
             throw new TypeError("Object.keys called on a non-object");
+        }
 
         var keys = [];
         for (var name in object) {
@@ -826,49 +958,16 @@ if (!Object.keys) {
                 }
             }
         }
-
         return keys;
     };
 
 }
 
 //
-// Date
+// most of es5-shim Date section is removed since ace doesn't need it, it is too intrusive and it causes problems for users
 // ====
 //
 
-// ES5 15.9.5.43
-// http://es5.github.com/#x15.9.5.43
-// This function returns a String value represent the instance in time 
-// represented by this Date object. The format of the String is the Date Time 
-// string format defined in 15.9.1.15. All fields are present in the String. 
-// The time zone is always UTC, denoted by the suffix Z. If the time value of 
-// this object is not a finite Number a RangeError exception is thrown.
-if (!Date.prototype.toISOString || (new Date(-62198755200000).toISOString().indexOf('-000001') === -1)) {
-    Date.prototype.toISOString = function toISOString() {
-        var result, length, value, year;
-        if (!isFinite(this))
-            throw new RangeError;
-
-        // the date time string format is specified in 15.9.1.15.
-        result = [this.getUTCMonth() + 1, this.getUTCDate(),
-            this.getUTCHours(), this.getUTCMinutes(), this.getUTCSeconds()];
-        year = this.getUTCFullYear();
-        year = (year < 0 ? '-' : (year > 9999 ? '+' : '')) + ('00000' + Math.abs(year)).slice(0 <= year && year <= 9999 ? -4 : -6);
-
-        length = result.length;
-        while (length--) {
-            value = result[length];
-            // pad months, days, hours, minutes, and seconds to have two digits.
-            if (value < 10)
-                result[length] = "0" + value;
-        }
-        // pad milliseconds to have three digits.
-        return year + "-" + result.slice(0, 2).join("-") + "T" + result.slice(2).join(":") + "." +
-            ("000" + this.getUTCMilliseconds()).slice(-3) + "Z";
-    }
-}
-
 // ES5 15.9.4.4
 // http://es5.github.com/#x15.9.4.4
 if (!Date.now) {
@@ -877,161 +976,51 @@ if (!Date.now) {
     };
 }
 
-// ES5 15.9.5.44
-// http://es5.github.com/#x15.9.5.44
-// This function provides a String representation of a Date object for use by 
-// JSON.stringify (15.12.3).
-if (!Date.prototype.toJSON) {
-    Date.prototype.toJSON = function toJSON(key) {
-        // When the toJSON method is called with argument key, the following 
-        // steps are taken:
-
-        // 1.  Let O be the result of calling ToObject, giving it the this
-        // value as its argument.
-        // 2. Let tv be ToPrimitive(O, hint Number).
-        // 3. If tv is a Number and is not finite, return null.
-        // XXX
-        // 4. Let toISO be the result of calling the [[Get]] internal method of
-        // O with argument "toISOString".
-        // 5. If IsCallable(toISO) is false, throw a TypeError exception.
-        if (typeof this.toISOString != "function")
-            throw new TypeError(); // TODO message
-        // 6. Return the result of calling the [[Call]] internal method of
-        //  toISO with O as the this value and an empty argument list.
-        return this.toISOString();
-
-        // NOTE 1 The argument is ignored.
-
-        // NOTE 2 The toJSON function is intentionally generic; it does not
-        // require that its this value be a Date object. Therefore, it can be
-        // transferred to other kinds of objects for use as a method. However,
-        // it does require that any such object have a toISOString method. An
-        // object is free to use the argument key to filter its
-        // stringification.
-    };
-}
-
-// ES5 15.9.4.2
-// http://es5.github.com/#x15.9.4.2
-// based on work shared by Daniel Friesen (dantman)
-// http://gist.github.com/303249
-if (Date.parse("+275760-09-13T00:00:00.000Z") !== 8.64e15) {
-    // XXX global assignment won't work in embeddings that use
-    // an alternate object for the context.
-    Date = (function(NativeDate) {
-
-        // Date.length === 7
-        var Date = function Date(Y, M, D, h, m, s, ms) {
-            var length = arguments.length;
-            if (this instanceof NativeDate) {
-                var date = length == 1 && String(Y) === Y ? // isString(Y)
-                    // We explicitly pass it through parse:
-                    new NativeDate(Date.parse(Y)) :
-                    // We have to manually make calls depending on argument
-                    // length here
-                    length >= 7 ? new NativeDate(Y, M, D, h, m, s, ms) :
-                    length >= 6 ? new NativeDate(Y, M, D, h, m, s) :
-                    length >= 5 ? new NativeDate(Y, M, D, h, m) :
-                    length >= 4 ? new NativeDate(Y, M, D, h) :
-                    length >= 3 ? new NativeDate(Y, M, D) :
-                    length >= 2 ? new NativeDate(Y, M) :
-                    length >= 1 ? new NativeDate(Y) :
-                                  new NativeDate();
-                // Prevent mixups with unfixed Date object
-                date.constructor = Date;
-                return date;
-            }
-            return NativeDate.apply(this, arguments);
-        };
-
-        // 15.9.1.15 Date Time String Format.
-        var isoDateExpression = new RegExp("^" +
-            "(\\d{4}|[\+\-]\\d{6})" + // four-digit year capture or sign + 6-digit extended year
-            "(?:-(\\d{2})" + // optional month capture
-            "(?:-(\\d{2})" + // optional day capture
-            "(?:" + // capture hours:minutes:seconds.milliseconds
-                "T(\\d{2})" + // hours capture
-                ":(\\d{2})" + // minutes capture
-                "(?:" + // optional :seconds.milliseconds
-                    ":(\\d{2})" + // seconds capture
-                    "(?:\\.(\\d{3}))?" + // milliseconds capture
-                ")?" +
-            "(?:" + // capture UTC offset component
-                "Z|" + // UTC capture
-                "(?:" + // offset specifier +/-hours:minutes
-                    "([-+])" + // sign capture
-                    "(\\d{2})" + // hours offset capture
-                    ":(\\d{2})" + // minutes offset capture
-                ")" +
-            ")?)?)?)?" +
-        "$");
-
-        // Copy any custom methods a 3rd party library may have added
-        for (var key in NativeDate)
-            Date[key] = NativeDate[key];
-
-        // Copy "native" methods explicitly; they may be non-enumerable
-        Date.now = NativeDate.now;
-        Date.UTC = NativeDate.UTC;
-        Date.prototype = NativeDate.prototype;
-        Date.prototype.constructor = Date;
-
-        // Upgrade Date.parse to handle simplified ISO 8601 strings
-        Date.parse = function parse(string) {
-            var match = isoDateExpression.exec(string);
-            if (match) {
-                match.shift(); // kill match[0], the full match
-                // parse months, days, hours, minutes, seconds, and milliseconds
-                for (var i = 1; i < 7; i++) {
-                    // provide default values if necessary
-                    match[i] = +(match[i] || (i < 3 ? 1 : 0));
-                    // match[1] is the month. Months are 0-11 in JavaScript
-                    // `Date` objects, but 1-12 in ISO notation, so we
-                    // decrement.
-                    if (i == 1)
-                        match[i]--;
-                }
-
-                // parse the UTC offset component
-                var minuteOffset = +match.pop(), hourOffset = +match.pop(), sign = match.pop();
-
-                // compute the explicit time zone offset if specified
-                var offset = 0;
-                if (sign) {
-                    // detect invalid offsets and return early
-                    if (hourOffset > 23 || minuteOffset > 59)
-                        return NaN;
-
-                    // express the provided time zone offset in minutes. The offset is
-                    // negative for time zones west of UTC; positive otherwise.
-                    offset = (hourOffset * 60 + minuteOffset) * 6e4 * (sign == "+" ? -1 : 1);
-                }
-
-                // Date.UTC for years between 0 and 99 converts year to 1900 + year
-                // The Gregorian calendar has a 400-year cycle, so 
-                // to Date.UTC(year + 400, .... ) - 12622780800000 == Date.UTC(year, ...),
-                // where 12622780800000 - number of milliseconds in Gregorian calendar 400 years
-                var year = +match[0];
-                if (0 <= year && year <= 99) {
-                    match[0] = year + 400;
-                    return NativeDate.UTC.apply(this, match) + offset - 12622780800000;
-                }
-
-                // compute a new UTC date value, accounting for the optional offset
-                return NativeDate.UTC.apply(this, match) + offset;
-            }
-            return NativeDate.parse.apply(this, arguments);
-        };
-
-        return Date;
-    })(Date);
-}
 
 //
 // String
 // ======
 //
 
+
+// ES5 15.5.4.14
+// http://es5.github.com/#x15.5.4.14
+// [bugfix, chrome]
+// If separator is undefined, then the result array contains just one String,
+// which is the this value (converted to a String). If limit is not undefined,
+// then the output array is truncated so that it contains no more than limit
+// elements.
+// "0".split(undefined, 0) -> []
+if("0".split(void 0, 0).length) {
+    var string_split = String.prototype.split;
+    String.prototype.split = function(separator, limit) {
+        if(separator === void 0 && limit === 0)return [];
+        return string_split.apply(this, arguments);
+    }
+}
+
+// ECMA-262, 3rd B.2.3
+// Note an ECMAScript standart, although ECMAScript 3rd Edition has a
+// non-normative section suggesting uniform semantics and it should be
+// normalized across all browsers
+// [bugfix, IE lt 9] IE < 9 substr() with negative value not working in IE
+if("".substr && "0b".substr(-1) !== "b") {
+    var string_substr = String.prototype.substr;
+    /**
+     *  Get the substring of a string
+     *  @param  {integer}  start   where to start the substring
+     *  @param  {integer}  length  how many characters to return
+     *  @return {string}
+     */
+    String.prototype.substr = function(start, length) {
+        return string_substr.call(
+            this,
+            start < 0 ? (start = this.length + start) < 0 ? 0 : start : start,
+            length
+        );
+    }
+}
+
 // ES5 15.5.4.20
 // http://es5.github.com/#x15.5.4.20
 var ws = "\x09\x0A\x0B\x0C\x0D\x20\xA0\u1680\u180E\u2000\u2001\u2002\u2003" +
@@ -1044,7 +1033,12 @@ if (!String.prototype.trim || ws.trim()) {
     var trimBeginRegexp = new RegExp("^" + ws + ws + "*"),
         trimEndRegexp = new RegExp(ws + ws + "*$");
     String.prototype.trim = function trim() {
-        return String(this).replace(trimBeginRegexp, "").replace(trimEndRegexp, "");
+        if (this === undefined || this === null) {
+            throw new TypeError("can't convert "+this+" to object");
+        }
+        return String(this)
+            .replace(trimBeginRegexp, "")
+            .replace(trimEndRegexp, "");
     };
 }
 
@@ -1056,27 +1050,57 @@ if (!String.prototype.trim || ws.trim()) {
 // ES5 9.4
 // http://es5.github.com/#x9.4
 // http://jsperf.com/to-integer
-var toInteger = function (n) {
+
+function toInteger(n) {
     n = +n;
-    if (n !== n) // isNaN
+    if (n !== n) { // isNaN
         n = 0;
-    else if (n !== 0 && n !== (1/0) && n !== -(1/0))
+    } else if (n !== 0 && n !== (1/0) && n !== -(1/0)) {
         n = (n > 0 || -1) * Math.floor(Math.abs(n));
+    }
     return n;
+}
+
+function isPrimitive(input) {
+    var type = typeof input;
+    return (
+        input === null ||
+        type === "undefined" ||
+        type === "boolean" ||
+        type === "number" ||
+        type === "string"
+    );
+}
+
+function toPrimitive(input) {
+    var val, valueOf, toString;
+    if (isPrimitive(input)) {
+        return input;
+    }
+    valueOf = input.valueOf;
+    if (typeof valueOf === "function") {
+        val = valueOf.call(input);
+        if (isPrimitive(val)) {
+            return val;
+        }
+    }
+    toString = input.toString;
+    if (typeof toString === "function") {
+        val = toString.call(input);
+        if (isPrimitive(val)) {
+            return val;
+        }
+    }
+    throw new TypeError();
+}
+
+// ES5 9.9
+// http://es5.github.com/#x9.9
+var toObject = function (o) {
+    if (o == null) { // this matches both null and undefined
+        throw new TypeError("can't convert "+o+" to object");
+    }
+    return Object(o);
 };
 
-var prepareString = "a"[0] != "a",
-    // ES5 9.9
-    // http://es5.github.com/#x9.9
-    toObject = function (o) {
-        if (o == null) { // this matches both null and undefined
-            throw new TypeError(); // TODO message
-        }
-        // If the implementation doesn't support by-index access of
-        // string characters (ex. IE < 7), split the string
-        if (prepareString && typeof o == "string" && o) {
-            return o.split("");
-        }
-        return Object(o);
-    };
 });

lib/ace/mode/_test/highlight_rules_test.js

@@ -112,7 +112,7 @@ function testMode(modeName, i) {
 }
 function testEqual(a) {
     var err;
-    if (a[0] !== a[1]) {
+    if (a[0] + "" !== a[1] + "") {
         console.log(a[0],a[1]);
         err = 1;
     }

lib/ace/mode/_test/text_markdown.txt

@@ -11,3 +11,9 @@ test: 6+ #s is not a valid header
 test: # followed be only space is not a valid header
 test: only space between #s is not a valid header
 #  #
+
+test links  [Cloud9 IDE](http://www.c9.io/) 
+* [demo](http://ajaxorg.github.com/ace/) 
+in lists
+
+in plain text <b>http://ace.ajaxorg.com<b>

lib/ace/mode/_test/tokens_c9search.json

@@ -10,55 +10,68 @@
 ],[
    "start",
   ["c9searchresults.constant.numeric","    1"],
-  ["c9searchresults.text",": var fs = require(\"fs\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var fs = require(\"fs\");"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t2"],
-  ["c9searchresults.text",": var argv = require('optimist').argv;"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var argv = require('optimist').argv;"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t3"],
-  ["c9searchresults.text",": var path = require(\"path\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var path = require(\"path\");"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t5"],
-  ["c9searchresults.text",": var clientExtensions = {};"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var clientExtensions = {};"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t6"],
-  ["c9searchresults.text",": var clientDirs = fs.readdirSync(__dirname + \"/../plugins-client\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var clientDirs = fs.readdirSync(__dirname + \"/../plugins-client\");"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t7"],
-  ["c9searchresults.text",": for (var i = 0; i < clientDirs.length; i++) {"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","for (var i = 0; i < clientDirs.length; i++) {"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t8"],
-  ["c9searchresults.text",":     var dir = clientDirs[i];"]
+  ["c9searchresults.text",":     "],
+  ["c9searchresults.text","var dir = clientDirs[i];"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t12"],
-  ["c9searchresults.text",":     var name = dir.split(\".\")[1];"]
+  ["c9searchresults.text",":     "],
+  ["c9searchresults.text","var name = dir.split(\".\")[1];"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t16"],
-  ["c9searchresults.text",": var projectDir = (argv.w && path.resolve(process.cwd(), argv.w)) || process.cwd();"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var projectDir = (argv.w && path.resolve(process.cwd(), argv.w)) || process.cwd();"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t17"],
-  ["c9searchresults.text",": var fsUrl = \"/workspace\";"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var fsUrl = \"/workspace\";"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t19"],
-  ["c9searchresults.text",": var port = argv.p || process.env.PORT || 3131;"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var port = argv.p || process.env.PORT || 3131;"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t20"],
-  ["c9searchresults.text",": var host = argv.l || \"localhost\";"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var host = argv.l || \"localhost\";"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t22"],
-  ["c9searchresults.text",": var config = {"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var config = {"]
 ],[
    "start"
 ],[
@@ -68,7 +81,8 @@
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t2"],
-  ["c9searchresults.text",": var config = require(\"./default\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var config = require(\"./default\");"]
 ],[
    "start"
 ],[
@@ -78,7 +92,8 @@
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t1"],
-  ["c9searchresults.text",": var config = require(\"./default\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var config = require(\"./default\");"]
 ],[
    "start"
 ],[

lib/ace/mode/_test/tokens_coffee.json

@@ -28,9 +28,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","args"],
+  ["variable.parameter","{args}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -40,9 +40,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","a1, a2"],
+  ["variable.parameter","{a1, a2}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -52,9 +52,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","@a1, a2"],
+  ["variable.parameter","{@a1, a2}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -62,11 +62,11 @@
   ["text","    "],
   ["entity.name.function","foo"],
   ["text"," "],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","args"],
+  ["variable.parameter","{args}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -76,22 +76,21 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","args"],
+  ["variable.parameter","{args}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
    "start",
   ["text","    "],
-  ["identifier","foo"],
+  ["entity.name.function","foo"],
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["constant.numeric","0"],
+  ["variable.parameter","{0abc}"],
-  ["identifier","abc"],
+  ["paren.rparen",")"],
-  ["paren.rparen","})"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -123,22 +122,27 @@
 ],[
    "start",
   ["text","    "],
-  ["identifier","foo"],
+  ["entity.name.function","foo"],
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["comment","#abc}) ->"]
+  ["variable.parameter","{#abc}"],
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["storage.type","->"]
 ],[
    "start",
   ["text","    "],
-  ["identifier","foo"],
+  ["entity.name.function","foo"],
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["identifier","abc"],
+  ["variable.parameter","{abc#}"],
-  ["comment","#}) ->"]
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["storage.type","->"]
 ],[
    "start",
   ["text","    "],
@@ -167,15 +171,13 @@
 ],[
    "start",
   ["text","    "],
-  ["identifier","foo"],
+  ["entity.name.function","foo"],
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["identifier","a"],
+  ["variable.parameter","{a{bc}"],
-  ["paren.lparen","{"],
+  ["paren.rparen",")"],
-  ["identifier","bc"],
-  ["paren.rparen","})"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -185,8 +187,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["paren.rparen","})"],
+  ["variable.parameter","{}"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -196,9 +199,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["text"," "],
+  ["variable.parameter","{ }"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -206,10 +209,11 @@
   ["text","    "],
   ["entity.name.function","foo"],
   ["text"," "],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["paren.rparen","})"],
+  ["variable.parameter","{}"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -294,7 +298,7 @@
   ["keyword.operator","="],
   ["text"," "],
   ["paren.lparen","("],
-  ["text"," "],
+  ["variable.parameter"," "],
   ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
@@ -303,10 +307,10 @@
   ["text","    "],
   ["entity.name.function","foo"],
   ["text"," "],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
   ["paren.lparen","("],
-  ["text"," "],
+  ["variable.parameter"," "],
   ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
@@ -345,7 +349,7 @@
   ["text","    "],
   ["entity.name.function","foo"],
   ["text"," "],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -358,7 +362,7 @@
   ["identifier","foo"],
   ["text"," "],
   ["identifier","bar"],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
   ["constant.numeric","1"],
   ["punctuation.operator",","],
@@ -422,7 +426,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["string","\"#{ 22 / 7 + {x: \""],
+  ["string.start","\""],
+  ["string","#{ 22 / 7 + {x: "],
+  ["string.end","\""],
   ["comment","#{a + b}\"} + 2}\""]
 ],[
    "qqdoc",
@@ -471,7 +477,7 @@
   ["keyword","this"],
   ["text"," "],
   ["keyword","isnt"],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
   ["string","`just "]
 ],[

lib/ace/mode/_test/tokens_curly.json

@@ -1,6 +1,9 @@
 [[
    "start",
-  ["text","tokenize Curly template{{test}}"]
+  ["text","tokenize Curly template"],
+  ["variable","{{"],
+  ["text","test"],
+  ["variable","}}"]
 ],[
    "start",
   ["text","tokenize embedded script"]

lib/ace/mode/_test/tokens_diff.json

@@ -1,6 +1,7 @@
 [[
    "start",
-  ["variable","diff --git"],
+  ["variable","diff"],
+  ["variable"," --git"],
   ["keyword"," a/lib/ace/edit_session.js"],
   ["variable"," b/lib/ace/edit_session.js"]
 ],[
@@ -52,7 +53,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -84,7 +86,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -191,7 +194,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -201,7 +205,8 @@
    "start"
 ],[
    "start",
-  ["variable","diff --git"],
+  ["variable","diff"],
+  ["variable"," --git"],
   ["keyword"," a/lib/ace/editor.js"],
   ["variable"," b/lib/ace/editor.js"]
 ],[
@@ -259,7 +264,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -291,7 +297,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -343,7 +350,8 @@
    "start"
 ],[
    "start",
-  ["variable","diff --git"],
+  ["variable","diff"],
+  ["variable"," --git"],
   ["keyword"," a/lib/ace/search_highlight.js"],
   ["variable"," b/lib/ace/search_highlight.js"]
 ],[

lib/ace/mode/_test/tokens_jade.json

@@ -28,7 +28,7 @@
    "start"
 ],[
    "comment_block",
-  ["punctuation.section.comment","//"]
+  ["punctuation.section.comment","// "]
 ],[
    "comment_block",
   ["comment.block.jade","  here it is. a block comment!"]
@@ -42,7 +42,7 @@
    "start"
 ],[
    "comment_block",
-  ["punctuation.section.comment","     //     "]
+  ["punctuation.section.comment","     // "]
 ],[
    "comment_block",
   ["comment.block.jade","        a far spaced"]

lib/ace/mode/_test/tokens_javascript.json

@@ -262,7 +262,7 @@
   ["keyword.operator","="],
   ["text"," "],
   ["string.regexp","/p"],
-  ["constant.language.escape","|"],
+  ["constant.language.delimiter","|"],
   ["string.regexp","p/"],
   ["text"," "],
   ["comment","// ends here"]
@@ -300,15 +300,17 @@
   ["string.regexp","rr--rr"],
   ["constant.language.escape","$^(?:"],
   ["string.regexp","d"],
-  ["constant.language.escape","|"],
+  ["constant.language.delimiter","|"],
   ["string.regexp","s"],
   ["constant.language.escape",")(?="],
   ["string.regexp","a"],
-  ["constant.language.escape","|)(?!"],
+  ["constant.language.delimiter","|"],
+  ["constant.language.escape",")(?!"],
   ["string.regexp","y"],
-  ["constant.language.escape",")[]|"],
+  ["constant.language.escape",")[]"],
+  ["constant.language.delimiter","|"],
   ["invalid","$?"],
-  ["constant.language.escape","|"],
+  ["constant.language.delimiter","|"],
   ["invalid","^*"],
   ["string.regexp","/"],
   ["text"," "],
@@ -509,7 +511,8 @@
    "start",
   ["text","\t"],
   ["string","'g'"],
-  ["text",": "],
+  ["text",":"],
+  ["text"," "],
   ["storage.type","function"],
   ["paren.lparen","("],
   ["variable.parameter","t"],

lib/ace/mode/_test/tokens_liquid.json

@@ -86,7 +86,16 @@
   ["meta.tag.r",">"]
 ],[
    "start",
-  ["text","        Only {{ product.price | format_as_money }}"]
+  ["text","        Only "],
+  ["variable","{{"],
+  ["text"," "],
+  ["identifier","product"],
+  ["text","."],
+  ["identifier","price"],
+  ["text"," | "],
+  ["identifier","format_as_money"],
+  ["text"," "],
+  ["variable","}}"]
 ],[
    "start"
 ],[
@@ -156,7 +165,15 @@
   ["meta.tag","<"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"],
-  ["text"," The word \"tobi\" in uppercase: {{ 'tobi' | upcase }} "],
+  ["text"," The word \"tobi\" in uppercase: "],
+  ["variable","{{"],
+  ["text"," "],
+  ["string","'tobi'"],
+  ["text"," | "],
+  ["support.function","upcase"],
+  ["text"," "],
+  ["variable","}}"],
+  ["text"," "],
   ["meta.tag","</"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"]
@@ -165,7 +182,15 @@
   ["meta.tag","<"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"],
-  ["text","The word \"tobi\" has {{ 'tobi' | size }} letters! "],
+  ["text","The word \"tobi\" has "],
+  ["variable","{{"],
+  ["text"," "],
+  ["string","'tobi'"],
+  ["text"," | "],
+  ["support.function","size"],
+  ["text"," "],
+  ["variable","}}"],
+  ["text"," letters! "],
   ["meta.tag","</"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"]
@@ -174,7 +199,19 @@
   ["meta.tag","<"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"],
-  ["text","Change \"Hello world\" to \"Hi world\": {{ 'Hello world' | replace: 'Hello', 'Hi' }} "],
+  ["text","Change \"Hello world\" to \"Hi world\": "],
+  ["variable","{{"],
+  ["text"," "],
+  ["string","'Hello world'"],
+  ["text"," | "],
+  ["support.function","replace"],
+  ["text",": "],
+  ["string","'Hello'"],
+  ["text",", "],
+  ["string","'Hi'"],
+  ["text"," "],
+  ["variable","}}"],
+  ["text"," "],
   ["meta.tag","</"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"]
@@ -183,7 +220,17 @@
   ["meta.tag","<"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"],
-  ["text","The date today is {{ 'now' | date: \"%Y %b %d\" }} "],
+  ["text","The date today is "],
+  ["variable","{{"],
+  ["text"," "],
+  ["string","'now'"],
+  ["text"," | "],
+  ["support.function","date"],
+  ["text",": "],
+  ["string","\"%Y %b %d\""],
+  ["text"," "],
+  ["variable","}}"],
+  ["text"," "],
   ["meta.tag","</"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"]
@@ -311,7 +358,14 @@
   ["identifier","link_to_vendor"],
   ["text"," "],
   ["variable","}}"],
-  ["text"," / {{ product.title }}"]
+  ["text"," / "],
+  ["variable","{{"],
+  ["text"," "],
+  ["identifier","product"],
+  ["text","."],
+  ["identifier","title"],
+  ["text"," "],
+  ["variable","}}"]
 ],[
    "start",
   ["text","    "],
@@ -445,7 +499,14 @@
   ["variable","%}"]
 ],[
    "start",
-  ["text","      First column: {{ item.variable }}"]
+  ["text","      First column: "],
+  ["variable","{{"],
+  ["text"," "],
+  ["identifier","item"],
+  ["text","."],
+  ["identifier","variable"],
+  ["text"," "],
+  ["variable","}}"]
 ],[
    "start",
   ["text","    "],
@@ -456,7 +517,14 @@
   ["variable","%}"]
 ],[
    "start",
-  ["text","      Different column: {{ item.variable }}"]
+  ["text","      Different column: "],
+  ["variable","{{"],
+  ["text"," "],
+  ["identifier","item"],
+  ["text","."],
+  ["identifier","variable"],
+  ["text"," "],
+  ["variable","}}"]
 ],[
    "start",
   ["text","    "],

lib/ace/mode/_test/tokens_lua.json

@@ -1,11 +1,11 @@
 [[
-   "qcomment",
+   ["bracketedComment",4,"start"],
   ["comment","--[[--"]
 ],[
-   "qcomment",
+   ["bracketedComment",4,"start"],
   ["comment","num_args takes in 5.1 byte code and extracts the number of arguments"]
 ],[
-   "qcomment",
+   ["bracketedComment",4,"start"],
   ["comment","from its function header."]
 ],[
    "start",
@@ -280,16 +280,16 @@
 ],[
    "start"
 ],[
-   "qstring3",
+   ["bracketedString",5,"start"],
   ["support.function","print"],
   ["paren.lparen","("],
-  ["string","[===["]
+  ["comment","[===["]
 ],[
-   "qstring3",
+   ["bracketedString",5,"start"],
-  ["string","    blah blah %s, (%d %d)"]
+  ["comment","    blah blah %s, (%d %d)"]
 ],[
    "start",
-  ["string","]===]"],
+  ["comment","]===]"],
   ["keyword.operator","%"],
   ["paren.lparen","{"],
   ["string","\"blah\""],
@@ -301,10 +301,10 @@
 ],[
    "start"
 ],[
-   "qcomment1",
+   ["bracketedComment",5,"start"],
   ["comment","--[=[--"]
 ],[
-   "qcomment1",
+   ["bracketedComment",5,"start"],
   ["comment","table.maxn is deprecated, use # instead."]
 ],[
    "start",

lib/ace/mode/_test/tokens_luapage.json

@@ -21,15 +21,15 @@
   ["meta.tag.tag-name","html"],
   ["meta.tag.r",">"]
 ],[
-   "lua-qcomment",
+   ["lua-bracketedComment",4,"lua-start"],
   ["keyword","<%"],
   ["text"," "],
   ["comment","--[[--"]
 ],[
-   "lua-qcomment",
+   ["lua-bracketedComment",4,"lua-start"],
   ["comment","    index.lp from the Kepler Project's LuaDoc HTML doclet."]
 ],[
-   "lua-qcomment",
+   ["lua-bracketedComment",4,"lua-start"],
   ["comment","    http://keplerproject.github.com/luadoc/"]
 ],[
    "start",

lib/ace/mode/_test/tokens_markdown.json

@@ -37,6 +37,39 @@
 ],[
    "start",
   ["text","#  #"]
+],[
+   "allowBlock"
+],[
+   "start",
+  ["text","test links  "],
+  ["text","["],
+  ["string","Cloud9 IDE"],
+  ["text","]("],
+  ["markup.underline","http://www.c9.io/"],
+  ["text",")"],
+  ["text"," "]
+],[
+   "listblock",
+  ["markup.list","* "],
+  ["text","["],
+  ["string","demo"],
+  ["text","]("],
+  ["markup.underline","http://ajaxorg.github.com/ace/"],
+  ["text",")"],
+  ["markup.list"," "]
+],[
+   "listblock",
+  ["markup.list","in lists"]
 ],[
    "start"
+],[
+   "start",
+  ["text","in plain text "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","b"],
+  ["meta.tag.r",">"],
+  ["text","http://ace.ajaxorg.com"],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","b"],
+  ["meta.tag.r",">"]
 ]]

lib/ace/mode/_test/tokens_scheme.json

@@ -0,0 +1,216 @@
+[[
+   "start",
+  ["text","("],
+  ["storage.type.function-type.scheme","define"],
+  ["text"," "],
+  ["text","("],
+  ["identifier","prompt-for-cd"],
+  ["text",")"]
+],[
+   "start",
+  ["text","   "],
+  ["string","\"Prompts"]
+],[
+   "start",
+  ["text","    "],
+  ["identifier","for"],
+  ["text"," "],
+  ["identifier","CD"],
+  ["text","\""]
+],[
+   "start",
+  ["text","   ("],
+  ["identifier","prompt-read"],
+  ["text"," "],
+  ["string","\"Title\""],
+  ["text"," "],
+  ["constant.numeric","1.53"],
+  ["text"," "],
+  ["constant.numeric","1"],
+  ["text"," "],
+  ["constant.numeric","2"],
+  ["text","/"],
+  ["constant.numeric","4"],
+  ["text"," "],
+  ["constant.numeric","1.7"],
+  ["text"," "],
+  ["constant.numeric","1.7e0"],
+  ["text"," "],
+  ["constant.numeric","2.9E-4"],
+  ["text"," "],
+  ["constant.numeric","+42"],
+  ["text"," "],
+  ["constant.numeric","-7"],
+  ["text"," "],
+  ["constant.numeric","#b001"],
+  ["text"," "],
+  ["constant.numeric","#b001"],
+  ["text","/"],
+  ["constant.numeric","100"],
+  ["text"," "],
+  ["constant.numeric","#o777"],
+  ["text"," "],
+  ["constant.numeric","#O777"],
+  ["text"," "],
+  ["constant.numeric","#xabc55"],
+  ["text"," "],
+  ["identifier","#c"],
+  ["text","("],
+  ["constant.numeric","0"],
+  ["text"," "],
+  ["constant.numeric","-5.6"],
+  ["text","))"]
+],[
+   "start",
+  ["text","   ("],
+  ["identifier","prompt-read"],
+  ["text"," "],
+  ["string","\"Artist\""],
+  ["text",")"]
+],[
+   "start",
+  ["text","   ("],
+  ["keyword.operator","or"],
+  ["text"," ("],
+  ["identifier","parse-integer"],
+  ["text"," ("],
+  ["identifier","prompt-read"],
+  ["text"," "],
+  ["string","\"Rating\""],
+  ["text",") "],
+  ["punctuation.definition.constant.character.scheme","#:junk-allowed"],
+  ["text"," "],
+  ["constant.language","#t"],
+  ["text",") "],
+  ["constant.numeric","0"],
+  ["text",")"]
+],[
+   "start",
+  ["text","  ("],
+  ["keyword.control","if"],
+  ["text"," "],
+  ["identifier","x"],
+  ["text"," ("],
+  ["support.function","format"],
+  ["text"," "],
+  ["constant.language","#t"],
+  ["text"," "],
+  ["string","\"yes\""],
+  ["text",") ("],
+  ["support.function","format"],
+  ["text"," "],
+  ["constant.language","#f"],
+  ["text"," "],
+  ["string","\"no\""],
+  ["text",") "],
+  ["comment",";and here comment"]
+],[
+   "start",
+  ["text","  ) "]
+],[
+   "start",
+  ["text","  "],
+  ["comment",";; second line comment"]
+],[
+   "start",
+  ["text","  '(+ "],
+  ["constant.numeric","1"],
+  ["text"," "],
+  ["constant.numeric","2"],
+  ["text",")"]
+],[
+   "start",
+  ["text","  ("],
+  ["identifier","position-if-not"],
+  ["text"," "],
+  ["identifier","char-set"],
+  ["text",":"],
+  ["identifier","whitespace"],
+  ["text"," "],
+  ["identifier","line"],
+  ["text"," "],
+  ["punctuation.definition.constant.character.scheme","#:start"],
+  ["text"," "],
+  ["identifier","beg"],
+  ["text","))"]
+],[
+   "start",
+  ["text","  ("],
+  ["support.function","quote"],
+  ["text"," ("],
+  ["identifier","privet"],
+  ["text"," "],
+  ["constant.numeric","1"],
+  ["text"," "],
+  ["constant.numeric","2"],
+  ["text"," "],
+  ["constant.numeric","3"],
+  ["text","))"]
+],[
+   "start",
+  ["text","  '("],
+  ["identifier","hello"],
+  ["text"," "],
+  ["identifier","world"],
+  ["text",")"]
+],[
+   "start",
+  ["text","  (* "],
+  ["constant.numeric","5"],
+  ["text"," "],
+  ["constant.numeric","7"],
+  ["text",")"]
+],[
+   "start",
+  ["text","  ("],
+  ["constant.numeric","1"],
+  ["text"," "],
+  ["constant.numeric","2"],
+  ["text"," "],
+  ["constant.numeric","34"],
+  ["text"," "],
+  ["constant.numeric","5"],
+  ["text",")"]
+],[
+   "start",
+  ["text","  ("],
+  ["punctuation.definition.constant.character.scheme","#:use"],
+  ["text"," "],
+  ["string","\"aaaa\""],
+  ["text",")"]
+],[
+   "start",
+  ["text","  ("],
+  ["keyword.control","let"],
+  ["text"," (("],
+  ["identifier","x"],
+  ["text"," "],
+  ["constant.numeric","10"],
+  ["text",") ("],
+  ["identifier","y"],
+  ["text"," "],
+  ["constant.numeric","20"],
+  ["text","))"]
+],[
+   "start",
+  ["text","    ("],
+  ["identifier","display"],
+  ["text"," (+ "],
+  ["identifier","x"],
+  ["text"," "],
+  ["identifier","y"],
+  ["text","))"]
+],[
+   "start",
+  ["text","  ) "]
+],[
+   "start"
+],[
+   "start",
+  ["text","  "],
+  ["string","\"asdad"],
+  ["constant.character.escape.scheme","\\0"],
+  ["string","eqweqe\""]
+],[
+   "start"
+]]

lib/ace/mode/_test/tokens_stylus.json

@@ -32,7 +32,8 @@
 ],[
    "start",
   ["entity.name.function.stylus","asdasdasdad"],
-  ["text","(df, ad"],
+  ["text","("],
+  ["text","df, ad"],
   ["keyword.operator.stylus","="],
   ["constant.numeric","23"],
   ["text",")"]

lib/ace/mode/_test/tokens_tcl.json

@@ -30,8 +30,7 @@
   ["identifier","distmap"],
   ["paren.rparen","}"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
@@ -43,8 +42,7 @@
   ["text"," "],
   ["identifier","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$vertex"],
-  ["variable.instance","vertex"],
   ["text"," "],
   ["identifier","Inf"]
 ],[
@@ -56,14 +54,14 @@
   ["text"," "],
   ["identifier","path"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$vertex"],
-  ["variable.instance","vertex"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["text","}"]
+  ["paren.rparen","}"]
 ],[
    "commandItem",
-  ["text","    }"]
+  ["text","    "],
+  ["paren.rparen","}"]
 ],[
    "start",
   ["text","    "],
@@ -73,8 +71,7 @@
   ["text"," "],
   ["identifier","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$origin"],
-  ["variable.instance","origin"],
   ["text"," 0"]
 ],[
    "start",
@@ -85,14 +82,12 @@
   ["text"," "],
   ["identifier","path"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$origin"],
-  ["variable.instance","origin"],
   ["text"," "],
   ["paren.lparen","["],
   ["keyword","list"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$origin"],
-  ["variable.instance","origin"],
   ["paren.rparen","]"]
 ],[
    "commandItem",
@@ -108,8 +103,7 @@
   ["text"," "],
   ["identifier","size"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["paren.rparen","]}"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -138,8 +132,7 @@
   ["support.function","-"],
   ["paren.rparen","}"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
@@ -148,8 +141,7 @@
   ["keyword","if"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["variable.instancce","$"],
+  ["variable.instance","$d"],
-  ["variable.instance","d"],
   ["text"," "],
   ["support.function",">"],
   ["text"," "],
@@ -163,11 +155,9 @@
   ["text"," "],
   ["identifier","get"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dist"],
-  ["variable.instance","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$uu"],
-  ["variable.instance","uu"],
   ["paren.rparen","]]}"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -178,8 +168,7 @@
   ["text"," "],
   ["identifier","u"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$uu"]
-  ["variable.instance","uu"]
 ],[
    "start",
   ["text","\t\t"],
@@ -187,14 +176,15 @@
   ["text"," "],
   ["identifier","d"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dd"]
-  ["variable.instance","dd"]
 ],[
    "commandItem",
-  ["text","\t    }"]
+  ["text","\t    "],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
-  ["text","\t}"]
+  ["text","\t"],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
   ["text"," "]
@@ -208,8 +198,7 @@
   ["keyword","if"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["variable.instancce","$"],
+  ["variable.instance","$d"],
-  ["variable.instance","d"],
   ["text"," "],
   ["support.function","=="],
   ["text"," "],
@@ -245,11 +234,9 @@
   ["text"," "],
   ["identifier","get"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$u"],
-  ["variable.instance","u"],
   ["paren.rparen","]"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -264,11 +251,9 @@
   ["text"," "],
   ["identifier","exists"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["paren.rparen","]}"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -283,13 +268,11 @@
   ["keyword","expr"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["variable.instancce","$"],
+  ["variable.instance","$d"],
-  ["variable.instance","d"],
   ["text"," "],
   ["support.function","+"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dd"],
-  ["variable.instance","dd"],
   ["paren.rparen","}]"]
 ],[
    "commandItem",
@@ -297,8 +280,7 @@
   ["keyword","if"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["variable.instancce","$"],
+  ["variable.instance","$alt"],
-  ["variable.instance","alt"],
   ["text"," "],
   ["support.function","<"],
   ["text"," "],
@@ -307,11 +289,9 @@
   ["text"," "],
   ["identifier","get"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dist"],
-  ["variable.instance","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["paren.rparen","]}"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -324,11 +304,9 @@
   ["text"," "],
   ["identifier","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$alt"]
-  ["variable.instance","alt"]
 ],[
    "start",
   ["text","\t\t    "],
@@ -338,8 +316,7 @@
   ["text"," "],
   ["identifier","path"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["text"," "],
   ["paren.lparen","["],
   ["keyword","list"],
@@ -350,25 +327,25 @@
   ["text"," "],
   ["identifier","get"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$path"],
-  ["variable.instance","path"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$u"],
-  ["variable.instance","u"],
   ["paren.rparen","]"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["paren.rparen","]"]
 ],[
    "commandItem",
-  ["text","\t\t}"]
+  ["text","\t\t"],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
-  ["text","\t    }"]
+  ["text","\t    "],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
-  ["text","\t}"]
+  ["text","\t"],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
   ["text"," "]
@@ -385,11 +362,11 @@
   ["text"," "],
   ["identifier","graph"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$u"]
-  ["variable.instance","u"]
 ],[
    "commandItem",
-  ["text","    }"]
+  ["text","    "],
+  ["paren.rparen","}"]
 ],[
    "start",
   ["text","    "],
@@ -398,13 +375,11 @@
   ["paren.lparen","["],
   ["keyword","list"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dist"],
-  ["variable.instance","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$path"],
-  ["variable.instance","path"],
   ["paren.rparen","]"]
 ],[
    "commandItem",
-  ["text","}"]
+  ["paren.rparen","}"]
 ]]

lib/ace/mode/_test/tokens_typescript.json

@@ -128,7 +128,8 @@
   ["paren.lparen","("],
   ["entity.name.function.ts","greeter.greet"],
   ["paren.lparen","("],
-  ["paren.rparen","))"]
+  ["paren.rparen",")"],
+  ["paren.rparen",")"]
 ],[
    "start",
   ["paren.rparen","}"]

lib/ace/mode/_test/tokens_xquery.json

@@ -20,12 +20,14 @@
 ],[
    "start",
   ["keyword","return"],
-  ["text"," <"],
+  ["text"," "],
+  ["text","<"],
   ["meta.tag","results"],
   ["text",">"]
 ],[
    "start",
-  ["text","  <"],
+  ["text","  "],
+  ["text","<"],
   ["meta.tag","message"],
   ["text",">"],
   ["lparen","{"],

lib/ace/mode/abap_highlight_rules.js

@@ -113,7 +113,7 @@ var AbapHighlightRules = function() {
             {token : "variable.parameter", regex : /sy|pa?\d\d\d\d\|t\d\d\d\.|innnn/}, 
             {token : "keyword", regex : compoundKeywords}, 
             {token : "variable.parameter", regex : /\w+-\w+(?:-\w+)*/}, 
-            {token : keywordMapper, regex : "\\w+\\b"}, 
+            {token : keywordMapper, regex : "\\b\\w+\\b"}, 
         ],
         "qstring" : [
             {token : "constant.language.escape",   regex : "''"},

lib/ace/mode/coffee_highlight_rules.js

@@ -38,10 +38,6 @@ define(function(require, exports, module) {
 
     function CoffeeHighlightRules() {
         var identifier = "[$A-Za-z_\\x7f-\\uffff][$\\w\\x7f-\\uffff]*";
-        var stringfill = {
-            token : "string",
-            regex : ".+"
-        };
 
         var keywords = (
             "this|throw|then|try|typeof|super|switch|return|break|by|continue|" +
@@ -86,50 +82,55 @@ define(function(require, exports, module) {
             "variable.language": variableLanguage
         }, "identifier");
 
-        var functionRules = {
+        var functionRule = {
-            "({args})->": {
+            token: ["paren.lparen", "variable.parameter", "paren.rparen", "text", "storage.type"],
-                token: ["paren.lparen", "text", "paren.lparen", "text", "variable.parameter", "text", "paren.rparen", "text", "paren.rparen", "text", "storage.type"],
+            regex: /(?:(\()((?:"[^")]*?"|'[^')]*?'|\/[^\/)]*?\/|[^()\"'\/])*?)(\))(\s*))?([\-=]>)/.source
-                regex: "(\\()(\\s*)(\\{)(\\s*)([$@A-Za-z_\\x7f-\\uffff][$@\\w\\s,\\x7f-\\uffff]*)(\\s*)(\\})(\\s*)(\\))(\\s*)([\\-=]>)"
-            },
-            "({})->": {
-                token: ["paren.lparen", "text", "paren.lparen", "text", "paren.rparen", "text", "paren.rparen", "text", "storage.type"],
-                regex: "(\\()(\\s*)(\\{)(\\s*)(\\})(\\s*)(\\))(\\s*)([\\-=]>)"
-            },
-            "(args)->": {
-                token: ["paren.lparen", "text", "variable.parameter", "text", "paren.rparen", "text", "storage.type"],
-                regex: "(\\()(\\s*)([$@A-Za-z_\\x7f-\\uffff][\\s\\x21-\\uffff]*)(\\s*)(\\))(\\s*)([\\-=]>)"
-            },
-            "()->": {
-                token: ["paren.lparen", "text", "paren.rparen", "text", "storage.type"],
-                regex: "(\\()(\\s*)(\\))(\\s*)([\\-=]>)"
-            }
         };
 
+        var stringEscape = /\\(?:x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)/;
+
         this.$rules = {
             start : [
                 {
                     token : "constant.numeric",
                     regex : "(?:0x[\\da-fA-F]+|(?:\\d+(?:\\.\\d+)?|\\.\\d+)(?:[eE][+-]?\\d+)?)"
                 }, {
-                    token : "string",
+                    stateName: "qdoc",
-                    regex : "'''",
+                    token : "string", regex : "'''", next : [
-                    next : "qdoc"
+                        {token : "string", regex : "'''", next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"},
+                    ]
                 }, {
+                    stateName: "qqdoc",
                     token : "string",
                     regex : '"""',
-                    next : "qqdoc"
+                    next : [
+                        {token : "string", regex : '"""', next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"}
+                    ]
                 }, {
-                    token : "string",
+                    stateName: "qstring",
-                    regex : "'",
+                    token : "string", regex : "'", next : [
-                    next : "qstring"
+                        {token : "string", regex : "'", next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"},
+                    ]
                 }, {
-                    token : "string",
+                    stateName: "qqstring",
-                    regex : '"',
+                    token : "string.start", regex : '"', next : [
-                    next : "qqstring"
+                        {token : "string.end", regex : '"', next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"},
+                    ]
                 }, {
-                    token : "string",
+                    stateName: "js",
-                    regex : "`",
+                    token : "string", regex : "`", next : [
-                    next : "js"
+                        {token : "string", regex : "`", next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"},
+                    ]
                 }, {
                     token : "string.regex",
                     regex : "///",
@@ -145,94 +146,23 @@ define(function(require, exports, module) {
                     token : "comment",
                     regex : "#.*"
                 }, {
-                    token : [
+                    token : ["punctuation.operator", "text", "identifier"],
-                        "punctuation.operator", "identifier"
+                    regex : "(\\.)(\\s*)(" + illegal + ")"
-                    ],
-                    regex : "(\\.)(" + illegal + ")"
                 }, {
                     token : "punctuation.operator",
                     regex : "\\."
                 }, {
                     //class A extends B
-                    token : [
+                    token : ["keyword", "text", "language.support.class",
-                        "keyword", "text", "language.support.class", "text", "keyword", "text", "language.support.class"
+                     "text", "keyword", "text", "language.support.class"],
-                    ],
+                    regex : "(class)(\\s+)(" + identifier + ")(?:(\\s+)(extends)(\\s+)(" + identifier + "))?"
-                    regex : "(class)(\\s+)(" + identifier + ")(\\s+)(extends)(\\s+)(" + identifier + ")"
                 }, {
-                    //class A
+                    //play = (...) ->
-                    token : [
+                    token : ["entity.name.function", "text", "keyword.operator", "text"].concat(functionRule.token),
-                        "keyword", "text", "language.support.class"
+                    regex : "(" + identifier + ")(\\s*)([=:])(\\s*)" + functionRule.regex
-                    ],
-                    regex : "(class)(\\s+)(" + identifier + ")"
-                }, {
-                    //play = ({args}) ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text"
-                    ].concat(functionRules["({args})->"].token),
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)" + functionRules["({args})->"].regex
-                }, {
-                    //play : ({args}) ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text"
-                    ].concat(functionRules["({args})->"].token),
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)" + functionRules["({args})->"].regex
-                }, {
-                    //play = ({}) ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text"
-                    ].concat(functionRules["({})->"].token),
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)" + functionRules["({})->"].regex
-                }, {
-                    //play : ({}) ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text"
-                    ].concat(functionRules["({})->"].token),
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)" + functionRules["({})->"].regex
-                }, {
-                    //play = (args) ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text"
-                    ].concat(functionRules["(args)->"].token),
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)" + functionRules["(args)->"].regex
-                }, {
-                    //play : (args) ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text"
-                    ].concat(functionRules["(args)->"].token),
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)" + functionRules["(args)->"].regex
-                }, {
-                    //play = () ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text"
-                    ].concat(functionRules["()->"].token),
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)" + functionRules["()->"].regex
-                }, {
-                    //play : () ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text"
-                    ].concat(functionRules["()->"].token),
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)" + functionRules["()->"].regex
-                }, {
-                    //play = ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text", "storage.type"
-                    ],
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)([\\-=]>)"
-                }, {
-                    //play : ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text", "storage.type"
-                    ],
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)([\\-=]>)"
                 }, 
-                functionRules["({args})->"],
+                functionRule, 
-                functionRules["({})->"],
+                {
-                functionRules["(args)->"],
-                functionRules["()->"]
-                , {
-                    token : "identifier",
-                    regex : "(?:(?:\\.|::)\\s*)" + identifier
-                }, {
                     token : "variable",
                     regex : "@(?:" + identifier + ")?"
                 }, {
@@ -240,7 +170,7 @@ define(function(require, exports, module) {
                     regex : identifier
                 }, {
                     token : "punctuation.operator",
-                    regex : "\\?|\\:|\\,|\\."
+                    regex : "\\,|\\."
                 }, {
                     token : "storage.type",
                     regex : "[\\-=]>"
@@ -258,35 +188,6 @@ define(function(require, exports, module) {
                     regex : "\\s+"
                 }],
 
-            qdoc : [{
-                token : "string",
-                regex : ".*?'''",
-                next : "start"
-            }, stringfill],
-
-            qqdoc : [{
-                token : "string",
-                regex : '.*?"""',
-                next : "start"
-            }, stringfill],
-
-            qstring : [{
-                token : "string",
-                regex : "[^\\\\']*(?:\\\\.[^\\\\']*)*'",
-                next : "start"
-            }, stringfill],
-
-            qqstring : [{
-                token : "string",
-                regex : '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
-                next : "start"
-            }, stringfill],
-
-            js : [{
-                token : "string",
-                regex : "[^\\\\`]*(?:\\\\.[^\\\\`]*)*`",
-                next : "start"
-            }, stringfill],
 
             heregex : [{
                 token : "string.regex",
@@ -302,13 +203,13 @@ define(function(require, exports, module) {
 
             comment : [{
                 token : "comment",
-                regex : '.*?###',
+                regex : '###',
                 next : "start"
             }, {
-                token : "comment",
+                defaultToken : "comment",
-                regex : ".+"
             }]
         };
+        this.normalizeRules();
     }
 
     exports.CoffeeHighlightRules = CoffeeHighlightRules;

lib/ace/mode/coldfusion_highlight_rules.js

@@ -66,24 +66,12 @@ var ColdfusionHighlightRules = function() {
             token : "meta.tag", // opening tag
             regex : "<\\/?",
             next : "tag"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : "[^<]+"
         } ],
     
         cdata : [ {
             token : "text",
             regex : "\\]\\]>",
             next : "start"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : ".+"
         } ],
 
         comment : [ {
@@ -91,8 +79,7 @@ var ColdfusionHighlightRules = function() {
             regex : ".*?-->",
             next : "start"
         }, {
-            token : "comment",
+            defaultToken : "comment"
-            regex : ".+"
         } ]
     };
     

lib/ace/mode/css_highlight_rules_test.js

@@ -1,92 +0,0 @@
-/* ***** BEGIN LICENSE BLOCK *****
- * Distributed under the BSD license:
- *
- * Copyright (c) 2010, Ajax.org B.V.
- * All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Ajax.org B.V. nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * ***** END LICENSE BLOCK ***** */
-
-if (typeof process !== "undefined") {
-    require("amd-loader");
-}
-
-define(function(require, exports, module) {
-"use strict";
-
-var CssMode = require("./css").Mode;
-var assert = require("../test/assertions");
-
-module.exports = {
-    
-    name: "CSS Tokenizer",
-    
-    setUp : function() {
-        this.tokenizer = new CssMode().getTokenizer();
-    },
-
-    "test: tokenize pixel number" : function() {
-        var line = "-12px";
-        var tokens = this.tokenizer.getLineTokens(line, "ruleset").tokens;
-
-        assert.equal(2, tokens.length);
-        assert.equal("constant.numeric", tokens[0].type);
-    },
-
-    "test: tokenize hex3 color" : function() {
-        var tokens = this.tokenizer.getLineTokens("#abc", "ruleset").tokens;
-
-        assert.equal(1, tokens.length);
-        assert.equal("constant.numeric", tokens[0].type);
-    },
-
-    "test: tokenize hex6 color" : function() {
-        var tokens = this.tokenizer.getLineTokens("#abc012", "ruleset").tokens;
-
-        assert.equal(1, tokens.length);
-        assert.equal("constant.numeric", tokens[0].type);
-    },
-
-    "test: tokenize parens" : function() {
-        var tokens = this.tokenizer.getLineTokens("{()}", "start").tokens;
-
-        assert.equal(3, tokens.length);
-        assert.equal("paren.lparen", tokens[0].type);
-        assert.equal("text", tokens[1].type);
-        assert.equal("paren.rparen", tokens[2].type);
-    },
-    
-    "test for last rule in ruleset to catch capturing group bugs" : function() {
-        var tokens = this.tokenizer.getLineTokens("top", "ruleset").tokens;
-        
-        assert.equal(1, tokens.length);
-        assert.equal("support.type", tokens[0].type);
-    }
-};
-
-});
-
-if (typeof module !== "undefined" && module === require.main) {
-    require("asyncjs").test.testcase(module.exports).exec()
-}

lib/ace/mode/dart_highlight_rules.js

@@ -38,101 +38,101 @@ var DartHighlightRules = function() {
         {
     "start": [
         {
-            "token" : "comment",
+            token : "comment",
-            "regex" : /\/\/.*$/
+            regex : /\/\/.*$/
         },
         {
-            "token" : "comment", // multi line comment
+            token : "comment", // multi line comment
-            "regex" : /\/\*/,
+            regex : /\/\*/,
-            "next" : "comment"
+            next : "comment"
         },
         {
-            "token": ["meta.preprocessor.script.dart"],
+            token: ["meta.preprocessor.script.dart"],
-            "regex": "^(#!.*)$"
+            regex: "^(#!.*)$"
         },
         {
-            "token": [ "keyword.other.import.dart", "meta.declaration.dart"],
+            token: "keyword.other.import.dart",
-            "regex": "#(?:\\b)(?:library|import|source|resource)(?:\\b)"
+            regex: "#(?:\\b)(?:library|import|source|resource)(?:\\b)"
         },
         {
-            "token" : ["keyword.other.import.dart", "text"],
+            token : ["keyword.other.import.dart", "text"],
-            "regex" : "(?:\\b)(prefix)(\\s*:)"
+            regex : "(?:\\b)(prefix)(\\s*:)"
         },
         {
-            "regex": "\\bas\\b",
+            regex: "\\bas\\b",
-            "token": "keyword.cast.dart"
+            token: "keyword.cast.dart"
         },
         {
-            "regex": "\\?|:",
+            regex: "\\?|:",
-            "token": "keyword.control.ternary.dart"
+            token: "keyword.control.ternary.dart"
         },
         {
-            "regex": "(?:\\b)(is\\!?)(?:\\b)",
+            regex: "(?:\\b)(is\\!?)(?:\\b)",
-            "token": ["keyword.operator.dart"]
+            token: ["keyword.operator.dart"]
         },
         {
-            "regex": "(<<|>>>?|~|\\^|\\||&)",
+            regex: "(<<|>>>?|~|\\^|\\||&)",
-            "token": ["keyword.operator.bitwise.dart"]
+            token: ["keyword.operator.bitwise.dart"]
         },
         {
-            "regex": "((?:&|\\^|\\||<<|>>>?)=)",
+            regex: "((?:&|\\^|\\||<<|>>>?)=)",
-            "token": ["keyword.operator.assignment.bitwise.dart"]
+            token: ["keyword.operator.assignment.bitwise.dart"]
         },
         {
-            "regex": "(===?|!==?|<=?|>=?)",
+            regex: "(===?|!==?|<=?|>=?)",
-            "token": ["keyword.operator.comparison.dart"]
+            token: ["keyword.operator.comparison.dart"]
         },
         {
-            "regex": "((?:[+*/%-]|\\~)=)",
+            regex: "((?:[+*/%-]|\\~)=)",
-            "token": ["keyword.operator.assignment.arithmetic.dart"]
+            token: ["keyword.operator.assignment.arithmetic.dart"]
         },
         {
-            "regex": "=",
+            regex: "=",
-            "token": "keyword.operator.assignment.dart"
+            token: "keyword.operator.assignment.dart"
         },
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "'''",
+            regex : "'''",
-            "next" : "qdoc"
+            next : "qdoc"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"""',
+            regex : '"""',
-            "next" : "qqdoc"
+            next : "qqdoc"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "'",
+            regex : "'",
-            "next" : "qstring"
+            next : "qstring"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"',
+            regex : '"',
-            "next" : "qqstring"
+            next : "qqstring"
         }, 
         {
-            "regex": "(\\-\\-|\\+\\+)",
+            regex: "(\\-\\-|\\+\\+)",
-            "token": ["keyword.operator.increment-decrement.dart"]
+            token: ["keyword.operator.increment-decrement.dart"]
         },
         {
-            "regex": "(\\-|\\+|\\*|\\/|\\~\\/|%)",
+            regex: "(\\-|\\+|\\*|\\/|\\~\\/|%)",
-            "token": ["keyword.operator.arithmetic.dart"]
+            token: ["keyword.operator.arithmetic.dart"]
         },
         {
-            "regex": "(!|&&|\\|\\|)",
+            regex: "(!|&&|\\|\\|)",
-            "token": ["keyword.operator.logical.dart"]
+            token: ["keyword.operator.logical.dart"]
         },
         {
-            "token" : "constant.numeric", // hex
+            token : "constant.numeric", // hex
-            "regex" : "0[xX][0-9a-fA-F]+\\b"
+            regex : "0[xX][0-9a-fA-F]+\\b"
         }, 
         {
-            "token" : "constant.numeric", // float
+            token : "constant.numeric", // float
-            "regex" : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
+            regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
         }, 
         {
-            "token" : keywordMapper,
+            token : keywordMapper,
-            "regex" : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
         }
     ],
     "comment" : [
@@ -147,30 +147,30 @@ var DartHighlightRules = function() {
     ],
     "qdoc" : [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : ".*?'''",
+            regex : ".*?'''",
-            "next" : "start"
+            next : "start"
         }, stringfill],
 
     "qqdoc" : [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '.*?"""',
+            regex : '.*?"""',
-            "next" : "start"
+            next : "start"
         }, stringfill],
 
     "qstring" : [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "[^\\\\']*(?:\\\\.[^\\\\']*)*'",
+            regex : "[^\\\\']*(?:\\\\.[^\\\\']*)*'",
-            "next" : "start"
+            next : "start"
         }, stringfill],
 
     "qqstring" : [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
+            regex : '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
-            "next" : "start"
+            next : "start"
         }, stringfill]
 }
 

lib/ace/mode/diff_highlight_rules.js

@@ -40,20 +40,20 @@ var DiffHighlightRules = function() {
 
     this.$rules = {
         "start" : [{
-                "regex": "^(?:\\*{15}|={67}|-{3}|\\+{3})$",
+                regex: "^(?:\\*{15}|={67}|-{3}|\\+{3})$",
-                "token": "punctuation.definition.separator.diff",
+                token: "punctuation.definition.separator.diff",
                 "name": "keyword"
             }, { //diff.range.unified
-                "regex": "^(@@)(\\s*.+?\\s*)(@@)(.*)$",
+                regex: "^(@@)(\\s*.+?\\s*)(@@)(.*)$",
-                "token": [
+                token: [
                     "constant",
                     "constant.numeric",
                     "constant",
                     "comment.doc.tag"
                 ]
             }, { //diff.range.normal
-                "regex": "^(\\d+)([,\\d]+)(a|d|c)(\\d+)([,\\d]+)(.*)$",
+                regex: "^(\\d+)([,\\d]+)(a|d|c)(\\d+)([,\\d]+)(.*)$",
-                "token": [
+                token: [
                     "constant.numeric",
                     "punctuation.definition.range.diff",
                     "constant.function",
@@ -63,34 +63,36 @@ var DiffHighlightRules = function() {
                 ],
                 "name": "meta."
             }, {
-                "regex": "^(?:(\\-{3}|\\+{3}|\\*{3})( .+))$",
+                regex: "^(\\-{3}|\\+{3}|\\*{3})( .+)$",
-                "token": [
+                token: [
                     "constant.numeric",
                     "meta.tag"
                 ]
             }, { // added
-                "regex": "^([!+>])(.*?)(\\s*)$",
+                regex: "^([!+>])(.*?)(\\s*)$",
-                "token": [
+                token: [
                     "support.constant",
                     "text",
                     "invalid"
                 ]
             }, { // removed
-                "regex": "^([<\\-])(.*?)(\\s*)$",
+                regex: "^([<\\-])(.*?)(\\s*)$",
-                "token": [
+                token: [
                     "support.function",
                     "string",
                     "invalid"
                 ]
             }, {
-                "regex": "^(diff)(\\s+--\\w+)?(.+?)( .+)?$",
+                regex: "^(diff)(\\s+--\\w+)?(.+?)( .+)?$",
-                "token": ["variable", "variable", "keyword", "variable"]
+                token: ["variable", "variable", "keyword", "variable"]
             }, {
-                "regex": "^Index.+$",
+                regex: "^Index.+$",
-                "token": "variable"
+                token: "variable"
             }, {
-                "regex": "^(.*?)(\\s*)$",
+                regex: "\\s*$",
-                "token": ["invisible", "invalid"]
+                token: "invalid"
+            }, {
+                defaultToken: "invisible"
             }
         ]
     };

lib/ace/mode/django.js

@@ -0,0 +1,117 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2012, Ajax.org B.V.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define('ace/mode/django', ['require', 'exports', 'module' , 'ace/lib/oop', 'ace/mode/html', 'ace/mode/text_highlight_rules', 'ace/tokenizer', 'ace/mode/html_highlight_rules'], function(require, exports, module) {
+
+var oop = require("../lib/oop");
+var HtmlMode = require("./html").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var HtmlHighlightRules = require("./html_highlight_rules").HtmlHighlightRules;
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var DjangoHighlightRules = function(){
+    this.$rules = {
+        'start': [{
+            token: "string",
+            regex: '".*?"'
+        }, {
+            token: "string",
+            regex: "'.*?'"
+        }, {
+            token: "constant",
+            regex: '[0-9]+'
+        }, {
+            token: "variable",
+            regex: "[-_a-zA-Z0-9:]+"
+        }],
+        'comment': [{
+            token : "comment.block",
+            merge: true,
+            regex : ".+?"
+        }],
+        'tag': [{
+            token: "entity.name.function",
+            regex: "[a-zA-Z][_a-zA-Z0-9]*",
+            next: "start"
+        }]
+    };
+};
+
+oop.inherits(DjangoHighlightRules, TextHighlightRules)
+
+var DjangoHtmlHighlightRules = function() {
+    this.$rules = new HtmlHighlightRules().getRules();
+
+    for (var i in this.$rules) {
+        this.$rules[i].unshift({
+            token: "comment.line",
+            regex: "\\{#.*?#\\}"
+        }, {
+            token: "comment.block",
+            regex: "\\{\\%\\s*comment\\s*\\%\\}",
+            merge: true,
+            next: "django-comment"
+        }, {
+            token: "constant.language",
+            regex: "\\{\\{",
+            next: "django-start"
+        }, {
+            token: "constant.language",
+            regex: "\\{\\%",
+            next: "django-tag"
+        });
+        this.embedRules(DjangoHighlightRules, "django-", [{
+                token: "comment.block",
+                regex: "\\{\\%\\s*endcomment\\s*\\%\\}",
+                merge: true,
+                next: "start"
+            }, {
+                token: "constant.language",
+                regex: "\\%\\}",
+                next: "start"
+            }, {
+                token: "constant.language",
+                regex: "\\}\\}",
+                next: "start"
+        }]);
+    }
+};
+
+oop.inherits(DjangoHtmlHighlightRules, HtmlHighlightRules);
+
+var Mode = function() {
+    var highlighter = new DjangoHtmlHighlightRules();
+    this.$tokenizer = new Tokenizer(highlighter.getRules());
+    this.$embeds = highlighter.getEmbeds();
+};
+oop.inherits(Mode, HtmlMode);
+
+exports.Mode = Mode;
+});

lib/ace/mode/doc_comment_highlight_rules.js

@@ -41,17 +41,10 @@ var DocCommentHighlightRules = function() {
             token : "comment.doc.tag",
             regex : "@[\\w\\d_]+" // TODO: fix email addresses
         }, {
-            token : "comment.doc",
+            token : "comment.doc.tag",
-            regex : "\\s+"
+            regex : "\\bTODO\\b"
         }, {
-            token : "comment.doc",
+            defaultToken : "comment.doc"
-            regex : "TODO"
-        }, {
-            token : "comment.doc",
-            regex : "[^@\\*]+"
-        }, {
-            token : "comment.doc",
-            regex : "."
         }]
     };
 };

lib/ace/mode/haml_highlight_rules.js

@@ -15,69 +15,69 @@ var HamlHighlightRules = function() {
         {
     "start": [
         {
-            "token" : "punctuation.section.comment",
+            token : "punctuation.section.comment",
-            "regex" : /^\s*\/.*/
+            regex : /^\s*\/.*/
         },
         {
-            "token" : "punctuation.section.comment",
+            token : "punctuation.section.comment",
-            "regex" : /^\s*#.*/
+            regex : /^\s*#.*/
         },
         {
-            "token": "string.quoted.double",
+            token: "string.quoted.double",
-            "regex": "==.+?=="
+            regex: "==.+?=="
         },
         {
-            "token": "keyword.other.doctype",
+            token: "keyword.other.doctype",
-            "regex": "^!!!\\s*(?:[a-zA-Z0-9-_]+)?"
+            regex: "^!!!\\s*(?:[a-zA-Z0-9-_]+)?"
         },
         RubyExports.qString,
         RubyExports.qqString,
         RubyExports.tString,
         {
-            "token": ["entity.name.tag.haml"],
+            token: ["entity.name.tag.haml"],
-            "regex": /^\s*%[\w:]+/,
+            regex: /^\s*%[\w:]+/,
-            "next": "tag_single"
+            next: "tag_single"
         },
         {
-            "token": [ "meta.escape.haml" ],
+            token: [ "meta.escape.haml" ],
-            "regex": "^\\s*\\\\."
+            regex: "^\\s*\\\\."
         },
         RubyExports.constantNumericHex,
         RubyExports.constantNumericFloat,
         
         RubyExports.constantOtherSymbol,
         {
-            "token": "text",
+            token: "text",
-            "regex": "=|-|~",
+            regex: "=|-|~",
-            "next": "embedded_ruby"
+            next: "embedded_ruby"
         }
     ],
     "tag_single": [
         {
-            "token": "entity.other.attribute-name.class.haml",
+            token: "entity.other.attribute-name.class.haml",
-            "regex": "\\.[\\w-]+"
+            regex: "\\.[\\w-]+"
         },
         {
-            "token": "entity.other.attribute-name.id.haml",
+            token: "entity.other.attribute-name.id.haml",
-            "regex": "#[\\w-]+"
+            regex: "#[\\w-]+"
         },
         {
-            "token": "punctuation.section",
+            token: "punctuation.section",
-            "regex": "\\{",
+            regex: "\\{",
-            "next": "section"
+            next: "section"
         },
         
         RubyExports.constantOtherSymbol,
         
         {
-            "token": "text",
+            token: "text",
-            "regex": /\s/,
+            regex: /\s/,
-            "next": "start"
+            next: "start"
         },
         {
-            "token": ["text", "punctuation"],
+            token: "empty",
-            "regex": "($)|((?!\\.|#|\\{|\\[|=|-|~|\\/))",
+            regex: "$|(?!\\.|#|\\{|\\[|=|-|~|\\/)",
-            "next": "start"
+            next: "start"
         }
     ],
     "section": [
@@ -90,9 +90,9 @@ var HamlHighlightRules = function() {
         RubyExports.constantNumericHex,
         RubyExports.constantNumericFloat,
         {
-            "token": "punctuation.section",
+            token: "punctuation.section",
-            "regex": "\\}",
+            regex: "\\}",
-            "next": "start"
+            next: "start"
         } 
     ],
     "embedded_ruby": [ 
@@ -103,23 +103,23 @@ var HamlHighlightRules = function() {
                 regex : "[A-Z][a-zA-Z_\\d]+"
         },    
         {
-            "token" : new RubyHighlightRules().getKeywords(),
+            token : new RubyHighlightRules().getKeywords(),
-            "regex" : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
         },
         {
-            "token" : ["keyword", "text", "text"],
+            token : ["keyword", "text", "text"],
-            "regex" : "(?:do|\\{)(?: \\|[^|]+\\|)?$",
+            regex : "(?:do|\\{)(?: \\|[^|]+\\|)?$",
-            "next"  : "start"
+            next  : "start"
         }, 
         {
-            "token" : ["text"],
+            token : ["text"],
-            "regex" : "^$",
+            regex : "^$",
-            "next"  : "start"
+            next  : "start"
         }, 
         {
-            "token" : ["text"],
+            token : ["text"],
-            "regex" : "^(?!.*\\|\\s*$)",
+            regex : "^(?!.*\\|\\s*$)",
-            "next"  : "start"
+            next  : "start"
         }
     ]
 }

lib/ace/mode/html_highlight_rules.js

@@ -94,21 +94,12 @@ var HtmlHighlightRules = function() {
         }, {
             token : "constant.character.entity",
             regex : "(?:&#[0-9]+;)|(?:&#x[0-9a-fA-F]+;)|(?:&[a-zA-Z0-9_:\\.-]+;)"
-        }, {
+        }],
-            token : "text",
-            regex : "[^<]+"
-        } ],
     
         cdata : [ {
             token : "text",
             regex : "\\]\\]>",
             next : "start"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : ".+"
         } ],
 
         comment : [ {
@@ -116,8 +107,7 @@ var HtmlHighlightRules = function() {
             regex : ".*?-->",
             next : "start"
         }, {
-            token : "comment",
+            defaultToken : "comment"
-            regex : ".+"
         } ]
     };
     

lib/ace/mode/html_highlight_rules_test.js

@@ -1,186 +0,0 @@
-/* ***** BEGIN LICENSE BLOCK *****
- * Distributed under the BSD license:
- *
- * Copyright (c) 2010, Ajax.org B.V.
- * All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Ajax.org B.V. nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * ***** END LICENSE BLOCK ***** */
-
-if (typeof process !== "undefined") {
-    require("amd-loader");
-}
-
-define(function(require, exports, module) {
-"use strict";
-
-var HtmlMode = require("./html").Mode;
-var assert = require("../test/assertions");
-
-var testData = {
-    "test: tokenize embedded script" : [{
-        text: "<script a='a'>var</script>'123'",
-        state: ["start", "start"],
-        tokens: [{
-            type: "meta.tag",
-            value: "<"
-        }, {
-            type: "meta.tag.tag-name.script",
-            value: "script"
-        }, {
-            type: "text",
-            value: " "
-        }, {
-            type: "entity.other.attribute-name",
-            value: "a"
-        }, {
-            type: "keyword.operator",
-            value: "="
-        }, {
-            type: "string",
-            value: "'a'"
-        }, {
-            type: "meta.tag.r",
-            value: ">"
-        }, {
-            type: "storage.type",
-            value: "var"
-        }, {
-            type: "meta.tag",
-            value: "</"
-        }, {
-            type: "meta.tag.tag-name.script",
-            value: "script"
-        }, {
-            type: "meta.tag.r",
-            value: ">"
-        }, {
-            type: "text",
-            value: "'123'"
-        }]
-    }],
-
-    "test: tokenize multiline attribute value with double quotes": [{
-        text: "<a href=\"abc",
-        state: [ "start", "tag_qqstring"],
-        tokens: [{
-                type: "meta.tag",
-                value: "<"
-            }, {
-                type: "meta.tag.tag-name.anchor",
-                value: "a"
-            }, {
-                type: "text",
-                value: " "
-            }, {
-                type: "entity.other.attribute-name",
-                value: "href"
-            }, {
-                type: "keyword.operator",
-                value: "="
-            }, {
-                type: "string",
-                value: "\"abc"
-            }
-        ]
-    }, {
-        text: "def\">",
-        state: [ "tag_qqstring", "start" ],
-        tokens: [ {
-                type: "string",
-                value: "def\""
-            }, {
-                type: "meta.tag.r",
-                value: ">"
-            }
-        ]
-    }],
-
-    "test: tokenize multiline attribute value with single quotes": [{
-        text: "<a href='abc",
-        state: ["start", "tag_qstring"],
-        tokens: [{
-                type: "meta.tag",
-                value: "<"
-            }, {
-                type: "meta.tag.tag-name.anchor",
-                value: "a"
-            }, {
-                type: "text",
-                value: " "
-            }, {
-                type: "entity.other.attribute-name",
-                value: "href"
-            }, {
-                type: "keyword.operator",
-                value: "="
-            }, {
-                type: "string",
-                value: "'abc"
-            }
-        ]
-    }, {
-        text: "def\"'>",
-        state: [ "tag_qstring", "start" ],
-        tokens: [ {
-                type: "string",
-                value: "def\"'"
-            }, {
-                type: "meta.tag.r",
-                value: ">"
-            }
-        ]
-    }]
-};
-
-function generateTest(exampleData) {
-    return function testTokenizer() {
-        for (var i = 0; i < exampleData.length; i++) {
-            var s = exampleData[i];
-            var lineTokens = tokenizer.getLineTokens(s.text, s.state[0]);
-
-            assert.equal(
-                JSON.stringify(lineTokens, null, 4),
-                JSON.stringify({tokens:s.tokens, state: s.state[1]}, null, 4)
-            );
-        }
-    }
-}
-
-var tokenizer;
-module.exports = {
-    setUp : function() {
-        tokenizer = new HtmlMode().getTokenizer();
-    }
-}
-
-for (var i in testData) {
-    module.exports[i] = generateTest(testData[i])
-}
-
-});
-
-if (typeof module !== "undefined" && module === require.main) {
-    require("asyncjs").test.testcase(module.exports).exec();
-}

lib/ace/mode/jade_highlight_rules.js

@@ -73,23 +73,23 @@ var JadeHighlightRules = function() {
         {
     "start": [
         {
-            "token": "keyword.control.import.include.jade",
+            token: "keyword.control.import.include.jade",
-            "regex": "\\s*\\binclude\\b"
+            regex: "\\s*\\binclude\\b"
         },
         {
-            "token": "keyword.other.doctype.jade",
+            token: "keyword.other.doctype.jade",
-            "regex": "^!!!\\s*(?:[a-zA-Z0-9-_]+)?"
+            regex: "^!!!\\s*(?:[a-zA-Z0-9-_]+)?"
         },
         {
-            "token" : "punctuation.section.comment",
+            token : "punctuation.section.comment",
-            "regex" : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)"
+            regex : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)"
         },
         {
-            "token" : function(space, text) {
+            token : function(space, text) {
                 return "punctuation.section.comment";
             },
-            "regex" : "^((\\s*)\/\/)(?:\\s*$)",
+            regex : "^((\\s*)\/\/)(?:\\s*$)",
-            "next": "comment_block"
+            next: "comment_block"
         },
         mixin_embed("markdown", "markdown-"),
         mixin_embed("sass", "sass-"),
@@ -97,100 +97,100 @@ var JadeHighlightRules = function() {
         mixin_embed("coffee", "coffee-"),
         /*
         {
-            "token": {
+            token: {
                 "2": {
                     "name": "entity.name.function.jade"
                 }
             },
-            "regex": "^(\\s*)(\\:cdata)",
+            regex: "^(\\s*)(\\:cdata)",
-            "next": "state_9"
+            next: "state_9"
         },*/
         // match stuff like: mixin dialog-title-desc(title, desc)
         {
-            "token": [ "storage.type.function.jade",
+            token: [ "storage.type.function.jade",
                        "entity.name.function.jade",
                        "punctuation.definition.parameters.begin.jade",
                        "variable.parameter.function.jade",
                        "punctuation.definition.parameters.end.jade"
                     ],
-            "regex": "^(\\s*mixin)( [\\w\\-]+)(\\s*\\()(.*?)(\\))"
+            regex: "^(\\s*mixin)( [\\w\\-]+)(\\s*\\()(.*?)(\\))"
         },
         // match stuff like: mixin dialog-title-desc
         {
-            "token": [ "storage.type.function.jade", "entity.name.function.jade"],
+            token: [ "storage.type.function.jade", "entity.name.function.jade"],
-            "regex": "^(\\s*mixin)( [\\w\\-]+)"
+            regex: "^(\\s*mixin)( [\\w\\-]+)"
         },
         {
-            "token": "source.js.embedded.jade",
+            token: "source.js.embedded.jade",
-            "regex": "^\\s*(?:-|=|!=)",
+            regex: "^\\s*(?:-|=|!=)",
-            "next": "js-start"
+            next: "js-start"
         },
         /*{
-            "token": "entity.name.tag.script.jade",
+            token: "entity.name.tag.script.jade",
-            "regex": "^\\s*script",
+            regex: "^\\s*script",
-            "next": "js_code_tag"
+            next: "js_code_tag"
         },*/
         {
-            "token": "string.interpolated.jade",
+            token: "string.interpolated.jade",
-            "regex": "[#!]\\{[^\\}]+\\}"
+            regex: "[#!]\\{[^\\}]+\\}"
         },
         // Match any tag, id or class. skip AST filters
         {
-            "token": ["meta.tag.any.jade", "entity.variable.tag.jade"],
+            token: "meta.tag.any.jade",
-            "regex": /^\s*(?!\w+\:)(?:[\w]+|(?=\.|#)])/,
+            regex: /^\s*(?!\w+\:)(?:[\w]+|(?=\.|#)])/,
-            "next": "tag_single"
+            next: "tag_single"
         },
         {
-            "token": "suport.type.attribute.id.jade",
+            token: "suport.type.attribute.id.jade",
-            "regex": "#\\w+"
+            regex: "#\\w+"
         },
         {
-            "token": "suport.type.attribute.class.jade",
+            token: "suport.type.attribute.class.jade",
-            "regex": "\\.\\w+"
+            regex: "\\.\\w+"
         },
         {
-            "token": "punctuation",
+            token: "punctuation",
-            "regex": "\\s*(?:\\()",
+            regex: "\\s*(?:\\()",
-            "next": "tag_attributes"
+            next: "tag_attributes"
         }
     ],
     "comment_block": [
         {
-            "token": function(text) {
+            token: function(text) {
                 return "text";
             },
-            "regex": "^(\\1\\S|$)", 
+            regex: "^(\\1\\S|$)", 
             "captures": "1",
-            "next": "start"
+            next: "start"
         },
         {
-            "token": "comment.block.jade",
+            token: "comment.block.jade",
-            "regex" : ".+"
+            regex : ".+"
         }
     ],
     /*
     
     "state_9": [
         {
-            "token": "TODO",
+            token: "TODO",
-            "regex": "^(?!\\1\\s+)",
+            regex: "^(?!\\1\\s+)",
-            "next": "start"
+            next: "start"
         },
         {
-            "token": "TODO",
+            token: "TODO",
-            "regex": ".+",
+            regex: ".+",
-            "next": "state_9"
+            next: "state_9"
         }
     ],*/
     /*"js_code": [
         {
-            "token": "keyword.control.js",
+            token: "keyword.control.js",
-            "regex": "\\beach\\b"
+            regex: "\\beach\\b"
         },
         {
-            "token": "text",
+            token: "text",
-            "regex": "$",
+            regex: "$",
-            "next": "start"
+            next: "start"
         }
     ],*/
     /*"js_code_tag": [
@@ -198,62 +198,62 @@ var JadeHighlightRules = function() {
             "include": "source.js"
         },
         {
-            "token": "TODO",
+            token: "TODO",
-            "regex": "^((?=(\\1)([\\w#\\.]|$\\n?))|^$\\n?)",
+            regex: "^((?=(\\1)([\\w#\\.]|$\\n?))|^$\\n?)",
-            "next": "start"
+            next: "start"
         }
     ],*/
     "tag_single": [
         {
-            "token": "entity.other.attribute-name.class.jade",
+            token: "entity.other.attribute-name.class.jade",
-            "regex": "\\.[\\w-]+"
+            regex: "\\.[\\w-]+"
         },
         {
-            "token": "entity.other.attribute-name.id.jade",
+            token: "entity.other.attribute-name.id.jade",
-            "regex": "#[\\w-]+"
+            regex: "#[\\w-]+"
         },
         {
-            "token": ["text", "punctuation"],
+            token: ["text", "punctuation"],
-            "regex": "($)|((?!\\.|#|=|-))",
+            regex: "($)|((?!\\.|#|=|-))",
-            "next": "start"
+            next: "start"
         }
     ],
     "tag_attributes": [ 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "'(?=.)",
+            regex : "'(?=.)",
-            "next"  : "qstring"
+            next  : "qstring"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"(?=.)',
+            regex : '"(?=.)',
-            "next"  : "qqstring"
+            next  : "qqstring"
         },
         {
-            "token": "entity.other.attribute-name.jade",
+            token: "entity.other.attribute-name.jade",
-            "regex": "\\b[a-zA-Z\\-:]+"
+            regex: "\\b[a-zA-Z\\-:]+"
         },
         {
-            "token": ["entity.other.attribute-name.jade", "punctuation"],
+            token: ["entity.other.attribute-name.jade", "punctuation"],
-            "regex": "\\b([a-zA-Z:\\.-]+)(=)",
+            regex: "\\b([a-zA-Z:\\.-]+)(=)",
-            "next": "attribute_strings"
+            next: "attribute_strings"
         },
         {
-            "token": "punctuation",
+            token: "punctuation",
-            "regex": "\\)",
+            regex: "\\)",
-            "next": "start"
+            next: "start"
         }
     ],
     "attribute_strings": [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "'(?=.)",
+            regex : "'(?=.)",
-            "next"  : "qstring"
+            next  : "qstring"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"(?=.)',
+            regex : '"(?=.)',
-            "next"  : "qqstring"
+            next  : "qqstring"
         }
     ],
     "qqstring" : [

lib/ace/mode/javascript_highlight_rules.js

@@ -61,7 +61,8 @@ var JavaScriptHighlightRules = function() {
         "constant.language":
             "null|Infinity|NaN|undefined",
         "support.function":
-            "alert"
+            "alert",
+        "constant.language.boolean": "true|false"
     }, "identifier");
 
     // keywords which can be followed by regular expressions
@@ -161,9 +162,6 @@ var JavaScriptHighlightRules = function() {
                 ],
                 regex : "(:)(\\s*)(function)(\\s*)(\\()",
                 next: "function_arguments"
-            }, {
-                token : "constant.language.boolean",
-                regex : /(?:true|false)\b/
             }, {
                 token : "keyword",
                 regex : "(?:" + kwBeforeRe + ")\\b",
@@ -205,9 +203,6 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token: "comment",
                 regex: /^#!.*$/
-            }, {
-                token : "text",
-                regex : /\s+/
             }
         ],
         // regular expressions are only allowed after certain tokens. This
@@ -253,18 +248,20 @@ var JavaScriptHighlightRules = function() {
             }, {
                 // operators
                 token : "constant.language.escape",
-                regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[(|)$^+*?]/
+                regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[()$^+*?]/
             }, {
-                token: "string.regexp",
+                token : "constant.language.delimiter",
-                regex: /{|[^{\[\/\\(|)$^+*?]+/,
+                regex: /\|/
             }, {
                 token: "constant.language.escape",
                 regex: /\[\^?/,
                 next: "regex_character_class",
             }, {
                 token: "empty",
-                regex: "",
+                regex: "$",
                 next: "start"
+            }, {
+                defaultToken: "string.regexp"
             }
         ],
         "regex_character_class": [
@@ -278,13 +275,12 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token: "constant.language.escape",
                 regex: "-"
-            }, {
-                token: "string.regexp.charachterclass",
-                regex: /[^\]\-\\]+/,
             }, {
                 token: "empty",
-                regex: "",
+                regex: "$",
                 next: "start"
+            }, {
+                defaultToken: "string.regexp.charachterclass"
             }
         ],
         "function_arguments": [
@@ -304,24 +300,12 @@ var JavaScriptHighlightRules = function() {
             }
         ],
         "comment_regex_allowed" : [
-            {
+            {token : "comment", regex : "\\*\\/", next : "regex_allowed"},
-                token : "comment", // closing comment
+            {defaultToken : "comment"}
-                regex : ".*?\\*\\/",
-                next : "regex_allowed"
-            }, {
-                token : "comment", // comment spanning whole line
-                regex : ".+"
-            }
         ],
         "comment" : [
-            {
+            {token : "comment", regex : "\\*\\/", next : "start"},
-                token : "comment", // closing comment
+            {defaultToken : "comment"}
-                regex : ".*?\\*\\/",
-                next : "start"
-            }, {
-                token : "comment", // comment spanning whole line
-                regex : ".+"
-            }
         ],
         "qqstring" : [
             {
@@ -336,8 +320,7 @@ var JavaScriptHighlightRules = function() {
                 regex : '"|$',
                 next  : "start",
             }, {
-                token : "string",
+                defaultToken: "string"
-                regex : '.|\\w+|\\s+',
             }
         ],
         "qstring" : [
@@ -353,8 +336,7 @@ var JavaScriptHighlightRules = function() {
                 regex : "'|$",
                 next  : "start",
             }, {
-                token : "string",
+                defaultToken: "string"
-                regex : '.|\\w+|\\s+',
             }
         ]
     };

lib/ace/mode/liquid_highlight_rules.js

@@ -104,24 +104,12 @@ var LiquidHighlightRules = function() {
             token : "meta.tag", // opening tag
             regex : "<\\/?",
             next : "tag"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : "[^<]+"
         } ],
 
         cdata : [ {
             token : "text",
             regex : "\\]\\]>",
             next : "start"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : ".+"
         } ],
 
         comment : [ {
@@ -129,8 +117,7 @@ var LiquidHighlightRules = function() {
             regex : ".*?-->",
             next : "start"
         }, {
-            token : "comment",
+            defaultToken : "comment"
-            regex : ".+"
         } ] ,
 
         liquid_start : [{

lib/ace/mode/lisp_highlight_rules.js

@@ -67,54 +67,51 @@ var LispHighlightRules = function() {
             regex : ";.*$"
         },
         {
-            "token": ["storage.type.function-type.lisp", "text", "entity.name.function.lisp"],
+            token: ["storage.type.function-type.lisp", "text", "entity.name.function.lisp"],
-            "regex": "(?:\\b(?:(defun|defmethod|defmacro))\\b)(\\s+)((?:\\w|\\-|\\!|\\?)*)"
+            regex: "(?:\\b(?:(defun|defmethod|defmacro))\\b)(\\s+)((?:\\w|\\-|\\!|\\?)*)"
         },
         {
-            "token": ["punctuation.definition.constant.character.lisp", "constant.character.lisp"],
+            token: ["punctuation.definition.constant.character.lisp", "constant.character.lisp"],
-            "regex": "(#)((?:\\w|[\\\\+-=<>'\"&#])+)"
+            regex: "(#)((?:\\w|[\\\\+-=<>'\"&#])+)"
         },
         {
-            "token": ["punctuation.definition.variable.lisp", "variable.other.global.lisp", "punctuation.definition.variable.lisp"],
+            token: ["punctuation.definition.variable.lisp", "variable.other.global.lisp", "punctuation.definition.variable.lisp"],
-            "regex": "(\\*)(\\S*)(\\*)"
+            regex: "(\\*)(\\S*)(\\*)"
         },
         {
-            "token" : "constant.numeric", // hex
+            token : "constant.numeric", // hex
-            "regex" : "0[xX][0-9a-fA-F]+(?:L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b"
+            regex : "0[xX][0-9a-fA-F]+(?:L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b"
         }, 
         {
-            "token" : "constant.numeric", // float
+            token : "constant.numeric", // float
-            "regex" : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?(?:L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b"
+            regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?(?:L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b"
         },
         {
-                "token" : keywordMapper,
+                token : keywordMapper,
-                "regex" : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+                regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
         },
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"(?=.)',
+            regex : '"(?=.)',
-            "next"  : "qqstring"
+            next  : "qqstring"
         }
     ],
     "qqstring": [
         {
-            "token": "constant.character.escape.lisp",
+            token: "constant.character.escape.lisp",
-            "regex": "\\\\."
+            regex: "\\\\."
         },
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '[^"\\\\]+',
+            regex : '[^"\\\\]+'
-            "merge" : true
         }, {
-            "token" : "string",
+            token : "string",
-            "regex" : "\\\\$",
+            regex : "\\\\$",
-            "next"  : "qqstring",
+            next  : "qqstring"
-            "merge" : true
         }, {
-            "token" : "string",
+            token : "string",
-            "regex" : '"|$',
+            regex : '"|$',
-            "next"  : "start",
+            next  : "start"
-            "merge" : true
         }
     ]
 }

lib/ace/mode/livescript.js

@@ -0,0 +1,248 @@
+define('ace/mode/ls', function(require, exports, module){
+  var identifier, LiveScriptMode, keywordend, stringfill;
+  identifier = '(?![\\d\\s])[$\\w\\xAA-\\uFFDC](?:(?!\\s)[$\\w\\xAA-\\uFFDC]|-[A-Za-z])*';
+  exports.Mode = LiveScriptMode = (function(superclass){
+    var indenter, prototype = extend$((import$(LiveScriptMode, superclass).displayName = 'LiveScriptMode', LiveScriptMode), superclass).prototype, constructor = LiveScriptMode;
+    function LiveScriptMode(){
+      var that;
+      this.$tokenizer = new (require('ace/tokenizer')).Tokenizer(LiveScriptMode.Rules);
+      if (that = require('ace/mode/matching_brace_outdent')) {
+        this.$outdent = new that.MatchingBraceOutdent;
+      }
+    }
+    indenter = RegExp('(?:[({[=:]|[-~]>|\\b(?:e(?:lse|xport)|d(?:o|efault)|t(?:ry|hen)|finally|import(?:\\s*all)?|const|var|let|new|catch(?:\\s*' + identifier + ')?))\\s*$');
+    prototype.getNextLineIndent = function(state, line, tab){
+      var indent, tokens;
+      indent = this.$getIndent(line);
+      tokens = this.$tokenizer.getLineTokens(line, state).tokens;
+      if (!(tokens.length && tokens[tokens.length - 1].type === 'comment')) {
+        if (state === 'start' && indenter.test(line)) {
+          indent += tab;
+        }
+      }
+      return indent;
+    };
+    prototype.toggleCommentLines = function(state, doc, startRow, endRow){
+      var comment, range, i$, i, out, line;
+      comment = /^(\s*)#/;
+      range = new (require('ace/range')).Range(0, 0, 0, 0);
+      for (i$ = startRow; i$ <= endRow; ++i$) {
+        i = i$;
+        if (out = comment.test(line = doc.getLine(i))) {
+          line = line.replace(comment, '$1');
+        } else {
+          line = line.replace(/^\s*/, '$&#');
+        }
+        range.end.row = range.start.row = i;
+        range.end.column = line.length + 1;
+        doc.replace(range, line);
+      }
+      return 1 - out * 2;
+    };
+    prototype.checkOutdent = function(state, line, input){
+      var ref$;
+      return (ref$ = this.$outdent) != null ? ref$.checkOutdent(line, input) : void 8;
+    };
+    prototype.autoOutdent = function(state, doc, row){
+      var ref$;
+      return (ref$ = this.$outdent) != null ? ref$.autoOutdent(doc, row) : void 8;
+    };
+    return LiveScriptMode;
+  }(require('ace/mode/text').Mode));
+  keywordend = '(?![$\\w]|-[A-Za-z]|\\s*:(?![:=]))';
+  stringfill = {
+    token: 'string',
+    regex: '.+'
+  };
+  LiveScriptMode.Rules = {
+    start: [
+      {
+        token: 'keyword',
+        regex: '(?:t(?:h(?:is|row|en)|ry|ypeof!?)|c(?:on(?:tinue|st)|a(?:se|tch)|lass)|i(?:n(?:stanceof)?|mp(?:ort(?:\\s+all)?|lements)|[fs])|d(?:e(?:fault|lete|bugger)|o)|f(?:or(?:\\s+own)?|inally|unction)|s(?:uper|witch)|e(?:lse|x(?:tends|port)|val)|a(?:nd|rguments)|n(?:ew|ot)|un(?:less|til)|w(?:hile|ith)|o[fr]|return|break|let|var|loop)' + keywordend
+      }, {
+        token: 'constant.language',
+        regex: '(?:true|false|yes|no|on|off|null|void|undefined)' + keywordend
+      }, {
+        token: 'invalid.illegal',
+        regex: '(?:p(?:ackage|r(?:ivate|otected)|ublic)|i(?:mplements|nterface)|enum|static|yield)' + keywordend
+      }, {
+        token: 'language.support.class',
+        regex: '(?:R(?:e(?:gExp|ferenceError)|angeError)|S(?:tring|yntaxError)|E(?:rror|valError)|Array|Boolean|Date|Function|Number|Object|TypeError|URIError)' + keywordend
+      }, {
+        token: 'language.support.function',
+        regex: '(?:is(?:NaN|Finite)|parse(?:Int|Float)|Math|JSON|(?:en|de)codeURI(?:Component)?)' + keywordend
+      }, {
+        token: 'variable.language',
+        regex: '(?:t(?:hat|il|o)|f(?:rom|allthrough)|it|by|e)' + keywordend
+      }, {
+        token: 'identifier',
+        regex: identifier + '\\s*:(?![:=])'
+      }, {
+        token: 'variable',
+        regex: identifier
+      }, {
+        token: 'keyword.operator',
+        regex: '(?:\\.{3}|\\s+\\?)'
+      }, {
+        token: 'keyword.variable',
+        regex: '(?:@+|::|\\.\\.)',
+        next: 'key'
+      }, {
+        token: 'keyword.operator',
+        regex: '\\.\\s*',
+        next: 'key'
+      }, {
+        token: 'string',
+        regex: '\\\\\\S[^\\s,;)}\\]]*'
+      }, {
+        token: 'string.doc',
+        regex: '\'\'\'',
+        next: 'qdoc'
+      }, {
+        token: 'string.doc',
+        regex: '"""',
+        next: 'qqdoc'
+      }, {
+        token: 'string',
+        regex: '\'',
+        next: 'qstring'
+      }, {
+        token: 'string',
+        regex: '"',
+        next: 'qqstring'
+      }, {
+        token: 'string',
+        regex: '`',
+        next: 'js'
+      }, {
+        token: 'string',
+        regex: '<\\[',
+        next: 'words'
+      }, {
+        token: 'string.regex',
+        regex: '//',
+        next: 'heregex'
+      }, {
+        token: 'comment.doc',
+        regex: '/\\*',
+        next: 'comment'
+      }, {
+        token: 'comment',
+        regex: '#.*'
+      }, {
+        token: 'string.regex',
+        regex: '\\/(?:[^[\\/\\n\\\\]*(?:(?:\\\\.|\\[[^\\]\\n\\\\]*(?:\\\\.[^\\]\\n\\\\]*)*\\])[^[\\/\\n\\\\]*)*)\\/[gimy$]{0,4}',
+        next: 'key'
+      }, {
+        token: 'constant.numeric',
+        regex: '(?:0x[\\da-fA-F][\\da-fA-F_]*|(?:[2-9]|[12]\\d|3[0-6])r[\\da-zA-Z][\\da-zA-Z_]*|(?:\\d[\\d_]*(?:\\.\\d[\\d_]*)?|\\.\\d[\\d_]*)(?:e[+-]?\\d[\\d_]*)?[\\w$]*)'
+      }, {
+        token: 'lparen',
+        regex: '[({[]'
+      }, {
+        token: 'rparen',
+        regex: '[)}\\]]',
+        next: 'key'
+      }, {
+        token: 'keyword.operator',
+        regex: '\\S+'
+      }, {
+        token: 'text',
+        regex: '\\s+'
+      }
+    ],
+    heregex: [
+      {
+        token: 'string.regex',
+        regex: '.*?//[gimy$?]{0,4}',
+        next: 'start'
+      }, {
+        token: 'string.regex',
+        regex: '\\s*#{'
+      }, {
+        token: 'comment.regex',
+        regex: '\\s+(?:#.*)?'
+      }, {
+        token: 'string.regex',
+        regex: '\\S+'
+      }
+    ],
+    key: [
+      {
+        token: 'keyword.operator',
+        regex: '[.?@!]+'
+      }, {
+        token: 'identifier',
+        regex: identifier,
+        next: 'start'
+      }, {
+        token: 'text',
+        regex: '.',
+        next: 'start'
+      }
+    ],
+    comment: [
+      {
+        token: 'comment.doc',
+        regex: '.*?\\*/',
+        next: 'start'
+      }, {
+        token: 'comment.doc',
+        regex: '.+'
+      }
+    ],
+    qdoc: [
+      {
+        token: 'string',
+        regex: ".*?'''",
+        next: 'key'
+      }, stringfill
+    ],
+    qqdoc: [
+      {
+        token: 'string',
+        regex: '.*?"""',
+        next: 'key'
+      }, stringfill
+    ],
+    qstring: [
+      {
+        token: 'string',
+        regex: '[^\\\\\']*(?:\\\\.[^\\\\\']*)*\'',
+        next: 'key'
+      }, stringfill
+    ],
+    qqstring: [
+      {
+        token: 'string',
+        regex: '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
+        next: 'key'
+      }, stringfill
+    ],
+    js: [
+      {
+        token: 'string',
+        regex: '[^\\\\`]*(?:\\\\.[^\\\\`]*)*`',
+        next: 'key'
+      }, stringfill
+    ],
+    words: [
+      {
+        token: 'string',
+        regex: '.*?\\]>',
+        next: 'key'
+      }, stringfill
+    ]
+  };
+function extend$(sub, sup){
+  function fun(){} fun.prototype = (sub.superclass = sup).prototype;
+  (sub.prototype = new fun).constructor = sub;
+  if (typeof sup.extended == 'function') sup.extended(sub);
+  return sub;
+}
+function import$(obj, src){
+  var own = {}.hasOwnProperty;
+  for (var key in src) if (own.call(src, key)) obj[key] = src[key];
+  return obj;
+}
+});

lib/ace/mode/lua_highlight_rules.js

@@ -86,8 +86,6 @@ var LuaHighlightRules = function() {
         "variable.language": "this"
     }, "identifier");
 
-    var strPre = "";
-
     var decimalInteger = "(?:(?:[1-9]\\d*)|(?:0))";
     var hexInteger = "(?:0[xX][\\dA-Fa-f]+)";
     var integer = "(?:" + decimalInteger + "|" + hexInteger + ")";
@@ -97,138 +95,71 @@ var LuaHighlightRules = function() {
     var pointFloat = "(?:(?:" + intPart + "?" + fraction + ")|(?:" + intPart + "\\.))";
     var floatNumber = "(?:" + pointFloat + ")";
 
-    var comment_stack = [];
-
     this.$rules = {
-        "start" :
+        "start" : [{
-
+            stateName: "bracketedComment",
-
+            token : function(value, currentState, stack){
-        // bracketed comments
+                stack.unshift(this.next, value.length, currentState);
-        [{
-            token : "comment",           // --[[ comment
-            regex : strPre + '\\-\\-\\[\\[.*\\]\\]'
-        }, {
-            token : "comment",           // --[=[ comment
-            regex : strPre + '\\-\\-\\[\\=\\[.*\\]\\=\\]'
-        }, {
-            token : "comment",           // --[==[ comment
-            regex : strPre + '\\-\\-\\[\\={2}\\[.*\\]\\={2}\\]'
-        }, {
-            token : "comment",           // --[===[ comment
-            regex : strPre + '\\-\\-\\[\\={3}\\[.*\\]\\={3}\\]'
-        }, {
-            token : "comment",           // --[====[ comment
-            regex : strPre + '\\-\\-\\[\\={4}\\[.*\\]\\={4}\\]'
-        }, {
-            token : "comment",           // --[====+[ comment
-            regex : strPre + '\\-\\-\\[\\={5}\\=*\\[.*\\]\\={5}\\=*\\]'
-        },
-
-        // multiline bracketed comments
-        {
-            token : "comment",           // --[[ comment
-            regex : strPre + '\\-\\-\\[\\[.*$',
-            next  : "qcomment"
-        }, {
-            token : "comment",           // --[=[ comment
-            regex : strPre + '\\-\\-\\[\\=\\[.*$',
-            next  : "qcomment1"
-        }, {
-            token : "comment",           // --[==[ comment
-            regex : strPre + '\\-\\-\\[\\={2}\\[.*$',
-            next  : "qcomment2"
-        }, {
-            token : "comment",           // --[===[ comment
-            regex : strPre + '\\-\\-\\[\\={3}\\[.*$',
-            next  : "qcomment3"
-        }, {
-            token : "comment",           // --[====[ comment
-            regex : strPre + '\\-\\-\\[\\={4}\\[.*$',
-            next  : "qcomment4"
-        }, {
-            token : function(value){     // --[====+[ comment
-                // WARNING: EXTREMELY SLOW, but this is the only way to circumvent the
-                // limits imposed by the current automaton.
-                // I've never personally seen any practical code where 5 or more '='s are
-                // used for string or commenting, so this will rarely be invoked.
-                var pattern = /\-\-\[(\=+)\[/, match;
-                // you can never be too paranoid ;)
-                if ((match = pattern.exec(value)) != null && (match = match[1]) != undefined)
-                    comment_stack.push(match.length);
-
                 return "comment";
             },
-            regex : strPre + '\\-\\-\\[\\={5}\\=*\\[.*$',
+            regex : /\-\-\[=*\[/,
-            next  : "qcomment5"
+            next  : [
+                {
+                    token : function(value, currentState, stack) {
+                        if (value.length == stack[1]) {
+                            stack.shift();
+                            stack.shift();
+                            this.next = stack.shift();
+                        } else {
+                            this.next = "";
+                        }
+                        return "comment";
+                    },
+                    regex : /(?:[^\\]|\\.)*?\]=*\]/,
+                    next  : "start"
+                }, {
+                    defaultToken : "comment"
+                }
+            ]
         },
 
-        // single line comments
         {
             token : "comment",
             regex : "\\-\\-.*$"
         },
-
-        // bracketed strings
         {
-            token : "string",           // [[ string
+            stateName: "bracketedString",
-            regex : strPre + '\\[\\[.*\\]\\]'
+            token : function(value, currentState, stack){
-        }, {
+                stack.unshift(this.next, value.length, currentState);
-            token : "string",           // [=[ string
+                return "comment";
-            regex : strPre + '\\[\\=\\[.*\\]\\=\\]'
-        }, {
-            token : "string",           // [==[ string
-            regex : strPre + '\\[\\={2}\\[.*\\]\\={2}\\]'
-        }, {
-            token : "string",           // [===[ string
-            regex : strPre + '\\[\\={3}\\[.*\\]\\={3}\\]'
-        }, {
-            token : "string",           // [====[ string
-            regex : strPre + '\\[\\={4}\\[.*\\]\\={4}\\]'
-        }, {
-            token : "string",           // [====+[ string
-            regex : strPre + '\\[\\={5}\\=*\\[.*\\]\\={5}\\=*\\]'
-        },
-
-        // multiline bracketed strings
-        {
-            token : "string",           // [[ string
-            regex : strPre + '\\[\\[.*$',
-            next  : "qstring"
-        }, {
-            token : "string",           // [=[ string
-            regex : strPre + '\\[\\=\\[.*$',
-            next  : "qstring1"
-        }, {
-            token : "string",           // [==[ string
-            regex : strPre + '\\[\\={2}\\[.*$',
-            next  : "qstring2"
-        }, {
-            token : "string",           // [===[ string
-            regex : strPre + '\\[\\={3}\\[.*$',
-            next  : "qstring3"
-        }, {
-            token : "string",           // [====[ string
-            regex : strPre + '\\[\\={4}\\[.*$',
-            next  : "qstring4"
-        }, {
-            token : function(value){     // --[====+[ string
-                // WARNING: EXTREMELY SLOW, see above.
-                var pattern = /\[(\=+)\[/, match;
-                if ((match = pattern.exec(value)) != null && (match = match[1]) != undefined)
-                    comment_stack.push(match.length);
-
-                return "string";
             },
-            regex : strPre + '\\[\\={5}\\=*\\[.*$',
+            regex : /\[=*\[/,
-            next  : "qstring5"
+            next  : [
-        },
+                {
+                    token : function(value, currentState, stack) {
+                        if (value.length == stack[1]) {
+                            stack.shift();
+                            stack.shift();
+                            this.next = stack.shift();
+                        } else {
+                            this.next = "";
+                        }
+                        return "comment";
+                    },
                     
+                    regex : /(?:[^\\]|\\.)*?\]=*\]/,
+                    next  : "start"
+                }, {
+                    defaultToken : "comment"
+                }
+            ]
+        },
         {
             token : "string",           // " string
-            regex : strPre + '"(?:[^\\\\]|\\\\.)*?"'
+            regex : '"(?:[^\\\\]|\\\\.)*?"'
         }, {
             token : "string",           // ' string
-            regex : strPre + "'(?:[^\\\\]|\\\\.)*?'"
+            regex : "'(?:[^\\\\]|\\\\.)*?'"
         }, {
             token : "constant.numeric", // float
             regex : floatNumber
@@ -249,135 +180,11 @@ var LuaHighlightRules = function() {
             regex : "[\\]\\)\\}]"
         }, {
             token : "text",
-            regex : "\\s+"
+            regex : "\\s+|\\w+"
-        } ],
-
-        "qcomment": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment1": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\=\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment2": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={2}\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment3": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={3}\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment4": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={4}\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment5": [ {
-            token : function(value){
-                // very hackish, mutates the qcomment5 field on the fly.
-                var pattern = /\](\=+)\]/, rule = this.rules.qcomment5[0], match;
-                rule.next = "start";
-                if ((match = pattern.exec(value)) != null && (match = match[1]) != undefined){
-                    var found = match.length, expected;
-                    if ((expected = comment_stack.pop()) != found){
-                        comment_stack.push(expected);
-                        rule.next = "qcomment5";
-                    }
-                }
-
-                return "comment";
-            },
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={5}\\=*\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-
-        "qstring": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring1": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\=\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring2": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={2}\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring3": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={3}\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring4": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={4}\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring5": [ {
-            token : function(value){
-                // very hackish, mutates the qstring5 field on the fly.
-                var pattern = /\](\=+)\]/, rule = this.rules.qstring5[0], match;
-                rule.next = "start";
-                if ((match = pattern.exec(value)) != null && (match = match[1]) != undefined){
-                    var found = match.length, expected;
-                    if ((expected = comment_stack.pop()) != found){
-                        comment_stack.push(expected);
-                        rule.next = "qstring5";
-                    }
-                }
-
-                return "string";
-            },
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={5}\\=*\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
         } ]
-
     };
     
+    this.normalizeRules();
 }
 
 oop.inherits(LuaHighlightRules, TextHighlightRules);

lib/ace/mode/makefile_highlight_rules.js

@@ -21,48 +21,48 @@ var MakefileHighlightRules = function() {
         {
     "start": [
         {
-            "token": "string.interpolated.backtick.makefile",
+            token: "string.interpolated.backtick.makefile",
-            "regex": "`",
+            regex: "`",
-            "next": "shell-start"
+            next: "shell-start"
         },
         {
-            "token": "punctuation.definition.comment.makefile",
+            token: "punctuation.definition.comment.makefile",
-            "regex": /#(?=.)/,
+            regex: /#(?=.)/,
-            "next": "comment"
+            next: "comment"
         },
         {
-            "token": [ "keyword.control.makefile"],
+            token: [ "keyword.control.makefile"],
-            "regex": "^(?:\\s*\\b)(\\-??include|ifeq|ifneq|ifdef|ifndef|else|endif|vpath|export|unexport|define|endef|override)(?:\\b)"
+            regex: "^(?:\\s*\\b)(\\-??include|ifeq|ifneq|ifdef|ifndef|else|endif|vpath|export|unexport|define|endef|override)(?:\\b)"
         },
         {// ^([^\t ]+(\s[^\t ]+)*:(?!\=))\s*.*
-            "token": ["entity.name.function.makefile", "text"],
+            token: ["entity.name.function.makefile", "text"],
-            "regex": "^([^\\t ]+(?:\\s[^\\t ]+)*:)(\\s*.*)"
+            regex: "^([^\\t ]+(?:\\s[^\\t ]+)*:)(\\s*.*)"
         }
     ],
     "comment": [
         {
-            "token" : "punctuation.definition.comment.makefile",
+            token : "punctuation.definition.comment.makefile",
-            "regex" : /.+\\/
+            regex : /.+\\/
         },
         {
-            "token" : "punctuation.definition.comment.makefile",
+            token : "punctuation.definition.comment.makefile",
-            "regex" : ".+",
+            regex : ".+",
-            "next"  : "start"
+            next  : "start"
         }
     ],
     "shell-start": [
         {
-            "token": keywordMapper,
+            token: keywordMapper,
-            "regex" : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
         }, 
         {
-            "token": "string",
+            token: "string",
-            "regex" : "\\w+"
+            regex : "\\w+"
         }, 
         {
-            "token" : "string.interpolated.backtick.makefile",
+            token : "string.interpolated.backtick.makefile",
-            "regex" : "`",
+            regex : "`",
-            "next"  : "start"
+            next  : "start"
         }
     ]
 }

lib/ace/mode/markdown_highlight_rules.js

@@ -52,38 +52,9 @@ var MarkdownHighlightRules = function() {
     // regexps are ordered -> the first match is used
 
     this.$rules = {
-        "start" : [ {
+        "basic" : [{ // code span `
-            token : "empty_line",
-            regex : '^$'
-        }, { // code span `
-            token : ["support.function", "support.function", "support.function"],
-            regex : "(`+)([^\\r]*?[^`])(\\1)"
-        }, { // code block
             token : "support.function",
-            regex : "^[ ]{4}.+"
+            regex : "(`+)(.*?[^`])(\\1)"
-        }, { // h1
-            token: "markup.heading.1",
-            regex: "^=+(?=\\s*$)"
-        }, { // h2
-            token: "markup.heading.2",
-            regex: "^\\-+(?=\\s*$)"
-        }, { // header
-            token : function(value) {
-                return "markup.heading." + value.search(/[^#]/);
-            },
-            regex : "^#{1,6}(?:[^ #].*| +.*(?:[^ #].*|[^ ]+.* +#+ *))$"
-        }, github_embed("(?:javascript|js)", "js-"),
-           github_embed("xml", "xml-"),
-           github_embed("html", "html-"),
-           github_embed("css", "css-"),
-        { // Github style block
-            token : "support.function",
-            regex : "^```\\s*[a-zA-Z]*(?:{.*?\\})?\\s*$",
-            next  : "githubblock"
-        }, { // block quote
-            token : "string",
-            regex : "^>[ ].+$",
-            next  : "blockquote"
         }, { // reference
             token : ["text", "constant", "text", "url", "string", "text"],
             regex : "^([ ]{0,3}\\[)([^\\]]+)(\\]:\\s*)([^ ]+)(\\s*(?:[\"][^\"]+[\"])?(\\s*))$"
@@ -98,25 +69,12 @@ var MarkdownHighlightRules = function() {
                     "(<?(?:(?:[^\\(]*?\\([^\\)]*?\\)\\S*?)|(?:.*?))>?)"+
                     "((?:[ \t]*\"(?:.*?)\"[ \\t]*)?)"+
                     "(\\))"
-        }, { // HR *
-            token : "constant",
-            regex : "^[ ]{0,2}(?:[ ]?\\*[ ]?){3,}\\s*$"
-        }, { // HR -
-            token : "constant",
-            regex : "^[ ]{0,2}(?:[ ]?\\-[ ]?){3,}\\s*$"
-        }, { // HR _
-            token : "constant",
-            regex : "^[ ]{0,2}(?:[ ]?\\_[ ]?){3,}\\s*$"
-        }, { // list
-            token : "markup.list",
-            regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
-            next  : "listblock"
         }, { // strong ** __
-            token : ["string", "string", "string"],
+            token : "string",
-            regex : "([*]{2}|[_]{2}(?=\\S))([^\\r]*?\\S[*_]*)(\\1)"
+            regex : "([*]{2}|[_]{2}(?=\\S))(.*?\\S[*_]*)(\\1)"
         }, { // emphasis * _
-            token : ["string", "string", "string"],
+            token : "string",
-            regex : "([*]|[_](?=\\S))([^\\r]*?\\S[*_]*)(\\1)"
+            regex : "([*]|[_](?=\\S))(.*?\\S[*_]*)(\\1)"
         }, { //
             token : ["text", "url", "text"],
             regex : "(<)("+
@@ -124,18 +82,72 @@ var MarkdownHighlightRules = function() {
                       "|"+
                       "(?:mailto:)?[-.\\w]+\\@[-a-z0-9]+(?:\\.[-a-z0-9]+)*\\.[a-z]+"+
                     ")(>)"
+        }],
+
+        // code block
+        "allowBlock": [
+            {token : "support.function", regex : "^ {4}.+", next : "allowBlock"},
+            {token : "empty", regex : "", next : "start"}
+        ],
+
+        "start" : [{
+            token : "empty_line",
+            regex : '^$',
+            next: "allowBlock"
+        }, { // h1
+            token: "markup.heading.1",
+            regex: "^=+(?=\\s*$)"
+        }, { // h2
+            token: "markup.heading.2",
+            regex: "^\\-+(?=\\s*$)"
+        }, { // header
+            token : function(value) {
+                return "markup.heading." + value.search(/[^#]/);
+            },
+            regex : "^#{1,6}(?:[^ #].*| +.*(?:[^ #].*|[^ ]+.* +#+ *))$"
+        },
+           github_embed("(?:javascript|js)", "js-"),
+           github_embed("xml", "xml-"),
+           github_embed("html", "html-"),
+           github_embed("css", "css-"),
+        { // Github style block
+            token : "support.function",
+            regex : "^```\\s*[a-zA-Z]*(?:{.*?\\})?\\s*$",
+            next  : "githubblock"
+        }, { // block quote
+            token : "string",
+            regex : "^>[ ].+$",
+            next  : "blockquote"
+        }, { // HR * - _
+            token : "constant",
+            regex : "^ {0,2}(?:(?: ?\\* ?){3,}|(?: ?\\- ?){3,}|(?: ?\\_ ?){3,})\\s*$",
+            next: "allowBlock"
+        }, { // list
+            token : "markup.list",
+            regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
+            next  : "listblock-start"
         }, {
-            token : "text",
+            include : "basic"
-            regex : "[^\\*_%$`\\[#<>]+"
+        }],
-        } ],
+
+        "listblock-start" : [{
+            token : "support.variable",
+            regex : /(?:\[[ x]\])?/,
+            next  : "listblock"
+        }],
 
         "listblock" : [ { // Lists only escape on completely blank lines.
             token : "empty_line",
             regex : "^$",
             next  : "start"
         }, {
+            include : "basic", noEscape: true
+        }, { // list
             token : "markup.list",
-            regex : ".+"
+            regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
+            next  : "listblock-start"
+        }, {
+            defaultToken : "markup.list"
         } ],
 
         "blockquote" : [ { // BLockquotes only escape on blank lines.
@@ -189,6 +201,7 @@ var MarkdownHighlightRules = function() {
             this.$rules[i] = html[i];
     }
 
+    this.normalizeRules();
 };
 oop.inherits(MarkdownHighlightRules, TextHighlightRules);
 

lib/ace/mode/php_highlight_rules.js

@@ -974,6 +974,16 @@ var PhpLangHighlightRules = function() {
                 // TODO: Unicode escape sequences
                 // TODO: Unicode identifiers
                 regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            }, {
+                token : function(value, currentSate, state) {
+                    value = value.substr(3);
+                    if (value[0] == "'" || value[0] == '"')
+                        value = value.slice(1, -1);
+                    state.unshift(this.next, value);
+                    return "markup.list";
+                },
+                regex : /<<<(?:\w+|'\w+'|"\w+")$/,
+                next: "heredoc"
             }, {
                 token : "keyword.operator",
                 regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"
@@ -988,6 +998,22 @@ var PhpLangHighlightRules = function() {
                 regex : "\\s+"
             }
         ],
+        "heredoc" : [
+            {
+                token : function(value, currentSate, stack) {
+                    if (stack[1]  + ";" != value)
+                        return "string";
+                    stack.shift();
+                    stack.shift();
+                    return "markup.list"
+                },
+                regex : "^\\w+;$",
+                next: "start"
+            }, {
+                token: "string",
+                regex : ".*",
+            }
+        ],
         "comment" : [
             {
                 token : "comment", // closing comment
@@ -1004,31 +1030,18 @@ var PhpLangHighlightRules = function() {
                 regex : '\\\\(?:[nrtvef\\\\"$]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2})'
             }, {
                 token : "constant.language.escape",
-                regex : /\$[\w\d]+(?:\[[\w\d]+\])?/
+                regex : /\$[\w]+(?:\[[\w\]+]|=>\w+)?/
             }, {
                 token : "constant.language.escape",
                 regex : /\$\{[^"\}]+\}?/           // this is wrong but ok for now
-            }, {
+            },
-                token : "string",
+            {token : "string", regex : '"', next : "start"},
-                regex : '"',
+            {defaultToken : "string"}
-                next : "start"
-            }, {
-                token : "string",
-                regex : '.+?'
-            }
         ],
         "qstring" : [
-            {
+            {token : "constant.language.escape", regex : /\\['\\]/},
-                token : "constant.language.escape",
+            {token : "string", regex : "'", next : "start"},
-                regex : "\\\\['\\\\]"
+            {defaultToken : "string"}
-            }, {
-                token : "string",
-                regex : "'",
-                next : "start"
-            }, {
-                token : "string",
-                regex : ".+?"
-            }
         ]
     };
 

lib/ace/mode/ruby_highlight_rules.js

@@ -36,34 +36,34 @@ var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
 
 // exports is for Haml
 var constantOtherSymbol = exports.constantOtherSymbol = {
-                token : "constant.other.symbol.ruby", // symbol
+    token : "constant.other.symbol.ruby", // symbol
-                regex : "[:](?:[A-Za-z_]|[@$](?=[a-zA-Z0-9_]))[a-zA-Z0-9_]*[!=?]?"
+    regex : "[:](?:[A-Za-z_]|[@$](?=[a-zA-Z0-9_]))[a-zA-Z0-9_]*[!=?]?"
-           };
+};
 
 var qString = exports.qString = {
-                token : "string", // single line
+    token : "string", // single line
-                regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
+    regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
-            };
+};
 
 var qqString = exports.qqString = {
-                token : "string", // single line
+    token : "string", // single line
-                regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
+    regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
-            };
+};
 
 var tString = exports.tString = {
-                token : "string", // backtick string
+    token : "string", // backtick string
-                regex : "[`](?:(?:\\\\.)|(?:[^'\\\\]))*?[`]"
+    regex : "[`](?:(?:\\\\.)|(?:[^'\\\\]))*?[`]"
-            };
+};
 
 var constantNumericHex = exports.constantNumericHex = {
-                token : "constant.numeric", // hex
+    token : "constant.numeric", // hex
-                regex : "0[xX][0-9a-fA-F](?:[0-9a-fA-F]|_(?=[0-9a-fA-F]))*\\b"
+    regex : "0[xX][0-9a-fA-F](?:[0-9a-fA-F]|_(?=[0-9a-fA-F]))*\\b"
-            };
+};
 
 var constantNumericFloat = exports.constantNumericFloat = {
-                token : "constant.numeric", // float
+    token : "constant.numeric", // float
-                regex : "[+-]?\\d(?:\\d|_(?=\\d))*(?:(?:\\.\\d(?:\\d|_(?=\\d))*)?(?:[eE][+-]?\\d+)?)?\\b"
+    regex : "[+-]?\\d(?:\\d|_(?=\\d))*(?:(?:\\.\\d(?:\\d|_(?=\\d))*)?(?:[eE][+-]?\\d+)?)?\\b"
-            };
+};
 
 var RubyHighlightRules = function() {
 
@@ -171,6 +171,49 @@ var RubyHighlightRules = function() {
             }, {
                 token : "punctuation.separator.key-value",
                 regex : "=>"
+            }, {
+                stateName: "heredoc",
+                token : function(value, currentState, stack) {
+                    var next = value[3] == '-' ? "heredoc" : "indentedHeredoc";
+                    var tokens = value.split(this.splitRegex);
+                    stack.push(next, tokens[3]);
+                    return [
+                        {type:"constant", value: "<<"},
+                        {type:"string", value: tokens[2]},
+                        {type:"support.class", value: tokens[3]},
+                        {type:"string", value: tokens[4]}
+                    ];
+                },
+                regex : "(<<-?)(['\"`]?)([\\w]+)(['\"`]?)",
+                rules: {
+                    heredoc: [{
+                        token:  function(value, currentState, stack) {
+                            if (value == stack[1]) {
+                                stack.shift();
+                                stack.shift();
+                                return "support.class";
+                            }
+                            return "string";
+                        },
+                        regex: ".*$",
+                        next: "start"
+                    }],
+                    indentedHeredoc: [{
+                        token: "string",
+                        regex: "^ +"
+                    }, {
+                        token:  function(value, currentState, stack) {
+                            if (value == stack[1]) {
+                                stack.shift();
+                                stack.shift();
+                                return "support.class";
+                            }
+                            return "string";
+                        },
+                        regex: ".*$",
+                        next: "start"
+                    }]
+                }
             }, {
                 token : "keyword.operator",
                 regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"
@@ -196,6 +239,8 @@ var RubyHighlightRules = function() {
             }
         ]
     };
+
+    this.normalizeRules();
 };
 
 oop.inherits(RubyHighlightRules, TextHighlightRules);

lib/ace/mode/scheme.js

@@ -1,7 +1,7 @@
 /* ***** BEGIN LICENSE BLOCK *****
  * Distributed under the BSD license:
  *
- * Copyright (c) 2010, Ajax.org B.V.
+ * Copyright (c) 2012, Ajax.org B.V.
  * All rights reserved.
  * 
  * Redistribution and use in source and binary forms, with or without
@@ -26,55 +26,31 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
+ *
+ * Contributor(s):
+ * 
+ * NalaGinrut@gmail.com
+ *
  * ***** END LICENSE BLOCK ***** */
 
-if (typeof process !== "undefined") {
-    require("amd-loader");
-}
-
 define(function(require, exports, module) {
+"use strict";
 
-var LiquidMode = require("./liquid").Mode;
+var oop = require("../lib/oop");
-var assert = require("../test/assertions");
+var TextMode = require("./text").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var SchemeHighlightRules = require("./scheme_highlight_rules").SchemeHighlightRules;
 
-module.exports = {
+var Mode = function() {
-
+    var highlighter = new SchemeHighlightRules();
-    name: "Liquid Tokenizer",
-
-    setUp : function() {
-        this.tokenizer = new LiquidMode().getTokenizer();
-    },
-
-    "test: tokenize tags" : function() {
-        var line = "for one in many";
-
-        var tokens = this.tokenizer.getLineTokens(line, "liquid_start").tokens;
-
-        assert.equal(7, tokens.length);
-        assert.equal("keyword", tokens[0].type);
-        assert.equal("text", tokens[1].type);
-        assert.equal("identifier", tokens[2].type);
-        assert.equal("text", tokens[3].type);
-        assert.equal("keyword", tokens[4].type);
-        assert.equal("text", tokens[5].type);
-        assert.equal("identifier", tokens[6].type);
-    },
-
-    "test: tokenize parens" : function() {
-        var line = "[{( )}]";
-
-        var tokens = this.tokenizer.getLineTokens(line, "liquid_start").tokens;
-
-        assert.equal(3, tokens.length);
-        assert.equal("paren.lparen", tokens[0].type);
-        assert.equal("text", tokens[1].type);
-        assert.equal("paren.rparen", tokens[2].type);
-    }
     
+    this.$tokenizer = new Tokenizer(highlighter.getRules());
 };
+oop.inherits(Mode, TextMode);
 
+(function() {
+    // Extra logic goes here. 
+}).call(Mode.prototype);
+
+exports.Mode = Mode;
 });
-
-if (typeof module !== "undefined" && module === require.main) {
-    require("asyncjs").test.testcase(module.exports).exec()
-}

lib/ace/mode/scheme_highlight_rules.js

@@ -0,0 +1,123 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2012, Ajax.org B.V.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *
+ * Contributor(s):
+ * 
+ * NalaGinrut@gmail.com
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var SchemeHighlightRules = function() {
+    var keywordControl = "case|do|let|loop|if|else|when";
+    var keywordOperator = "eq?|eqv?|equal?|and|or|not|null?";
+    var constantLanguage = "#t|#f";
+    var supportFunctions = "cons|car|cdr|cond|lambda|lambda*|syntax-rules|format|set!|quote|eval|append|list|list?|member?|load";
+
+    var keywordMapper = this.createKeywordMapper({
+        "keyword.control": keywordControl,
+        "keyword.operator": keywordOperator,
+        "constant.language": constantLanguage,
+        "support.function": supportFunctions
+    }, "identifier", true);
+
+    // regexp must not have capturing parentheses. Use (?:) instead.
+    // regexps are ordered -> the first match is used
+
+    this.$rules = 
+        {
+    "start": [
+        {
+            token : "comment",
+            regex : ";.*$"
+        },
+        {
+            "token": ["storage.type.function-type.scheme", "text", "entity.name.function.scheme"],
+            "regex": "(?:\\b(?:(define|define-syntax|define-macro))\\b)(\\s+)((?:\\w|\\-|\\!|\\?)*)"
+        },
+        {
+            "token": "punctuation.definition.constant.character.scheme",
+            "regex": "#:\\S+"
+        },
+        {
+            "token": ["punctuation.definition.variable.scheme", "variable.other.global.scheme", "punctuation.definition.variable.scheme"],
+            "regex": "(\\*)(\\S*)(\\*)"
+        },
+        {
+            "token" : "constant.numeric", // hex
+            "regex" : "#[xXoObB][0-9a-fA-F]+"
+        }, 
+        {
+            "token" : "constant.numeric", // float
+            "regex" : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?"
+        },
+        {
+                "token" : keywordMapper,
+                "regex" : "[a-zA-Z_#][a-zA-Z0-9_\\-\\?\\!\\*]*"
+        },
+        {
+            "token" : "string",
+            "regex" : '"(?=.)',
+            "next"  : "qqstring"
+        }
+    ],
+    "qqstring": [
+        {
+            "token": "constant.character.escape.scheme",
+            "regex": "\\\\."
+        },
+        {
+            "token" : "string",
+            "regex" : '[^"\\\\]+',
+            "merge" : true
+        }, {
+            "token" : "string",
+            "regex" : "\\\\$",
+            "next"  : "qqstring",
+            "merge" : true
+        }, {
+            "token" : "string",
+            "regex" : '"|$',
+            "next"  : "start",
+            "merge" : true
+        }
+    ]
+}
+
+};
+
+oop.inherits(SchemeHighlightRules, TextHighlightRules);
+
+exports.SchemeHighlightRules = SchemeHighlightRules;
+});

lib/ace/mode/stylus_highlight_rules.js

@@ -22,7 +22,7 @@ var StylusHighlightRules = function() {
     }, "text", true);
 
     this.$rules = {
-    "start": [
+    start: [
         {
             token : "comment",
             regex : /\/\/.*$/
@@ -33,24 +33,24 @@ var StylusHighlightRules = function() {
             next : "comment"
         },
         {
-            "token": ["entity.name.function.stylus", "text"],
+            token: ["entity.name.function.stylus", "text"],
-            "regex": "^([-a-zA-Z_][-\\w]*)?(\\()"
+            regex: "^([-a-zA-Z_][-\\w]*)?(\\()"
         },
         {
-            "token": ["entity.other.attribute-name.class.stylus"],
+            token: ["entity.other.attribute-name.class.stylus"],
-            "regex": "\\.-?[_a-zA-Z]+[_a-zA-Z0-9-]*"
+            regex: "\\.-?[_a-zA-Z]+[_a-zA-Z0-9-]*"
         },
         {
-            "token": ["entity.language.stylus"],
+            token: ["entity.language.stylus"],
-            "regex": "^ *&"
+            regex: "^ *&"
         },
         {
-            "token": ["variable.language.stylus"],
+            token: ["variable.language.stylus"],
-            "regex": "(arguments)"
+            regex: "(arguments)"
         },
         {
-            "token": ["keyword.stylus"],
+            token: ["keyword.stylus"],
-            "regex": "@[-\\w]+"
+            regex: "@[-\\w]+"
         },
         {
             token : ["punctuation", "entity.other.attribute-name.pseudo-element.css"],
@@ -60,8 +60,8 @@ var StylusHighlightRules = function() {
             regex : CssHighlightRules.pseudoClasses
         }, 
         {
-            "token": ["entity.name.tag.stylus"],
+            token: ["entity.name.tag.stylus"],
-            "regex": "(?:\\b)(a|abbr|acronym|address|area|article|aside|audio|b|base|big|blockquote|body|br|button|canvas|caption|cite|code|col|colgroup|datalist|dd|del|details|dfn|dialog|div|dl|dt|em|eventsource|fieldset|figure|figcaption|footer|form|frame|frameset|(?:h[1-6])|head|header|hgroup|hr|html|i|iframe|img|input|ins|kbd|label|legend|li|link|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|pre|progress|q|samp|script|section|select|small|span|strike|strong|style|sub|summary|sup|table|tbody|td|textarea|tfoot|th|thead|time|title|tr|tt|ul|var|video)(?:\\b)"
+            regex: "(?:\\b)(a|abbr|acronym|address|area|article|aside|audio|b|base|big|blockquote|body|br|button|canvas|caption|cite|code|col|colgroup|datalist|dd|del|details|dfn|dialog|div|dl|dt|em|eventsource|fieldset|figure|figcaption|footer|form|frame|frameset|(?:h[1-6])|head|header|hgroup|hr|html|i|iframe|img|input|ins|kbd|label|legend|li|link|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|pre|progress|q|samp|script|section|select|small|span|strike|strong|style|sub|summary|sup|table|tbody|td|textarea|tfoot|th|thead|time|title|tr|tt|ul|var|video)(?:\\b)"
         },
         {
             token : "constant.numeric",  // hex6 color
@@ -72,24 +72,24 @@ var StylusHighlightRules = function() {
             regex : "#[a-f0-9]{3}"
         }, 
         {
-            "token": ["punctuation.definition.entity.stylus", "entity.other.attribute-name.id.stylus"],
+            token: ["punctuation.definition.entity.stylus", "entity.other.attribute-name.id.stylus"],
-            "regex": "(#)([a-zA-Z][a-zA-Z0-9_-]*)"
+            regex: "(#)([a-zA-Z][a-zA-Z0-9_-]*)"
         },
         {
-            "token": "meta.vendor-prefix.stylus",
+            token: "meta.vendor-prefix.stylus",
-            "regex": "-webkit-|-moz\\-|-ms-|-o-"
+            regex: "-webkit-|-moz\\-|-ms-|-o-"
         },
         {
-            "token": "keyword.control.stylus",
+            token: "keyword.control.stylus",
-            "regex": "(?:!important|for|in|return|true|false|null|if|else|unless|return)\\b"
+            regex: "(?:!important|for|in|return|true|false|null|if|else|unless|return)\\b"
         },
         {
-            "token": "keyword.operator.stylus",
+            token: "keyword.operator.stylus",
-            "regex": "!|~|\\+|-|(?:\\*)?\\*|\\/|%|(?:\\.)\\.\\.|<|>|(?:=|:|\\?|\\+|-|\\*|\\/|%|<|>)?=|!="
+            regex: "!|~|\\+|-|(?:\\*)?\\*|\\/|%|(?:\\.)\\.\\.|<|>|(?:=|:|\\?|\\+|-|\\*|\\/|%|<|>)?=|!="
         },
         {
-            "token": "keyword.operator.stylus",
+            token: "keyword.operator.stylus",
-            "regex": "(?:in|is(?:nt)?|not)\\b"
+            regex: "(?:in|is(?:nt)?|not)\\b"
         },
         {
             token : "string",

lib/ace/mode/tcl_highlight_rules.js

@@ -66,7 +66,7 @@ var TclHighlightRules = function() {
                 regex : '[ ]*["]',
                 next  : "qqstring"
             }, {
-                token : "variable.instancce", // variable xotcl with braces
+                token : "variable.instance", // variable xotcl with braces
                 regex : "[$]",
                 next  : "variable"
             }, {
@@ -103,7 +103,7 @@ var TclHighlightRules = function() {
                 token : "string", // single line
                 regex : '[ ]*["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
             }, {
-                token : "variable.instancce", // variable xotcl with braces
+                token : "variable.instance", // variable xotcl with braces
                 regex : "[$]",
                 next  : "variable"
             }, {
@@ -118,6 +118,9 @@ var TclHighlightRules = function() {
                 token : "support.function",
                 regex : "(?:[:][:])",
                 next  : "commandItem"
+            }, {
+                token : "paren.rparen",
+                regex : "[\\])}]"
             }, {
                 token : "support.function",
                 regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|{\\*}|;|::"

lib/ace/mode/text_highlight_rules.js

@@ -54,7 +54,7 @@ var TextHighlightRules = function() {
     this.addRules = function(rules, prefix) {
         for (var key in rules) {
             var state = rules[key];
-            for (var i=0; i<state.length; i++) {
+            for (var i = 0; i < state.length; i++) {
                 var rule = state[i];
                 if (rule.next) {
                     rule.next = prefix + rule.next;
@@ -94,7 +94,38 @@ var TextHighlightRules = function() {
 
     this.getEmbeds = function() {
         return this.$embeds;
-    }
+    };
+    
+    this.normalizeRules = function() {
+        var id = 0;
+        for (var key in this.$rules) {
+            var state = this.$rules[key];
+            for (var i = 0; i < state.length; i++) {
+                var rule = state[i];
+                if (rule.next && Array.isArray(rule.next)) {
+                    var stateName = rule.stateName || ("state" + id++);
+                    this.$rules[stateName] = rule.next;
+                    rule.next = stateName;
+                }
+                if (rule.rules) {
+                     for (var r in rule.rules) {
+                        if (this.$rules[r]) {
+                            if (this.$rules[r].push)
+                                this.$rules[r].push.apply(this.$rules[r], rule.rules[r]);
+                        } else {
+                            this.$rules[r] = rule.rules[r];
+                        }
+                    }
+                }
+                if (rule.include || typeof rule == "string") {
+                    var args = [i, 1].concat(this.$rules[rule.include || rule]);
+                    if (rule.noEscape)
+                        args = args.filter(function(x) {return !x.next;});
+                    state.splice.apply(state, args);
+                }
+            }
+        }
+    };
 
     this.createKeywordMapper = function(map, defaultToken, ignoreCase, splitChar) {
         var keywords = Object.create(null);

lib/ace/mode/tm_snippet.js

@@ -0,0 +1,86 @@
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextMode = require("./text").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var SnippetHighlightRules = function() {
+
+    var builtins = "SELECTION|CURRENT_WORD|SELECTED_TEXT|CURRENT_LINE|LINE_INDEX|" +
+        "LINE_NUMBER|SOFT_TABS|TAB_SIZE|FILENAME|FILEPATH|FULLNAME";
+
+    this.$rules = {
+        "start" : [
+            {token:"constant.language.escape", regex: /\\[\$}`\\]/},
+            {token:"keyword", regex: "\\$(?:TM_)?(?:" + builtins + ")\\b"},
+            {token:"variable", regex: "\\$\\w+"},
+            {token: function(value, state, stack) {
+                if (stack[1])
+                    stack[1]++;
+                else
+                    stack.unshift("start", 1);
+                return this.tokenName;
+            }, tokenName: "markup.list", regex: "\\${", next: "varDecl"},
+            {token: function(value, state, stack) {
+                if (!stack[1])
+                    return "text";
+                stack[1]--;
+                if (!stack[1])
+                    stack.splice(0,2);
+                return this.tokenName;
+            }, tokenName: "markup.list", regex: "}"},
+            {token: "doc,comment", regex:/^\${2}-{5,}$/}
+        ],
+        "varDecl" : [
+            {regex: /\d+\b/, token: "constant.numeric"},
+            {token:"keyword", regex: "(?:TM_)?(?:" + builtins + ")\\b"},
+            {token:"variable", regex: "\\w+"},
+            {regex: /:/, token: "punctuation.operator", next: "start"},
+            {regex: /\//, token: "string.regex", next: "regexp"},
+            {regex: "", next: "start"}
+        ],
+        "regexp" : [
+            {regex: /\\./, token: "escape"},
+            {regex: /\[/, token: "regex.start", next: "charClass"},
+            {regex: "/", token: "string.regex", next: "format"},
+            //{"default": "string.regex"},
+            {"token": "string.regex", regex:"."},
+        ],
+        charClass : [
+            {regex: "\\.", token: "escape"},
+            {regex: "\\]", token: "regex.end", next: "regexp"},
+            {"token": "string.regex", regex:"."},
+        ],
+        "format" : [
+            {regex: /\\[ulULE]/, token: "keyword"},
+            {regex: /\$\d+/, token: "variable"},
+            {regex: "/[gim]*:?", token: "string.regex", next: "start"},
+            // {"default": "string"},
+            {"token": "string", regex:"."},
+        ]
+    };
+};
+
+oop.inherits(SnippetHighlightRules, TextHighlightRules);
+
+exports.SnippetHighlightRules = SnippetHighlightRules;
+
+
+var Mode = function() {
+    var highlighter = new SnippetHighlightRules();
+
+    this.$tokenizer = new Tokenizer(highlighter.getRules());
+};
+oop.inherits(Mode, TextMode);
+
+(function() {
+    this.getNextLineIndent = function(state, line, tab) {
+        return this.$getIndent(line);
+    };
+}).call(Mode.prototype);
+exports.Mode = Mode;
+
+
+});

lib/ace/mode/typescript_highlight_rules.js

@@ -50,39 +50,39 @@ var TypeScriptHighlightRules = function() {
     var tsRules =  [
         // Match stuff like: module name {...}
         {
-            "token": ["keyword.operator.ts", "text", "variable.parameter.function.ts", "text"],
+            token: ["keyword.operator.ts", "text", "variable.parameter.function.ts", "text"],
-            "regex": "\\b(module)(\\s*)([a-zA-Z0-9_?.$][\\w?.$]*)(\\s*\\{)"
+            regex: "\\b(module)(\\s*)([a-zA-Z0-9_?.$][\\w?.$]*)(\\s*\\{)"
         }, 
         // Match stuff like: super(argument, list)
         {
-            "token": ["storage.type.variable.ts", "text", "keyword.other.ts", "text"],
+            token: ["storage.type.variable.ts", "text", "keyword.other.ts", "text"],
-            "regex": "(super)(\\s*\\()([a-zA-Z0-9,_?.$\\s]+\\s*)(\\))"
+            regex: "(super)(\\s*\\()([a-zA-Z0-9,_?.$\\s]+\\s*)(\\))"
         },
         // Match stuff like: function() {...}
         {
-            "token": ["entity.name.function.ts","paren.lparen", "paren.rparen"],
+            token: ["entity.name.function.ts","paren.lparen", "paren.rparen"],
-            "regex": "([a-zA-Z_?.$][\\w?.$]*)(\\()(\\))"
+            regex: "([a-zA-Z_?.$][\\w?.$]*)(\\()(\\))"
         },
         // Match stuff like: (function: return type)
         {
-            "token": ["variable.parameter.function.ts", "text", "variable.parameter.function.ts"],
+            token: ["variable.parameter.function.ts", "text", "variable.parameter.function.ts"],
-            "regex": "([a-zA-Z0-9_?.$][\\w?.$]*)(\\s*:\\s*)([a-zA-Z0-9_?.$][\\w?.$]*)"
+            regex: "([a-zA-Z0-9_?.$][\\w?.$]*)(\\s*:\\s*)([a-zA-Z0-9_?.$][\\w?.$]*)"
         },  
         {
-            "token": ["keyword.operator.ts"],
+            token: ["keyword.operator.ts"],
-            "regex": "(?:\\b(constructor|declare|interface|as|AS|public|private|class|extends|export|super)\\b)"
+            regex: "(?:\\b(constructor|declare|interface|as|AS|public|private|class|extends|export|super)\\b)"
         }, 
         {
-            "token": ["storage.type.variable.ts"],
+            token: ["storage.type.variable.ts"],
-            "regex": "(?:\\b(this\\.|string\\b|bool\\b|number)\\b)"
+            regex: "(?:\\b(this\\.|string\\b|bool\\b|number)\\b)"
         }, 
         {
-            "token": ["keyword.operator.ts", "storage.type.variable.ts", "keyword.operator.ts", "storage.type.variable.ts"],
+            token: ["keyword.operator.ts", "storage.type.variable.ts", "keyword.operator.ts", "storage.type.variable.ts"],
-            "regex": "(class)(\\s+[a-zA-Z0-9_?.$][\\w?.$]*\\s+)(extends)(\\s+[a-zA-Z0-9_?.$][\\w?.$]*\\s+)?"
+            regex: "(class)(\\s+[a-zA-Z0-9_?.$][\\w?.$]*\\s+)(extends)(\\s+[a-zA-Z0-9_?.$][\\w?.$]*\\s+)?"
         },
         {
-            "token": "keyword",
+            token: "keyword",
-            "regex": "(?:super|export|class|extends|import)\\b"
+            regex: "(?:super|export|class|extends|import)\\b"
         }
     ];
 

lib/ace/mode/xml_highlight_rules.js

@@ -47,8 +47,7 @@ var XmlHighlightRules = function() {
             {
                 token : "constant.character.entity", 
                 regex : "(?:&#[0-9]+;)|(?:&#x[0-9a-fA-F]+;)|(?:&[a-zA-Z0-9_:\\.-]+;)" 
-            },
+            }
-            {token : "text", regex : "[^<]+"}
         ],
         
         cdata : [

lib/ace/mode/xml_util.js

@@ -50,7 +50,7 @@ function multiLineString(quote, state) {
             token : "constant.language.escape",
             regex : "(?:&#[0-9]+;)|(?:&#x[0-9a-fA-F]+;)|(?:&[a-zA-Z0-9_:\\.-]+;)" 
         },
-        {token : "string", regex : '\\w+|.|\\s+'}
+        {defaultToken : "string"}
     ];
 }
 

lib/ace/mode/yaml_highlight_rules.js

@@ -95,12 +95,6 @@ var YamlHighlightRules = function() {
             }, {
                 token : "paren.rparen",
                 regex : "[\\])}]"
-            }, {
-                token : "text",
-                regex : "\\s+"
-            }, {
-                token : "text",
-                regex : "\\w+"
             }
         ],
         "qqstring" : [

lib/ace/tokenizer.js

@@ -50,133 +50,186 @@ define(function(require, exports, module) {
  **/
 var Tokenizer = function(rules, flag) {
     flag = flag ? "g" + flag : "g";
-    this.rules = rules;
+    this.states = rules;
 
     this.regExps = {};
     this.matchMappings = {};
-    for ( var key in this.rules) {
+    for (var key in this.states) {
-        var rule = this.rules[key];
+        var state = this.states[key];
-        var state = rule;
         var ruleRegExps = [];
         var matchTotal = 0;
-        var mapping = this.matchMappings[key] = {};
+        var mapping = this.matchMappings[key] = {defaultToken: "text"};
 
-        for ( var i = 0; i < state.length; i++) {
+        for (var i = 0; i < state.length; i++) {
-
+            var rule = state[i];
-            if (state[i].regex instanceof RegExp)
+            if (rule.defaultToken) {
-                state[i].regex = state[i].regex.toString().slice(1, -1);
+                mapping.defaultToken = rule.defaultToken;
+                continue;
+            }
+            if (rule.regex instanceof RegExp)
+                rule.regex = rule.regex.toString().slice(1, -1);
 
             // Count number of matching groups. 2 extra groups from the full match
             // And the catch-all on the end (used to force a match);
-            var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
+            var adjustedregex = rule.regex;
+            var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
+            if (Array.isArray(rule.token)) {
+                if (rule.token.length == 1) {
+                    rule.token = rule.token[0];
+                } else {
+                    rule.tokenArray = rule.token;
+                    rule.token = this.$arrayTokens;
+                }
+            }
 
-            // Replace any backreferences and offset appropriately.
+            if (matchcount > 1) {
-            var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
+                if (/\\\d/.test(rule.regex)) {
-                return "\\" + (parseInt(digit, 10) + matchTotal + 1);
+                    // Replace any backreferences and offset appropriately.
-            });
+                    adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) {
+                        return "\\" + (parseInt(digit, 10) + matchTotal + 1);
+                    });
+                } else {
+                    matchcount = 1;
+                    adjustedregex = this.removeCapturingGroups(rule.regex);
+                }
+                if (!rule.splitRegex)
+                    rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
+            }
 
-            if (matchcount > 1 && state[i].token.length !== matchcount-1)
+            mapping[matchTotal] = i;
-                throw new Error("For " + state[i].regex + " the matching groups (" +(matchcount-1) + ") and length of the token array (" + state[i].token.length + ") don't match (rule #" + i + " of state " + key + ")");
-
-            mapping[matchTotal] = {
-                rule: i,
-                len: matchcount
-            };
             matchTotal += matchcount;
 
             ruleRegExps.push(adjustedregex);
         }
 
-        this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
+        this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
     }
 };
 
 (function() {
+    this.$arrayTokens = function(str) {
+        if (!str)
+            return [];
+        var values = str.split(this.splitRegex)
+        var tokens = [];
+        var types = this.tokenArray;
+        if (types.length != values.length - 2) {
+            if (window.console)
+                console.error(types.length , values.length - 2, str, this.splitRegex);
+            return [{type: "error.invalid", value: str}];
+        }
+        for (var i = 0; i < types.length; i++) {
+            if (values[i + 1]) {
+                tokens[tokens.length] = {
+                    type: types[i],
+                    value: values[i + 1]
+                };
+            }
+        }
+        return tokens;
+    };
+    
+    this.removeCapturingGroups = function(src) {
+        var r = src.replace(
+            /\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
+            function(x, y) {return y ? "(?:" : x;}
+        );
+        return r;
+    };
+    
+    this.createSplitterRegexp = function(src, flag) {
+        src = src.replace(/\(\?=([^()]|\\.)*?\)$/, "");
+        return new RegExp(src, flag);
+    };
 
     /**
     * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
     * @returns {Object}
     **/
     this.getLineTokens = function(line, startState) {
+        if (startState && typeof startState != "string") {
+            var stack = startState.slice(0);
+            startState = stack[0];
+        } else
+            var stack = [];
+
         var currentState = startState || "start";
-        var state = this.rules[currentState];
+        var state = this.states[currentState];
         var mapping = this.matchMappings[currentState];
         var re = this.regExps[currentState];
         re.lastIndex = 0;
 
         var match, tokens = [];
-
         var lastIndex = 0;
 
-        var token = {
+        var token = {type: null, value: ""};
-            type: null,
-            value: ""
-        };
 
         while (match = re.exec(line)) {
-            var type = "text";
+            var type = mapping.defaultToken;
             var rule = null;
-            var value = [match[0]];
+            var value = match[0];
+            var index = re.lastIndex;
+
+            if (index - value.length > lastIndex) {
+                var skipped = line.substring(lastIndex, index - value.length);
+                if (token.type == type) {
+                    token.value += skipped;
+                } else {
+                    if (token.type)
+                        tokens.push(token);
+                    token = {type: type, value: skipped};
+                }
+            }
 
             for (var i = 0; i < match.length-2; i++) {
                 if (match[i + 1] === undefined)
                     continue;
 
-                rule = state[mapping[i].rule];
+                rule = state[mapping[i]];
-
-                if (mapping[i].len > 1)
-                    value = match.slice(i+2, i+1+mapping[i].len);
 
                 // compute token type
-                if (typeof rule.token == "function")
+                type = typeof rule.token == "function"
-                    type = rule.token.apply(this, value);
+                    ? rule.token(value, currentState, stack)
-                else
+                    : rule.token;
-                    type = rule.token;
 
                 if (rule.next) {
                     currentState = rule.next;
-                    state = this.rules[currentState];
+                    state = this.states[currentState];
-                    mapping = this.matchMappings[currentState];
+                    if (!state) {
-                    lastIndex = re.lastIndex;
+                        window.console && console.error && console.error(currentState, "doesn't exist");
-
+                        currentState = "start";
-                    re = this.regExps[currentState];
+                        state = this.states[currentState];
-
-                    if (re === undefined) {
-                         throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
                     }
-
+                    mapping = this.matchMappings[currentState];
-                    re.lastIndex = lastIndex;
+                    lastIndex = index;
+                    re = this.regExps[currentState];
+                    re.lastIndex = index;
                 }
                 break;
             }
 
-            if (value[0]) {
+            if (value) {
                 if (typeof type == "string") {
-                    value = [value.join("")];
+                    if ((!rule || rule.merge !== false) && token.type === type) {
-                    type = [type];
+                        token.value += value;
-                }
-                for (var i = 0; i < value.length; i++) {
-                    if (!value[i])
-                        continue;
-
-                    if ((!rule || rule.merge !== false) && token.type === type[i]) {
-                        token.value += value[i];
                     } else {
                         if (token.type)
                             tokens.push(token);
-
+                        token = {type: type, value: value};
-                        token = {
-                            type: type[i],
-                            value: value[i]
-                        };
                     }
+                } else {
+                    if (token.type)
+                        tokens.push(token);
+                    token = {type: null, value: ""};
+                    for (var i = 0; i < type.length; i++)
+                        tokens.push(type[i]);
                 }
             }
 
             if (lastIndex == line.length)
                 break;
 
-            lastIndex = re.lastIndex;
+            lastIndex = index;
         }
 
         if (token.type)
@@ -184,7 +237,7 @@ var Tokenizer = function(rules, flag) {
 
         return {
             tokens : tokens,
-            state : currentState
+            state : stack.length ? stack : currentState
         };
     };
 

lib/ace/tokenizer_dev.js

@@ -37,51 +37,111 @@ define(function(require, exports, module) {
  **/
 var Tokenizer = function(rules, flag) {
     flag = flag ? "g" + flag : "g";
-    this.rules = rules;
+    this.states = rules;
 
     this.regExps = {};
     this.matchMappings = {};
-    for ( var key in this.rules) {
+    for (var key in this.states) {
-        var rule = this.rules[key];
+        var state = this.states[key];
-        var state = rule;
         var ruleRegExps = [];
         var matchTotal = 0;
-        var mapping = this.matchMappings[key] = {};
+        var mapping = this.matchMappings[key] = {defaultToken: "default.text"};
 
-        for ( var i = 0; i < state.length; i++) {
+        for (var i = 0; i < state.length; i++) {
-
+            var rule = state[i];
-            if (state[i].regex instanceof RegExp)
+            if (rule.defaultToken) {
-                state[i].regex = state[i].regex.toString().slice(1, -1);
+                mapping.defaultToken = rule.defaultToken;
+                continue;
+            }
+            if (rule.regex instanceof RegExp)
+                rule.regex = rule.regex.toString().slice(1, -1);
 
             // Count number of matching groups. 2 extra groups from the full match
             // And the catch-all on the end (used to force a match);
-            var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
+            var adjustedregex = rule.regex;
+            var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
+            if (Array.isArray(rule.token)) {
+                if (rule.token.length == 1) {
+                    rule.token = rule.token[0];
+                } else {
+                    rule.tokenArray = rule.token;
+                    rule.token = this.$arrayTokens;
+                }
+            }
 
-            // Replace any backreferences and offset appropriately.
+            if (matchcount > 1) {
-            var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
+                if (/\\\d/.test(rule.regex)) {
-                return "\\" + (parseInt(digit, 10) + matchTotal + 1);
+                    // Replace any backreferences and offset appropriately.
-            });
+                    adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) {
+                        return "\\" + (parseInt(digit, 10) + matchTotal + 1);
+                    });
+                } else {
+                    matchcount = 1;
+                    adjustedregex = this.removeCapturingGroups(rule.regex);
+                }
+                if (!rule.splitRegex)
+                    rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
+            }
 
-            if (matchcount > 1 && state[i].token.length !== matchcount-1)
+            mapping[matchTotal] = i;
-                throw new Error("For " + state[i].regex + " the matching groups and length of the token array don't match (rule #" + i + " of state " + key + ")");
-
-            mapping[matchTotal] = {
-                rule: i,
-                len: matchcount
-            };
             matchTotal += matchcount;
 
             ruleRegExps.push(adjustedregex);
         }
 
-        this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
+        this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
     }
 };
 
 (function() {
+    this.$arrayTokens = function(str) {
+        if (!str)
+            return [];
+        var values = str.split(this.splitRegex)
+        var tokens = [];
+        var types = this.tokenArray;
+        if (types.length != values.length - 2) {
+            if (window.console)
+                console.error(types.length , values.length - 2, str, this.splitRegex);
+            return [{type: "error.invalid", value: str}];
+        }
+        for (var i = 0; i < types.length; i++) {
+            if (values[i + 1]) {
+                tokens[tokens.length] = {
+                    type: types[i],
+                    value: values[i + 1]
+                };
+            }
+        }
+        return tokens;
+    };
+    
+    this.removeCapturingGroups = function(src) {
+        var r = src.replace(
+            /\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
+            function(x, y) {return y ? "(?:" : x;}
+        );
+        return r;
+    };
+    
+    this.createSplitterRegexp = function(src, flag) {
+        src = src.replace(/\(\?=([^()]|\\.)*?\)$/, "");
+        return new RegExp(src, flag);
+    };
+
+    /**
+    * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
+    * @returns {Object}
+    **/
     this.getLineTokens = function(line, startState) {
+        if (startState && typeof startState != "string") {
+            var stack = startState.slice(0);
+            startState = stack[0];
+        } else
+            var stack = [];
+
         var currentState = startState || "start";
-        var state = this.rules[currentState];
+        var state = this.states[currentState];
         var mapping = this.matchMappings[currentState];
         var re = this.regExps[currentState];
         re.lastIndex = 0;
@@ -110,92 +170,85 @@ var Tokenizer = function(rules, flag) {
         var maxRecur = 10000;
         
         while (match = re.exec(line)) {
-            var type = "default.text";
+            var type = mapping.defaultToken;
             var rule = null;
-            var value = [match[0]];
+            var value = match[0];
+            var index = re.lastIndex;
+
+            if (index - value.length > lastIndex) {
+                var skipped = line.substring(lastIndex, index - value.length);
+                if (token.type == type) {
+                    token.value += skipped;
+                } else {
+                    if (token.type)
+                        tokens.push(token);
+                    token = {type: type, value: skipped};
+                }
+            }
 
             for (var i = 0; i < match.length-2; i++) {
                 if (match[i + 1] === undefined)
                     continue;
                 
                 if (!maxRecur--) {
-                    throw "infinite" + mapping[i].rule + currentState
+                    throw "infinite" + state[mapping[i]] + currentState
                 }
 
-                rule = state[mapping[i].rule];
+                rule = state[mapping[i]];
-
-                if (mapping[i].len > 1)
-                    value = match.slice(i+2, i+1+mapping[i].len);
 
                 // compute token type
-                if (typeof rule.token == "function")
+                type = typeof rule.token == "function"
-                    type = rule.token.apply(this, value);
+                    ? rule.token(value, currentState, stack)
-                else
+                    : rule.token;
-                    type = rule.token;
 
                 if (rule.next) {
                     currentState = rule.next;
-                    state = this.rules[currentState];
+                    state = this.states[currentState];
-                    mapping = this.matchMappings[currentState];
+                    if (!state) {
-                    lastIndex = re.lastIndex;
+                        window.console && console.error && console.error(currentState, "doesn't exist");
-
+                        currentState = "start";
-                    re = this.regExps[currentState];
+                        state = this.states[currentState];
-
-                    if (re === undefined) {
-                         throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
                     }
-
+                    mapping = this.matchMappings[currentState];
-                    re.lastIndex = lastIndex;
+                    lastIndex = index;
+                    re = this.regExps[currentState];
+                    re.lastIndex = index;
 
                     onStateChange();
                 }
                 break;
             }
 
-            if (value[0]) {
+            if (value) {
                 if (typeof type == "string") {
-                    value = [value.join("")];
+                    if ((!rule || rule.merge !== false) && token.type === type) {
-                    type = [type];
+                        token.value += value;
-                }
-                for (var i = 0; i < value.length; i++) {
-                    if (!value[i])
-                        continue;
-
-                    var mergeable = (!rule || rule.merge || type[i] === "text") && token.type === type[i];
-                    
-                    if (false && mergeable) {
-                        token.value += value[i];
                     } else {
-                        if (token.type) {
+                        if (token.type)
-                            token.stateTransitions = stateTransitions;
                             tokens.push(token);
-                            initState()
+                        token = {type: type, value: value};
-                        }
-                        
-                        token = {
-                            type: type[i],
-                            value: value[i],
-                            state: currentState,
-                            mergeable: mergeable
-                        };
                     }
+                } else {
+                    if (token.type)
+                        tokens.push(token);
+                    token = {type: null, value: ""};
+                    for (var i = 0; i < type.length; i++)
+                        tokens.push(type[i]);
                 }
             }
 
             if (lastIndex == line.length)
                 break;
 
-            lastIndex = re.lastIndex;
+            lastIndex = index;
         }
 
-        if (token.type) {
+        if (token.type)
-            token.stateTransitions = stateTransitions;
             tokens.push(token);
-        }
 
         return {
             tokens : tokens,
-            state : currentState
+            state : stack.length ? stack : currentState
         };
     };
 

tool/mode_creator.js

@@ -118,7 +118,7 @@ function run() {
     var path = "ace/mode/new";
     var deps = getDeps(src, path);
     src = src.replace("define(", 'define("' + path +'", ["require","exports","module",' + deps +'],');
-    src += ';require(["ace/mode/new"], continueRun, function(e){console.log(e);require.undef("ace/mode/new")})';
+    src += ';require(["ace/mode/new"], continueRun, function(e){console.log(e);window.require.undef("ace/mode/new")})';
     try {
         eval(src);
     } catch(e) {