Merge pull request #1276 from ajaxorg/higlighting/onMatch

restore old behavior of token and add onMatch. fixes #1269

lib/ace/mode/_test/tokens_jade.json

@@ -27,28 +27,29 @@
 ],[
    "start"
 ],[
-   "comment_block",
+   ["comment_block",0,"start"],
-  ["punctuation.section.comment","// "]
+  ["comment","// "]
 ],[
-   "comment_block",
+   ["comment_block",0,"start"],
-  ["comment.block.jade","  here it is. a block comment!"]
+  ["comment","  here it is. a block comment!"]
 ],[
-   "comment_block",
+   ["comment_block",0,"start"],
-  ["comment.block.jade"," and another row!"]
+  ["comment"," and another row!"]
 ],[
    "start",
-  ["text","but not here."]
+  ["meta.tag.any.jade","but"],
+  ["text"," not here."]
 ],[
    "start"
 ],[
-   "comment_block",
+   ["comment_block",5,"start"],
-  ["punctuation.section.comment","     // "]
+  ["comment","     // "]
 ],[
-   "comment_block",
+   ["comment_block",5,"start"],
-  ["comment.block.jade","        a far spaced"]
+  ["comment","        a far spaced"]
 ],[
-   "comment_block",
+   "start",
-  ["comment.block.jade","    should be lack of block"]
+  ["text","    should be lack of block"]
 ],[
    "start"
 ],[

lib/ace/mode/_test/tokens_pascal.json

@@ -0,0 +1,297 @@
+[[
+   "punctuation.definition.comment.pascal2",
+  ["punctuation.definition.comment.pascal","(*"],
+  ["comment.block.pascal.one","****************************************************************************"]
+],[
+   "punctuation.definition.comment.pascal2",
+  ["comment.block.pascal.one"," * A simple bubble sort program.  Reads integers, one per line, and prints   *"]
+],[
+   "punctuation.definition.comment.pascal2",
+  ["comment.block.pascal.one"," * them out in sorted order.  Blows up if there are more than 49.            *"]
+],[
+   "start",
+  ["comment.block.pascal.one"," ****************************************************************************"],
+  ["punctuation.definition.comment.pascal","*)"]
+],[
+   "start",
+  ["keyword.control.pascal","PROGRAM"],
+  ["text"," Sort(input"],
+  ["keyword.operator",","],
+  ["text"," output)"],
+  ["keyword.operator",";"]
+],[
+   "start",
+  ["text","    "],
+  ["keyword.control.pascal","CONST"]
+],[
+   "start",
+  ["text","        "],
+  ["punctuation.definition.comment.pascal","(*"],
+  ["comment.block.pascal.one"," Max array size. "],
+  ["punctuation.definition.comment.pascal","*)"]
+],[
+   "start",
+  ["text","        MaxElts "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["constant.numeric.pascal","50"],
+  ["keyword.operator",";"]
+],[
+   "start",
+  ["text","    "],
+  ["keyword.control.pascal","TYPE"],
+  ["text"," "]
+],[
+   "start",
+  ["text","        "],
+  ["punctuation.definition.comment.pascal","(*"],
+  ["comment.block.pascal.one"," Type of the element array. "],
+  ["punctuation.definition.comment.pascal","*)"]
+],[
+   "start",
+  ["text","        IntArrType "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["keyword.control.pascal","ARRAY"],
+  ["text"," ["],
+  ["constant.numeric.pascal","1"],
+  ["text","..MaxElts] "],
+  ["keyword.control.pascal","OF"],
+  ["text"," Integer"],
+  ["keyword.operator",";"]
+],[
+   "start"
+],[
+   "start",
+  ["text","    "],
+  ["keyword.control.pascal","VAR"]
+],[
+   "start",
+  ["text","        "],
+  ["punctuation.definition.comment.pascal","(*"],
+  ["comment.block.pascal.one"," Indexes, exchange temp, array size. "],
+  ["punctuation.definition.comment.pascal","*)"]
+],[
+   "start",
+  ["text","        i"],
+  ["keyword.operator",","],
+  ["text"," j"],
+  ["keyword.operator",","],
+  ["text"," tmp"],
+  ["keyword.operator",","],
+  ["text"," size: integer"],
+  ["keyword.operator",";"]
+],[
+   "start"
+],[
+   "start",
+  ["text","        "],
+  ["punctuation.definition.comment.pascal","(*"],
+  ["comment.block.pascal.one"," Array of ints "],
+  ["punctuation.definition.comment.pascal","*)"]
+],[
+   "start",
+  ["text","        arr: IntArrType"],
+  ["keyword.operator",";"]
+],[
+   "start"
+],[
+   "start",
+  ["text","    "],
+  ["punctuation.definition.comment.pascal","(*"],
+  ["comment.block.pascal.one"," Read in the integers. "],
+  ["punctuation.definition.comment.pascal","*)"]
+],[
+   "start",
+  ["text","    "],
+  ["variable.pascal","PROCEDURE"],
+  ["text"," "],
+  ["storage.type.function.pascal","ReadArr"],
+  ["text","("],
+  ["keyword.control.pascal","VAR"],
+  ["text"," size: Integer"],
+  ["keyword.operator",";"],
+  ["text"," "],
+  ["keyword.control.pascal","VAR"],
+  ["text"," a: IntArrType)"],
+  ["keyword.operator",";"],
+  ["text"," "]
+],[
+   "start",
+  ["text","        "],
+  ["keyword.control.pascal","BEGIN"]
+],[
+   "start",
+  ["text","            size "],
+  ["keyword.operator",":="],
+  ["text"," "],
+  ["constant.numeric.pascal","1"],
+  ["keyword.operator",";"]
+],[
+   "start",
+  ["text","            "],
+  ["keyword.control.pascal","WHILE"],
+  ["text"," "],
+  ["keyword.control.pascal","NOT"],
+  ["text"," eof "],
+  ["keyword.control.pascal","DO"],
+  ["text"," "],
+  ["keyword.control.pascal","BEGIN"]
+],[
+   "start",
+  ["text","                readln(a[size])"],
+  ["keyword.operator",";"]
+],[
+   "start",
+  ["text","                "],
+  ["keyword.control.pascal","IF"],
+  ["text"," "],
+  ["keyword.control.pascal","NOT"],
+  ["text"," eof "],
+  ["keyword.control.pascal","THEN"],
+  ["text"," "]
+],[
+   "start",
+  ["text","                    size "],
+  ["keyword.operator",":="],
+  ["text"," size "],
+  ["keyword.operator","+"],
+  ["text"," "],
+  ["constant.numeric.pascal","1"]
+],[
+   "start",
+  ["text","            "],
+  ["keyword.control.pascal","END"]
+],[
+   "start",
+  ["text","        "],
+  ["keyword.control.pascal","END"],
+  ["keyword.operator",";"]
+],[
+   "start"
+],[
+   "start",
+  ["text","    "],
+  ["keyword.control.pascal","BEGIN"]
+],[
+   "start",
+  ["text","        "],
+  ["punctuation.definition.comment.pascal","(*"],
+  ["comment.block.pascal.one"," Read "],
+  ["punctuation.definition.comment.pascal","*)"]
+],[
+   "start",
+  ["text","        ReadArr(size"],
+  ["keyword.operator",","],
+  ["text"," arr)"],
+  ["keyword.operator",";"]
+],[
+   "start"
+],[
+   "start",
+  ["text","        "],
+  ["punctuation.definition.comment.pascal","(*"],
+  ["comment.block.pascal.one"," Sort using bubble sort. "],
+  ["punctuation.definition.comment.pascal","*)"]
+],[
+   "start",
+  ["text","        "],
+  ["keyword.control.pascal","FOR"],
+  ["text"," i "],
+  ["keyword.operator",":="],
+  ["text"," size "],
+  ["keyword.operator","-"],
+  ["text"," "],
+  ["constant.numeric.pascal","1"],
+  ["text"," DOWNTO "],
+  ["constant.numeric.pascal","1"],
+  ["text"," "],
+  ["keyword.control.pascal","DO"]
+],[
+   "start",
+  ["text","            "],
+  ["keyword.control.pascal","FOR"],
+  ["text"," j "],
+  ["keyword.operator",":="],
+  ["text"," "],
+  ["constant.numeric.pascal","1"],
+  ["text"," "],
+  ["keyword.control.pascal","TO"],
+  ["text"," i "],
+  ["keyword.control.pascal","DO"],
+  ["text"," "]
+],[
+   "start",
+  ["text","                "],
+  ["keyword.control.pascal","IF"],
+  ["text"," arr[j] > arr[j "],
+  ["keyword.operator","+"],
+  ["text"," "],
+  ["constant.numeric.pascal","1"],
+  ["text","] "],
+  ["keyword.control.pascal","THEN"],
+  ["text"," "],
+  ["keyword.control.pascal","BEGIN"]
+],[
+   "start",
+  ["text","                    tmp "],
+  ["keyword.operator",":="],
+  ["text"," arr[j]"],
+  ["keyword.operator",";"]
+],[
+   "start",
+  ["text","                    arr[j] "],
+  ["keyword.operator",":="],
+  ["text"," arr[j "],
+  ["keyword.operator","+"],
+  ["text"," "],
+  ["constant.numeric.pascal","1"],
+  ["text","]"],
+  ["keyword.operator",";"]
+],[
+   "start",
+  ["text","                    arr[j "],
+  ["keyword.operator","+"],
+  ["text"," "],
+  ["constant.numeric.pascal","1"],
+  ["text","] "],
+  ["keyword.operator",":="],
+  ["text"," tmp"],
+  ["keyword.operator",";"]
+],[
+   "start",
+  ["text","                "],
+  ["keyword.control.pascal","END"],
+  ["keyword.operator",";"]
+],[
+   "start"
+],[
+   "start",
+  ["text","        "],
+  ["punctuation.definition.comment.pascal","(*"],
+  ["comment.block.pascal.one"," Print. "],
+  ["punctuation.definition.comment.pascal","*)"]
+],[
+   "start",
+  ["text","        "],
+  ["keyword.control.pascal","FOR"],
+  ["text"," i "],
+  ["keyword.operator",":="],
+  ["text"," "],
+  ["constant.numeric.pascal","1"],
+  ["text"," "],
+  ["keyword.control.pascal","TO"],
+  ["text"," size "],
+  ["keyword.control.pascal","DO"]
+],[
+   "start",
+  ["text","            writeln(arr[i])"]
+],[
+   "start",
+  ["text","    "],
+  ["keyword.control.pascal","END"],
+  ["text","."]
+],[
+   "start",
+  ["text","            "]
+]]

lib/ace/mode/_test/tokens_sass.json

@@ -0,0 +1,229 @@
+[[
+   "start",
+  ["comment","// sass ace mode;"]
+],[
+   "start"
+],[
+   "start",
+  ["keyword","@import"],
+  ["text"," "],
+  ["support.function","url("],
+  ["string","http://fonts.googleapis.com/css?family=Ace:700"],
+  ["support.function",")"]
+],[
+   "start"
+],[
+   "start",
+  ["variable.language","html"],
+  ["text",", "],
+  ["variable.language","body"]
+],[
+   "start",
+  ["support.type","  :background-color "],
+  ["constant.numeric","#ace"]
+],[
+   "start",
+  ["text","  "],
+  ["support.type","text-align"],
+  ["text",": "],
+  ["constant.language","center"]
+],[
+   "start",
+  ["text","  "],
+  ["support.type","height"],
+  ["text",": "],
+  ["constant.numeric","100%"]
+],[
+   ["comment",-1,2,"start"],
+  ["comment","  /*;*********;"]
+],[
+   ["comment",3,2,"start"],
+  ["comment","    ;comment  ;"]
+],[
+   ["comment",3,2,"start"],
+  ["comment","    ;*********;"]
+],[
+   "start"
+],[
+   "start",
+  ["variable.language",".toggle"]
+],[
+   "start",
+  ["text","  "],
+  ["variable","$size"],
+  ["text",": "],
+  ["constant.numeric","14px"]
+],[
+   "start"
+],[
+   "start",
+  ["support.type","  :background "],
+  ["support.function","url("],
+  ["string","http://subtlepatterns.com/patterns/dark_stripes.png"],
+  ["support.function",")"]
+],[
+   "start",
+  ["text","  "],
+  ["support.type","border-radius"],
+  ["text",": "],
+  ["constant.numeric","8px"]
+],[
+   "start",
+  ["text","  "],
+  ["support.type","height"],
+  ["text",": "],
+  ["variable","$size"]
+],[
+   "start"
+],[
+   "start",
+  ["text","  &"],
+  ["variable.language",":before"]
+],[
+   "start",
+  ["text","    "],
+  ["variable","$radius"],
+  ["text",": "],
+  ["variable","$size"],
+  ["text"," "],
+  ["keyword.operator","*"],
+  ["text"," "],
+  ["constant.numeric","0.845"]
+],[
+   "start",
+  ["text","    "],
+  ["variable","$glow"],
+  ["text",": "],
+  ["variable","$size"],
+  ["text"," "],
+  ["keyword.operator","*"],
+  ["text"," "],
+  ["constant.numeric","0.125"]
+],[
+   "start"
+],[
+   "start",
+  ["text","    "],
+  ["support.type","box-shadow"],
+  ["text",": "],
+  ["constant.numeric","0"],
+  ["text"," "],
+  ["constant.numeric","0"],
+  ["text"," "],
+  ["variable","$glow"],
+  ["text"," "],
+  ["variable","$glow"],
+  ["text"," / "],
+  ["constant.numeric","2"],
+  ["text"," "],
+  ["constant.numeric","#fff"]
+],[
+   "start",
+  ["text","    "],
+  ["support.type","border-radius"],
+  ["text",": "],
+  ["variable","$radius"]
+],[
+   "start",
+  ["text","    "]
+],[
+   "start",
+  ["text","    &"],
+  ["variable.language",":active"]
+],[
+   "start",
+  ["text","      ~ "],
+  ["variable.language",".button"]
+],[
+   "start",
+  ["text","        "],
+  ["support.type","box-shadow"],
+  ["text",": "],
+  ["constant.numeric","0"],
+  ["text"," "],
+  ["constant.numeric","15px"],
+  ["text"," "],
+  ["constant.numeric","25px"],
+  ["text"," "],
+  ["constant.numeric","-4px"],
+  ["text"," "],
+  ["support.function","rgba"],
+  ["paren.lparen","("],
+  ["constant.numeric","0"],
+  ["text",","],
+  ["constant.numeric","0"],
+  ["text",","],
+  ["constant.numeric","0"],
+  ["text",","],
+  ["constant.numeric","0.4"],
+  ["paren.rparen",")"],
+  ["text","      "]
+],[
+   "start",
+  ["text","      ~ "],
+  ["variable.language",".label"]
+],[
+   "start",
+  ["text","        "],
+  ["support.type","font-size"],
+  ["text",": "],
+  ["constant.numeric","40px"]
+],[
+   "start",
+  ["text","        "],
+  ["support.type","color"],
+  ["text",": "],
+  ["support.function","rgba"],
+  ["paren.lparen","("],
+  ["constant.numeric","0"],
+  ["text",","],
+  ["constant.numeric","0"],
+  ["text",","],
+  ["constant.numeric","0"],
+  ["text",","],
+  ["constant.numeric","0.45"],
+  ["paren.rparen",")"]
+],[
+   "start"
+],[
+   "start",
+  ["text","    &"],
+  ["variable.language",":checked"],
+  ["text","      "]
+],[
+   "start",
+  ["text","      ~ "],
+  ["variable.language",".button"]
+],[
+   "start",
+  ["text","        "],
+  ["support.type","box-shadow"],
+  ["text",": "],
+  ["constant.numeric","0"],
+  ["text"," "],
+  ["constant.numeric","15px"],
+  ["text"," "],
+  ["constant.numeric","25px"],
+  ["text"," "],
+  ["constant.numeric","-4px"],
+  ["text"," "],
+  ["constant.numeric","#ace"]
+],[
+   "start",
+  ["text","      ~ "],
+  ["variable.language",".label"]
+],[
+   "start",
+  ["text","        "],
+  ["support.type","font-size"],
+  ["text",": "],
+  ["constant.numeric","40px"]
+],[
+   "start",
+  ["text","        "],
+  ["support.type","color"],
+  ["text",": "],
+  ["constant.numeric","#c9c9c9"]
+],[
+   "start"
+]]

lib/ace/mode/_test/tokens_tmsnippet.json

@@ -0,0 +1,132 @@
+[[
+   "start",
+  ["doc,comment","$$------------------------------------"]
+],[
+   "start"
+],[
+   "start",
+  ["text","tabTrigger: t"]
+],[
+   "start"
+],[
+   "start",
+  ["text","name: Heading 3"]
+],[
+   "start"
+],[
+   "start",
+  ["text","scope: language"]
+],[
+   "start"
+],[
+   "start",
+  ["text","content: -----------------------------"]
+],[
+   "start"
+],[
+   "start",
+  ["text","\\begin{"],
+  ["markup.list","${"],
+  ["constant.numeric","1"],
+  ["punctuation.operator",":"],
+  ["text","documnet"],
+  ["markup.list","}"],
+  ["text","}"]
+],[
+   "start"
+],[
+   "start",
+  ["text","    "],
+  ["markup.list","${"],
+  ["constant.numeric","2"],
+  ["punctuation.operator",":"],
+  ["keyword","$TM_SELECTED_TEXT"],
+  ["text",":some latex"],
+  ["markup.list","}"]
+],[
+   "start"
+],[
+   "start",
+  ["text","    "],
+  ["markup.list","${"],
+  ["constant.numeric","3"],
+  ["punctuation.operator",":"],
+  ["keyword","$TM_SELECTED_TEXT"],
+  ["text","/a/b/c"],
+  ["markup.list","}"]
+],[
+   "start"
+],[
+   "start",
+  ["text","    "],
+  ["markup.list","${"],
+  ["constant.numeric","4"],
+  ["punctuation.operator",":"],
+  ["markup.list","${"],
+  ["keyword","TM_SELECTED_TEXT"],
+  ["string.regex","/(.)/"],
+  ["string","\\"],
+  ["keyword","\\u"],
+  ["variable","$1"],
+  ["string.regex","/g:"],
+  ["text","7"],
+  ["markup.list","}}"]
+],[
+   "start"
+],[
+   "start",
+  ["text","\\end{"],
+  ["variable","$1"],
+  ["text","}"]
+],[
+   "start"
+],[
+   "start",
+  ["variable","$0"],
+  ["constant.language.escape","\\\\"],
+  ["text","$$"]
+],[
+   "start"
+],[
+   "start",
+  ["doc,comment","$$------------------------------------"]
+],[
+   "start"
+],[
+   "start",
+  ["text","tabTrigger: ^3"]
+],[
+   "start"
+],[
+   "start",
+  ["text","name: Heading 3"]
+],[
+   "start"
+],[
+   "start",
+  ["text","scope: language"]
+],[
+   "start"
+],[
+   "start",
+  ["text","content: -----------------------------"]
+],[
+   "start"
+],[
+   "start"
+],[
+   "start"
+],[
+   "start",
+  ["markup.list","${"],
+  ["keyword","TM_CURRENT_LINE"],
+  ["string.regex","/./"],
+  ["string","^"],
+  ["string.regex","/g"],
+  ["markup.list","}"]
+],[
+   "start"
+],[
+   "start",
+  ["doc,comment","$$------------------------------------"]
+]]

lib/ace/mode/jade_highlight_rules.js

@@ -85,10 +85,11 @@ var JadeHighlightRules = function() {
             regex : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)"
         },
         {
-            token : function(space, text) {
+            onMatch: function(value, currentState, stack) {
-                return "punctuation.section.comment";
+                stack.unshift(this.next, value.length - 2, currentState);
+                return "comment";
             },
-            regex : "^((\\s*)\/\/)(?:\\s*$)",
+            regex: /^\s*\/\//,
             next: "comment_block"
         },
         mixin_embed("markdown", "markdown-"),
@@ -155,18 +156,18 @@ var JadeHighlightRules = function() {
         }
     ],
     "comment_block": [
-        {
+        {regex: /^\s*/, onMatch: function(value, currentState, stack) {
-            token: function(text) {
+            if (value.length <= stack[1]) {
+                stack.shift();
+                stack.shift();
+                this.next = stack.shift();
                 return "text";
-            },
+            } else {
-            regex: "^(\\1\\S|$)", 
+                this.next = "";
-            "captures": "1",
+                return "comment";
-            next: "start"
+            }
-        },
+        }, next: "start"},
-        {
+        {defaultToken: "comment"}
-            token: "comment.block.jade",
-            regex : ".+"
-        }
     ],
     /*
     

lib/ace/mode/javascript_highlight_rules.js

@@ -246,11 +246,11 @@ var JavaScriptHighlightRules = function() {
             }, {
                 // invalid operators
                 token : "invalid",
-                regex: /\{\d+,?(?:\d+)?}[+*]|[+*$^?][+*]|[$^][?]|\?{3,}/
+                regex: /\{\d+\b,?\d*\}[+*]|[+*$^?][+*]|[$^][?]|\?{3,}/
             }, {
                 // operators
                 token : "constant.language.escape",
-                regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[()$^+*?]/
+                regex: /\(\?[:=!]|\)|\{\d+\b,?\d*\}|[+*]\?|[()$^+*?]/
             }, {
                 token : "constant.language.delimiter",
                 regex: /\|/

lib/ace/mode/lua_highlight_rules.js

@@ -98,14 +98,14 @@ var LuaHighlightRules = function() {
     this.$rules = {
         "start" : [{
             stateName: "bracketedComment",
-            token : function(value, currentState, stack){
+            onMatch : function(value, currentState, stack){
                 stack.unshift(this.next, value.length, currentState);
                 return "comment";
             },
             regex : /\-\-\[=*\[/,
             next  : [
                 {
-                    token : function(value, currentState, stack) {
+                    onMatch : function(value, currentState, stack) {
                         if (value.length == stack[1]) {
                             stack.shift();
                             stack.shift();
@@ -129,14 +129,14 @@ var LuaHighlightRules = function() {
         },
         {
             stateName: "bracketedString",
-            token : function(value, currentState, stack){
+            onMatch : function(value, currentState, stack){
                 stack.unshift(this.next, value.length, currentState);
                 return "comment";
             },
             regex : /\[=*\[/,
             next  : [
                 {
-                    token : function(value, currentState, stack) {
+                    onMatch : function(value, currentState, stack) {
                         if (value.length == stack[1]) {
                             stack.shift();
                             stack.shift();

lib/ace/mode/php_highlight_rules.js

@@ -971,7 +971,7 @@ var PhpLangHighlightRules = function() {
                 // TODO: Unicode identifiers
                 regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
             }, {
-                token : function(value, currentSate, state) {
+                onMatch : function(value, currentSate, state) {
                     value = value.substr(3);
                     if (value[0] == "'" || value[0] == '"')
                         value = value.slice(1, -1);
@@ -996,7 +996,7 @@ var PhpLangHighlightRules = function() {
         ],
         "heredoc" : [
             {
-                token : function(value, currentSate, stack) {
+                onMatch : function(value, currentSate, stack) {
                     if (stack[1]  + ";" != value)
                         return "string";
                     stack.shift();

lib/ace/mode/r_highlight_rules.js

@@ -91,8 +91,7 @@ define(function(require, exports, module)
                regex : "`.*?`"
             },
             {
-               token : function(value)
+               onMatch : function(value) {
-               {
                   if (keywords[value])
                      return "keyword";
                   else if (buildinConstants[value])

lib/ace/mode/ruby_highlight_rules.js

@@ -173,7 +173,7 @@ var RubyHighlightRules = function() {
                 regex : "=>"
             }, {
                 stateName: "heredoc",
-                token : function(value, currentState, stack) {
+                onMatch : function(value, currentState, stack) {
                     var next = value[2] == '-' ? "indentedHeredoc" : "heredoc";
                     var tokens = value.split(this.splitRegex);
                     stack.push(next, tokens[3]);
@@ -187,7 +187,7 @@ var RubyHighlightRules = function() {
                 regex : "(<<-?)(['\"`]?)([\\w]+)(['\"`]?)",
                 rules: {
                     heredoc: [{
-                        token:  function(value, currentState, stack) {
+                        onMatch:  function(value, currentState, stack) {
                             if (value == stack[1]) {
                                 stack.shift();
                                 stack.shift();
@@ -202,7 +202,7 @@ var RubyHighlightRules = function() {
                         token: "string",
                         regex: "^ +"
                     }, {
-                        token:  function(value, currentState, stack) {
+                        onMatch:  function(value, currentState, stack) {
                             if (value == stack[1]) {
                                 stack.shift();
                                 stack.shift();

lib/ace/mode/sass_highlight_rules.js

@@ -40,8 +40,8 @@ var SassHighlightRules = function() {
     var start = this.$rules.start;
     if (start[1].token == "comment") {
         start.splice(1, 1, {
-            token: function(value, currentState, stack) {
+            onMatch: function(value, currentState, stack) {
-                stack.unshift(this.next, value.length - 2, currentState);
+                stack.unshift(this.next, -1, value.length - 2, currentState);
                 return "comment";
             },
             regex: /^\s*\/\*/,
@@ -55,14 +55,14 @@ var SassHighlightRules = function() {
         });
         
         this.$rules.comment = [
-            {regex: /^\s*/, token: function(value, currentState, stack) {
+            {regex: /^\s*/, onMatch: function(value, currentState, stack) {
-                if (value.length < stack[1]) {
+                if (stack[1] === -1)
-                    stack.shift();
+                    stack[1] = Math.max(stack[2], value.length - 1);
-                    stack.shift();
+                if (value.length <= stack[1]) {
+                    /*shift3x*/stack.shift();stack.shift();stack.shift();
                     this.next = stack.shift();
                     return "text";
                 } else {
-                    stack[1] = value.length;
                     this.next = "";
                     return "comment";
                 }

lib/ace/mode/tmsnippet.js

@@ -16,14 +16,14 @@ var SnippetHighlightRules = function() {
             {token:"constant.language.escape", regex: /\\[\$}`\\]/},
             {token:"keyword", regex: "\\$(?:TM_)?(?:" + builtins + ")\\b"},
             {token:"variable", regex: "\\$\\w+"},
-            {token: function(value, state, stack) {
+            {onMatch: function(value, state, stack) {
                 if (stack[1])
                     stack[1]++;
                 else
                     stack.unshift("start", 1);
                 return this.tokenName;
             }, tokenName: "markup.list", regex: "\\${", next: "varDecl"},
-            {token: function(value, state, stack) {
+            {onMatch: function(value, state, stack) {
                 if (!stack[1])
                     return "text";
                 stack[1]--;

lib/ace/tokenizer.js

@@ -79,10 +79,15 @@ var Tokenizer = function(rules) {
                     rule.token = rule.token[0];
                 } else {
                     rule.tokenArray = rule.token;
-                    rule.token = this.$arrayTokens;
+                    rule.onMatch = this.$arrayTokens;
                 }
+            } else if (typeof rule.token == "function" && !rule.onMatch) {
+                if (matchcount > 1)
+                    rule.onMatch = this.$applyToken;
+                else
+                    rule.onMatch = rule.token;
             }
-
+            
             if (matchcount > 1) {
                 if (/\\\d/.test(rule.regex)) {
                     // Replace any backreferences and offset appropriately.
@@ -101,6 +106,11 @@ var Tokenizer = function(rules) {
             matchTotal += matchcount;
 
             ruleRegExps.push(adjustedregex);
+            
+            // makes property access faster
+            if (!rule.onMatch)
+                rule.onMatch = null;
+            rule.__proto__ = null;
         }
 
         this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
@@ -108,6 +118,25 @@ var Tokenizer = function(rules) {
 };
 
 (function() {
+    this.$applyToken = function(str) {
+        var values = this.splitRegex.exec(str).slice(1);
+        var types = this.token.apply(this, values);
+        
+        // required for compatibility with old modes
+        if (typeof types === "string")
+            return [{type: types, value: str}];
+
+        var tokens = [];
+        for (var i = 0, l = types.length; i < l; i++) {
+            if (values[i])
+                tokens[tokens.length] = {
+                    type: types[i],
+                    value: values[i]
+                };
+        }
+        return tokens;
+    },
+
     this.$arrayTokens = function(str) {
         if (!str)
             return [];
@@ -119,13 +148,12 @@ var Tokenizer = function(rules) {
                 console.error(types , values, str, this.splitRegex, this);
             return [{type: "error.invalid", value: str}];
         }
-        for (var i = 0; i < types.length; i++) {
+        for (var i = 0, l = types.length; i < l; i++) {
-            if (values[i + 1]) {
+            if (values[i + 1])
                 tokens[tokens.length] = {
                     type: types[i],
                     value: values[i + 1]
                 };
-            }
         }
         return tokens;
     };
@@ -188,10 +216,10 @@ var Tokenizer = function(rules) {
 
                 rule = state[mapping[i]];
 
-                // compute token type
+                if (rule.onMatch)
-                type = typeof rule.token == "function"
+                    type = rule.onMatch(value, currentState, stack);
-                    ? rule.token(value, currentState, stack)
+                else
-                    : rule.token;
+                    type = rule.token;
 
                 if (rule.next) {
                     if (typeof rule.next == "string")