Merge pull request #1185 from ajaxorg/highlighting/2.0

Highlighting/2.0

demo/kitchen-sink/doclist.js

@@ -119,6 +119,7 @@ var docs = {
     "docs/tcl.tcl": "Tcl",
     "docs/tex.tex": "Tex",
     "docs/textile.textile": {name: "Textile", wrapped: true},
+    "docs/tmSnippet.tmSnippet": "tmSnippet",
     "docs/typescript.ts": "Typescript",
     "docs/vbscript.vbs": "VBScript",
     "docs/xml.xml": "XML",

demo/kitchen-sink/docs/ruby.rb

@@ -20,4 +20,16 @@ class Range
   end
 end
 
-{:id => 34, :sdfasdfasdf => "asdasdads"}
+{:id => 34, :key => "value"}
+
+
+    herDocs = [<<'FOO', <<BAR, <<-BAZ, <<-`EXEC`] #comment
+  FOO #{literal}
+FOO
+  BAR #{fact(10)}
+BAR
+  BAZ indented
+    BAZ
+        echo hi
+    EXEC
+puts herDocs

demo/kitchen-sink/docs/tmSnippet.tmSnippet

@@ -0,0 +1,19 @@
+$$------------------------------------
+tabTrigger: t
+name: Heading 3
+scope: language
+content: -----------------------------
+\begin{${1:documnet}}
+    ${2:$TM_SELECTED_TEXT:some latex}
+    ${3:$TM_SELECTED_TEXT/a/b/c}
+    ${4:${TM_SELECTED_TEXT/(.)/\\u$1/g:7}}
+\end{$1}
+$0\\$$
+$$------------------------------------
+tabTrigger: ^3
+name: Heading 3
+scope: language
+content: -----------------------------
+
+${TM_CURRENT_LINE/./^/g}
+$$------------------------------------

demo/kitchen-sink/modelist.js

@@ -91,6 +91,7 @@ var modesByName = {
     tex:        ["Tex"          , "tex"],
     text:       ["Text"         , "txt"],
     textile:    ["Textile"      , "textile"],
+    tm_snippet: ["tmSnippet"    , "tmSnippet"],
     typescript: ["Typescript"   , "typescript|ts|str"],
     vbscript:   ["VBScript"     , "vbs"],
     xml:        ["XML"          , "xml|rdf|rss|wsdl|xslt|atom|mathml|mml|xul|xbl"],

lib/ace/background_tokenizer.js

@@ -236,7 +236,7 @@ var BackgroundTokenizer = function(tokenizer, editor) {
             data.state = "start";
         }
 
-        if (this.states[row] !== data.state) {
+        if (this.states[row] + "" !== data.state + "") {
             this.states[row] = data.state;
             this.lines[row + 1] = null;
             if (this.currentLine > row + 1)

lib/ace/mode/_test/highlight_rules_test.js

@@ -112,7 +112,7 @@ function testMode(modeName, i) {
 }
 function testEqual(a) {
     var err;
-    if (a[0] !== a[1]) {
+    if (a[0] + "" !== a[1] + "") {
         console.log(a[0],a[1]);
         err = 1;
     }

lib/ace/mode/_test/text_markdown.txt

@@ -11,3 +11,9 @@ test: 6+ #s is not a valid header
 test: # followed be only space is not a valid header
 test: only space between #s is not a valid header
 #  #
+
+test links  [Cloud9 IDE](http://www.c9.io/) 
+* [demo](http://ajaxorg.github.com/ace/) 
+in lists
+
+in plain text <b>http://ace.ajaxorg.com<b>

lib/ace/mode/_test/tokens_c9search.json

@@ -10,55 +10,68 @@
 ],[
    "start",
   ["c9searchresults.constant.numeric","    1"],
-  ["c9searchresults.text",": var fs = require(\"fs\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var fs = require(\"fs\");"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t2"],
-  ["c9searchresults.text",": var argv = require('optimist').argv;"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var argv = require('optimist').argv;"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t3"],
-  ["c9searchresults.text",": var path = require(\"path\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var path = require(\"path\");"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t5"],
-  ["c9searchresults.text",": var clientExtensions = {};"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var clientExtensions = {};"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t6"],
-  ["c9searchresults.text",": var clientDirs = fs.readdirSync(__dirname + \"/../plugins-client\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var clientDirs = fs.readdirSync(__dirname + \"/../plugins-client\");"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t7"],
-  ["c9searchresults.text",": for (var i = 0; i < clientDirs.length; i++) {"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","for (var i = 0; i < clientDirs.length; i++) {"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t8"],
-  ["c9searchresults.text",":     var dir = clientDirs[i];"]
+  ["c9searchresults.text",":     "],
+  ["c9searchresults.text","var dir = clientDirs[i];"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t12"],
-  ["c9searchresults.text",":     var name = dir.split(\".\")[1];"]
+  ["c9searchresults.text",":     "],
+  ["c9searchresults.text","var name = dir.split(\".\")[1];"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t16"],
-  ["c9searchresults.text",": var projectDir = (argv.w && path.resolve(process.cwd(), argv.w)) || process.cwd();"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var projectDir = (argv.w && path.resolve(process.cwd(), argv.w)) || process.cwd();"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t17"],
-  ["c9searchresults.text",": var fsUrl = \"/workspace\";"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var fsUrl = \"/workspace\";"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t19"],
-  ["c9searchresults.text",": var port = argv.p || process.env.PORT || 3131;"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var port = argv.p || process.env.PORT || 3131;"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t20"],
-  ["c9searchresults.text",": var host = argv.l || \"localhost\";"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var host = argv.l || \"localhost\";"]
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t22"],
-  ["c9searchresults.text",": var config = {"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var config = {"]
 ],[
    "start"
 ],[
@@ -68,7 +81,8 @@
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t2"],
-  ["c9searchresults.text",": var config = require(\"./default\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var config = require(\"./default\");"]
 ],[
    "start"
 ],[
@@ -78,7 +92,8 @@
 ],[
    "start",
   ["c9searchresults.constant.numeric","\t1"],
-  ["c9searchresults.text",": var config = require(\"./default\");"]
+  ["c9searchresults.text",": "],
+  ["c9searchresults.text","var config = require(\"./default\");"]
 ],[
    "start"
 ],[

lib/ace/mode/_test/tokens_coffee.json

@@ -28,9 +28,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","args"],
+  ["variable.parameter","{args}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -40,9 +40,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","a1, a2"],
+  ["variable.parameter","{a1, a2}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -52,9 +52,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","@a1, a2"],
+  ["variable.parameter","{@a1, a2}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -62,11 +62,11 @@
   ["text","    "],
   ["entity.name.function","foo"],
   ["text"," "],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","args"],
+  ["variable.parameter","{args}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -76,22 +76,21 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["variable.parameter","args"],
+  ["variable.parameter","{args}"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
    "start",
   ["text","    "],
-  ["identifier","foo"],
+  ["entity.name.function","foo"],
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["constant.numeric","0"],
+  ["variable.parameter","{0abc}"],
-  ["identifier","abc"],
+  ["paren.rparen",")"],
-  ["paren.rparen","})"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -123,22 +122,27 @@
 ],[
    "start",
   ["text","    "],
-  ["identifier","foo"],
+  ["entity.name.function","foo"],
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["comment","#abc}) ->"]
+  ["variable.parameter","{#abc}"],
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["storage.type","->"]
 ],[
    "start",
   ["text","    "],
-  ["identifier","foo"],
+  ["entity.name.function","foo"],
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["identifier","abc"],
+  ["variable.parameter","{abc#}"],
-  ["comment","#}) ->"]
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["storage.type","->"]
 ],[
    "start",
   ["text","    "],
@@ -167,15 +171,13 @@
 ],[
    "start",
   ["text","    "],
-  ["identifier","foo"],
+  ["entity.name.function","foo"],
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["identifier","a"],
+  ["variable.parameter","{a{bc}"],
-  ["paren.lparen","{"],
+  ["paren.rparen",")"],
-  ["identifier","bc"],
-  ["paren.rparen","})"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -185,8 +187,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["paren.rparen","})"],
+  ["variable.parameter","{}"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -196,9 +199,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["text"," "],
+  ["variable.parameter","{ }"],
-  ["paren.rparen","})"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -206,10 +209,11 @@
   ["text","    "],
   ["entity.name.function","foo"],
   ["text"," "],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
-  ["paren.lparen","({"],
+  ["paren.lparen","("],
-  ["paren.rparen","})"],
+  ["variable.parameter","{}"],
+  ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -294,7 +298,7 @@
   ["keyword.operator","="],
   ["text"," "],
   ["paren.lparen","("],
-  ["text"," "],
+  ["variable.parameter"," "],
   ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
@@ -303,10 +307,10 @@
   ["text","    "],
   ["entity.name.function","foo"],
   ["text"," "],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
   ["paren.lparen","("],
-  ["text"," "],
+  ["variable.parameter"," "],
   ["paren.rparen",")"],
   ["text"," "],
   ["storage.type","->"]
@@ -345,7 +349,7 @@
   ["text","    "],
   ["entity.name.function","foo"],
   ["text"," "],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
   ["storage.type","->"]
 ],[
@@ -358,7 +362,7 @@
   ["identifier","foo"],
   ["text"," "],
   ["identifier","bar"],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
   ["constant.numeric","1"],
   ["punctuation.operator",","],
@@ -422,7 +426,9 @@
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
-  ["string","\"#{ 22 / 7 + {x: \""],
+  ["string.start","\""],
+  ["string","#{ 22 / 7 + {x: "],
+  ["string.end","\""],
   ["comment","#{a + b}\"} + 2}\""]
 ],[
    "qqdoc",
@@ -466,14 +472,17 @@
    "start",
   ["string.regex","    ///imgy"]
 ],[
-   "start",
+   "js",
   ["text","    "],
   ["keyword","this"],
   ["text"," "],
   ["keyword","isnt"],
-  ["punctuation.operator",":"],
+  ["keyword.operator",":"],
   ["text"," "],
-  ["string","`just JavaScript`"]
+  ["string","`just "]
+],[
+   "start",
+  ["string","       JavaScript`"]
 ],[
    "start",
   ["text","    "],

lib/ace/mode/_test/tokens_curly.json

@@ -1,106 +1,55 @@
 [[
    "start",
-  ["meta.tag","<"],
+  ["text","tokenize Curly template"],
-  ["meta.tag.tag-name","html"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","head"],
-  ["meta.tag.r",">"]
-],[
-   "start"
-],[
-   "css-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.style","style"],
-  ["text"," "],
-  ["entity.other.attribute-name","type"],
-  ["keyword.operator","="],
-  ["string","\"text/css\""],
-  ["meta.tag.r",">"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["variable",".text-layer"],
-  ["text"," "],
-  ["paren.lparen","{"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","font-family"],
-  ["text",": Monaco, "],
-  ["string","\"Courier New\""],
-  ["text",", "],
-  ["support.constant.fonts","monospace"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","font-size"],
-  ["text",": "],
-  ["constant.numeric","12"],
-  ["keyword","px"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","cursor"],
-  ["text",": "],
-  ["support.constant","text"],
-  ["text",";"]
-],[
-   "css-start",
-  ["text","        "],
-  ["paren.rparen","}"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.style","style"],
-  ["meta.tag.r",">"]
-],[
-   "start"
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","head"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","body"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","h1"],
-  ["text"," "],
-  ["entity.other.attribute-name","style"],
-  ["keyword.operator","="],
-  ["string","\"color:red\""],
-  ["meta.tag.r",">"],
   ["variable","{{"],
-  ["text","author_name"],
+  ["text","test"],
-  ["variable","}}"],
+  ["variable","}}"]
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","h1"],
-  ["meta.tag.r",">"]
 ],[
    "start",
+  ["text","tokenize embedded script"]
+],[
+   "start",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.script","script"],
   ["text"," "],
+  ["entity.other.attribute-name","a"],
+  ["keyword.operator","="],
+  ["string","'a'"],
+  ["meta.tag.r",">"],
+  ["storage.type","var"],
   ["meta.tag","</"],
-  ["meta.tag.tag-name","body"],
+  ["meta.tag.tag-name.script","script"],
+  ["meta.tag.r",">"],
+  ["text","'123'"]
+],[
+   "start",
+  ["text","tokenize multiline attribute value with double quotes"]
+],[
+   "tag_qqstring",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.anchor","a"],
+  ["text"," "],
+  ["entity.other.attribute-name","href"],
+  ["keyword.operator","="],
+  ["string","\"abc{{xyz}}"]
+],[
+   "start",
+  ["string","def\""],
   ["meta.tag.r",">"]
 ],[
    "start",
-  ["meta.tag","</"],
+  ["text","tokenize multiline attribute value with single quotes"]
-  ["meta.tag.tag-name","html"],
+],[
+   "tag_qstring",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.anchor","a"],
+  ["text"," "],
+  ["entity.other.attribute-name","href"],
+  ["keyword.operator","="],
+  ["string","'abc"]
+],[
+   "start",
+  ["string","def\\\"'"],
   ["meta.tag.r",">"]
 ],[
    "start"

lib/ace/mode/_test/tokens_diff.json

@@ -1,6 +1,7 @@
 [[
    "start",
-  ["variable","diff --git"],
+  ["variable","diff"],
+  ["variable"," --git"],
   ["keyword"," a/lib/ace/edit_session.js"],
   ["variable"," b/lib/ace/edit_session.js"]
 ],[
@@ -52,7 +53,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -84,7 +86,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -191,7 +194,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -201,7 +205,8 @@
    "start"
 ],[
    "start",
-  ["variable","diff --git"],
+  ["variable","diff"],
+  ["variable"," --git"],
   ["keyword"," a/lib/ace/editor.js"],
   ["variable"," b/lib/ace/editor.js"]
 ],[
@@ -259,7 +264,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -291,7 +297,8 @@
 ],[
    "start"
 ],[
-   "start"
+   "start",
+  ["invalid"," "]
 ],[
    "start"
 ],[
@@ -343,7 +350,8 @@
    "start"
 ],[
    "start",
-  ["variable","diff --git"],
+  ["variable","diff"],
+  ["variable"," --git"],
   ["keyword"," a/lib/ace/search_highlight.js"],
   ["variable"," b/lib/ace/search_highlight.js"]
 ],[

lib/ace/mode/_test/tokens_html.json

@@ -5,96 +5,44 @@
   ["meta.tag.r",">"]
 ],[
    "start",
-  ["text","    "],
   ["meta.tag","<"],
-  ["meta.tag.tag-name","head"],
+  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "start"
-],[
-   "css-start",
   ["text"," "],
-  ["meta.tag","<"],
+  ["entity.other.attribute-name","a"],
-  ["meta.tag.tag-name.style","style"],
-  ["text"," "],
-  ["entity.other.attribute-name","type"],
   ["keyword.operator","="],
-  ["string","\"text/css\""],
+  ["string","'a'"],
-  ["meta.tag.r",">"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["variable",".text-layer"],
-  ["text"," "],
-  ["paren.lparen","{"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","font-family"],
-  ["text",": Monaco, "],
-  ["string","\"Courier New\""],
-  ["text",", "],
-  ["support.constant.fonts","monospace"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","font-size"],
-  ["text",": "],
-  ["constant.numeric","12"],
-  ["keyword","px"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","cursor"],
-  ["text",": "],
-  ["support.constant","text"],
-  ["text",";"]
-],[
-   "css-start",
-  ["text","        "],
-  ["paren.rparen","}"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.style","style"],
-  ["meta.tag.r",">"]
-],[
-   "start"
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","head"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","body"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","h1"],
-  ["text"," "],
-  ["entity.other.attribute-name","style"],
-  ["keyword.operator","="],
-  ["string","\"color:red\""],
   ["meta.tag.r",">"],
-  ["text","Juhu Kinners"],
+  ["storage.type","var"],
   ["meta.tag","</"],
-  ["meta.tag.tag-name","h1"],
+  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
+  ["meta.tag.r",">"],
+  ["text","'123'"]
+],[
+   "tag_qqstring",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.anchor","a"],
+  ["text"," "],
+  ["entity.other.attribute-name","href"],
+  ["keyword.operator","="],
+  ["string","\"abc"]
 ],[
    "start",
-  ["text","    "],
+  ["string","  def\""],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","body"],
   ["meta.tag.r",">"]
+],[
+   "tag_qstring",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.anchor","a"],
+  ["text"," "],
+  ["entity.other.attribute-name","href"],
+  ["keyword.operator","="],
+  ["string","'abc"]
+],[
+   "start",
+  ["string","def\\'"],
+  ["meta.tag.r",">"]
+],[
+   "start"
 ],[
    "start",
   ["meta.tag","</"],

lib/ace/mode/_test/tokens_javascript.json

@@ -1,4 +1,196 @@
 [[
+   "start",
+  ["comment","//test: tokenize 'standard' functions"]
+],[
+   "start",
+  ["identifier","string"],
+  ["punctuation.operator","."],
+  ["support.function","charCodeAt"],
+  ["paren.lparen","("],
+  ["constant.numeric","23"],
+  ["paren.rparen",")"],
+  ["punctuation.operator",";"],
+  ["text"," "],
+  ["variable.language","document"],
+  ["punctuation.operator","."],
+  ["support.function.dom","getElementById"],
+  ["paren.lparen","("],
+  ["string","'test'"],
+  ["paren.rparen",")"],
+  ["punctuation.operator",";"],
+  ["text"," "],
+  ["storage.type","console"],
+  ["punctuation.operator","."],
+  ["support.function.firebug","log"],
+  ["paren.lparen","("],
+  ["string","'Here it is'"],
+  ["paren.rparen",")"],
+  ["punctuation.operator",";"],
+  ["string","\";"]
+],[
+   "start",
+  ["identifier","test"],
+  ["punctuation.operator",":"],
+  ["text"," "],
+  ["comment.doc","/**tokenize doc*/"],
+  ["text"," "],
+  ["identifier","comment"]
+],[
+   "start",
+  ["comment.doc","/**tokenize doc comment with "],
+  ["comment.doc.tag","@tag"],
+  ["comment.doc"," {}*/"]
+],[
+   "start",
+  ["comment","//test: tokenize parens"]
+],[
+   "regex_allowed",
+  ["text","    "],
+  ["storage.type","var"],
+  ["text"," "],
+  ["identifier","line"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["string","\"[{( )}]\""],
+  ["punctuation.operator",";"]
+],[
+   "start",
+  ["comment","//test tokenize arithmetic expression which looks like a regexp"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","/"],
+  ["identifier","b"],
+  ["keyword.operator","/"],
+  ["identifier","c"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","/="],
+  ["identifier","b"],
+  ["keyword.operator","/"],
+  ["identifier","c"]
+],[
+   "start",
+  ["comment","//test tokenize reg exps"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","="],
+  ["string.regexp","/b/g"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","+"],
+  ["string.regexp","/b/g"]
+],[
+   "start",
+  ["identifier","a"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["constant.numeric","1"],
+  ["text"," "],
+  ["keyword.operator","+"],
+  ["text"," "],
+  ["string.regexp","/2 "],
+  ["constant.language.escape","+"],
+  ["string.regexp"," 1/b"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","="],
+  ["string.regexp","/a/"],
+  ["text"," "],
+  ["keyword.operator","/"],
+  ["text"," "],
+  ["string.regexp","/a/"]
+],[
+   "start",
+  ["keyword","case"],
+  ["text"," "],
+  ["string.regexp","/a/"],
+  ["punctuation.operator","."],
+  ["support.function","test"],
+  ["paren.lparen","("],
+  ["identifier","c"],
+  ["paren.rparen",")"]
+],[
+   "start",
+  ["comment","//test tokenize multi-line comment containing a single line comment"]
+],[
+   "start",
+  ["identifier","noRegex"]
+],[
+   "start",
+  ["comment","/* foo // bar */"]
+],[
+   "regex_allowed",
+  ["identifier","canBeRegex"],
+  ["punctuation.operator",";"]
+],[
+   "regex_allowed",
+  ["comment","/* foo // bar */"]
+],[
+   "start",
+  ["comment","// test tokenize identifier with umlauts"]
+],[
+   "start",
+  ["identifier","fu"],
+  ["punctuation.operator","?"],
+  ["identifier","e"]
+],[
+   "start",
+  ["comment","// test // is not a regexp"]
+],[
+   "start",
+  ["paren.lparen","{"],
+  ["text"," "],
+  ["comment","// 123"]
+],[
+   "start",
+  ["comment","//test skipping escaped chars"]
+],[
+   "start",
+  ["string","'Meh"],
+  ["constant.language.escape","\\\\"],
+  ["string","nNeh'"]
+],[
+   "start",
+  ["storage.type","console"],
+  ["punctuation.operator","."],
+  ["support.function.firebug","log"],
+  ["paren.lparen","("],
+  ["string","'"],
+  ["constant.language.escape","\\\\"],
+  ["string","u1232Feh'"]
+],[
+   "qqstring",
+  ["string","\"test multiline\\"]
+],[
+   "start",
+  ["string"," strings\""]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","="],
+  ["text","'"]
+],[
+   "qqstring",
+  ["identifier","b"],
+  ["keyword.operator","="],
+  ["string","\"\\"]
+],[
+   "start",
+  ["string","still a string"]
+],[
+   "start",
+  ["text"," "]
+],[
+   "start",
+  ["text"," "]
+],[
    "regex_allowed",
   ["storage.type","function"],
   ["text"," "],
@@ -70,7 +262,7 @@
   ["keyword.operator","="],
   ["text"," "],
   ["string.regexp","/p"],
-  ["constant.language.escape","|"],
+  ["constant.language.delimiter","|"],
   ["string.regexp","p/"],
   ["text"," "],
   ["comment","// ends here"]
@@ -108,15 +300,17 @@
   ["string.regexp","rr--rr"],
   ["constant.language.escape","$^(?:"],
   ["string.regexp","d"],
-  ["constant.language.escape","|"],
+  ["constant.language.delimiter","|"],
   ["string.regexp","s"],
   ["constant.language.escape",")(?="],
   ["string.regexp","a"],
-  ["constant.language.escape","|)(?!"],
+  ["constant.language.delimiter","|"],
+  ["constant.language.escape",")(?!"],
   ["string.regexp","y"],
-  ["constant.language.escape",")[]|"],
+  ["constant.language.escape",")[]"],
+  ["constant.language.delimiter","|"],
   ["invalid","$?"],
-  ["constant.language.escape","|"],
+  ["constant.language.delimiter","|"],
   ["invalid","^*"],
   ["string.regexp","/"],
   ["text"," "],
@@ -318,6 +512,7 @@
   ["text","\t"],
   ["string","'g'"],
   ["text",":"],
+  ["text"," "],
   ["storage.type","function"],
   ["paren.lparen","("],
   ["variable.parameter","t"],

lib/ace/mode/_test/tokens_liquid.json

@@ -86,7 +86,16 @@
   ["meta.tag.r",">"]
 ],[
    "start",
-  ["text","        Only {{ product.price | format_as_money }}"]
+  ["text","        Only "],
+  ["variable","{{"],
+  ["text"," "],
+  ["identifier","product"],
+  ["text","."],
+  ["identifier","price"],
+  ["text"," | "],
+  ["identifier","format_as_money"],
+  ["text"," "],
+  ["variable","}}"]
 ],[
    "start"
 ],[
@@ -156,7 +165,15 @@
   ["meta.tag","<"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"],
-  ["text"," The word \"tobi\" in uppercase: {{ 'tobi' | upcase }} "],
+  ["text"," The word \"tobi\" in uppercase: "],
+  ["variable","{{"],
+  ["text"," "],
+  ["string","'tobi'"],
+  ["text"," | "],
+  ["support.function","upcase"],
+  ["text"," "],
+  ["variable","}}"],
+  ["text"," "],
   ["meta.tag","</"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"]
@@ -165,7 +182,15 @@
   ["meta.tag","<"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"],
-  ["text","The word \"tobi\" has {{ 'tobi' | size }} letters! "],
+  ["text","The word \"tobi\" has "],
+  ["variable","{{"],
+  ["text"," "],
+  ["string","'tobi'"],
+  ["text"," | "],
+  ["support.function","size"],
+  ["text"," "],
+  ["variable","}}"],
+  ["text"," letters! "],
   ["meta.tag","</"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"]
@@ -174,7 +199,19 @@
   ["meta.tag","<"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"],
-  ["text","Change \"Hello world\" to \"Hi world\": {{ 'Hello world' | replace: 'Hello', 'Hi' }} "],
+  ["text","Change \"Hello world\" to \"Hi world\": "],
+  ["variable","{{"],
+  ["text"," "],
+  ["string","'Hello world'"],
+  ["text"," | "],
+  ["support.function","replace"],
+  ["text",": "],
+  ["string","'Hello'"],
+  ["text",", "],
+  ["string","'Hi'"],
+  ["text"," "],
+  ["variable","}}"],
+  ["text"," "],
   ["meta.tag","</"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"]
@@ -183,7 +220,17 @@
   ["meta.tag","<"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"],
-  ["text","The date today is {{ 'now' | date: \"%Y %b %d\" }} "],
+  ["text","The date today is "],
+  ["variable","{{"],
+  ["text"," "],
+  ["string","'now'"],
+  ["text"," | "],
+  ["support.function","date"],
+  ["text",": "],
+  ["string","\"%Y %b %d\""],
+  ["text"," "],
+  ["variable","}}"],
+  ["text"," "],
   ["meta.tag","</"],
   ["meta.tag.tag-name","p"],
   ["meta.tag.r",">"]
@@ -311,7 +358,14 @@
   ["identifier","link_to_vendor"],
   ["text"," "],
   ["variable","}}"],
-  ["text"," / {{ product.title }}"]
+  ["text"," / "],
+  ["variable","{{"],
+  ["text"," "],
+  ["identifier","product"],
+  ["text","."],
+  ["identifier","title"],
+  ["text"," "],
+  ["variable","}}"]
 ],[
    "start",
   ["text","    "],
@@ -445,7 +499,14 @@
   ["variable","%}"]
 ],[
    "start",
-  ["text","      First column: {{ item.variable }}"]
+  ["text","      First column: "],
+  ["variable","{{"],
+  ["text"," "],
+  ["identifier","item"],
+  ["text","."],
+  ["identifier","variable"],
+  ["text"," "],
+  ["variable","}}"]
 ],[
    "start",
   ["text","    "],
@@ -456,7 +517,14 @@
   ["variable","%}"]
 ],[
    "start",
-  ["text","      Different column: {{ item.variable }}"]
+  ["text","      Different column: "],
+  ["variable","{{"],
+  ["text"," "],
+  ["identifier","item"],
+  ["text","."],
+  ["identifier","variable"],
+  ["text"," "],
+  ["variable","}}"]
 ],[
    "start",
   ["text","    "],

lib/ace/mode/_test/tokens_lua.json

@@ -1,11 +1,11 @@
 [[
-   "qcomment",
+   ["bracketedComment",4,"start"],
   ["comment","--[[--"]
 ],[
-   "qcomment",
+   ["bracketedComment",4,"start"],
   ["comment","num_args takes in 5.1 byte code and extracts the number of arguments"]
 ],[
-   "qcomment",
+   ["bracketedComment",4,"start"],
   ["comment","from its function header."]
 ],[
    "start",
@@ -280,16 +280,16 @@
 ],[
    "start"
 ],[
-   "qstring3",
+   ["bracketedString",5,"start"],
   ["support.function","print"],
   ["paren.lparen","("],
-  ["string","[===["]
+  ["comment","[===["]
 ],[
-   "qstring3",
+   ["bracketedString",5,"start"],
-  ["string","    blah blah %s, (%d %d)"]
+  ["comment","    blah blah %s, (%d %d)"]
 ],[
    "start",
-  ["string","]===]"],
+  ["comment","]===]"],
   ["keyword.operator","%"],
   ["paren.lparen","{"],
   ["string","\"blah\""],
@@ -301,10 +301,10 @@
 ],[
    "start"
 ],[
-   "qcomment1",
+   ["bracketedComment",5,"start"],
   ["comment","--[=[--"]
 ],[
-   "qcomment1",
+   ["bracketedComment",5,"start"],
   ["comment","table.maxn is deprecated, use # instead."]
 ],[
    "start",

lib/ace/mode/_test/tokens_luapage.json

@@ -21,15 +21,15 @@
   ["meta.tag.tag-name","html"],
   ["meta.tag.r",">"]
 ],[
-   "lua-qcomment",
+   ["lua-bracketedComment",4,"lua-start"],
   ["keyword","<%"],
   ["text"," "],
   ["comment","--[[--"]
 ],[
-   "lua-qcomment",
+   ["lua-bracketedComment",4,"lua-start"],
   ["comment","    index.lp from the Kepler Project's LuaDoc HTML doclet."]
 ],[
-   "lua-qcomment",
+   ["lua-bracketedComment",4,"lua-start"],
   ["comment","    http://keplerproject.github.com/luadoc/"]
 ],[
    "start",

lib/ace/mode/_test/tokens_lucene.json

@@ -1,17 +1,92 @@
 [[
    "start",
-  ["paren.lparen","("],
+  ["keyword","test:"],
-  ["keyword","title:"],
+  ["text"," recognises "],
-  ["string","\"foo bar\""],
-  ["text"," "],
   ["keyword.operator","AND"],
-  ["text"," "],
+  ["text"," as keyword"]
-  ["keyword","body:"],
+],[
-  ["string","\"quick fox\""],
+   "start",
-  ["paren.rparen",")"],
+  ["keyword","test:"],
-  ["text"," "],
+  ["text"," recognises "],
   ["keyword.operator","OR"],
-  ["text"," "],
+  ["text"," as keyword"]
-  ["keyword","title:"],
+],[
-  ["text","fox"]
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["keyword.operator","NOT"],
+  ["text"," as keyword"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["string","\"hello this is dog\""],
+  ["text"," as string"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["constant.character.negation","-"],
+  ["string","\"hello this is dog\""],
+  ["text"," as negation with string"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["constant.character.proximity","~100"],
+  ["text"," as text with proximity"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["string","\"hello this is dog\""],
+  ["constant.character.proximity","~100"],
+  ["text"," as string with proximity"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["keyword","raw:"],
+  ["string","\"hello this is dog\""],
+  ["text"," as keyword"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["keyword","raw:"],
+  ["text","foo as\"keyword'"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["string","\"(\""],
+  ["text"," as opening parenthesis"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["string","\")\""],
+  ["text"," as closing parenthesis"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises foo"],
+  ["constant.character.asterisk","*"],
+  ["text"," as text with asterisk"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises foo"],
+  ["constant.character.interro","?"],
+  ["text"," as text with interro"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises single word as text"]
+],[
+   "start",
+  ["text"," foo"]
+],[
+   "start",
+  ["text"," "]
 ]]

lib/ace/mode/_test/tokens_markdown.json

@@ -1,690 +1,75 @@
 [[
    "start",
-  ["text","Ace (Ajax.org Cloud9 Editor)"]
+  ["text","test: header 1 "]
 ],[
    "start",
-  ["markup.heading.1","============================"]
+  ["markup.heading.1","#f"]
-],[
-   "start"
 ],[
    "start",
-  ["text","Ace is a standalone code editor written in JavaScript. Our goal is to create a browser based editor that matches and extends the features, usability and performance of existing native editors such as TextMate, Vim or Eclipse. It can be easily embedded in any web page or JavaScript application. Ace is developed as the primary editor for ["],
+  ["text","test: header 2"]
+],[
+   "start",
+  ["markup.heading.2","## foo"]
+],[
+   "start",
+  ["text","test: header ends with ' #'"]
+],[
+   "start",
+  ["markup.heading.1","# # # "]
+],[
+   "start",
+  ["text","test: header ends with '#'"]
+],[
+   "start",
+  ["markup.heading.1","#foo# "]
+],[
+   "start",
+  ["text","test: 6+ #s is not a valid header"]
+],[
+   "start",
+  ["text","####### foo"]
+],[
+   "start",
+  ["text","test: # followed be only space is not a valid header"]
+],[
+   "start",
+  ["text","test: only space between #s is not a valid header"]
+],[
+   "start",
+  ["text","#  #"]
+],[
+   "allowBlock"
+],[
+   "start",
+  ["text","test links  "],
+  ["text","["],
   ["string","Cloud9 IDE"],
   ["text","]("],
-  ["markup.underline","http://www.cloud9ide.com/"],
+  ["markup.underline","http://www.c9.io/"],
-  ["text",") and the successor of the Mozilla Skywriter (Bespin) Project."]
+  ["text",")"],
-],[
+  ["text"," "]
-   "start"
-],[
-   "start",
-  ["text","Features"]
-],[
-   "start",
-  ["markup.heading.2","--------"]
-],[
-   "start"
 ],[
    "listblock",
-  ["markup.list","* Syntax highlighting"]
+  ["markup.list","* "],
-],[
+  ["text","["],
-   "listblock",
-  ["markup.list","* Automatic indent and outdent"]
-],[
-   "listblock",
-  ["markup.list","* An optional command line"]
-],[
-   "listblock",
-  ["markup.list","* Handles huge documents (100,000 lines and more are no problem)"]
-],[
-   "listblock",
-  ["markup.list","* Fully customizable key bindings including VI and Emacs modes"]
-],[
-   "listblock",
-  ["markup.list","* Themes (TextMate themes can be imported)"]
-],[
-   "listblock",
-  ["markup.list","* Search and replace with regular expressions"]
-],[
-   "listblock",
-  ["markup.list","* Highlight matching parentheses"]
-],[
-   "listblock",
-  ["markup.list","* Toggle between soft tabs and real tabs"]
-],[
-   "listblock",
-  ["markup.list","* Displays hidden characters"]
-],[
-   "listblock",
-  ["markup.list","* Drag and drop text using the mouse"]
-],[
-   "listblock",
-  ["markup.list","* Line wrapping"]
-],[
-   "listblock",
-  ["markup.list","* Unstructured / user code folding"]
-],[
-   "listblock",
-  ["markup.list","* Live syntax checker (currently JavaScript/CoffeeScript)"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Take Ace for a spin!"]
-],[
-   "start",
-  ["markup.heading.2","--------------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Check out the Ace live ["],
   ["string","demo"],
   ["text","]("],
   ["markup.underline","http://ajaxorg.github.com/ace/"],
-  ["text",") or get a ["],
+  ["text",")"],
-  ["string","Cloud9 IDE account"],
+  ["markup.list"," "]
-  ["text","]("],
-  ["markup.underline","http://run.cloud9ide.com"],
-  ["text",") to experience Ace while editing one of your own GitHub projects."]
-],[
-   "start"
-],[
-   "start",
-  ["text","If you want, you can use Ace as a textarea replacement thanks to the ["],
-  ["string","Ace Bookmarklet"],
-  ["text","]("],
-  ["markup.underline","http://ajaxorg.github.com/ace/build/textarea/editor.html"],
-  ["text",")."]
-],[
-   "start"
-],[
-   "start",
-  ["text","History"]
-],[
-   "start",
-  ["markup.heading.2","-------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Previously known as “Bespin” and “Skywriter” it’s now known as Ace (Ajax.org Cloud9 Editor)! Bespin and Ace started as two independent projects, both aiming to build a no-compromise code editor component for the web. Bespin started as part of Mozilla Labs and was based on the canvas tag, while Ace is the Editor component of the Cloud9 IDE and is using the DOM for rendering. After the release of Ace at JSConf.eu 2010 in Berlin the Skywriter team decided to merge Ace with a simplified version of Skywriter's plugin system and some of Skywriter's extensibility points. All these changes have been merged back to Ace. Both Ajax.org and Mozilla are actively developing and maintaining Ace."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Getting the code"]
-],[
-   "start",
-  ["markup.heading.2","----------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Ace is a community project. We actively encourage and support contributions. The Ace source code is hosted on GitHub. It is released under the BSD License. This license is very simple, and is friendly to all kinds of projects, whether open source or not. Take charge of your editor and add your favorite language highlighting and keybindings!"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    git clone git://github.com/ajaxorg/ace.git"]
-],[
-   "githubblock",
-  ["support.function","    cd ace"]
-],[
-   "githubblock",
-  ["support.function","    git submodule update --init --recursive"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Embedding Ace"]
-],[
-   "start",
-  ["markup.heading.2","-------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Ace can be easily embedded into any existing web page. The Ace git repository ships with a pre-packaged version of Ace inside of the "],
-  ["support.function","`build`"],
-  ["text"," directory. The same packaged files are also available as a separate ["],
-  ["string","download"],
-  ["text","]("],
-  ["markup.underline","https://github.com/ajaxorg/ace/downloads"],
-  ["text","). Simply copy the contents of the "],
-  ["support.function","`src`"],
-  ["text"," subdirectory somewhere into your project and take a look at the included demos of how to use Ace."]
-],[
-   "start"
-],[
-   "start",
-  ["text","The easiest version is simply:"]
-],[
-   "start"
-],[
-   "html-start",
-  ["support.function","```html"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","div"],
-  ["text"," "],
-  ["entity.other.attribute-name","id"],
-  ["keyword.operator","="],
-  ["string","\"editor\""],
-  ["meta.tag.r",">"],
-  ["text","some text"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","div"],
-  ["meta.tag.r",">"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.script","script"],
-  ["text"," "],
-  ["entity.other.attribute-name","src"],
-  ["keyword.operator","="],
-  ["string","\"src/ace.js\""],
-  ["text"," "],
-  ["entity.other.attribute-name","type"],
-  ["keyword.operator","="],
-  ["string","\"text/javascript\""],
-  ["text"," "],
-  ["entity.other.attribute-name","charset"],
-  ["keyword.operator","="],
-  ["string","\"utf-8\""],
-  ["meta.tag.r",">"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "html-js-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "html-js-regex_allowed",
-  ["text","    "],
-  ["storage.type","window"],
-  ["punctuation.operator","."],
-  ["entity.name.function","onload"],
-  ["text"," "],
-  ["keyword.operator","="],
-  ["text"," "],
-  ["storage.type","function"],
-  ["paren.lparen","("],
-  ["paren.rparen",")"],
-  ["text"," "],
-  ["paren.lparen","{"]
-],[
-   "html-js-regex_allowed",
-  ["text","        "],
-  ["storage.type","var"],
-  ["text"," "],
-  ["identifier","editor"],
-  ["text"," "],
-  ["keyword.operator","="],
-  ["text"," "],
-  ["identifier","ace"],
-  ["punctuation.operator","."],
-  ["identifier","edit"],
-  ["paren.lparen","("],
-  ["string","\"editor\""],
-  ["paren.rparen",")"],
-  ["punctuation.operator",";"]
-],[
-   "html-js-regex_allowed",
-  ["text","    "],
-  ["paren.rparen","}"],
-  ["punctuation.operator",";"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","With \"editor\" being the id of the DOM element, which should be converted to an editor. Note that this element must be explicitly sized and positioned "],
-  ["support.function","`absolute`"],
-  ["text"," or "],
-  ["support.function","`relative`"],
-  ["text"," for Ace to work. e.g."]
-],[
-   "start"
-],[
-   "css-start",
-  ["support.function","```css"]
-],[
-   "css-ruleset",
-  ["text","    "],
-  ["keyword","#editor"],
-  ["text"," "],
-  ["paren.lparen","{"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["support.type","position"],
-  ["text",": "],
-  ["support.constant","absolute"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["support.type","width"],
-  ["text",": "],
-  ["constant.numeric","500"],
-  ["keyword","px"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["support.type","height"],
-  ["text",": "],
-  ["constant.numeric","400"],
-  ["keyword","px"],
-  ["text",";"]
-],[
-   "css-start",
-  ["text","    "],
-  ["paren.rparen","}"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","To change the theme simply include the Theme's JavaScript file"]
-],[
-   "start"
-],[
-   "html-start",
-  ["support.function","```html"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.script","script"],
-  ["text"," "],
-  ["entity.other.attribute-name","src"],
-  ["keyword.operator","="],
-  ["string","\"src/theme-twilight.js\""],
-  ["text"," "],
-  ["entity.other.attribute-name","type"],
-  ["keyword.operator","="],
-  ["string","\"text/javascript\""],
-  ["text"," "],
-  ["entity.other.attribute-name","charset"],
-  ["keyword.operator","="],
-  ["string","\"utf-8\""],
-  ["meta.tag.r",">"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","and configure the editor to use the theme:"]
-],[
-   "start"
-],[
-   "js-start",
-  ["support.function","```javascript"]
-],[
-   "js-regex_allowed",
-  ["text","    "],
-  ["identifier","editor"],
-  ["punctuation.operator","."],
-  ["identifier","setTheme"],
-  ["paren.lparen","("],
-  ["string","\"ace/theme/twilight\""],
-  ["paren.rparen",")"],
-  ["punctuation.operator",";"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","By default the editor only supports plain text mode; many other languages are available as separate modules. After including the mode's JavaScript file:"]
-],[
-   "start"
-],[
-   "html-start",
-  ["support.function","```html"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.script","script"],
-  ["text"," "],
-  ["entity.other.attribute-name","src"],
-  ["keyword.operator","="],
-  ["string","\"src/mode-javascript.js\""],
-  ["text"," "],
-  ["entity.other.attribute-name","type"],
-  ["keyword.operator","="],
-  ["string","\"text/javascript\""],
-  ["text"," "],
-  ["entity.other.attribute-name","charset"],
-  ["keyword.operator","="],
-  ["string","\"utf-8\""],
-  ["meta.tag.r",">"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Then the mode can be used like this:"]
-],[
-   "start"
-],[
-   "js-start",
-  ["support.function","```javascript"]
-],[
-   "js-regex_allowed",
-  ["text","    "],
-  ["storage.type","var"],
-  ["text"," "],
-  ["identifier","JavaScriptMode"],
-  ["text"," "],
-  ["keyword.operator","="],
-  ["text"," "],
-  ["identifier","require"],
-  ["paren.lparen","("],
-  ["string","\"ace/mode/javascript\""],
-  ["paren.rparen",")"],
-  ["punctuation.operator","."],
-  ["identifier","Mode"],
-  ["punctuation.operator",";"]
-],[
-   "js-regex_allowed",
-  ["text","    "],
-  ["identifier","editor"],
-  ["punctuation.operator","."],
-  ["identifier","getSession"],
-  ["paren.lparen","("],
-  ["paren.rparen",")"],
-  ["punctuation.operator","."],
-  ["identifier","setMode"],
-  ["paren.lparen","("],
-  ["keyword","new"],
-  ["text"," "],
-  ["identifier","JavaScriptMode"],
-  ["paren.lparen","("],
-  ["paren.rparen","))"],
-  ["punctuation.operator",";"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Documentation"]
-],[
-   "start",
-  ["markup.heading.2","-------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","You find a lot more sample code in the ["],
-  ["string","demo app"],
-  ["text","]("],
-  ["markup.underline","https://github.com/ajaxorg/ace/blob/master/demo/demo.js"],
-  ["text",")."]
-],[
-   "start"
-],[
-   "start",
-  ["text","There is also some documentation on the ["],
-  ["string","wiki page"],
-  ["text","]("],
-  ["markup.underline","https://github.com/ajaxorg/ace/wiki"],
-  ["text",")."]
-],[
-   "start"
-],[
-   "start",
-  ["text","If you still need help, feel free to drop a mail on the ["],
-  ["string","ace mailing list"],
-  ["text","]("],
-  ["markup.underline","http://groups.google.com/group/ace-discuss"],
-  ["text",")."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Running Ace"]
-],[
-   "start",
-  ["markup.heading.2","-----------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","After the checkout Ace works out of the box. No build step is required. Open 'editor.html' in any browser except Google Chrome. Google Chrome doesn't allow XMLHTTPRequests from files loaded from disc (i.e. with a file:/// URL). To open Ace in Chrome simply start the bundled mini HTTP server:"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    ./static.py"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Or using Node.JS"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    ./static.js"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","The editor can then be opened at http://localhost:8888/index.html."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Package Ace"]
-],[
-   "start",
-  ["markup.heading.2","-----------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","To package Ace we use the dryice build tool developed by the Mozilla Skywriter team. Before you can build you need to make sure that the submodules are up to date."]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    git submodule update --init --recursive"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Afterwards Ace can be built by calling"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    ./Makefile.dryice.js normal"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","The packaged Ace will be put in the 'build' folder."]
-],[
-   "start"
-],[
-   "start",
-  ["text","To build the bookmarklet version execute"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    ./Makefile.dryice.js bm"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Running the Unit Tests"]
-],[
-   "start",
-  ["markup.heading.2","----------------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","The Ace unit tests run on node.js. Before the first run a couple of node modules have to be installed. The easiest way to do this is by using the node package manager (npm). In the Ace base directory simply call"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    npm link ."]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","To run the tests call:"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    node lib/ace/test/all.js"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","You can also run the tests in your browser by serving:"]
-],[
-   "start"
-],[
-   "start",
-  ["support.function","    http://localhost:8888/lib/ace/test/tests.html"]
-],[
-   "start"
-],[
-   "start",
-  ["text","This makes debugging failing tests way more easier."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Contributing"]
-],[
-   "start",
-  ["markup.heading.2","------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Ace wouldn't be what it is without contributions! Feel free to fork and improve/enhance Ace any way you want. If you feel that the editor or the Ace community will benefit from your changes, please open a pull request. To protect the interests of the Ace contributors and users we require contributors to sign a Contributors License Agreement (CLA) before we pull the changes into the main repository. Our CLA is the simplest of agreements, requiring that the contributions you make to an ajax.org project are only those you're allowed to make. This helps us significantly reduce future legal risk for everyone involved. It is easy, helps everyone, takes ten minutes, and only needs to be completed once.  There are two versions of the agreement:"]
-],[
-   "start"
 ],[
    "listblock",
-  ["markup.list","1. [The Individual CLA](https://github.com/ajaxorg/ace/raw/master/doc/Contributor_License_Agreement-v2.pdf): use this version if you're working on an ajax.org in your spare time, or can clearly claim ownership of copyright in what you'll be submitting."]
+  ["markup.list","in lists"]
-],[
-   "listblock",
-  ["markup.list","2. [The Corporate CLA](https://github.com/ajaxorg/ace/raw/master/doc/Corporate_Contributor_License_Agreement-v2.pdf): have your corporate lawyer review and submit this if your company is going to be contributing to ajax.org  projects"]
 ],[
    "start"
 ],[
    "start",
-  ["text","If you want to contribute to an ajax.org project please print the CLA and fill it out and sign it. Then either send it by snail mail or fax to us or send it back scanned (or as a photo) by email."]
+  ["text","in plain text "],
-],[
+  ["meta.tag","<"],
-   "start"
+  ["meta.tag.tag-name","b"],
-],[
+  ["meta.tag.r",">"],
-   "start",
+  ["text","http://ace.ajaxorg.com"],
-  ["text","Email: fabian.jakobs@web.de"]
+  ["meta.tag","<"],
-],[
+  ["meta.tag.tag-name","b"],
-   "start"
+  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","Fax: +31 (0) 206388953"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Address: Ajax.org B.V."]
-],[
-   "start",
-  ["text","  Keizersgracht 241"]
-],[
-   "start",
-  ["text","  1016 EA, Amsterdam"]
-],[
-   "start",
-  ["text","  the Netherlands"]
 ]]

lib/ace/mode/_test/tokens_ruby.json

@@ -1,154 +1,135 @@
 [[
    "start",
-  ["comment","#!/usr/bin/ruby"]
+  ["text"," "],
-],[
+  ["comment","#test: symbol tokenizer"]
-   "start"
 ],[
    "start",
-  ["comment","# Program to find the factorial of a number"]
-],[
-   "start",
-  ["keyword","def"],
-  ["text"," "],
-  ["identifier","fact"],
-  ["paren.lparen","("],
-  ["identifier","n"],
-  ["paren.rparen",")"]
-],[
-   "start",
-  ["text","    "],
-  ["keyword","if"],
-  ["text"," "],
-  ["identifier","n"],
-  ["text"," "],
-  ["keyword.operator","=="],
-  ["text"," "],
-  ["constant.numeric","0"]
-],[
-   "start",
-  ["text","        "],
-  ["constant.numeric","1"]
-],[
-   "start",
-  ["text","    "],
-  ["keyword","else"]
-],[
-   "start",
-  ["text","        "],
-  ["identifier","n"],
-  ["text"," "],
-  ["keyword.operator","*"],
-  ["text"," "],
-  ["identifier","fact"],
-  ["paren.lparen","("],
-  ["identifier","n"],
-  ["constant.numeric","-1"],
-  ["paren.rparen",")"]
-],[
-   "start",
-  ["text","    "],
-  ["keyword","end"]
-],[
-   "start",
-  ["keyword","end"]
-],[
-   "start"
-],[
-   "start",
-  ["support.function","puts"],
-  ["text"," "],
-  ["identifier","fact"],
-  ["paren.lparen","("],
-  ["support.class","ARGV"],
-  ["paren.lparen","["],
-  ["constant.numeric","0"],
-  ["paren.rparen","]"],
-  ["text","."],
-  ["identifier","to_i"],
-  ["paren.rparen",")"]
-],[
-   "start"
-],[
-   "start",
-  ["keyword","class"],
-  ["text"," "],
-  ["support.class","Range"]
-],[
-   "start",
-  ["text","  "],
-  ["keyword","def"],
-  ["text"," "],
-  ["identifier","to_json"],
-  ["paren.lparen","("],
-  ["keyword.operator","*"],
-  ["identifier","a"],
-  ["paren.rparen",")"]
-],[
-   "start",
-  ["text","    "],
-  ["paren.lparen","{"]
-],[
-   "start",
-  ["text","      "],
-  ["string","'json_class'"],
-  ["text","   "],
-  ["punctuation.separator.key-value","=>"],
-  ["text"," "],
-  ["variable.language","self"],
-  ["text","."],
-  ["keyword","class"],
-  ["text","."],
-  ["identifier","name"],
-  ["text",", "],
-  ["comment","# = 'Range'"]
-],[
-   "start",
-  ["text","      "],
-  ["string","'data'"],
-  ["text","         "],
-  ["punctuation.separator.key-value","=>"],
   ["text","  "],
   ["paren.lparen","["],
+  ["constant.other.symbol.ruby",":@thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":$thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":_thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":Thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing1"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing_a"],
+  ["text",","]
+],[
+   "start",
   ["text","              "],
-  ["identifier","first"],
+  ["constant.other.symbol.ruby",":THING"],
   ["text",", "],
-  ["identifier","last"],
+  ["constant.other.symbol.ruby",":thing!"],
   ["text",", "],
-  ["identifier","exclude_end"],
+  ["constant.other.symbol.ruby",":thing="],
-  ["text","? "],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing?"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":t?"],
+  ["text",","]
+],[
+   "start",
+  ["text","              :, :@, :"],
+  ["keyword.operator","$"],
+  ["text",", :"],
+  ["constant.numeric","1"],
+  ["text",", :1"],
+  ["identifier","thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":th?"],
+  ["identifier","ing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thi="],
+  ["identifier","ng"],
+  ["text",", :1"],
+  ["identifier","thing"],
+  ["text",","]
+],[
+   "start",
+  ["text","            "],
+  ["constant.other.symbol.ruby",":th!"],
+  ["identifier","ing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing"],
+  ["comment","#"]
+],[
+   "start",
+  ["text","    "],
   ["paren.rparen","]"]
-],[
-   "start",
-  ["text","    "],
-  ["paren.rparen","}"],
-  ["text","."],
-  ["identifier","to_json"],
-  ["paren.lparen","("],
-  ["keyword.operator","*"],
-  ["identifier","a"],
-  ["paren.rparen",")"]
-],[
-   "start",
-  ["text","  "],
-  ["keyword","end"]
-],[
-   "start",
-  ["keyword","end"]
 ],[
    "start"
 ],[
    "start",
-  ["paren.lparen","{"],
-  ["constant.other.symbol.ruby",":id"],
   ["text"," "],
-  ["punctuation.separator.key-value","=>"],
+  ["comment","#test: namespaces aren't symbols\" : function() {"]
+],[
+   "start",
   ["text","   "],
-  ["constant.numeric","34"],
+  ["support.class","Namespaced"],
+  ["text","::"],
+  ["support.class","Class"]
+],[
+   "start",
+  ["text"," "],
+  ["comment","#test: hex tokenizer "]
+],[
+   "start",
+  ["text","    "],
+  ["constant.numeric","0x9a"],
   ["text",", "],
-  ["constant.other.symbol.ruby",":sdfasdfasdf"],
+  ["constant.numeric","0XA1"],
+  ["text",", "],
+  ["constant.numeric","0x9_a"],
+  ["text",", 0"],
+  ["identifier","x"],
+  ["text",", 0"],
+  ["identifier","x_9a"],
+  ["text",", 0"],
+  ["identifier","x9a_"],
+  ["text",","]
+],[
+   "start",
   ["text"," "],
-  ["punctuation.separator.key-value","=>"],
+  ["comment","#test: float tokenizer"]
+],[
+   "start",
   ["text","    "],
-  ["string","\"asdasdads\""],
+  ["paren.lparen","["],
-  ["paren.rparen","}"]
+  ["constant.numeric","1"],
+  ["text",", "],
+  ["constant.numeric","+1"],
+  ["text",", "],
+  ["constant.numeric","-1"],
+  ["text",", "],
+  ["constant.numeric","12_345"],
+  ["text",", "],
+  ["constant.numeric","0.000_1"],
+  ["text",","]
+],[
+   "start",
+  ["text","    "],
+  ["identifier","_"],
+  ["text",", "],
+  ["constant.numeric","3_1"],
+  ["text",", "],
+  ["constant.numeric","1_2"],
+  ["text",", 1"],
+  ["identifier","_"],
+  ["text","."],
+  ["constant.numeric","0"],
+  ["text",", "],
+  ["constant.numeric","0"],
+  ["text","."],
+  ["identifier","_1"],
+  ["paren.rparen","]"],
+  ["text",";"]
+],[
+   "start",
+  ["text"," "]
 ]]

lib/ace/mode/_test/tokens_stylus.json

@@ -32,7 +32,8 @@
 ],[
    "start",
   ["entity.name.function.stylus","asdasdasdad"],
-  ["text","(df, ad"],
+  ["text","("],
+  ["text","df, ad"],
   ["keyword.operator.stylus","="],
   ["constant.numeric","23"],
   ["text",")"]

lib/ace/mode/_test/tokens_tcl.json

@@ -30,8 +30,7 @@
   ["identifier","distmap"],
   ["paren.rparen","}"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
@@ -43,8 +42,7 @@
   ["text"," "],
   ["identifier","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$vertex"],
-  ["variable.instance","vertex"],
   ["text"," "],
   ["identifier","Inf"]
 ],[
@@ -56,14 +54,14 @@
   ["text"," "],
   ["identifier","path"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$vertex"],
-  ["variable.instance","vertex"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["text","}"]
+  ["paren.rparen","}"]
 ],[
    "commandItem",
-  ["text","    }"]
+  ["text","    "],
+  ["paren.rparen","}"]
 ],[
    "start",
   ["text","    "],
@@ -73,8 +71,7 @@
   ["text"," "],
   ["identifier","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$origin"],
-  ["variable.instance","origin"],
   ["text"," 0"]
 ],[
    "start",
@@ -85,14 +82,12 @@
   ["text"," "],
   ["identifier","path"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$origin"],
-  ["variable.instance","origin"],
   ["text"," "],
   ["paren.lparen","["],
   ["keyword","list"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$origin"],
-  ["variable.instance","origin"],
   ["paren.rparen","]"]
 ],[
    "commandItem",
@@ -108,8 +103,7 @@
   ["text"," "],
   ["identifier","size"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["paren.rparen","]}"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -138,8 +132,7 @@
   ["support.function","-"],
   ["paren.rparen","}"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
@@ -148,8 +141,7 @@
   ["keyword","if"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["variable.instancce","$"],
+  ["variable.instance","$d"],
-  ["variable.instance","d"],
   ["text"," "],
   ["support.function",">"],
   ["text"," "],
@@ -163,11 +155,9 @@
   ["text"," "],
   ["identifier","get"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dist"],
-  ["variable.instance","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$uu"],
-  ["variable.instance","uu"],
   ["paren.rparen","]]}"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -178,8 +168,7 @@
   ["text"," "],
   ["identifier","u"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$uu"]
-  ["variable.instance","uu"]
 ],[
    "start",
   ["text","\t\t"],
@@ -187,14 +176,15 @@
   ["text"," "],
   ["identifier","d"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dd"]
-  ["variable.instance","dd"]
 ],[
    "commandItem",
-  ["text","\t    }"]
+  ["text","\t    "],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
-  ["text","\t}"]
+  ["text","\t"],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
   ["text"," "]
@@ -208,8 +198,7 @@
   ["keyword","if"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["variable.instancce","$"],
+  ["variable.instance","$d"],
-  ["variable.instance","d"],
   ["text"," "],
   ["support.function","=="],
   ["text"," "],
@@ -245,11 +234,9 @@
   ["text"," "],
   ["identifier","get"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$u"],
-  ["variable.instance","u"],
   ["paren.rparen","]"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -264,11 +251,9 @@
   ["text"," "],
   ["identifier","exists"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$graph"],
-  ["variable.instance","graph"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["paren.rparen","]}"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -283,13 +268,11 @@
   ["keyword","expr"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["variable.instancce","$"],
+  ["variable.instance","$d"],
-  ["variable.instance","d"],
   ["text"," "],
   ["support.function","+"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dd"],
-  ["variable.instance","dd"],
   ["paren.rparen","}]"]
 ],[
    "commandItem",
@@ -297,8 +280,7 @@
   ["keyword","if"],
   ["text"," "],
   ["paren.lparen","{"],
-  ["variable.instancce","$"],
+  ["variable.instance","$alt"],
-  ["variable.instance","alt"],
   ["text"," "],
   ["support.function","<"],
   ["text"," "],
@@ -307,11 +289,9 @@
   ["text"," "],
   ["identifier","get"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dist"],
-  ["variable.instance","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["paren.rparen","]}"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -324,11 +304,9 @@
   ["text"," "],
   ["identifier","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$alt"]
-  ["variable.instance","alt"]
 ],[
    "start",
   ["text","\t\t    "],
@@ -338,8 +316,7 @@
   ["text"," "],
   ["identifier","path"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["text"," "],
   ["paren.lparen","["],
   ["keyword","list"],
@@ -350,25 +327,25 @@
   ["text"," "],
   ["identifier","get"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$path"],
-  ["variable.instance","path"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$u"],
-  ["variable.instance","u"],
   ["paren.rparen","]"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$v"],
-  ["variable.instance","v"],
   ["paren.rparen","]"]
 ],[
    "commandItem",
-  ["text","\t\t}"]
+  ["text","\t\t"],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
-  ["text","\t    }"]
+  ["text","\t    "],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
-  ["text","\t}"]
+  ["text","\t"],
+  ["paren.rparen","}"]
 ],[
    "commandItem",
   ["text"," "]
@@ -385,11 +362,11 @@
   ["text"," "],
   ["identifier","graph"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$u"]
-  ["variable.instance","u"]
 ],[
    "commandItem",
-  ["text","    }"]
+  ["text","    "],
+  ["paren.rparen","}"]
 ],[
    "start",
   ["text","    "],
@@ -398,13 +375,11 @@
   ["paren.lparen","["],
   ["keyword","list"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$dist"],
-  ["variable.instance","dist"],
   ["text"," "],
-  ["variable.instancce","$"],
+  ["variable.instance","$path"],
-  ["variable.instance","path"],
   ["paren.rparen","]"]
 ],[
    "commandItem",
-  ["text","}"]
+  ["paren.rparen","}"]
 ]]

lib/ace/mode/_test/tokens_typescript.json

@@ -128,7 +128,8 @@
   ["paren.lparen","("],
   ["entity.name.function.ts","greeter.greet"],
   ["paren.lparen","("],
-  ["paren.rparen","))"]
+  ["paren.rparen",")"],
+  ["paren.rparen",")"]
 ],[
    "start",
   ["paren.rparen","}"]

lib/ace/mode/_test/tokens_xml.json

@@ -1,539 +1,43 @@
 [[
    "start",
-  ["xml-pe","<?xml version=\"1.0\" encoding=\"UTF-8\"?>"]
-],[
-   "tag_embed_attribute_list",
   ["meta.tag","<"],
-  ["meta.tag.tag-name","query"],
+  ["meta.tag.tag-name","Juhu"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns:yahoo"],
-  ["keyword.operator","="],
-  ["string","\"http://www.yahooapis.com/v1/base.rng\""]
-],[
-   "start",
-  ["text","    "],
-  ["entity.other.attribute-name","yahoo:count"],
-  ["keyword.operator","="],
-  ["string","\"7\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:created"],
-  ["keyword.operator","="],
-  ["string","\"2011-10-11T08:40:23Z\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","diagnostics"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","publiclyCallable"],
   ["meta.tag.r",">"],
-  ["text","true"],
+  ["text","//Juhu Kinners"],
   ["meta.tag","</"],
-  ["meta.tag.tag-name","publiclyCallable"],
+  ["meta.tag.tag-name","Kinners"],
   ["meta.tag.r",">"]
 ],[
    "start",
-  ["text","        "],
+  ["text","test: two tags in the same lines should be in separate tokens\""]
+],[
+   "start",
   ["meta.tag","<"],
-  ["meta.tag.tag-name","url"],
+  ["meta.tag.tag-name","Juhu"],
-  ["text"," "],
-  ["entity.other.attribute-name","execution-start-time"],
-  ["keyword.operator","="],
-  ["string","\"0\""],
-  ["text"," "],
-  ["entity.other.attribute-name","execution-stop-time"],
-  ["keyword.operator","="],
-  ["string","\"25\""],
-  ["text"," "],
-  ["entity.other.attribute-name","execution-time"],
-  ["keyword.operator","="],
-  ["string","\"25\""],
   ["meta.tag.r",">"],
-  ["text","<![CDATA[http://where.yahooapis.com/v1/continents;start=0;count=10]]>"],
+  ["meta.tag","<"],
-  ["meta.tag","</"],
+  ["meta.tag.tag-name","Kinners"],
-  ["meta.tag.tag-name","url"],
   ["meta.tag.r",">"]
 ],[
    "start",
-  ["text","        "],
+  ["text","test: multiline attributes\""]
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","user-time"],
-  ["meta.tag.r",">"],
-  ["text","26"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","user-time"],
-  ["meta.tag.r",">"]
 ],[
-   "start",
+   "tag_qqstring",
-  ["text","        "],
   ["meta.tag","<"],
-  ["meta.tag.tag-name","service-time"],
+  ["meta.tag.tag-name","copy"],
-  ["meta.tag.r",">"],
-  ["text","25"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","service-time"],
-  ["meta.tag.r",">"]
-],[
-   "start",
   ["text"," "],
-  ["meta.tag","<"],
+  ["entity.other.attribute-name","set"],
-  ["meta.tag.tag-name","build-version"],
-  ["meta.tag.r",">"],
-  ["text","21978"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","build-version"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","diagnostics"],
-  ["meta.tag.r",">"],
-  ["text"," "]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","results"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
   ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
+  ["string","\"{"]
 ],[
-   "start",
+   "tag_qqstring",
+  ["string","}\""],
   ["text"," "],
-  ["entity.other.attribute-name","xml:lang"],
+  ["entity.other.attribute-name","undo"],
   ["keyword.operator","="],
-  ["string","\"en-US\""],
+  ["string","\"{"]
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865670\""],
-  ["meta.tag.r",">"]
 ],[
    "start",
-  ["text","            "],
+  ["string","}\""],
-  ["meta.tag","<"],
+  ["meta.tag.r","/>"]
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865670"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Africa"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865675\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865675"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Europe"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865673\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865673"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","South America"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/28289421\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","28289421"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Antarctic"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865671\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865671"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Asia"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865672\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865672"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","North America"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/55949070\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","55949070"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Australia"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","results"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","query"],
-  ["meta.tag.r",">"]
 ]]

lib/ace/mode/_test/tokens_xquery.json

@@ -20,11 +20,13 @@
 ],[
    "start",
   ["keyword","return"],
+  ["text"," "],
   ["text","<"],
   ["meta.tag","results"],
   ["text",">"]
 ],[
    "start",
+  ["text","  "],
   ["text","<"],
   ["meta.tag","message"],
   ["text",">"],

lib/ace/mode/abap_highlight_rules.js

@@ -113,7 +113,7 @@ var AbapHighlightRules = function() {
             {token : "variable.parameter", regex : /sy|pa?\d\d\d\d\|t\d\d\d\.|innnn/}, 
             {token : "keyword", regex : compoundKeywords}, 
             {token : "variable.parameter", regex : /\w+-\w+(?:-\w+)*/}, 
-            {token : keywordMapper, regex : "\\w+\\b"}, 
+            {token : keywordMapper, regex : "\\b\\w+\\b"}, 
         ],
         "qstring" : [
             {token : "constant.language.escape",   regex : "''"},

lib/ace/mode/coffee_highlight_rules.js

@@ -38,10 +38,6 @@ define(function(require, exports, module) {
 
     function CoffeeHighlightRules() {
         var identifier = "[$A-Za-z_\\x7f-\\uffff][$\\w\\x7f-\\uffff]*";
-        var stringfill = {
-            token : "string",
-            regex : ".+"
-        };
 
         var keywords = (
             "this|throw|then|try|typeof|super|switch|return|break|by|continue|" +
@@ -86,50 +82,55 @@ define(function(require, exports, module) {
             "variable.language": variableLanguage
         }, "identifier");
 
-        var functionRules = {
+        var functionRule = {
-            "({args})->": {
+            token: ["paren.lparen", "variable.parameter", "paren.rparen", "text", "storage.type"],
-                token: ["paren.lparen", "text", "paren.lparen", "text", "variable.parameter", "text", "paren.rparen", "text", "paren.rparen", "text", "storage.type"],
+            regex: /(?:(\()((?:"[^")]*?"|'[^')]*?'|\/[^\/)]*?\/|[^()\"'\/])*?)(\))(\s*))?([\-=]>)/.source
-                regex: "(\\()(\\s*)(\\{)(\\s*)([$@A-Za-z_\\x7f-\\uffff][$@\\w\\s,\\x7f-\\uffff]*)(\\s*)(\\})(\\s*)(\\))(\\s*)([\\-=]>)"
-            },
-            "({})->": {
-                token: ["paren.lparen", "text", "paren.lparen", "text", "paren.rparen", "text", "paren.rparen", "text", "storage.type"],
-                regex: "(\\()(\\s*)(\\{)(\\s*)(\\})(\\s*)(\\))(\\s*)([\\-=]>)"
-            },
-            "(args)->": {
-                token: ["paren.lparen", "text", "variable.parameter", "text", "paren.rparen", "text", "storage.type"],
-                regex: "(\\()(\\s*)([$@A-Za-z_\\x7f-\\uffff][\\s\\x21-\\uffff]*)(\\s*)(\\))(\\s*)([\\-=]>)"
-            },
-            "()->": {
-                token: ["paren.lparen", "text", "paren.rparen", "text", "storage.type"],
-                regex: "(\\()(\\s*)(\\))(\\s*)([\\-=]>)"
-            }
         };
 
+        var stringEscape = /\\(?:x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)/;
+
         this.$rules = {
             start : [
                 {
                     token : "constant.numeric",
                     regex : "(?:0x[\\da-fA-F]+|(?:\\d+(?:\\.\\d+)?|\\.\\d+)(?:[eE][+-]?\\d+)?)"
                 }, {
-                    token : "string",
+                    stateName: "qdoc",
-                    regex : "'''",
+                    token : "string", regex : "'''", next : [
-                    next : "qdoc"
+                        {token : "string", regex : "'''", next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"},
+                    ]
                 }, {
+                    stateName: "qqdoc",
                     token : "string",
                     regex : '"""',
-                    next : "qqdoc"
+                    next : [
+                        {token : "string", regex : '"""', next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"}
+                    ]
                 }, {
-                    token : "string",
+                    stateName: "qstring",
-                    regex : "'",
+                    token : "string", regex : "'", next : [
-                    next : "qstring"
+                        {token : "string", regex : "'", next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"},
+                    ]
                 }, {
-                    token : "string",
+                    stateName: "qqstring",
-                    regex : '"',
+                    token : "string.start", regex : '"', next : [
-                    next : "qqstring"
+                        {token : "string.end", regex : '"', next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"},
+                    ]
                 }, {
-                    token : "string",
+                    stateName: "js",
-                    regex : "`",
+                    token : "string", regex : "`", next : [
-                    next : "js"
+                        {token : "string", regex : "`", next : "start"},
+                        {token : "constant.language.escape", regex : stringEscape},
+                        {defaultToken: "string"},
+                    ]
                 }, {
                     token : "string.regex",
                     regex : "///",
@@ -145,94 +146,23 @@ define(function(require, exports, module) {
                     token : "comment",
                     regex : "#.*"
                 }, {
-                    token : [
+                    token : ["punctuation.operator", "text", "identifier"],
-                        "punctuation.operator", "identifier"
+                    regex : "(\\.)(\\s*)(" + illegal + ")"
-                    ],
-                    regex : "(\\.)(" + illegal + ")"
                 }, {
                     token : "punctuation.operator",
                     regex : "\\."
                 }, {
                     //class A extends B
-                    token : [
+                    token : ["keyword", "text", "language.support.class",
-                        "keyword", "text", "language.support.class", "text", "keyword", "text", "language.support.class"
+                     "text", "keyword", "text", "language.support.class"],
-                    ],
+                    regex : "(class)(\\s+)(" + identifier + ")(?:(\\s+)(extends)(\\s+)(" + identifier + "))?"
-                    regex : "(class)(\\s+)(" + identifier + ")(\\s+)(extends)(\\s+)(" + identifier + ")"
                 }, {
-                    //class A
+                    //play = (...) ->
-                    token : [
+                    token : ["entity.name.function", "text", "keyword.operator", "text"].concat(functionRule.token),
-                        "keyword", "text", "language.support.class"
+                    regex : "(" + identifier + ")(\\s*)([=:])(\\s*)" + functionRule.regex
-                    ],
-                    regex : "(class)(\\s+)(" + identifier + ")"
-                }, {
-                    //play = ({args}) ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text"
-                    ].concat(functionRules["({args})->"].token),
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)" + functionRules["({args})->"].regex
-                }, {
-                    //play : ({args}) ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text"
-                    ].concat(functionRules["({args})->"].token),
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)" + functionRules["({args})->"].regex
-                }, {
-                    //play = ({}) ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text"
-                    ].concat(functionRules["({})->"].token),
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)" + functionRules["({})->"].regex
-                }, {
-                    //play : ({}) ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text"
-                    ].concat(functionRules["({})->"].token),
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)" + functionRules["({})->"].regex
-                }, {
-                    //play = (args) ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text"
-                    ].concat(functionRules["(args)->"].token),
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)" + functionRules["(args)->"].regex
-                }, {
-                    //play : (args) ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text"
-                    ].concat(functionRules["(args)->"].token),
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)" + functionRules["(args)->"].regex
-                }, {
-                    //play = () ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text"
-                    ].concat(functionRules["()->"].token),
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)" + functionRules["()->"].regex
-                }, {
-                    //play : () ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text"
-                    ].concat(functionRules["()->"].token),
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)" + functionRules["()->"].regex
-                }, {
-                    //play = ->
-                    token : [
-                        "entity.name.function", "text", "keyword.operator", "text", "storage.type"
-                    ],
-                    regex : "(" + identifier + ")(\\s*)(=)(\\s*)([\\-=]>)"
-                }, {
-                    //play : ->
-                    token : [
-                        "entity.name.function", "text", "punctuation.operator", "text", "storage.type"
-                    ],
-                    regex : "(" + identifier + ")(\\s*)(:)(\\s*)([\\-=]>)"
                 }, 
-                functionRules["({args})->"],
+                functionRule, 
-                functionRules["({})->"],
+                {
-                functionRules["(args)->"],
-                functionRules["()->"]
-                , {
-                    token : "identifier",
-                    regex : "(?:(?:\\.|::)\\s*)" + identifier
-                }, {
                     token : "variable",
                     regex : "@(?:" + identifier + ")?"
                 }, {
@@ -240,7 +170,7 @@ define(function(require, exports, module) {
                     regex : identifier
                 }, {
                     token : "punctuation.operator",
-                    regex : "\\?|\\:|\\,|\\."
+                    regex : "\\,|\\."
                 }, {
                     token : "storage.type",
                     regex : "[\\-=]>"
@@ -258,35 +188,6 @@ define(function(require, exports, module) {
                     regex : "\\s+"
                 }],
 
-            qdoc : [{
-                token : "string",
-                regex : ".*?'''",
-                next : "start"
-            }, stringfill],
-
-            qqdoc : [{
-                token : "string",
-                regex : '.*?"""',
-                next : "start"
-            }, stringfill],
-
-            qstring : [{
-                token : "string",
-                regex : "[^\\\\']*(?:\\\\.[^\\\\']*)*'",
-                next : "start"
-            }, stringfill],
-
-            qqstring : [{
-                token : "string",
-                regex : '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
-                next : "start"
-            }, stringfill],
-
-            js : [{
-                token : "string",
-                regex : "[^\\\\`]*(?:\\\\.[^\\\\`]*)*`",
-                next : "start"
-            }, stringfill],
 
             heregex : [{
                 token : "string.regex",
@@ -302,13 +203,13 @@ define(function(require, exports, module) {
 
             comment : [{
                 token : "comment",
-                regex : '.*?###',
+                regex : '###',
                 next : "start"
             }, {
-                token : "comment",
+                defaultToken : "comment",
-                regex : ".+"
             }]
         };
+        this.normalizeRules();
     }
 
     exports.CoffeeHighlightRules = CoffeeHighlightRules;

lib/ace/mode/coldfusion_highlight_rules.js

@@ -66,24 +66,12 @@ var ColdfusionHighlightRules = function() {
             token : "meta.tag", // opening tag
             regex : "<\\/?",
             next : "tag"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : "[^<]+"
         } ],
     
         cdata : [ {
             token : "text",
             regex : "\\]\\]>",
             next : "start"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : ".+"
         } ],
 
         comment : [ {
@@ -91,8 +79,7 @@ var ColdfusionHighlightRules = function() {
             regex : ".*?-->",
             next : "start"
         }, {
-            token : "comment",
+            defaultToken : "comment"
-            regex : ".+"
         } ]
     };
     

lib/ace/mode/css_highlight_rules_test.js

@@ -1,92 +0,0 @@
-/* ***** BEGIN LICENSE BLOCK *****
- * Distributed under the BSD license:
- *
- * Copyright (c) 2010, Ajax.org B.V.
- * All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Ajax.org B.V. nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * ***** END LICENSE BLOCK ***** */
-
-if (typeof process !== "undefined") {
-    require("amd-loader");
-}
-
-define(function(require, exports, module) {
-"use strict";
-
-var CssMode = require("./css").Mode;
-var assert = require("../test/assertions");
-
-module.exports = {
-    
-    name: "CSS Tokenizer",
-    
-    setUp : function() {
-        this.tokenizer = new CssMode().getTokenizer();
-    },
-
-    "test: tokenize pixel number" : function() {
-        var line = "-12px";
-        var tokens = this.tokenizer.getLineTokens(line, "ruleset").tokens;
-
-        assert.equal(2, tokens.length);
-        assert.equal("constant.numeric", tokens[0].type);
-    },
-
-    "test: tokenize hex3 color" : function() {
-        var tokens = this.tokenizer.getLineTokens("#abc", "ruleset").tokens;
-
-        assert.equal(1, tokens.length);
-        assert.equal("constant.numeric", tokens[0].type);
-    },
-
-    "test: tokenize hex6 color" : function() {
-        var tokens = this.tokenizer.getLineTokens("#abc012", "ruleset").tokens;
-
-        assert.equal(1, tokens.length);
-        assert.equal("constant.numeric", tokens[0].type);
-    },
-
-    "test: tokenize parens" : function() {
-        var tokens = this.tokenizer.getLineTokens("{()}", "start").tokens;
-
-        assert.equal(3, tokens.length);
-        assert.equal("paren.lparen", tokens[0].type);
-        assert.equal("text", tokens[1].type);
-        assert.equal("paren.rparen", tokens[2].type);
-    },
-    
-    "test for last rule in ruleset to catch capturing group bugs" : function() {
-        var tokens = this.tokenizer.getLineTokens("top", "ruleset").tokens;
-        
-        assert.equal(1, tokens.length);
-        assert.equal("support.type", tokens[0].type);
-    }
-};
-
-});
-
-if (typeof module !== "undefined" && module === require.main) {
-    require("asyncjs").test.testcase(module.exports).exec()
-}

lib/ace/mode/dart_highlight_rules.js

@@ -38,101 +38,101 @@ var DartHighlightRules = function() {
         {
     "start": [
         {
-            "token" : "comment",
+            token : "comment",
-            "regex" : /\/\/.*$/
+            regex : /\/\/.*$/
         },
         {
-            "token" : "comment", // multi line comment
+            token : "comment", // multi line comment
-            "regex" : /\/\*/,
+            regex : /\/\*/,
-            "next" : "comment"
+            next : "comment"
         },
         {
-            "token": ["meta.preprocessor.script.dart"],
+            token: ["meta.preprocessor.script.dart"],
-            "regex": "^(#!.*)$"
+            regex: "^(#!.*)$"
         },
         {
-            "token": [ "keyword.other.import.dart", "meta.declaration.dart"],
+            token: "keyword.other.import.dart",
-            "regex": "#(?:\\b)(?:library|import|source|resource)(?:\\b)"
+            regex: "#(?:\\b)(?:library|import|source|resource)(?:\\b)"
         },
         {
-            "token" : ["keyword.other.import.dart", "text"],
+            token : ["keyword.other.import.dart", "text"],
-            "regex" : "(?:\\b)(prefix)(\\s*:)"
+            regex : "(?:\\b)(prefix)(\\s*:)"
         },
         {
-            "regex": "\\bas\\b",
+            regex: "\\bas\\b",
-            "token": "keyword.cast.dart"
+            token: "keyword.cast.dart"
         },
         {
-            "regex": "\\?|:",
+            regex: "\\?|:",
-            "token": "keyword.control.ternary.dart"
+            token: "keyword.control.ternary.dart"
         },
         {
-            "regex": "(?:\\b)(is\\!?)(?:\\b)",
+            regex: "(?:\\b)(is\\!?)(?:\\b)",
-            "token": ["keyword.operator.dart"]
+            token: ["keyword.operator.dart"]
         },
         {
-            "regex": "(<<|>>>?|~|\\^|\\||&)",
+            regex: "(<<|>>>?|~|\\^|\\||&)",
-            "token": ["keyword.operator.bitwise.dart"]
+            token: ["keyword.operator.bitwise.dart"]
         },
         {
-            "regex": "((?:&|\\^|\\||<<|>>>?)=)",
+            regex: "((?:&|\\^|\\||<<|>>>?)=)",
-            "token": ["keyword.operator.assignment.bitwise.dart"]
+            token: ["keyword.operator.assignment.bitwise.dart"]
         },
         {
-            "regex": "(===?|!==?|<=?|>=?)",
+            regex: "(===?|!==?|<=?|>=?)",
-            "token": ["keyword.operator.comparison.dart"]
+            token: ["keyword.operator.comparison.dart"]
         },
         {
-            "regex": "((?:[+*/%-]|\\~)=)",
+            regex: "((?:[+*/%-]|\\~)=)",
-            "token": ["keyword.operator.assignment.arithmetic.dart"]
+            token: ["keyword.operator.assignment.arithmetic.dart"]
         },
         {
-            "regex": "=",
+            regex: "=",
-            "token": "keyword.operator.assignment.dart"
+            token: "keyword.operator.assignment.dart"
         },
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "'''",
+            regex : "'''",
-            "next" : "qdoc"
+            next : "qdoc"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"""',
+            regex : '"""',
-            "next" : "qqdoc"
+            next : "qqdoc"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "'",
+            regex : "'",
-            "next" : "qstring"
+            next : "qstring"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"',
+            regex : '"',
-            "next" : "qqstring"
+            next : "qqstring"
         }, 
         {
-            "regex": "(\\-\\-|\\+\\+)",
+            regex: "(\\-\\-|\\+\\+)",
-            "token": ["keyword.operator.increment-decrement.dart"]
+            token: ["keyword.operator.increment-decrement.dart"]
         },
         {
-            "regex": "(\\-|\\+|\\*|\\/|\\~\\/|%)",
+            regex: "(\\-|\\+|\\*|\\/|\\~\\/|%)",
-            "token": ["keyword.operator.arithmetic.dart"]
+            token: ["keyword.operator.arithmetic.dart"]
         },
         {
-            "regex": "(!|&&|\\|\\|)",
+            regex: "(!|&&|\\|\\|)",
-            "token": ["keyword.operator.logical.dart"]
+            token: ["keyword.operator.logical.dart"]
         },
         {
-            "token" : "constant.numeric", // hex
+            token : "constant.numeric", // hex
-            "regex" : "0[xX][0-9a-fA-F]+\\b"
+            regex : "0[xX][0-9a-fA-F]+\\b"
         }, 
         {
-            "token" : "constant.numeric", // float
+            token : "constant.numeric", // float
-            "regex" : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
+            regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
         }, 
         {
-            "token" : keywordMapper,
+            token : keywordMapper,
-            "regex" : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
         }
     ],
     "comment" : [
@@ -147,30 +147,30 @@ var DartHighlightRules = function() {
     ],
     "qdoc" : [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : ".*?'''",
+            regex : ".*?'''",
-            "next" : "start"
+            next : "start"
         }, stringfill],
 
     "qqdoc" : [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '.*?"""',
+            regex : '.*?"""',
-            "next" : "start"
+            next : "start"
         }, stringfill],
 
     "qstring" : [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "[^\\\\']*(?:\\\\.[^\\\\']*)*'",
+            regex : "[^\\\\']*(?:\\\\.[^\\\\']*)*'",
-            "next" : "start"
+            next : "start"
         }, stringfill],
 
     "qqstring" : [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
+            regex : '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
-            "next" : "start"
+            next : "start"
         }, stringfill]
 }
 

lib/ace/mode/diff_highlight_rules.js

@@ -40,20 +40,20 @@ var DiffHighlightRules = function() {
 
     this.$rules = {
         "start" : [{
-                "regex": "^(?:\\*{15}|={67}|-{3}|\\+{3})$",
+                regex: "^(?:\\*{15}|={67}|-{3}|\\+{3})$",
-                "token": "punctuation.definition.separator.diff",
+                token: "punctuation.definition.separator.diff",
                 "name": "keyword"
             }, { //diff.range.unified
-                "regex": "^(@@)(\\s*.+?\\s*)(@@)(.*)$",
+                regex: "^(@@)(\\s*.+?\\s*)(@@)(.*)$",
-                "token": [
+                token: [
                     "constant",
                     "constant.numeric",
                     "constant",
                     "comment.doc.tag"
                 ]
             }, { //diff.range.normal
-                "regex": "^(\\d+)([,\\d]+)(a|d|c)(\\d+)([,\\d]+)(.*)$",
+                regex: "^(\\d+)([,\\d]+)(a|d|c)(\\d+)([,\\d]+)(.*)$",
-                "token": [
+                token: [
                     "constant.numeric",
                     "punctuation.definition.range.diff",
                     "constant.function",
@@ -63,34 +63,36 @@ var DiffHighlightRules = function() {
                 ],
                 "name": "meta."
             }, {
-                "regex": "^(?:(\\-{3}|\\+{3}|\\*{3})( .+))$",
+                regex: "^(\\-{3}|\\+{3}|\\*{3})( .+)$",
-                "token": [
+                token: [
                     "constant.numeric",
                     "meta.tag"
                 ]
             }, { // added
-                "regex": "^([!+>])(.*?)(\\s*)$",
+                regex: "^([!+>])(.*?)(\\s*)$",
-                "token": [
+                token: [
                     "support.constant",
                     "text",
                     "invalid"
                 ]
             }, { // removed
-                "regex": "^([<\\-])(.*?)(\\s*)$",
+                regex: "^([<\\-])(.*?)(\\s*)$",
-                "token": [
+                token: [
                     "support.function",
                     "string",
                     "invalid"
                 ]
             }, {
-                "regex": "^(diff)(\\s+--\\w+)?(.+?)( .+)?$",
+                regex: "^(diff)(\\s+--\\w+)?(.+?)( .+)?$",
-                "token": ["variable", "variable", "keyword", "variable"]
+                token: ["variable", "variable", "keyword", "variable"]
             }, {
-                "regex": "^Index.+$",
+                regex: "^Index.+$",
-                "token": "variable"
+                token: "variable"
             }, {
-                "regex": "^(.*?)(\\s*)$",
+                regex: "\\s*$",
-                "token": ["invisible", "invalid"]
+                token: "invalid"
+            }, {
+                defaultToken: "invisible"
             }
         ]
     };

lib/ace/mode/django.js

@@ -0,0 +1,117 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2012, Ajax.org B.V.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define('ace/mode/django', ['require', 'exports', 'module' , 'ace/lib/oop', 'ace/mode/html', 'ace/mode/text_highlight_rules', 'ace/tokenizer', 'ace/mode/html_highlight_rules'], function(require, exports, module) {
+
+var oop = require("../lib/oop");
+var HtmlMode = require("./html").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var HtmlHighlightRules = require("./html_highlight_rules").HtmlHighlightRules;
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var DjangoHighlightRules = function(){
+    this.$rules = {
+        'start': [{
+            token: "string",
+            regex: '".*?"'
+        }, {
+            token: "string",
+            regex: "'.*?'"
+        }, {
+            token: "constant",
+            regex: '[0-9]+'
+        }, {
+            token: "variable",
+            regex: "[-_a-zA-Z0-9:]+"
+        }],
+        'comment': [{
+            token : "comment.block",
+            merge: true,
+            regex : ".+?"
+        }],
+        'tag': [{
+            token: "entity.name.function",
+            regex: "[a-zA-Z][_a-zA-Z0-9]*",
+            next: "start"
+        }]
+    };
+};
+
+oop.inherits(DjangoHighlightRules, TextHighlightRules)
+
+var DjangoHtmlHighlightRules = function() {
+    this.$rules = new HtmlHighlightRules().getRules();
+
+    for (var i in this.$rules) {
+        this.$rules[i].unshift({
+            token: "comment.line",
+            regex: "\\{#.*?#\\}"
+        }, {
+            token: "comment.block",
+            regex: "\\{\\%\\s*comment\\s*\\%\\}",
+            merge: true,
+            next: "django-comment"
+        }, {
+            token: "constant.language",
+            regex: "\\{\\{",
+            next: "django-start"
+        }, {
+            token: "constant.language",
+            regex: "\\{\\%",
+            next: "django-tag"
+        });
+        this.embedRules(DjangoHighlightRules, "django-", [{
+                token: "comment.block",
+                regex: "\\{\\%\\s*endcomment\\s*\\%\\}",
+                merge: true,
+                next: "start"
+            }, {
+                token: "constant.language",
+                regex: "\\%\\}",
+                next: "start"
+            }, {
+                token: "constant.language",
+                regex: "\\}\\}",
+                next: "start"
+        }]);
+    }
+};
+
+oop.inherits(DjangoHtmlHighlightRules, HtmlHighlightRules);
+
+var Mode = function() {
+    var highlighter = new DjangoHtmlHighlightRules();
+    this.$tokenizer = new Tokenizer(highlighter.getRules());
+    this.$embeds = highlighter.getEmbeds();
+};
+oop.inherits(Mode, HtmlMode);
+
+exports.Mode = Mode;
+});

lib/ace/mode/doc_comment_highlight_rules.js

@@ -41,17 +41,10 @@ var DocCommentHighlightRules = function() {
             token : "comment.doc.tag",
             regex : "@[\\w\\d_]+" // TODO: fix email addresses
         }, {
-            token : "comment.doc",
+            token : "comment.doc.tag",
-            regex : "\\s+"
+            regex : "\\bTODO\\b"
         }, {
-            token : "comment.doc",
+            defaultToken : "comment.doc"
-            regex : "TODO"
-        }, {
-            token : "comment.doc",
-            regex : "[^@\\*]+"
-        }, {
-            token : "comment.doc",
-            regex : "."
         }]
     };
 };

lib/ace/mode/haml_highlight_rules.js

@@ -15,69 +15,69 @@ var HamlHighlightRules = function() {
         {
     "start": [
         {
-            "token" : "punctuation.section.comment",
+            token : "punctuation.section.comment",
-            "regex" : /^\s*\/.*/
+            regex : /^\s*\/.*/
         },
         {
-            "token" : "punctuation.section.comment",
+            token : "punctuation.section.comment",
-            "regex" : /^\s*#.*/
+            regex : /^\s*#.*/
         },
         {
-            "token": "string.quoted.double",
+            token: "string.quoted.double",
-            "regex": "==.+?=="
+            regex: "==.+?=="
         },
         {
-            "token": "keyword.other.doctype",
+            token: "keyword.other.doctype",
-            "regex": "^!!!\\s*(?:[a-zA-Z0-9-_]+)?"
+            regex: "^!!!\\s*(?:[a-zA-Z0-9-_]+)?"
         },
         RubyExports.qString,
         RubyExports.qqString,
         RubyExports.tString,
         {
-            "token": ["entity.name.tag.haml"],
+            token: ["entity.name.tag.haml"],
-            "regex": /^\s*%[\w:]+/,
+            regex: /^\s*%[\w:]+/,
-            "next": "tag_single"
+            next: "tag_single"
         },
         {
-            "token": [ "meta.escape.haml" ],
+            token: [ "meta.escape.haml" ],
-            "regex": "^\\s*\\\\."
+            regex: "^\\s*\\\\."
         },
         RubyExports.constantNumericHex,
         RubyExports.constantNumericFloat,
         
         RubyExports.constantOtherSymbol,
         {
-            "token": "text",
+            token: "text",
-            "regex": "=|-|~",
+            regex: "=|-|~",
-            "next": "embedded_ruby"
+            next: "embedded_ruby"
         }
     ],
     "tag_single": [
         {
-            "token": "entity.other.attribute-name.class.haml",
+            token: "entity.other.attribute-name.class.haml",
-            "regex": "\\.[\\w-]+"
+            regex: "\\.[\\w-]+"
         },
         {
-            "token": "entity.other.attribute-name.id.haml",
+            token: "entity.other.attribute-name.id.haml",
-            "regex": "#[\\w-]+"
+            regex: "#[\\w-]+"
         },
         {
-            "token": "punctuation.section",
+            token: "punctuation.section",
-            "regex": "\\{",
+            regex: "\\{",
-            "next": "section"
+            next: "section"
         },
         
         RubyExports.constantOtherSymbol,
         
         {
-            "token": "text",
+            token: "text",
-            "regex": /\s/,
+            regex: /\s/,
-            "next": "start"
+            next: "start"
         },
         {
-            "token": ["text", "punctuation"],
+            token: "empty",
-            "regex": "($)|((?!\\.|#|\\{|\\[|=|-|~|\\/))",
+            regex: "$|(?!\\.|#|\\{|\\[|=|-|~|\\/)",
-            "next": "start"
+            next: "start"
         }
     ],
     "section": [
@@ -90,9 +90,9 @@ var HamlHighlightRules = function() {
         RubyExports.constantNumericHex,
         RubyExports.constantNumericFloat,
         {
-            "token": "punctuation.section",
+            token: "punctuation.section",
-            "regex": "\\}",
+            regex: "\\}",
-            "next": "start"
+            next: "start"
         } 
     ],
     "embedded_ruby": [ 
@@ -103,23 +103,23 @@ var HamlHighlightRules = function() {
                 regex : "[A-Z][a-zA-Z_\\d]+"
         },    
         {
-            "token" : new RubyHighlightRules().getKeywords(),
+            token : new RubyHighlightRules().getKeywords(),
-            "regex" : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
         },
         {
-            "token" : ["keyword", "text", "text"],
+            token : ["keyword", "text", "text"],
-            "regex" : "(?:do|\\{)(?: \\|[^|]+\\|)?$",
+            regex : "(?:do|\\{)(?: \\|[^|]+\\|)?$",
-            "next"  : "start"
+            next  : "start"
         }, 
         {
-            "token" : ["text"],
+            token : ["text"],
-            "regex" : "^$",
+            regex : "^$",
-            "next"  : "start"
+            next  : "start"
         }, 
         {
-            "token" : ["text"],
+            token : ["text"],
-            "regex" : "^(?!.*\\|\\s*$)",
+            regex : "^(?!.*\\|\\s*$)",
-            "next"  : "start"
+            next  : "start"
         }
     ]
 }

lib/ace/mode/html_highlight_rules.js

@@ -94,21 +94,12 @@ var HtmlHighlightRules = function() {
         }, {
             token : "constant.character.entity",
             regex : "(?:&#[0-9]+;)|(?:&#x[0-9a-fA-F]+;)|(?:&[a-zA-Z0-9_:\\.-]+;)"
-        }, {
-            token : "text",
-            regex : "[^<]+"
         }],
     
         cdata : [ {
             token : "text",
             regex : "\\]\\]>",
             next : "start"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : ".+"
         } ],
 
         comment : [ {
@@ -116,8 +107,7 @@ var HtmlHighlightRules = function() {
             regex : ".*?-->",
             next : "start"
         }, {
-            token : "comment",
+            defaultToken : "comment"
-            regex : ".+"
         } ]
     };
     

lib/ace/mode/html_highlight_rules_test.js

@@ -1,186 +0,0 @@
-/* ***** BEGIN LICENSE BLOCK *****
- * Distributed under the BSD license:
- *
- * Copyright (c) 2010, Ajax.org B.V.
- * All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Ajax.org B.V. nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * ***** END LICENSE BLOCK ***** */
-
-if (typeof process !== "undefined") {
-    require("amd-loader");
-}
-
-define(function(require, exports, module) {
-"use strict";
-
-var HtmlMode = require("./html").Mode;
-var assert = require("../test/assertions");
-
-var testData = {
-    "test: tokenize embedded script" : [{
-        text: "<script a='a'>var</script>'123'",
-        state: ["start", "start"],
-        tokens: [{
-            type: "meta.tag",
-            value: "<"
-        }, {
-            type: "meta.tag.tag-name.script",
-            value: "script"
-        }, {
-            type: "text",
-            value: " "
-        }, {
-            type: "entity.other.attribute-name",
-            value: "a"
-        }, {
-            type: "keyword.operator",
-            value: "="
-        }, {
-            type: "string",
-            value: "'a'"
-        }, {
-            type: "meta.tag.r",
-            value: ">"
-        }, {
-            type: "storage.type",
-            value: "var"
-        }, {
-            type: "meta.tag",
-            value: "</"
-        }, {
-            type: "meta.tag.tag-name.script",
-            value: "script"
-        }, {
-            type: "meta.tag.r",
-            value: ">"
-        }, {
-            type: "text",
-            value: "'123'"
-        }]
-    }],
-
-    "test: tokenize multiline attribute value with double quotes": [{
-        text: "<a href=\"abc",
-        state: [ "start", "tag_qqstring"],
-        tokens: [{
-                type: "meta.tag",
-                value: "<"
-            }, {
-                type: "meta.tag.tag-name.anchor",
-                value: "a"
-            }, {
-                type: "text",
-                value: " "
-            }, {
-                type: "entity.other.attribute-name",
-                value: "href"
-            }, {
-                type: "keyword.operator",
-                value: "="
-            }, {
-                type: "string",
-                value: "\"abc"
-            }
-        ]
-    }, {
-        text: "def\">",
-        state: [ "tag_qqstring", "start" ],
-        tokens: [ {
-                type: "string",
-                value: "def\""
-            }, {
-                type: "meta.tag.r",
-                value: ">"
-            }
-        ]
-    }],
-
-    "test: tokenize multiline attribute value with single quotes": [{
-        text: "<a href='abc",
-        state: ["start", "tag_qstring"],
-        tokens: [{
-                type: "meta.tag",
-                value: "<"
-            }, {
-                type: "meta.tag.tag-name.anchor",
-                value: "a"
-            }, {
-                type: "text",
-                value: " "
-            }, {
-                type: "entity.other.attribute-name",
-                value: "href"
-            }, {
-                type: "keyword.operator",
-                value: "="
-            }, {
-                type: "string",
-                value: "'abc"
-            }
-        ]
-    }, {
-        text: "def\"'>",
-        state: [ "tag_qstring", "start" ],
-        tokens: [ {
-                type: "string",
-                value: "def\"'"
-            }, {
-                type: "meta.tag.r",
-                value: ">"
-            }
-        ]
-    }]
-};
-
-function generateTest(exampleData) {
-    return function testTokenizer() {
-        for (var i = 0; i < exampleData.length; i++) {
-            var s = exampleData[i];
-            var lineTokens = tokenizer.getLineTokens(s.text, s.state[0]);
-
-            assert.equal(
-                JSON.stringify(lineTokens, null, 4),
-                JSON.stringify({tokens:s.tokens, state: s.state[1]}, null, 4)
-            );
-        }
-    }
-}
-
-var tokenizer;
-module.exports = {
-    setUp : function() {
-        tokenizer = new HtmlMode().getTokenizer();
-    }
-}
-
-for (var i in testData) {
-    module.exports[i] = generateTest(testData[i])
-}
-
-});
-
-if (typeof module !== "undefined" && module === require.main) {
-    require("asyncjs").test.testcase(module.exports).exec();
-}

lib/ace/mode/jade_highlight_rules.js

@@ -73,23 +73,23 @@ var JadeHighlightRules = function() {
         {
     "start": [
         {
-            "token": "keyword.control.import.include.jade",
+            token: "keyword.control.import.include.jade",
-            "regex": "\\s*\\binclude\\b"
+            regex: "\\s*\\binclude\\b"
         },
         {
-            "token": "keyword.other.doctype.jade",
+            token: "keyword.other.doctype.jade",
-            "regex": "^!!!\\s*(?:[a-zA-Z0-9-_]+)?"
+            regex: "^!!!\\s*(?:[a-zA-Z0-9-_]+)?"
         },
         {
-            "token" : "punctuation.section.comment",
+            token : "punctuation.section.comment",
-            "regex" : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)"
+            regex : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)"
         },
         {
-            "token" : function(space, text) {
+            token : function(space, text) {
                 return "punctuation.section.comment";
             },
-            "regex" : "^((\\s*)\/\/)(?:\\s*$)",
+            regex : "^((\\s*)\/\/)(?:\\s*$)",
-            "next": "comment_block"
+            next: "comment_block"
         },
         mixin_embed("markdown", "markdown-"),
         mixin_embed("sass", "sass-"),
@@ -97,100 +97,100 @@ var JadeHighlightRules = function() {
         mixin_embed("coffee", "coffee-"),
         /*
         {
-            "token": {
+            token: {
                 "2": {
                     "name": "entity.name.function.jade"
                 }
             },
-            "regex": "^(\\s*)(\\:cdata)",
+            regex: "^(\\s*)(\\:cdata)",
-            "next": "state_9"
+            next: "state_9"
         },*/
         // match stuff like: mixin dialog-title-desc(title, desc)
         {
-            "token": [ "storage.type.function.jade",
+            token: [ "storage.type.function.jade",
                        "entity.name.function.jade",
                        "punctuation.definition.parameters.begin.jade",
                        "variable.parameter.function.jade",
                        "punctuation.definition.parameters.end.jade"
                     ],
-            "regex": "^(\\s*mixin)( [\\w\\-]+)(\\s*\\()(.*?)(\\))"
+            regex: "^(\\s*mixin)( [\\w\\-]+)(\\s*\\()(.*?)(\\))"
         },
         // match stuff like: mixin dialog-title-desc
         {
-            "token": [ "storage.type.function.jade", "entity.name.function.jade"],
+            token: [ "storage.type.function.jade", "entity.name.function.jade"],
-            "regex": "^(\\s*mixin)( [\\w\\-]+)"
+            regex: "^(\\s*mixin)( [\\w\\-]+)"
         },
         {
-            "token": "source.js.embedded.jade",
+            token: "source.js.embedded.jade",
-            "regex": "^\\s*(?:-|=|!=)",
+            regex: "^\\s*(?:-|=|!=)",
-            "next": "js-start"
+            next: "js-start"
         },
         /*{
-            "token": "entity.name.tag.script.jade",
+            token: "entity.name.tag.script.jade",
-            "regex": "^\\s*script",
+            regex: "^\\s*script",
-            "next": "js_code_tag"
+            next: "js_code_tag"
         },*/
         {
-            "token": "string.interpolated.jade",
+            token: "string.interpolated.jade",
-            "regex": "[#!]\\{[^\\}]+\\}"
+            regex: "[#!]\\{[^\\}]+\\}"
         },
         // Match any tag, id or class. skip AST filters
         {
-            "token": ["meta.tag.any.jade", "entity.variable.tag.jade"],
+            token: "meta.tag.any.jade",
-            "regex": /^\s*(?!\w+\:)(?:[\w]+|(?=\.|#)])/,
+            regex: /^\s*(?!\w+\:)(?:[\w]+|(?=\.|#)])/,
-            "next": "tag_single"
+            next: "tag_single"
         },
         {
-            "token": "suport.type.attribute.id.jade",
+            token: "suport.type.attribute.id.jade",
-            "regex": "#\\w+"
+            regex: "#\\w+"
         },
         {
-            "token": "suport.type.attribute.class.jade",
+            token: "suport.type.attribute.class.jade",
-            "regex": "\\.\\w+"
+            regex: "\\.\\w+"
         },
         {
-            "token": "punctuation",
+            token: "punctuation",
-            "regex": "\\s*(?:\\()",
+            regex: "\\s*(?:\\()",
-            "next": "tag_attributes"
+            next: "tag_attributes"
         }
     ],
     "comment_block": [
         {
-            "token": function(text) {
+            token: function(text) {
                 return "text";
             },
-            "regex": "^(\\1\\S|$)", 
+            regex: "^(\\1\\S|$)", 
             "captures": "1",
-            "next": "start"
+            next: "start"
         },
         {
-            "token": "comment.block.jade",
+            token: "comment.block.jade",
-            "regex" : ".+"
+            regex : ".+"
         }
     ],
     /*
     
     "state_9": [
         {
-            "token": "TODO",
+            token: "TODO",
-            "regex": "^(?!\\1\\s+)",
+            regex: "^(?!\\1\\s+)",
-            "next": "start"
+            next: "start"
         },
         {
-            "token": "TODO",
+            token: "TODO",
-            "regex": ".+",
+            regex: ".+",
-            "next": "state_9"
+            next: "state_9"
         }
     ],*/
     /*"js_code": [
         {
-            "token": "keyword.control.js",
+            token: "keyword.control.js",
-            "regex": "\\beach\\b"
+            regex: "\\beach\\b"
         },
         {
-            "token": "text",
+            token: "text",
-            "regex": "$",
+            regex: "$",
-            "next": "start"
+            next: "start"
         }
     ],*/
     /*"js_code_tag": [
@@ -198,62 +198,62 @@ var JadeHighlightRules = function() {
             "include": "source.js"
         },
         {
-            "token": "TODO",
+            token: "TODO",
-            "regex": "^((?=(\\1)([\\w#\\.]|$\\n?))|^$\\n?)",
+            regex: "^((?=(\\1)([\\w#\\.]|$\\n?))|^$\\n?)",
-            "next": "start"
+            next: "start"
         }
     ],*/
     "tag_single": [
         {
-            "token": "entity.other.attribute-name.class.jade",
+            token: "entity.other.attribute-name.class.jade",
-            "regex": "\\.[\\w-]+"
+            regex: "\\.[\\w-]+"
         },
         {
-            "token": "entity.other.attribute-name.id.jade",
+            token: "entity.other.attribute-name.id.jade",
-            "regex": "#[\\w-]+"
+            regex: "#[\\w-]+"
         },
         {
-            "token": ["text", "punctuation"],
+            token: ["text", "punctuation"],
-            "regex": "($)|((?!\\.|#|=|-))",
+            regex: "($)|((?!\\.|#|=|-))",
-            "next": "start"
+            next: "start"
         }
     ],
     "tag_attributes": [ 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "'(?=.)",
+            regex : "'(?=.)",
-            "next"  : "qstring"
+            next  : "qstring"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"(?=.)',
+            regex : '"(?=.)',
-            "next"  : "qqstring"
+            next  : "qqstring"
         },
         {
-            "token": "entity.other.attribute-name.jade",
+            token: "entity.other.attribute-name.jade",
-            "regex": "\\b[a-zA-Z\\-:]+"
+            regex: "\\b[a-zA-Z\\-:]+"
         },
         {
-            "token": ["entity.other.attribute-name.jade", "punctuation"],
+            token: ["entity.other.attribute-name.jade", "punctuation"],
-            "regex": "\\b([a-zA-Z:\\.-]+)(=)",
+            regex: "\\b([a-zA-Z:\\.-]+)(=)",
-            "next": "attribute_strings"
+            next: "attribute_strings"
         },
         {
-            "token": "punctuation",
+            token: "punctuation",
-            "regex": "\\)",
+            regex: "\\)",
-            "next": "start"
+            next: "start"
         }
     ],
     "attribute_strings": [
         {
-            "token" : "string",
+            token : "string",
-            "regex" : "'(?=.)",
+            regex : "'(?=.)",
-            "next"  : "qstring"
+            next  : "qstring"
         }, 
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"(?=.)',
+            regex : '"(?=.)',
-            "next"  : "qqstring"
+            next  : "qqstring"
         }
     ],
     "qqstring" : [

lib/ace/mode/javascript_highlight_rules.js

@@ -61,7 +61,8 @@ var JavaScriptHighlightRules = function() {
         "constant.language":
             "null|Infinity|NaN|undefined",
         "support.function":
-            "alert"
+            "alert",
+        "constant.language.boolean": "true|false"
     }, "identifier");
 
     // keywords which can be followed by regular expressions
@@ -161,9 +162,6 @@ var JavaScriptHighlightRules = function() {
                 ],
                 regex : "(:)(\\s*)(function)(\\s*)(\\()",
                 next: "function_arguments"
-            }, {
-                token : "constant.language.boolean",
-                regex : /(?:true|false)\b/
             }, {
                 token : "keyword",
                 regex : "(?:" + kwBeforeRe + ")\\b",
@@ -205,9 +203,6 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token: "comment",
                 regex: /^#!.*$/
-            }, {
-                token : "text",
-                regex : /\s+/
             }
         ],
         // regular expressions are only allowed after certain tokens. This
@@ -253,18 +248,20 @@ var JavaScriptHighlightRules = function() {
             }, {
                 // operators
                 token : "constant.language.escape",
-                regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[(|)$^+*?]/
+                regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[()$^+*?]/
             }, {
-                token: "string.regexp",
+                token : "constant.language.delimiter",
-                regex: /{|[^{\[\/\\(|)$^+*?]+/,
+                regex: /\|/
             }, {
                 token: "constant.language.escape",
                 regex: /\[\^?/,
                 next: "regex_character_class",
             }, {
                 token: "empty",
-                regex: "",
+                regex: "$",
                 next: "start"
+            }, {
+                defaultToken: "string.regexp"
             }
         ],
         "regex_character_class": [
@@ -278,13 +275,12 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token: "constant.language.escape",
                 regex: "-"
-            }, {
-                token: "string.regexp.charachterclass",
-                regex: /[^\]\-\\]+/,
             }, {
                 token: "empty",
-                regex: "",
+                regex: "$",
                 next: "start"
+            }, {
+                defaultToken: "string.regexp.charachterclass"
             }
         ],
         "function_arguments": [
@@ -304,24 +300,12 @@ var JavaScriptHighlightRules = function() {
             }
         ],
         "comment_regex_allowed" : [
-            {
+            {token : "comment", regex : "\\*\\/", next : "regex_allowed"},
-                token : "comment", // closing comment
+            {defaultToken : "comment"}
-                regex : ".*?\\*\\/",
-                next : "regex_allowed"
-            }, {
-                token : "comment", // comment spanning whole line
-                regex : ".+"
-            }
         ],
         "comment" : [
-            {
+            {token : "comment", regex : "\\*\\/", next : "start"},
-                token : "comment", // closing comment
+            {defaultToken : "comment"}
-                regex : ".*?\\*\\/",
-                next : "start"
-            }, {
-                token : "comment", // comment spanning whole line
-                regex : ".+"
-            }
         ],
         "qqstring" : [
             {
@@ -336,8 +320,7 @@ var JavaScriptHighlightRules = function() {
                 regex : '"|$',
                 next  : "start",
             }, {
-                token : "string",
+                defaultToken: "string"
-                regex : '.|\\w+|\\s+',
             }
         ],
         "qstring" : [
@@ -353,8 +336,7 @@ var JavaScriptHighlightRules = function() {
                 regex : "'|$",
                 next  : "start",
             }, {
-                token : "string",
+                defaultToken: "string"
-                regex : '.|\\w+|\\s+',
             }
         ]
     };

lib/ace/mode/liquid_highlight_rules.js

@@ -104,24 +104,12 @@ var LiquidHighlightRules = function() {
             token : "meta.tag", // opening tag
             regex : "<\\/?",
             next : "tag"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : "[^<]+"
         } ],
 
         cdata : [ {
             token : "text",
             regex : "\\]\\]>",
             next : "start"
-        }, {
-            token : "text",
-            regex : "\\s+"
-        }, {
-            token : "text",
-            regex : ".+"
         } ],
 
         comment : [ {
@@ -129,8 +117,7 @@ var LiquidHighlightRules = function() {
             regex : ".*?-->",
             next : "start"
         }, {
-            token : "comment",
+            defaultToken : "comment"
-            regex : ".+"
         } ] ,
 
         liquid_start : [{

lib/ace/mode/liquid_highlight_rules_test.js

@@ -1,80 +0,0 @@
-/* ***** BEGIN LICENSE BLOCK *****
- * Distributed under the BSD license:
- *
- * Copyright (c) 2010, Ajax.org B.V.
- * All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Ajax.org B.V. nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * ***** END LICENSE BLOCK ***** */
-
-if (typeof process !== "undefined") {
-    require("amd-loader");
-}
-
-define(function(require, exports, module) {
-
-var LiquidMode = require("./liquid").Mode;
-var assert = require("../test/assertions");
-
-module.exports = {
-
-    name: "Liquid Tokenizer",
-
-    setUp : function() {
-        this.tokenizer = new LiquidMode().getTokenizer();
-    },
-
-    "test: tokenize tags" : function() {
-        var line = "for one in many";
-
-        var tokens = this.tokenizer.getLineTokens(line, "liquid_start").tokens;
-
-        assert.equal(7, tokens.length);
-        assert.equal("keyword", tokens[0].type);
-        assert.equal("text", tokens[1].type);
-        assert.equal("identifier", tokens[2].type);
-        assert.equal("text", tokens[3].type);
-        assert.equal("keyword", tokens[4].type);
-        assert.equal("text", tokens[5].type);
-        assert.equal("identifier", tokens[6].type);
-    },
-
-    "test: tokenize parens" : function() {
-        var line = "[{( )}]";
-
-        var tokens = this.tokenizer.getLineTokens(line, "liquid_start").tokens;
-
-        assert.equal(3, tokens.length);
-        assert.equal("paren.lparen", tokens[0].type);
-        assert.equal("text", tokens[1].type);
-        assert.equal("paren.rparen", tokens[2].type);
-    }
-
-};
-
-});
-
-if (typeof module !== "undefined" && module === require.main) {
-    require("asyncjs").test.testcase(module.exports).exec()
-}

lib/ace/mode/lisp_highlight_rules.js

@@ -67,54 +67,51 @@ var LispHighlightRules = function() {
             regex : ";.*$"
         },
         {
-            "token": ["storage.type.function-type.lisp", "text", "entity.name.function.lisp"],
+            token: ["storage.type.function-type.lisp", "text", "entity.name.function.lisp"],
-            "regex": "(?:\\b(?:(defun|defmethod|defmacro))\\b)(\\s+)((?:\\w|\\-|\\!|\\?)*)"
+            regex: "(?:\\b(?:(defun|defmethod|defmacro))\\b)(\\s+)((?:\\w|\\-|\\!|\\?)*)"
         },
         {
-            "token": ["punctuation.definition.constant.character.lisp", "constant.character.lisp"],
+            token: ["punctuation.definition.constant.character.lisp", "constant.character.lisp"],
-            "regex": "(#)((?:\\w|[\\\\+-=<>'\"&#])+)"
+            regex: "(#)((?:\\w|[\\\\+-=<>'\"&#])+)"
         },
         {
-            "token": ["punctuation.definition.variable.lisp", "variable.other.global.lisp", "punctuation.definition.variable.lisp"],
+            token: ["punctuation.definition.variable.lisp", "variable.other.global.lisp", "punctuation.definition.variable.lisp"],
-            "regex": "(\\*)(\\S*)(\\*)"
+            regex: "(\\*)(\\S*)(\\*)"
         },
         {
-            "token" : "constant.numeric", // hex
+            token : "constant.numeric", // hex
-            "regex" : "0[xX][0-9a-fA-F]+(?:L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b"
+            regex : "0[xX][0-9a-fA-F]+(?:L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b"
         }, 
         {
-            "token" : "constant.numeric", // float
+            token : "constant.numeric", // float
-            "regex" : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?(?:L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b"
+            regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?(?:L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b"
         },
         {
-                "token" : keywordMapper,
+                token : keywordMapper,
-                "regex" : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+                regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
         },
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '"(?=.)',
+            regex : '"(?=.)',
-            "next"  : "qqstring"
+            next  : "qqstring"
         }
     ],
     "qqstring": [
         {
-            "token": "constant.character.escape.lisp",
+            token: "constant.character.escape.lisp",
-            "regex": "\\\\."
+            regex: "\\\\."
         },
         {
-            "token" : "string",
+            token : "string",
-            "regex" : '[^"\\\\]+',
+            regex : '[^"\\\\]+'
-            "merge" : true
         }, {
-            "token" : "string",
+            token : "string",
-            "regex" : "\\\\$",
+            regex : "\\\\$",
-            "next"  : "qqstring",
+            next  : "qqstring"
-            "merge" : true
         }, {
-            "token" : "string",
+            token : "string",
-            "regex" : '"|$',
+            regex : '"|$',
-            "next"  : "start",
+            next  : "start"
-            "merge" : true
         }
     ]
 }

lib/ace/mode/livescript.js

@@ -0,0 +1,248 @@
+define('ace/mode/ls', function(require, exports, module){
+  var identifier, LiveScriptMode, keywordend, stringfill;
+  identifier = '(?![\\d\\s])[$\\w\\xAA-\\uFFDC](?:(?!\\s)[$\\w\\xAA-\\uFFDC]|-[A-Za-z])*';
+  exports.Mode = LiveScriptMode = (function(superclass){
+    var indenter, prototype = extend$((import$(LiveScriptMode, superclass).displayName = 'LiveScriptMode', LiveScriptMode), superclass).prototype, constructor = LiveScriptMode;
+    function LiveScriptMode(){
+      var that;
+      this.$tokenizer = new (require('ace/tokenizer')).Tokenizer(LiveScriptMode.Rules);
+      if (that = require('ace/mode/matching_brace_outdent')) {
+        this.$outdent = new that.MatchingBraceOutdent;
+      }
+    }
+    indenter = RegExp('(?:[({[=:]|[-~]>|\\b(?:e(?:lse|xport)|d(?:o|efault)|t(?:ry|hen)|finally|import(?:\\s*all)?|const|var|let|new|catch(?:\\s*' + identifier + ')?))\\s*$');
+    prototype.getNextLineIndent = function(state, line, tab){
+      var indent, tokens;
+      indent = this.$getIndent(line);
+      tokens = this.$tokenizer.getLineTokens(line, state).tokens;
+      if (!(tokens.length && tokens[tokens.length - 1].type === 'comment')) {
+        if (state === 'start' && indenter.test(line)) {
+          indent += tab;
+        }
+      }
+      return indent;
+    };
+    prototype.toggleCommentLines = function(state, doc, startRow, endRow){
+      var comment, range, i$, i, out, line;
+      comment = /^(\s*)#/;
+      range = new (require('ace/range')).Range(0, 0, 0, 0);
+      for (i$ = startRow; i$ <= endRow; ++i$) {
+        i = i$;
+        if (out = comment.test(line = doc.getLine(i))) {
+          line = line.replace(comment, '$1');
+        } else {
+          line = line.replace(/^\s*/, '$&#');
+        }
+        range.end.row = range.start.row = i;
+        range.end.column = line.length + 1;
+        doc.replace(range, line);
+      }
+      return 1 - out * 2;
+    };
+    prototype.checkOutdent = function(state, line, input){
+      var ref$;
+      return (ref$ = this.$outdent) != null ? ref$.checkOutdent(line, input) : void 8;
+    };
+    prototype.autoOutdent = function(state, doc, row){
+      var ref$;
+      return (ref$ = this.$outdent) != null ? ref$.autoOutdent(doc, row) : void 8;
+    };
+    return LiveScriptMode;
+  }(require('ace/mode/text').Mode));
+  keywordend = '(?![$\\w]|-[A-Za-z]|\\s*:(?![:=]))';
+  stringfill = {
+    token: 'string',
+    regex: '.+'
+  };
+  LiveScriptMode.Rules = {
+    start: [
+      {
+        token: 'keyword',
+        regex: '(?:t(?:h(?:is|row|en)|ry|ypeof!?)|c(?:on(?:tinue|st)|a(?:se|tch)|lass)|i(?:n(?:stanceof)?|mp(?:ort(?:\\s+all)?|lements)|[fs])|d(?:e(?:fault|lete|bugger)|o)|f(?:or(?:\\s+own)?|inally|unction)|s(?:uper|witch)|e(?:lse|x(?:tends|port)|val)|a(?:nd|rguments)|n(?:ew|ot)|un(?:less|til)|w(?:hile|ith)|o[fr]|return|break|let|var|loop)' + keywordend
+      }, {
+        token: 'constant.language',
+        regex: '(?:true|false|yes|no|on|off|null|void|undefined)' + keywordend
+      }, {
+        token: 'invalid.illegal',
+        regex: '(?:p(?:ackage|r(?:ivate|otected)|ublic)|i(?:mplements|nterface)|enum|static|yield)' + keywordend
+      }, {
+        token: 'language.support.class',
+        regex: '(?:R(?:e(?:gExp|ferenceError)|angeError)|S(?:tring|yntaxError)|E(?:rror|valError)|Array|Boolean|Date|Function|Number|Object|TypeError|URIError)' + keywordend
+      }, {
+        token: 'language.support.function',
+        regex: '(?:is(?:NaN|Finite)|parse(?:Int|Float)|Math|JSON|(?:en|de)codeURI(?:Component)?)' + keywordend
+      }, {
+        token: 'variable.language',
+        regex: '(?:t(?:hat|il|o)|f(?:rom|allthrough)|it|by|e)' + keywordend
+      }, {
+        token: 'identifier',
+        regex: identifier + '\\s*:(?![:=])'
+      }, {
+        token: 'variable',
+        regex: identifier
+      }, {
+        token: 'keyword.operator',
+        regex: '(?:\\.{3}|\\s+\\?)'
+      }, {
+        token: 'keyword.variable',
+        regex: '(?:@+|::|\\.\\.)',
+        next: 'key'
+      }, {
+        token: 'keyword.operator',
+        regex: '\\.\\s*',
+        next: 'key'
+      }, {
+        token: 'string',
+        regex: '\\\\\\S[^\\s,;)}\\]]*'
+      }, {
+        token: 'string.doc',
+        regex: '\'\'\'',
+        next: 'qdoc'
+      }, {
+        token: 'string.doc',
+        regex: '"""',
+        next: 'qqdoc'
+      }, {
+        token: 'string',
+        regex: '\'',
+        next: 'qstring'
+      }, {
+        token: 'string',
+        regex: '"',
+        next: 'qqstring'
+      }, {
+        token: 'string',
+        regex: '`',
+        next: 'js'
+      }, {
+        token: 'string',
+        regex: '<\\[',
+        next: 'words'
+      }, {
+        token: 'string.regex',
+        regex: '//',
+        next: 'heregex'
+      }, {
+        token: 'comment.doc',
+        regex: '/\\*',
+        next: 'comment'
+      }, {
+        token: 'comment',
+        regex: '#.*'
+      }, {
+        token: 'string.regex',
+        regex: '\\/(?:[^[\\/\\n\\\\]*(?:(?:\\\\.|\\[[^\\]\\n\\\\]*(?:\\\\.[^\\]\\n\\\\]*)*\\])[^[\\/\\n\\\\]*)*)\\/[gimy$]{0,4}',
+        next: 'key'
+      }, {
+        token: 'constant.numeric',
+        regex: '(?:0x[\\da-fA-F][\\da-fA-F_]*|(?:[2-9]|[12]\\d|3[0-6])r[\\da-zA-Z][\\da-zA-Z_]*|(?:\\d[\\d_]*(?:\\.\\d[\\d_]*)?|\\.\\d[\\d_]*)(?:e[+-]?\\d[\\d_]*)?[\\w$]*)'
+      }, {
+        token: 'lparen',
+        regex: '[({[]'
+      }, {
+        token: 'rparen',
+        regex: '[)}\\]]',
+        next: 'key'
+      }, {
+        token: 'keyword.operator',
+        regex: '\\S+'
+      }, {
+        token: 'text',
+        regex: '\\s+'
+      }
+    ],
+    heregex: [
+      {
+        token: 'string.regex',
+        regex: '.*?//[gimy$?]{0,4}',
+        next: 'start'
+      }, {
+        token: 'string.regex',
+        regex: '\\s*#{'
+      }, {
+        token: 'comment.regex',
+        regex: '\\s+(?:#.*)?'
+      }, {
+        token: 'string.regex',
+        regex: '\\S+'
+      }
+    ],
+    key: [
+      {
+        token: 'keyword.operator',
+        regex: '[.?@!]+'
+      }, {
+        token: 'identifier',
+        regex: identifier,
+        next: 'start'
+      }, {
+        token: 'text',
+        regex: '.',
+        next: 'start'
+      }
+    ],
+    comment: [
+      {
+        token: 'comment.doc',
+        regex: '.*?\\*/',
+        next: 'start'
+      }, {
+        token: 'comment.doc',
+        regex: '.+'
+      }
+    ],
+    qdoc: [
+      {
+        token: 'string',
+        regex: ".*?'''",
+        next: 'key'
+      }, stringfill
+    ],
+    qqdoc: [
+      {
+        token: 'string',
+        regex: '.*?"""',
+        next: 'key'
+      }, stringfill
+    ],
+    qstring: [
+      {
+        token: 'string',
+        regex: '[^\\\\\']*(?:\\\\.[^\\\\\']*)*\'',
+        next: 'key'
+      }, stringfill
+    ],
+    qqstring: [
+      {
+        token: 'string',
+        regex: '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
+        next: 'key'
+      }, stringfill
+    ],
+    js: [
+      {
+        token: 'string',
+        regex: '[^\\\\`]*(?:\\\\.[^\\\\`]*)*`',
+        next: 'key'
+      }, stringfill
+    ],
+    words: [
+      {
+        token: 'string',
+        regex: '.*?\\]>',
+        next: 'key'
+      }, stringfill
+    ]
+  };
+function extend$(sub, sup){
+  function fun(){} fun.prototype = (sub.superclass = sup).prototype;
+  (sub.prototype = new fun).constructor = sub;
+  if (typeof sup.extended == 'function') sup.extended(sub);
+  return sub;
+}
+function import$(obj, src){
+  var own = {}.hasOwnProperty;
+  for (var key in src) if (own.call(src, key)) obj[key] = src[key];
+  return obj;
+}
+});

lib/ace/mode/lua_highlight_rules.js

@@ -86,8 +86,6 @@ var LuaHighlightRules = function() {
         "variable.language": "this"
     }, "identifier");
 
-    var strPre = "";
-
     var decimalInteger = "(?:(?:[1-9]\\d*)|(?:0))";
     var hexInteger = "(?:0[xX][\\dA-Fa-f]+)";
     var integer = "(?:" + decimalInteger + "|" + hexInteger + ")";
@@ -97,138 +95,71 @@ var LuaHighlightRules = function() {
     var pointFloat = "(?:(?:" + intPart + "?" + fraction + ")|(?:" + intPart + "\\.))";
     var floatNumber = "(?:" + pointFloat + ")";
 
-    var comment_stack = [];
-
     this.$rules = {
-        "start" :
+        "start" : [{
-
+            stateName: "bracketedComment",
-
+            token : function(value, currentState, stack){
-        // bracketed comments
+                stack.unshift(this.next, value.length, currentState);
-        [{
-            token : "comment",           // --[[ comment
-            regex : strPre + '\\-\\-\\[\\[.*\\]\\]'
-        }, {
-            token : "comment",           // --[=[ comment
-            regex : strPre + '\\-\\-\\[\\=\\[.*\\]\\=\\]'
-        }, {
-            token : "comment",           // --[==[ comment
-            regex : strPre + '\\-\\-\\[\\={2}\\[.*\\]\\={2}\\]'
-        }, {
-            token : "comment",           // --[===[ comment
-            regex : strPre + '\\-\\-\\[\\={3}\\[.*\\]\\={3}\\]'
-        }, {
-            token : "comment",           // --[====[ comment
-            regex : strPre + '\\-\\-\\[\\={4}\\[.*\\]\\={4}\\]'
-        }, {
-            token : "comment",           // --[====+[ comment
-            regex : strPre + '\\-\\-\\[\\={5}\\=*\\[.*\\]\\={5}\\=*\\]'
-        },
-
-        // multiline bracketed comments
-        {
-            token : "comment",           // --[[ comment
-            regex : strPre + '\\-\\-\\[\\[.*$',
-            next  : "qcomment"
-        }, {
-            token : "comment",           // --[=[ comment
-            regex : strPre + '\\-\\-\\[\\=\\[.*$',
-            next  : "qcomment1"
-        }, {
-            token : "comment",           // --[==[ comment
-            regex : strPre + '\\-\\-\\[\\={2}\\[.*$',
-            next  : "qcomment2"
-        }, {
-            token : "comment",           // --[===[ comment
-            regex : strPre + '\\-\\-\\[\\={3}\\[.*$',
-            next  : "qcomment3"
-        }, {
-            token : "comment",           // --[====[ comment
-            regex : strPre + '\\-\\-\\[\\={4}\\[.*$',
-            next  : "qcomment4"
-        }, {
-            token : function(value){     // --[====+[ comment
-                // WARNING: EXTREMELY SLOW, but this is the only way to circumvent the
-                // limits imposed by the current automaton.
-                // I've never personally seen any practical code where 5 or more '='s are
-                // used for string or commenting, so this will rarely be invoked.
-                var pattern = /\-\-\[(\=+)\[/, match;
-                // you can never be too paranoid ;)
-                if ((match = pattern.exec(value)) != null && (match = match[1]) != undefined)
-                    comment_stack.push(match.length);
-
                 return "comment";
             },
-            regex : strPre + '\\-\\-\\[\\={5}\\=*\\[.*$',
+            regex : /\-\-\[=*\[/,
-            next  : "qcomment5"
+            next  : [
+                {
+                    token : function(value, currentState, stack) {
+                        if (value.length == stack[1]) {
+                            stack.shift();
+                            stack.shift();
+                            this.next = stack.shift();
+                        } else {
+                            this.next = "";
+                        }
+                        return "comment";
+                    },
+                    regex : /(?:[^\\]|\\.)*?\]=*\]/,
+                    next  : "start"
+                }, {
+                    defaultToken : "comment"
+                }
+            ]
         },
 
-        // single line comments
         {
             token : "comment",
             regex : "\\-\\-.*$"
         },
-
-        // bracketed strings
         {
-            token : "string",           // [[ string
+            stateName: "bracketedString",
-            regex : strPre + '\\[\\[.*\\]\\]'
+            token : function(value, currentState, stack){
-        }, {
+                stack.unshift(this.next, value.length, currentState);
-            token : "string",           // [=[ string
+                return "comment";
-            regex : strPre + '\\[\\=\\[.*\\]\\=\\]'
-        }, {
-            token : "string",           // [==[ string
-            regex : strPre + '\\[\\={2}\\[.*\\]\\={2}\\]'
-        }, {
-            token : "string",           // [===[ string
-            regex : strPre + '\\[\\={3}\\[.*\\]\\={3}\\]'
-        }, {
-            token : "string",           // [====[ string
-            regex : strPre + '\\[\\={4}\\[.*\\]\\={4}\\]'
-        }, {
-            token : "string",           // [====+[ string
-            regex : strPre + '\\[\\={5}\\=*\\[.*\\]\\={5}\\=*\\]'
             },
-
+            regex : /\[=*\[/,
-        // multiline bracketed strings
+            next  : [
                 {
-            token : "string",           // [[ string
+                    token : function(value, currentState, stack) {
-            regex : strPre + '\\[\\[.*$',
+                        if (value.length == stack[1]) {
-            next  : "qstring"
+                            stack.shift();
-        }, {
+                            stack.shift();
-            token : "string",           // [=[ string
+                            this.next = stack.shift();
-            regex : strPre + '\\[\\=\\[.*$',
+                        } else {
-            next  : "qstring1"
+                            this.next = "";
-        }, {
+                        }
-            token : "string",           // [==[ string
+                        return "comment";
-            regex : strPre + '\\[\\={2}\\[.*$',
-            next  : "qstring2"
-        }, {
-            token : "string",           // [===[ string
-            regex : strPre + '\\[\\={3}\\[.*$',
-            next  : "qstring3"
-        }, {
-            token : "string",           // [====[ string
-            regex : strPre + '\\[\\={4}\\[.*$',
-            next  : "qstring4"
-        }, {
-            token : function(value){     // --[====+[ string
-                // WARNING: EXTREMELY SLOW, see above.
-                var pattern = /\[(\=+)\[/, match;
-                if ((match = pattern.exec(value)) != null && (match = match[1]) != undefined)
-                    comment_stack.push(match.length);
-
-                return "string";
-            },
-            regex : strPre + '\\[\\={5}\\=*\\[.*$',
-            next  : "qstring5"
                     },
                     
+                    regex : /(?:[^\\]|\\.)*?\]=*\]/,
+                    next  : "start"
+                }, {
+                    defaultToken : "comment"
+                }
+            ]
+        },
         {
             token : "string",           // " string
-            regex : strPre + '"(?:[^\\\\]|\\\\.)*?"'
+            regex : '"(?:[^\\\\]|\\\\.)*?"'
         }, {
             token : "string",           // ' string
-            regex : strPre + "'(?:[^\\\\]|\\\\.)*?'"
+            regex : "'(?:[^\\\\]|\\\\.)*?'"
         }, {
             token : "constant.numeric", // float
             regex : floatNumber
@@ -249,135 +180,11 @@ var LuaHighlightRules = function() {
             regex : "[\\]\\)\\}]"
         }, {
             token : "text",
-            regex : "\\s+"
+            regex : "\\s+|\\w+"
-        } ],
-
-        "qcomment": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment1": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\=\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment2": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={2}\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment3": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={3}\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment4": [ {
-            token : "comment",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={4}\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-        "qcomment5": [ {
-            token : function(value){
-                // very hackish, mutates the qcomment5 field on the fly.
-                var pattern = /\](\=+)\]/, rule = this.rules.qcomment5[0], match;
-                rule.next = "start";
-                if ((match = pattern.exec(value)) != null && (match = match[1]) != undefined){
-                    var found = match.length, expected;
-                    if ((expected = comment_stack.pop()) != found){
-                        comment_stack.push(expected);
-                        rule.next = "qcomment5";
-                    }
-                }
-
-                return "comment";
-            },
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={5}\\=*\\]",
-            next  : "start"
-        }, {
-            token : "comment",
-            regex : '.+'
-        } ],
-
-        "qstring": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring1": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\=\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring2": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={2}\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring3": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={3}\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring4": [ {
-            token : "string",
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={4}\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
-        } ],
-        "qstring5": [ {
-            token : function(value){
-                // very hackish, mutates the qstring5 field on the fly.
-                var pattern = /\](\=+)\]/, rule = this.rules.qstring5[0], match;
-                rule.next = "start";
-                if ((match = pattern.exec(value)) != null && (match = match[1]) != undefined){
-                    var found = match.length, expected;
-                    if ((expected = comment_stack.pop()) != found){
-                        comment_stack.push(expected);
-                        rule.next = "qstring5";
-                    }
-                }
-
-                return "string";
-            },
-            regex : "(?:[^\\\\]|\\\\.)*?\\]\\={5}\\=*\\]",
-            next  : "start"
-        }, {
-            token : "string",
-            regex : '.+'
         } ]
-
     };
     
+    this.normalizeRules();
 }
 
 oop.inherits(LuaHighlightRules, TextHighlightRules);

lib/ace/mode/makefile_highlight_rules.js

@@ -21,48 +21,48 @@ var MakefileHighlightRules = function() {
         {
     "start": [
         {
-            "token": "string.interpolated.backtick.makefile",
+            token: "string.interpolated.backtick.makefile",
-            "regex": "`",
+            regex: "`",
-            "next": "shell-start"
+            next: "shell-start"
         },
         {
-            "token": "punctuation.definition.comment.makefile",
+            token: "punctuation.definition.comment.makefile",
-            "regex": /#(?=.)/,
+            regex: /#(?=.)/,
-            "next": "comment"
+            next: "comment"
         },
         {
-            "token": [ "keyword.control.makefile"],
+            token: [ "keyword.control.makefile"],
-            "regex": "^(?:\\s*\\b)(\\-??include|ifeq|ifneq|ifdef|ifndef|else|endif|vpath|export|unexport|define|endef|override)(?:\\b)"
+            regex: "^(?:\\s*\\b)(\\-??include|ifeq|ifneq|ifdef|ifndef|else|endif|vpath|export|unexport|define|endef|override)(?:\\b)"
         },
         {// ^([^\t ]+(\s[^\t ]+)*:(?!\=))\s*.*
-            "token": ["entity.name.function.makefile", "text"],
+            token: ["entity.name.function.makefile", "text"],
-            "regex": "^([^\\t ]+(?:\\s[^\\t ]+)*:)(\\s*.*)"
+            regex: "^([^\\t ]+(?:\\s[^\\t ]+)*:)(\\s*.*)"
         }
     ],
     "comment": [
         {
-            "token" : "punctuation.definition.comment.makefile",
+            token : "punctuation.definition.comment.makefile",
-            "regex" : /.+\\/
+            regex : /.+\\/
         },
         {
-            "token" : "punctuation.definition.comment.makefile",
+            token : "punctuation.definition.comment.makefile",
-            "regex" : ".+",
+            regex : ".+",
-            "next"  : "start"
+            next  : "start"
         }
     ],
     "shell-start": [
         {
-            "token": keywordMapper,
+            token: keywordMapper,
-            "regex" : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
         }, 
         {
-            "token": "string",
+            token: "string",
-            "regex" : "\\w+"
+            regex : "\\w+"
         }, 
         {
-            "token" : "string.interpolated.backtick.makefile",
+            token : "string.interpolated.backtick.makefile",
-            "regex" : "`",
+            regex : "`",
-            "next"  : "start"
+            next  : "start"
         }
     ]
 }

lib/ace/mode/markdown_highlight_rules.js

@@ -52,38 +52,9 @@ var MarkdownHighlightRules = function() {
     // regexps are ordered -> the first match is used
 
     this.$rules = {
-        "start" : [ {
+        "basic" : [{ // code span `
-            token : "empty_line",
-            regex : '^$'
-        }, { // code span `
-            token : ["support.function", "support.function", "support.function"],
-            regex : "(`+)([^\\r]*?[^`])(\\1)"
-        }, { // code block
             token : "support.function",
-            regex : "^[ ]{4}.+"
+            regex : "(`+)(.*?[^`])(\\1)"
-        }, { // h1
-            token: "markup.heading.1",
-            regex: "^=+(?=\\s*$)"
-        }, { // h2
-            token: "markup.heading.2",
-            regex: "^\\-+(?=\\s*$)"
-        }, { // header
-            token : function(value) {
-                return "markup.heading." + value.search(/[^#]/);
-            },
-            regex : "^#{1,6}(?:[^ #].*| +.*(?:[^ #].*|[^ ]+.* +#+ *))$"
-        }, github_embed("(?:javascript|js)", "js-"),
-           github_embed("xml", "xml-"),
-           github_embed("html", "html-"),
-           github_embed("css", "css-"),
-        { // Github style block
-            token : "support.function",
-            regex : "^```\\s*[a-zA-Z]*(?:{.*?\\})?\\s*$",
-            next  : "githubblock"
-        }, { // block quote
-            token : "string",
-            regex : "^>[ ].+$",
-            next  : "blockquote"
         }, { // reference
             token : ["text", "constant", "text", "url", "string", "text"],
             regex : "^([ ]{0,3}\\[)([^\\]]+)(\\]:\\s*)([^ ]+)(\\s*(?:[\"][^\"]+[\"])?(\\s*))$"
@@ -98,25 +69,12 @@ var MarkdownHighlightRules = function() {
                     "(<?(?:(?:[^\\(]*?\\([^\\)]*?\\)\\S*?)|(?:.*?))>?)"+
                     "((?:[ \t]*\"(?:.*?)\"[ \\t]*)?)"+
                     "(\\))"
-        }, { // HR *
-            token : "constant",
-            regex : "^[ ]{0,2}(?:[ ]?\\*[ ]?){3,}\\s*$"
-        }, { // HR -
-            token : "constant",
-            regex : "^[ ]{0,2}(?:[ ]?\\-[ ]?){3,}\\s*$"
-        }, { // HR _
-            token : "constant",
-            regex : "^[ ]{0,2}(?:[ ]?\\_[ ]?){3,}\\s*$"
-        }, { // list
-            token : "markup.list",
-            regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
-            next  : "listblock"
         }, { // strong ** __
-            token : ["string", "string", "string"],
+            token : "string",
-            regex : "([*]{2}|[_]{2}(?=\\S))([^\\r]*?\\S[*_]*)(\\1)"
+            regex : "([*]{2}|[_]{2}(?=\\S))(.*?\\S[*_]*)(\\1)"
         }, { // emphasis * _
-            token : ["string", "string", "string"],
+            token : "string",
-            regex : "([*]|[_](?=\\S))([^\\r]*?\\S[*_]*)(\\1)"
+            regex : "([*]|[_](?=\\S))(.*?\\S[*_]*)(\\1)"
         }, { //
             token : ["text", "url", "text"],
             regex : "(<)("+
@@ -124,9 +82,58 @@ var MarkdownHighlightRules = function() {
                       "|"+
                       "(?:mailto:)?[-.\\w]+\\@[-a-z0-9]+(?:\\.[-a-z0-9]+)*\\.[a-z]+"+
                     ")(>)"
+        }],
+
+        // code block
+        "allowBlock": [
+            {token : "support.function", regex : "^ {4}.+", next : "allowBlock"},
+            {token : "empty", regex : "", next : "start"}
+        ],
+
+        "start" : [{
+            token : "empty_line",
+            regex : '^$',
+            next: "allowBlock"
+        }, { // h1
+            token: "markup.heading.1",
+            regex: "^=+(?=\\s*$)"
+        }, { // h2
+            token: "markup.heading.2",
+            regex: "^\\-+(?=\\s*$)"
+        }, { // header
+            token : function(value) {
+                return "markup.heading." + value.search(/[^#]/);
+            },
+            regex : "^#{1,6}(?:[^ #].*| +.*(?:[^ #].*|[^ ]+.* +#+ *))$"
+        },
+           github_embed("(?:javascript|js)", "js-"),
+           github_embed("xml", "xml-"),
+           github_embed("html", "html-"),
+           github_embed("css", "css-"),
+        { // Github style block
+            token : "support.function",
+            regex : "^```\\s*[a-zA-Z]*(?:{.*?\\})?\\s*$",
+            next  : "githubblock"
+        }, { // block quote
+            token : "string",
+            regex : "^>[ ].+$",
+            next  : "blockquote"
+        }, { // HR * - _
+            token : "constant",
+            regex : "^ {0,2}(?:(?: ?\\* ?){3,}|(?: ?\\- ?){3,}|(?: ?\\_ ?){3,})\\s*$",
+            next: "allowBlock"
+        }, { // list
+            token : "markup.list",
+            regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
+            next  : "listblock-start"
         }, {
-            token : "text",
+            include : "basic"
-            regex : "[^\\*_%$`\\[#<>]+"
+        }],
+
+        "listblock-start" : [{
+            token : "support.variable",
+            regex : /(?:\[[ x]\])?/,
+            next  : "listblock"
         }],
 
         "listblock" : [ { // Lists only escape on completely blank lines.
@@ -134,8 +141,13 @@ var MarkdownHighlightRules = function() {
             regex : "^$",
             next  : "start"
         }, {
+            include : "basic", noEscape: true
+        }, { // list
             token : "markup.list",
-            regex : ".+"
+            regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
+            next  : "listblock-start"
+        }, {
+            defaultToken : "markup.list"
         } ],
 
         "blockquote" : [ { // BLockquotes only escape on blank lines.
@@ -189,6 +201,7 @@ var MarkdownHighlightRules = function() {
             this.$rules[i] = html[i];
     }
 
+    this.normalizeRules();
 };
 oop.inherits(MarkdownHighlightRules, TextHighlightRules);
 

lib/ace/mode/php_highlight_rules.js

@@ -974,6 +974,16 @@ var PhpLangHighlightRules = function() {
                 // TODO: Unicode escape sequences
                 // TODO: Unicode identifiers
                 regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            }, {
+                token : function(value, currentSate, state) {
+                    value = value.substr(3);
+                    if (value[0] == "'" || value[0] == '"')
+                        value = value.slice(1, -1);
+                    state.unshift(this.next, value);
+                    return "markup.list";
+                },
+                regex : /<<<(?:\w+|'\w+'|"\w+")$/,
+                next: "heredoc"
             }, {
                 token : "keyword.operator",
                 regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"
@@ -988,6 +998,22 @@ var PhpLangHighlightRules = function() {
                 regex : "\\s+"
             }
         ],
+        "heredoc" : [
+            {
+                token : function(value, currentSate, stack) {
+                    if (stack[1]  + ";" != value)
+                        return "string";
+                    stack.shift();
+                    stack.shift();
+                    return "markup.list"
+                },
+                regex : "^\\w+;$",
+                next: "start"
+            }, {
+                token: "string",
+                regex : ".*",
+            }
+        ],
         "comment" : [
             {
                 token : "comment", // closing comment
@@ -1004,31 +1030,18 @@ var PhpLangHighlightRules = function() {
                 regex : '\\\\(?:[nrtvef\\\\"$]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2})'
             }, {
                 token : "constant.language.escape",
-                regex : /\$[\w\d]+(?:\[[\w\d]+\])?/
+                regex : /\$[\w]+(?:\[[\w\]+]|=>\w+)?/
             }, {
                 token : "constant.language.escape",
                 regex : /\$\{[^"\}]+\}?/           // this is wrong but ok for now
-            }, {
+            },
-                token : "string",
+            {token : "string", regex : '"', next : "start"},
-                regex : '"',
+            {defaultToken : "string"}
-                next : "start"
-            }, {
-                token : "string",
-                regex : '.+?'
-            }
         ],
         "qstring" : [
-            {
+            {token : "constant.language.escape", regex : /\\['\\]/},
-                token : "constant.language.escape",
+            {token : "string", regex : "'", next : "start"},
-                regex : "\\\\['\\\\]"
+            {defaultToken : "string"}
-            }, {
-                token : "string",
-                regex : "'",
-                next : "start"
-            }, {
-                token : "string",
-                regex : ".+?"
-            }
         ]
     };
 

lib/ace/mode/ruby_highlight_rules.js

@@ -171,6 +171,43 @@ var RubyHighlightRules = function() {
             }, {
                 token : "punctuation.separator.key-value",
                 regex : "=>"
+            }, {
+                stateName: "heredoc",
+                token : function(value, currentState, stack) {
+                    var next = value[0].length == 2 ? "heredoc" : "indentedHeredoc"
+                    stack.push(next, value[2])
+                    return ["constant", "string", "support.class", "string"]
+                },
+                regex : "(<<-?)(['\"`]?)([\\w]+)(['\"`]?)",
+                rules: {
+                    heredoc: [{
+                        token:  function(value, currentState, stack) {
+                            if (value == stack[1]) {
+                                stack.shift();
+                                stack.shift();
+                                return "support.class";
+                            }
+                            return "string";
+                        },
+                        regex: ".*$",
+                        next: "start"
+                    }],
+                    indentedHeredoc: [{
+                        token: "string",
+                        regex: "^ +"
+                    }, {
+                        token:  function(value, currentState, stack) {
+                            if (value == stack[1]) {
+                                stack.shift();
+                                stack.shift();
+                                return "support.class";
+                            }
+                            return "string";
+                        },
+                        regex: ".*$",
+                        next: "start"
+                    }]
+                }
             }, {
                 token : "keyword.operator",
                 regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"
@@ -196,6 +233,8 @@ var RubyHighlightRules = function() {
             }
         ]
     };
+
+    this.normalizeRules();
 };
 
 oop.inherits(RubyHighlightRules, TextHighlightRules);

lib/ace/mode/stylus_highlight_rules.js

@@ -22,7 +22,7 @@ var StylusHighlightRules = function() {
     }, "text", true);
 
     this.$rules = {
-    "start": [
+    start: [
         {
             token : "comment",
             regex : /\/\/.*$/
@@ -33,24 +33,24 @@ var StylusHighlightRules = function() {
             next : "comment"
         },
         {
-            "token": ["entity.name.function.stylus", "text"],
+            token: ["entity.name.function.stylus", "text"],
-            "regex": "^([-a-zA-Z_][-\\w]*)?(\\()"
+            regex: "^([-a-zA-Z_][-\\w]*)?(\\()"
         },
         {
-            "token": ["entity.other.attribute-name.class.stylus"],
+            token: ["entity.other.attribute-name.class.stylus"],
-            "regex": "\\.-?[_a-zA-Z]+[_a-zA-Z0-9-]*"
+            regex: "\\.-?[_a-zA-Z]+[_a-zA-Z0-9-]*"
         },
         {
-            "token": ["entity.language.stylus"],
+            token: ["entity.language.stylus"],
-            "regex": "^ *&"
+            regex: "^ *&"
         },
         {
-            "token": ["variable.language.stylus"],
+            token: ["variable.language.stylus"],
-            "regex": "(arguments)"
+            regex: "(arguments)"
         },
         {
-            "token": ["keyword.stylus"],
+            token: ["keyword.stylus"],
-            "regex": "@[-\\w]+"
+            regex: "@[-\\w]+"
         },
         {
             token : ["punctuation", "entity.other.attribute-name.pseudo-element.css"],
@@ -60,8 +60,8 @@ var StylusHighlightRules = function() {
             regex : CssHighlightRules.pseudoClasses
         }, 
         {
-            "token": ["entity.name.tag.stylus"],
+            token: ["entity.name.tag.stylus"],
-            "regex": "(?:\\b)(a|abbr|acronym|address|area|article|aside|audio|b|base|big|blockquote|body|br|button|canvas|caption|cite|code|col|colgroup|datalist|dd|del|details|dfn|dialog|div|dl|dt|em|eventsource|fieldset|figure|figcaption|footer|form|frame|frameset|(?:h[1-6])|head|header|hgroup|hr|html|i|iframe|img|input|ins|kbd|label|legend|li|link|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|pre|progress|q|samp|script|section|select|small|span|strike|strong|style|sub|summary|sup|table|tbody|td|textarea|tfoot|th|thead|time|title|tr|tt|ul|var|video)(?:\\b)"
+            regex: "(?:\\b)(a|abbr|acronym|address|area|article|aside|audio|b|base|big|blockquote|body|br|button|canvas|caption|cite|code|col|colgroup|datalist|dd|del|details|dfn|dialog|div|dl|dt|em|eventsource|fieldset|figure|figcaption|footer|form|frame|frameset|(?:h[1-6])|head|header|hgroup|hr|html|i|iframe|img|input|ins|kbd|label|legend|li|link|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|pre|progress|q|samp|script|section|select|small|span|strike|strong|style|sub|summary|sup|table|tbody|td|textarea|tfoot|th|thead|time|title|tr|tt|ul|var|video)(?:\\b)"
         },
         {
             token : "constant.numeric",  // hex6 color
@@ -72,24 +72,24 @@ var StylusHighlightRules = function() {
             regex : "#[a-f0-9]{3}"
         }, 
         {
-            "token": ["punctuation.definition.entity.stylus", "entity.other.attribute-name.id.stylus"],
+            token: ["punctuation.definition.entity.stylus", "entity.other.attribute-name.id.stylus"],
-            "regex": "(#)([a-zA-Z][a-zA-Z0-9_-]*)"
+            regex: "(#)([a-zA-Z][a-zA-Z0-9_-]*)"
         },
         {
-            "token": "meta.vendor-prefix.stylus",
+            token: "meta.vendor-prefix.stylus",
-            "regex": "-webkit-|-moz\\-|-ms-|-o-"
+            regex: "-webkit-|-moz\\-|-ms-|-o-"
         },
         {
-            "token": "keyword.control.stylus",
+            token: "keyword.control.stylus",
-            "regex": "(?:!important|for|in|return|true|false|null|if|else|unless|return)\\b"
+            regex: "(?:!important|for|in|return|true|false|null|if|else|unless|return)\\b"
         },
         {
-            "token": "keyword.operator.stylus",
+            token: "keyword.operator.stylus",
-            "regex": "!|~|\\+|-|(?:\\*)?\\*|\\/|%|(?:\\.)\\.\\.|<|>|(?:=|:|\\?|\\+|-|\\*|\\/|%|<|>)?=|!="
+            regex: "!|~|\\+|-|(?:\\*)?\\*|\\/|%|(?:\\.)\\.\\.|<|>|(?:=|:|\\?|\\+|-|\\*|\\/|%|<|>)?=|!="
         },
         {
-            "token": "keyword.operator.stylus",
+            token: "keyword.operator.stylus",
-            "regex": "(?:in|is(?:nt)?|not)\\b"
+            regex: "(?:in|is(?:nt)?|not)\\b"
         },
         {
             token : "string",

lib/ace/mode/tcl_highlight_rules.js

@@ -66,7 +66,7 @@ var TclHighlightRules = function() {
                 regex : '[ ]*["]',
                 next  : "qqstring"
             }, {
-                token : "variable.instancce", // variable xotcl with braces
+                token : "variable.instance", // variable xotcl with braces
                 regex : "[$]",
                 next  : "variable"
             }, {
@@ -103,7 +103,7 @@ var TclHighlightRules = function() {
                 token : "string", // single line
                 regex : '[ ]*["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
             }, {
-                token : "variable.instancce", // variable xotcl with braces
+                token : "variable.instance", // variable xotcl with braces
                 regex : "[$]",
                 next  : "variable"
             }, {
@@ -118,6 +118,9 @@ var TclHighlightRules = function() {
                 token : "support.function",
                 regex : "(?:[:][:])",
                 next  : "commandItem"
+            }, {
+                token : "paren.rparen",
+                regex : "[\\])}]"
             }, {
                 token : "support.function",
                 regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|{\\*}|;|::"

lib/ace/mode/text_highlight_rules.js

@@ -94,7 +94,38 @@ var TextHighlightRules = function() {
 
     this.getEmbeds = function() {
         return this.$embeds;
+    };
+    
+    this.normalizeRules = function() {
+        var id = 0;
+        for (var key in this.$rules) {
+            var state = this.$rules[key];
+            for (var i = 0; i < state.length; i++) {
+                var rule = state[i];
+                if (rule.next && Array.isArray(rule.next)) {
+                    var stateName = rule.stateName || ("state" + id++);
+                    this.$rules[stateName] = rule.next;
+                    rule.next = stateName;
                 }
+                if (rule.rules) {
+                     for (var r in rule.rules) {
+                        if (this.$rules[r]) {
+                            if (this.$rules[r].push)
+                                this.$rules[r].push.apply(this.$rules[r], rule.rules[r]);
+                        } else {
+                            this.$rules[r] = rule.rules[r];
+                        }
+                    }
+                }
+                if (rule.include || typeof rule == "string") {
+                    var args = [i, 1].concat(this.$rules[rule.include || rule]);
+                    if (rule.noEscape)
+                        args = args.filter(function(x) {return !x.next;});
+                    state.splice.apply(state, args);
+                }
+            }
+        }
+    };
 
     this.createKeywordMapper = function(map, defaultToken, ignoreCase, splitChar) {
         var keywords = Object.create(null);

lib/ace/mode/tm_snippet.js

@@ -0,0 +1,86 @@
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextMode = require("./text").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var SnippetHighlightRules = function() {
+
+    var builtins = "SELECTION|CURRENT_WORD|SELECTED_TEXT|CURRENT_LINE|LINE_INDEX|" +
+        "LINE_NUMBER|SOFT_TABS|TAB_SIZE|FILENAME|FILEPATH|FULLNAME";
+
+    this.$rules = {
+        "start" : [
+            {token:"constant.language.escape", regex: /\\[\$}`\\]/},
+            {token:"keyword", regex: "\\$(?:TM_)?(?:" + builtins + ")\\b"},
+            {token:"variable", regex: "\\$\\w+"},
+            {token: function(value, state, stack) {
+                if (stack[1])
+                    stack[1]++;
+                else
+                    stack.unshift("start", 1);
+                return this.tokenName;
+            }, tokenName: "markup.list", regex: "\\${", next: "varDecl"},
+            {token: function(value, state, stack) {
+                if (!stack[1])
+                    return "text";
+                stack[1]--;
+                if (!stack[1])
+                    stack.splice(0,2);
+                return this.tokenName;
+            }, tokenName: "markup.list", regex: "}"},
+            {token: "doc,comment", regex:/^\${2}-{5,}$/}
+        ],
+        "varDecl" : [
+            {regex: /\d+\b/, token: "constant.numeric"},
+            {token:"keyword", regex: "(?:TM_)?(?:" + builtins + ")\\b"},
+            {token:"variable", regex: "\\w+"},
+            {regex: /:/, token: "punctuation.operator", next: "start"},
+            {regex: /\//, token: "string.regex", next: "regexp"},
+            {regex: "", next: "start"}
+        ],
+        "regexp" : [
+            {regex: /\\./, token: "escape"},
+            {regex: /\[/, token: "regex.start", next: "charClass"},
+            {regex: "/", token: "string.regex", next: "format"},
+            //{"default": "string.regex"},
+            {"token": "string.regex", regex:"."},
+        ],
+        charClass : [
+            {regex: "\\.", token: "escape"},
+            {regex: "\\]", token: "regex.end", next: "regexp"},
+            {"token": "string.regex", regex:"."},
+        ],
+        "format" : [
+            {regex: /\\[ulULE]/, token: "keyword"},
+            {regex: /\$\d+/, token: "variable"},
+            {regex: "/[gim]*:?", token: "string.regex", next: "start"},
+            // {"default": "string"},
+            {"token": "string", regex:"."},
+        ]
+    };
+};
+
+oop.inherits(SnippetHighlightRules, TextHighlightRules);
+
+exports.SnippetHighlightRules = SnippetHighlightRules;
+
+
+var Mode = function() {
+    var highlighter = new SnippetHighlightRules();
+
+    this.$tokenizer = new Tokenizer(highlighter.getRules());
+};
+oop.inherits(Mode, TextMode);
+
+(function() {
+    this.getNextLineIndent = function(state, line, tab) {
+        return this.$getIndent(line);
+    };
+}).call(Mode.prototype);
+exports.Mode = Mode;
+
+
+});

lib/ace/mode/typescript_highlight_rules.js

@@ -50,39 +50,39 @@ var TypeScriptHighlightRules = function() {
     var tsRules =  [
         // Match stuff like: module name {...}
         {
-            "token": ["keyword.operator.ts", "text", "variable.parameter.function.ts", "text"],
+            token: ["keyword.operator.ts", "text", "variable.parameter.function.ts", "text"],
-            "regex": "\\b(module)(\\s*)([a-zA-Z0-9_?.$][\\w?.$]*)(\\s*\\{)"
+            regex: "\\b(module)(\\s*)([a-zA-Z0-9_?.$][\\w?.$]*)(\\s*\\{)"
         }, 
         // Match stuff like: super(argument, list)
         {
-            "token": ["storage.type.variable.ts", "text", "keyword.other.ts", "text"],
+            token: ["storage.type.variable.ts", "text", "keyword.other.ts", "text"],
-            "regex": "(super)(\\s*\\()([a-zA-Z0-9,_?.$\\s]+\\s*)(\\))"
+            regex: "(super)(\\s*\\()([a-zA-Z0-9,_?.$\\s]+\\s*)(\\))"
         },
         // Match stuff like: function() {...}
         {
-            "token": ["entity.name.function.ts","paren.lparen", "paren.rparen"],
+            token: ["entity.name.function.ts","paren.lparen", "paren.rparen"],
-            "regex": "([a-zA-Z_?.$][\\w?.$]*)(\\()(\\))"
+            regex: "([a-zA-Z_?.$][\\w?.$]*)(\\()(\\))"
         },
         // Match stuff like: (function: return type)
         {
-            "token": ["variable.parameter.function.ts", "text", "variable.parameter.function.ts"],
+            token: ["variable.parameter.function.ts", "text", "variable.parameter.function.ts"],
-            "regex": "([a-zA-Z0-9_?.$][\\w?.$]*)(\\s*:\\s*)([a-zA-Z0-9_?.$][\\w?.$]*)"
+            regex: "([a-zA-Z0-9_?.$][\\w?.$]*)(\\s*:\\s*)([a-zA-Z0-9_?.$][\\w?.$]*)"
         },  
         {
-            "token": ["keyword.operator.ts"],
+            token: ["keyword.operator.ts"],
-            "regex": "(?:\\b(constructor|declare|interface|as|AS|public|private|class|extends|export|super)\\b)"
+            regex: "(?:\\b(constructor|declare|interface|as|AS|public|private|class|extends|export|super)\\b)"
         }, 
         {
-            "token": ["storage.type.variable.ts"],
+            token: ["storage.type.variable.ts"],
-            "regex": "(?:\\b(this\\.|string\\b|bool\\b|number)\\b)"
+            regex: "(?:\\b(this\\.|string\\b|bool\\b|number)\\b)"
         }, 
         {
-            "token": ["keyword.operator.ts", "storage.type.variable.ts", "keyword.operator.ts", "storage.type.variable.ts"],
+            token: ["keyword.operator.ts", "storage.type.variable.ts", "keyword.operator.ts", "storage.type.variable.ts"],
-            "regex": "(class)(\\s+[a-zA-Z0-9_?.$][\\w?.$]*\\s+)(extends)(\\s+[a-zA-Z0-9_?.$][\\w?.$]*\\s+)?"
+            regex: "(class)(\\s+[a-zA-Z0-9_?.$][\\w?.$]*\\s+)(extends)(\\s+[a-zA-Z0-9_?.$][\\w?.$]*\\s+)?"
         },
         {
-            "token": "keyword",
+            token: "keyword",
-            "regex": "(?:super|export|class|extends|import)\\b"
+            regex: "(?:super|export|class|extends|import)\\b"
         }
     ];
 

lib/ace/mode/xml_highlight_rules.js

@@ -47,8 +47,7 @@ var XmlHighlightRules = function() {
             {
                 token : "constant.character.entity", 
                 regex : "(?:&#[0-9]+;)|(?:&#x[0-9a-fA-F]+;)|(?:&[a-zA-Z0-9_:\\.-]+;)" 
-            },
+            }
-            {token : "text", regex : "[^<]+"}
         ],
         
         cdata : [

lib/ace/mode/xml_util.js

@@ -50,7 +50,7 @@ function multiLineString(quote, state) {
             token : "constant.language.escape",
             regex : "(?:&#[0-9]+;)|(?:&#x[0-9a-fA-F]+;)|(?:&[a-zA-Z0-9_:\\.-]+;)" 
         },
-        {token : "string", regex : '\\w+|.|\\s+'}
+        {defaultToken : "string"}
     ];
 }
 

lib/ace/mode/yaml_highlight_rules.js

@@ -95,12 +95,6 @@ var YamlHighlightRules = function() {
             }, {
                 token : "paren.rparen",
                 regex : "[\\])}]"
-            }, {
-                token : "text",
-                regex : "\\s+"
-            }, {
-                token : "text",
-                regex : "\\w+"
             }
         ],
         "qqstring" : [

lib/ace/tokenizer.js

@@ -50,133 +50,186 @@ define(function(require, exports, module) {
  **/
 var Tokenizer = function(rules, flag) {
     flag = flag ? "g" + flag : "g";
-    this.rules = rules;
+    this.states = rules;
 
     this.regExps = {};
     this.matchMappings = {};
-    for ( var key in this.rules) {
+    for (var key in this.states) {
-        var rule = this.rules[key];
+        var state = this.states[key];
-        var state = rule;
         var ruleRegExps = [];
         var matchTotal = 0;
-        var mapping = this.matchMappings[key] = {};
+        var mapping = this.matchMappings[key] = {defaultToken: "text"};
 
         for (var i = 0; i < state.length; i++) {
-
+            var rule = state[i];
-            if (state[i].regex instanceof RegExp)
+            if (rule.defaultToken) {
-                state[i].regex = state[i].regex.toString().slice(1, -1);
+                mapping.defaultToken = rule.defaultToken;
+                continue;
+            }
+            if (rule.regex instanceof RegExp)
+                rule.regex = rule.regex.toString().slice(1, -1);
 
             // Count number of matching groups. 2 extra groups from the full match
             // And the catch-all on the end (used to force a match);
-            var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
+            var adjustedregex = rule.regex;
+            var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
+            if (Array.isArray(rule.token)) {
+                if (rule.token.length == 1) {
+                    rule.token = rule.token[0];
+                } else {
+                    rule.tokenArray = rule.token;
+                    rule.token = this.$arrayTokens;
+                }
+            }
 
+            if (matchcount > 1) {
+                if (/\\\d/.test(rule.regex)) {
                     // Replace any backreferences and offset appropriately.
-            var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
+                    adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) {
                         return "\\" + (parseInt(digit, 10) + matchTotal + 1);
                     });
+                } else {
+                    matchcount = 1;
+                    adjustedregex = this.removeCapturingGroups(rule.regex);
+                }
+                if (!rule.splitRegex)
+                    rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
+            }
 
-            if (matchcount > 1 && state[i].token.length !== matchcount-1)
+            mapping[matchTotal] = i;
-                throw new Error("For " + state[i].regex + " the matching groups (" +(matchcount-1) + ") and length of the token array (" + state[i].token.length + ") don't match (rule #" + i + " of state " + key + ")");
-
-            mapping[matchTotal] = {
-                rule: i,
-                len: matchcount
-            };
             matchTotal += matchcount;
 
             ruleRegExps.push(adjustedregex);
         }
 
-        this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
+        this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
     }
 };
 
 (function() {
+    this.$arrayTokens = function(str) {
+        if (!str)
+            return [];
+        var values = str.split(this.splitRegex)
+        var tokens = [];
+        var types = this.tokenArray;
+        if (types.length != values.length - 2) {
+            if (window.console)
+                console.error(types.length , values.length - 2, str, this.splitRegex);
+            return [{type: "error.invalid", value: str}];
+        }
+        for (var i = 0; i < types.length; i++) {
+            if (values[i + 1]) {
+                tokens[tokens.length] = {
+                    type: types[i],
+                    value: values[i + 1]
+                };
+            }
+        }
+        return tokens;
+    };
+    
+    this.removeCapturingGroups = function(src) {
+        var r = src.replace(
+            /\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
+            function(x, y) {return y ? "(?:" : x;}
+        );
+        return r;
+    };
+    
+    this.createSplitterRegexp = function(src, flag) {
+        src = src.replace(/\(\?=([^()]|\\.)*?\)$/, "");
+        return new RegExp(src, flag);
+    };
 
     /**
     * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
     * @returns {Object}
     **/
     this.getLineTokens = function(line, startState) {
+        if (startState && typeof startState != "string") {
+            var stack = startState.slice(0);
+            startState = stack[0];
+        } else
+            var stack = [];
+
         var currentState = startState || "start";
-        var state = this.rules[currentState];
+        var state = this.states[currentState];
         var mapping = this.matchMappings[currentState];
         var re = this.regExps[currentState];
         re.lastIndex = 0;
 
         var match, tokens = [];
-
         var lastIndex = 0;
 
-        var token = {
+        var token = {type: null, value: ""};
-            type: null,
-            value: ""
-        };
 
         while (match = re.exec(line)) {
-            var type = "text";
+            var type = mapping.defaultToken;
             var rule = null;
-            var value = [match[0]];
+            var value = match[0];
+            var index = re.lastIndex;
+
+            if (index - value.length > lastIndex) {
+                var skipped = line.substring(lastIndex, index - value.length);
+                if (token.type == type) {
+                    token.value += skipped;
+                } else {
+                    if (token.type)
+                        tokens.push(token);
+                    token = {type: type, value: skipped};
+                }
+            }
 
             for (var i = 0; i < match.length-2; i++) {
                 if (match[i + 1] === undefined)
                     continue;
 
-                rule = state[mapping[i].rule];
+                rule = state[mapping[i]];
-
-                if (mapping[i].len > 1)
-                    value = match.slice(i+2, i+1+mapping[i].len);
 
                 // compute token type
-                if (typeof rule.token == "function")
+                type = typeof rule.token == "function"
-                    type = rule.token.apply(this, value);
+                    ? rule.token(value, currentState, stack)
-                else
+                    : rule.token;
-                    type = rule.token;
 
                 if (rule.next) {
                     currentState = rule.next;
-                    state = this.rules[currentState];
+                    state = this.states[currentState];
-                    mapping = this.matchMappings[currentState];
+                    if (!state) {
-                    lastIndex = re.lastIndex;
+                        window.console && console.error && console.error(currentState, "doesn't exist");
-
+                        currentState = "start";
-                    re = this.regExps[currentState];
+                        state = this.states[currentState];
-
-                    if (re === undefined) {
-                         throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
                     }
-
+                    mapping = this.matchMappings[currentState];
-                    re.lastIndex = lastIndex;
+                    lastIndex = index;
+                    re = this.regExps[currentState];
+                    re.lastIndex = index;
                 }
                 break;
             }
 
-            if (value[0]) {
+            if (value) {
                 if (typeof type == "string") {
-                    value = [value.join("")];
+                    if ((!rule || rule.merge !== false) && token.type === type) {
-                    type = [type];
+                        token.value += value;
-                }
-                for (var i = 0; i < value.length; i++) {
-                    if (!value[i])
-                        continue;
-
-                    if ((!rule || rule.merge !== false) && token.type === type[i]) {
-                        token.value += value[i];
                     } else {
                         if (token.type)
                             tokens.push(token);
-
+                        token = {type: type, value: value};
-                        token = {
-                            type: type[i],
-                            value: value[i]
-                        };
                     }
+                } else {
+                    if (token.type)
+                        tokens.push(token);
+                    token = {type: null, value: ""};
+                    for (var i = 0; i < type.length; i++)
+                        tokens.push(type[i]);
                 }
             }
 
             if (lastIndex == line.length)
                 break;
 
-            lastIndex = re.lastIndex;
+            lastIndex = index;
         }
 
         if (token.type)
@@ -184,7 +237,7 @@ var Tokenizer = function(rules, flag) {
 
         return {
             tokens : tokens,
-            state : currentState
+            state : stack.length ? stack : currentState
         };
     };
 

lib/ace/tokenizer_dev.js

@@ -37,51 +37,111 @@ define(function(require, exports, module) {
  **/
 var Tokenizer = function(rules, flag) {
     flag = flag ? "g" + flag : "g";
-    this.rules = rules;
+    this.states = rules;
 
     this.regExps = {};
     this.matchMappings = {};
-    for ( var key in this.rules) {
+    for (var key in this.states) {
-        var rule = this.rules[key];
+        var state = this.states[key];
-        var state = rule;
         var ruleRegExps = [];
         var matchTotal = 0;
-        var mapping = this.matchMappings[key] = {};
+        var mapping = this.matchMappings[key] = {defaultToken: "default.text"};
 
         for (var i = 0; i < state.length; i++) {
-
+            var rule = state[i];
-            if (state[i].regex instanceof RegExp)
+            if (rule.defaultToken) {
-                state[i].regex = state[i].regex.toString().slice(1, -1);
+                mapping.defaultToken = rule.defaultToken;
+                continue;
+            }
+            if (rule.regex instanceof RegExp)
+                rule.regex = rule.regex.toString().slice(1, -1);
 
             // Count number of matching groups. 2 extra groups from the full match
             // And the catch-all on the end (used to force a match);
-            var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
+            var adjustedregex = rule.regex;
+            var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
+            if (Array.isArray(rule.token)) {
+                if (rule.token.length == 1) {
+                    rule.token = rule.token[0];
+                } else {
+                    rule.tokenArray = rule.token;
+                    rule.token = this.$arrayTokens;
+                }
+            }
 
+            if (matchcount > 1) {
+                if (/\\\d/.test(rule.regex)) {
                     // Replace any backreferences and offset appropriately.
-            var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
+                    adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) {
                         return "\\" + (parseInt(digit, 10) + matchTotal + 1);
                     });
+                } else {
+                    matchcount = 1;
+                    adjustedregex = this.removeCapturingGroups(rule.regex);
+                }
+                if (!rule.splitRegex)
+                    rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
+            }
 
-            if (matchcount > 1 && state[i].token.length !== matchcount-1)
+            mapping[matchTotal] = i;
-                throw new Error("For " + state[i].regex + " the matching groups and length of the token array don't match (rule #" + i + " of state " + key + ")");
-
-            mapping[matchTotal] = {
-                rule: i,
-                len: matchcount
-            };
             matchTotal += matchcount;
 
             ruleRegExps.push(adjustedregex);
         }
 
-        this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
+        this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
     }
 };
 
 (function() {
+    this.$arrayTokens = function(str) {
+        if (!str)
+            return [];
+        var values = str.split(this.splitRegex)
+        var tokens = [];
+        var types = this.tokenArray;
+        if (types.length != values.length - 2) {
+            if (window.console)
+                console.error(types.length , values.length - 2, str, this.splitRegex);
+            return [{type: "error.invalid", value: str}];
+        }
+        for (var i = 0; i < types.length; i++) {
+            if (values[i + 1]) {
+                tokens[tokens.length] = {
+                    type: types[i],
+                    value: values[i + 1]
+                };
+            }
+        }
+        return tokens;
+    };
+    
+    this.removeCapturingGroups = function(src) {
+        var r = src.replace(
+            /\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
+            function(x, y) {return y ? "(?:" : x;}
+        );
+        return r;
+    };
+    
+    this.createSplitterRegexp = function(src, flag) {
+        src = src.replace(/\(\?=([^()]|\\.)*?\)$/, "");
+        return new RegExp(src, flag);
+    };
+
+    /**
+    * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
+    * @returns {Object}
+    **/
     this.getLineTokens = function(line, startState) {
+        if (startState && typeof startState != "string") {
+            var stack = startState.slice(0);
+            startState = stack[0];
+        } else
+            var stack = [];
+
         var currentState = startState || "start";
-        var state = this.rules[currentState];
+        var state = this.states[currentState];
         var mapping = this.matchMappings[currentState];
         var re = this.regExps[currentState];
         re.lastIndex = 0;
@@ -110,92 +170,85 @@ var Tokenizer = function(rules, flag) {
         var maxRecur = 10000;
         
         while (match = re.exec(line)) {
-            var type = "default.text";
+            var type = mapping.defaultToken;
             var rule = null;
-            var value = [match[0]];
+            var value = match[0];
+            var index = re.lastIndex;
+
+            if (index - value.length > lastIndex) {
+                var skipped = line.substring(lastIndex, index - value.length);
+                if (token.type == type) {
+                    token.value += skipped;
+                } else {
+                    if (token.type)
+                        tokens.push(token);
+                    token = {type: type, value: skipped};
+                }
+            }
 
             for (var i = 0; i < match.length-2; i++) {
                 if (match[i + 1] === undefined)
                     continue;
                 
                 if (!maxRecur--) {
-                    throw "infinite" + mapping[i].rule + currentState
+                    throw "infinite" + state[mapping[i]] + currentState
                 }
 
-                rule = state[mapping[i].rule];
+                rule = state[mapping[i]];
-
-                if (mapping[i].len > 1)
-                    value = match.slice(i+2, i+1+mapping[i].len);
 
                 // compute token type
-                if (typeof rule.token == "function")
+                type = typeof rule.token == "function"
-                    type = rule.token.apply(this, value);
+                    ? rule.token(value, currentState, stack)
-                else
+                    : rule.token;
-                    type = rule.token;
 
                 if (rule.next) {
                     currentState = rule.next;
-                    state = this.rules[currentState];
+                    state = this.states[currentState];
-                    mapping = this.matchMappings[currentState];
+                    if (!state) {
-                    lastIndex = re.lastIndex;
+                        window.console && console.error && console.error(currentState, "doesn't exist");
-
+                        currentState = "start";
-                    re = this.regExps[currentState];
+                        state = this.states[currentState];
-
-                    if (re === undefined) {
-                         throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
                     }
-
+                    mapping = this.matchMappings[currentState];
-                    re.lastIndex = lastIndex;
+                    lastIndex = index;
+                    re = this.regExps[currentState];
+                    re.lastIndex = index;
 
                     onStateChange();
                 }
                 break;
             }
 
-            if (value[0]) {
+            if (value) {
                 if (typeof type == "string") {
-                    value = [value.join("")];
+                    if ((!rule || rule.merge !== false) && token.type === type) {
-                    type = [type];
+                        token.value += value;
-                }
-                for (var i = 0; i < value.length; i++) {
-                    if (!value[i])
-                        continue;
-
-                    var mergeable = (!rule || rule.merge || type[i] === "text") && token.type === type[i];
-                    
-                    if (false && mergeable) {
-                        token.value += value[i];
                     } else {
-                        if (token.type) {
+                        if (token.type)
-                            token.stateTransitions = stateTransitions;
                             tokens.push(token);
-                            initState()
+                        token = {type: type, value: value};
-                        }
-                        
-                        token = {
-                            type: type[i],
-                            value: value[i],
-                            state: currentState,
-                            mergeable: mergeable
-                        };
                     }
+                } else {
+                    if (token.type)
+                        tokens.push(token);
+                    token = {type: null, value: ""};
+                    for (var i = 0; i < type.length; i++)
+                        tokens.push(type[i]);
                 }
             }
 
             if (lastIndex == line.length)
                 break;
 
-            lastIndex = re.lastIndex;
+            lastIndex = index;
         }
 
-        if (token.type) {
+        if (token.type)
-            token.stateTransitions = stateTransitions;
             tokens.push(token);
-        }
 
         return {
             tokens : tokens,
-            state : currentState
+            state : stack.length ? stack : currentState
         };
     };
 

tool/mode_creator.js

@@ -118,7 +118,7 @@ function run() {
     var path = "ace/mode/new";
     var deps = getDeps(src, path);
     src = src.replace("define(", 'define("' + path +'", ["require","exports","module",' + deps +'],');
-    src += ';require(["ace/mode/new"], continueRun, function(e){console.log(e);require.undef("ace/mode/new")})';
+    src += ';require(["ace/mode/new"], continueRun, function(e){console.log(e);window.require.undef("ace/mode/new")})';
     try {
         eval(src);
     } catch(e) {