From 4b398798c0c43c5aa13cfa804a073fe0bd431ba3 Mon Sep 17 00:00:00 2001
From: nightwing <amirjanyan@gmail.com>
Date: Sat, 22 Jun 2013 22:38:54 +0400
Subject: [PATCH 1/4] highlight escapes in sh mode

---
 lib/ace/mode/sh_highlight_rules.js | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/lib/ace/mode/sh_highlight_rules.js b/lib/ace/mode/sh_highlight_rules.js
index a73efb5e..d9ed5600 100644
--- a/lib/ace/mode/sh_highlight_rules.js
+++ b/lib/ace/mode/sh_highlight_rules.js
@@ -77,12 +77,28 @@ var ShHighlightRules = function() {
     var func = "(?:" + variableName + "\\s*\\(\\))";
 
     this.$rules = {
-        "start" : [ {
+        "start" : [{
+            token : "constant",
+            regex : /\\./
+        }, {
             token : ["text", "comment"],
             regex : /(^|\s)(#.*)$/
         }, {
-            token : "string",           // " string
-            regex : '"(?:[^\\\\]|\\\\.)*?"'
+            token : "string",
+            regex : '"',
+            push : [{
+                token : "constant.language.escape",
+                regex : /\\(?:[$abeEfnrtv\\'"]|x[a-fA-F\d]{1,2}|u[a-fA-F\d]{4}([a-fA-F\d]{4})?|c.|\d{1,3})/
+            }, {
+                token : "constant",
+                regex : /\$\w+/
+            }, {
+                token : "string",
+                regex : '"',
+                next: "pop"
+            }, {
+                defaultToken: "string"
+            }]
         }, {
             token : "variable.language",
             regex : builtinVariable
@@ -97,7 +113,7 @@ var ShHighlightRules = function() {
             regex : fileDescriptor
         }, {
             token : "string",           // ' string
-            regex : "'(?:[^\\\\]|\\\\.)*?'"
+            start : "'", end : "'"
         }, {
             token : "constant.numeric", // float
             regex : floatNumber
@@ -118,6 +134,8 @@ var ShHighlightRules = function() {
             regex : "[\\]\\)\\}]"
         } ]
     };
+    
+    this.normalizeRules();
 };
 
 oop.inherits(ShHighlightRules, TextHighlightRules);

From b9190d1e8de0485b04b4a5254160d954fb22cf6a Mon Sep 17 00:00:00 2001
From: nightwing <amirjanyan@gmail.com>
Date: Sun, 23 Jun 2013 22:25:50 +0400
Subject: [PATCH 2/4] fix go string highlighting

---
 lib/ace/mode/golang_highlight_rules.js | 44 +++++++++-----------------
 1 file changed, 15 insertions(+), 29 deletions(-)

diff --git a/lib/ace/mode/golang_highlight_rules.js b/lib/ace/mode/golang_highlight_rules.js
index 26010bd1..b323e9e4 100644
--- a/lib/ace/mode/golang_highlight_rules.js
+++ b/lib/ace/mode/golang_highlight_rules.js
@@ -5,7 +5,7 @@ define(function(require, exports, module) {
 
     var GolangHighlightRules = function() {
         var keywords = (
-            "true|else|false|break|case|return|goto|if|const|" +
+            "else|break|case|return|goto|if|const|" +
             "continue|struct|default|switch|for|" +
             "func|import|package|chan|defer|fallthrough|go|interface|map|range" +
             "select|type|var"
@@ -13,7 +13,6 @@ define(function(require, exports, module) {
         var buildinConstants = ("nil|true|false|iota");
 
         var keywordMapper = this.createKeywordMapper({
-            "variable.language": "this",
             "keyword": keywords,
             "constant.language": buildinConstants
         }, "identifier");
@@ -32,35 +31,29 @@ define(function(require, exports, module) {
                 }, {
                     token : "string", // single line
                     regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
-                }, {
-                    token : "string", // multi line string start
-                    regex : '["].*\\\\$',
-                    next : "qqstring"
                 }, {
                     token : "string", // single line
-                    regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
+                    regex : '[`](?:[^`]*)[`]'
                 }, {
                     token : "string", // multi line string start
-                    regex : "['].*\\\\$",
-                    next : "qstring"
+                    merge : true,
+                    regex : '[`](?:[^`]*)$',
+                    next : "bqstring"
+                }, {
+                    token : "constant.numeric", // rune
+                    regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))[']"
                 }, {
                     token : "constant.numeric", // hex
                     regex : "0[xX][0-9a-fA-F]+\\b"
                 }, {
                     token : "constant.numeric", // float
                     regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
-                }, {
-                    token : "constant", // <CONSTANT>
-                    regex : "<[a-zA-Z0-9.]+>"
-                }, {
-                    token : "keyword", // pre-compiler directivs
-                    regex : "(?:#include|#pragma|#line|#define|#undef|#ifdef|#else|#elif|#endif|#ifndef)"
                 }, {
                     token : keywordMapper,
                     regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
                 }, {
                     token : "keyword.operator",
-                    regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|==|=|!=|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|new|delete|typeof|void)"
+                    regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|==|=|!=|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^="
                 }, {
                     token : "punctuation.operator",
                     regex : "\\?|\\:|\\,|\\;|\\."
@@ -70,6 +63,9 @@ define(function(require, exports, module) {
                 }, {
                     token : "paren.rparen",
                     regex : "[\\])}]"
+                }, {
+                    token: "invalid",
+                    regex: "\\s+$"
                 }, {
                     token : "text",
                     regex : "\\s+"
@@ -85,20 +81,10 @@ define(function(require, exports, module) {
                     regex : ".+"
                 }
             ],
-            "qqstring" : [
+            "bqstring" : [
                 {
                     token : "string",
-                    regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
-                    next : "start"
-                }, {
-                    token : "string",
-                    regex : '.+'
-                }
-            ],
-            "qstring" : [
-                {
-                    token : "string",
-                    regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
+                    regex : '(?:[^`]*)`',
                     next : "start"
                 }, {
                     token : "string",
@@ -109,7 +95,7 @@ define(function(require, exports, module) {
 
         this.embedRules(DocCommentHighlightRules, "doc-",
             [ DocCommentHighlightRules.getEndRule("start") ]);
-    }
+    };
     oop.inherits(GolangHighlightRules, TextHighlightRules);
 
     exports.GolangHighlightRules = GolangHighlightRules;

From a6654eee9ea188aa23ea6c48b985a3f6ed921361 Mon Sep 17 00:00:00 2001
From: nightwing <amirjanyan@gmail.com>
Date: Sun, 23 Jun 2013 22:35:23 +0400
Subject: [PATCH 3/4] update tests

---
 lib/ace/mode/_test/tokens_assembly_x86.json | 114 +++
 lib/ace/mode/_test/tokens_batchfile.json    |  80 ++
 lib/ace/mode/_test/tokens_html_ruby.json    | 241 ++++++
 lib/ace/mode/_test/tokens_lsl.json          |   2 +-
 lib/ace/mode/_test/tokens_mushcode.json     | 790 ++++++++++++++++++++
 lib/ace/mode/_test/tokens_properties.json   |  68 ++
 lib/ace/mode/_test/tokens_sh.json           |   4 +-
 lib/ace/mode/_test/tokens_snippets.json     |  50 --
 lib/ace/mode/_test/tokens_twig.json         |  82 ++
 lib/ace/mode/_test/tokens_velocity.json     | 281 +++++++
 10 files changed, 1660 insertions(+), 52 deletions(-)
 create mode 100644 lib/ace/mode/_test/tokens_assembly_x86.json
 create mode 100644 lib/ace/mode/_test/tokens_batchfile.json
 create mode 100644 lib/ace/mode/_test/tokens_html_ruby.json
 create mode 100644 lib/ace/mode/_test/tokens_mushcode.json
 create mode 100644 lib/ace/mode/_test/tokens_properties.json
 create mode 100644 lib/ace/mode/_test/tokens_twig.json
 create mode 100644 lib/ace/mode/_test/tokens_velocity.json

diff --git a/lib/ace/mode/_test/tokens_assembly_x86.json b/lib/ace/mode/_test/tokens_assembly_x86.json
new file mode 100644
index 00000000..5da0470e
--- /dev/null
+++ b/lib/ace/mode/_test/tokens_assembly_x86.json
@@ -0,0 +1,114 @@
+[[
+   "start",
+  ["support.function.directive.assembly","section"],
+  ["text","\t.text"]
+],[
+   "start",
+  ["text","    "],
+  ["support.function.directive.assembly","global"],
+  ["text"," "],
+  ["entity.name.function.assembly","main"],
+  ["text","         "],
+  ["comment.assembly",";must be declared for using gcc"]
+],[
+   "start"
+],[
+   "start",
+  ["entity.name.function.assembly","main:"],
+  ["text","\t                "],
+  ["comment.assembly",";tell linker entry point"]
+],[
+   "start"
+],[
+   "start",
+  ["text","\t"],
+  ["keyword.control.assembly","mov"],
+  ["text","\t"],
+  ["variable.parameter.register.assembly","edx"],
+  ["text",", len\t    "],
+  ["comment.assembly",";message length"]
+],[
+   "start",
+  ["text","\t"],
+  ["keyword.control.assembly","mov"],
+  ["text","\t"],
+  ["variable.parameter.register.assembly","ecx"],
+  ["text",", msg\t    "],
+  ["comment.assembly",";message to write"]
+],[
+   "start",
+  ["text","\t"],
+  ["keyword.control.assembly","mov"],
+  ["text","\t"],
+  ["variable.parameter.register.assembly","ebx"],
+  ["text",", "],
+  ["constant.character.decimal.assembly","1"],
+  ["text","\t        "],
+  ["comment.assembly",";file descriptor (stdout)"]
+],[
+   "start",
+  ["text","\t"],
+  ["keyword.control.assembly","mov"],
+  ["text","\t"],
+  ["variable.parameter.register.assembly","eax"],
+  ["text",", "],
+  ["constant.character.decimal.assembly","4"],
+  ["text","\t        "],
+  ["comment.assembly",";system call number (sys_write)"]
+],[
+   "start",
+  ["text","\t"],
+  ["keyword.control.assembly","int"],
+  ["text","\t"],
+  ["constant.character.hexadecimal.assembly","0x80"],
+  ["text","\t        "],
+  ["comment.assembly",";call kernel"]
+],[
+   "start"
+],[
+   "start",
+  ["text","\t"],
+  ["keyword.control.assembly","mov"],
+  ["text","\t"],
+  ["variable.parameter.register.assembly","eax"],
+  ["text",", "],
+  ["constant.character.decimal.assembly","1"],
+  ["text","\t        "],
+  ["comment.assembly",";system call number (sys_exit)"]
+],[
+   "start",
+  ["text","\t"],
+  ["keyword.control.assembly","int"],
+  ["text","\t"],
+  ["constant.character.hexadecimal.assembly","0x80"],
+  ["text","\t        "],
+  ["comment.assembly",";call kernel"]
+],[
+   "start"
+],[
+   "start",
+  ["support.function.directive.assembly","section"],
+  ["text","\t.data"]
+],[
+   "start"
+],[
+   "start",
+  ["entity.name.function.assembly","msg"],
+  ["text","\t"],
+  ["support.function.directive.assembly","db"],
+  ["text","\t"],
+  ["string.assembly","'Hello, world!'"],
+  ["text",","],
+  ["constant.character.hexadecimal.assembly","0xa"],
+  ["text","\t"],
+  ["comment.assembly",";our dear string"]
+],[
+   "start",
+  ["entity.name.function.assembly","len"],
+  ["text","\t"],
+  ["support.function.directive.assembly","equ"],
+  ["text","\t$ - msg\t\t\t"],
+  ["comment.assembly",";length of our dear string"]
+],[
+   "start"
+]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_batchfile.json b/lib/ace/mode/_test/tokens_batchfile.json
new file mode 100644
index 00000000..d685d835
--- /dev/null
+++ b/lib/ace/mode/_test/tokens_batchfile.json
@@ -0,0 +1,80 @@
+[[
+   "start",
+  ["comment.line.colons.dosbatch",":: batch file highlighting in Ace!"]
+],[
+   "start",
+  ["text","@"],
+  ["keyword.command.dosbatch","echo"],
+  ["text"," off"]
+],[
+   "start"
+],[
+   "start",
+  ["keyword.control.statement.dosbatch","CALL"],
+  ["text"," "],
+  ["keyword.command.dosbatch","set"],
+  ["text"," var1="],
+  ["markup.list","%"],
+  ["constant.other","cd"],
+  ["markup.list","%"]
+],[
+   "start",
+  ["keyword.command.dosbatch","echo"],
+  ["text"," unhide everything in "],
+  ["markup.list","%"],
+  ["constant.other","var1"],
+  ["markup.list","%"],
+  ["text","!"]
+],[
+   "start"
+],[
+   "start",
+  ["comment.line.colons.dosbatch",":: FOR loop in bat is super strange!"]
+],[
+   "start",
+  ["keyword.control.repeat.dosbatch","FOR"],
+  ["text"," /f "],
+  ["punctuation.definition.string.begin.shell","\""],
+  ["string.quoted.double.dosbatch","tokens=*"],
+  ["punctuation.definition.string.end.shell","\""],
+  ["text"," "],
+  ["constant.numeric","%%G"],
+  ["text"," IN ('"],
+  ["keyword.command.dosbatch","dir"],
+  ["text"," /A:D /b') DO ("]
+],[
+   "start",
+  ["keyword.command.dosbatch","echo"],
+  ["text"," "],
+  ["markup.list","%"],
+  ["constant.other","var1"],
+  ["markup.list","%"],
+  ["constant.numeric","%%G"]
+],[
+   "start",
+  ["keyword.command.dosbatch","attrib"],
+  ["text"," -r -a -h -s "],
+  ["punctuation.definition.string.begin.shell","\""],
+  ["markup.list","%"],
+  ["constant.other","var1"],
+  ["markup.list","%"],
+  ["constant.numeric","%%G"],
+  ["punctuation.definition.string.end.shell","\""],
+  ["text"," /D /S"]
+],[
+   "start",
+  ["text",")"]
+],[
+   "start"
+],[
+   "start",
+  ["keyword.command.dosbatch","pause"]
+],[
+   "start"
+],[
+   "start",
+  ["doc.comment","REM"],
+  ["comment"," that's all"]
+],[
+   "start"
+]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_html_ruby.json b/lib/ace/mode/_test/tokens_html_ruby.json
new file mode 100644
index 00000000..203d4b4b
--- /dev/null
+++ b/lib/ace/mode/_test/tokens_html_ruby.json
@@ -0,0 +1,241 @@
+[[
+   "start",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","h1"],
+  ["meta.tag.r",">"],
+  ["text","Listing Books"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name","h1"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text"," "]
+],[
+   "start",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","table"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","  "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","tr"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"],
+  ["text","Title"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"],
+  ["text","Summary"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","th"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","  "],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","tr"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text"," "]
+],[
+   "start",
+  ["support.ruby_tag","<%"],
+  ["text"," "],
+  ["variable.instance","@books"],
+  ["text","."],
+  ["identifier","each"],
+  ["text"," "],
+  ["keyword","do"],
+  ["text"," |"],
+  ["identifier","book"],
+  ["text","| "],
+  ["support.ruby_tag","%>"]
+],[
+   "start",
+  ["text","  "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","tr"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"],
+  ["support.ruby_tag","<%="],
+  ["text"," "],
+  ["identifier","book"],
+  ["text","."],
+  ["identifier","title"],
+  ["text"," "],
+  ["support.ruby_tag","%>"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"],
+  ["support.ruby_tag","<%="],
+  ["text"," "],
+  ["identifier","book"],
+  ["text","."],
+  ["identifier","content"],
+  ["text"," "],
+  ["support.ruby_tag","%>"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"],
+  ["support.ruby_tag","<%="],
+  ["text"," "],
+  ["support.function","link_to"],
+  ["text"," "],
+  ["string","'Show'"],
+  ["text",", "],
+  ["identifier","book"],
+  ["text"," "],
+  ["support.ruby_tag","%>"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"],
+  ["support.ruby_tag","<%="],
+  ["text"," "],
+  ["support.function","link_to"],
+  ["text"," "],
+  ["string","'Edit'"],
+  ["text",", "],
+  ["identifier","edit_book_path"],
+  ["paren.lparen","("],
+  ["identifier","book"],
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["support.ruby_tag","%>"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","    "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"],
+  ["support.ruby_tag","<%="],
+  ["text"," "],
+  ["support.function","link_to"],
+  ["text"," "],
+  ["string","'Remove'"],
+  ["text",", "],
+  ["identifier","book"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":confirm"],
+  ["text"," "],
+  ["punctuation.separator.key-value","=>"],
+  ["text"," "],
+  ["string","'Are you sure?'"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":method"],
+  ["text"," "],
+  ["punctuation.separator.key-value","=>"],
+  ["text"," "],
+  ["constant.other.symbol.ruby",":delete"],
+  ["text"," "],
+  ["support.ruby_tag","%>"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","td"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","  "],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","tr"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["support.ruby_tag","<%"],
+  ["text"," "],
+  ["keyword","end"],
+  ["text"," "],
+  ["support.ruby_tag","%>"]
+],[
+   "start",
+  ["meta.tag","</"],
+  ["meta.tag.tag-name.table","table"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text"," "]
+],[
+   "start",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","br"],
+  ["text"," "],
+  ["meta.tag.r","/>"]
+],[
+   "start",
+  ["text"," "]
+],[
+   "start",
+  ["support.ruby_tag","<%="],
+  ["text"," "],
+  ["support.function","link_to"],
+  ["text"," "],
+  ["string","'New book'"],
+  ["text",", "],
+  ["identifier","new_book_path"],
+  ["text"," "],
+  ["support.ruby_tag","%>"]
+]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_lsl.json b/lib/ace/mode/_test/tokens_lsl.json
index a34106a6..b007cf11 100644
--- a/lib/ace/mode/_test/tokens_lsl.json
+++ b/lib/ace/mode/_test/tokens_lsl.json
@@ -492,4 +492,4 @@
   ["paren.rparen.lsl","}"]
 ],[
    "start"
-]]
+]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_mushcode.json b/lib/ace/mode/_test/tokens_mushcode.json
new file mode 100644
index 00000000..9f8e7cc2
--- /dev/null
+++ b/lib/ace/mode/_test/tokens_mushcode.json
@@ -0,0 +1,790 @@
+[[
+   "start",
+  ["text","@"],
+  ["support.function","create"],
+  ["text"," "],
+  ["identifier","phone"]
+],[
+   "start",
+  ["text","&"],
+  ["identifier","pickup"],
+  ["text"," "],
+  ["identifier","phone"],
+  ["keyword.operator","="],
+  ["identifier","$pick"],
+  ["text"," "],
+  ["identifier","up"],
+  ["text",":@"],
+  ["support.function","ifelse"],
+  ["text"," "],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","is"],
+  ["text",","],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","mode"],
+  ["paren.rparen",")"],
+  ["text",","],
+  ["identifier","ICC"],
+  ["paren.rparen",")]"],
+  ["keyword.operator","="],
+  ["paren.lparen","{"],
+  ["text","@"],
+  ["support.function","pemit"],
+  ["text"," "],
+  ["keyword.operator","%#="],
+  ["identifier","You"],
+  ["text"," "],
+  ["support.function","pick"],
+  ["text"," "],
+  ["identifier","up"],
+  ["text"," "],
+  ["identifier","the"],
+  ["text"," "],
+  ["paren.lparen","["],
+  ["support.function","fullname"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["paren.rparen",")]"],
+  ["text","."],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["text",","],
+  ["identifier","PHONER"],
+  ["text",":"],
+  ["keyword.operator","%#"],
+  ["paren.rparen",")]"],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["text",","],
+  ["identifier","MODE"],
+  ["text",":"],
+  ["identifier","CIP"],
+  ["paren.rparen",")]"],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","(["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","INCOMING"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["identifier","CONNECTED"],
+  ["text",":"],
+  ["paren.lparen","["],
+  ["support.function","num"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["paren.rparen",")])]"],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["text",","],
+  ["identifier","CONNECTED"],
+  ["text",":"],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","INCOMING"],
+  ["paren.rparen",")])]"],
+  ["variable","%r"],
+  ["paren.lparen","["],
+  ["support.function","showpicture"],
+  ["paren.lparen","("],
+  ["identifier","PICPICKUP"],
+  ["paren.rparen",")]"],
+  ["variable","%r"],
+  ["identifier","Use"],
+  ["text"," '"],
+  ["paren.lparen","["],
+  ["identifier","color"],
+  ["paren.lparen","("],
+  ["identifier","green"],
+  ["text",","],
+  ["identifier","black"],
+  ["text",","],
+  ["identifier","psay"],
+  ["text"," "],
+  ["keyword.operator","<"],
+  ["identifier","message"],
+  ["keyword.operator",">"],
+  ["paren.rparen",")]"],
+  ["text","' "],
+  ["paren.lparen","("],
+  ["support.function","or"],
+  ["text"," '"],
+  ["paren.lparen","["],
+  ["identifier","color"],
+  ["paren.lparen","("],
+  ["identifier","green"],
+  ["text",","],
+  ["identifier","black"],
+  ["text",","],
+  ["identifier","p"],
+  ["text"," "],
+  ["keyword.operator","<"],
+  ["identifier","message"],
+  ["keyword.operator",">"],
+  ["paren.rparen",")]"],
+  ["text","'"],
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["identifier","to"],
+  ["text"," "],
+  ["identifier","talk"],
+  ["text"," "],
+  ["identifier","into"],
+  ["text"," "],
+  ["identifier","the"],
+  ["text"," "],
+  ["identifier","phone"],
+  ["text",".;@"],
+  ["support.function","oemit"],
+  ["text"," "],
+  ["keyword.operator","%#="],
+  ["variable","%N"],
+  ["text"," "],
+  ["identifier","picks"],
+  ["text"," "],
+  ["identifier","up"],
+  ["text"," "],
+  ["identifier","the"],
+  ["text"," "],
+  ["paren.lparen","["],
+  ["support.function","fullname"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["paren.rparen",")]"],
+  ["text","."],
+  ["paren.rparen","}"],
+  ["text",","],
+  ["paren.lparen","{"],
+  ["text","@"],
+  ["support.function","pemit"],
+  ["text"," "],
+  ["keyword.operator","%#="],
+  ["identifier","You"],
+  ["text"," "],
+  ["support.function","pick"],
+  ["text"," "],
+  ["identifier","up"],
+  ["text"," "],
+  ["identifier","the"],
+  ["text"," "],
+  ["identifier","phone"],
+  ["text"," "],
+  ["identifier","but"],
+  ["text"," "],
+  ["identifier","no"],
+  ["text"," "],
+  ["identifier","one"],
+  ["text"," "],
+  ["identifier","is"],
+  ["text"," "],
+  ["identifier","there"],
+  ["text",". "],
+  ["identifier","You"],
+  ["text"," "],
+  ["identifier","hear"],
+  ["text"," "],
+  ["identifier","a"],
+  ["text"," "],
+  ["identifier","dialtone"],
+  ["text"," "],
+  ["support.function","and"],
+  ["text"," "],
+  ["identifier","then"],
+  ["text"," "],
+  ["identifier","hang"],
+  ["text"," "],
+  ["identifier","up"],
+  ["text",". "],
+  ["paren.lparen","["],
+  ["support.function","play"],
+  ["paren.lparen","("],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","DIALTONE"],
+  ["paren.rparen","))]"],
+  ["text",";@"],
+  ["support.function","oemit"],
+  ["text"," "],
+  ["keyword.operator","%#="],
+  ["variable","%N"],
+  ["text"," "],
+  ["identifier","picks"],
+  ["text"," "],
+  ["identifier","up"],
+  ["text"," "],
+  ["identifier","the"],
+  ["text"," "],
+  ["identifier","phone"],
+  ["text",", "],
+  ["identifier","but"],
+  ["text"," "],
+  ["identifier","no"],
+  ["text"," "],
+  ["identifier","one"],
+  ["text"," "],
+  ["identifier","is"],
+  ["text"," "],
+  ["identifier","on"],
+  ["text"," "],
+  ["identifier","the"],
+  ["text"," "],
+  ["identifier","other"],
+  ["text"," "],
+  ["identifier","end"],
+  ["text","."],
+  ["paren.rparen","}"]
+],[
+   "start",
+  ["text","&"],
+  ["identifier","ringfun"],
+  ["text"," "],
+  ["identifier","phone"],
+  ["keyword.operator","="],
+  ["paren.lparen","["],
+  ["support.function","ifelse"],
+  ["paren.lparen","("],
+  ["support.function","eq"],
+  ["paren.lparen","("],
+  ["support.function","comp"],
+  ["paren.lparen","(["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["identifier","off"],
+  ["paren.rparen",")"],
+  ["text",","],
+  ["constant.numeric","0"],
+  ["paren.rparen",")"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["identifier","color"],
+  ["paren.lparen","("],
+  ["identifier","black"],
+  ["text",","],
+  ["identifier","cyan"],
+  ["text",","],
+  ["identifier","INCOMING"],
+  ["text"," "],
+  ["identifier","CALL"],
+  ["text"," "],
+  ["identifier","FROM"],
+  ["text"," "],
+  ["variable","%1"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","play"],
+  ["paren.lparen","(["],
+  ["support.function","switch"],
+  ["paren.lparen","(["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["constant.numeric","1"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone1"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["constant.numeric","2"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone2"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["constant.numeric","3"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone3"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["constant.numeric","4"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone4"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["constant.numeric","5"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone5"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["constant.numeric","6"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone6"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["constant.numeric","7"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone7"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["constant.numeric","8"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone8"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["constant.numeric","9"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","ringtone9"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["identifier","custom"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","customtone"],
+  ["paren.rparen",")]"],
+  ["text",","],
+  ["identifier","vibrate"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%0"],
+  ["keyword.operator","/"],
+  ["identifier","vibrate"],
+  ["paren.rparen",")])])]"]
+],[
+   "start",
+  ["text","&"],
+  ["identifier","ringloop"],
+  ["text"," "],
+  ["identifier","phone"],
+  ["keyword.operator","="],
+  ["text","@"],
+  ["support.function","switch"],
+  ["text"," "],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","ringstate"],
+  ["paren.rparen",")]"],
+  ["keyword.operator","="],
+  ["constant.numeric","1"],
+  ["text",","],
+  ["paren.lparen","{"],
+  ["text","@"],
+  ["support.function","emit"],
+  ["text"," "],
+  ["paren.lparen","["],
+  ["support.function","setq"],
+  ["paren.lparen","("],
+  ["identifier","q"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","connecting"],
+  ["paren.rparen",")])]"],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["text",","],
+  ["identifier","rangs"],
+  ["text",":"],
+  ["constant.numeric","0"],
+  ["paren.rparen",")]"],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["text",","],
+  ["identifier","mode"],
+  ["text",":"],
+  ["identifier","WFC"],
+  ["paren.rparen",")]"],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["text",","],
+  ["identifier","INCOMING"],
+  ["text",":"],
+  ["paren.rparen",")]"],
+  ["text",";@"],
+  ["support.function","ifelse"],
+  ["text"," "],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["keyword.operator","/"],
+  ["identifier","HASVMB"],
+  ["paren.rparen",")]"],
+  ["keyword.operator","="],
+  ["paren.lparen","{"],
+  ["text","@"],
+  ["support.function","tr"],
+  ["text"," "],
+  ["identifier","me"],
+  ["keyword.operator","/"],
+  ["identifier","ROUTEVMB"],
+  ["keyword.operator","="],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","connecting"],
+  ["paren.rparen",")]"],
+  ["text",";"],
+  ["paren.rparen","}"],
+  ["text",","],
+  ["paren.lparen","{"],
+  ["text","@"],
+  ["support.function","pemit"],
+  ["text"," "],
+  ["keyword.operator","%#="],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","MSGCNC"],
+  ["paren.rparen",")]"],
+  ["text",";"],
+  ["paren.rparen","}}"],
+  ["text",","],
+  ["constant.numeric","2"],
+  ["text",","],
+  ["paren.lparen","{"],
+  ["text","@"],
+  ["support.function","pemit"],
+  ["text"," "],
+  ["keyword.operator","%#="],
+  ["identifier","The"],
+  ["text"," "],
+  ["identifier","call"],
+  ["text"," "],
+  ["identifier","is"],
+  ["text"," "],
+  ["identifier","connected"],
+  ["text","."],
+  ["paren.lparen","["],
+  ["support.function","setq"],
+  ["paren.lparen","("],
+  ["identifier","q"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","CONNECTING"],
+  ["paren.rparen",")])]"],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["text",","],
+  ["identifier","CONNECTED"],
+  ["text",":"],
+  ["variable","%qq"],
+  ["paren.rparen",")]"],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["text",","],
+  ["identifier","CONNECTED"],
+  ["text",":"],
+  ["paren.lparen","["],
+  ["support.function","num"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["paren.rparen",")])]"],
+  ["paren.lparen","["],
+  ["support.function","set"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["text",","],
+  ["identifier","MODE"],
+  ["text",":"],
+  ["identifier","CIP"],
+  ["paren.rparen",")]"],
+  ["text",";@"],
+  ["support.function","tr"],
+  ["text"," "],
+  ["identifier","me"],
+  ["keyword.operator","/"],
+  ["identifier","ciploop"],
+  ["text",";@"],
+  ["support.function","tr"],
+  ["text"," "],
+  ["variable","%qq"],
+  ["keyword.operator","/"],
+  ["identifier","ciploop"],
+  ["text",";"],
+  ["paren.rparen","}"],
+  ["text",","],
+  ["constant.numeric","3"],
+  ["text",","],
+  ["paren.lparen","{"],
+  ["text","@"],
+  ["support.function","emit"],
+  ["text"," "],
+  ["identifier","On"],
+  ["text"," "],
+  ["paren.lparen","["],
+  ["support.function","fullname"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["paren.rparen",")]"],
+  ["text","'"],
+  ["support.function","s"],
+  ["text"," "],
+  ["identifier","earpiece"],
+  ["text"," "],
+  ["identifier","you"],
+  ["text"," "],
+  ["identifier","hear"],
+  ["text"," "],
+  ["identifier","a"],
+  ["text"," "],
+  ["identifier","ringing"],
+  ["text"," "],
+  ["identifier","sound"],
+  ["text","."],
+  ["paren.lparen","["],
+  ["support.function","play"],
+  ["paren.lparen","("],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","LINETONE"],
+  ["paren.rparen","))]"],
+  ["text",";@"],
+  ["support.function","tr"],
+  ["text"," "],
+  ["identifier","me"],
+  ["keyword.operator","/"],
+  ["identifier","ringhere"],
+  ["text",";@"],
+  ["identifier","increment"],
+  ["text"," "],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","connecting"],
+  ["paren.rparen",")]"],
+  ["keyword.operator","/"],
+  ["identifier","RANGS"],
+  ["text",";@"],
+  ["identifier","wait"],
+  ["text"," "],
+  ["constant.numeric","5"],
+  ["keyword.operator","="],
+  ["paren.lparen","{"],
+  ["text","@"],
+  ["support.function","tr"],
+  ["text"," "],
+  ["identifier","me"],
+  ["keyword.operator","/"],
+  ["identifier","ringloop"],
+  ["paren.rparen","}"],
+  ["text",";"],
+  ["paren.rparen","}"],
+  ["text",","],
+  ["constant.numeric","4"],
+  ["text",","],
+  ["paren.lparen","{"],
+  ["paren.rparen","}"]
+],[
+   "start",
+  ["text","&"],
+  ["identifier","ringstate"],
+  ["text"," "],
+  ["identifier","phone"],
+  ["keyword.operator","="],
+  ["paren.lparen","["],
+  ["support.function","setq"],
+  ["paren.lparen","("],
+  ["identifier","q"],
+  ["text",","],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","connecting"],
+  ["paren.rparen","))]"],
+  ["paren.lparen","["],
+  ["support.function","setq"],
+  ["paren.lparen","("],
+  ["constant.numeric","1"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","gt"],
+  ["paren.lparen","("],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["keyword.operator","/"],
+  ["identifier","rangs"],
+  ["paren.rparen",")"],
+  ["text",","],
+  ["support.function","sub"],
+  ["paren.lparen","("],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["keyword.operator","/"],
+  ["identifier","rings"],
+  ["paren.rparen",")"],
+  ["text",","],
+  ["constant.numeric","1"],
+  ["paren.rparen","))])]"],
+  ["paren.lparen","["],
+  ["support.function","setq"],
+  ["paren.lparen","("],
+  ["constant.numeric","2"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","and"],
+  ["paren.lparen","("],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","is"],
+  ["text",","],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["keyword.operator","/"],
+  ["identifier","MODE"],
+  ["paren.rparen",")"],
+  ["text",","],
+  ["identifier","CIP"],
+  ["paren.rparen",")"],
+  ["text",","],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","is"],
+  ["text",","],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["keyword.operator","/"],
+  ["identifier","INCOMING"],
+  ["paren.rparen",")"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","num"],
+  ["paren.lparen","("],
+  ["identifier","me"],
+  ["paren.rparen",")]))]"],
+  ["paren.lparen","["],
+  ["support.function","setq"],
+  ["paren.lparen","("],
+  ["constant.numeric","3"],
+  ["text",","],
+  ["paren.lparen","["],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["identifier","is"],
+  ["text",","],
+  ["support.function","u"],
+  ["paren.lparen","("],
+  ["variable","%qq"],
+  ["keyword.operator","/"],
+  ["identifier","MODE"],
+  ["paren.rparen",")"],
+  ["text",","],
+  ["identifier","ICC"],
+  ["paren.rparen",")])]"],
+  ["paren.lparen","["],
+  ["support.function","ifelse"],
+  ["paren.lparen","("],
+  ["variable","%q1"],
+  ["text",","],
+  ["constant.numeric","1"],
+  ["text",","],
+  ["support.function","ifelse"],
+  ["paren.lparen","("],
+  ["variable","%q2"],
+  ["text",","],
+  ["constant.numeric","2"],
+  ["text",","],
+  ["support.function","ifelse"],
+  ["paren.lparen","("],
+  ["variable","%q3"],
+  ["text",","],
+  ["constant.numeric","3"],
+  ["text",","],
+  ["constant.numeric","4"],
+  ["paren.rparen",")))]"]
+],[
+   "start",
+  ["text",";"],
+  ["identifier","comment"]
+],[
+   "start",
+  ["text","@@"],
+  ["paren.lparen","("],
+  ["identifier","comment"],
+  ["paren.rparen",")"]
+],[
+   "start",
+  ["keyword","say"],
+  ["text"," "],
+  ["paren.lparen","["],
+  ["support.function","time"],
+  ["paren.lparen","("],
+  ["paren.rparen",")]"]
+],[
+   "start"
+]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_properties.json b/lib/ace/mode/_test/tokens_properties.json
new file mode 100644
index 00000000..8831045e
--- /dev/null
+++ b/lib/ace/mode/_test/tokens_properties.json
@@ -0,0 +1,68 @@
+[[
+   "start",
+  ["comment","# You are reading the \".properties\" entry."]
+],[
+   "start",
+  ["comment","! The exclamation mark can also mark text as comments."]
+],[
+   "start",
+  ["comment","# The key and element characters #, !, =, and : are written with a preceding backslash to ensure that they are properly loaded."]
+],[
+   "start",
+  ["variable","website "],
+  ["keyword","="],
+  ["string"," http"],
+  ["constant.language.escape","\\"],
+  ["string","://en.wikipedia.org/"]
+],[
+   "start",
+  ["variable","language "],
+  ["keyword","="],
+  ["string"," English"]
+],[
+   "start",
+  ["comment","# The backslash below tells the application to continue reading"]
+],[
+   "start",
+  ["comment","# the value onto the next line."]
+],[
+   "value",
+  ["variable","message "],
+  ["keyword","="],
+  ["string"," Welcome to \\"]
+],[
+   "start",
+  ["string","          Wikipedia!"]
+],[
+   "start",
+  ["comment","# Add spaces to the key"]
+],[
+   "start",
+  ["variable","key"],
+  ["constant.language.escape","\\"],
+  ["variable"," with"],
+  ["constant.language.escape","\\"],
+  ["variable"," spaces "],
+  ["keyword","="],
+  ["string"," This is the value that could be looked up with the key \"key with spaces\"."]
+],[
+   "start",
+  ["comment","# Unicode"]
+],[
+   "start",
+  ["variable","tab "],
+  ["keyword",":"],
+  ["string"," "],
+  ["constant.language.escape","\\u0009"]
+],[
+   "start",
+  ["variable","empty-key"],
+  ["keyword","="]
+],[
+   "start",
+  ["variable","last.line"],
+  ["keyword","="],
+  ["string","value"]
+],[
+   "start"
+]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_sh.json b/lib/ace/mode/_test/tokens_sh.json
index 811f7db0..e4ab3412 100644
--- a/lib/ace/mode/_test/tokens_sh.json
+++ b/lib/ace/mode/_test/tokens_sh.json
@@ -314,7 +314,9 @@
   ["text","    "],
   ["support.function.builtin","echo"],
   ["text"," "],
-  ["string","\"http://github.com/$repo/pull/new/${branch##refs/heads/}\""]
+  ["string","\"http://github.com/"],
+  ["constant","$repo"],
+  ["string","/pull/new/${branch##refs/heads/}\""]
 ],[
    "start",
   ["keyword","else"]
diff --git a/lib/ace/mode/_test/tokens_snippets.json b/lib/ace/mode/_test/tokens_snippets.json
index f695403d..308683b7 100644
--- a/lib/ace/mode/_test/tokens_snippets.json
+++ b/lib/ace/mode/_test/tokens_snippets.json
@@ -1,14 +1,10 @@
 [[
    "start",
   ["comment","# Function"]
-],[
-   "start"
 ],[
    "start",
   ["constant.language.escape","snippet"],
   ["text"," fun"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\tfunction "],
@@ -23,8 +19,6 @@
   ["text","argument"],
   ["markup.list","}"],
   ["text",") {"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t\t"],
@@ -33,18 +27,12 @@
   ["punctuation.operator",":"],
   ["text","// body..."],
   ["markup.list","}"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t}"]
-],[
-   "start"
 ],[
    "start",
   ["comment","# Anonymous Function"]
-],[
-   "start"
 ],[
    "start",
   ["constant.language.escape","regex "],
@@ -55,14 +43,10 @@
   ["keyword","/"],
   ["text","(\\))?"],
   ["keyword","/"]
-],[
-   "start"
 ],[
    "start",
   ["constant.language.escape","name"],
   ["text"," f"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\tfunction"],
@@ -77,8 +61,6 @@
   ["text","("],
   ["variable","$2"],
   ["text",") {"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t\t"],
@@ -87,8 +69,6 @@
   ["punctuation.operator",":"],
   ["keyword","$TM_SELECTED_TEXT"],
   ["markup.list","}"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t}"],
@@ -102,31 +82,21 @@
   ["variable","M4"],
   ["text","?)"],
   ["markup.list","}"]
-],[
-   "start"
 ],[
    "start",
   ["comment","# Immediate function"]
-],[
-   "start"
 ],[
    "start",
   ["constant.language.escape","trigger"],
   ["text"," \\(?f\\("]
-],[
-   "start"
 ],[
    "start",
   ["constant.language.escape","endTrigger"],
   ["text"," \\)?"]
-],[
-   "start"
 ],[
    "start",
   ["constant.language.escape","snippet"],
   ["text"," f("]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t(function("],
@@ -134,8 +104,6 @@
   ["constant.numeric","1"],
   ["markup.list","}"],
   ["text",") {"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t\t"],
@@ -147,8 +115,6 @@
   ["punctuation.operator",":"],
   ["text","/* code */"],
   ["markup.list","}}"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t}("],
@@ -156,19 +122,13 @@
   ["constant.numeric","1"],
   ["markup.list","}"],
   ["text","));"]
-],[
-   "start"
 ],[
    "start",
   ["comment","# if"]
-],[
-   "start"
 ],[
    "start",
   ["constant.language.escape","snippet"],
   ["text"," if"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\tif ("],
@@ -178,31 +138,21 @@
   ["text","true"],
   ["markup.list","}"],
   ["text",") {"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t\t"],
   ["markup.list","${"],
   ["constant.numeric","0"],
   ["markup.list","}"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t}"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t"]
-],[
-   "start"
 ],[
    "sn-start",
   ["text","\t"]
diff --git a/lib/ace/mode/_test/tokens_twig.json b/lib/ace/mode/_test/tokens_twig.json
new file mode 100644
index 00000000..2d72b070
--- /dev/null
+++ b/lib/ace/mode/_test/tokens_twig.json
@@ -0,0 +1,82 @@
+[[
+   "start",
+  ["meta.tag.twig","{%"],
+  ["text"," "],
+  ["keyword.control.twig","autoescape"],
+  ["text"," "],
+  ["constant.language.boolean","true"],
+  ["text"," "],
+  ["meta.tag.twig","%}"]
+],[
+   "start",
+  ["text","    "],
+  ["variable.other.readwrite.local.twig","{{"],
+  ["text"," "],
+  ["identifier","var"],
+  ["text"," "],
+  ["variable.other.readwrite.local.twig","}}"]
+],[
+   "start",
+  ["text","    "],
+  ["variable.other.readwrite.local.twig","{{"],
+  ["text"," "],
+  ["identifier","var"],
+  ["keyword.operator.other","|"],
+  ["support.function.twig","raw"],
+  ["text"," "],
+  ["variable.other.readwrite.local.twig","}}"],
+  ["text","     "],
+  ["comment.block.twig","{# var won't be escaped #}"]
+],[
+   "start",
+  ["text","    "],
+  ["variable.other.readwrite.local.twig","{{"],
+  ["text"," "],
+  ["identifier","var"],
+  ["keyword.operator.other","|"],
+  ["support.function.twig","escape"],
+  ["text"," "],
+  ["variable.other.readwrite.local.twig","}}"],
+  ["text","  "],
+  ["comment.block.twig","{# var won't be doubled-escaped #}"]
+],[
+   "start",
+  ["meta.tag.twig","{%"],
+  ["text"," "],
+  ["keyword.control.twig","endautoescape"],
+  ["text"," "],
+  ["meta.tag.twig","%}"]
+],[
+   "start"
+],[
+   "start",
+  ["variable.other.readwrite.local.twig","{{"],
+  ["text"," "],
+  ["keyword.control.twig","include"],
+  ["paren.lparen","("],
+  ["string","'twig.html'"],
+  ["punctuation.operator",","],
+  ["text"," "],
+  ["identifier","sandboxed"],
+  ["text"," "],
+  ["keyword.operator.assignment","="],
+  ["text"," "],
+  ["constant.language.boolean","true"],
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["variable.other.readwrite.local.twig","}}"]
+],[
+   "start"
+],[
+   "start",
+  ["variable.other.readwrite.local.twig","{{"],
+  ["string","\"string "],
+  ["constant.language.escape","#{with}"],
+  ["string"," "],
+  ["constant.language.escape","\\\""],
+  ["string"," escapes\""],
+  ["text"," "],
+  ["string","'another#one'"],
+  ["text"," "],
+  ["variable.other.readwrite.local.twig","}}"]
+]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_velocity.json b/lib/ace/mode/_test/tokens_velocity.json
new file mode 100644
index 00000000..231b192b
--- /dev/null
+++ b/lib/ace/mode/_test/tokens_velocity.json
@@ -0,0 +1,281 @@
+[[
+   "comment",
+  ["comment.block","#*"]
+],[
+   "comment",
+  ["comment","  This is a sample comment block that"]
+],[
+   "comment",
+  ["comment","  spans multiple lines."]
+],[
+   "start",
+  ["comment","*#"]
+],[
+   "start"
+],[
+   "start",
+  ["keyword","#macro"],
+  ["text"," "],
+  ["lparen","("],
+  ["text"," "],
+  ["identifier","outputItem"],
+  ["text"," "],
+  ["variable","$item"],
+  ["text"," "],
+  ["rparen",")"]
+],[
+   "start",
+  ["text","  "],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","li"],
+  ["meta.tag.r",">"],
+  ["variable","${"],
+  ["identifier","item"],
+  ["variable","}"],
+  ["meta.tag","</"],
+  ["meta.tag.tag-name","li"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["keyword","#end"]
+],[
+   "start"
+],[
+   "start",
+  ["comment","## Define the items to iterate"]
+],[
+   "start",
+  ["keyword","#set"],
+  ["text"," "],
+  ["lparen","("],
+  ["text"," "],
+  ["variable","$items"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["lparen","["],
+  ["constant.numeric","1"],
+  ["text",", "],
+  ["constant.numeric","2"],
+  ["text",", "],
+  ["constant.numeric","3"],
+  ["text",", "],
+  ["constant.numeric","4"],
+  ["rparen","]"],
+  ["text"," "],
+  ["rparen",")"]
+],[
+   "start"
+],[
+   "start",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","ul"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","  "],
+  ["comment","## Iterate over the items and output the evens."]
+],[
+   "start",
+  ["text","  "],
+  ["keyword","#foreach"],
+  ["text"," "],
+  ["lparen","("],
+  ["text"," "],
+  ["variable","$item"],
+  ["text"," "],
+  ["identifier","in"],
+  ["text"," "],
+  ["variable","$items"],
+  ["text"," "],
+  ["rparen",")"]
+],[
+   "start",
+  ["text","    "],
+  ["keyword","#if"],
+  ["text"," "],
+  ["lparen","("],
+  ["text"," "],
+  ["support.function","$_MathTool"],
+  ["text","."],
+  ["identifier","mod"],
+  ["lparen","("],
+  ["variable","$item"],
+  ["text",", "],
+  ["constant.numeric","2"],
+  ["rparen",")"],
+  ["text"," "],
+  ["keyword.operator","=="],
+  ["text"," "],
+  ["constant.numeric","0"],
+  ["text"," "],
+  ["rparen",")"]
+],[
+   "start",
+  ["text","      "],
+  ["identifier","#outputItem"],
+  ["text"," "],
+  ["lparen","("],
+  ["variable","$item"],
+  ["rparen",")"]
+],[
+   "start",
+  ["text","    "],
+  ["keyword","#end"]
+],[
+   "start",
+  ["text","  "],
+  ["keyword","#end"]
+],[
+   "start",
+  ["meta.tag","</"],
+  ["meta.tag.tag-name","ul"],
+  ["meta.tag.r",">"]
+],[
+   "start"
+],[
+   "js-start",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","script"],
+  ["meta.tag.r",">"]
+],[
+   "js-comment_regex_allowed",
+  ["text","  "],
+  ["comment","/*"]
+],[
+   "js-comment_regex_allowed",
+  ["comment","    A sample function to decomstrate"]
+],[
+   "js-comment_regex_allowed",
+  ["comment","    JavaScript highlighting and folding."]
+],[
+   "js-start",
+  ["comment","  */"]
+],[
+   "js-start",
+  ["text","  "],
+  ["storage.type","function"],
+  ["text"," "],
+  ["entity.name.function","foo"],
+  ["paren.lparen","("],
+  ["variable.parameter","items"],
+  ["punctuation.operator",", "],
+  ["variable.parameter","nada"],
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["paren.lparen","{"]
+],[
+   "js-start",
+  ["text","    "],
+  ["keyword","for"],
+  ["text"," "],
+  ["paren.lparen","("],
+  ["storage.type","var"],
+  ["text"," "],
+  ["identifier","i"],
+  ["keyword.operator","="],
+  ["constant.numeric","0"],
+  ["punctuation.operator",";"],
+  ["text"," "],
+  ["identifier","i"],
+  ["keyword.operator","<"],
+  ["identifier","items"],
+  ["punctuation.operator","."],
+  ["support.constant","length"],
+  ["punctuation.operator",";"],
+  ["text"," "],
+  ["identifier","i"],
+  ["keyword.operator","++"],
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["paren.lparen","{"]
+],[
+   "js-start",
+  ["text","      "],
+  ["support.function","alert"],
+  ["paren.lparen","("],
+  ["identifier","items"],
+  ["paren.lparen","["],
+  ["identifier","i"],
+  ["paren.rparen","]"],
+  ["text"," "],
+  ["keyword.operator","+"],
+  ["text"," "],
+  ["string","\"juhu"],
+  ["constant.language.escape","\\n"],
+  ["string","\""],
+  ["paren.rparen",")"],
+  ["punctuation.operator",";"]
+],[
+   "js-no_regex",
+  ["text","    "],
+  ["paren.rparen","}"]
+],[
+   "js-no_regex",
+  ["text","  "],
+  ["paren.rparen","}"]
+],[
+   "start",
+  ["meta.tag","</"],
+  ["meta.tag.tag-name","script"],
+  ["meta.tag.r",">"]
+],[
+   "start"
+],[
+   "css-start",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","style"],
+  ["meta.tag.r",">"]
+],[
+   "css-comment",
+  ["text","  "],
+  ["comment","/*"]
+],[
+   "css-comment",
+  ["comment","    A sample style to decomstrate"]
+],[
+   "css-comment",
+  ["comment","    CSS highlighting and folding."]
+],[
+   "css-start",
+  ["comment","  */"]
+],[
+   "css-ruleset",
+  ["text","  "],
+  ["variable",".class"],
+  ["text"," "],
+  ["paren.lparen","{"]
+],[
+   "css-ruleset",
+  ["text","    "],
+  ["support.type","font-family"],
+  ["text",": Monaco, "],
+  ["string","\"Courier New\""],
+  ["text",", "],
+  ["support.constant.fonts","monospace"],
+  ["text",";"]
+],[
+   "css-ruleset",
+  ["text","    "],
+  ["support.type","font-size"],
+  ["text",": "],
+  ["constant.numeric","12"],
+  ["keyword","px"],
+  ["text",";"]
+],[
+   "css-ruleset",
+  ["text","    "],
+  ["support.type","cursor"],
+  ["text",": "],
+  ["support.constant","text"],
+  ["text",";"]
+],[
+   "css-start",
+  ["text","  "],
+  ["paren.rparen","}"]
+],[
+   "start",
+  ["meta.tag","</"],
+  ["meta.tag.tag-name","style"],
+  ["meta.tag.r",">"]
+]]
\ No newline at end of file

From f1e809294c78dc4f6c80dd14f5a696fa70c69d3b Mon Sep 17 00:00:00 2001
From: nightwing <amirjanyan@gmail.com>
Date: Wed, 26 Jun 2013 20:23:58 +0400
Subject: [PATCH 4/4] fix #1490 Parsing Lua comments in the middle of line

---
 demo/kitchen-sink/docs/lua.lua         |  2 ++
 lib/ace/mode/_test/tokens_lua.json     | 18 +++++++++++++-----
 lib/ace/mode/_test/tokens_luapage.json |  6 +++---
 lib/ace/mode/lua_highlight_rules.js    |  6 +++---
 4 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/demo/kitchen-sink/docs/lua.lua b/demo/kitchen-sink/docs/lua.lua
index 381b782c..6c927197 100644
--- a/demo/kitchen-sink/docs/lua.lua
+++ b/demo/kitchen-sink/docs/lua.lua
@@ -34,3 +34,5 @@ print([===[
 table.maxn is deprecated, use # instead.
 --]=]--
 print(table.maxn{1,2,[4]=4,[8]=8) -- outputs 8 instead of 2
+
+print(5 --[[ blah ]])
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_lua.json b/lib/ace/mode/_test/tokens_lua.json
index cc2676ea..276b3ffc 100644
--- a/lib/ace/mode/_test/tokens_lua.json
+++ b/lib/ace/mode/_test/tokens_lua.json
@@ -1,11 +1,11 @@
 [[
-   ["bracketedComment",4,"start"],
+   ["bracketedComment",2,"start"],
   ["comment","--[[--"]
 ],[
-   ["bracketedComment",4,"start"],
+   ["bracketedComment",2,"start"],
   ["comment","num_args takes in 5.1 byte code and extracts the number of arguments"]
 ],[
-   ["bracketedComment",4,"start"],
+   ["bracketedComment",2,"start"],
   ["comment","from its function header."]
 ],[
    "start",
@@ -301,10 +301,10 @@
 ],[
    "start"
 ],[
-   ["bracketedComment",5,"start"],
+   ["bracketedComment",3,"start"],
   ["comment","--[=[--"]
 ],[
-   ["bracketedComment",5,"start"],
+   ["bracketedComment",3,"start"],
   ["comment","table.maxn is deprecated, use # instead."]
 ],[
    "start",
@@ -337,4 +337,12 @@
   ["comment","-- outputs 8 instead of 2"]
 ],[
    "start"
+],[
+   "start",
+  ["support.function","print"],
+  ["paren.lparen","("],
+  ["constant.numeric","5"],
+  ["text"," "],
+  ["comment","--[[ blah ]]"],
+  ["paren.rparen",")"]
 ]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_luapage.json b/lib/ace/mode/_test/tokens_luapage.json
index d63f30e9..c9e51514 100644
--- a/lib/ace/mode/_test/tokens_luapage.json
+++ b/lib/ace/mode/_test/tokens_luapage.json
@@ -21,15 +21,15 @@
   ["meta.tag.tag-name","html"],
   ["meta.tag.r",">"]
 ],[
-   ["lua-bracketedComment",4,"lua-start"],
+   ["lua-bracketedComment",2,"lua-start"],
   ["keyword","<%"],
   ["text"," "],
   ["comment","--[[--"]
 ],[
-   ["lua-bracketedComment",4,"lua-start"],
+   ["lua-bracketedComment",2,"lua-start"],
   ["comment","    index.lp from the Kepler Project's LuaDoc HTML doclet."]
 ],[
-   ["lua-bracketedComment",4,"lua-start"],
+   ["lua-bracketedComment",2,"lua-start"],
   ["comment","    http://keplerproject.github.com/luadoc/"]
 ],[
    "start",
diff --git a/lib/ace/mode/lua_highlight_rules.js b/lib/ace/mode/lua_highlight_rules.js
index e9768d6a..5eb969d3 100644
--- a/lib/ace/mode/lua_highlight_rules.js
+++ b/lib/ace/mode/lua_highlight_rules.js
@@ -99,7 +99,7 @@ var LuaHighlightRules = function() {
         "start" : [{
             stateName: "bracketedComment",
             onMatch : function(value, currentState, stack){
-                stack.unshift(this.next, value.length, currentState);
+                stack.unshift(this.next, value.length - 2, currentState);
                 return "comment";
             },
             regex : /\-\-\[=*\[/,
@@ -115,7 +115,7 @@ var LuaHighlightRules = function() {
                         }
                         return "comment";
                     },
-                    regex : /(?:[^\\]|\\.)*?\]=*\]/,
+                    regex : /\]=*\]/,
                     next  : "start"
                 }, {
                     defaultToken : "comment"
@@ -147,7 +147,7 @@ var LuaHighlightRules = function() {
                         return "comment";
                     },
                     
-                    regex : /(?:[^\\]|\\.)*?\]=*\]/,
+                    regex : /\]=*\]/,
                     next  : "start"
                 }, {
                     defaultToken : "comment"