Merge pull request #1172 from ajaxorg/highlighting/js

Highlighting bugfix
2012-12-26 00:45:25 -08:00 · 2012-12-26 00:45:25 -08:00 · 06911396d6
commit 06911396d6
parent b7da773bec 8f6f6c6e3d
108 changed files with 7687 additions and 1129 deletions
--- a/.gitignore
+++ b/.gitignore
@ -16,7 +16,8 @@
 # A handy place to put stuff that git should ignore:
 /ignore/
 node_modules/
-tool/node_modules/
+jam/
+* *

 .git-ref
 npm-debug.log
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 9f433321a8cca9270753842c2517bd3588354dfe
+Subproject commit 97bae1d132effbeca5eef8aaef2a2a1fa836b181
--- a/demo/kitchen-sink/docs/abap.abap
+++ b/demo/kitchen-sink/docs/abap.abap
@ -0,0 +1,36 @@
+***************************************
+** Program: EXAMPLE                  **
+** Author: Joe Byte, 07-Jul-2007     **
+***************************************
+ 
+REPORT BOOKINGS.
+ 
+* Read flight bookings from the database
+SELECT * FROM FLIGHTINFO
+  WHERE CLASS = 'Y'       "Y = economy
+  OR    CLASS = 'C'.      "C = business
+(...)
+
+REPORT TEST.
+WRITE 'Hello World'.
+
+USERPROMPT = 'Please double-click on a line in the output list ' &
+             'to see the complete details of the transaction.'.
+
+
+DATA LAST_EOM    TYPE D.  "last end-of-month date
+ 
+* Start from today's date
+  LAST_EOM = SY-DATUM.
+* Set characters 6 and 7 (0-relative) of the YYYYMMDD string to "01",
+* giving the first day of the current month
+  LAST_EOM+6(2) = '01'.
+* Subtract one day
+  LAST_EOM = LAST_EOM - 1.
+ 
+  WRITE: 'Last day of previous month was', LAST_EOM.
+  
+DATA : BEGIN OF I_VBRK OCCURS 0,
+         VBELN LIKE VBRK-VBELN,
+         ZUONR LIKE VBRK-ZUONR,
+       END OF I_VBRK.
--- a/lib/ace/mode/_test/text_asciidoc.txt
+++ b/lib/ace/mode/_test/text_asciidoc.txt
@ -0,0 +1,111 @@
+------------------------------------
+-----------------------------------
+_ita_lic_, *bo*ld*, +mo+no+, normal. 
+``double quoted'', `single quoted'.
+  normal, ^super^, ~sub~.
+
+''''
+Escaped:
+\_italic_, +++_italic_+++,
+t\__e__st, +++t__e__st+++,
+++<b>bold</b>+++, $$<b>normal</b>$$
+\&#182; &#182;
+\`not single quoted'
+\`\`not double quoted''
+
+
+[fffff]
+++++++++++++++++++++++++++++++++++++
+(C) {ss} ss
+++++++++++++++++++++++++++++++++++++
+
+.............
+callout <1>
+..............
+
+> 1
+1> 2
+<2> 3
+
+Escaped:
+\_italic_,
+t\__e__st, o__
+
+.optional title
+.............
+callout <1>
+..............
+
+
+:macro: dddd
+
+.lists
+. 1
+.. 2
+... d
+..... big
+
+continue +
+next line
+xi) no ++la+tin++
+
+xi) la++tin++
+2. num__ber__  [red]#red#
+--
+  5. f <<x6,ssss>> {macro}
+--
+image::path[beauty] ->--<= -- replacements
+
+ image::s
+sssss
+sss 
+sssss
+
+
+== 1
+  heading
+=== not a heading
+====================================
+
+==================================
+====4 
+NOTE: above
+
+NOTE: above
+
+[[x6]]
+WARNING: 
+
+[options[]]
+---------------------------
+literal
+---------------------------
+
+
+= Tables
+|====================================
+| _italic_ |  *bold* | text | (R)
+
+|====================================
+
+
+[more, options]
+///////////
+comment
+///////////
+// one line comment
+
+
+
+[red]#red text# [yellow-background]#on yellow#
+[big]#large# [red yellow-background big]*all bold*
+
+	
+https://site text <mail@i.am> callto:ace http://ace.ajaxorg.com[awesome]
+  .still normal text
+.break out thoug should not
+---------------------------
+///////////////////////////
+---------------------------
+
+
--- a/lib/ace/mode/_test/tokens_abap.json
+++ b/lib/ace/mode/_test/tokens_abap.json
@ -0,0 +1,294 @@
+[
+ {
+  "state": "start",
+  "data": [
+   [ "doc.comment", "***************************************" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "doc.comment", "** Program: EXAMPLE                  **" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "doc.comment", "** Author: Joe Byte, 07-Jul-2007     **" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "doc.comment", "***************************************" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "REPORT" ],
+   [ "text", " BOOKINGS" ],
+   [ "keyword.operator", "." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "doc.comment", "* Read flight bookings from the database" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "SELECT" ],
+   [ "keyword.operator", " * " ],
+   [ "keyword", "FROM" ],
+   [ "text", " FLIGHTINFO" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "keyword", "WHERE" ],
+   [ "text", " " ],
+   [ "keyword", "CLASS" ],
+   [ "keyword.operator", " = " ],
+   [ "string", "'Y'" ],
+   [ "text", "       " ],
+   [ "comment", "\"Y = economy" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "keyword", "OR" ],
+   [ "text", "    " ],
+   [ "keyword", "CLASS" ],
+   [ "keyword.operator", " = " ],
+   [ "string", "'C'" ],
+   [ "keyword.operator", "." ],
+   [ "text", "      " ],
+   [ "comment", "\"C = business" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "paren.lparen", "(" ],
+   [ "invalid", "..." ],
+   [ "paren.rparen", ")" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "REPORT" ],
+   [ "text", " TEST" ],
+   [ "keyword.operator", "." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "WRITE" ],
+   [ "text", " " ],
+   [ "string", "'Hello World'" ],
+   [ "keyword.operator", "." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "USERPROMPT" ],
+   [ "keyword.operator", " = " ],
+   [ "string", "'Please double-click on a line in the output list '" ],
+   [ "text", " " ],
+   [ "keyword.operator", "&" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "             " ],
+   [ "string", "'to see the complete details of the transaction.'" ],
+   [ "keyword.operator", "." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "DATA" ],
+   [ "text", " LAST_EOM    " ],
+   [ "keyword", "TYPE" ],
+   [ "text", " " ],
+   [ "support.type", "D" ],
+   [ "keyword.operator", "." ],
+   [ "text", "  " ],
+   [ "comment", "\"last end-of-month date" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "doc.comment", "* Start from today's date" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  LAST_EOM" ],
+   [ "keyword.operator", " = " ],
+   [ "variable.parameter", "SY" ],
+   [ "keyword.operator", "-" ],
+   [ "text", "DATUM" ],
+   [ "keyword.operator", "." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "doc.comment", "* Set characters 6 and 7 (0-relative) of the YYYYMMDD string to \"01\"," ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "doc.comment", "* giving the first day of the current month" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  LAST_EOM" ],
+   [ "constant.numeric", "+6" ],
+   [ "paren.lparen", "(" ],
+   [ "constant.numeric", "2" ],
+   [ "paren.rparen", ")" ],
+   [ "keyword.operator", " = " ],
+   [ "string", "'01'" ],
+   [ "keyword.operator", "." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "doc.comment", "* Subtract one day" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  LAST_EOM" ],
+   [ "keyword.operator", " = " ],
+   [ "text", "LAST_EOM" ],
+   [ "keyword.operator", " - " ],
+   [ "constant.numeric", "1" ],
+   [ "keyword.operator", "." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "keyword", "WRITE" ],
+   [ "keyword.operator", ":" ],
+   [ "text", " " ],
+   [ "string", "'Last day of previous month was'" ],
+   [ "keyword.operator", "," ],
+   [ "text", " LAST_EOM" ],
+   [ "keyword.operator", "." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "DATA" ],
+   [ "text", " " ],
+   [ "keyword.operator", ":" ],
+   [ "text", " " ],
+   [ "keyword", "BEGIN OF" ],
+   [ "text", " I_VBRK " ],
+   [ "keyword", "OCCURS" ],
+   [ "text", " " ],
+   [ "constant.numeric", "0" ],
+   [ "keyword.operator", "," ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "         VBELN " ],
+   [ "keyword", "LIKE" ],
+   [ "text", " " ],
+   [ "variable.parameter", "VBRK-VBELN" ],
+   [ "keyword.operator", "," ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "         ZUONR " ],
+   [ "keyword", "LIKE" ],
+   [ "text", " " ],
+   [ "variable.parameter", "VBRK-ZUONR" ],
+   [ "keyword.operator", "," ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "       " ],
+   [ "keyword", "END OF" ],
+   [ "text", " I_VBRK" ],
+   [ "keyword.operator", "." ]
+  ]
+ }
+]
--- a/lib/ace/mode/_test/tokens_asciidoc.json
+++ b/lib/ace/mode/_test/tokens_asciidoc.json
@ -0,0 +1,729 @@
+[
+ {
+  "state": "literalBlock",
+  "data": [
+   [ "support.function", "------------------------------------" ]
+  ]
+ },
+ {
+  "state": "dissallowDelimitedBlock",
+  "data": [
+   [ "support.function", "-----------------------------------" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "string.italic", "_ita_lic_" ],
+   [ "text", ", " ],
+   [ "keyword.bold", "*bo*ld*" ],
+   [ "text", ", " ],
+   [ "support.function", "+mo+no+" ],
+   [ "text", ", normal. " ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "keyword", "``" ],
+   [ "text", "double quoted" ],
+   [ "keyword", "''" ],
+   [ "text", ", `single quoted'." ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "  normal, ^super^, " ],
+   [ "keyword", "~sub~" ],
+   [ "text", "." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "''''" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "Escaped:" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "constant.language.escape", "\\_" ],
+   [ "text", "italic_, " ],
+   [ "support.function", "+++" ],
+   [ "text", "_italic_" ],
+   [ "support.function", "+++" ],
+   [ "text", "," ]
+  ]
+ },
+ {
+  "state": "smallPassthrough",
+  "data": [
+   [ "text", "t" ],
+   [ "constant.language.escape", "\\_" ],
+   [ "text", "_e" ],
+   [ "string.italic", "__st, +++t__" ],
+   [ "text", "e__st" ],
+   [ "support.function", "+++" ],
+   [ "text", "," ]
+  ]
+ },
+ {
+  "state": "smallPassthrough",
+  "data": [
+   [ "support.function", "+++" ],
+   [ "text", "<b>bold</b>" ],
+   [ "support.function", "+++" ],
+   [ "text", ", $$<" ],
+   [ "support.function", "b" ],
+   [ "text", ">" ],
+   [ "support.function", "normal" ],
+   [ "text", "</" ],
+   [ "support.function", "b" ],
+   [ "text", ">$$" ]
+  ]
+ },
+ {
+  "state": "smallPassthrough",
+  "data": [
+   [ "text", "\\&#" ],
+   [ "support.function", "182" ],
+   [ "text", "; &#" ],
+   [ "support.function", "182" ],
+   [ "text", ";" ]
+  ]
+ },
+ {
+  "state": "smallPassthrough",
+  "data": [
+   [ "text", "\\`" ],
+   [ "support.function", "not" ],
+   [ "text", " " ],
+   [ "support.function", "single" ],
+   [ "text", " " ],
+   [ "support.function", "quoted" ],
+   [ "text", "'" ]
+  ]
+ },
+ {
+  "state": "smallPassthrough",
+  "data": [
+   [ "text", "\\`\\`" ],
+   [ "support.function", "not" ],
+   [ "text", " " ],
+   [ "support.function", "double" ],
+   [ "text", " " ],
+   [ "support.function", "quoted" ],
+   [ "text", "''" ]
+  ]
+ },
+ {
+  "state": "dissallowDelimitedBlock",
+  "data": []
+ },
+ {
+  "state": "text",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "string.regexp", "[fffff]" ]
+  ]
+ },
+ {
+  "state": "passthroughBlock",
+  "data": [
+   [ "string", "+++++++++++++++++++++++++++++++++++++" ]
+  ]
+ },
+ {
+  "state": "passthroughBlock",
+  "data": [
+   [ "support.function", "(C) " ],
+   [ "constant.character", "{ss}" ],
+   [ "support.function", " ss" ]
+  ]
+ },
+ {
+  "state": "dissallowDelimitedBlock",
+  "data": [
+   [ "support.function", "+++++++++++++++++++++++++++++++++++++" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": []
+ },
+ {
+  "state": "listingBlock",
+  "data": [
+   [ "support.function", "............." ]
+  ]
+ },
+ {
+  "state": "listingBlock",
+  "data": [
+   [ "support.function", "callout " ],
+   [ "constant.numeric", "<1>" ]
+  ]
+ },
+ {
+  "state": "dissallowDelimitedBlock",
+  "data": [
+   [ "support.function", ".............." ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": []
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "> 1" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "1> 2" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "<2> 3" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "Escaped:" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "constant.language.escape", "\\_" ],
+   [ "text", "italic_," ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "t" ],
+   [ "constant.language.escape", "\\_" ],
+   [ "text", "_e" ],
+   [ "string.italic", "__st, o__" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.numeric", ".optional title" ]
+  ]
+ },
+ {
+  "state": "listingBlock",
+  "data": [
+   [ "support.function", "............." ]
+  ]
+ },
+ {
+  "state": "listingBlock",
+  "data": [
+   [ "support.function", "callout " ],
+   [ "constant.numeric", "<1>" ]
+  ]
+ },
+ {
+  "state": "dissallowDelimitedBlock",
+  "data": [
+   [ "support.function", ".............." ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "keyword", ":macro:" ],
+   [ "text", " dddd" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.numeric", ".lists" ]
+  ]
+ },
+ {
+  "state": "listText",
+  "data": [
+   [ "keyword", ". " ],
+   [ "text", "1" ]
+  ]
+ },
+ {
+  "state": "listText",
+  "data": [
+   [ "keyword", ".. " ],
+   [ "text", "2" ]
+  ]
+ },
+ {
+  "state": "listText",
+  "data": [
+   [ "keyword", "... " ],
+   [ "text", "d" ]
+  ]
+ },
+ {
+  "state": "listText",
+  "data": [
+   [ "keyword", "..... " ],
+   [ "text", "big" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "+" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "continue" ],
+   [ "keyword", " +" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "next line" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "xi) no " ],
+   [ "support.function", "++la+tin++" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "listText",
+  "data": [
+   [ "keyword", "xi) " ],
+   [ "text", "la" ],
+   [ "support.function", "++tin++" ]
+  ]
+ },
+ {
+  "state": "listText",
+  "data": [
+   [ "keyword", "2. " ],
+   [ "text", "num" ],
+   [ "string.italic", "__ber__" ],
+   [ "text", "  [red]" ],
+   [ "keyword", "#" ],
+   [ "text", "red" ],
+   [ "keyword", "#" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "--" ]
+  ]
+ },
+ {
+  "state": "listText",
+  "data": [
+   [ "keyword", "  5. " ],
+   [ "text", "f " ],
+   [ "keyword", "<<x6," ],
+   [ "string", "ssss" ],
+   [ "keyword", ">>" ],
+   [ "text", " " ],
+   [ "constant.character", "{macro}" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "--" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "markup.list.macro", "image::" ],
+   [ "keyword", "path" ],
+   [ "string", "[beauty]" ],
+   [ "text", " " ],
+   [ "constant.language.escape", "->" ],
+   [ "text", "--" ],
+   [ "constant.language.escape", "<= --" ],
+   [ "text", " replacements" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "indentedBlock",
+  "data": [
+   [ "support.function", " image::s" ]
+  ]
+ },
+ {
+  "state": "indentedBlock",
+  "data": [
+   [ "support.function", "sssss" ]
+  ]
+ },
+ {
+  "state": "indentedBlock",
+  "data": [
+   [ "support.function", "sss " ]
+  ]
+ },
+ {
+  "state": "indentedBlock",
+  "data": [
+   [ "support.function", "sssss" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "markup.heading", "== 1" ]
+  ]
+ },
+ {
+  "state": "indentedBlock",
+  "data": [
+   [ "support.function", "  heading" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "markup.heading", "=== not a heading" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "====================================" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "==================================" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "====4 " ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "NOTE: above" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "keyword", "NOTE:" ],
+   [ "text", " above" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "string.regexp", "[[x6]]" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "keyword", "WARNING:" ],
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "string.regexp", "[options[]]" ]
+  ]
+ },
+ {
+  "state": "literalBlock",
+  "data": [
+   [ "support.function", "---------------------------" ]
+  ]
+ },
+ {
+  "state": "literalBlock",
+  "data": [
+   [ "support.function", "literal" ]
+  ]
+ },
+ {
+  "state": "dissallowDelimitedBlock",
+  "data": [
+   [ "support.function", "---------------------------" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "markup.heading", "= Tables" ]
+  ]
+ },
+ {
+  "state": "tableBlock",
+  "data": [
+   [ "doc.comment", "|====================================" ]
+  ]
+ },
+ {
+  "state": "tableBlock",
+  "data": [
+   [ "doc.comment", "|" ],
+   [ "text", " " ],
+   [ "string.italic", "_italic_" ],
+   [ "text", " " ],
+   [ "doc.comment", "|" ],
+   [ "text", "  " ],
+   [ "keyword.bold", "*bold*" ],
+   [ "text", " " ],
+   [ "doc.comment", "|" ],
+   [ "text", " text " ],
+   [ "doc.comment", "|" ],
+   [ "text", " " ],
+   [ "constant.language.escape", "(R)" ]
+  ]
+ },
+ {
+  "state": "tableBlock",
+  "data": []
+ },
+ {
+  "state": "dissallowDelimitedBlock",
+  "data": [
+   [ "doc.comment", "|====================================" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "string.regexp", "[more, options]" ]
+  ]
+ },
+ {
+  "state": "commentBlock",
+  "data": [
+   [ "doc.comment", "///////////" ]
+  ]
+ },
+ {
+  "state": "commentBlock",
+  "data": [
+   [ "doc.comment", "comment" ]
+  ]
+ },
+ {
+  "state": "dissallowDelimitedBlock",
+  "data": [
+   [ "doc.comment", "///////////" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "comment", "// one line comment" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "[red]" ],
+   [ "keyword", "#" ],
+   [ "text", "red text" ],
+   [ "keyword", "#" ],
+   [ "text", " [yellow-background]" ],
+   [ "keyword", "#" ],
+   [ "text", "on yellow" ],
+   [ "keyword", "#" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "[big]" ],
+   [ "keyword", "#" ],
+   [ "text", "large" ],
+   [ "keyword", "#" ],
+   [ "text", " [red yellow-background big]" ],
+   [ "keyword.bold", "*all bold*" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "\t" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "markup.underline.list", "https://site" ],
+   [ "text", " text <" ],
+   [ "markup.underline.list", "mail@i.am" ],
+   [ "text", "> " ],
+   [ "markup.underline.list", "callto:ace" ],
+   [ "text", " " ],
+   [ "link", "http://ace.ajaxorg.com" ],
+   [ "variable.language", "[awesome]" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": [
+   [ "text", "  .still normal text" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.numeric", ".break out thoug should not" ]
+  ]
+ },
+ {
+  "state": "literalBlock",
+  "data": [
+   [ "support.function", "---------------------------" ]
+  ]
+ },
+ {
+  "state": "literalBlock",
+  "data": [
+   [ "support.function", "///////////////////////////" ]
+  ]
+ },
+ {
+  "state": "dissallowDelimitedBlock",
+  "data": [
+   [ "support.function", "---------------------------" ]
+  ]
+ },
+ {
+  "state": "text",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ }
+]
--- a/lib/ace/mode/_test/tokens_c9search.json
+++ b/lib/ace/mode/_test/tokens_c9search.json
@ -20,104 +20,91 @@
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "    1" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var fs = require(\"fs\");" ]
+   [ "c9searchresults.text", ": var fs = require(\"fs\");" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t2" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var argv = require('optimist').argv;" ]
+   [ "c9searchresults.text", ": var argv = require('optimist').argv;" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t3" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var path = require(\"path\");" ]
+   [ "c9searchresults.text", ": var path = require(\"path\");" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t5" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var clientExtensions = {};" ]
+   [ "c9searchresults.text", ": var clientExtensions = {};" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t6" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var clientDirs = fs.readdirSync(__dirname + \"/../plugins-client\");" ]
+   [ "c9searchresults.text", ": var clientDirs = fs.readdirSync(__dirname + \"/../plugins-client\");" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t7" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "for (var i = 0; i < clientDirs.length; i++) {" ]
+   [ "c9searchresults.text", ": for (var i = 0; i < clientDirs.length; i++) {" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t8" ],
-   [ "c9searchresults.text", ":     " ],
-   [ "c9searchresults.text", "var dir = clientDirs[i];" ]
+   [ "c9searchresults.text", ":     var dir = clientDirs[i];" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t12" ],
-   [ "c9searchresults.text", ":     " ],
-   [ "c9searchresults.text", "var name = dir.split(\".\")[1];" ]
+   [ "c9searchresults.text", ":     var name = dir.split(\".\")[1];" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t16" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var projectDir = (argv.w && path.resolve(process.cwd(), argv.w)) || process.cwd();" ]
+   [ "c9searchresults.text", ": var projectDir = (argv.w && path.resolve(process.cwd(), argv.w)) || process.cwd();" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t17" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var fsUrl = \"/workspace\";" ]
+   [ "c9searchresults.text", ": var fsUrl = \"/workspace\";" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t19" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var port = argv.p || process.env.PORT || 3131;" ]
+   [ "c9searchresults.text", ": var port = argv.p || process.env.PORT || 3131;" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t20" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var host = argv.l || \"localhost\";" ]
+   [ "c9searchresults.text", ": var host = argv.l || \"localhost\";" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t22" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var config = {" ]
+   [ "c9searchresults.text", ": var config = {" ]
  ]
 },
 {
@ -135,8 +122,7 @@
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t2" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var config = require(\"./default\");" ]
+   [ "c9searchresults.text", ": var config = require(\"./default\");" ]
  ]
 },
 {
@ -154,8 +140,7 @@
  "state": "start",
  "data": [
   [ "c9searchresults.constant.numeric", "\t1" ],
-   [ "c9searchresults.text", ": " ],
-   [ "c9searchresults.text", "var config = require(\"./default\");" ]
+   [ "c9searchresults.text", ": var config = require(\"./default\");" ]
  ]
 },
 {
--- a/lib/ace/mode/_test/tokens_c_cpp.json
+++ b/lib/ace/mode/_test/tokens_c_cpp.json
@ -13,10 +13,94 @@
  "state": "start",
  "data": [
   [ "keyword", "#include" ],
-   [ "text", " " ],
-   [ "constant", "<iostream>" ]
+   [ "constant.other", " <iostream>" ]
  ]
 },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "directive",
+  "data": [
+   [ "keyword", "#include" ],
+   [ "constant.other.multiline", " \\" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.other", "   <iostream>" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "directive",
+  "data": [
+   [ "keyword", "#include" ],
+   [ "constant.other.multiline", " \\" ]
+  ]
+ },
+ {
+  "state": "directive",
+  "data": [
+   [ "constant.other.multiline", "   \\" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.other", "   <iostream>" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "directive",
+  "data": [
+   [ "keyword", "#include" ],
+   [ "constant.other.multiline", " \\" ]
+  ]
+ },
+ {
+  "state": "directive",
+  "data": [
+   [ "constant.other.multiline", "   \\" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.other", "   \"iostream\"" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "#include" ],
+   [ "constant.other", " <boost/asio/io_service.hpp>" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword", "#include" ],
+   [ "constant.other", " \"boost/asio/io_service.hpp\"" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
 {
  "state": "start",
  "data": [
@ -37,7 +121,7 @@
  "data": [
   [ "storage.type", "int" ],
   [ "text", " " ],
-   [ "entity.name.function", "main" ],
+   [ "identifier", "main" ],
   [ "text", " " ],
   [ "paren.lparen", "(" ],
   [ "paren.rparen", ")" ]
@ -63,8 +147,7 @@
   [ "constant.numeric", "3" ],
   [ "punctuation.operator", ";" ],
   [ "text", " " ],
-   [ "comment", "/*" ],
-   [ "comment", " foobar */" ]
+   [ "comment", "/* foobar */" ]
  ]
 },
 {
@ -84,8 +167,7 @@
  "data": [
   [ "text", "    " ],
   [ "identifier", "a" ],
-   [ "keyword.operator", "+" ],
-   [ "keyword.operator", "=" ],
+   [ "keyword.operator", "+=" ],
   [ "constant.numeric", "2" ],
   [ "punctuation.operator", ";" ],
   [ "text", " " ],
@ -98,13 +180,38 @@
   [ "text", "    " ],
   [ "identifier", "cout" ],
   [ "text", " " ],
-   [ "keyword.operator", "<" ],
-   [ "keyword.operator", "<" ],
+   [ "keyword.operator", "<<" ],
   [ "text", " " ],
   [ "identifier", "a" ],
   [ "punctuation.operator", ";" ]
  ]
 },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "    " ],
+   [ "keyword", "#if" ],
+   [ "constant.other", " VERBOSE >= 2" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "        " ],
+   [ "identifier", "prints" ],
+   [ "paren.lparen", "(" ],
+   [ "string", "\"trace message\"" ],
+   [ "paren.rparen", ")" ],
+   [ "punctuation.operator", ";" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "    " ],
+   [ "keyword", "#endif" ]
+  ]
+ },
 {
  "state": "start",
  "data": [
@ -120,5 +227,47 @@
  "data": [
   [ "paren.rparen", "}" ]
  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "comment", "/* Print an error message and get out */" ]
+  ]
+ },
+ {
+  "state": "directive",
+  "data": [
+   [ "keyword", "#define" ],
+   [ "constant.other.multiline", " ABORT                             \\" ]
+  ]
+ },
+ {
+  "state": "directive",
+  "data": [
+   [ "constant.other.multiline", "    do {                                  \\" ]
+  ]
+ },
+ {
+  "state": "directive",
+  "data": [
+   [ "constant.other.multiline", "        print( \"Abort\\n\" );                \\" ]
+  ]
+ },
+ {
+  "state": "directive",
+  "data": [
+   [ "constant.other.multiline", "        exit(8);                          \\" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.other", "} while (0)                      " ],
+   [ "comment", "/* Note: No semicolon */" ]
+  ]
 }
 ]
--- a/lib/ace/mode/_test/tokens_clojure.json
+++ b/lib/ace/mode/_test/tokens_clojure.json
@ -12,33 +12,27 @@
  "state": "start",
  "data": [
   [ "text", "  " ],
-   [ "string", "\"returns a String parting in a given language" ],
-   [ "string", "\"" ]
+   [ "string", "\"returns a String parting in a given language\"" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "text", "  " ],
-   [ "keyword", "(" ],
-   [ "keyword", "[" ],
-   [ "keyword", "]" ],
+   [ "keyword", "([]" ],
   [ "text", " " ],
   [ "keyword", "(" ],
   [ "identifier", "parting" ],
   [ "text", " " ],
-   [ "string", "\"World" ],
-   [ "string", "\"" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ]
+   [ "string", "\"World\"" ],
+   [ "keyword", "))" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "text", "  " ],
-   [ "keyword", "(" ],
-   [ "keyword", "[" ],
+   [ "keyword", "([" ],
   [ "support.function", "name" ],
   [ "keyword", "]" ],
   [ "text", " " ],
@ -47,18 +41,15 @@
   [ "text", " " ],
   [ "support.function", "name" ],
   [ "text", " " ],
-   [ "string", "\"en" ],
-   [ "string", "\"" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ]
+   [ "string", "\"en\"" ],
+   [ "keyword", "))" ]
  ]
 },
 {
  "state": "start",
  "data": [
   [ "text", "  " ],
-   [ "keyword", "(" ],
-   [ "keyword", "[" ],
+   [ "keyword", "([" ],
   [ "support.function", "name" ],
   [ "text", " " ],
   [ "identifier", "language" ],
@ -109,14 +100,12 @@
  "state": "start",
  "data": [
   [ "text", "      " ],
-   [ "string", "\"en" ],
-   [ "string", "\"" ],
+   [ "string", "\"en\"" ],
   [ "text", " " ],
   [ "keyword", "(" ],
   [ "support.function", "str" ],
   [ "text", " " ],
-   [ "string", "\"Goodbye, " ],
-   [ "string", "\"" ],
+   [ "string", "\"Goodbye, \"" ],
   [ "text", " " ],
   [ "support.function", "name" ],
   [ "keyword", ")" ]
@ -126,14 +115,12 @@
  "state": "start",
  "data": [
   [ "text", "      " ],
-   [ "string", "\"es" ],
-   [ "string", "\"" ],
+   [ "string", "\"es\"" ],
   [ "text", " " ],
   [ "keyword", "(" ],
   [ "support.function", "str" ],
   [ "text", " " ],
-   [ "string", "\"Adios, " ],
-   [ "string", "\"" ],
+   [ "string", "\"Adios, \"" ],
   [ "text", " " ],
   [ "support.function", "name" ],
   [ "keyword", ")" ]
@ -143,8 +130,7 @@
  "state": "start",
  "data": [
   [ "text", "      " ],
-   [ "keyword", "(" ],
-   [ "keyword", "throw" ],
+   [ "keyword", "(throw" ],
   [ "text", " " ],
   [ "keyword", "(" ],
   [ "identifier", "IllegalArgumentException" ],
@ -158,16 +144,10 @@
   [ "keyword", "(" ],
   [ "support.function", "str" ],
   [ "text", " " ],
-   [ "string", "\"unsupported language " ],
-   [ "string", "\"" ],
+   [ "string", "\"unsupported language \"" ],
   [ "text", " " ],
   [ "identifier", "language" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ]
+   [ "keyword", "))))))" ]
  ]
 },
 {
@ -182,8 +162,7 @@
   [ "text", " " ],
   [ "keyword", "(" ],
   [ "identifier", "parting" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ],
+   [ "keyword", "))" ],
   [ "text", " " ],
   [ "comment", "; -> Goodbye, World" ]
  ]
@ -197,10 +176,8 @@
   [ "keyword", "(" ],
   [ "identifier", "parting" ],
   [ "text", " " ],
-   [ "string", "\"Mark" ],
-   [ "string", "\"" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ],
+   [ "string", "\"Mark\"" ],
+   [ "keyword", "))" ],
   [ "text", " " ],
   [ "comment", "; -> Goodbye, Mark" ]
  ]
@ -214,13 +191,10 @@
   [ "keyword", "(" ],
   [ "identifier", "parting" ],
   [ "text", " " ],
-   [ "string", "\"Mark" ],
-   [ "string", "\"" ],
+   [ "string", "\"Mark\"" ],
   [ "text", " " ],
-   [ "string", "\"es" ],
-   [ "string", "\"" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ],
+   [ "string", "\"es\"" ],
+   [ "keyword", "))" ],
   [ "text", " " ],
   [ "comment", "; -> Adios, Mark" ]
  ]
@ -234,13 +208,10 @@
   [ "keyword", "(" ],
   [ "identifier", "parting" ],
   [ "text", " " ],
-   [ "string", "\"Mark" ],
-   [ "string", "\"" ],
+   [ "string", "\"Mark\"" ],
   [ "text", ", " ],
-   [ "string", "\"xy" ],
-   [ "string", "\"" ],
-   [ "keyword", ")" ],
-   [ "keyword", ")" ],
+   [ "string", "\"xy\"" ],
+   [ "keyword", "))" ],
   [ "text", " " ],
   [ "comment", "; -> java.lang.IllegalArgumentException: unsupported language xy" ]
  ]
--- a/lib/ace/mode/_test/tokens_coffee.json
+++ b/lib/ace/mode/_test/tokens_coffee.json
@ -50,7 +50,8 @@
  "data": [
   [ "text", "    " ],
   [ "identifier", "console" ],
-   [ "identifier", ".log" ],
+   [ "punctuation.operator", "." ],
+   [ "identifier", "log" ],
   [ "text", " " ],
   [ "string", "'qstring'" ],
   [ "text", " " ],
@ -100,7 +101,7 @@
  "data": [
   [ "keyword", "do" ],
   [ "text", " " ],
-   [ "keyword.operator", "->" ]
+   [ "storage.type", "->" ]
  ]
 },
 {
@ -131,7 +132,8 @@
   [ "keyword.operator", "=" ],
   [ "text", " " ],
   [ "string.regex", "/regex/imgy" ],
-   [ "identifier", ".test" ],
+   [ "punctuation.operator", "." ],
+   [ "identifier", "test" ],
   [ "text", " " ],
   [ "string.regex", "///" ]
  ]
--- a/lib/ace/mode/_test/tokens_coldfusion.json
+++ b/lib/ace/mode/_test/tokens_coldfusion.json
@ -2,8 +2,7 @@
 {
  "state": "start",
  "data": [
-   [ "comment", "<!--" ],
-   [ "comment", "- hello world --->" ]
+   [ "comment", "<!--- hello world --->" ]
  ]
 },
 {
@ -19,7 +18,7 @@
   [ "entity.other.attribute-name", "welcome" ],
   [ "keyword.operator", "=" ],
   [ "string", "\"Hello World!\"" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 },
 {
@ -31,11 +30,11 @@
  "data": [
   [ "meta.tag", "<" ],
   [ "meta.tag.tag-name", "cfoutput" ],
-   [ "meta.tag", ">" ],
+   [ "meta.tag.r", ">" ],
   [ "text", "#welcome#" ],
   [ "meta.tag", "</" ],
   [ "meta.tag.tag-name", "cfoutput" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 }
 ]
--- a/lib/ace/mode/_test/tokens_dart.json
+++ b/lib/ace/mode/_test/tokens_dart.json
@ -0,0 +1,537 @@
+[
+ {
+  "state": "start",
+  "data": [
+   [ "identifier", "main" ],
+   [ "text", "() {" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "identifier", "print" ],
+   [ "text", "(" ],
+   [ "string", "'Hello World!'" ],
+   [ "text", ");" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "}" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "storage.type.primitive.dart", "int" ],
+   [ "text", " " ],
+   [ "identifier", "fib" ],
+   [ "text", "(" ],
+   [ "storage.type.primitive.dart", "int" ],
+   [ "text", " " ],
+   [ "identifier", "n" ],
+   [ "text", ") " ],
+   [ "keyword.operator.assignment.dart", "=" ],
+   [ "keyword.operator.comparison.dart", ">" ],
+   [ "text", " (" ],
+   [ "identifier", "n" ],
+   [ "text", " " ],
+   [ "keyword.operator.comparison.dart", ">" ],
+   [ "text", " " ],
+   [ "constant.numeric", "1" ],
+   [ "text", ") " ],
+   [ "keyword.control.ternary.dart", "?" ],
+   [ "text", " (" ],
+   [ "identifier", "fib" ],
+   [ "text", "(" ],
+   [ "identifier", "n" ],
+   [ "text", " " ],
+   [ "keyword.operator.arithmetic.dart", "-" ],
+   [ "text", " " ],
+   [ "constant.numeric", "1" ],
+   [ "text", ") " ],
+   [ "keyword.operator.arithmetic.dart", "+" ],
+   [ "text", " " ],
+   [ "identifier", "fib" ],
+   [ "text", "(" ],
+   [ "identifier", "n" ],
+   [ "text", " " ],
+   [ "keyword.operator.arithmetic.dart", "-" ],
+   [ "text", " " ],
+   [ "constant.numeric", "2" ],
+   [ "text", ")) " ],
+   [ "keyword.control.ternary.dart", ":" ],
+   [ "text", " " ],
+   [ "identifier", "n" ],
+   [ "text", ";" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "identifier", "main" ],
+   [ "text", "() {" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "identifier", "print" ],
+   [ "text", "(" ],
+   [ "string", "'fib(20) = ${fib(20)}'" ],
+   [ "text", ");" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "}" ]
+  ]
+ },
+ {
+  "state": "comment",
+  "data": [
+   [ "comment", "/*asd" ]
+  ]
+ },
+ {
+  "state": "comment",
+  "data": [
+   [ "comment", "asdad" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "comment", "*/" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.numeric", "0.67" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.numeric", "77" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "." ],
+   [ "constant.numeric", "86" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword.other.import.dart", "#import" ],
+   [ "text", "(" ],
+   [ "string", "\"http://dartwatch.com/myOtherLibrary.dart\"" ],
+   [ "text", ");" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword.other.import.dart", "#import" ],
+   [ "text", "(" ],
+   [ "string", "\"myOtherLibrary.dart\"" ],
+   [ "text", ", " ],
+   [ "keyword.other.import.dart", "prefix" ],
+   [ "text", ":" ],
+   [ "string", "\"lib1\"" ],
+   [ "text", ");" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "qqdoc",
+  "data": [
+   [ "string", "\"\"\"asdasdads" ]
+  ]
+ },
+ {
+  "state": "qqdoc",
+  "data": [
+   [ "string", "asdadsadsasd" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "string", "asdasdasdad\"\"\"" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "string", "'23424'" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "constant.numeric", "0x234" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "identifier", "foo" ],
+   [ "text", " " ],
+   [ "keyword.operator.dart", "is" ],
+   [ "text", " " ],
+   [ "identifier", "bar" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "storage.type.primitive.dart", "int" ],
+   [ "text", " " ],
+   [ "identifier", "x" ],
+   [ "text", " " ],
+   [ "keyword.operator.assignment.dart", "=" ],
+   [ "text", " " ],
+   [ "constant.numeric", "4" ],
+   [ "text", " " ],
+   [ "keyword.operator.bitwise.dart", "<<" ],
+   [ "text", " " ],
+   [ "constant.numeric", "10" ],
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "comment", "// Create a class for Point." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword.declaration.dart", "class" ],
+   [ "text", " " ],
+   [ "identifier", "Point" ],
+   [ "text", " {" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "comment", "// Final variables cannot be changed once they are assigned." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "comment", "// Create two instance variables." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "storage.modifier.dart", "final" ],
+   [ "text", " " ],
+   [ "storage.type.primitive.dart", "num" ],
+   [ "text", " " ],
+   [ "identifier", "x" ],
+   [ "text", ", " ],
+   [ "identifier", "y" ],
+   [ "text", ";" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "comment", "// A constructor, with syntactic sugar for setting instance variables." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "identifier", "Point" ],
+   [ "text", "(" ],
+   [ "variable.language.dart", "this" ],
+   [ "text", "." ],
+   [ "identifier", "x" ],
+   [ "text", ", " ],
+   [ "variable.language.dart", "this" ],
+   [ "text", "." ],
+   [ "identifier", "y" ],
+   [ "text", ");" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "comment", "// A named constructor with an initializer list." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "identifier", "Point" ],
+   [ "text", "." ],
+   [ "identifier", "origin" ],
+   [ "text", "() " ],
+   [ "keyword.control.ternary.dart", ":" ],
+   [ "text", " " ],
+   [ "identifier", "x" ],
+   [ "text", " " ],
+   [ "keyword.operator.assignment.dart", "=" ],
+   [ "text", " " ],
+   [ "constant.numeric", "0" ],
+   [ "text", ", " ],
+   [ "identifier", "y" ],
+   [ "text", " " ],
+   [ "keyword.operator.assignment.dart", "=" ],
+   [ "text", " " ],
+   [ "constant.numeric", "0" ],
+   [ "text", ";" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "comment", "// A method." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "storage.type.primitive.dart", "num" ],
+   [ "text", " " ],
+   [ "identifier", "distanceTo" ],
+   [ "text", "(" ],
+   [ "identifier", "Point" ],
+   [ "text", " " ],
+   [ "identifier", "other" ],
+   [ "text", ") {" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "    " ],
+   [ "storage.type.primitive.dart", "var" ],
+   [ "text", " " ],
+   [ "identifier", "dx" ],
+   [ "text", " " ],
+   [ "keyword.operator.assignment.dart", "=" ],
+   [ "text", " " ],
+   [ "identifier", "x" ],
+   [ "text", " " ],
+   [ "keyword.operator.arithmetic.dart", "-" ],
+   [ "text", " " ],
+   [ "identifier", "other" ],
+   [ "text", "." ],
+   [ "identifier", "x" ],
+   [ "text", ";" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "    " ],
+   [ "storage.type.primitive.dart", "var" ],
+   [ "text", " " ],
+   [ "identifier", "dy" ],
+   [ "text", " " ],
+   [ "keyword.operator.assignment.dart", "=" ],
+   [ "text", " " ],
+   [ "identifier", "y" ],
+   [ "text", " " ],
+   [ "keyword.operator.arithmetic.dart", "-" ],
+   [ "text", " " ],
+   [ "identifier", "other" ],
+   [ "text", "." ],
+   [ "identifier", "y" ],
+   [ "text", ";" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "    " ],
+   [ "keyword.control.dart", "return" ],
+   [ "text", " " ],
+   [ "identifier", "sqrt" ],
+   [ "text", "(" ],
+   [ "identifier", "dx" ],
+   [ "text", " " ],
+   [ "keyword.operator.arithmetic.dart", "*" ],
+   [ "text", " " ],
+   [ "identifier", "dx" ],
+   [ "text", " " ],
+   [ "keyword.operator.arithmetic.dart", "+" ],
+   [ "text", " " ],
+   [ "identifier", "dy" ],
+   [ "text", " " ],
+   [ "keyword.operator.arithmetic.dart", "*" ],
+   [ "text", " " ],
+   [ "identifier", "dy" ],
+   [ "text", ");" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  }" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "}" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " " ],
+   [ "comment", "// Check for null." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "storage.type.primitive.dart", "var" ],
+   [ "text", " " ],
+   [ "identifier", "unicorn" ],
+   [ "text", ";" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "identifier", "assert" ],
+   [ "text", "(" ],
+   [ "identifier", "unicorn" ],
+   [ "text", " " ],
+   [ "keyword.operator.comparison.dart", "==" ],
+   [ "text", " " ],
+   [ "constant.language.dart", "null" ],
+   [ "text", ");" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "comment", "// Check for NaN." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "storage.type.primitive.dart", "var" ],
+   [ "text", " " ],
+   [ "identifier", "iMeantToDoThis" ],
+   [ "text", " " ],
+   [ "keyword.operator.assignment.dart", "=" ],
+   [ "text", " " ],
+   [ "constant.numeric", "0" ],
+   [ "keyword.operator.arithmetic.dart", "/" ],
+   [ "constant.numeric", "0" ],
+   [ "text", ";" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "identifier", "assert" ],
+   [ "text", "(" ],
+   [ "identifier", "iMeantToDoThis" ],
+   [ "text", "." ],
+   [ "identifier", "isNaN" ],
+   [ "text", "());" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ }
+]
--- a/lib/ace/mode/_test/tokens_diff.json
+++ b/lib/ace/mode/_test/tokens_diff.json
@ -2,8 +2,7 @@
 {
  "state": "start",
  "data": [
-   [ "variable", "diff" ],
-   [ "variable", " --git" ],
+   [ "variable", "diff --git" ],
   [ "keyword", " a/lib/ace/edit_session.js" ],
   [ "variable", " b/lib/ace/edit_session.js" ]
  ]
@ -381,8 +380,7 @@
 {
  "state": "start",
  "data": [
-   [ "variable", "diff" ],
-   [ "variable", " --git" ],
+   [ "variable", "diff --git" ],
   [ "keyword", " a/lib/ace/editor.js" ],
   [ "variable", " b/lib/ace/editor.js" ]
  ]
@ -647,8 +645,7 @@
 {
  "state": "start",
  "data": [
-   [ "variable", "diff" ],
-   [ "variable", " --git" ],
+   [ "variable", "diff --git" ],
   [ "keyword", " a/lib/ace/search_highlight.js" ],
   [ "variable", " b/lib/ace/search_highlight.js" ]
  ]
--- a/lib/ace/mode/_test/tokens_glsl.json
+++ b/lib/ace/mode/_test/tokens_glsl.json
@ -41,7 +41,7 @@
  "data": [
   [ "keyword", "void" ],
   [ "text", " " ],
-   [ "entity.name.function", "main" ],
+   [ "identifier", "main" ],
   [ "paren.lparen", "(" ],
   [ "paren.rparen", ")" ],
   [ "text", " " ],
--- a/lib/ace/mode/_test/tokens_golang.json
+++ b/lib/ace/mode/_test/tokens_golang.json
@ -102,8 +102,7 @@
   [ "identifier", "pi" ],
   [ "paren.lparen", "(" ],
   [ "constant.numeric", "5000" ],
-   [ "paren.rparen", ")" ],
-   [ "paren.rparen", ")" ]
+   [ "paren.rparen", "))" ]
  ]
 },
 {
@ -203,8 +202,7 @@
   [ "identifier", "float64" ],
   [ "paren.lparen", "(" ],
   [ "identifier", "k" ],
-   [ "paren.rparen", ")" ],
-   [ "paren.rparen", ")" ]
+   [ "paren.rparen", "))" ]
  ]
 },
 {
@ -259,11 +257,9 @@
   [ "text", "        " ],
   [ "identifier", "f" ],
   [ "text", " " ],
-   [ "keyword.operator", "+" ],
-   [ "keyword.operator", "=" ],
+   [ "keyword.operator", "+=" ],
   [ "text", " " ],
-   [ "keyword.operator", "<" ],
-   [ "keyword.operator", "-" ],
+   [ "keyword.operator", "<-" ],
   [ "identifier", "ch" ]
  ]
 },
@ -321,8 +317,7 @@
   [ "text", "    " ],
   [ "identifier", "ch" ],
   [ "text", " " ],
-   [ "keyword.operator", "<" ],
-   [ "keyword.operator", "-" ],
+   [ "keyword.operator", "<-" ],
   [ "text", " " ],
   [ "constant.numeric", "4" ],
   [ "text", " " ],
--- a/lib/ace/mode/_test/tokens_groovy.json
+++ b/lib/ace/mode/_test/tokens_groovy.json
@ -162,8 +162,7 @@
   [ "identifier", "salary" ],
   [ "text", ":" ],
   [ "constant.numeric", "50" ],
-   [ "rparen", ")" ],
-   [ "rparen", "]" ]
+   [ "rparen", ")]" ]
  ]
 },
 {
@ -195,8 +194,7 @@
   [ "text", " " ],
   [ "identifier", "e" ],
   [ "text", " " ],
-   [ "keyword.operator", "-" ],
-   [ "keyword.operator", ">" ],
+   [ "keyword.operator", "->" ],
   [ "text", " " ],
   [ "identifier", "e" ],
   [ "text", "." ],
@ -279,8 +277,7 @@
   [ "text", " " ],
   [ "identifier", "e" ],
   [ "text", " " ],
-   [ "keyword.operator", "-" ],
-   [ "keyword.operator", ">" ],
+   [ "keyword.operator", "->" ],
   [ "text", " " ],
   [ "identifier", "e" ],
   [ "text", "." ],
@ -350,8 +347,7 @@
   [ "text", " " ],
   [ "identifier", "e" ],
   [ "text", " " ],
-   [ "keyword.operator", "-" ],
-   [ "keyword.operator", ">" ],
+   [ "keyword.operator", "->" ],
   [ "text", " " ],
   [ "identifier", "e" ],
   [ "text", "." ],
@ -398,8 +394,7 @@
   [ "identifier", "emps" ],
   [ "lparen", "[" ],
   [ "constant.numeric", "0" ],
-   [ "rparen", "]" ],
-   [ "rparen", ")" ],
+   [ "rparen", "])" ],
   [ "text", " " ],
   [ "comment", "// true" ]
  ]
@ -457,8 +452,7 @@
   [ "text", " " ],
   [ "identifier", "reader" ],
   [ "text", " " ],
-   [ "keyword.operator", "-" ],
-   [ "keyword.operator", ">" ],
+   [ "keyword.operator", "->" ],
   [ "text", " " ],
   [ "identifier", "doSomethingWith" ],
   [ "lparen", "(" ],
--- a/lib/ace/mode/_test/tokens_haml.json
+++ b/lib/ace/mode/_test/tokens_haml.json
@ -0,0 +1,280 @@
+[
+ {
+  "state": "start",
+  "data": [
+   [ "keyword.other.doctype", "!!!5" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "# <!--[if lt IE 7]> <html class=\"no-js lt-ie9 lt-ie8 lt-ie7\" lang=\"en\"> <![endif]-->" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "# <!--[if IE 7]>    <html class=\"no-js lt-ie9 lt-ie8\" lang=\"en\"> <![endif]-->" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "# <!--[if IE 8]>    <html class=\"no-js lt-ie9\" lang=\"en\"> <![endif]-->" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "# <!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en\"> <!--<![endif]-->" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "/adasdasdad" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "%div" ],
+   [ "punctuation.section", "{" ],
+   [ "constant.other.symbol.ruby", ":id" ],
+   [ "text", " => " ],
+   [ "string", "\"#{@item.type}_#{@item.number}\"" ],
+   [ "text", ", " ],
+   [ "constant.other.symbol.ruby", ":class" ],
+   [ "text", " => " ],
+   [ "string", "'#{@item.type} #{@item.urgency}'" ],
+   [ "text", ", " ],
+   [ "constant.other.symbol.ruby", ":phoney" ],
+   [ "text", " => " ],
+   [ "string", "`asdasdasd`" ],
+   [ "punctuation.section", "}" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "/ file: app/views/movies/index.html.haml" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "meta.escape.haml", "\\d" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "%ads:" ],
+   [ "punctuation.section", "{" ],
+   [ "constant.other.symbol.ruby", ":bleh" ],
+   [ "text", " => " ],
+   [ "constant.numeric", "33" ],
+   [ "punctuation.section", "}" ]
+  ]
+ },
+ {
+  "state": "embedded_ruby",
+  "data": [
+   [ "entity.name.tag.haml", "%p" ],
+   [ "text", "==ddd==" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  Date/Time:" ]
+  ]
+ },
+ {
+  "state": "embedded_ruby",
+  "data": [
+   [ "text", "  - " ],
+   [ "identifier", "now" ],
+   [ "text", " = " ],
+   [ "support.class", "DateTime" ],
+   [ "text", "." ],
+   [ "identifier", "now" ],
+   [ "text", " " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "  %strong" ],
+   [ "text", "= now" ]
+  ]
+ },
+ {
+  "state": "embedded_ruby",
+  "data": [
+   [ "text", "   = " ],
+   [ "keyword", "if" ],
+   [ "text", " " ],
+   [ "identifier", "now" ],
+   [ "text", "  " ],
+   [ "support.class", "DateTime" ],
+   [ "text", "." ],
+   [ "identifier", "parse" ],
+   [ "text", "(\"" ],
+   [ "support.class", "December" ],
+   [ "text", " " ],
+   [ "constant.numeric", "31" ],
+   [ "text", ", " ],
+   [ "constant.numeric", "2006" ],
+   [ "text", "\")" ]
+  ]
+ },
+ {
+  "state": "embedded_ruby",
+  "data": [
+   [ "text", "    = \"" ],
+   [ "support.class", "Happy" ],
+   [ "text", " " ],
+   [ "identifier", "new" ],
+   [ "text", " \" + \"" ],
+   [ "identifier", "year" ],
+   [ "text", "!\"" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "%sfd" ],
+   [ "entity.other.attribute-name.class.haml", ".dfdfg" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "#content" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", " .title" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "   %h1" ],
+   [ "text", "= @title" ]
+  ]
+ },
+ {
+  "state": "embedded_ruby",
+  "data": [
+   [ "text", "   = " ],
+   [ "support.function", "link_to" ],
+   [ "text", " '" ],
+   [ "support.class", "Home" ],
+   [ "text", "', " ],
+   [ "identifier", "home_url" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "   #contents" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "%div" ],
+   [ "entity.other.attribute-name.id.haml", "#content" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "  %div" ],
+   [ "entity.other.attribute-name.class.haml", ".articles" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "    %div" ],
+   [ "entity.other.attribute-name.class.haml", ".article.title" ],
+   [ "text", " Blah" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "    %div" ],
+   [ "entity.other.attribute-name.class.haml", ".article.date" ],
+   [ "text", " " ],
+   [ "constant.numeric", "2006-11-05" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "    %div" ],
+   [ "entity.other.attribute-name.class.haml", ".article.entry" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "      Neil Patrick Harris " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "%div" ],
+   [ "text", "[@user, " ],
+   [ "constant.other.symbol.ruby", ":greeting" ],
+   [ "text", "]" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "entity.name.tag.haml", "  %bar" ],
+   [ "text", "[" ],
+   [ "constant.numeric", "290" ],
+   [ "text", "]/" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "string.quoted.double", "==Hello!==" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ }
+]
--- a/lib/ace/mode/_test/tokens_html.json
+++ b/lib/ace/mode/_test/tokens_html.json
@ -4,7 +4,7 @@
  "data": [
   [ "meta.tag", "<" ],
   [ "meta.tag.tag-name", "html" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 },
 {
@ -13,7 +13,7 @@
   [ "text", "    " ],
   [ "meta.tag", "<" ],
   [ "meta.tag.tag-name", "head" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 },
 {
@ -30,7 +30,7 @@
   [ "entity.other.attribute-name", "type" ],
   [ "keyword.operator", "=" ],
   [ "string", "\"text/css\"" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 },
 {
@ -88,7 +88,7 @@
   [ "text", "    " ],
   [ "meta.tag", "</" ],
   [ "meta.tag.tag-name.style", "style" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 },
 {
@ -101,7 +101,7 @@
   [ "text", "    " ],
   [ "meta.tag", "</" ],
   [ "meta.tag.tag-name", "head" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 },
 {
@ -110,7 +110,7 @@
   [ "text", "    " ],
   [ "meta.tag", "<" ],
   [ "meta.tag.tag-name", "body" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 },
 {
@ -123,11 +123,11 @@
   [ "entity.other.attribute-name", "style" ],
   [ "keyword.operator", "=" ],
   [ "string", "\"color:red\"" ],
-   [ "meta.tag", ">" ],
+   [ "meta.tag.r", ">" ],
   [ "text", "Juhu Kinners" ],
   [ "meta.tag", "</" ],
   [ "meta.tag.tag-name", "h1" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 },
 {
@ -136,7 +136,7 @@
   [ "text", "    " ],
   [ "meta.tag", "</" ],
   [ "meta.tag.tag-name", "body" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 },
 {
@ -144,7 +144,7 @@
  "data": [
   [ "meta.tag", "</" ],
   [ "meta.tag.tag-name", "html" ],
-   [ "meta.tag", ">" ]
+   [ "meta.tag.r", ">" ]
  ]
 }
 ]
--- a/lib/ace/mode/_test/tokens_jade.json
+++ b/lib/ace/mode/_test/tokens_jade.json
@ -0,0 +1,309 @@
+[
+ {
+  "state": "start",
+  "data": [
+   [ "keyword.other.doctype.jade", "!!!doctype" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword.other.doctype.jade", "!!!5" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword.other.doctype.jade", "!!!" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword.control.import.include.jade", "include" ],
+   [ "text", " something" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "keyword.control.import.include.jade", "         include" ],
+   [ "text", " another_thing" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "  // let's talk about it" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "comment_block",
+  "data": [
+   [ "punctuation.section.comment", "//" ]
+  ]
+ },
+ {
+  "state": "comment_block",
+  "data": [
+   [ "comment.block.jade", "  here it is. a block comment!" ]
+  ]
+ },
+ {
+  "state": "comment_block",
+  "data": [
+   [ "comment.block.jade", " and another row!" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "but not here." ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "comment_block",
+  "data": [
+   [ "punctuation.section.comment", "     //     " ]
+  ]
+ },
+ {
+  "state": "comment_block",
+  "data": [
+   [ "comment.block.jade", "        a far spaced" ]
+  ]
+ },
+ {
+  "state": "comment_block",
+  "data": [
+   [ "comment.block.jade", "    should be lack of block" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "punctuation.section.comment", "   // also not a comment" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "meta.tag.any.jade", "     div" ],
+   [ "entity.other.attribute-name.class.jade", ".attemptAtBlock" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "meta.tag.any.jade", "  span" ],
+   [ "entity.other.attribute-name.id.jade", "#myName" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "string.interpolated.jade", "#{implicit}" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "     " ],
+   [ "string.interpolated.jade", "!{more_explicit}" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "  " ],
+   [ "suport.type.attribute.id.jade", "#idDiv" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "text", "    " ],
+   [ "suport.type.attribute.class.jade", ".idDiv" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "meta.tag.any.jade", "    test" ],
+   [ "punctuation", "(" ],
+   [ "entity.other.attribute-name.jade", "id" ],
+   [ "text", "=" ],
+   [ "string", "\"tag\"" ],
+   [ "punctuation", ")" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "meta.tag.any.jade", "    header" ],
+   [ "punctuation", "(" ],
+   [ "entity.other.attribute-name.jade", "id" ],
+   [ "text", "=" ],
+   [ "string", "\"tag\"" ],
+   [ "text", ", " ],
+   [ "entity.other.attribute-name.jade", "blah" ],
+   [ "text", "=" ],
+   [ "string", "\"foo\"" ],
+   [ "text", ", " ],
+   [ "entity.other.attribute-name.jade", "meh" ],
+   [ "text", "=" ],
+   [ "string", "\"aads\"" ],
+   [ "punctuation", ")" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "storage.type.function.jade", "mixin" ],
+   [ "entity.name.function.jade", " article" ],
+   [ "punctuation.definition.parameters.begin.jade", "(" ],
+   [ "variable.parameter.function.jade", "obj, parents" ],
+   [ "punctuation.definition.parameters.end.jade", ")" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "storage.type.function.jade", "  mixin" ],
+   [ "entity.name.function.jade", " bleh" ],
+   [ "punctuation.definition.parameters.begin.jade", "(" ],
+   [ "punctuation.definition.parameters.end.jade", ")" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "storage.type.function.jade", "    mixin" ],
+   [ "entity.name.function.jade", " clever-name" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "source.js.embedded.jade", " -" ],
+   [ "storage.type", "var" ],
+   [ "text", " " ],
+   [ "identifier", "x" ],
+   [ "text", " " ],
+   [ "keyword.operator", "=" ],
+   [ "text", " " ],
+   [ "string", "\"0\"" ],
+   [ "text", ";" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "source.js.embedded.jade", " -" ],
+   [ "text", " " ],
+   [ "identifier", "y" ],
+   [ "text", " " ],
+   [ "identifier", "each" ],
+   [ "text", " z" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": []
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "source.js.embedded.jade", " -" ],
+   [ "text", " " ],
+   [ "storage.type", "var" ],
+   [ "text", " " ],
+   [ "identifier", "items" ],
+   [ "text", " " ],
+   [ "keyword.operator", "=" ],
+   [ "text", " " ],
+   [ "paren.lparen", "[" ],
+   [ "string", "\"one\"" ],
+   [ "punctuation.operator", "," ],
+   [ "text", " " ],
+   [ "string", "\"two\"" ],
+   [ "punctuation.operator", "," ],
+   [ "text", " " ],
+   [ "string", "\"three\"" ],
+   [ "text", "]" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "meta.tag.any.jade", "   each" ],
+   [ "text", " item in items" ]
+  ]
+ },
+ {
+  "state": "start",
+  "data": [
+   [ "meta.tag.any.jade", "    li" ],
+   [ "text", "= item" ]
+  ]
+ }
+]
--- a/lib/ace/mode/_test/tokens_javascript.json
+++ b/lib/ace/mode/_test/tokens_javascript.json
@ -46,7 +46,7 @@
  "state": "regex_allowed",
  "data": [
   [ "text", "        " ],
-   [ "identifier", "alert" ],
+   [ "support.function", "alert" ],
   [ "paren.lparen", "(" ],
   [ "identifier", "items" ],
   [ "paren.lparen", "[" ],
@ -107,8 +107,7 @@
   [ "keyword.operator", "=" ],
   [ "text", " " ],
   [ "string.regexp", "/d" ],
-   [ "constant.language.escape", "{1,2}" ],
-   [ "constant.language.escape", "?" ],
+   [ "constant.language.escape", "{1,2}?" ],
   [ "string.regexp", "f{e}" ],
   [ "invalid", "++" ],
   [ "string.regexp", "r" ],
@ -120,12 +119,10 @@
   [ "string.regexp.charachterclass", "r" ],
   [ "constant.language.escape", "-" ],
   [ "string.regexp.charachterclass", "o" ],
-   [ "regexp.keyword.operator", "\\f" ],
-   [ "regexp.keyword.operator", "\\f" ],
+   [ "regexp.keyword.operator", "\\f\\f" ],
   [ "string.regexp.charachterclass", "[" ],
   [ "regexp.keyword.operator", "\\f" ],
-   [ "constant.language.escape", "]" ],
-   [ "constant.language.escape", "?" ],
+   [ "constant.language.escape", "]?" ],
   [ "string.regexp", "r" ],
   [ "invalid", "{7}+" ],
   [ "string.regexp", "r" ],
@ -133,23 +130,16 @@
   [ "string.regexp", "7}" ],
   [ "constant.language.escape", "+" ],
   [ "string.regexp", "rr--rr" ],
-   [ "constant.language.escape", "$" ],
-   [ "constant.language.escape", "^" ],
-   [ "constant.language.escape", "(?:" ],
+   [ "constant.language.escape", "$^(?:" ],
   [ "string.regexp", "d" ],
   [ "constant.language.escape", "|" ],
   [ "string.regexp", "s" ],
-   [ "constant.language.escape", ")" ],
-   [ "constant.language.escape", "(?=" ],
+   [ "constant.language.escape", ")(?=" ],
   [ "string.regexp", "a" ],
-   [ "constant.language.escape", "|" ],
-   [ "constant.language.escape", ")" ],
-   [ "constant.language.escape", "(?!" ],
+   [ "constant.language.escape", "|)(?!" ],
   [ "string.regexp", "y" ],
-   [ "constant.language.escape", ")[]" ],
-   [ "constant.language.escape", "|" ],
-   [ "constant.language.escape", "$" ],
-   [ "constant.language.escape", "?" ],
+   [ "constant.language.escape", ")[]|" ],
+   [ "invalid", "$?" ],
   [ "constant.language.escape", "|" ],
   [ "invalid", "^*" ],
   [ "string.regexp", "/" ],
--- a/Show more
+++ b/Show more