From 0c99e826e9fec054b4b941f8a22014b42e6647f4 Mon Sep 17 00:00:00 2001
From: Garen Torikian <gjtorikian@gmail.com>
Date: Mon, 4 Feb 2013 17:09:07 -0800
Subject: [PATCH] Update dem tests

---
 lib/ace/mode/_test/tokens_c_cpp.json      |  19 +
 lib/ace/mode/_test/tokens_coffee.json     |   7 +-
 lib/ace/mode/_test/tokens_curly.json      | 122 +---
 lib/ace/mode/_test/tokens_html.json       | 112 +---
 lib/ace/mode/_test/tokens_javascript.json | 192 ++++++
 lib/ace/mode/_test/tokens_lucene.json     |  99 ++-
 lib/ace/mode/_test/tokens_markdown.json   | 718 ++--------------------
 lib/ace/mode/_test/tokens_ruby.json       | 251 ++++----
 lib/ace/mode/_test/tokens_sh.json         | 100 ++-
 lib/ace/mode/_test/tokens_xml.json        | 550 +----------------
 10 files changed, 639 insertions(+), 1531 deletions(-)

diff --git a/lib/ace/mode/_test/tokens_c_cpp.json b/lib/ace/mode/_test/tokens_c_cpp.json
index 6ddda060..2818526f 100644
--- a/lib/ace/mode/_test/tokens_c_cpp.json
+++ b/lib/ace/mode/_test/tokens_c_cpp.json
@@ -52,6 +52,25 @@
   ["constant.other"," \"boost/asio/io_service.hpp\""]
 ],[
    "start"
+],[
+   "directive",
+  ["keyword","#include"],
+  ["constant.other.multiline"," \\"]
+],[
+   "directive",
+  ["constant.other.multiline","   \\"]
+],[
+   "directive",
+  ["constant.other.multiline","   \"iostream\" \\"]
+],[
+   "directive",
+  ["constant.other.multiline","   \"string\" \\"]
+],[
+   "start",
+  ["constant.other","   <vector>"]
+],[
+   "start",
+  ["text","   "]
 ],[
    "start",
   ["keyword.control","using"],
diff --git a/lib/ace/mode/_test/tokens_coffee.json b/lib/ace/mode/_test/tokens_coffee.json
index 5d9d8645..a4a90e7b 100644
--- a/lib/ace/mode/_test/tokens_coffee.json
+++ b/lib/ace/mode/_test/tokens_coffee.json
@@ -466,14 +466,17 @@
    "start",
   ["string.regex","    ///imgy"]
 ],[
-   "start",
+   "js",
   ["text","    "],
   ["keyword","this"],
   ["text"," "],
   ["keyword","isnt"],
   ["punctuation.operator",":"],
   ["text"," "],
-  ["string","`just JavaScript`"]
+  ["string","`just "]
+],[
+   "start",
+  ["string","       JavaScript`"]
 ],[
    "start",
   ["text","    "],
diff --git a/lib/ace/mode/_test/tokens_curly.json b/lib/ace/mode/_test/tokens_curly.json
index 6d086717..b43f9087 100644
--- a/lib/ace/mode/_test/tokens_curly.json
+++ b/lib/ace/mode/_test/tokens_curly.json
@@ -1,106 +1,52 @@
 [[
    "start",
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","html"],
-  ["meta.tag.r",">"]
+  ["text","tokenize Curly template{{test}}"]
 ],[
    "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","head"],
-  ["meta.tag.r",">"]
+  ["text","tokenize embedded script"]
 ],[
-   "start"
-],[
-   "css-start",
-  ["text","    "],
+   "start",
   ["meta.tag","<"],
-  ["meta.tag.tag-name.style","style"],
+  ["meta.tag.tag-name.script","script"],
   ["text"," "],
-  ["entity.other.attribute-name","type"],
+  ["entity.other.attribute-name","a"],
   ["keyword.operator","="],
-  ["string","\"text/css\""],
-  ["meta.tag.r",">"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["variable",".text-layer"],
-  ["text"," "],
-  ["paren.lparen","{"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","font-family"],
-  ["text",": Monaco, "],
-  ["string","\"Courier New\""],
-  ["text",", "],
-  ["support.constant.fonts","monospace"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","font-size"],
-  ["text",": "],
-  ["constant.numeric","12"],
-  ["keyword","px"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","cursor"],
-  ["text",": "],
-  ["support.constant","text"],
-  ["text",";"]
-],[
-   "css-start",
-  ["text","        "],
-  ["paren.rparen","}"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.style","style"],
-  ["meta.tag.r",">"]
-],[
-   "start"
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","head"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","body"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","h1"],
-  ["text"," "],
-  ["entity.other.attribute-name","style"],
-  ["keyword.operator","="],
-  ["string","\"color:red\""],
+  ["string","'a'"],
   ["meta.tag.r",">"],
-  ["variable","{{"],
-  ["text","author_name"],
-  ["variable","}}"],
+  ["storage.type","var"],
   ["meta.tag","</"],
-  ["meta.tag.tag-name","h1"],
+  ["meta.tag.tag-name.script","script"],
+  ["meta.tag.r",">"],
+  ["text","'123'"]
+],[
+   "start",
+  ["text","tokenize multiline attribute value with double quotes"]
+],[
+   "tag_qqstring",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.anchor","a"],
+  ["text"," "],
+  ["entity.other.attribute-name","href"],
+  ["keyword.operator","="],
+  ["string","\"abc{{xyz}}"]
+],[
+   "start",
+  ["string","def\""],
   ["meta.tag.r",">"]
 ],[
    "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","body"],
-  ["meta.tag.r",">"]
+  ["text","tokenize multiline attribute value with single quotes"]
+],[
+   "tag_qstring",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.anchor","a"],
+  ["text"," "],
+  ["entity.other.attribute-name","href"],
+  ["keyword.operator","="],
+  ["string","'abc"]
 ],[
    "start",
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","html"],
+  ["string","def\\\"'"],
   ["meta.tag.r",">"]
 ],[
    "start"
diff --git a/lib/ace/mode/_test/tokens_html.json b/lib/ace/mode/_test/tokens_html.json
index df77b5d3..726c704f 100644
--- a/lib/ace/mode/_test/tokens_html.json
+++ b/lib/ace/mode/_test/tokens_html.json
@@ -5,96 +5,44 @@
   ["meta.tag.r",">"]
 ],[
    "start",
-  ["text","    "],
   ["meta.tag","<"],
-  ["meta.tag.tag-name","head"],
-  ["meta.tag.r",">"]
-],[
-   "start"
-],[
-   "css-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.style","style"],
+  ["meta.tag.tag-name.script","script"],
   ["text"," "],
-  ["entity.other.attribute-name","type"],
+  ["entity.other.attribute-name","a"],
   ["keyword.operator","="],
-  ["string","\"text/css\""],
-  ["meta.tag.r",">"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["variable",".text-layer"],
-  ["text"," "],
-  ["paren.lparen","{"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","font-family"],
-  ["text",": Monaco, "],
-  ["string","\"Courier New\""],
-  ["text",", "],
-  ["support.constant.fonts","monospace"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","font-size"],
-  ["text",": "],
-  ["constant.numeric","12"],
-  ["keyword","px"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","            "],
-  ["support.type","cursor"],
-  ["text",": "],
-  ["support.constant","text"],
-  ["text",";"]
-],[
-   "css-start",
-  ["text","        "],
-  ["paren.rparen","}"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.style","style"],
-  ["meta.tag.r",">"]
-],[
-   "start"
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","head"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","body"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","h1"],
-  ["text"," "],
-  ["entity.other.attribute-name","style"],
-  ["keyword.operator","="],
-  ["string","\"color:red\""],
+  ["string","'a'"],
   ["meta.tag.r",">"],
-  ["text","Juhu Kinners"],
+  ["storage.type","var"],
   ["meta.tag","</"],
-  ["meta.tag.tag-name","h1"],
-  ["meta.tag.r",">"]
+  ["meta.tag.tag-name.script","script"],
+  ["meta.tag.r",">"],
+  ["text","'123'"]
+],[
+   "tag_qqstring",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.anchor","a"],
+  ["text"," "],
+  ["entity.other.attribute-name","href"],
+  ["keyword.operator","="],
+  ["string","\"abc"]
 ],[
    "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","body"],
+  ["string","  def\""],
   ["meta.tag.r",">"]
+],[
+   "tag_qstring",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name.anchor","a"],
+  ["text"," "],
+  ["entity.other.attribute-name","href"],
+  ["keyword.operator","="],
+  ["string","'abc"]
+],[
+   "start",
+  ["string","def\\'"],
+  ["meta.tag.r",">"]
+],[
+   "start"
 ],[
    "start",
   ["meta.tag","</"],
diff --git a/lib/ace/mode/_test/tokens_javascript.json b/lib/ace/mode/_test/tokens_javascript.json
index 85ae0a42..ae5e21b6 100644
--- a/lib/ace/mode/_test/tokens_javascript.json
+++ b/lib/ace/mode/_test/tokens_javascript.json
@@ -1,4 +1,196 @@
 [[
+   "start",
+  ["comment","//test: tokenize 'standard' functions"]
+],[
+   "start",
+  ["identifier","string"],
+  ["punctuation.operator","."],
+  ["support.function","charCodeAt"],
+  ["paren.lparen","("],
+  ["constant.numeric","23"],
+  ["paren.rparen",")"],
+  ["punctuation.operator",";"],
+  ["text"," "],
+  ["variable.language","document"],
+  ["punctuation.operator","."],
+  ["support.function.dom","getElementById"],
+  ["paren.lparen","("],
+  ["string","'test'"],
+  ["paren.rparen",")"],
+  ["punctuation.operator",";"],
+  ["text"," "],
+  ["storage.type","console"],
+  ["punctuation.operator","."],
+  ["support.function.firebug","log"],
+  ["paren.lparen","("],
+  ["string","'Here it is'"],
+  ["paren.rparen",")"],
+  ["punctuation.operator",";"],
+  ["string","\";"]
+],[
+   "start",
+  ["identifier","test"],
+  ["punctuation.operator",":"],
+  ["text"," "],
+  ["comment.doc","/**tokenize doc*/"],
+  ["text"," "],
+  ["identifier","comment"]
+],[
+   "start",
+  ["comment.doc","/**tokenize doc comment with "],
+  ["comment.doc.tag","@tag"],
+  ["comment.doc"," {}*/"]
+],[
+   "start",
+  ["comment","//test: tokenize parens"]
+],[
+   "regex_allowed",
+  ["text","    "],
+  ["storage.type","var"],
+  ["text"," "],
+  ["identifier","line"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["string","\"[{( )}]\""],
+  ["punctuation.operator",";"]
+],[
+   "start",
+  ["comment","//test tokenize arithmetic expression which looks like a regexp"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","/"],
+  ["identifier","b"],
+  ["keyword.operator","/"],
+  ["identifier","c"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","/="],
+  ["identifier","b"],
+  ["keyword.operator","/"],
+  ["identifier","c"]
+],[
+   "start",
+  ["comment","//test tokenize reg exps"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","="],
+  ["string.regexp","/b/g"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","+"],
+  ["string.regexp","/b/g"]
+],[
+   "start",
+  ["identifier","a"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["constant.numeric","1"],
+  ["text"," "],
+  ["keyword.operator","+"],
+  ["text"," "],
+  ["string.regexp","/2 "],
+  ["constant.language.escape","+"],
+  ["string.regexp"," 1/b"]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","="],
+  ["string.regexp","/a/"],
+  ["text"," "],
+  ["keyword.operator","/"],
+  ["text"," "],
+  ["string.regexp","/a/"]
+],[
+   "start",
+  ["keyword","case"],
+  ["text"," "],
+  ["string.regexp","/a/"],
+  ["punctuation.operator","."],
+  ["support.function","test"],
+  ["paren.lparen","("],
+  ["identifier","c"],
+  ["paren.rparen",")"]
+],[
+   "start",
+  ["comment","//test tokenize multi-line comment containing a single line comment"]
+],[
+   "start",
+  ["identifier","noRegex"]
+],[
+   "start",
+  ["comment","/* foo // bar */"]
+],[
+   "regex_allowed",
+  ["identifier","canBeRegex"],
+  ["punctuation.operator",";"]
+],[
+   "regex_allowed",
+  ["comment","/* foo // bar */"]
+],[
+   "start",
+  ["comment","// test tokenize identifier with umlauts"]
+],[
+   "start",
+  ["identifier","fu"],
+  ["punctuation.operator","?"],
+  ["identifier","e"]
+],[
+   "start",
+  ["comment","// test // is not a regexp"]
+],[
+   "start",
+  ["paren.lparen","{"],
+  ["text"," "],
+  ["comment","// 123"]
+],[
+   "start",
+  ["comment","//test skipping escaped chars"]
+],[
+   "start",
+  ["string","'Meh"],
+  ["constant.language.escape","\\\\"],
+  ["string","nNeh'"]
+],[
+   "start",
+  ["storage.type","console"],
+  ["punctuation.operator","."],
+  ["support.function.firebug","log"],
+  ["paren.lparen","("],
+  ["string","'"],
+  ["constant.language.escape","\\\\"],
+  ["string","u1232Feh'"]
+],[
+   "qqstring",
+  ["string","\"test multiline\\"]
+],[
+   "start",
+  ["string"," strings\""]
+],[
+   "start",
+  ["identifier","a"],
+  ["keyword.operator","="],
+  ["text","'"]
+],[
+   "qqstring",
+  ["identifier","b"],
+  ["keyword.operator","="],
+  ["string","\"\\"]
+],[
+   "start",
+  ["string","still a string"]
+],[
+   "start",
+  ["text"," "]
+],[
+   "start",
+  ["text"," "]
+],[
    "regex_allowed",
   ["storage.type","function"],
   ["text"," "],
diff --git a/lib/ace/mode/_test/tokens_lucene.json b/lib/ace/mode/_test/tokens_lucene.json
index 4b75447b..1f6d2985 100644
--- a/lib/ace/mode/_test/tokens_lucene.json
+++ b/lib/ace/mode/_test/tokens_lucene.json
@@ -1,17 +1,92 @@
 [[
    "start",
-  ["paren.lparen","("],
-  ["keyword","title:"],
-  ["string","\"foo bar\""],
-  ["text"," "],
+  ["keyword","test:"],
+  ["text"," recognises "],
   ["keyword.operator","AND"],
-  ["text"," "],
-  ["keyword","body:"],
-  ["string","\"quick fox\""],
-  ["paren.rparen",")"],
-  ["text"," "],
+  ["text"," as keyword"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
   ["keyword.operator","OR"],
-  ["text"," "],
-  ["keyword","title:"],
-  ["text","fox"]
+  ["text"," as keyword"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["keyword.operator","NOT"],
+  ["text"," as keyword"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["string","\"hello this is dog\""],
+  ["text"," as string"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["constant.character.negation","-"],
+  ["string","\"hello this is dog\""],
+  ["text"," as negation with string"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["constant.character.proximity","~100"],
+  ["text"," as text with proximity"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["string","\"hello this is dog\""],
+  ["constant.character.proximity","~100"],
+  ["text"," as string with proximity"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["keyword","raw:"],
+  ["string","\"hello this is dog\""],
+  ["text"," as keyword"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["keyword","raw:"],
+  ["text","foo as\"keyword'"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["string","\"(\""],
+  ["text"," as opening parenthesis"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises "],
+  ["string","\")\""],
+  ["text"," as closing parenthesis"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises foo"],
+  ["constant.character.asterisk","*"],
+  ["text"," as text with asterisk"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises foo"],
+  ["constant.character.interro","?"],
+  ["text"," as text with interro"]
+],[
+   "start",
+  ["keyword","test:"],
+  ["text"," recognises single word as text"]
+],[
+   "start",
+  ["text"," foo"]
+],[
+   "start",
+  ["text"," "]
 ]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_markdown.json b/lib/ace/mode/_test/tokens_markdown.json
index 5fe3be82..e8d1464f 100644
--- a/lib/ace/mode/_test/tokens_markdown.json
+++ b/lib/ace/mode/_test/tokens_markdown.json
@@ -1,690 +1,42 @@
 [[
    "start",
-  ["text","Ace (Ajax.org Cloud9 Editor)"]
+  ["text","test: header 1 "]
 ],[
    "start",
-  ["markup.heading.1","============================"]
+  ["markup.heading.1","#f"]
+],[
+   "start",
+  ["text","test: header 2"]
+],[
+   "start",
+  ["markup.heading.2","## foo"]
+],[
+   "start",
+  ["text","test: header ends with ' #'"]
+],[
+   "start",
+  ["markup.heading.1","# # # "]
+],[
+   "start",
+  ["text","test: header ends with '#'"]
+],[
+   "start",
+  ["markup.heading.1","#foo# "]
+],[
+   "start",
+  ["text","test: 6+ #s is not a valid header"]
+],[
+   "start",
+  ["text","####### foo"]
+],[
+   "start",
+  ["text","test: # followed be only space is not a valid header"]
+],[
+   "start",
+  ["text","test: only space between #s is not a valid header"]
+],[
+   "start",
+  ["text","#  #"]
 ],[
    "start"
-],[
-   "start",
-  ["text","Ace is a standalone code editor written in JavaScript. Our goal is to create a browser based editor that matches and extends the features, usability and performance of existing native editors such as TextMate, Vim or Eclipse. It can be easily embedded in any web page or JavaScript application. Ace is developed as the primary editor for ["],
-  ["string","Cloud9 IDE"],
-  ["text","]("],
-  ["markup.underline","http://www.cloud9ide.com/"],
-  ["text",") and the successor of the Mozilla Skywriter (Bespin) Project."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Features"]
-],[
-   "start",
-  ["markup.heading.2","--------"]
-],[
-   "start"
-],[
-   "listblock",
-  ["markup.list","* Syntax highlighting"]
-],[
-   "listblock",
-  ["markup.list","* Automatic indent and outdent"]
-],[
-   "listblock",
-  ["markup.list","* An optional command line"]
-],[
-   "listblock",
-  ["markup.list","* Handles huge documents (100,000 lines and more are no problem)"]
-],[
-   "listblock",
-  ["markup.list","* Fully customizable key bindings including VI and Emacs modes"]
-],[
-   "listblock",
-  ["markup.list","* Themes (TextMate themes can be imported)"]
-],[
-   "listblock",
-  ["markup.list","* Search and replace with regular expressions"]
-],[
-   "listblock",
-  ["markup.list","* Highlight matching parentheses"]
-],[
-   "listblock",
-  ["markup.list","* Toggle between soft tabs and real tabs"]
-],[
-   "listblock",
-  ["markup.list","* Displays hidden characters"]
-],[
-   "listblock",
-  ["markup.list","* Drag and drop text using the mouse"]
-],[
-   "listblock",
-  ["markup.list","* Line wrapping"]
-],[
-   "listblock",
-  ["markup.list","* Unstructured / user code folding"]
-],[
-   "listblock",
-  ["markup.list","* Live syntax checker (currently JavaScript/CoffeeScript)"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Take Ace for a spin!"]
-],[
-   "start",
-  ["markup.heading.2","--------------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Check out the Ace live ["],
-  ["string","demo"],
-  ["text","]("],
-  ["markup.underline","http://ajaxorg.github.com/ace/"],
-  ["text",") or get a ["],
-  ["string","Cloud9 IDE account"],
-  ["text","]("],
-  ["markup.underline","http://run.cloud9ide.com"],
-  ["text",") to experience Ace while editing one of your own GitHub projects."]
-],[
-   "start"
-],[
-   "start",
-  ["text","If you want, you can use Ace as a textarea replacement thanks to the ["],
-  ["string","Ace Bookmarklet"],
-  ["text","]("],
-  ["markup.underline","http://ajaxorg.github.com/ace/build/textarea/editor.html"],
-  ["text",")."]
-],[
-   "start"
-],[
-   "start",
-  ["text","History"]
-],[
-   "start",
-  ["markup.heading.2","-------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Previously known as “Bespin” and “Skywriter” it’s now known as Ace (Ajax.org Cloud9 Editor)! Bespin and Ace started as two independent projects, both aiming to build a no-compromise code editor component for the web. Bespin started as part of Mozilla Labs and was based on the canvas tag, while Ace is the Editor component of the Cloud9 IDE and is using the DOM for rendering. After the release of Ace at JSConf.eu 2010 in Berlin the Skywriter team decided to merge Ace with a simplified version of Skywriter's plugin system and some of Skywriter's extensibility points. All these changes have been merged back to Ace. Both Ajax.org and Mozilla are actively developing and maintaining Ace."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Getting the code"]
-],[
-   "start",
-  ["markup.heading.2","----------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Ace is a community project. We actively encourage and support contributions. The Ace source code is hosted on GitHub. It is released under the BSD License. This license is very simple, and is friendly to all kinds of projects, whether open source or not. Take charge of your editor and add your favorite language highlighting and keybindings!"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    git clone git://github.com/ajaxorg/ace.git"]
-],[
-   "githubblock",
-  ["support.function","    cd ace"]
-],[
-   "githubblock",
-  ["support.function","    git submodule update --init --recursive"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Embedding Ace"]
-],[
-   "start",
-  ["markup.heading.2","-------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Ace can be easily embedded into any existing web page. The Ace git repository ships with a pre-packaged version of Ace inside of the "],
-  ["support.function","`build`"],
-  ["text"," directory. The same packaged files are also available as a separate ["],
-  ["string","download"],
-  ["text","]("],
-  ["markup.underline","https://github.com/ajaxorg/ace/downloads"],
-  ["text","). Simply copy the contents of the "],
-  ["support.function","`src`"],
-  ["text"," subdirectory somewhere into your project and take a look at the included demos of how to use Ace."]
-],[
-   "start"
-],[
-   "start",
-  ["text","The easiest version is simply:"]
-],[
-   "start"
-],[
-   "html-start",
-  ["support.function","```html"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","div"],
-  ["text"," "],
-  ["entity.other.attribute-name","id"],
-  ["keyword.operator","="],
-  ["string","\"editor\""],
-  ["meta.tag.r",">"],
-  ["text","some text"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","div"],
-  ["meta.tag.r",">"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.script","script"],
-  ["text"," "],
-  ["entity.other.attribute-name","src"],
-  ["keyword.operator","="],
-  ["string","\"src/ace.js\""],
-  ["text"," "],
-  ["entity.other.attribute-name","type"],
-  ["keyword.operator","="],
-  ["string","\"text/javascript\""],
-  ["text"," "],
-  ["entity.other.attribute-name","charset"],
-  ["keyword.operator","="],
-  ["string","\"utf-8\""],
-  ["meta.tag.r",">"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "html-js-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "html-js-regex_allowed",
-  ["text","    "],
-  ["storage.type","window"],
-  ["punctuation.operator","."],
-  ["entity.name.function","onload"],
-  ["text"," "],
-  ["keyword.operator","="],
-  ["text"," "],
-  ["storage.type","function"],
-  ["paren.lparen","("],
-  ["paren.rparen",")"],
-  ["text"," "],
-  ["paren.lparen","{"]
-],[
-   "html-js-regex_allowed",
-  ["text","        "],
-  ["storage.type","var"],
-  ["text"," "],
-  ["identifier","editor"],
-  ["text"," "],
-  ["keyword.operator","="],
-  ["text"," "],
-  ["identifier","ace"],
-  ["punctuation.operator","."],
-  ["identifier","edit"],
-  ["paren.lparen","("],
-  ["string","\"editor\""],
-  ["paren.rparen",")"],
-  ["punctuation.operator",";"]
-],[
-   "html-js-regex_allowed",
-  ["text","    "],
-  ["paren.rparen","}"],
-  ["punctuation.operator",";"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","With \"editor\" being the id of the DOM element, which should be converted to an editor. Note that this element must be explicitly sized and positioned "],
-  ["support.function","`absolute`"],
-  ["text"," or "],
-  ["support.function","`relative`"],
-  ["text"," for Ace to work. e.g."]
-],[
-   "start"
-],[
-   "css-start",
-  ["support.function","```css"]
-],[
-   "css-ruleset",
-  ["text","    "],
-  ["keyword","#editor"],
-  ["text"," "],
-  ["paren.lparen","{"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["support.type","position"],
-  ["text",": "],
-  ["support.constant","absolute"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["support.type","width"],
-  ["text",": "],
-  ["constant.numeric","500"],
-  ["keyword","px"],
-  ["text",";"]
-],[
-   "css-ruleset",
-  ["text","        "],
-  ["support.type","height"],
-  ["text",": "],
-  ["constant.numeric","400"],
-  ["keyword","px"],
-  ["text",";"]
-],[
-   "css-start",
-  ["text","    "],
-  ["paren.rparen","}"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","To change the theme simply include the Theme's JavaScript file"]
-],[
-   "start"
-],[
-   "html-start",
-  ["support.function","```html"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.script","script"],
-  ["text"," "],
-  ["entity.other.attribute-name","src"],
-  ["keyword.operator","="],
-  ["string","\"src/theme-twilight.js\""],
-  ["text"," "],
-  ["entity.other.attribute-name","type"],
-  ["keyword.operator","="],
-  ["string","\"text/javascript\""],
-  ["text"," "],
-  ["entity.other.attribute-name","charset"],
-  ["keyword.operator","="],
-  ["string","\"utf-8\""],
-  ["meta.tag.r",">"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","and configure the editor to use the theme:"]
-],[
-   "start"
-],[
-   "js-start",
-  ["support.function","```javascript"]
-],[
-   "js-regex_allowed",
-  ["text","    "],
-  ["identifier","editor"],
-  ["punctuation.operator","."],
-  ["identifier","setTheme"],
-  ["paren.lparen","("],
-  ["string","\"ace/theme/twilight\""],
-  ["paren.rparen",")"],
-  ["punctuation.operator",";"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","By default the editor only supports plain text mode; many other languages are available as separate modules. After including the mode's JavaScript file:"]
-],[
-   "start"
-],[
-   "html-start",
-  ["support.function","```html"]
-],[
-   "html-start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name.script","script"],
-  ["text"," "],
-  ["entity.other.attribute-name","src"],
-  ["keyword.operator","="],
-  ["string","\"src/mode-javascript.js\""],
-  ["text"," "],
-  ["entity.other.attribute-name","type"],
-  ["keyword.operator","="],
-  ["string","\"text/javascript\""],
-  ["text"," "],
-  ["entity.other.attribute-name","charset"],
-  ["keyword.operator","="],
-  ["string","\"utf-8\""],
-  ["meta.tag.r",">"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name.script","script"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Then the mode can be used like this:"]
-],[
-   "start"
-],[
-   "js-start",
-  ["support.function","```javascript"]
-],[
-   "js-regex_allowed",
-  ["text","    "],
-  ["storage.type","var"],
-  ["text"," "],
-  ["identifier","JavaScriptMode"],
-  ["text"," "],
-  ["keyword.operator","="],
-  ["text"," "],
-  ["identifier","require"],
-  ["paren.lparen","("],
-  ["string","\"ace/mode/javascript\""],
-  ["paren.rparen",")"],
-  ["punctuation.operator","."],
-  ["identifier","Mode"],
-  ["punctuation.operator",";"]
-],[
-   "js-regex_allowed",
-  ["text","    "],
-  ["identifier","editor"],
-  ["punctuation.operator","."],
-  ["identifier","getSession"],
-  ["paren.lparen","("],
-  ["paren.rparen",")"],
-  ["punctuation.operator","."],
-  ["identifier","setMode"],
-  ["paren.lparen","("],
-  ["keyword","new"],
-  ["text"," "],
-  ["identifier","JavaScriptMode"],
-  ["paren.lparen","("],
-  ["paren.rparen","))"],
-  ["punctuation.operator",";"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Documentation"]
-],[
-   "start",
-  ["markup.heading.2","-------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","You find a lot more sample code in the ["],
-  ["string","demo app"],
-  ["text","]("],
-  ["markup.underline","https://github.com/ajaxorg/ace/blob/master/demo/demo.js"],
-  ["text",")."]
-],[
-   "start"
-],[
-   "start",
-  ["text","There is also some documentation on the ["],
-  ["string","wiki page"],
-  ["text","]("],
-  ["markup.underline","https://github.com/ajaxorg/ace/wiki"],
-  ["text",")."]
-],[
-   "start"
-],[
-   "start",
-  ["text","If you still need help, feel free to drop a mail on the ["],
-  ["string","ace mailing list"],
-  ["text","]("],
-  ["markup.underline","http://groups.google.com/group/ace-discuss"],
-  ["text",")."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Running Ace"]
-],[
-   "start",
-  ["markup.heading.2","-----------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","After the checkout Ace works out of the box. No build step is required. Open 'editor.html' in any browser except Google Chrome. Google Chrome doesn't allow XMLHTTPRequests from files loaded from disc (i.e. with a file:/// URL). To open Ace in Chrome simply start the bundled mini HTTP server:"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    ./static.py"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Or using Node.JS"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    ./static.js"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","The editor can then be opened at http://localhost:8888/index.html."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Package Ace"]
-],[
-   "start",
-  ["markup.heading.2","-----------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","To package Ace we use the dryice build tool developed by the Mozilla Skywriter team. Before you can build you need to make sure that the submodules are up to date."]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    git submodule update --init --recursive"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Afterwards Ace can be built by calling"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    ./Makefile.dryice.js normal"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","The packaged Ace will be put in the 'build' folder."]
-],[
-   "start"
-],[
-   "start",
-  ["text","To build the bookmarklet version execute"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    ./Makefile.dryice.js bm"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Running the Unit Tests"]
-],[
-   "start",
-  ["markup.heading.2","----------------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","The Ace unit tests run on node.js. Before the first run a couple of node modules have to be installed. The easiest way to do this is by using the node package manager (npm). In the Ace base directory simply call"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    npm link ."]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","To run the tests call:"]
-],[
-   "start"
-],[
-   "githubblock",
-  ["support.function","```bash"]
-],[
-   "githubblock",
-  ["support.function","    node lib/ace/test/all.js"]
-],[
-   "start",
-  ["support.function","```"]
-],[
-   "start"
-],[
-   "start",
-  ["text","You can also run the tests in your browser by serving:"]
-],[
-   "start"
-],[
-   "start",
-  ["support.function","    http://localhost:8888/lib/ace/test/tests.html"]
-],[
-   "start"
-],[
-   "start",
-  ["text","This makes debugging failing tests way more easier."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Contributing"]
-],[
-   "start",
-  ["markup.heading.2","------------"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Ace wouldn't be what it is without contributions! Feel free to fork and improve/enhance Ace any way you want. If you feel that the editor or the Ace community will benefit from your changes, please open a pull request. To protect the interests of the Ace contributors and users we require contributors to sign a Contributors License Agreement (CLA) before we pull the changes into the main repository. Our CLA is the simplest of agreements, requiring that the contributions you make to an ajax.org project are only those you're allowed to make. This helps us significantly reduce future legal risk for everyone involved. It is easy, helps everyone, takes ten minutes, and only needs to be completed once.  There are two versions of the agreement:"]
-],[
-   "start"
-],[
-   "listblock",
-  ["markup.list","1. [The Individual CLA](https://github.com/ajaxorg/ace/raw/master/doc/Contributor_License_Agreement-v2.pdf): use this version if you're working on an ajax.org in your spare time, or can clearly claim ownership of copyright in what you'll be submitting."]
-],[
-   "listblock",
-  ["markup.list","2. [The Corporate CLA](https://github.com/ajaxorg/ace/raw/master/doc/Corporate_Contributor_License_Agreement-v2.pdf): have your corporate lawyer review and submit this if your company is going to be contributing to ajax.org  projects"]
-],[
-   "start"
-],[
-   "start",
-  ["text","If you want to contribute to an ajax.org project please print the CLA and fill it out and sign it. Then either send it by snail mail or fax to us or send it back scanned (or as a photo) by email."]
-],[
-   "start"
-],[
-   "start",
-  ["text","Email: fabian.jakobs@web.de"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Fax: +31 (0) 206388953"]
-],[
-   "start"
-],[
-   "start",
-  ["text","Address: Ajax.org B.V."]
-],[
-   "start",
-  ["text","  Keizersgracht 241"]
-],[
-   "start",
-  ["text","  1016 EA, Amsterdam"]
-],[
-   "start",
-  ["text","  the Netherlands"]
 ]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_ruby.json b/lib/ace/mode/_test/tokens_ruby.json
index 5f4b9d16..f1d80f46 100644
--- a/lib/ace/mode/_test/tokens_ruby.json
+++ b/lib/ace/mode/_test/tokens_ruby.json
@@ -1,154 +1,135 @@
 [[
    "start",
-  ["comment","#!/usr/bin/ruby"]
-],[
-   "start"
-],[
-   "start",
-  ["comment","# Program to find the factorial of a number"]
-],[
-   "start",
-  ["keyword","def"],
   ["text"," "],
-  ["identifier","fact"],
-  ["paren.lparen","("],
-  ["identifier","n"],
-  ["paren.rparen",")"]
-],[
-   "start",
-  ["text","    "],
-  ["keyword","if"],
-  ["text"," "],
-  ["identifier","n"],
-  ["text"," "],
-  ["keyword.operator","=="],
-  ["text"," "],
-  ["constant.numeric","0"]
-],[
-   "start",
-  ["text","        "],
-  ["constant.numeric","1"]
-],[
-   "start",
-  ["text","    "],
-  ["keyword","else"]
-],[
-   "start",
-  ["text","        "],
-  ["identifier","n"],
-  ["text"," "],
-  ["keyword.operator","*"],
-  ["text"," "],
-  ["identifier","fact"],
-  ["paren.lparen","("],
-  ["identifier","n"],
-  ["constant.numeric","-1"],
-  ["paren.rparen",")"]
-],[
-   "start",
-  ["text","    "],
-  ["keyword","end"]
-],[
-   "start",
-  ["keyword","end"]
-],[
-   "start"
-],[
-   "start",
-  ["support.function","puts"],
-  ["text"," "],
-  ["identifier","fact"],
-  ["paren.lparen","("],
-  ["support.class","ARGV"],
-  ["paren.lparen","["],
-  ["constant.numeric","0"],
-  ["paren.rparen","]"],
-  ["text","."],
-  ["identifier","to_i"],
-  ["paren.rparen",")"]
-],[
-   "start"
-],[
-   "start",
-  ["keyword","class"],
-  ["text"," "],
-  ["support.class","Range"]
+  ["comment","#test: symbol tokenizer"]
 ],[
    "start",
   ["text","  "],
-  ["keyword","def"],
-  ["text"," "],
-  ["identifier","to_json"],
-  ["paren.lparen","("],
-  ["keyword.operator","*"],
-  ["identifier","a"],
-  ["paren.rparen",")"]
+  ["paren.lparen","["],
+  ["constant.other.symbol.ruby",":@thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":$thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":_thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":Thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing1"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing_a"],
+  ["text",","]
+],[
+   "start",
+  ["text","              "],
+  ["constant.other.symbol.ruby",":THING"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing!"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing="],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing?"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":t?"],
+  ["text",","]
+],[
+   "start",
+  ["text","              :, :@, :"],
+  ["keyword.operator","$"],
+  ["text",", :"],
+  ["constant.numeric","1"],
+  ["text",", :1"],
+  ["identifier","thing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":th?"],
+  ["identifier","ing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thi="],
+  ["identifier","ng"],
+  ["text",", :1"],
+  ["identifier","thing"],
+  ["text",","]
+],[
+   "start",
+  ["text","            "],
+  ["constant.other.symbol.ruby",":th!"],
+  ["identifier","ing"],
+  ["text",", "],
+  ["constant.other.symbol.ruby",":thing"],
+  ["comment","#"]
 ],[
    "start",
   ["text","    "],
-  ["paren.lparen","{"]
-],[
-   "start",
-  ["text","      "],
-  ["string","'json_class'"],
-  ["text","   "],
-  ["punctuation.separator.key-value","=>"],
-  ["text"," "],
-  ["variable.language","self"],
-  ["text","."],
-  ["keyword","class"],
-  ["text","."],
-  ["identifier","name"],
-  ["text",", "],
-  ["comment","# = 'Range'"]
-],[
-   "start",
-  ["text","      "],
-  ["string","'data'"],
-  ["text","         "],
-  ["punctuation.separator.key-value","=>"],
-  ["text"," "],
-  ["paren.lparen","["],
-  ["text"," "],
-  ["identifier","first"],
-  ["text",", "],
-  ["identifier","last"],
-  ["text",", "],
-  ["identifier","exclude_end"],
-  ["text","? "],
   ["paren.rparen","]"]
-],[
-   "start",
-  ["text","    "],
-  ["paren.rparen","}"],
-  ["text","."],
-  ["identifier","to_json"],
-  ["paren.lparen","("],
-  ["keyword.operator","*"],
-  ["identifier","a"],
-  ["paren.rparen",")"]
-],[
-   "start",
-  ["text","  "],
-  ["keyword","end"]
-],[
-   "start",
-  ["keyword","end"]
 ],[
    "start"
 ],[
    "start",
-  ["paren.lparen","{"],
-  ["constant.other.symbol.ruby",":id"],
   ["text"," "],
-  ["punctuation.separator.key-value","=>"],
+  ["comment","#test: namespaces aren't symbols\" : function() {"]
+],[
+   "start",
+  ["text","   "],
+  ["support.class","Namespaced"],
+  ["text","::"],
+  ["support.class","Class"]
+],[
+   "start",
   ["text"," "],
-  ["constant.numeric","34"],
+  ["comment","#test: hex tokenizer "]
+],[
+   "start",
+  ["text","    "],
+  ["constant.numeric","0x9a"],
   ["text",", "],
-  ["constant.other.symbol.ruby",":sdfasdfasdf"],
+  ["constant.numeric","0XA1"],
+  ["text",", "],
+  ["constant.numeric","0x9_a"],
+  ["text",", 0"],
+  ["identifier","x"],
+  ["text",", 0"],
+  ["identifier","x_9a"],
+  ["text",", 0"],
+  ["identifier","x9a_"],
+  ["text",","]
+],[
+   "start",
   ["text"," "],
-  ["punctuation.separator.key-value","=>"],
-  ["text"," "],
-  ["string","\"asdasdads\""],
-  ["paren.rparen","}"]
+  ["comment","#test: float tokenizer"]
+],[
+   "start",
+  ["text","    "],
+  ["paren.lparen","["],
+  ["constant.numeric","1"],
+  ["text",", "],
+  ["constant.numeric","+1"],
+  ["text",", "],
+  ["constant.numeric","-1"],
+  ["text",", "],
+  ["constant.numeric","12_345"],
+  ["text",", "],
+  ["constant.numeric","0.000_1"],
+  ["text",","]
+],[
+   "start",
+  ["text","    "],
+  ["identifier","_"],
+  ["text",", "],
+  ["constant.numeric","3_1"],
+  ["text",", "],
+  ["constant.numeric","1_2"],
+  ["text",", 1"],
+  ["identifier","_"],
+  ["text","."],
+  ["constant.numeric","0"],
+  ["text",", "],
+  ["constant.numeric","0"],
+  ["text","."],
+  ["identifier","_1"],
+  ["paren.rparen","]"],
+  ["text",";"]
+],[
+   "start",
+  ["text"," "]
 ]]
\ No newline at end of file
diff --git a/lib/ace/mode/_test/tokens_sh.json b/lib/ace/mode/_test/tokens_sh.json
index d490692e..11e8698b 100644
--- a/lib/ace/mode/_test/tokens_sh.json
+++ b/lib/ace/mode/_test/tokens_sh.json
@@ -1,23 +1,89 @@
 [[
    "start",
-  ["comment","#!/bin/sh"]
+  ["text","#!"],
+  ["keyword.operator","/"],
+  ["identifier","bin"],
+  ["keyword.operator","/"],
+  ["identifier","sh"]
 ],[
    "start"
 ],[
    "start",
-  ["comment","# Script to open a browser to current branch"]
+  ["text","# "],
+  ["identifier","Script"],
+  ["text"," "],
+  ["identifier","to"],
+  ["text"," "],
+  ["identifier","open"],
+  ["text"," "],
+  ["identifier","a"],
+  ["text"," "],
+  ["identifier","browser"],
+  ["text"," "],
+  ["identifier","to"],
+  ["text"," "],
+  ["identifier","current"],
+  ["text"," "],
+  ["identifier","branch"]
 ],[
    "start",
-  ["comment","# Repo formats:"]
+  ["text","# "],
+  ["identifier","Repo"],
+  ["text"," "],
+  ["identifier","formats"],
+  ["text",":"]
 ],[
    "start",
-  ["comment","# ssh   git@github.com:richoH/gh_pr.git"]
+  ["text","# "],
+  ["identifier","ssh"],
+  ["text","   "],
+  ["identifier","git"],
+  ["text","@"],
+  ["identifier","github"],
+  ["text","."],
+  ["identifier","com"],
+  ["text",":"],
+  ["identifier","richoH"],
+  ["keyword.operator","/"],
+  ["identifier","gh_pr"],
+  ["text","."],
+  ["identifier","git"]
 ],[
    "start",
-  ["comment","# http  https://richoH@github.com/richoH/gh_pr.git"]
+  ["text","# "],
+  ["identifier","http"],
+  ["text","  "],
+  ["identifier","https"],
+  ["text",":"],
+  ["keyword.operator","//"],
+  ["identifier","richoH"],
+  ["text","@"],
+  ["identifier","github"],
+  ["text","."],
+  ["identifier","com"],
+  ["keyword.operator","/"],
+  ["identifier","richoH"],
+  ["keyword.operator","/"],
+  ["identifier","gh_pr"],
+  ["text","."],
+  ["identifier","git"]
 ],[
    "start",
-  ["comment","# git   git://github.com/richoH/gh_pr.git"]
+  ["text","# "],
+  ["identifier","git"],
+  ["text","   "],
+  ["identifier","git"],
+  ["text",":"],
+  ["keyword.operator","//"],
+  ["identifier","github"],
+  ["text","."],
+  ["identifier","com"],
+  ["keyword.operator","/"],
+  ["identifier","richoH"],
+  ["keyword.operator","/"],
+  ["identifier","gh_pr"],
+  ["text","."],
+  ["identifier","git"]
 ],[
    "start"
 ],[
@@ -269,6 +335,28 @@
   ["paren.rparen","}"]
 ],[
    "start"
+],[
+   "start",
+  ["variable","x="],
+  ["paren.lparen","("],
+  ["constant.numeric","1"],
+  ["text"," "],
+  ["constant.numeric","2"],
+  ["text"," "],
+  ["constant.numeric","3"],
+  ["paren.rparen",")"]
+],[
+   "start",
+  ["support.function.builtin","echo"],
+  ["text"," $"],
+  ["paren.lparen","{"],
+  ["text","#"],
+  ["identifier","x"],
+  ["paren.lparen","["],
+  ["text","@"],
+  ["paren.rparen","]}"]
+],[
+   "start"
 ],[
    "start",
   ["keyword","if"],
diff --git a/lib/ace/mode/_test/tokens_xml.json b/lib/ace/mode/_test/tokens_xml.json
index 19762eca..f7c1ed64 100644
--- a/lib/ace/mode/_test/tokens_xml.json
+++ b/lib/ace/mode/_test/tokens_xml.json
@@ -1,539 +1,43 @@
 [[
    "start",
-  ["xml-pe","<?xml version=\"1.0\" encoding=\"UTF-8\"?>"]
-],[
-   "tag_embed_attribute_list",
   ["meta.tag","<"],
-  ["meta.tag.tag-name","query"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns:yahoo"],
-  ["keyword.operator","="],
-  ["string","\"http://www.yahooapis.com/v1/base.rng\""]
-],[
-   "start",
-  ["text","    "],
-  ["entity.other.attribute-name","yahoo:count"],
-  ["keyword.operator","="],
-  ["string","\"7\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:created"],
-  ["keyword.operator","="],
-  ["string","\"2011-10-11T08:40:23Z\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","diagnostics"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","publiclyCallable"],
+  ["meta.tag.tag-name","Juhu"],
   ["meta.tag.r",">"],
-  ["text","true"],
+  ["text","//Juhu Kinners"],
   ["meta.tag","</"],
-  ["meta.tag.tag-name","publiclyCallable"],
+  ["meta.tag.tag-name","Kinners"],
   ["meta.tag.r",">"]
 ],[
    "start",
-  ["text","        "],
+  ["text","test: two tags in the same lines should be in separate tokens\""]
+],[
+   "start",
   ["meta.tag","<"],
-  ["meta.tag.tag-name","url"],
+  ["meta.tag.tag-name","Juhu"],
+  ["meta.tag.r",">"],
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","Kinners"],
+  ["meta.tag.r",">"]
+],[
+   "start",
+  ["text","test: multiline attributes\""]
+],[
+   "tag_qqstring",
+  ["meta.tag","<"],
+  ["meta.tag.tag-name","copy"],
   ["text"," "],
-  ["entity.other.attribute-name","execution-start-time"],
+  ["entity.other.attribute-name","set"],
   ["keyword.operator","="],
-  ["string","\"0\""],
+  ["string","\"{"]
+],[
+   "tag_qqstring",
+  ["string","}\""],
   ["text"," "],
-  ["entity.other.attribute-name","execution-stop-time"],
+  ["entity.other.attribute-name","undo"],
   ["keyword.operator","="],
-  ["string","\"25\""],
-  ["text"," "],
-  ["entity.other.attribute-name","execution-time"],
-  ["keyword.operator","="],
-  ["string","\"25\""],
-  ["meta.tag.r",">"],
-  ["text","<![CDATA[http://where.yahooapis.com/v1/continents;start=0;count=10]]>"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","url"],
-  ["meta.tag.r",">"]
+  ["string","\"{"]
 ],[
    "start",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","user-time"],
-  ["meta.tag.r",">"],
-  ["text","26"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","user-time"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","service-time"],
-  ["meta.tag.r",">"],
-  ["text","25"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","service-time"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","build-version"],
-  ["meta.tag.r",">"],
-  ["text","21978"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","build-version"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","diagnostics"],
-  ["meta.tag.r",">"],
-  ["text"," "]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","results"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865670\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865670"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Africa"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865675\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865675"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Europe"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865673\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865673"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","South America"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/28289421\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","28289421"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Antarctic"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865671\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865671"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Asia"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/24865672\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","24865672"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","North America"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "tag_embed_attribute_list",
-  ["text","        "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","place"],
-  ["text"," "],
-  ["entity.other.attribute-name","xmlns"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/schema.rng\""]
-],[
-   "start",
-  ["text","            "],
-  ["entity.other.attribute-name","xml:lang"],
-  ["keyword.operator","="],
-  ["string","\"en-US\""],
-  ["text"," "],
-  ["entity.other.attribute-name","yahoo:uri"],
-  ["keyword.operator","="],
-  ["string","\"http://where.yahooapis.com/v1/place/55949070\""],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"],
-  ["text","55949070"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","woeid"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["text"," "],
-  ["entity.other.attribute-name","code"],
-  ["keyword.operator","="],
-  ["string","\"29\""],
-  ["meta.tag.r",">"],
-  ["text","Continent"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","placeTypeName"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","            "],
-  ["meta.tag","<"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"],
-  ["text","Australia"],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","name"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","        "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","place"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["text","    "],
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","results"],
-  ["meta.tag.r",">"]
-],[
-   "start",
-  ["meta.tag","</"],
-  ["meta.tag.tag-name","query"],
-  ["meta.tag.r",">"]
+  ["string","}\""],
+  ["meta.tag.r","/>"]
 ]]
\ No newline at end of file