Merge pull request #1249 from ajaxorg/highlighting/2.0

Highlighting 2.0 (continuation)

demo/kitchen-sink/doclist.js

@@ -100,6 +100,7 @@ var docs = {
     "docs/objectivec.m": {name: "Objective-C"},
     "docs/ocaml.ml": "OCaml",
     "docs/OpenSCAD.scad": "OpenSCAD",
+    "docs/pascal.pas": "Pascal",
     "docs/perl.pl": "Perl",
     "docs/pgsql.pgsql": {name: "pgSQL", wrapped: true},
     "docs/php.php": "PHP",
@@ -113,6 +114,7 @@ var docs = {
     "docs/abap.abap": "SAP - ABAP",
     "docs/scala.scala": "Scala",
     "docs/scss.scss": "SCSS",
+    "docs/sass.sass": "SASS",
     "docs/sh.sh": "SH",
     "docs/stylus.styl": "Stylus",
     "docs/sql.sql": {name: "SQL", wrapped: true},

demo/kitchen-sink/docs/pascal.pas

@@ -0,0 +1,48 @@
+(*****************************************************************************
+ * A simple bubble sort program.  Reads integers, one per line, and prints   *
+ * them out in sorted order.  Blows up if there are more than 49.            *
+ *****************************************************************************)
+PROGRAM Sort(input, output);
+    CONST
+        (* Max array size. *)
+        MaxElts = 50;
+    TYPE 
+        (* Type of the element array. *)
+        IntArrType = ARRAY [1..MaxElts] OF Integer;
+
+    VAR
+        (* Indexes, exchange temp, array size. *)
+        i, j, tmp, size: integer;
+
+        (* Array of ints *)
+        arr: IntArrType;
+
+    (* Read in the integers. *)
+    PROCEDURE ReadArr(VAR size: Integer; VAR a: IntArrType); 
+        BEGIN
+            size := 1;
+            WHILE NOT eof DO BEGIN
+                readln(a[size]);
+                IF NOT eof THEN 
+                    size := size + 1
+            END
+        END;
+
+    BEGIN
+        (* Read *)
+        ReadArr(size, arr);
+
+        (* Sort using bubble sort. *)
+        FOR i := size - 1 DOWNTO 1 DO
+            FOR j := 1 TO i DO 
+                IF arr[j] > arr[j + 1] THEN BEGIN
+                    tmp := arr[j];
+                    arr[j] := arr[j + 1];
+                    arr[j + 1] := tmp;
+                END;
+
+        (* Print. *)
+        FOR i := 1 TO size DO
+            writeln(arr[i])
+    END.
+            

demo/kitchen-sink/docs/sass.sass

@@ -0,0 +1,39 @@
+// sass ace mode;
+
+@import url(http://fonts.googleapis.com/css?family=Ace:700)
+
+html, body
+  :background-color #ace
+  text-align: center
+  height: 100%
+  /*;*********;
+    ;comment  ;
+    ;*********;
+
+.toggle
+  $size: 14px
+
+  :background url(http://subtlepatterns.com/patterns/dark_stripes.png)
+  border-radius: 8px
+  height: $size
+
+  &:before
+    $radius: $size * 0.845
+    $glow: $size * 0.125
+
+    box-shadow: 0 0 $glow $glow / 2 #fff
+    border-radius: $radius
+    
+    &:active
+      ~ .button
+        box-shadow: 0 15px 25px -4px rgba(0,0,0,0.4)      
+      ~ .label
+        font-size: 40px
+        color: rgba(0,0,0,0.45)
+
+    &:checked      
+      ~ .button
+        box-shadow: 0 15px 25px -4px #ace
+      ~ .label
+        font-size: 40px
+        color: #c9c9c9

demo/kitchen-sink/modelist.js

@@ -72,6 +72,7 @@ var modesByName = {
     markdown:   ["Markdown"     , "md|markdown"],
     objectivec: ["Objective-C"  , "m"],
     ocaml:      ["OCaml"        , "ml|mli"],
+    pascal:     ["Pascal"       , "pas|p"],
     perl:       ["Perl"         , "pl|pm"],
     pgsql:      ["pgSQL"        , "pgsql"],
     php:        ["PHP"          , "php|phtml"],
@@ -83,7 +84,8 @@ var modesByName = {
     ruby:       ["Ruby"         , "ru|gemspec|rake|rb"],
     scad:       ["OpenSCAD"     , "scad"],
     scala:      ["Scala"        , "scala"],
-    scss:       ["SCSS"         , "scss|sass"],
+    scss:       ["SCSS"         , "scss"],
+    sass:       ["SASS"         , "sass"],
     sh:         ["SH"           , "sh|bash|bat"],
     sql:        ["SQL"          , "sql"],
     stylus:     ["Stylus"       , "styl|stylus"],

demo/kitchen-sink/util.js

@@ -40,6 +40,10 @@ var Renderer = require("ace/virtual_renderer").VirtualRenderer;
 var Editor = require("ace/editor").Editor;
 var MultiSelect = require("ace/multi_select").MultiSelect;
 
+exports.createEditor = function(el) {
+    return new Editor(new Renderer(el));
+}
+
 exports.createSplitEditor = function(el) {
     if (typeof(el) == "string")
         el = document.getElementById(el);
@@ -54,8 +58,8 @@ exports.createSplitEditor = function(el) {
     el.style.position = "relative";
     var split = {$container: el};
 
-    split.editor0 = split[0] = new Editor(new Renderer(e0, require("ace/theme/textmate")));
+    split.editor0 = split[0] = new Editor(new Renderer(e0));
-    split.editor1 = split[1] = new Editor(new Renderer(e1, require("ace/theme/textmate")));
+    split.editor1 = split[1] = new Editor(new Renderer(e1));
     split.splitter = s;
 
     MultiSelect(split.editor0);

lib/ace/background_tokenizer.js

@@ -34,8 +34,6 @@ define(function(require, exports, module) {
 var oop = require("./lib/oop");
 var EventEmitter = require("./lib/event_emitter").EventEmitter;
 
-// tokenizing lines longer than this makes editor very slow
-var MAX_LINE_LENGTH = 5000;
 
 /**
  * 
@@ -226,15 +224,7 @@ var BackgroundTokenizer = function(tokenizer, editor) {
         var line = this.doc.getLine(row);
         var state = this.states[row - 1];
 
-        if (line.length > MAX_LINE_LENGTH) {
-            var overflow = {value: line.substr(MAX_LINE_LENGTH), type: "text"};
-            line = line.slice(0, MAX_LINE_LENGTH);
-        }
         var data = this.tokenizer.getLineTokens(line, state, row);
-        if (overflow) {
-            data.tokens.push(overflow);
-            data.state = "start";
-        }
 
         if (this.states[row] + "" !== data.state + "") {
             this.states[row] = data.state;

lib/ace/background_tokenizer_test.js

@@ -62,19 +62,19 @@ module.exports = {
         doc.setMode("./mode/javascript")  
         
         forceTokenize(doc)
-        testStates(doc, ["comment", "start", "start"])
+        testStates(doc, ["comment_regex_allowed", "start", "no_regex"])
         
         doc.remove(new Range(0,2,1,2))
-        testStates(doc, [null, "start"])
+        testStates(doc, [null, "no_regex"])
         
         forceTokenize(doc)
-        testStates(doc, ["comment", "comment"])
+        testStates(doc, ["comment_regex_allowed", "comment_regex_allowed"])
         
         doc.insert({row:0, column:2}, "\n*/")
-        testStates(doc, [undefined, undefined, "comment"])
+        testStates(doc, [undefined, undefined, "comment_regex_allowed"])
         
         forceTokenize(doc)
-        testStates(doc, ["comment", "start", "start"])
+        testStates(doc, ["comment_regex_allowed", "start", "no_regex"])
     }
 };
 

lib/ace/keyboard/vim/commands.js

@@ -310,7 +310,7 @@ var actions = exports.actions = {
         fn: function(editor, range, count, param) {
             editor.modifyNumber(count || 1);
         }
-    },
+    }
 };
 
 var inputBuffer = exports.inputBuffer = {

lib/ace/mode/_test/text_markdown.txt

@@ -12,8 +12,9 @@ test: # followed be only space is not a valid header
 test: only space between #s is not a valid header
 #  #
 
-test links  [Cloud9 IDE](http://www.c9.io/) 
+# test links  [Cloud9 IDE](http://www.c9.io/) #
 * [demo](http://ajaxorg.github.com/ace/) 
 in lists
 
 in plain text <b>http://ace.ajaxorg.com<b>
+

lib/ace/mode/_test/tokens_csharp.json

@@ -20,7 +20,9 @@
   ["punctuation.operator","."],
   ["identifier","WriteLine"],
   ["paren.lparen","("],
-  ["string","\"Hello World\""],
+  ["string.start","\""],
+  ["string","Hello World"],
+  ["string.end","\""],
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[

lib/ace/mode/_test/tokens_javascript.json

@@ -2,7 +2,7 @@
    "start",
   ["comment","//test: tokenize 'standard' functions"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","string"],
   ["punctuation.operator","."],
   ["support.function","charCodeAt"],
@@ -28,7 +28,7 @@
   ["punctuation.operator",";"],
   ["string","\";"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","test"],
   ["punctuation.operator",":"],
   ["text"," "],
@@ -36,15 +36,15 @@
   ["text"," "],
   ["identifier","comment"]
 ],[
-   "start",
+   "no_regex",
   ["comment.doc","/**tokenize doc comment with "],
   ["comment.doc.tag","@tag"],
   ["comment.doc"," {}*/"]
 ],[
-   "start",
+   "no_regex",
   ["comment","//test: tokenize parens"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","    "],
   ["storage.type","var"],
   ["text"," "],
@@ -58,34 +58,34 @@
    "start",
   ["comment","//test tokenize arithmetic expression which looks like a regexp"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","a"],
   ["keyword.operator","/"],
   ["identifier","b"],
   ["keyword.operator","/"],
   ["identifier","c"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","a"],
   ["keyword.operator","/="],
   ["identifier","b"],
   ["keyword.operator","/"],
   ["identifier","c"]
 ],[
-   "start",
+   "no_regex",
   ["comment","//test tokenize reg exps"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","a"],
   ["keyword.operator","="],
   ["string.regexp","/b/g"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","a"],
   ["keyword.operator","+"],
   ["string.regexp","/b/g"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","a"],
   ["text"," "],
   ["keyword.operator","="],
@@ -98,7 +98,7 @@
   ["constant.language.escape","+"],
   ["string.regexp"," 1/b"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","a"],
   ["keyword.operator","="],
   ["string.regexp","/a/"],
@@ -107,7 +107,7 @@
   ["text"," "],
   ["string.regexp","/a/"]
 ],[
-   "start",
+   "no_regex",
   ["keyword","case"],
   ["text"," "],
   ["string.regexp","/a/"],
@@ -117,31 +117,31 @@
   ["identifier","c"],
   ["paren.rparen",")"]
 ],[
-   "start",
+   "no_regex",
   ["comment","//test tokenize multi-line comment containing a single line comment"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","noRegex"]
 ],[
-   "start",
+   "no_regex",
   ["comment","/* foo // bar */"]
 ],[
-   "regex_allowed",
+   "start",
   ["identifier","canBeRegex"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["comment","/* foo // bar */"]
 ],[
    "start",
   ["comment","// test tokenize identifier with umlauts"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","fu"],
   ["punctuation.operator","?"],
   ["identifier","e"]
 ],[
-   "start",
+   "no_regex",
   ["comment","// test // is not a regexp"]
 ],[
    "start",
@@ -152,12 +152,12 @@
    "start",
   ["comment","//test skipping escaped chars"]
 ],[
-   "start",
+   "no_regex",
   ["string","'Meh"],
   ["constant.language.escape","\\\\"],
   ["string","nNeh'"]
 ],[
-   "start",
+   "no_regex",
   ["storage.type","console"],
   ["punctuation.operator","."],
   ["support.function.firebug","log"],
@@ -169,10 +169,10 @@
    "qqstring",
   ["string","\"test multiline\\"]
 ],[
-   "start",
+   "no_regex",
   ["string"," strings\""]
 ],[
-   "start",
+   "no_regex",
   ["identifier","a"],
   ["keyword.operator","="],
   ["text","'"]
@@ -182,16 +182,16 @@
   ["keyword.operator","="],
   ["string","\"\\"]
 ],[
-   "start",
+   "no_regex",
   ["string","still a string"]
 ],[
-   "start",
+   "no_regex",
+  ["text"," "]
+],[
+   "no_regex",
   ["text"," "]
 ],[
    "start",
-  ["text"," "]
-],[
-   "regex_allowed",
   ["storage.type","function"],
   ["text"," "],
   ["entity.name.function","foo"],
@@ -203,7 +203,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","    "],
   ["keyword","for"],
   ["text"," "],
@@ -228,7 +228,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","        "],
   ["support.function","alert"],
   ["paren.lparen","("],
@@ -245,18 +245,18 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","    "],
   ["paren.rparen","}"],
   ["text","\t"],
   ["comment","// Real Tab."]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "start",
+   "no_regex",
   ["identifier","regexp"],
   ["text"," "],
   ["keyword.operator","="],
@@ -267,9 +267,9 @@
   ["text"," "],
   ["comment","// ends here"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "start",
+   "no_regex",
   ["identifier","r"],
   ["text"," "],
   ["keyword.operator","="],
@@ -316,7 +316,7 @@
   ["text"," "],
   ["identifier","o"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","a"],
   ["keyword.operator","="],
   ["string.regexp","/a/"],
@@ -331,16 +331,16 @@
   ["text"," "],
   ["string.regexp","/ /"]
 ],[
-   "start",
+   "no_regex",
   ["text"," "],
   ["comment.doc","/************************************/"]
 ],[
-   "start",
+   "no_regex",
   ["comment.doc","/** total mess, tricky to highlight**/"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "regex_allowed",
+   "start",
   ["storage.type","function"],
   ["text"," "],
   ["paren.lparen","("],
@@ -355,10 +355,10 @@
    "doc-start",
   ["comment.doc","\t * docComment"]
 ],[
-   "start",
+   "no_regex",
   ["comment.doc","\t **/"]
 ],[
-   "start",
+   "no_regex",
   ["text","\t"],
   ["identifier","r"],
   ["text"," "],
@@ -369,7 +369,7 @@
   ["constant.language.escape","*"],
   ["string.regexp","/"]
 ],[
-   "start",
+   "no_regex",
   ["text","\t"],
   ["identifier","g"],
   ["text"," "],
@@ -401,7 +401,7 @@
   ["text"," "],
   ["constant.numeric","0x25"]
 ],[
-   "start",
+   "no_regex",
   ["text","\t"],
   ["identifier","t"],
   ["text"," "],
@@ -414,10 +414,10 @@
   ["string","''"],
   ["paren.rparen","]"]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"]
 ],[
-   "regex_allowed",
+   "start",
   ["storage.type","function"],
   ["text"," "],
   ["paren.lparen","("],
@@ -425,50 +425,50 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","\t"],
   ["comment","/* eee */"]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"]
 ],[
-   "start"
+   "no_regex"
 ],[
    "qqstring",
   ["string","\"s\\"]
 ],[
-   "start",
+   "no_regex",
   ["string","s"],
   ["constant.language.escape","\\u7824"],
   ["string","sss"],
   ["constant.language.escape","\\u"],
   ["string","1\""]
 ],[
-   "start"
+   "no_regex"
 ],[
    "qstring",
   ["string","'\\"]
 ],[
-   "start",
+   "no_regex",
   ["string","string'"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "start",
+   "no_regex",
   ["text","'"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","string"],
   ["text","'"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "start",
+   "no_regex",
   ["string","\"trailing space"],
   ["constant.language.escape","\\ "],
   ["string","  "]
 ],[
-   "start",
+   "no_regex",
   ["string","\"         \""],
   ["text","    "],
   ["keyword.operator","/"],
@@ -480,7 +480,7 @@
   ["keyword.operator","/"],
   ["identifier","g"]
 ],[
-   "start"
+   "no_regex"
 ],[
    "doc-start",
   ["comment.doc","/**"]
@@ -488,19 +488,19 @@
    "doc-start",
   ["comment.doc"," *doc"]
 ],[
-   "start",
+   "no_regex",
   ["comment.doc"," */"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "regex_allowed",
+   "start",
   ["identifier","a"],
   ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","\t"],
   ["string","'a'"],
   ["punctuation.operator",":"],
@@ -508,7 +508,7 @@
   ["identifier","b"],
   ["punctuation.operator",","]
 ],[
-   "start",
+   "no_regex",
   ["text","\t"],
   ["string","'g'"],
   ["text",":"],
@@ -518,7 +518,7 @@
   ["variable.parameter","t"],
   ["paren.rparen",")"]
 ],[
-   "start",
+   "no_regex",
   ["text","\t"],
   ["entity.name.function","gta"],
   ["punctuation.operator",":"],
@@ -529,12 +529,12 @@
   ["variable.parameter","b"],
   ["paren.rparen",")"]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "start"
+   "no_regex"
 ],[
    "function_arguments",
   ["identifier","foo"],
@@ -552,14 +552,14 @@
   ["variable.parameter","b"],
   ["punctuation.operator",","]
 ],[
-   "start",
+   "no_regex",
   ["punctuation.operator","                          "],
   ["variable.parameter","c"],
   ["punctuation.operator",", "],
   ["variable.parameter","d"],
   ["paren.rparen",")"]
 ],[
-   "start",
+   "no_regex",
   ["storage.type","foo"],
   ["punctuation.operator","."],
   ["entity.name.function","d"],
@@ -572,7 +572,7 @@
   ["variable.parameter","b"],
   ["paren.rparen",")"]
 ],[
-   "start",
+   "no_regex",
   ["storage.type","foo"],
   ["punctuation.operator","."],
   ["entity.name.function","d"],
@@ -588,5 +588,5 @@
   ["string","\"string\""],
   ["text","   "]
 ],[
-   "start"
+   "no_regex"
 ]]

lib/ace/mode/_test/tokens_jsp.json

@@ -15,7 +15,7 @@
   ["meta.tag.tag-name.script","script"],
   ["meta.tag.r",">"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["storage.type","var"],
   ["text"," "],
@@ -26,7 +26,7 @@
   ["string","\"abc\""],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["storage.type","function"],
   ["text"," "],
@@ -34,7 +34,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","        "],
   ["paren.rparen","}"]
 ],[

lib/ace/mode/_test/tokens_markdown.json

@@ -3,25 +3,29 @@
   ["text","test: header 1 "]
 ],[
    "start",
-  ["markup.heading.1","#f"]
+  ["markup.heading.1","#"],
+  ["markup.heading","f"]
 ],[
    "start",
   ["text","test: header 2"]
 ],[
    "start",
-  ["markup.heading.2","## foo"]
+  ["markup.heading.2","##"],
+  ["markup.heading"," foo"]
 ],[
    "start",
   ["text","test: header ends with ' #'"]
 ],[
    "start",
-  ["markup.heading.1","# # # "]
+  ["markup.heading.1","#"],
+  ["markup.heading"," # # "]
 ],[
    "start",
   ["text","test: header ends with '#'"]
 ],[
    "start",
-  ["markup.heading.1","#foo# "]
+  ["markup.heading.1","#"],
+  ["markup.heading","foo# "]
 ],[
    "start",
   ["text","test: 6+ #s is not a valid header"]
@@ -41,13 +45,14 @@
    "allowBlock"
 ],[
    "start",
-  ["text","test links  "],
+  ["markup.heading.1","#"],
+  ["markup.heading"," test links  "],
   ["text","["],
   ["string","Cloud9 IDE"],
   ["text","]("],
   ["markup.underline","http://www.c9.io/"],
   ["text",")"],
-  ["text"," "]
+  ["markup.heading"," #"]
 ],[
    "listblock",
   ["markup.list","* "],
@@ -72,4 +77,8 @@
   ["meta.tag","<"],
   ["meta.tag.tag-name","b"],
   ["meta.tag.r",">"]
+],[
+   "allowBlock"
+],[
+   "start"
 ]]

lib/ace/mode/_test/tokens_svg.json

@@ -50,7 +50,7 @@
 ],[
    "start"
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","  "],
   ["meta.tag","<"],
   ["meta.tag.tag-name","script"],
@@ -64,7 +64,7 @@
   ["identifier","CDATA"],
   ["paren.lparen","["]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","    "],
   ["storage.type","var"],
   ["text"," "],
@@ -75,7 +75,7 @@
   ["constant.numeric","0"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","    "],
   ["storage.type","var"],
   ["text"," "],
@@ -86,7 +86,7 @@
   ["constant.numeric","1"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","    "],
   ["storage.type","var"],
   ["text"," "],
@@ -97,28 +97,28 @@
   ["constant.numeric","100"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","    "],
   ["storage.type","var"],
   ["text"," "],
   ["identifier","hickory"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","    "],
   ["storage.type","var"],
   ["text"," "],
   ["identifier","dickory"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","    "],
   ["storage.type","var"],
   ["text"," "],
   ["identifier","dock"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","    "],
   ["storage.type","var"],
   ["text"," "],
@@ -127,7 +127,7 @@
 ],[
    "js-start"
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","    "],
   ["storage.type","function"],
   ["text"," "],
@@ -138,7 +138,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["identifier","hickory"],
   ["text","  "],
@@ -156,7 +156,7 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["identifier","dickory"],
   ["text"," "],
@@ -174,7 +174,7 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["identifier","dock"],
   ["text"," "],
@@ -194,18 +194,18 @@
 ],[
    "js-start"
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["identifier","ShowAndGrowElement"],
   ["paren.lparen","("],
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","    "],
   ["paren.rparen","}"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","    "],
   ["storage.type","function"],
   ["text"," "],
@@ -215,7 +215,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["identifier","timevalue"],
   ["text"," "],
@@ -228,7 +228,7 @@
   ["identifier","timer_increment"],
   ["punctuation.operator",";"]
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","        "],
   ["keyword","if"],
   ["text"," "],
@@ -240,7 +240,7 @@
   ["identifier","max_time"],
   ["paren.rparen",")"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","            "],
   ["keyword","return"],
   ["punctuation.operator",";"]
@@ -249,7 +249,7 @@
   ["text","        "],
   ["comment","// Scale the text string gradually until it is 20 times larger"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["identifier","scalefactor"],
   ["text"," "],
@@ -270,7 +270,7 @@
 ],[
    "js-start"
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["keyword","if"],
   ["text"," "],
@@ -284,7 +284,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","            "],
   ["identifier","hickory"],
   ["punctuation.operator","."],
@@ -297,7 +297,7 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","            "],
   ["identifier","hickory"],
   ["punctuation.operator","."],
@@ -327,13 +327,13 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","        "],
   ["paren.rparen","}"]
 ],[
-   "js-start"
+   "js-no_regex"
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["keyword","if"],
   ["text"," "],
@@ -355,7 +355,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","            "],
   ["identifier","dickory"],
   ["punctuation.operator","."],
@@ -368,7 +368,7 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","            "],
   ["identifier","dickory"],
   ["punctuation.operator","."],
@@ -395,11 +395,11 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","        "],
   ["paren.rparen","}"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","        "],
   ["keyword","if"],
   ["text"," "],
@@ -413,7 +413,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","            "],
   ["identifier","dock"],
   ["punctuation.operator","."],
@@ -426,7 +426,7 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "js-regex_allowed",
+   "js-start",
   ["text","            "],
   ["identifier","dock"],
   ["punctuation.operator","."],
@@ -455,17 +455,17 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","        "],
   ["paren.rparen","}"]
 ],[
-   "js-start"
+   "js-no_regex"
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","        "],
   ["comment","// Call ShowAndGrowElement again <timer_increment> milliseconds later."]
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","        "],
   ["identifier","setTimeout"],
   ["paren.lparen","("],
@@ -475,11 +475,11 @@
   ["identifier","timer_increment"],
   ["paren.rparen",")"]
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","    "],
   ["paren.rparen","}"]
 ],[
-   "js-start",
+   "js-no_regex",
   ["text","    "],
   ["variable.language","window"],
   ["punctuation.operator","."],

lib/ace/mode/_test/tokens_typescript.json

@@ -1,19 +1,19 @@
 [[
-   "regex_allowed",
+   "start",
   ["keyword.operator.ts","class"],
   ["text"," "],
   ["identifier","Greeter"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","\t"],
   ["variable.parameter.function.ts","greeting"],
   ["text",": "],
   ["variable.parameter.function.ts","string"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","\t"],
   ["keyword.operator.ts","constructor"],
   ["text"," "],
@@ -25,7 +25,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","\t\t"],
   ["storage.type.variable.ts","this."],
   ["identifier","greeting"],
@@ -35,19 +35,19 @@
   ["identifier","message"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","\t"],
   ["paren.rparen","}"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","\t"],
-  ["entity.name.function.ts","greet"],
+  ["identifier","greet"],
   ["paren.lparen","("],
   ["paren.rparen",")"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","\t\t"],
   ["keyword","return"],
   ["text"," "],
@@ -59,17 +59,17 @@
   ["identifier","greeting"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","\t"],
   ["paren.rparen","}"]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"],
   ["text","   "]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "regex_allowed",
+   "start",
   ["storage.type","var"],
   ["text"," "],
   ["identifier","greeter"],
@@ -86,7 +86,7 @@
 ],[
    "start"
 ],[
-   "start",
+   "no_regex",
   ["storage.type","var"],
   ["text"," "],
   ["identifier","button"],
@@ -100,7 +100,7 @@
   ["string","'button'"],
   ["paren.rparen",")"]
 ],[
-   "start",
+   "no_regex",
   ["identifier","button"],
   ["punctuation.operator","."],
   ["identifier","innerText"],
@@ -109,7 +109,7 @@
   ["text"," "],
   ["string","\"Say Hello\""]
 ],[
-   "regex_allowed",
+   "start",
   ["storage.type","button"],
   ["punctuation.operator","."],
   ["entity.name.function","onclick"],
@@ -122,7 +122,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "start",
+   "no_regex",
   ["text","\t"],
   ["support.function","alert"],
   ["paren.lparen","("],
@@ -131,12 +131,12 @@
   ["paren.rparen",")"],
   ["paren.rparen",")"]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "start",
+   "no_regex",
   ["variable.language","document"],
   ["punctuation.operator","."],
   ["identifier","body"],
@@ -146,20 +146,20 @@
   ["identifier","button"],
   ["paren.rparen",")"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "regex_allowed",
+   "start",
-  ["keyword.operator.ts","class"],
+  ["keyword","class"],
   ["text"," "],
   ["identifier","Snake"],
   ["text"," "],
-  ["keyword.operator.ts","extends"],
+  ["keyword","extends"],
   ["text"," "],
   ["identifier","Animal"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","   "],
   ["entity.name.function.ts","move"],
   ["paren.lparen","("],
@@ -167,7 +167,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
   ["support.function","alert"],
   ["paren.lparen","("],
@@ -175,7 +175,7 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
   ["storage.type.variable.ts","super"],
   ["text","("],
@@ -183,27 +183,27 @@
   ["text",")"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","   "],
   ["paren.rparen","}"]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "regex_allowed",
+   "start",
-  ["keyword.operator.ts","class"],
+  ["keyword","class"],
   ["text"," "],
   ["identifier","Horse"],
   ["text"," "],
-  ["keyword.operator.ts","extends"],
+  ["keyword","extends"],
   ["text"," "],
   ["identifier","Animal"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","   "],
   ["entity.name.function.ts","move"],
   ["paren.lparen","("],
@@ -211,7 +211,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
   ["support.function","alert"],
   ["paren.lparen","("],
@@ -219,7 +219,7 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
   ["keyword.operator.ts","super"],
   ["punctuation.operator","."],
@@ -229,22 +229,23 @@
   ["paren.rparen",")"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","   "],
   ["paren.rparen","}"]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"]
 ],[
-   "start"
+   "no_regex"
 ],[
    "start",
-  ["keyword.operator.ts","module"],
+  ["identifier","module"],
   ["text"," "],
-  ["variable.parameter.function.ts","Sayings"],
+  ["identifier","Sayings"],
-  ["text"," {"]
+  ["text"," "],
+  ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","    "],
   ["keyword.operator.ts","export"],
   ["text"," "],
@@ -254,14 +255,14 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","        "],
   ["variable.parameter.function.ts","greeting"],
   ["text",": "],
   ["variable.parameter.function.ts","string"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","        "],
   ["keyword.operator.ts","constructor"],
   ["text"," "],
@@ -273,7 +274,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","            "],
   ["storage.type.variable.ts","this."],
   ["identifier","greeting"],
@@ -283,19 +284,19 @@
   ["identifier","message"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","        "],
   ["paren.rparen","}"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","        "],
-  ["entity.name.function.ts","greet"],
+  ["identifier","greet"],
   ["paren.lparen","("],
   ["paren.rparen",")"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","            "],
   ["keyword","return"],
   ["text"," "],
@@ -307,24 +308,25 @@
   ["identifier","greeting"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","        "],
   ["paren.rparen","}"]
 ],[
-   "start",
+   "no_regex",
   ["text","    "],
   ["paren.rparen","}"]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"]
 ],[
    "start",
-  ["keyword.operator.ts","module"],
+  ["identifier","module"],
   ["text"," "],
-  ["variable.parameter.function.ts","Mankala"],
+  ["identifier","Mankala"],
-  ["text"," {"]
+  ["text"," "],
+  ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","   "],
   ["keyword.operator.ts","export"],
   ["text"," "],
@@ -334,7 +336,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
   ["keyword.operator.ts","public"],
   ["text"," "],
@@ -345,7 +347,7 @@
   ["constant.language.boolean","false"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
   ["keyword.operator.ts","public"],
   ["text"," "],
@@ -356,7 +358,7 @@
   ["constant.numeric","0"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
   ["keyword.operator.ts","public"],
   ["text"," "],
@@ -367,7 +369,7 @@
   ["constant.numeric","0"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
   ["keyword.operator.ts","public"],
   ["text"," "],
@@ -380,7 +382,7 @@
 ],[
    "start"
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
   ["keyword.operator.ts","public"],
   ["text"," "],
@@ -390,7 +392,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","           "],
   ["storage.type.variable.ts","this."],
   ["identifier","turnContinues"],
@@ -400,7 +402,7 @@
   ["constant.language.boolean","false"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","           "],
   ["storage.type.variable.ts","this."],
   ["identifier","seedStoredCount"],
@@ -410,7 +412,7 @@
   ["constant.numeric","0"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","           "],
   ["storage.type.variable.ts","this."],
   ["identifier","capturedCount"],
@@ -420,7 +422,7 @@
   ["constant.numeric","0"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","           "],
   ["storage.type.variable.ts","this."],
   ["identifier","spaceCaptured"],
@@ -430,23 +432,23 @@
   ["identifier","NoSpace"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","       "],
   ["paren.rparen","}"]
 ],[
-   "start"
+   "no_regex"
 ],[
-   "regex_allowed",
+   "start",
   ["text","       "],
-  ["keyword.operator.ts","public"],
+  ["keyword","public"],
   ["text"," "],
-  ["entity.name.function.ts","toString"],
+  ["identifier","toString"],
   ["paren.lparen","("],
   ["paren.rparen",")"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","           "],
   ["storage.type","var"],
   ["text"," "],
@@ -457,7 +459,7 @@
   ["string","\"\""],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","           "],
   ["keyword","if"],
   ["text"," "],
@@ -468,7 +470,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","               "],
   ["identifier","stringBuilder"],
   ["text"," "],
@@ -477,11 +479,11 @@
   ["string","\" turn continues,\""],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","           "],
   ["paren.rparen","}"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","           "],
   ["identifier","stringBuilder"],
   ["text"," "],
@@ -495,7 +497,7 @@
   ["identifier","seedStoredCount"],
   ["punctuation.operator",";"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","           "],
   ["keyword","if"],
   ["text"," "],
@@ -510,7 +512,7 @@
   ["text"," "],
   ["paren.lparen","{"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","               "],
   ["identifier","stringBuilder"],
   ["text"," "],
@@ -533,25 +535,25 @@
   ["identifier","spaceCaptured"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","           "],
   ["paren.rparen","}"]
 ],[
-   "regex_allowed",
+   "start",
   ["text","           "],
   ["keyword","return"],
   ["text"," "],
   ["identifier","stringBuilder"],
   ["punctuation.operator",";"]
 ],[
-   "start",
+   "no_regex",
   ["text","       "],
   ["paren.rparen","}"]
 ],[
-   "start",
+   "no_regex",
   ["text","   "],
   ["paren.rparen","}"]
 ],[
-   "start",
+   "no_regex",
   ["paren.rparen","}"]
 ]]

lib/ace/mode/abap.js

@@ -39,7 +39,7 @@ var TextMode = require("./text").Mode;
 var oop = require("../lib/oop");
 
 function Mode() {
-    this.$tokenizer = new Tokenizer(new Rules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new Rules().getRules());
     this.foldingRules = new FoldMode();
 }
 

lib/ace/mode/abap_highlight_rules.js

@@ -114,16 +114,17 @@ var AbapHighlightRules = function() {
             {token : "keyword", regex : compoundKeywords}, 
             {token : "variable.parameter", regex : /\w+-\w+(?:-\w+)*/}, 
             {token : keywordMapper, regex : "\\b\\w+\\b"},
+            {caseInsensitive: true}
         ],
         "qstring" : [
             {token : "constant.language.escape",   regex : "''"},
             {token : "string", regex : "'",     next  : "start"},
-            {token : "string", regex : ".|\w+"}
+            {defaultToken : "string"}
         ],
         "string" : [
             {token : "constant.language.escape",   regex : "``"},
             {token : "string", regex : "`",     next  : "start"},
-            {token : "string", regex : ".|\w+"}
+            {defaultToken : "string"}
         ]
     }
 };

lib/ace/mode/c9search.js

@@ -39,7 +39,7 @@ var MatchingBraceOutdent = require("./matching_brace_outdent").MatchingBraceOutd
 var C9StyleFoldMode = require("./folding/c9search").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new C9SearchHighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new C9SearchHighlightRules().getRules());
     this.$outdent = new MatchingBraceOutdent();
     this.foldingRules = new C9StyleFoldMode();
 };

lib/ace/mode/coffee_highlight_rules.js

@@ -99,7 +99,7 @@ define(function(require, exports, module) {
                     token : "string", regex : "'''", next : [
                         {token : "string", regex : "'''", next : "start"},
                         {token : "constant.language.escape", regex : stringEscape},
-                        {defaultToken: "string"},
+                        {defaultToken: "string"}
                     ]
                 }, {
                     stateName: "qqdoc",
@@ -115,21 +115,21 @@ define(function(require, exports, module) {
                     token : "string", regex : "'", next : [
                         {token : "string", regex : "'", next : "start"},
                         {token : "constant.language.escape", regex : stringEscape},
-                        {defaultToken: "string"},
+                        {defaultToken: "string"}
                     ]
                 }, {
                     stateName: "qqstring",
                     token : "string.start", regex : '"', next : [
                         {token : "string.end", regex : '"', next : "start"},
                         {token : "constant.language.escape", regex : stringEscape},
-                        {defaultToken: "string"},
+                        {defaultToken: "string"}
                     ]
                 }, {
                     stateName: "js",
                     token : "string", regex : "`", next : [
                         {token : "string", regex : "`", next : "start"},
                         {token : "constant.language.escape", regex : stringEscape},
-                        {defaultToken: "string"},
+                        {defaultToken: "string"}
                     ]
                 }, {
                     token : "string.regex",

lib/ace/mode/csharp_highlight_rules.js

@@ -30,11 +30,17 @@ var CSharpHighlightRules = function() {
                 token : "string.regexp",
                 regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
             }, {
-                token : "string", // single line
+                token : "string", // character
-                regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
+                regex : /'(?:.|\\(:?u[\da-fA-F]+|x[\da-fA-F]+|[tbrf'"n]))'/
             }, {
-                token : "string", // single line
+                token : "string", start : '"', end : '"|$', next: [
-                regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
+                    {token: "constant.language.escape", regex: /\\(:?u[\da-fA-F]+|x[\da-fA-F]+|[tbrf'"n])/},
+                    {token: "invalid", regex: /\\./}
+                ]
+            }, {
+                token : "string", start : '@"', end : '"', next:[
+                    {token: "constant.language.escape", regex: '""'}
+                ]
             }, {
                 token : "constant.numeric", // hex
                 regex : "0[xX][0-9a-fA-F]+\\b"
@@ -46,8 +52,6 @@ var CSharpHighlightRules = function() {
                 regex : "(?:true|false)\\b"
             }, {
                 token : keywordMapper,
-                // TODO: Unicode escape sequences
-                // TODO: Unicode identifiers
                 regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
             }, {
                 token : "keyword.operator",
@@ -80,6 +84,7 @@ var CSharpHighlightRules = function() {
 
     this.embedRules(DocCommentHighlightRules, "doc-",
         [ DocCommentHighlightRules.getEndRule("start") ]);
+    this.normalizeRules();
 };
 
 oop.inherits(CSharpHighlightRules, TextHighlightRules);

lib/ace/mode/css.js

@@ -41,7 +41,7 @@ var CssBehaviour = require("./behaviour/css").CssBehaviour;
 var CStyleFoldMode = require("./folding/cstyle").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new CssHighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new CssHighlightRules().getRules());
     this.$outdent = new MatchingBraceOutdent();
     this.$behaviour = new CssBehaviour();
     this.foldingRules = new CStyleFoldMode();

lib/ace/mode/css_highlight_rules.js

@@ -89,9 +89,14 @@ var CssHighlightRules = function() {
         }, {
             token : ["punctuation", "entity.other.attribute-name.pseudo-class.css"],
             regex : pseudoClasses
+        }, {
+            token : ["support.function", "string", "support.function"],
+            regex : "(url\\()(.*)(\\))"
         }, {
             token : keywordMapper,
             regex : "\\-?[a-zA-Z_][a-zA-Z0-9_\\-]*"
+        }, {
+            caseInsensitive: true
         }
       ];
 
@@ -160,6 +165,8 @@ var CssHighlightRules = function() {
         },{
             token: "constant",
             regex: "[a-z0-9-_]+"
+        },{
+            caseInsensitive: true
         }],
 
         "media" : [ {
@@ -186,6 +193,8 @@ var CssHighlightRules = function() {
         },{
             token: "constant",
             regex: "[a-z0-9-_]+"
+        },{
+            caseInsensitive: true
         }],
 
         "comment" : comment,

lib/ace/mode/diff.js

@@ -38,7 +38,7 @@ var HighlightRules = require("./diff_highlight_rules").DiffHighlightRules;
 var FoldMode = require("./folding/diff").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new HighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new HighlightRules().getRules());
     this.foldingRules = new FoldMode(["diff", "index", "\\+{3}", "@@|\\*{5}"], "i");
 };
 oop.inherits(Mode, TextMode);

lib/ace/mode/diff_highlight_rules.js

@@ -92,7 +92,8 @@ var DiffHighlightRules = function() {
                 regex: "\\s*$",
                 token: "invalid"
             }, {
-                defaultToken: "invisible"
+                defaultToken: "invisible",
+                caseInsensitive: true
             }
         ]
     };

lib/ace/mode/javascript.js

@@ -91,13 +91,13 @@ oop.inherits(Mode, TextMode);
             return indent;
         }
         
-        if (state == "start" || state == "regex_allowed") {
+        if (state == "start" || state == "no_regex") {
             var match = line.match(/^.*(?:\bcase\b.*\:|[\{\(\[])\s*$/);
             if (match) {
                 indent += tab;
             }
         } else if (state == "doc-start") {
-            if (endState == "start" || state == "regex_allowed") {
+            if (endState == "start" || endState == "no_regex") {
                 return "";
             }
             var match = line.match(/^\s*(\/?)\*/);

lib/ace/mode/javascript_highlight_rules.js

@@ -83,7 +83,7 @@ var JavaScriptHighlightRules = function() {
     // regexps are ordered -> the first match is used
 
     this.$rules = {
-        "start" : [
+        "no_regex" : [
             {
                 token : "comment",
                 regex : /\/\/.*$/
@@ -165,7 +165,7 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token : "keyword",
                 regex : "(?:" + kwBeforeRe + ")\\b",
-                next : "regex_allowed"
+                next : "start"
             }, {
                 token : ["punctuation.operator", "support.function"],
                 regex : /(\.)(s(?:h(?:ift|ow(?:Mod(?:elessDialog|alDialog)|Help))|croll(?:X|By(?:Pages|Lines)?|Y|To)?|t(?:opzzzz|rike)|i(?:n|zeToContent|debar|gnText)|ort|u(?:p|b(?:str(?:ing)?)?)|pli(?:ce|t)|e(?:nd|t(?:Re(?:sizable|questHeader)|M(?:i(?:nutes|lliseconds)|onth)|Seconds|Ho(?:tKeys|urs)|Year|Cursor|Time(?:out)?|Interval|ZOptions|Date|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Date|FullYear)|FullYear|Active)|arch)|qrt|lice|avePreferences|mall)|h(?:ome|andleEvent)|navigate|c(?:har(?:CodeAt|At)|o(?:s|n(?:cat|textual|firm)|mpile)|eil|lear(?:Timeout|Interval)?|a(?:ptureEvents|ll)|reate(?:StyleSheet|Popup|EventObject))|t(?:o(?:GMTString|S(?:tring|ource)|U(?:TCString|pperCase)|Lo(?:caleString|werCase))|est|a(?:n|int(?:Enabled)?))|i(?:s(?:NaN|Finite)|ndexOf|talics)|d(?:isableExternalCapture|ump|etachEvent)|u(?:n(?:shift|taint|escape|watch)|pdateCommands)|j(?:oin|avaEnabled)|p(?:o(?:p|w)|ush|lugins.refresh|a(?:ddings|rse(?:Int|Float)?)|r(?:int|ompt|eference))|e(?:scape|nableExternalCapture|val|lementFromPoint|x(?:p|ec(?:Script|Command)?))|valueOf|UTC|queryCommand(?:State|Indeterm|Enabled|Value)|f(?:i(?:nd|le(?:ModifiedDate|Size|CreatedDate|UpdatedDate)|xed)|o(?:nt(?:size|color)|rward)|loor|romCharCode)|watch|l(?:ink|o(?:ad|g)|astIndexOf)|a(?:sin|nchor|cos|t(?:tachEvent|ob|an(?:2)?)|pply|lert|b(?:s|ort))|r(?:ou(?:nd|teEvents)|e(?:size(?:By|To)|calc|turnValue|place|verse|l(?:oad|ease(?:Capture|Events)))|andom)|g(?:o|et(?:ResponseHeader|M(?:i(?:nutes|lliseconds)|onth)|Se(?:conds|lection)|Hours|Year|Time(?:zoneOffset)?|Da(?:y|te)|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Da(?:y|te)|FullYear)|FullYear|A(?:ttention|llResponseHeaders)))|m(?:in|ove(?:B(?:y|elow)|To(?:Absolute)?|Above)|ergeAttributes|a(?:tch|rgins|x))|b(?:toa|ig|o(?:ld|rderWidths)|link|ack))\b(?=\()/
@@ -184,22 +184,22 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token : "keyword.operator",
                 regex : /--|\+\+|[!$%&*+\-~]|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\|\||\?\:|\*=|%=|\+=|\-=|&=|\^=/,
-                next  : "regex_allowed"
+                next  : "start"
             }, {
                 token : "punctuation.operator",
                 regex : /\?|\:|\,|\;|\./,
-                next  : "regex_allowed"
+                next  : "start"
             }, {
                 token : "paren.lparen",
                 regex : /[\[({]/,
-                next  : "regex_allowed"
+                next  : "start"
             }, {
                 token : "paren.rparen",
                 regex : /[\])}]/
             }, {
                 token : "keyword.operator",
                 regex : /\/=?/,
-                next  : "regex_allowed"
+                next  : "start"
             }, {
                 token: "comment",
                 regex: /^#!.*$/
@@ -207,7 +207,7 @@ var JavaScriptHighlightRules = function() {
         ],
         // regular expressions are only allowed after certain tokens. This
         // makes sure we don't mix up regexps with the divison operator
-        "regex_allowed": [
+        "start": [
             DocCommentHighlightRules.getStartRule("doc-start"),
             {
                 token : "comment", // multi line comment
@@ -215,20 +215,22 @@ var JavaScriptHighlightRules = function() {
                 next : "comment_regex_allowed"
             }, {
                 token : "comment",
-                regex : "\\/\\/.*$"
+                regex : "\\/\\/.*$",
+                next : "start"
             }, {
                 token: "string.regexp",
                 regex: "\\/",
                 next: "regex",
             }, {
                 token : "text",
-                regex : "\\s+"
+                regex : "\\s+|^$",
+                next : "start"
             }, {
                 // immediately return to the start mode without matching
                 // anything
                 token: "empty",
                 regex: "",
-                next: "start"
+                next: "no_regex"
             }
         ],
         "regex": [
@@ -240,7 +242,7 @@ var JavaScriptHighlightRules = function() {
                 // flag
                 token: "string.regexp",
                 regex: "/\\w*",
-                next: "start",
+                next: "no_regex",
             }, {
                 // invalid operators
                 token : "invalid",
@@ -259,7 +261,7 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token: "empty",
                 regex: "$",
-                next: "start"
+                next: "no_regex"
             }, {
                 defaultToken: "string.regexp"
             }
@@ -278,7 +280,7 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token: "empty",
                 regex: "$",
-                next: "start"
+                next: "no_regex"
             }, {
                 defaultToken: "string.regexp.charachterclass"
             }
@@ -296,15 +298,15 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token: "empty",
                 regex: "",
-                next: "start"
+                next: "no_regex"
             }
         ],
         "comment_regex_allowed" : [
-            {token : "comment", regex : "\\*\\/", next : "regex_allowed"},
+            {token : "comment", regex : "\\*\\/", next : "start"},
             {defaultToken : "comment"}
         ],
         "comment" : [
-            {token : "comment", regex : "\\*\\/", next : "start"},
+            {token : "comment", regex : "\\*\\/", next : "no_regex"},
             {defaultToken : "comment"}
         ],
         "qqstring" : [
@@ -318,7 +320,7 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token : "string",
                 regex : '"|$',
-                next  : "start",
+                next  : "no_regex",
             }, {
                 defaultToken: "string"
             }
@@ -334,7 +336,7 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token : "string",
                 regex : "'|$",
-                next  : "start",
+                next  : "no_regex",
             }, {
                 defaultToken: "string"
             }
@@ -342,7 +344,7 @@ var JavaScriptHighlightRules = function() {
     };
 
     this.embedRules(DocCommentHighlightRules, "doc-",
-        [ DocCommentHighlightRules.getEndRule("start") ]);
+        [ DocCommentHighlightRules.getEndRule("no_regex") ]);
 };
 
 oop.inherits(JavaScriptHighlightRules, TextHighlightRules);

lib/ace/mode/javascript_test.js

@@ -115,7 +115,7 @@ module.exports = {
     "test: no auto indent should add to existing indent" : function() {
         assert.equal("      ", this.mode.getNextLineIndent("start", "    if () {", "  "));
         assert.equal("    ", this.mode.getNextLineIndent("start", "    cde", "  "));
-        assert.equal("    ", this.mode.getNextLineIndent("regex_allowed", "function foo(items) {", "    "));
+        assert.equal("    ", this.mode.getNextLineIndent("start", "function foo(items) {", "    "));
     },
 
     "test: special indent in doc comments" : function() {

lib/ace/mode/less.js

@@ -40,7 +40,7 @@ var CssBehaviour = require("./behaviour/css").CssBehaviour;
 var CStyleFoldMode = require("./folding/cstyle").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new LessHighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new LessHighlightRules().getRules());
     this.$outdent = new MatchingBraceOutdent();
     this.$behaviour = new CssBehaviour();
     this.foldingRules = new CStyleFoldMode();

lib/ace/mode/less_highlight_rules.js

@@ -247,6 +247,8 @@ var LessHighlightRules = function() {
             }, {
                 token : "text",
                 regex : "\\s+"
+            }, {
+                caseInsensitive: true
             }
         ],
         "comment" : [

lib/ace/mode/livescript.js

@@ -1,4 +1,4 @@
-define('ace/mode/ls', function(require, exports, module){
+define(function(require, exports, module){
   var identifier, LiveScriptMode, keywordend, stringfill;
   identifier = '(?![\\d\\s])[$\\w\\xAA-\\uFFDC](?:(?!\\s)[$\\w\\xAA-\\uFFDC]|-[A-Za-z])*';
   exports.Mode = LiveScriptMode = (function(superclass){

lib/ace/mode/markdown_highlight_rules.js

@@ -100,11 +100,12 @@ var MarkdownHighlightRules = function() {
         }, { // h2
             token: "markup.heading.2",
             regex: "^\\-+(?=\\s*$)"
-        }, { // header
+        }, {
             token : function(value) {
-                return "markup.heading." + value.search(/[^#]/);
+                return "markup.heading." + value.length;
             },
-            regex : "^#{1,6}(?:[^ #].*| +.*(?:[^ #].*|[^ ]+.* +#+ *))$"
+            regex : /^#{1,6}(?=\s*[^ #]|\s+#.)/,
+            next : "header"
         },
            github_embed("(?:javascript|js)", "js-"),
            github_embed("xml", "xml-"),
@@ -130,6 +131,15 @@ var MarkdownHighlightRules = function() {
             include : "basic"
         }],
         
+        "header" : [{
+            regex: "$",
+            next : "start"
+        }, {
+            include: "basic"
+        }, {
+            defaultToken : "markup.heading"
+        } ],
+
         "listblock-start" : [{
             token : "support.variable",
             regex : /(?:\[[ x]\])?/,

lib/ace/mode/pascal.js

@@ -0,0 +1,62 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2012, Ajax.org B.V.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *
+ * Contributor(s):
+ * 
+ *
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/*
+  THIS FILE WAS AUTOGENERATED BY mode.tmpl.js
+*/
+
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextMode = require("./text").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var PascalHighlightRules = require("./pascal_highlight_rules").PascalHighlightRules;
+// TODO: pick appropriate fold mode
+var FoldMode = require("./folding/coffee").FoldMode;
+
+var Mode = function() {
+    var highlighter = new PascalHighlightRules();
+    this.foldingRules = new FoldMode();
+    this.$tokenizer = new Tokenizer(highlighter.getRules());
+};
+oop.inherits(Mode, TextMode);
+
+(function() {
+    // Extra logic goes here. 
+}).call(Mode.prototype);
+
+exports.Mode = Mode;
+});

lib/ace/mode/pascal_highlight_rules.js

@@ -0,0 +1,127 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2012, Ajax.org B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/* THIS FILE WAS AUTOGENERATED FROM tool\tm bundles\pascal.tmbundle\Syntaxes\Pascal.plist (UUID: F42FA544-6B1C-11D9-9517-000D93589AF6) */
+/****************************************************************
+ * IT MIGHT NOT BE PERFECT, PARTICULARLY:                       *
+ * IN DECIDING STATES TO TRANSITION TO,                         *
+ * IGNORING WHITESPACE,                                         *
+ * IGNORING GROUPS WITH ?:,                                     *
+ * EXTENDING EXISTING MODES,                                    *
+ * GATHERING KEYWORDS, OR                                       *
+ * DECIDING WHEN TO USE PUSH.                                   *
+ * ...But it's a good start from an existing *.tmlanguage file. *
+ ****************************************************************/
+
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var PascalHighlightRules = function() {
+    // regexp must not have capturing parentheses. Use (?:) instead.
+    // regexps are ordered -> the first match is used
+
+    this.$rules = { start: 
+       [ { caseInsensitive: true,
+           token: 'keyword.control.pascal',
+           regex: '\\b(?:(absolute|abstract|all|and|and_then|array|as|asm|attribute|begin|bindable|case|class|const|constructor|destructor|div|do|do|else|end|except|export|exports|external|far|file|finalization|finally|for|forward|goto|if|implementation|import|in|inherited|initialization|interface|interrupt|is|label|library|mod|module|name|near|nil|not|object|of|only|operator|or|or_else|otherwise|packed|pow|private|program|property|protected|public|published|qualified|record|repeat|resident|restricted|segment|set|shl|shr|then|to|try|type|unit|until|uses|value|var|view|virtual|while|with|xor))\\b' },
+         { caseInsensitive: true,           
+           token: 
+            [ 'variable.pascal', "text",
+              'storage.type.prototype.pascal',
+              'entity.name.function.prototype.pascal' ],
+           regex: '\\b(function|procedure)(\\s+)(\\w+)(\\.\\w+)?(?=(?:\\(.*?\\))?;\\s*(?:attribute|forward|external))' },
+         { token: 'keyword.operator',
+           regex: '[+\\-;,/*%]|:=|=' },
+         { caseInsensitive: true,
+           token: 
+            [ 'variable.pascal', "text",
+              'storage.type.function.pascal',
+              'entity.name.function.pascal' ],
+           regex: '\\b(function|procedure)(\\s+)(\\w+)(\\.\\w+)?' },
+         { token: 'constant.numeric.pascal',
+           regex: '\\b((0(x|X)[0-9a-fA-F]*)|(([0-9]+\\.?[0-9]*)|(\\.[0-9]+))((e|E)(\\+|-)?[0-9]+)?)(L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b' },
+         { token: 'punctuation.definition.comment.pascal',
+           regex: '--',
+           push: 
+            [ { token: 'comment.line.double-dash.pascal.one',
+                regex: '$',
+                next: 'pop' },
+              { defaultToken: 'comment.line.double-dash.pascal.one' } ] },
+         { token: 'punctuation.definition.comment.pascal',
+           regex: '//',
+           push: 
+            [ { token: 'comment.line.double-slash.pascal.two',
+                regex: '$',
+                next: 'pop' },
+              { defaultToken: 'comment.line.double-slash.pascal.two' } ] },
+         { token: 'punctuation.definition.comment.pascal',
+           regex: '\\(\\*',
+           push: 
+            [ { token: 'punctuation.definition.comment.pascal',
+                regex: '\\*\\)',
+                next: 'pop' },
+              { defaultToken: 'comment.block.pascal.one' } ] },
+         { token: 'punctuation.definition.comment.pascal',
+           regex: '\\{',
+           push: 
+            [ { token: 'punctuation.definition.comment.pascal',
+                regex: '\\}',
+                next: 'pop' },
+              { defaultToken: 'comment.block.pascal.two' } ] },
+         { token: 'punctuation.definition.string.begin.pascal',
+           regex: '"',
+           push: 
+            [ { token: 'constant.character.escape.pascal', regex: '\\\\.' },
+              { token: 'punctuation.definition.string.end.pascal',
+                regex: '"',
+                next: 'pop' },
+              { defaultToken: 'string.quoted.double.pascal' } ],
+           //Double quoted strings are an extension and (generally) support C-style escape sequences.
+            },
+         { token: 'punctuation.definition.string.begin.pascal',
+           regex: '\'',
+           push: 
+            [ { token: 'constant.character.escape.apostrophe.pascal',
+                regex: '\'\'' },
+              { token: 'punctuation.definition.string.end.pascal',
+                regex: '\'',
+                next: 'pop' },
+              { defaultToken: 'string.quoted.single.pascal' } ] } ] }
+    
+    this.normalizeRules();
+};
+
+oop.inherits(PascalHighlightRules, TextHighlightRules);
+
+exports.PascalHighlightRules = PascalHighlightRules;
+});

lib/ace/mode/php_highlight_rules.js

@@ -899,11 +899,7 @@ var PhpLangHighlightRules = function() {
         "start" : [
             {
                 token : "comment",
-                regex : "\\/\\/.*$"
+                regex : /(?:#|\/\/)(?:[^?]|\?[^>])*/
-            },
-            {
-               token : "comment",
-               regex : "#.*$"
             },
             docComment.getStartRule("doc-start"),
             {
@@ -1058,8 +1054,8 @@ var PhpHighlightRules = function() {
     for (var i in this.$rules) {
         this.$rules[i].unshift({
             token : "support.php_tag", // php open tag
-            regex : "<\\?(?:php|\\=)?",
+            regex : "<\\?(?:php|=)?",
-            next  : "php-start"
+            push  : "php-start"
         });
     }
 
@@ -1068,8 +1064,9 @@ var PhpHighlightRules = function() {
     this.$rules["php-start"].unshift({
         token : "support.php_tag", // php close tag
         regex : "\\?>",
-        next  : "start"
+        next  : "pop"
     });
+    this.normalizeRules();
 };
 
 oop.inherits(PhpHighlightRules, HtmlHighlightRules);

lib/ace/mode/ruby_highlight_rules.js

@@ -174,11 +174,11 @@ var RubyHighlightRules = function() {
             }, {
                 stateName: "heredoc",
                 token : function(value, currentState, stack) {
-                    var next = value[3] == '-' ? "heredoc" : "indentedHeredoc";
+                    var next = value[2] == '-' ? "indentedHeredoc" : "heredoc";
                     var tokens = value.split(this.splitRegex);
                     stack.push(next, tokens[3]);
                     return [
-                        {type:"constant", value: "<<"},
+                        {type:"constant", value: tokens[1]},
                         {type:"string", value: tokens[2]},
                         {type:"support.class", value: tokens[3]},
                         {type:"string", value: tokens[4]}

lib/ace/mode/sass.js

@@ -0,0 +1,52 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2010, Ajax.org B.V.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextMode = require("./text").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var SassHighlightRules = require("./sass_highlight_rules").SassHighlightRules;
+var FoldMode = require("./folding/coffee").FoldMode;
+
+var Mode = function() {
+    this.$tokenizer = new Tokenizer(new SassHighlightRules().getRules());
+    this.foldingRules = new FoldMode();
+};
+oop.inherits(Mode, TextMode);
+
+(function() {
+    
+}).call(Mode.prototype);
+
+exports.Mode = Mode;
+
+});

lib/ace/mode/sass_highlight_rules.js

@@ -0,0 +1,79 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2010, Ajax.org B.V.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var lang = require("../lib/lang");
+var ScssHighlightRules = require("./scss_highlight_rules").ScssHighlightRules;
+
+var SassHighlightRules = function() {
+    ScssHighlightRules.call(this);
+    var start = this.$rules.start;
+    if (start[1].token == "comment") {
+        start.splice(1, 1, {
+            token: function(value, currentState, stack) {
+                stack.unshift(this.next, value.length - 2, currentState);
+                return "comment";
+            },
+            regex: /^\s*\/\*/,
+            next: "comment"
+        }, {
+            token: "error.invalid",
+            regex: "/\\*|[{;}]"
+        }, {
+            token: "support.type",
+            regex: /^\s*:[\w\-]+\s/,
+        });
+        
+        this.$rules.comment = [
+            {regex: /^\s*/, token: function(value, currentState, stack) {
+                if (value.length < stack[1]) {
+                    stack.shift();
+                    stack.shift();
+                    this.next = stack.shift();
+                    return "text";
+                } else {
+                    stack[1] = value.length;
+                    this.next = "";
+                    return "comment";
+                }
+            }, next: "start"},
+            {defaultToken: "comment"}
+        ]
+    }
+};
+
+oop.inherits(SassHighlightRules, ScssHighlightRules);
+
+exports.SassHighlightRules = SassHighlightRules;
+
+});

lib/ace/mode/scss.js

@@ -40,7 +40,7 @@ var CssBehaviour = require("./behaviour/css").CssBehaviour;
 var CStyleFoldMode = require("./folding/cstyle").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new ScssHighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new ScssHighlightRules().getRules());
     this.$outdent = new MatchingBraceOutdent();
     this.$behaviour = new CssBehaviour();
     this.foldingRules = new CStyleFoldMode();

lib/ace/mode/scss_highlight_rules.js

@@ -66,8 +66,8 @@ var ScssHighlightRules = function() {
             "border-color|border-left-color|border-left-style|border-left-width|" +
             "border-left|border-right-color|border-right-style|border-right-width|" +
             "border-right|border-spacing|border-style|border-top-color|" +
-            "border-top-style|border-top-width|border-top|border-width|border|" +
+            "border-top-style|border-top-width|border-top|border-width|border|bottom|" +
-            "bottom|box-sizing|caption-side|clear|clip|color|content|counter-increment|" +
+            "box-shadow|box-sizing|caption-side|clear|clip|color|content|counter-increment|" +
             "counter-reset|cue-after|cue-before|cue|cursor|direction|display|" +
             "elevation|empty-cells|float|font-family|font-size-adjust|font-size|" +
             "font-stretch|font-style|font-variant|font-weight|font|height|left|" +
@@ -204,6 +204,9 @@ var ScssHighlightRules = function() {
             }, {
                 token : "constant.numeric",
                 regex : numRe
+            }, {
+                token : ["support.function", "string", "support.function"],
+                regex : "(url\\()(.*)(\\))"
             }, {
                 token : function(value) {
                     if (properties.hasOwnProperty(value.toLowerCase()))
@@ -249,6 +252,8 @@ var ScssHighlightRules = function() {
             }, {
                 token : "text",
                 regex : "\\s+"
+            }, {
+                caseInsensitive: true
             }
         ],
         "comment" : [

lib/ace/mode/text.js

@@ -65,7 +65,7 @@ var Mode = function() {
     };
 
     this.getNextLineIndent = function(state, line, tab) {
-        return "";
+        return this.$getIndent(line);
     };
 
     this.checkOutdent = function(state, line, input) {

lib/ace/mode/text_highlight_rules.js

@@ -43,8 +43,7 @@ var TextHighlightRules = function() {
             token : "empty_line",
             regex : '^$'
         }, {
-            token : "text",
+            defaultToken : "text"
-            regex : ".+"
         }]
     };
 };
@@ -96,35 +95,85 @@ var TextHighlightRules = function() {
         return this.$embeds;
     };
 
+    var pushState = function(currentState, stack) {
+        if (currentState != "start")
+            stack.unshift(this.nextState, currentState);
+        return this.nextState;
+    };
+    var popState = function(currentState, stack) {
+        if (stack[0] !== currentState)
+            return "start";
+        stack.shift();
+        return stack.shift();
+    };
+
     this.normalizeRules = function() {
         var id = 0;
-        for (var key in this.$rules) {
+        var rules = this.$rules;
-            var state = this.$rules[key];
+        function processState(key) {
+            var state = rules[key];
+            state.processed = true;
             for (var i = 0; i < state.length; i++) {
                 var rule = state[i];
-                if (rule.next && Array.isArray(rule.next)) {
+                if (!rule.regex && rule.start) {
-                    var stateName = rule.stateName || ("state" + id++);
+                    rule.regex = rule.start;
-                    this.$rules[stateName] = rule.next;
+                    if (!rule.next)
-                    rule.next = stateName;
+                        rule.next = [];
+                    rule.next.push({
+                        defaultToken: rule.token
+                    }, {
+                        token: rule.token + ".end",
+                        regex: rule.end || rule.start,
+                        next: "pop"
+                    });
+                    rule.token = rule.token + ".start";
+                    rule.push = true;
                 }
+                var next = rule.next || rule.push;
+                if (next && Array.isArray(next)) {
+                    var stateName = rule.stateName || (rule.token + id++);
+                    rules[stateName] = next;
+                    rule.next = stateName;
+                    processState(stateName);
+                } else if (next == "pop") {
+                    rule.next = popState;
+                }
+
+                if (rule.push) {
+                    rule.nextState = rule.next || rule.push;
+                    rule.next = pushState;
+                    delete rule.push;
+                }
+
                 if (rule.rules) {
                     for (var r in rule.rules) {
-                        if (this.$rules[r]) {
+                        if (rules[r]) {
-                            if (this.$rules[r].push)
+                            if (rules[r].push)
-                                this.$rules[r].push.apply(this.$rules[r], rule.rules[r]);
+                                rules[r].push.apply(rules[r], rule.rules[r]);
                         } else {
-                            this.$rules[r] = rule.rules[r];
+                            rules[r] = rule.rules[r];
                         }
                     }
                 }
                 if (rule.include || typeof rule == "string") {
-                    var args = [i, 1].concat(this.$rules[rule.include || rule]);
+                    var includeName = rule.include || rule;
+                    var toInsert = rules[includeName];
+                } else if (Array.isArray(rule))
+                    toInsert = rule;
+
+                if (toInsert) {
+                    var args = [i, 1].concat(toInsert);
                     if (rule.noEscape)
                         args = args.filter(function(x) {return !x.next;});
                     state.splice.apply(state, args);
+                    // skip included rules since they are already processed
+                    //i += args.length - 3;
+                    i--;
+                    toInsert = null
                 }
             }
-        }
+        };
+        Object.keys(rules).forEach(processState);
     };
 
     this.createKeywordMapper = function(map, defaultToken, ignoreCase, splitChar) {

lib/ace/mode/tmsnippet.js

@@ -46,19 +46,19 @@ var SnippetHighlightRules = function() {
             {regex: /\[/, token: "regex.start", next: "charClass"},
             {regex: "/", token: "string.regex", next: "format"},
             //{"default": "string.regex"},
-            {"token": "string.regex", regex:"."},
+            {"token": "string.regex", regex:"."}
         ],
         charClass : [
             {regex: "\\.", token: "escape"},
             {regex: "\\]", token: "regex.end", next: "regexp"},
-            {"token": "string.regex", regex:"."},
+            {"token": "string.regex", regex:"."}
         ],
         "format" : [
             {regex: /\\[ulULE]/, token: "keyword"},
             {regex: /\$\d+/, token: "variable"},
             {regex: "/[gim]*:?", token: "string.regex", next: "start"},
             // {"default": "string"},
-            {"token": "string", regex:"."},
+            {"token": "string", regex:"."}
         ]
     };
 };

lib/ace/search_test.js

@@ -451,7 +451,7 @@ module.exports = {
         assert.position(ranges[1].end, 0, 11);
         assert.position(ranges[0].start, 0, 0);
         assert.position(ranges[0].end, 0, 3);
-    },
+    }
 };
 
 });

lib/ace/tokenizer.js

@@ -31,6 +31,8 @@
 define(function(require, exports, module) {
 "use strict";
 
+// tokenizing lines longer than this makes editor very slow
+var MAX_TOKEN_COUNT = 1000;
 /**
  *
  *
@@ -41,15 +43,10 @@ define(function(require, exports, module) {
 /**
  * Constructs a new tokenizer based on the given rules and flags.
  * @param {Object} rules The highlighting rules
- * @param {String} flag Any additional regular expression flags to pass (like "i" for case insensitive)
- *
- *
- *
  *
  * @constructor
  **/
-var Tokenizer = function(rules, flag) {
+var Tokenizer = function(rules) {
-    flag = flag ? "g" + flag : "g";
     this.states = rules;
 
     this.regExps = {};
@@ -59,13 +56,17 @@ var Tokenizer = function(rules, flag) {
         var ruleRegExps = [];
         var matchTotal = 0;
         var mapping = this.matchMappings[key] = {defaultToken: "text"};
+        var flag = "g";
 
         for (var i = 0; i < state.length; i++) {
             var rule = state[i];
-            if (rule.defaultToken) {
+            if (rule.defaultToken)
                 mapping.defaultToken = rule.defaultToken;
+            if (rule.caseInsensitive)
+                flag = "gi";
+            if (rule.regex == null)
                 continue;
-            }
+            
             if (rule.regex instanceof RegExp)
                 rule.regex = rule.regex.toString().slice(1, -1);
 
@@ -110,12 +111,12 @@ var Tokenizer = function(rules, flag) {
     this.$arrayTokens = function(str) {
         if (!str)
             return [];
-        var values = str.split(this.splitRegex)
+        var values = this.splitRegex.exec(str);
         var tokens = [];
         var types = this.tokenArray;
-        if (types.length != values.length - 2) {
+        if (types.length != values.length - 1) {
             if (window.console)
-                console.error(types.length , values.length - 2, str, this.splitRegex);
+                console.error(types , values, str, this.splitRegex, this);
             return [{type: "error.invalid", value: str}];
         }
         for (var i = 0; i < types.length; i++) {
@@ -138,8 +139,8 @@ var Tokenizer = function(rules, flag) {
     };
     
     this.createSplitterRegexp = function(src, flag) {
-        src = src.replace(/\(\?=([^()]|\\.)*?\)$/, "");
+        src = src.replace(/\(\?=([^()]|\\.|\(([^()]|\\.)*?\))*\)(?=\)*$)/, "");
-        return new RegExp(src, flag);
+        return new RegExp(src, (flag||"").replace("g", ""));
     };
 
     /**
@@ -193,7 +194,11 @@ var Tokenizer = function(rules, flag) {
                     : rule.token;
 
                 if (rule.next) {
+                    if (typeof rule.next == "string")
                         currentState = rule.next;
+                    else
+                        currentState = rule.next(currentState, stack);
+
                     state = this.states[currentState];
                     if (!state) {
                         window.console && console.error && console.error(currentState, "doesn't exist");
@@ -230,6 +235,12 @@ var Tokenizer = function(rules, flag) {
                 break;
 
             lastIndex = index;
+
+            if (tokens.length > MAX_TOKEN_COUNT) {
+                token.value += line.substr(lastIndex);
+                currentState = "start"
+                break;
+            }
         }
 
         if (token.type)

lib/ace/tokenizer_dev.js

@@ -30,13 +30,15 @@
 
 define(function(require, exports, module) {
 
+// tokenizing lines longer than this makes editor very slow
+var MAX_TOKEN_COUNT = 1000;
 /*
  * version of Tokenizer with additional logging
  * and infinite loop checks
  * can be used for developing/testing new modes
  **/
-var Tokenizer = function(rules, flag) {
+
-    flag = flag ? "g" + flag : "g";
+var Tokenizer = function(rules) {
     this.states = rules;
 
     this.regExps = {};
@@ -46,13 +48,17 @@ var Tokenizer = function(rules, flag) {
         var ruleRegExps = [];
         var matchTotal = 0;
         var mapping = this.matchMappings[key] = {defaultToken: "default.text"};
+        var flag = "g";
 
         for (var i = 0; i < state.length; i++) {
             var rule = state[i];
-            if (rule.defaultToken) {
+            if (rule.defaultToken)
                 mapping.defaultToken = rule.defaultToken;
+            if (rule.caseInsensitive)
+                flag = "gi";
+            if (rule.regex == null)
                 continue;
-            }
+            
             if (rule.regex instanceof RegExp)
                 rule.regex = rule.regex.toString().slice(1, -1);
 
@@ -202,7 +208,11 @@ var Tokenizer = function(rules, flag) {
                     : rule.token;
 
                 if (rule.next) {
+                    if (typeof rule.next == "string")
                         currentState = rule.next;
+                    else
+                        currentState = rule.next(currentState, stack);
+
                     state = this.states[currentState];
                     if (!state) {
                         window.console && console.error && console.error(currentState, "doesn't exist");
@@ -241,6 +251,12 @@ var Tokenizer = function(rules, flag) {
                 break;
 
             lastIndex = index;
+
+            if (tokens.length > MAX_TOKEN_COUNT) {
+                token.value += line.substr(lastIndex);
+                currentState = "start"
+                break;
+            }
         }
 
         if (token.type)

tool/lib.js

@@ -0,0 +1,38 @@
+var plist = require("plist");
+var util = require("util");
+exports.parsePlist = function(themeXml, callback) {    
+    var result = ""
+    plist.parseString(themeXml, function(_, theme) {
+        result = theme[0];
+        callback && callback(theme[0]);
+    });
+    return result;
+}
+
+exports.formatJSON = function(object, initialIndent) {
+    return util.inspect(object, false, 40).replace(/^/gm, initialIndent||"")
+    
+}
+
+
+exports.fillTemplate = function(template, replacements) {
+    return template.replace(/%(.+?)%/g, function(str, m) {
+        return replacements[m] || "";
+    });
+}
+
+exports.hyphenate = function(str) {
+    return str.replace(/([A-Z])/g, "-$1").replace(/_/g, "-").toLowerCase();
+}
+
+exports.quoteString = function(str) {
+    return '"' + str.replace(/\\/, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\\n") + '"';
+}
+
+
+exports.restoreJSONComments = function(objStr) {
+    return objStr.replace(/^(\s*)comment: '(.*)'/gm, function(_, i, c) {
+        return i + "//" +  c.replace(/\\n(\\t)*/g, "\n" + i + "//") + "\n" + i
+    }).replace(/ \/\/ ERROR/g, '", // ERROR');
+}
+

tool/mode_creator.html

@@ -21,6 +21,34 @@
         .separator-h {
             padding: 0 20px;
         }
+        #closeBtn {
+            background: rgba(245, 146, 146, 0.5);
+            border: 1px solid #F48A8A;
+            border-radius: 50%;
+            padding: 7px;
+            position: absolute;
+            right: -8px;
+            top: -8px;
+            z-index: 1000;
+        }
+        #closeBtn:hover {
+            background: rgba(245, 146, 146, 0.9);
+        }
+        #console{
+            /border: 1px solid lightblue;
+            bottom: 0;
+            height: 80px;
+            margin: 0 4%;
+            position: absolute;
+            width: 92%;
+            z-index: 1000;
+            box-shadow: 0 0 1px 2px gray
+        }
+        #consoleEditor{
+            height: 100%;
+            position: relative;
+            width: 100%;
+        }
     </style>
     <link href="../doc/site/images/favicon.ico" rel="icon" type="image/x-icon">
 </head>
@@ -29,10 +57,13 @@
 <div id="header">
     <label for="modeEl">mode</label>
     <select id="modeEl" size="1"></select>
+    <input type="button" value="&#10227;" title="sync" id="syncToMode"></select>
     <span id="tablist"></span>
     <span class="separator-h"></span>
     <label for="autorunEl">live preview</label>
     <input type="checkbox" label="autorun" id="autorunEl" checked>
+    <span class="separator-h"></span>
+    <input type="button" value="measure speed" id="perfTest"></select>
 
 
     <div style='float:right'>
@@ -44,6 +75,10 @@
     </div>
 </div>
 <div id="editor"></div>
+<div id="console" style="display:none">
+    <span id="closeBtn" onclick="this.parentNode.style.display='none'"></span>
+    <div id="consoleEditor"></div>
+</div>
 
 
 <script type="text/javascript">

tool/mode_creator.js

@@ -14,6 +14,7 @@ var EditSession = require("ace/edit_session").EditSession;
 var UndoManager = require("ace/undomanager").UndoManager;
 
 var DebugTokenizer = require("ace/tokenizer_dev").Tokenizer;
+var Tokenizer = require("ace/tokenizer").Tokenizer;
 
 // createEditor
 var splitEditor = window.splitEditor = util.createSplitEditor("editor");
@@ -74,6 +75,33 @@ util.bindDropdown(modeEl, function(value) {
     });
 });
 
+document.getElementById("syncToMode").onclick = function() {
+    docEl.value = modelist.modesByName[modeEl.value].desc;
+    docEl.onchange();
+    run();
+}
+document.getElementById("perfTest").onclick = function() {
+    var lines = editor2.session.doc.getAllLines()
+    if (!lines.length)
+        return
+    while (lines.length < 1000) {
+        lines = lines.concat(lines)
+    }
+
+    var tk = new Tokenizer(currentRules);
+    var testPerf = function(lines, tk){
+        var state = "start"
+        for (var i=0, l = lines.length; i <l; i++) {
+            state = tk.getLineTokens(lines[i], state).state
+        }
+    }
+
+    var t = performance.now();
+    testPerf(lines, tk);
+    t = t - performance.now(t);
+    log("tokenized " + lines.length + " lines in " + t + " ms");
+}
+
 util.fillDropdown("themeEl", {
     bright: [
         "chrome", "clouds", "crimson_editor", "dawn", "dreamweaver", "eclipse", "github",
@@ -118,18 +146,26 @@ function run() {
     var path = "ace/mode/new";
     var deps = getDeps(src, path);
     src = src.replace("define(", 'define("' + path +'", ["require","exports","module",' + deps +'],');
-    src += ';require(["ace/mode/new"], continueRun, function(e){console.log(e);window.require.undef("ace/mode/new")})';
+    src += ';require(["ace/mode/new"], function(e) {\
+        try{continueRun(e)}catch(e){log(e)}\
+    }, function(e){\
+        log(e);\
+        window.require.undef("ace/mode/new")\
+    });';
     try {
         eval(src);
+        hideLog()
     } catch(e) {
-        console.log(e);
+        log(e);
     }
 }
+var currentRules
 var continueRun = function(rules) {
     rules = rules[Object.keys(rules)[0]];
+    currentRules = new rules().getRules()
     var Tokenizer = DebugTokenizer;
 
-    var tk = new Tokenizer(new rules().getRules());
+    var tk = new Tokenizer(currentRules);
     editor2.session.$mode.$tokenizer = tk;
     editor2.session.bgTokenizer.setTokenizer(tk);
     editor2.renderer.updateText();
@@ -137,5 +173,23 @@ var continueRun = function(rules) {
 
 editor1.commands.bindKey("ctrl-Return", run);
 
+var logEditor
+function log(e) {
+    console.log(e)
+    if (!logEditor) {
+        logEditor = util.createEditor(document.getElementById("consoleEditor"));
+        logEditor.session.setMode("ace/mode/javascript")
+        logEditor.session.setUseWorker(false)
+    }
+    logEditor.container.parentNode.style.display = '';
+    logEditor.resize()
+    logEditor.navigateFileEnd(e);
+    logEditor.insert(e + "\n");
+}
+function hideLog() {
+    if (logEditor)
+        logEditor.container.parentNode.style.display = 'none';
+}
+
 });
 

tool/mode_highlight_rules.tmpl.js

@@ -26,35 +26,19 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- *
- * Contributor(s):
- * 
- *
- *
  * ***** END LICENSE BLOCK ***** */
 
-/*
+/* THIS FILE WAS AUTOGENERATED FROM %name% (UUID: %uuid%) */
-  THIS FILE WAS AUTOGENERATED BY %name% (UUID: %uuid%) */
+/****************************************************************
-
+ * IT MIGHT NOT BE PERFECT, PARTICULARLY:                       *
-/*******
+ * IN DECIDING STATES TO TRANSITION TO,                         *
-
+ * IGNORING WHITESPACE,                                         *
-  THIS FILE MIGHT NOT BE PERFECT, PARTICULARLY:
+ * IGNORING GROUPS WITH ?:,                                     *
-
+ * EXTENDING EXISTING MODES,                                    *
-  IN DECIDING STATES TO TRANSITION TO,
+ * GATHERING KEYWORDS, OR                                       *
-
+ * DECIDING WHEN TO USE PUSH.                                   *
-  IGNORING WHITESPACE, 
+ * ...But it's a good start from an existing *.tmlanguage file. *
-
+ ****************************************************************/
-  IGNORING GROUPS WITH ?:, 
-
-  EXTENDING EXISTING MODES,
-
-  GATHERING KEYWORDS, OR
-
-  RULE PREFERENCE ORDER.
-
-  ...But it's a good start from an existing *.tmlanguage file.
-
-*******/
 
 define(function(require, exports, module) {
 "use strict";
@@ -63,14 +47,12 @@ var oop = require("../lib/oop");
 var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
 
 var %language%HighlightRules = function() {
-
     // regexp must not have capturing parentheses. Use (?:) instead.
     // regexps are ordered -> the first match is used
 
-    this.$rules = 
+    this.$rules = %languageTokens%
-        %languageTokens%
     
-    %repositoryRules%
+    this.normalizeRules();
 };
 
 oop.inherits(%language%HighlightRules, TextHighlightRules);

tool/tmlanguage.js

@@ -1,89 +1,110 @@
 var fs = require("fs"); 
 var util = require("util");
+var lib = require("./lib");
+var parseLanguage = lib.parsePlist;
 
-// for tracking token states
-var startState = { start: [] }, statesObj = { };
-
-
-var args = process.argv.splice(2);
-var tmLanguageFile  = args[0];
-var devMode         = args[1];
-
-var parseString = require("plist").parseString;
-function parseLanguage(languageXml, callback) {
-  parseString(languageXml, function(_, language) {
-    callback(language[0])
-  });
-}
 
 function logDebug(string, obj) {
     console.log(string, obj);
 }
 
-String.prototype.splice = function( idx, rem, s ) {
-    return (this.slice(0,idx) + s + this.slice(idx + Math.abs(rem)));
-};
 
-String.prototype.replaceAt = function (index, char) {
+// tmLanguage processor
-  return this.substr(0, index) + char + this.substr(index + 1);
+
+// for tracking token states
+var states = {start: []};
+var stateName = "start";
+
+function processRules(rules){
+    if (rules.patterns)
+        states.start = processPatterns(rules.patterns);
+    if (rules.repository)
+        processRepository(rules.repository);
+    return states;
+}
+function processRepository(r) {
+    for (var key in r) {
+        var p = r[key];
+        if (p.begin)
+            var stateObj = [processPattern(r[key])];
+        else if (p.patterns && !p.repository)
+            var stateObj = processPatterns(p.patterns);
+        else 
+            var stateObj = [processPattern(r[key])];
+
+        if (stateObj)
+            states["#" + key] = stateObj;
+    }
+}
+function processPatterns(pl) {
+    return pl.map(processPattern);
+}
+function processPattern(p) {
+
+    if (p.end == "(?!\\G)" && p.patterns && p.patterns.length == 1) {
+        var rule = processPattern(p.patterns[0]);
+    }
+    else if (p.begin && p.end) {
+        var rule = simpleRule(p.begin, p.name, p.beginCaptures || p.captures)
+        
+        var next = processPatterns(p.patterns || []);
+        var endRule = simpleRule(p.end, p.name, p.endCaptures || p.captures);
+        endRule.next = "pop";
+        next.push(endRule);
+        
+        if (p.name || p.contentName)
+            next.push({defaultToken: p.name || p.contentName});
+
+        rule.push = next;
+    }
+    else if (p.match) {
+        var rule = simpleRule(p.match, p.name, p.captures)
+    }
+    else if (p.include) {
+        var rule =  {include: p.include};
     }
     
-function keyCount(obj) {
+    if (p.comment)
-  return Object.keys(obj).length;
+        rule.comment = (rule.comment || "") +  p.comment;
+
+    if (p.repository)
+        processRepository(p.repository);
+    return rule;
 }
+function simpleRule(regex, name, captures) {
+    name = name || "text";
+    var rule = {};
     
-/**
+    regex = transformRegExp(regex, rule);
-
+    if (captures) {
-Scrubbing is sometimes necessary, but there appears to be no
+        var tokenArray = [];
-automated way to do it...
+        Object.keys(captures).forEach(function(x){
-
+            tokenArray[x] = captures[x] && captures[x].name;
-
+        });
-function cleanSingleCapture(match) {
+        if (tokenArray.length == 1) {
-  // if there's a single "( )", screw that and make it "(?: )"
+            name = tokenArray[0];
-  return match.replace("(", "(?:");
+        } else {
-}
+            for (var i = 0; i < tokenArray.length; i++)
-
+                if (!tokenArray[i])
-function cleanMultiCapture(match) {
+                    tokenArray[i] = name;
-  // regexp will be a quoted string, so turn "\" into "\\"
+            name = tokenArray;
-  var spaceFinderRegExp = new RegExp("\\\\s.| .", "g");
+            rule.todo = "fix grouping";
-  var m;
-  /* 
-   essentially turns things like
-   
-    \\s*(mixin) ([\\w\\-]+)\\s*(\\()
-   
-   into
-   
-   (\\s*mixin)( [\\w\\-]+)(\\s*\\()
-   
-   so that mode parser stops complaining
-  
-  while ((m = spaceFinderRegExp.exec(match)) != null) {
-    var idx = m.index;
-    var nextParenIdx = match.indexOf("(", idx);
-
-    if (nextParenIdx > idx) {
-      match = match.splice(idx, 0, "(").replaceAt(nextParenIdx + 1, '');
         }
     }
     
-  //console.log("match", match);
+    try {new RegExp(regex);} catch(e) {
-  return match;
+        rule.TODO = "FIXME: regexp doesn't have js equivalent";
     }
-*/
+    rule.token = name;
-
+    rule.regex = regex;
-// stupid yet necessary function, to transform JSON id comments into real comments
+    return rule;
-function restoreComments(objStr) {
-  return objStr.replace(/"\s+(\/\/.+)",/g, "\$1").replace(/ \/\/ ERROR/g, '", // ERROR');
 }
 
-function checkForLookBehind(str) {
+
-  var lookbehindRegExp = new RegExp("\\?<[=|!]", "g");
+// regex transformation
-  return lookbehindRegExp.test(str) ? str + " // ERROR: This contains a lookbehind, which JS does not support :(" : str;
-}
 
 function removeXFlag(str) {
-  if (str.slice(0,4) == "(?x)") {
+    if (str && str.slice(0,4) == "(?x)") {
         str = str.replace(/\\.|\[([^\]\\]|\\.)*?\]|\s+|(?:#[^\n]*)/g, function(s) {
             if (s[0] == "[")
                 return s;
@@ -95,149 +116,31 @@ function removeXFlag(str) {
     return str;
 }
 
-function transformRegExp(str) {
+function transformRegExp(str, rule) {
     str = removeXFlag(str);
-  str = checkForLookBehind(str);
+    str = str.replace(/\\n(?!\?).?/g, '$'); // replace newlines by $ except if its postfixed by ?
+    if (/\(\?[i]\:/g.test(str)) {
+        str = str.replace(/\(\?[ims\-]\:/g, "(?:"); // checkForInvariantRegex
+        rule && (rule.caseInsensitive = true);
+    }
     return str;
 }
 
-function assembleStateObjs(strState, pattern) {
+// 
-  var patterns = pattern.patterns;
+function extractPatterns(tmRules) {
-  var stateObj = {};
+    var patterns = processRules(tmRules);
-  var tokenElem = [];
+    return lib.restoreJSONComments(lib.formatJSON(patterns, "    "));
     
-  if (patterns) {
-    for (var p in patterns) {
-      stateObj = {}; // this is apparently necessary
-
-      if (patterns[p].include) {
-        stateObj.include = patterns[p].include;
-      }
-      else {
-        stateObj.token = patterns[p].name;
-        stateObj.regex = transformRegExp(patterns[p].match);
-      }
-      statesObj[strState].push(stateObj);
 }
 
-    stateObj = {};
-    stateObj.token = "TODO";
-    stateObj.regex = transformRegExp(pattern.end);
-    stateObj.next = "start";
-  }
-  else {
-    stateObj.token = "TODO";
-    stateObj.regex = transformRegExp(pattern.end);
-    stateObj.next = "start";
 
-    statesObj[strState].push(stateObj);
-
-    stateObj = {};
-    stateObj.token = "TODO";
-    stateObj.regex = ".+";
-    stateObj.next = strState;
-  }
-
-  return stateObj;
-}
-
-function extractPatterns(patterns) {
-  var state = 0;
-  patterns.forEach(function(pattern) {
-    state++;
-    var i = 1;
-    var tokenArray = [];
-    var tokenObj = { token: [] };
-    var stateObj = {};
-
-    if (pattern.comment) {
-      startState.start.push("          // " + pattern.comment.trim());
-    }
-
-    // it needs a state transition
-    if (pattern.begin && pattern.end) {
-      var strState = "state_" + state;
-      if ( pattern.beginCaptures === undefined && pattern.endCaptures === undefined) {
-        tokenObj.token.push(pattern.captures);
-      }
-      else if (pattern.beginCaptures) {
-        tokenObj.token.push(pattern.beginCaptures);
-      }
-      else if (pattern.endCaptures) {
-        tokenObj.token.push(pattern.endCaptures);
-      }
-
-      if (tokenObj.token === undefined) {
-        if (pattern.name)
-          tokenObj.token.push(pattern.name);
-        else 
-          logDebug("There's no token name for this state transition", pattern)
-      }
-
-      if (tokenObj.token === undefined) {
-        tokenObj.token.push(pattern.name);
-      }
-
-      statesObj[strState] = [ ];
-      statesObj[strState].push(assembleStateObjs(strState, pattern));
-      
-      tokenObj.regex = transformRegExp(pattern.begin);
-      tokenObj.next = strState;
-    }
-    else if( ( pattern.begin || pattern.end ) && !( pattern.begin && pattern.end ) ) {
-      logDebug("Somehow, there's pattern.begin or pattern.end--but not both?", pattern);
-    }
-
-    else if (pattern.captures) {
-      tokenObj.token.push([]);
-      tokenObj.token.push(pattern.captures);
-      tokenObj.regex = transformRegExp(pattern.match);
-    }
-
-    else if (pattern.match) {
-      tokenObj.token.push(pattern.name);
-      tokenObj.regex = transformRegExp(pattern.match);
-    }
-
-    else if (pattern.include) {
-      tokenObj.token.push(pattern.include);
-      tokenObj.regex = "";
-    }
-
-    else {
-      tokenObj.token.push("");
-      tokenObj.regex = "";
-      logDebug("I've gone through every choice, and have no clue what this is:", pattern);
-    }
-
-    // sometimes captures have names--not sure when or why
-    if (pattern.name) {
-      tokenObj.token.push(pattern.name);
-    }
-
-    startState.start.push(tokenObj);
-  });
-
-  var resultingObj = startState;
-
-  for (var state in statesObj) { 
-    resultingObj[state] = statesObj[state]; 
-  }
-
-  return restoreComments(JSON.stringify(resultingObj, null, "    "));
-}
-
-function fillTemplate(template, replacements) {
-    return template.replace(/%(.+?)%/g, function(str, m) {
-        return replacements[m] || "";
-    });
-}
 
+// cli stuff
 var modeTemplate = fs.readFileSync(__dirname + "/mode.tmpl.js", "utf8");
 var modeHighlightTemplate = fs.readFileSync(__dirname + "/mode_highlight_rules.tmpl.js", "utf8");
 
-function convertLanguage(name) {
+function convertLanguageFile(name) {
-    var tmLanguage = fs.readFileSync(__dirname + "/" + name, "utf8");
+    var tmLanguage = fs.readFileSync(process.cwd() + "/" + name, "utf8");
     parseLanguage(tmLanguage, function(language) {
         var languageHighlightFilename = language.name.replace(/[-_]/g, "").toLowerCase();
         var languageNameSanitized = language.name.replace(/-/g, "");
@@ -252,32 +155,23 @@ function convertLanguage(name) {
             console.log(util.inspect(language.repository, false, 4));
         }
         
-        var languageMode = fillTemplate(modeTemplate, {
+        var languageMode = lib.fillTemplate(modeTemplate, {
             language: languageNameSanitized,
             languageHighlightFilename: languageHighlightFilename
         });
 
-        var patterns = extractPatterns(language.patterns);
+        var patterns = extractPatterns(language);
-        var repository = {};
 
-        if (language.repository) {
+        var languageHighlightRules = lib.fillTemplate(modeHighlightTemplate, {
-          for (var r in language.repository) {
-            repository[r] = language.repository[r];
-          }
-          repository = restoreComments(JSON.stringify(repository, null, "    "));
-        }
-
-        var languageHighlightRules = fillTemplate(modeHighlightTemplate, {
             language: languageNameSanitized,
-              languageTokens: patterns,
+            languageTokens: patterns.trim(),
-              repositoryRules: "/*** START REPOSITORY RULES\n" + repository + "\nEND REPOSITORY RULES ***/",
             uuid: language.uuid,
             name: name
         });
 
         if (devMode) {
-          console.log(languageMode)
+            console.log(languageMode);
-          console.log(languageHighlightRules)
+            console.log(languageHighlightRules);
             console.log("Not writing, 'cause we're in dev mode, baby.");
         }
         else {
@@ -287,8 +181,11 @@ function convertLanguage(name) {
     });
 }
 
+var args = process.argv.splice(2);
+var tmLanguageFile  = args[0];
+var devMode         = args[1];
 if (tmLanguageFile === undefined) {
     console.error("Please pass in a language file via the command line.");
     process.exit(1);
 }
-convertLanguage(tmLanguageFile);
+convertLanguageFile(tmLanguageFile);