don't merge tokens (Includes coding style cleanups)

fix #355

index.html

@@ -31,7 +31,6 @@
           <option value="java">Java</option>
           <option value="csharp">C#</option>
           <option value="c_cpp">C++</option>
-          <option value="svg">SVG</option>
           <option value="clojure">Clojure</option>
           <option value="ocaml">OCaml</option>
           <option value="textile">Textile</option>
@@ -221,9 +220,7 @@
     for (var i=0; i<items.length; i++) {
         alert(items[i] + "juhu");
     }	// Real Tab.
-}
+}</script>
-// A magic character: >> ぁ <<
-</script>
 
     <script type="text/editor" id="csstext">.text-layer {
     font-family: Monaco, "Courier New", monospace;
@@ -302,8 +299,7 @@ for i in sys.argv[1:]:
         print repr(i), "not a numeric value"
     else:
         celsius=(fahrenheit-32)*5.0/9.0
-        print '%i\260F = %i\260C' % (int(fahrenheit), int(celsius+.5))
+        print '%i\260F = %i\260C' % (int(fahrenheit), int(celsius+.5))</script>
-</script>
 
     <script type="text/editor" id="phptext"><?php
 
@@ -674,8 +670,7 @@ th.join()
 assert counter.get() == 12
 </script>
 
-<script type="text/editor" id="scala">
+<script type="text/editor" id="scala">//http://www.scala-lang.org/node/227
-//http://www.scala-lang.org/node/227
 /* Defines a new method 'sort' for array objects */
 object implicits extends Application {
   implicit def arrayWrapper[A : ClassManifest](x: Array[A]) =
@@ -686,8 +681,7 @@ object implicits extends Application {
     }
   val x = Array(2, 3, 1, 4)
   println("x = "+ x.sort((x: Int, y: Int) => x < y))
-}
+}</script>
-</script>
   <input id="cockpitInput" type="text"/>
 
   <script type="text/javascript">

lib/ace/layer/text.js

@@ -384,17 +384,10 @@ var Text = function(parentEl) {
         var screenColumn = 0;
         var self = this;
 
-        function addToken(token, value) {
+        if (!splits || splits.length == 0)
-            screenColumn = self.$renderToken(
-                stringBuilder, screenColumn, token, value
-            );
-        }
-
-        if (!splits || splits.length == 0) {
             splitChars = Number.MAX_VALUE;
-        } else {
+        else
             splitChars = splits[0];
-        }
 
         if (!onlyContents) {
             stringBuilder.push("<div class='ace_line' style='height:",
@@ -408,11 +401,17 @@ var Text = function(parentEl) {
             var value = token.value;
 
             if (chars + value.length < splitChars) {
-                addToken(token, value);
+                screenColumn = self.$renderToken(
+                    stringBuilder, screenColumn, token, value
+                );
                 chars += value.length;
-            } else {
+            }
+            else {
                 while (chars + value.length >= splitChars) {
-                    addToken(token, value.substring(0, splitChars - chars));
+                    screenColumn = self.$renderToken(
+                        stringBuilder, screenColumn, 
+                        token, value.substring(0, splitChars - chars)
+                    );
                     value = value.substring(splitChars - chars);
                     chars = splitChars;
                     
@@ -430,18 +429,19 @@ var Text = function(parentEl) {
                 }
                 if (value.length != 0) {
                     chars += value.length;
-                    addToken(token, value);
+                    screenColumn = self.$renderToken(
+                        stringBuilder, screenColumn, token, value
+                    );
                 }
             }
         }
 
         if (this.showInvisibles) {
-            if (lastRow !== this.session.getLength() - 1) {
+            if (lastRow !== this.session.getLength() - 1)
                 stringBuilder.push("<span class='ace_invisible'>" + this.EOL_CHAR + "</span>");
-            } else {
+            else
                 stringBuilder.push("<span class='ace_invisible'>" + this.EOF_CHAR + "</span>");
         }
-        }
         stringBuilder.push("</div>");
     };
 

lib/ace/mode/c_cpp_highlight_rules.js

@@ -74,6 +74,7 @@ var c_cppHighlightRules = function() {
             new DocCommentHighlightRules().getStartRule("doc-start"),
             {
                 token : "comment", // multi line comment
+                merge : true,
                 regex : "\\/\\*",
                 next : "comment"
             }, {
@@ -81,6 +82,7 @@ var c_cppHighlightRules = function() {
                 regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : '["].*\\\\$',
                 next : "qqstring"
             }, {
@@ -88,6 +90,7 @@ var c_cppHighlightRules = function() {
                 regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : "['].*\\\\$",
                 next : "qstring"
             }, {
@@ -135,6 +138,7 @@ var c_cppHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
         ],
@@ -145,6 +149,7 @@ var c_cppHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ],
@@ -155,6 +160,7 @@ var c_cppHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ]

lib/ace/mode/clojure_highlight_rules.js

@@ -133,7 +133,6 @@ var ClojureHighlightRules = function() {
 
     this.$rules = {
         "start" : [
-
             {
                 token : "comment",
                 regex : ";.*$"
@@ -141,53 +140,40 @@ var ClojureHighlightRules = function() {
                     token : "comment", // multi line comment
                     regex : "^\=begin$",
                     next : "comment"
-                },
+            }, {
-                {             
                 token : "keyword", //parens
                 regex : "[\\(|\\)]"
-	        },
+            }, {
-                {             
                 token : "keyword", //lists
                 regex : "[\\'\\(]"
-	        },
+            }, {
-                {             
                 token : "keyword", //vectors
                 regex : "[\\[|\\]]"
-	        },
+            }, {
-                {             
                 token : "keyword", //sets and maps
                 regex : "[\\{|\\}|\\#\\{|\\#\\}]"
-	        },
+            }, {
-                {
                     token : "keyword", // ampersands
                     regex : '[\\&]'
-	        },
+            }, {
-                {
                     token : "keyword", // metadata
                     regex : '[\\#\\^\\{]'
-	        },
+            }, {
-                {
                     token : "keyword", // anonymous fn syntactic sugar
                     regex : '[\\%]'
-	        },
+            }, {
-                {
                     token : "keyword", // deref reader macro
                     regex : '[@]'
-	        },
+            }, {
-                {
                 token : "constant.numeric", // hex
                 regex : "0[xX][0-9a-fA-F]+\\b"
-	        },
+            }, {
-                {
                 token : "constant.numeric", // float
                 regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
-	        },
+            }, {
-                {
                 token : "constant.language",
                 regex : '[!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+||=|!=|<=|>=|<>|<|>|!|&&]'
-	        },
+            }, {
-                
-           {
                 token : function(value) {
                     if (keywords.hasOwnProperty(value))
                         return "keyword";
@@ -201,16 +187,13 @@ var ClojureHighlightRules = function() {
                 // TODO: Unicode escape sequences
                 // TODO: Unicode identifiers
                 regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
-	        },
+            }, {
-                {
                 token : "string", // single line
                 regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
-	        },
+            }, {
-                {
                 token : "string", // symbol
                 regex : "[:](?:[a-zA-Z]|\d)+"
-	        },
+            }, {
-                {
             token : "string.regexp", //Regular Expressions
             regex : '/#"(?:\.|(\\\")|[^\""\n])*"/g'
             }
@@ -223,6 +206,7 @@ var ClojureHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
         ]

lib/ace/mode/coffee_highlight_rules.js

@@ -37,146 +37,158 @@
 
 define(function(require, exports, module) {
 
-require("pilot/oop").inherits(
+    require("pilot/oop").inherits(CoffeeHighlightRules,
-  CoffeeHighlightRules,
             require("ace/mode/text_highlight_rules").TextHighlightRules);
 
-function CoffeeHighlightRules() {
+    function CoffeeHighlightRules() {
-var identifier = "[$A-Za-z_\\x7f-\\uffff][$\\w\\x7f-\\uffff]*"
+        var identifier = "[$A-Za-z_\\x7f-\\uffff][$\\w\\x7f-\\uffff]*";
-  , keywordend = "(?![$\\w]|\\s*:)"
+        var keywordend = "(?![$\\w]|\\s*:)";
-  , stringfill = {token: "string", regex: ".+"}
+        var stringfill = {
-;
+            token : "string",
-this.$rules =
+            merge : true,
-  { start:
+            regex : ".+"
-    [ { token: "identifier"
-      , regex: "(?:@|(?:\\.|::)\\s*)" + identifier
-      }
-    , { token: "keyword"
-      , regex: "(?:t(?:h(?:is|row|en)|ry|ypeof)|s(?:uper|witch)|return|b(?:reak|y)|c(?:ontinue|atch|lass)|i(?:n(?:stanceof)?|s(?:nt)?|f)|e(?:lse|xtends)|f(?:or (?:own)?|inally|unction)|wh(?:ile|en)|n(?:ew|ot?)|d(?:e(?:lete|bugger)|o)|loop|o(?:ff?|[rn])|un(?:less|til)|and|yes)" + keywordend
-      }
-    , { token: "constant.language"
-      , regex: "(?:true|false|null|undefined)" + keywordend
-      }
-    , { token: "invalid.illegal"
-      , regex: "(?:c(?:ase|onst)|default|function|v(?:ar|oid)|with|e(?:num|xport)|i(?:mplements|nterface)|let|p(?:ackage|r(?:ivate|otected)|ublic)|static|yield|__(?:hasProp|extends|slice|bind|indexOf))" + keywordend
-      }
-    , { token: "language.support.class"
-      , regex: "(?:Array|Boolean|Date|Function|Number|Object|R(?:e(?:gExp|ferenceError)|angeError)|S(?:tring|yntaxError)|E(?:rror|valError)|TypeError|URIError)" + keywordend
-      }
-    , { token: "language.support.function"
-      , regex: "(?:Math|JSON|is(?:NaN|Finite)|parse(?:Int|Float)|encodeURI(?:Component)?|decodeURI(?:Component)?)" + keywordend
-      }
-    , { token: "identifier"
-      , regex: identifier
-      }
-    , { token: "constant.numeric"
-      , regex: "(?:0x[\\da-fA-F]+|(?:\\d+(?:\\.\\d+)?|\\.\\d+)(?:[eE][+-]?\\d+)?)"
-      }
-    , { token: "string"
-      , regex: "'''"
-      , next : "qdoc"
-      }
-    , { token: "string"
-      , regex: '"""'
-      , next : "qqdoc"
-      }
-    , { token: "string"
-      , regex: "'"
-      , next : "qstring"
-      }
-    , { token: "string"
-      , regex: '"'
-      , next : "qqstring"
-      }
-    , { token: "string"
-      , regex: "`"
-      , next : "js"
-      }
-    , { token: "string.regex"
-      , regex: "///"
-      , next : "heregex"
-      }
-    , { token: "string.regex"
-      , regex: "/(?!\\s)[^[/\\n\\\\]*(?: (?:\\\\.|\\[[^\\]\\n\\\\]*(?:\\\\.[^\\]\\n\\\\]*)*\\])[^[/\\n\\\\]*)*/[imgy]{0,4}(?!\\w)"
-      }
-    , { token: "comment"
-      , regex: "###(?!#)"
-      , next : "comment"
-      }
-    , { token: "comment"
-      , regex: "#.*"
-      }
-    , { token: "lparen"
-      , regex: "[({[]"
-      }
-    , { token: "rparen"
-      , regex: "[\\]})]"
-      }
-    , { token: "keyword.operator"
-      , regex: "\\S+"
-      }
-    , { token: "text"
-      , regex: "\\s+"
-      }
-    ]
-  , qdoc:
-    [ { token: "string"
-      , regex: ".*?'''"
-      , next : "start"
-      }
-    , stringfill
-    ]
-  , qqdoc:
-    [ { token: "string"
-      , regex: '.*?"""'
-      , next : "start"
-      }
-    , stringfill
-    ]
-  , qstring:
-    [ { token: "string"
-      , regex: "[^\\\\']*(?:\\\\.[^\\\\']*)*'"
-      , next : "start"
-      }
-    , stringfill
-    ]
-  , qqstring:
-    [ { token: "string"
-      , regex: '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"'
-      , next : "start"
-      }
-    , stringfill
-    ]
-  , js:
-    [ { token: "string"
-      , regex: "[^\\\\`]*(?:\\\\.[^\\\\`]*)*`"
-      , next : "start"
-      }
-    , stringfill
-    ]
-  , heregex:
-    [ { token: "string.regex"
-      , regex: '.*?///[imgy]{0,4}'
-      , next : "start"
-      }
-    , { token: "comment.regex"
-      , regex: "\\s+(?:#.*)?"
-      }
-    , { token: "string.regex"
-      , regex: "\\S+"
-      }
-    ]
-  , comment:
-    [ { token: "comment"
-      , regex: '.*?###'
-      , next : "start"
-      }
-    , { token: "comment"
-      , regex: ".+"
-      }
-    ]
         };
-}
         
-exports.CoffeeHighlightRules = CoffeeHighlightRules;
+        this.$rules = {
+            start : [
+                {
+                    token : "identifier",
+                    regex : "(?:@|(?:\\.|::)\\s*)" + identifier
+                }, {
+                    token : "keyword",
+                    regex : "(?:t(?:h(?:is|row|en)|ry|ypeof)|s(?:uper|witch)|return|b(?:reak|y)|c(?:ontinue|atch|lass)|i(?:n(?:stanceof)?|s(?:nt)?|f)|e(?:lse|xtends)|f(?:or (?:own)?|inally|unction)|wh(?:ile|en)|n(?:ew|ot?)|d(?:e(?:lete|bugger)|o)|loop|o(?:ff?|[rn])|un(?:less|til)|and|yes)"
+                            + keywordend
+                }, {
+                    token : "constant.language",
+                    regex : "(?:true|false|null|undefined)" + keywordend
+                }, {
+                    token : "invalid.illegal",
+                    regex : "(?:c(?:ase|onst)|default|function|v(?:ar|oid)|with|e(?:num|xport)|i(?:mplements|nterface)|let|p(?:ackage|r(?:ivate|otected)|ublic)|static|yield|__(?:hasProp|extends|slice|bind|indexOf))"
+                            + keywordend
+                }, {
+                    token : "language.support.class",
+                    regex : "(?:Array|Boolean|Date|Function|Number|Object|R(?:e(?:gExp|ferenceError)|angeError)|S(?:tring|yntaxError)|E(?:rror|valError)|TypeError|URIError)"
+                            + keywordend
+                }, {
+                    token : "language.support.function",
+                    regex : "(?:Math|JSON|is(?:NaN|Finite)|parse(?:Int|Float)|encodeURI(?:Component)?|decodeURI(?:Component)?)"
+                            + keywordend
+                }, {
+                    token : "identifier",
+                    regex : identifier
+                }, {
+                    token : "constant.numeric",
+                    regex : "(?:0x[\\da-fA-F]+|(?:\\d+(?:\\.\\d+)?|\\.\\d+)(?:[eE][+-]?\\d+)?)"
+                }, {
+                    token : "string",
+                    merge : true,
+                    regex : "'''",
+                    next : "qdoc"
+                }, {
+                    token : "string",
+                    merge : true,
+                    regex : '"""',
+                    next : "qqdoc"
+                }, {
+                    token : "string",
+                    merge : true,
+                    regex : "'",
+                    next : "qstring"
+                }, {
+                    token : "string",
+                    merge : true,
+                    regex : '"',
+                    next : "qqstring"
+                }, {
+                    token : "string",
+                    merge : true,
+                    regex : "`",
+                    next : "js"
+                }, {
+                    token : "string.regex",
+                    merge : true,
+                    regex : "///",
+                    next : "heregex"
+                }, {
+                    token : "string.regex",
+                    regex : "/(?!\\s)[^[/\\n\\\\]*(?: (?:\\\\.|\\[[^\\]\\n\\\\]*(?:\\\\.[^\\]\\n\\\\]*)*\\])[^[/\\n\\\\]*)*/[imgy]{0,4}(?!\\w)"
+                }, {
+                    token : "comment",
+                    merge : true,
+                    regex : "###(?!#)",
+                    next : "comment"
+                }, {
+                    token : "comment",
+                    regex : "#.*"
+                }, {
+                    token : "lparen",
+                    regex : "[({[]"
+                }, {
+                    token : "rparen",
+                    regex : "[\\]})]"
+                }, {
+                    token : "keyword.operator",
+                    regex : "\\S+"
+                }, {
+                    token : "text",
+                    regex : "\\s+"
+                }],
+            
+            qdoc : [{
+                token : "string",
+                regex : ".*?'''",
+                next : "start"
+            }, stringfill],
+            
+            qqdoc : [{
+                token : "string",
+                regex : '.*?"""',
+                next : "start"
+            }, stringfill],
+            
+            qstring : [{
+                token : "string",
+                regex : "[^\\\\']*(?:\\\\.[^\\\\']*)*'",
+                next : "start"
+            }, stringfill],
+            
+            qqstring : [{
+                token : "string",
+                regex : '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
+                next : "start"
+            }, stringfill],
+            
+            js : [{
+                token : "string",
+                merge : true,
+                regex : "[^\\\\`]*(?:\\\\.[^\\\\`]*)*`",
+                next : "start"
+            }, stringfill],
+            
+            heregex : [{
+                token : "string.regex",
+                regex : '.*?///[imgy]{0,4}',
+                next : "start"
+            }, {
+                token : "comment.regex",
+                regex : "\\s+(?:#.*)?"
+            }, {
+                token : "string.regex",
+                merge : true,
+                regex : "\\S+"
+            }],
+            
+            comment : [{
+                token : "comment",
+                regex : '.*?###',
+                next : "start"
+            }, {
+                token : "comment",
+                merge : true,
+                regex : ".+"
+            }]
+        };
+    }
+
+    exports.CoffeeHighlightRules = CoffeeHighlightRules;
 });

lib/ace/mode/csharp_highlight_rules.js

@@ -29,10 +29,7 @@ var CSharpHighlightRules = function() {
             {
                 token : "comment", // multi line comment
                 regex : "\\/\\*",
-                next : "comment"
+                merge : true,
-            }, {
-	            token : "comment", // multi line comment
-	            regex : "\\/\\*\\*",
                 next : "comment"
             }, {
                 token : "string.regexp",
@@ -87,28 +84,9 @@ var CSharpHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
-        ],
-        "qqstring" : [
-            {
-	            token : "string",
-	            regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
-	            next : "start"
-	        }, {
-	            token : "string",
-	            regex : '.+'
-	        }
-        ],
-        "qstring" : [
-	        {
-	            token : "string",
-	            regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
-	            next : "start"
-	        }, {
-	            token : "string",
-	            regex : '.+'
-	        }
         ]
     };
     

lib/ace/mode/css_highlight_rules.js

@@ -125,6 +125,7 @@ var CssHighlightRules = function() {
     var base_ruleset = [
         {
             token : "comment", // multi line comment
+            merge : true,
             regex : "\\/\\*",
             next : "ruleset_comment"
         },{
@@ -212,66 +213,56 @@ var CssHighlightRules = function() {
         }
       ];
 
-    var ruleset = lang.copyArray( base_ruleset );
+    var ruleset = lang.copyArray(base_ruleset);
-    ruleset.unshift(
+    ruleset.unshift({
-      {
         token : "rparen",
-        regex : "\}",
+        regex : "\\}",
         next:   "start"
-      }
+    });
-    );
 
     var media_ruleset = lang.copyArray( base_ruleset );
-    media_ruleset.unshift(
+    media_ruleset.unshift({
-      {
         token : "rparen",
-        regex : "\}",
+        regex : "\\}",
         next:   "media"
-      }
+    });
-    );
 
-    var base_comment = [
+    var base_comment = [{
-      {
           token : "comment", // comment spanning whole line
+          merge : true,
           regex : ".+"
-      }
+    }];
-    ];
 
-    var comment = lang.copyArray( base_comment );
+    var comment = lang.copyArray(base_comment);
-    comment.unshift(
+    comment.unshift({
-      {
           token : "comment", // closing comment
           regex : ".*?\\*\\/",
           next : "start"
-      }
+    });
-    );
 
-    var media_comment = lang.copyArray( base_comment );
+    var media_comment = lang.copyArray(base_comment);
-    media_comment.unshift(
+    media_comment.unshift({
-      {
           token : "comment", // closing comment
           regex : ".*?\\*\\/",
           next : "media"
-      }
+    });
-    );
 
-    var ruleset_comment = lang.copyArray( base_comment );
+    var ruleset_comment = lang.copyArray(base_comment);
-    ruleset_comment.unshift(
+    ruleset_comment.unshift({
-      {
           token : "comment", // closing comment
           regex : ".*?\\*\\/",
           next : "ruleset"
-      }
+    });
-    );
 
     this.$rules = {
-        "start" : [ {
+        "start" : [{
             token : "comment", // multi line comment
+            merge : true,
             regex : "\\/\\*",
             next : "comment"
         }, {
             token: "lparen",
-            regex: "{",
+            regex: "\\{",
             next:  "ruleset"
         }, {
             token: "string",
@@ -293,15 +284,16 @@ var CssHighlightRules = function() {
 
         "media" : [ {
             token : "comment", // multi line comment
+            merge : true,
             regex : "\\/\\*",
             next : "media_comment"
         }, {
             token: "lparen",
-            regex: "{",
+            regex: "\\{",
             next:  "media_ruleset"
         },{
             token: "string",
-            regex: "}",
+            regex: "\\}",
             next:  "start"
         },{
             token: "keyword",

lib/ace/mode/css_test.js

@@ -46,6 +46,9 @@ var CssMode = require("ace/mode/css").Mode;
 var assert = require("ace/test/assertions");
 
 module.exports = {
+    
+    name: "CSS",
+    
     setUp : function() {
         this.mode = new CssMode();
     },

lib/ace/mode/css_tokenizer_test.js

@@ -45,6 +45,9 @@ var CssMode = require("ace/mode/css").Mode;
 var assert = require("ace/test/assertions");
 
 module.exports = {
+    
+    name: "CSS Tokenizer",
+    
     setUp : function() {
         this.tokenizer = new CssMode().getTokenizer();
     },

lib/ace/mode/doc_comment_highlight_rules.js

@@ -48,15 +48,19 @@ var DocCommentHighlightRules = function() {
             regex : "@[\\w\\d_]+" // TODO: fix email addresses
         }, {
             token : "comment.doc",
+            merge : true,
             regex : "\\s+"
         }, {
             token : "comment.doc",
+            merge : true,
             regex : "TODO"
         }, {
             token : "comment.doc",
+            merge : true,
             regex : "[^@\\*]+"
         }, {
             token : "comment.doc",
+            merge : true,
             regex : "."
         }]
     };
@@ -69,14 +73,16 @@ oop.inherits(DocCommentHighlightRules, TextHighlightRules);
     this.getStartRule = function(start) {
         return {
             token : "comment.doc", // doc comment
+            merge : true,
             regex : "\\/\\*(?=\\*)",
-            next: start
+            next  : start
         };
     };
     
     this.getEndRule = function (start) {
         return {
             token : "comment.doc", // closing comment
+            merge : true,
             regex : "\\*\\/",
             next  : start
         };

lib/ace/mode/groovy_highlight_rules.js

@@ -66,12 +66,9 @@ var GroovyHighlightRules = function() {
             new DocCommentHighlightRules().getStartRule("doc-start"),
             {
                 token : "comment", // multi line comment
+                merge : true,
                 regex : "\\/\\*",
                 next : "comment"
-            }, {
-	            token : "comment", // multi line comment
-	            regex : "\\/\\*\\*",
-	            next : "comment"
             }, {
                 token : "string.regexp",
                 regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
@@ -129,28 +126,9 @@ var GroovyHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
-        ],
-        "qqstring" : [
-            {
-	            token : "string",
-	            regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
-	            next : "start"
-	        }, {
-	            token : "string",
-	            regex : '.+'
-	        }
-        ],
-        "qstring" : [
-	        {
-	            token : "string",
-	            regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
-	            next : "start"
-	        }, {
-	            token : "string",
-	            regex : '.+'
-	        }
         ]
     };
     

lib/ace/mode/html_highlight_rules.js

@@ -54,6 +54,7 @@ var HtmlHighlightRules = function() {
             regex : '".*?"'
         }, {
             token : "string", // multi line string start
+            merge : true,
             regex : '["].*$',
             next : state + "-qqstring"
         }, {
@@ -61,6 +62,7 @@ var HtmlHighlightRules = function() {
             regex : "'.*?'"
         }, {
             token : "string", // multi line string start
+            merge : true,
             regex : "['].*$",
             next : state + "-qstring"
         }]
@@ -69,10 +71,12 @@ var HtmlHighlightRules = function() {
     function multiLineString(quote, state) {
         return [{
             token : "string",
+            merge : true,
             regex : ".*" + quote,
             next : state
         }, {
             token : "string",
+            merge : true,
             regex : '.+'
         }]
     }
@@ -80,6 +84,7 @@ var HtmlHighlightRules = function() {
     this.$rules = {
         start : [ {
             token : "text",
+            merge : true,
             regex : "<\\!\\[CDATA\\[",
             next : "cdata"
         }, {
@@ -87,6 +92,7 @@ var HtmlHighlightRules = function() {
             regex : "<\\?.*?\\?>"
         }, {
             token : "comment",
+            merge : true,
             regex : "<\\!--",
             next : "comment"
         }, {
@@ -158,9 +164,11 @@ var HtmlHighlightRules = function() {
             next : "start"
         }, {
             token : "text",
+            merge : true,
             regex : "\\s+"
         }, {
             token : "text",
+            merge : true,
             regex : ".+"
         } ],
 
@@ -170,6 +178,7 @@ var HtmlHighlightRules = function() {
             next : "start"
         }, {
             token : "comment",
+            merge : true,
             regex : ".+"
         } ]
     };

lib/ace/mode/java_highlight_rules.js

@@ -67,12 +67,9 @@ var JavaHighlightRules = function() {
             new DocCommentHighlightRules().getStartRule("doc-start"),
             {
                 token : "comment", // multi line comment
+                merge : true,
                 regex : "\\/\\*",
                 next : "comment"
-            }, {
-	            token : "comment", // multi line comment
-	            regex : "\\/\\*\\*",
-	            next : "comment"
             }, {
                 token : "string.regexp",
                 regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
@@ -130,28 +127,9 @@ var JavaHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
-        ],
-        "qqstring" : [
-            {
-	            token : "string",
-	            regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
-	            next : "start"
-	        }, {
-	            token : "string",
-	            regex : '.+'
-	        }
-        ],
-        "qstring" : [
-	        {
-	            token : "string",
-	            regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
-	            next : "start"
-	        }, {
-	            token : "string",
-	            regex : '.+'
-	        }
         ]
     };
     

lib/ace/mode/javascript_highlight_rules.js

@@ -80,6 +80,7 @@ var JavaScriptHighlightRules = function() {
             new DocCommentHighlightRules().getStartRule("doc-start"),
             {
                 token : "comment", // multi line comment
+                merge : true,
                 regex : "\\/\\*",
                 next : "comment"
             }, {
@@ -87,6 +88,7 @@ var JavaScriptHighlightRules = function() {
                 regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : '["].*\\\\$',
                 next : "qqstring"
             }, {
@@ -94,6 +96,7 @@ var JavaScriptHighlightRules = function() {
                 regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : "['].*\\\\$",
                 next : "qstring"
             }, {
@@ -157,7 +160,7 @@ var JavaScriptHighlightRules = function() {
                 token : "text",
                 regex : "\\s+"
             }, {
-                // immediately return to the start mode without mathcing
+                // immediately return to the start mode without matching
                 // anything
                 token: "empty", 
                 regex: "",
@@ -171,6 +174,7 @@ var JavaScriptHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
         ],
@@ -181,6 +185,7 @@ var JavaScriptHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ],
@@ -191,6 +196,7 @@ var JavaScriptHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ]

lib/ace/mode/javascript_tokenizer_test.js

@@ -45,6 +45,9 @@ var JavaScriptMode = require("ace/mode/javascript").Mode;
 var assert = require("ace/test/assertions");
 
 module.exports = {
+    
+    name: "JavaScript Tokenizer",
+    
     setUp : function() {
         this.tokenizer = new JavaScriptMode().getTokenizer();
     },
@@ -91,10 +94,14 @@ module.exports = {
 
         var tokens = this.tokenizer.getLineTokens(line, "start").tokens;
 
-        assert.equal(3, tokens.length);
+        assert.equal(7, tokens.length);
         assert.equal("lparen", tokens[0].type);
-        assert.equal("text", tokens[1].type);
+        assert.equal("lparen", tokens[1].type);
-        assert.equal("rparen", tokens[2].type);
+        assert.equal("lparen", tokens[2].type);
+        assert.equal("text", tokens[3].type);
+        assert.equal("rparen", tokens[4].type);
+        assert.equal("rparen", tokens[5].type);
+        assert.equal("rparen", tokens[6].type);
     },
 
     "test for last rule in ruleset to catch capturing group bugs" : function() {

lib/ace/mode/ocaml_highlight_rules.js

@@ -237,9 +237,8 @@ var OcamlHighlightRules = function() {
         "Int32|Int64|LargeFile|Lazy|Lexing|List|ListLabels|Make|Map|Marshal|" +
         "MoreLabels|Mutex|Nativeint|Num|Obj|Oo|Parsing|Pervasives|Printexc|" +
         "Printf|Queue|Random|Scanf|Scanning|Set|Sort|Stack|State|StdLabels|Str|" +
-        "Stream|String|StringLabels|Sys|Thread|ThreadUnix|Tk|Unix|UnixLabels|Weak")
+        "Stream|String|StringLabels|Sys|Thread|ThreadUnix|Tk|Unix|UnixLabels|Weak"
-
+    ).split("|"));
-        .split("|"));
 
     var decimalInteger = "(?:(?:[1-9]\\d*)|(?:0))";
     var octInteger = "(?:0[oO]?[0-7]+)";
@@ -262,6 +261,7 @@ var OcamlHighlightRules = function() {
             },
             {
                 token : "comment",
+                merge : true,
                 regex : '\\(\\*.*',
                 next : "comment"
             },
@@ -275,6 +275,7 @@ var OcamlHighlightRules = function() {
             },
             {
                 token : "string", // " string
+                merge : true,
                 regex : '"',
                 next  : "qstring"
             },
@@ -328,6 +329,7 @@ var OcamlHighlightRules = function() {
             },
             {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
         ],
@@ -339,6 +341,7 @@ var OcamlHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ]

lib/ace/mode/perl_highlight_rules.js

@@ -93,6 +93,7 @@ var PerlHighlightRules = function() {
                 regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : '["].*\\\\$',
                 next : "qqstring"
             }, {
@@ -100,6 +101,7 @@ var PerlHighlightRules = function() {
                 regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : "['].*\\\\$",
                 next : "qstring"
             }, {
@@ -141,6 +143,7 @@ var PerlHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ],
@@ -151,6 +154,7 @@ var PerlHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ]

lib/ace/mode/php_highlight_rules.js

@@ -921,6 +921,7 @@ var PhpHighlightRules = function() {
             new DocCommentHighlightRules().getStartRule("doc-start"),
             {
                 token : "comment", // multi line comment
+                merge : true,
                 regex : "\\/\\*",
                 next : "comment"
             }, {
@@ -931,6 +932,7 @@ var PhpHighlightRules = function() {
                 regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : '["].*\\\\$',
                 next : "qqstring"
             }, {
@@ -938,6 +940,7 @@ var PhpHighlightRules = function() {
                 regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : "['].*\\\\$",
                 next : "qstring"
             }, {
@@ -1018,6 +1021,7 @@ var PhpHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
         ],
@@ -1028,6 +1032,7 @@ var PhpHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ],
@@ -1038,6 +1043,7 @@ var PhpHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ]

lib/ace/mode/python_highlight_rules.js

@@ -96,6 +96,7 @@ var PythonHighlightRules = function() {
             regex : strPre + '"{3}(?:[^\\\\]|\\\\.)*?"{3}'
         }, {
             token : "string",           // multi line """ string start
+            merge : true,
             regex : strPre + '"{3}.*$',
             next : "qqstring"
         }, {
@@ -106,6 +107,7 @@ var PythonHighlightRules = function() {
             regex : strPre + "'{3}(?:[^\\\\]|\\\\.)*?'{3}"
         }, {
             token : "string",           // multi line ''' string start
+            merge : true,
             regex : strPre + "'{3}.*$",
             next : "qstring"
         }, {
@@ -158,6 +160,7 @@ var PythonHighlightRules = function() {
             next : "start"
         }, {
             token : "string",
+            merge : true,
             regex : '.+'
         } ],
         "qstring" : [ {
@@ -166,6 +169,7 @@ var PythonHighlightRules = function() {
             next : "start"
         }, {
             token : "string",
+            merge : true,
             regex : '.+'
         } ]
     };

lib/ace/mode/ruby_highlight_rules.js

@@ -103,6 +103,7 @@ var RubyHighlightRules = function() {
                 regex : "#.*$"
             }, {
                 token : "comment", // multi line comment
+                merge : true,
                 regex : "^\=begin$",
                 next : "comment"
             }, {
@@ -179,6 +180,7 @@ var RubyHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
         ]

lib/ace/mode/ruby_tokenizer_test.js

@@ -46,6 +46,9 @@ var RubyMode = require("ace/mode/ruby").Mode;
 var assert = require("ace/test/assertions");
 
 module.exports = {
+    
+    name: "Ruby Tokenizer",
+    
     setUp : function() {
         this.tokenizer = new RubyMode().getTokenizer();
     },

lib/ace/mode/scala_highlight_rules.js

@@ -67,12 +67,9 @@ var ScalaHighlightRules = function() {
             new DocCommentHighlightRules().getStartRule("doc-start"),
             {
                 token : "comment", // multi line comment
+                merge : true,
                 regex : "\\/\\*",
                 next : "comment"
-            }, {
-	            token : "comment", // multi line comment
-	            regex : "\\/\\*\\*",
-	            next : "comment"
             }, {
                 token : "string.regexp",
                 regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
@@ -130,28 +127,9 @@ var ScalaHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
-        ],
-        "qqstring" : [
-            {
-	            token : "string",
-	            regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
-	            next : "start"
-	        }, {
-	            token : "string",
-	            regex : '.+'
-	        }
-        ],
-        "qstring" : [
-	        {
-	            token : "string",
-	            regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
-	            next : "start"
-	        }, {
-	            token : "string",
-	            regex : '.+'
-	        }
         ]
     };
     

lib/ace/mode/scss_highlight_rules.js

@@ -197,6 +197,7 @@ var ScssHighlightRules = function() {
             },
             {
                 token : "comment", // multi line comment
+                merge : true,
                 regex : "\\/\\*",
                 next : "comment"
             }, {
@@ -204,6 +205,7 @@ var ScssHighlightRules = function() {
                 regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : '["].*\\\\$',
                 next : "qqstring"
             }, {
@@ -211,6 +213,7 @@ var ScssHighlightRules = function() {
                 regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
             }, {
                 token : "string", // multi line string start
+                merge : true,
                 regex : "['].*\\\\$",
                 next : "qstring"
             }, {
@@ -324,6 +327,7 @@ var ScssHighlightRules = function() {
                 next : "start"
             }, {
                 token : "comment", // comment spanning whole line
+                merge : true,
                 regex : ".+"
             }
         ],
@@ -334,6 +338,7 @@ var ScssHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ],
@@ -344,11 +349,11 @@ var ScssHighlightRules = function() {
                 next : "start"
             }, {
                 token : "string",
+                merge : true,
                 regex : '.+'
             }
         ]
     };
-
 };
 
 oop.inherits(ScssHighlightRules, TextHighlightRules);

lib/ace/mode/text_highlight_rules.js

@@ -45,13 +45,13 @@ var TextHighlightRules = function() {
     // regexps are ordered -> the first match is used
 
     this.$rules = {
-        "start" : [ {
+        "start" : [{
             token : "empty_line",
             regex : '^$'
         }, {
             token : "text",
             regex : ".+"
-        } ]
+        }]
     };
 };
 

lib/ace/mode/textile_highlight_rules.js

@@ -40,23 +40,7 @@ define(function(require, exports, module) {
 var oop = require("pilot/oop");
 var TextHighlightRules = require("ace/mode/text_highlight_rules").TextHighlightRules;
 
-var TextileHighlightRules = function()
+var TextileHighlightRules = function() {
-{
-/*
-    var phraseModifiers = lang.arrayToMap(
-        ("_|*|__|**|??|-|+|^|%|@").split("|")
-    );
-    
-    var blockModifiers = lang.arrayToMap(
-        ("h1|h2|h3|h4|h5|h6|bq|p|bc|pre").split("|")
-    );
-    */
-    /*
-    var punctuation = lang.arrayToMap(
-        ("-|--|(tm)|(r)|(c)").split("|")
-    );
-    */
-    
     this.$rules = {
         "start" : [
             {

lib/ace/mode/xml_highlight_rules.js

@@ -46,7 +46,7 @@ var XmlHighlightRules = function() {
     // regexps are ordered -> the first match is used
 
     this.$rules = {
-        start : [ {
+        start : [{
             token : "text",
             regex : "<\\!\\[CDATA\\[",
             next : "cdata"
@@ -55,6 +55,7 @@ var XmlHighlightRules = function() {
             regex : "<\\?.*?\\?>"
         }, {
             token : "comment",
+            merge : true,
             regex : "<\\!--",
             next : "comment"
         }, {
@@ -67,9 +68,9 @@ var XmlHighlightRules = function() {
         }, {
             token : "text",
             regex : "[^<]+"
-        } ],
+        }],
 
-        tag : [ {
+        tag : [{
             token : "text",
             regex : ">",
             next : "start"
@@ -84,6 +85,7 @@ var XmlHighlightRules = function() {
             regex : '".*?"'
         }, {
             token : "string", // multi line string start
+            merge : true,
             regex : '["].*$',
             next : "qqstring"
         }, {
@@ -91,6 +93,7 @@ var XmlHighlightRules = function() {
             regex : "'.*?'"
         }, {
             token : "string", // multi line string start
+            merge : true,
             regex : "['].*$",
             next : "qstring"
         }],
@@ -101,6 +104,7 @@ var XmlHighlightRules = function() {
             next : "tag"
         }, {
             token : "string",
+            merge : true,
             regex : '.+'
         }],
         
@@ -110,10 +114,11 @@ var XmlHighlightRules = function() {
             next : "tag"
         }, {
             token : "string",
+            merge : true,
             regex : '.+'
         }],
         
-        cdata : [ {
+        cdata : [{
             token : "text",
             regex : "\\]\\]>",
             next : "start"
@@ -123,16 +128,17 @@ var XmlHighlightRules = function() {
         }, {
             token : "text",
             regex : "(?:[^\\]]|\\](?!\\]>))+"
-        } ],
+        }],
 
-        comment : [ {
+        comment : [{
             token : "comment",
             regex : ".*?-->",
             next : "start"
         }, {
             token : "comment",
+            merge : true,
             regex : ".+"
-        } ]
+        }]
     };
 };
 

lib/ace/mode/xml_tokenizer_test.js

@@ -45,12 +45,14 @@ var XmlMode = require("ace/mode/xml").Mode;
 var assert = require("ace/test/assertions");
 
 module.exports = {
+    
+    name: "XML Tokenizer",
+    
     setUp : function() {
         this.tokenizer = new XmlMode().getTokenizer();
     },
 
     "test: tokenize1" : function() {
-
         var line = "<Juhu>//Juhu Kinners</Kinners>";
         var tokens = this.tokenizer.getLineTokens(line, "start").tokens;
 

lib/ace/tokenizer.js

@@ -69,7 +69,6 @@ var Tokenizer = function(rules) {
         }
 
         this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
-        
     }
 };
 
@@ -93,11 +92,12 @@ var Tokenizer = function(rules) {
         
         while (match = re.exec(line)) {
             var type = "text";
+            var rule = null;
             var value = [match[0]];
 
-            for ( var i = 0; i < match.length-2; i++) {
+            for (var i = 0; i < match.length-2; i++) {
                 if (match[i + 1] !== undefined) {
-                    var rule = state[mapping[i].rule];
+                    rule = state[mapping[i].rule];
                     
                     if (mapping[i].len > 1) {
                         value = match.slice(i+2, i+1+mapping[i].len);
@@ -128,9 +128,10 @@ var Tokenizer = function(rules) {
                     value = [value.join("")];
                     type = [type];
                 }
-            
                 for (var i = 0; i < value.length; i++) {
-                    if (token.type !== type[i]) {
+                    if ((!rule || rule.merge || type[i] === "text") && token.type === type[i]) {
+                        token.value += value[i];
+                    } else {
                         if (token.type) {
                             tokens.push(token);
                         }
@@ -139,8 +140,6 @@ var Tokenizer = function(rules) {
                             type: type[i],
                             value: value[i]
                         }
-                    } else {
-                        token.value += value[i];
                     }
                 }
             }