fix tokenizer on ie8

changes $arrayTokens to use exec instead of split
 1 it works on ie8
 2 it is faster on chrome http://jsperf.com/capturing-split-vs-exec
unfortunately exec is slow on firefox but ff regexes are a bit rusty atm (v21), and whole tokenizer is slow on it anyway.

lib/ace/keyboard/vim/commands.js

@@ -310,7 +310,7 @@ var actions = exports.actions = {
         fn: function(editor, range, count, param) {
             editor.modifyNumber(count || 1);
         }
-    },
+    }
 };
 
 var inputBuffer = exports.inputBuffer = {

lib/ace/mode/coffee_highlight_rules.js

@@ -99,7 +99,7 @@ define(function(require, exports, module) {
                     token : "string", regex : "'''", next : [
                         {token : "string", regex : "'''", next : "start"},
                         {token : "constant.language.escape", regex : stringEscape},
-                        {defaultToken: "string"},
+                        {defaultToken: "string"}
                     ]
                 }, {
                     stateName: "qqdoc",
@@ -115,21 +115,21 @@ define(function(require, exports, module) {
                     token : "string", regex : "'", next : [
                         {token : "string", regex : "'", next : "start"},
                         {token : "constant.language.escape", regex : stringEscape},
-                        {defaultToken: "string"},
+                        {defaultToken: "string"}
                     ]
                 }, {
                     stateName: "qqstring",
                     token : "string.start", regex : '"', next : [
                         {token : "string.end", regex : '"', next : "start"},
                         {token : "constant.language.escape", regex : stringEscape},
-                        {defaultToken: "string"},
+                        {defaultToken: "string"}
                     ]
                 }, {
                     stateName: "js",
                     token : "string", regex : "`", next : [
                         {token : "string", regex : "`", next : "start"},
                         {token : "constant.language.escape", regex : stringEscape},
-                        {defaultToken: "string"},
+                        {defaultToken: "string"}
                     ]
                 }, {
                     token : "string.regex",

lib/ace/mode/tmsnippet.js

@@ -46,19 +46,19 @@ var SnippetHighlightRules = function() {
             {regex: /\[/, token: "regex.start", next: "charClass"},
             {regex: "/", token: "string.regex", next: "format"},
             //{"default": "string.regex"},
-            {"token": "string.regex", regex:"."},
+            {"token": "string.regex", regex:"."}
         ],
         charClass : [
             {regex: "\\.", token: "escape"},
             {regex: "\\]", token: "regex.end", next: "regexp"},
-            {"token": "string.regex", regex:"."},
+            {"token": "string.regex", regex:"."}
         ],
         "format" : [
             {regex: /\\[ulULE]/, token: "keyword"},
             {regex: /\$\d+/, token: "variable"},
             {regex: "/[gim]*:?", token: "string.regex", next: "start"},
             // {"default": "string"},
-            {"token": "string", regex:"."},
+            {"token": "string", regex:"."}
         ]
     };
 };

lib/ace/search_test.js

@@ -451,7 +451,7 @@ module.exports = {
         assert.position(ranges[1].end, 0, 11);
         assert.position(ranges[0].start, 0, 0);
         assert.position(ranges[0].end, 0, 3);
-    },
+    }
 };
 
 });

lib/ace/tokenizer.js

@@ -111,12 +111,12 @@ var Tokenizer = function(rules) {
     this.$arrayTokens = function(str) {
         if (!str)
             return [];
-        var values = str.split(this.splitRegex)
+        var values = this.splitRegex.exec(str);
         var tokens = [];
         var types = this.tokenArray;
-        if (types.length != values.length - 2) {
+        if (types.length != values.length - 1) {
             if (window.console)
-                console.error(types.length , values.length - 2, str, this.splitRegex);
+                console.error(types , values, str, this.splitRegex, this);
             return [{type: "error.invalid", value: str}];
         }
         for (var i = 0; i < types.length; i++) {
@@ -140,7 +140,7 @@ var Tokenizer = function(rules) {
     
     this.createSplitterRegexp = function(src, flag) {
         src = src.replace(/\(\?=([^()]|\\.|\(([^()]|\\.)*?\))*\)(?=\)*$)/, "");
-        return new RegExp(src, flag);
+        return new RegExp(src, (flag||"").replace("g", ""));
     };
 
     /**

tool/lib.js

@@ -0,0 +1,38 @@
+var plist = require("plist");
+var util = require("util");
+exports.parsePlist = function(themeXml, callback) {    
+    var result = ""
+    plist.parseString(themeXml, function(_, theme) {
+        result = theme[0];
+        callback && callback(theme[0]);
+    });
+    return result;
+}
+
+exports.formatJSON = function(object, initialIndent) {
+    return util.inspect(object, false, 40).replace(/^/gm, initialIndent||"")
+    
+}
+
+
+exports.fillTemplate = function(template, replacements) {
+    return template.replace(/%(.+?)%/g, function(str, m) {
+        return replacements[m] || "";
+    });
+}
+
+exports.hyphenate = function(str) {
+    return str.replace(/([A-Z])/g, "-$1").replace(/_/g, "-").toLowerCase();
+}
+
+exports.quoteString = function(str) {
+    return '"' + str.replace(/\\/, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\\n") + '"';
+}
+
+
+exports.restoreJSONComments = function(objStr) {
+    return objStr.replace(/^(\s*)comment: '(.*)'/gm, function(_, i, c) {
+        return i + "//" +  c.replace(/\\n(\\t)*/g, "\n" + i + "//") + "\n" + i
+    }).replace(/ \/\/ ERROR/g, '", // ERROR');
+}
+