tweak mode_creator

lib/ace/tokenizer_dev.js

@@ -30,13 +30,15 @@
 
 define(function(require, exports, module) {
 
+// tokenizing lines longer than this makes editor very slow
+var MAX_TOKEN_COUNT = 1000;
 /*
  * version of Tokenizer with additional logging
  * and infinite loop checks
  * can be used for developing/testing new modes
  **/
-var Tokenizer = function(rules, flag) {
-    flag = flag ? "g" + flag : "g";
+
+var Tokenizer = function(rules) {
     this.states = rules;
 
     this.regExps = {};
@@ -46,13 +48,17 @@ var Tokenizer = function(rules, flag) {
         var ruleRegExps = [];
         var matchTotal = 0;
         var mapping = this.matchMappings[key] = {defaultToken: "default.text"};
+        var flag = "g";
 
         for (var i = 0; i < state.length; i++) {
             var rule = state[i];
-            if (rule.defaultToken) {
+            if (rule.defaultToken)
                 mapping.defaultToken = rule.defaultToken;
+            if (rule.caseInsensitive)
+                flag = "gi";
+            if (rule.regex == null)
                 continue;
-            }
+            
             if (rule.regex instanceof RegExp)
                 rule.regex = rule.regex.toString().slice(1, -1);
 
@@ -241,6 +247,12 @@ var Tokenizer = function(rules, flag) {
                 break;
 
             lastIndex = index;
+
+            if (tokens.length > MAX_TOKEN_COUNT) {
+                token.value += line.substr(lastIndex);
+                currentState = "start"
+                break;
+            }
         }
 
         if (token.type)