workaround for chrome not displaying middle of very long lines

lib/ace/tokenizer.js

@@ -83,6 +83,7 @@ var Tokenizer = function(rules) {
                         + (matchcount - 1) + "!=" + rule.token.length);
                 } else {
                     rule.tokenArray = rule.token;
+                    rule.token = null;
                     rule.onMatch = this.$arrayTokens;
                 }
             } else if (typeof rule.token == "function" && !rule.onMatch) {
@@ -271,7 +272,7 @@ var Tokenizer = function(rules) {
 
             if (value) {
                 if (typeof type == "string") {
-                    if ((!rule || rule.merge !== false) && token.type === type) {
+                    if ((!rule || rule.merge) && token.type === type) {
                         token.value += value;
                     } else {
                         if (token.type)
@@ -293,8 +294,17 @@ var Tokenizer = function(rules) {
             lastIndex = index;
 
             if (tokens.length > MAX_TOKEN_COUNT) {
-                token.value += line.substr(lastIndex);
+                // chrome doens't show contents of text nodes with very long text
-                currentState = "start"
+                while (lastIndex < line.length) {
+                    if (token.type)
+                        tokens.push(token);
+                    token = {
+                        value: line.substring(lastIndex, lastIndex += 2000),
+                        type: "overflow"
+                    }    
+                }
+                currentState = "start";
+                stack = [];
                 break;
             }
         }

lib/ace/tokenizer_dev.js

@@ -32,7 +32,7 @@ define(function(require, exports, module) {
 var BaseTokenizer = require("./tokenizer").Tokenizer;
 
 // tokenizing lines longer than this makes editor very slow
-var MAX_TOKEN_COUNT = 1000;
+var MAX_TOKEN_COUNT = 100000;
 /*
  * version of Tokenizer with additional logging
  * and infinite loop checks
@@ -80,7 +80,7 @@ var Tokenizer = function(rules) {
         };
         initState();
         
-        var maxRecur = 10000;
+        var maxRecur = 100000;
         
         while (match = re.exec(line)) {
             var type = mapping.defaultToken;
@@ -177,5 +177,7 @@ var Tokenizer = function(rules) {
 
 };
 
+Tokenizer.prototype = BaseTokenizer.prototype;
+
 exports.Tokenizer = Tokenizer;
 });