From e084dc1303b3b4e9c33c3b9c7123933df82bb0d8 Mon Sep 17 00:00:00 2001
From: nightwing <amirjanyan@gmail.com>
Date: Thu, 7 Feb 2013 22:29:37 +0400
Subject: [PATCH] move line length checking to tokenizer to not break long data
 uri's in css

---
 lib/ace/background_tokenizer.js | 10 ----------
 lib/ace/tokenizer.js            |  8 ++++++++
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/lib/ace/background_tokenizer.js b/lib/ace/background_tokenizer.js
index c1bd8047..8f936d1a 100644
--- a/lib/ace/background_tokenizer.js
+++ b/lib/ace/background_tokenizer.js
@@ -34,8 +34,6 @@ define(function(require, exports, module) {
 var oop = require("./lib/oop");
 var EventEmitter = require("./lib/event_emitter").EventEmitter;
 
-// tokenizing lines longer than this makes editor very slow
-var MAX_LINE_LENGTH = 5000;
 
 /**
  * 
@@ -226,15 +224,7 @@ var BackgroundTokenizer = function(tokenizer, editor) {
         var line = this.doc.getLine(row);
         var state = this.states[row - 1];
 
-        if (line.length > MAX_LINE_LENGTH) {
-            var overflow = {value: line.substr(MAX_LINE_LENGTH), type: "text"};
-            line = line.slice(0, MAX_LINE_LENGTH);
-        }
         var data = this.tokenizer.getLineTokens(line, state, row);
-        if (overflow) {
-            data.tokens.push(overflow);
-            data.state = "start";
-        }
 
         if (this.states[row] + "" !== data.state + "") {
             this.states[row] = data.state;
diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js
index c7d6bee9..628b3a29 100644
--- a/lib/ace/tokenizer.js
+++ b/lib/ace/tokenizer.js
@@ -31,6 +31,8 @@
 define(function(require, exports, module) {
 "use strict";
 
+// tokenizing lines longer than this makes editor very slow
+var MAX_TOKEN_COUNT = 1000;
 /**
  *
  *
@@ -230,6 +232,12 @@ var Tokenizer = function(rules, flag) {
                 break;
 
             lastIndex = index;
+
+            if (tokens.length > MAX_TOKEN_COUNT) {
+                token.value += line.substr(lastIndex);
+                currentState = "start"
+                break;
+            }
         }
 
         if (token.type)