From 61ac1944722fbb7ef39988360cac65e707f33430 Mon Sep 17 00:00:00 2001
From: Fabian Jakobs <fabian.jakobs@web.de>
Date: Tue, 27 Apr 2010 12:20:16 +0200
Subject: [PATCH] fix nasty tokenizer bugs

---
 src/ace/BackgroundTokenizer.js | 20 ++++++++++----------
 src/ace/Editor.js              |  1 +
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/ace/BackgroundTokenizer.js b/src/ace/BackgroundTokenizer.js
index b024706a..64b124d8 100644
--- a/src/ace/BackgroundTokenizer.js
+++ b/src/ace/BackgroundTokenizer.js
@@ -18,20 +18,15 @@ ace.BackgroundTokenizer = function(tokenizer) {
         var processedLines = 0;
 
         while (self.currentLine < textLines.length) {
-            var line = textLines[self.currentLine];
-
-            var state = self.currentLine == 0 ? "start"
-                    : self.lines[self.currentLine - 1].state;
-            self.lines[self.currentLine] = self.tokenizer.getLineTokens(line, state);
+            self.lines[self.currentLine] = self.$tokenizeRow(self.currentLine);
+            self.currentLine++;
 
             // only check every 30 lines
             processedLines += 1;
             if ((processedLines % 30 == 0) && (new Date() - workerStart) > 20) {
-                self.fireUpdateEvent(startLine, self.currentLine);
+                self.fireUpdateEvent(startLine, self.currentLine-1);
                 return setTimeout(self.$worker, 10);
             }
-
-            self.currentLine++;
         }
 
         self.running = false;
@@ -95,11 +90,16 @@ ace.BackgroundTokenizer = function(tokenizer) {
 
     this.$tokenizeRow = function(row) {
         if (!this.lines[row]) {
-            var state = "start";
+            var state = null;
             if (row > 0 && this.lines[row - 1]) {
                 state = this.lines[row - 1].state;
             }
-            this.lines[row] = this.tokenizer.getLineTokens(this.textLines[row] || "", state);
+            var tokens = this.tokenizer.getLineTokens(this.textLines[row] || "", state || "start");
+            if (state) {
+                this.lines[row] = tokens;
+            } else {
+                return tokens;
+            }
         }
         return this.lines[row];
     };
diff --git a/src/ace/Editor.js b/src/ace/Editor.js
index e856688f..6e6ef4d9 100644
--- a/src/ace/Editor.js
+++ b/src/ace/Editor.js
@@ -65,6 +65,7 @@ ace.Editor = function(renderer, doc) {
 
         this.onDocumentModeChange();
         this.bgTokenizer.setLines(this.doc.lines);
+        this.bgTokenizer.start(0);
 
         this.renderer.draw();
         this.onCursorChange();