From 554dafd1bff40568261761d27aa7f3e612b627ff Mon Sep 17 00:00:00 2001
From: Fabian Jakobs <fabian.jakobs@web.de>
Date: Tue, 8 Feb 2011 12:32:47 +0100
Subject: [PATCH] fix tokenizer check

---
 lib/ace/tokenizer.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js
index de31c298..1bd8ab5d 100644
--- a/lib/ace/tokenizer.js
+++ b/lib/ace/tokenizer.js
@@ -74,6 +74,10 @@ var Tokenizer = function(rules) {
             var type = "text";
             var value = match[0];
 
+            if (re.lastIndex == lastIndex) {
+                throw new Error("tokenizer error before line: '" + line + "'");
+            }
+
             for ( var i = 0; i < state.length; i++) {
                 if (match[i + 1] !== undefined) {
                     if (typeof state[i].token == "function") {
@@ -110,9 +114,7 @@ var Tokenizer = function(rules) {
             
             if (lastIndex == line.length) {
 		        break;
-	        } else if (re.lastIndex == lastIndex) {
-                throw new Error("tokenizer error");
-            }
+	        }
             
 	        lastIndex = re.lastIndex;
         };