From ae4b171065f1a7a6a3f926842cb36a5eb319db5f Mon Sep 17 00:00:00 2001
From: Daniel Krech <eikeon@eikeon.com>
Date: Sun, 6 Feb 2011 23:53:02 +0800
Subject: [PATCH] Fixed to allow tokenizing of an empty line. eg. a rule with a
 regex of ^$

---
 lib/ace/tokenizer.js | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js
index a5f847ff..fa70e2d5 100644
--- a/lib/ace/tokenizer.js
+++ b/lib/ace/tokenizer.js
@@ -74,11 +74,8 @@ var Tokenizer = function(rules) {
             var type = "text";
             var value = match[0];
 
-            if (re.lastIndex == lastIndex) { throw new Error("tokenizer error"); }
-            lastIndex = re.lastIndex;
-
             for ( var i = 0; i < state.length; i++) {
-                if (match[i + 1]) {
+                if (match[i + 1] !== undefined) {
                     if (typeof state[i].token == "function") {
                         type = state[i].token(match[0]);
                     }
@@ -110,6 +107,10 @@ var Tokenizer = function(rules) {
             } else {
                 token.value += value;
             }
+            if (lastIndex == line.length) {
+		break;
+	    }
+	    lastIndex = re.lastIndex;
         };
 
         if (token.type) {