From 90f57f682ba3c64d6ee125555e20f1320f18f6bf Mon Sep 17 00:00:00 2001
From: Fabian Jakobs <fabian.jakobs@web.de>
Date: Tue, 22 Feb 2011 23:39:01 +0100
Subject: [PATCH] add unit test to match regular expression literals (still
 fialing)

---
 .../test/mode/javascript_tokenizer_test.js    |  5 +++
 lib/ace/tokenizer.js                          | 41 +++++++++----------
 2 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/lib/ace/test/mode/javascript_tokenizer_test.js b/lib/ace/test/mode/javascript_tokenizer_test.js
index be0fe1ea..c4aa922a 100644
--- a/lib/ace/test/mode/javascript_tokenizer_test.js
+++ b/lib/ace/test/mode/javascript_tokenizer_test.js
@@ -98,6 +98,11 @@ var Test = {
         
         assert.equal(1, tokens.length);
         assert.equal("rparen", tokens[0].type);
+    },
+    
+    "test tokenize regular expressions": function() {
+        var tokens = this.tokenizer.getLineTokens("a/b/c", "start").tokens;
+        assert.equal(5, tokens.length);
     }
 };
 
diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js
index 00cdd6de..66e9b847 100644
--- a/lib/ace/tokenizer.js
+++ b/lib/ace/tokenizer.js
@@ -42,14 +42,15 @@ var Tokenizer = function(rules) {
 
     this.regExps = {};
     for ( var key in this.rules) {
-        var state = this.rules[key];
+        var rule = this.rules[key];
+        var state = rule;
         var ruleRegExps = [];
 
-        for ( var i = 0; i < state.length; i++) {
+        for ( var i = 0; i < state.length; i++)
             ruleRegExps.push(state[i].regex);
-        };
 
         this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
+        
     }
 };
 
@@ -76,19 +77,19 @@ var Tokenizer = function(rules) {
 
             for ( var i = 0; i < state.length; i++) {
                 if (match[i + 1]) {
-                    if (typeof state[i].token == "function") {
-                        type = state[i].token(match[0]);
-                    }
-                    else {
-                        type = state[i].token;
-                    }
+                    var rule = state[i];
+                    
+                    if (typeof rule.token == "function")
+                        type = rule.token(match[0]);
+                    else
+                        type = rule.token;
 
-                    if (state[i].next && state[i].next !== currentState) {
-                        currentState = state[i].next;
-                        var state = this.rules[currentState];
-                        var lastIndex = re.lastIndex;
+                    if (rule.next && rule.next !== currentState) {
+                        currentState = rule.next;
+                        state = this.rules[currentState];
+                        lastIndex = re.lastIndex;
 
-                        var re = this.regExps[currentState];
+                        re = this.regExps[currentState];
                         re.lastIndex = lastIndex;
                     }
                     break;
@@ -97,9 +98,9 @@ var Tokenizer = function(rules) {
             
                   
             if (token.type !== type) {
-                if (token.type) {
+                if (token.type)
                     tokens.push(token);
-                }
+                    
                 token = {
                     type: type,
                     value: value
@@ -108,16 +109,14 @@ var Tokenizer = function(rules) {
                 token.value += value;
             }
             
-            if (lastIndex == line.length) {
-    	        break;
-            }
+            if (lastIndex == line.length)
+                break;
             
             lastIndex = re.lastIndex;
         };
 
-        if (token.type) {
+        if (token.type)
             tokens.push(token);
-        }
 
         return {
             tokens : tokens,