From 08e621b443d6db92e81f4ec5e5d668eee2d13824 Mon Sep 17 00:00:00 2001
From: nightwing <amirjanyan@gmail.com>
Date: Fri, 28 Dec 2012 19:13:30 +0400
Subject: [PATCH] allow array of states in tokenizer

---
 lib/ace/background_tokenizer.js |  2 +-
 lib/ace/tokenizer.js            | 19 ++++++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/lib/ace/background_tokenizer.js b/lib/ace/background_tokenizer.js
index 28d11994..d75305f9 100644
--- a/lib/ace/background_tokenizer.js
+++ b/lib/ace/background_tokenizer.js
@@ -236,7 +236,7 @@ var BackgroundTokenizer = function(tokenizer, editor) {
             data.state = "start";
         }
 
-        if (this.states[row] !== data.state) {
+        if (this.states[row] + "" !== data.state + "") {
             this.states[row] = data.state;
             this.lines[row + 1] = null;
             if (this.currentLine > row + 1)
diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js
index f468602c..3f21c475 100644
--- a/lib/ace/tokenizer.js
+++ b/lib/ace/tokenizer.js
@@ -54,15 +54,14 @@ var Tokenizer = function(rules, flag) {
 
     this.regExps = {};
     this.matchMappings = {};
-    for ( var key in this.rules) {
+    for (var key in this.rules) {
         var rule = this.rules[key];
         var state = rule;
         var ruleRegExps = [];
         var matchTotal = 0;
         var mapping = this.matchMappings[key] = {};
 
-        for ( var i = 0; i < state.length; i++) {
-
+        for (var i = 0; i < state.length; i++) {
             if (state[i].regex instanceof RegExp)
                 state[i].regex = state[i].regex.toString().slice(1, -1);
 
@@ -75,7 +74,7 @@ var Tokenizer = function(rules, flag) {
                 return "\\" + (parseInt(digit, 10) + matchTotal + 1);
             });
 
-            if (matchcount > 1 && state[i].token.length !== matchcount-1)
+            if (matchcount > 1 && typeof state[i].token == "string" && state[i].token.length !== matchcount-1)
                 throw new Error("For " + state[i].regex + " the matching groups (" +(matchcount-1) + ") and length of the token array (" + state[i].token.length + ") don't match (rule #" + i + " of state " + key + ")");
 
             mapping[matchTotal] = {
@@ -98,6 +97,12 @@ var Tokenizer = function(rules, flag) {
     * @returns {Object}
     **/
     this.getLineTokens = function(line, startState) {
+        if (startState && typeof startState != "string") {
+            var stack = startState.slice(0);
+            startState = stack[0];
+        } else {
+            var stack = []
+        }
         var currentState = startState || "start";
         var state = this.rules[currentState];
         var mapping = this.matchMappings[currentState];
@@ -129,7 +134,7 @@ var Tokenizer = function(rules, flag) {
 
                 // compute token type
                 if (typeof rule.token == "function")
-                    type = rule.token.apply(this, value);
+                    type = rule.token(value.length == 1 ? value[0] : value, currentState, stack);
                 else
                     type = rule.token;
 
@@ -142,7 +147,7 @@ var Tokenizer = function(rules, flag) {
                     re = this.regExps[currentState];
 
                     if (re === undefined) {
-                         throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
+                        throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
                     }
 
                     re.lastIndex = lastIndex;
@@ -184,7 +189,7 @@ var Tokenizer = function(rules, flag) {
 
         return {
             tokens : tokens,
-            state : currentState
+            state : stack.length ? stack : currentState
         };
     };