Merge pull request #2125 from ajaxorg/highlighter-fixes

Highlighter fixes

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "doc/wiki"]
 	path = doc/wiki
-	url = git://github.com/ajaxorg/ace.wiki.git
+	url = https://github.com/ajaxorg/ace.wiki.git
 [submodule "build"]
 	path = build
-	url = git://github.com/ajaxorg/ace-builds.git
+	url = https://github.com/ajaxorg/ace-builds.git

demo/kitchen-sink/docs/io.io

@@ -0,0 +1,6 @@
+// computes factorial of a number
+factorial := method(n,
+    if(n == 0, return 1)
+    res := 1
+    Range 1 to(n) foreach(i, res = res * i)
+)

lib/ace/edit_session/bracket_match.js

@@ -117,6 +117,7 @@ function BracketMatch() {
             typeRe = new RegExp(
                 "(\\.?" +
                 token.type.replace(".", "\\.").replace("rparen", ".paren")
+                    .replace(/\b(?:end|start|begin)\b/, "")
                 + ")+"
             );
         }
@@ -173,6 +174,7 @@ function BracketMatch() {
             typeRe = new RegExp(
                 "(\\.?" +
                 token.type.replace(".", "\\.").replace("lparen", ".paren")
+                    .replace(/\b(?:end|start|begin)\b/, "")
                 + ")+"
             );
         }

lib/ace/ext/modelist.js

@@ -66,8 +66,8 @@ var supportedModes = {
     Dockerfile:  ["^Dockerfile"],
     Dot:         ["dot"],
     Eiffel:      ["e"],
-    Erlang:      ["erl|hrl"],
     EJS:         ["ejs"],
+    Erlang:      ["erl|hrl"],
     Forth:       ["frt|fs|ldr"],
     FTL:         ["ftl"],
     Gcode:       ["gcode"],
@@ -83,6 +83,7 @@ var supportedModes = {
     HTML:        ["html|htm|xhtml"],
     HTML_Ruby:   ["erb|rhtml|html.erb"],
     INI:         ["ini|conf|cfg|prefs"],
+    Io:          ["io"],
     Jack:        ["jack"],
     Jade:        ["jade"],
     Java:        ["java"],
@@ -103,11 +104,11 @@ var supportedModes = {
     LuaPage:     ["lp"],
     Lucene:      ["lucene"],
     Makefile:    ["^Makefile|^GNUmakefile|^makefile|^OCamlMakefile|make"],
-    MATLAB:      ["matlab"],
     Markdown:    ["md|markdown"],
+    MATLAB:      ["matlab"],
     MEL:         ["mel"],
-    MySQL:       ["mysql"],
     MUSHCode:    ["mc|mush"],
+    MySQL:       ["mysql"],
     Nix:         ["nix"],
     ObjectiveC:  ["m|mm"],
     OCaml:       ["ml|mli"],
@@ -129,14 +130,14 @@ var supportedModes = {
     SASS:        ["sass"],
     SCAD:        ["scad"],
     Scala:       ["scala"],
-    Smarty:      ["smarty|tpl"],
     Scheme:      ["scm|rkt"],
     SCSS:        ["scss"],
     SH:          ["sh|bash|^.bashrc"],
     SJS:         ["sjs"],
-    Space:       ["space"],
+    Smarty:      ["smarty|tpl"],
     snippets:    ["snippets"],
     Soy_Template:["soy"],
+    Space:       ["space"],
     SQL:         ["sql"],
     Stylus:      ["styl|stylus"],
     SVG:         ["svg"],

lib/ace/mode/_test/text_coffee.txt

@@ -42,6 +42,7 @@ class Foo extends Bar
 foo.static.function
 #!test tokenize string with interpolation
 a = "#{ 22 / 7 + {x: "#{a + b}"} + 2}"
+" #{ "" + {} } )"  
 """heredoc
    """
 do ->

lib/ace/mode/_test/tokens_coffee.json

@@ -466,6 +466,23 @@
   ["constant.numeric","2"],
   ["paren.string","}"],
   ["string.end","\""]
+],[
+   "start",
+  ["string.start","\""],
+  ["string"," "],
+  ["paren.string","#{"],
+  ["text"," "],
+  ["string.start","\""],
+  ["string.end","\""],
+  ["text"," "],
+  ["keyword.operator","+"],
+  ["text"," "],
+  ["paren","{}"],
+  ["text"," "],
+  ["paren.string","}"],
+  ["string"," )"],
+  ["string.end","\""],
+  ["text","  "]
 ],[
    "qqdoc",
   ["string","\"\"\"heredoc"]

lib/ace/mode/_test/tokens_javascript.json

@@ -379,7 +379,7 @@
   ["constant.numeric","1."],
   ["text","00"],
   ["identifier","E"],
-  ["text","^"],
+  ["keyword.operator","^"],
   ["constant.numeric","1"],
   ["punctuation.operator",","],
   ["text"," "],

lib/ace/mode/_test/tokens_latex.json

@@ -92,7 +92,9 @@
 ],[
    "start",
   ["text","    E &= mc^2                              "],
-  ["constant.character.escape","\\\\"],
+  ["constant.character.escape","\\\\"]
+],[
+   "start",
   ["text","    m &= "],
   ["storage.type","\\frac"],
   ["lparen","{"],

lib/ace/mode/_test/tokens_php.json

@@ -138,7 +138,7 @@
   ["text","  "],
   ["support.php_tag","<?php"]
 ],[
-   ["php-qstring","php-start","js-start"],
+   ["#tmp","php-qstring","php-start","js-start"],
   ["support.function","echo"],
   ["text"," "],
   ["string","'"]

lib/ace/mode/io.js

@@ -0,0 +1,58 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2012, Ajax.org B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/*
+  THIS FILE WAS AUTOGENERATED BY mode.tmpl.js
+*/
+
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextMode = require("./text").Mode;
+var Tokenizer = require("../tokenizer").Tokenizer;
+var IoHighlightRules = require("./io_highlight_rules").IoHighlightRules;
+var FoldMode = require("./folding/cstyle").FoldMode;
+
+var Mode = function() {
+    this.HighlightRules = IoHighlightRules;
+    this.foldingRules = new FoldMode();
+};
+oop.inherits(Mode, TextMode);
+
+(function() {
+    this.lineCommentStart = "//";
+    this.blockComment = {start: "/*", end: "*/"};
+    // Extra logic goes here.
+    this.$id = "ace/mode/io"
+}).call(Mode.prototype);
+
+exports.Mode = Mode;
+});

lib/ace/mode/io_highlight_rules.js

@@ -0,0 +1,120 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2012, Ajax.org B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/* This file was autogenerated from tm bundles\io.tmbundle/Syntaxes/io.plist (uuid: ) */
+/****************************************************************************************
+ * IT MIGHT NOT BE PERFECT ...But it's a good start from an existing *.tmlanguage file. *
+ * fileTypes                                                                            *
+ ****************************************************************************************/
+
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var IoHighlightRules = function() {
+    // regexp must not have capturing parentheses. Use (?:) instead.
+    // regexps are ordered -> the first match is used
+
+    this.$rules = { start: 
+       [ { token: [ 'text', 'meta.empty-parenthesis.io' ],
+           regex: '(\\()(\\))',
+           comment: 'we match this to overload return inside () --Allan; scoping rules for what gets the scope have changed, so we now group the ) instead of the ( -- Rob' },
+         { token: [ 'text', 'meta.comma-parenthesis.io' ],
+           regex: '(\\,)(\\))',
+           comment: 'We want to do the same for ,) -- Seckar; same as above -- Rob' },
+         { token: 'keyword.control.io',
+           regex: '\\b(?:if|ifTrue|ifFalse|ifTrueIfFalse|for|loop|reverseForeach|foreach|map|continue|break|while|do|return)\\b' },
+         { token: 'punctuation.definition.comment.io',
+           regex: '/\\*',
+           push: 
+            [ { token: 'punctuation.definition.comment.io',
+                regex: '\\*/',
+                next: 'pop' },
+              { defaultToken: 'comment.block.io' } ] },
+         { token: 'punctuation.definition.comment.io',
+           regex: '//',
+           push: 
+            [ { token: 'comment.line.double-slash.io',
+                regex: '$',
+                next: 'pop' },
+              { defaultToken: 'comment.line.double-slash.io' } ] },
+         { token: 'punctuation.definition.comment.io',
+           regex: '#',
+           push: 
+            [ { token: 'comment.line.number-sign.io', regex: '$', next: 'pop' },
+              { defaultToken: 'comment.line.number-sign.io' } ] },
+         { token: 'variable.language.io',
+           regex: '\\b(?:self|sender|target|proto|protos|parent)\\b',
+           comment: 'I wonder if some of this isn\'t variable.other.language? --Allan; scoping this as variable.language to match Objective-C\'s handling of \'self\', which is inconsistent with C++\'s handling of \'this\' but perhaps intentionally so -- Rob' },
+         { token: 'keyword.operator.io',
+           regex: '<=|>=|=|:=|\\*|\\||\\|\\||\\+|-|/|&|&&|>|<|\\?|@|@@|\\b(?:and|or)\\b' },
+         { token: 'constant.other.io', regex: '\\bGL[\\w_]+\\b' },
+         { token: 'support.class.io', regex: '\\b[A-Z](?:\\w+)?\\b' },
+         { token: 'support.function.io',
+           regex: '\\b(?:clone|call|init|method|list|vector|block|\\w+(?=\\s*\\())\\b' },
+         { token: 'support.function.open-gl.io',
+           regex: '\\bgl(?:u|ut)?[A-Z]\\w+\\b' },
+         { token: 'punctuation.definition.string.begin.io',
+           regex: '"""',
+           push: 
+            [ { token: 'punctuation.definition.string.end.io',
+                regex: '"""',
+                next: 'pop' },
+              { token: 'constant.character.escape.io', regex: '\\\\.' },
+              { defaultToken: 'string.quoted.triple.io' } ] },
+         { token: 'punctuation.definition.string.begin.io',
+           regex: '"',
+           push: 
+            [ { token: 'punctuation.definition.string.end.io',
+                regex: '"',
+                next: 'pop' },
+              { token: 'constant.character.escape.io', regex: '\\\\.' },
+              { defaultToken: 'string.quoted.double.io' } ] },
+         { token: 'constant.numeric.io',
+           regex: '\\b(?:0(?:x|X)[0-9a-fA-F]*|(?:[0-9]+\\.?[0-9]*|\\.[0-9]+)(?:(?:e|E)(?:\\+|-)?[0-9]+)?)(?:L|l|UL|ul|u|U|F|f)?\\b' },
+         { token: 'variable.other.global.io', regex: 'Lobby\\b' },
+         { token: 'constant.language.io',
+           regex: '\\b(?:TRUE|true|FALSE|false|NULL|null|Null|Nil|nil|YES|NO)\\b' } ] }
+    
+    this.normalizeRules();
+};
+
+IoHighlightRules.metaData = { fileTypes: [ 'io' ],
+      keyEquivalent: '^~I',
+      name: 'Io',
+      scopeName: 'source.io' }
+
+
+oop.inherits(IoHighlightRules, TextHighlightRules);
+
+exports.IoHighlightRules = IoHighlightRules;
+});

lib/ace/mode/javascript_highlight_rules.js

@@ -35,7 +35,7 @@ var oop = require("../lib/oop");
 var DocCommentHighlightRules = require("./doc_comment_highlight_rules").DocCommentHighlightRules;
 var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
 
-var JavaScriptHighlightRules = function() {
+var JavaScriptHighlightRules = function(options) {
     // see: https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects
     var keywordMapper = this.createKeywordMapper({
         "variable.language":
@@ -187,11 +187,11 @@ var JavaScriptHighlightRules = function() {
                 regex : identifierRe
             }, {
                 token : "keyword.operator",
-                regex : /--|\+\+|[!$%&*+\-~]|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\|\||\?\:|\*=|%=|\+=|\-=|&=|\^=/,
+                regex : /--|\+\+|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\|\||\?\:|[!$%&*+\-~\/^]=?/,
                 next  : "start"
             }, {
                 token : "punctuation.operator",
-                regex : /\?|\:|\,|\;|\./,
+                regex : /[?:,;.]/,
                 next  : "start"
             }, {
                 token : "paren.lparen",
@@ -200,10 +200,6 @@ var JavaScriptHighlightRules = function() {
             }, {
                 token : "paren.rparen",
                 regex : /[\])}]/
-            }, {
-                token : "keyword.operator",
-                regex : /\/=?/,
-                next  : "start"
             }, {
                 token: "comment",
                 regex: /^#!.*$/
@@ -355,8 +351,48 @@ var JavaScriptHighlightRules = function() {
         ]
     };
     
+    
+    if (!options || !options.noES6) {
+        this.$rules.no_regex.unshift({
+            regex: "[{}]", onMatch: function(val, state, stack) {
+                this.next = val == "{" ? this.nextState : "";
+                if (val == "{" && stack.length) {
+                    stack.unshift("start", state);
+                    return "paren";
+                }
+                if (val == "}" && stack.length) {
+                    stack.shift();
+                    this.next = stack.shift();
+                    if (this.next.indexOf("string") != -1)
+                        return "paren.quasi.end";
+                }
+                return val == "{" ? "paren.lparen" : "paren.rparen";
+            },
+            nextState: "start"
+        }, {
+            token : "string.quasi.start",
+            regex : /`/,
+            push  : [{
+                token : "constant.language.escape",
+                regex : escapedRe
+            }, {
+                token : "paren.quasi.start",
+                regex : /\${/,
+                push  : "start"
+            }, {
+                token : "string.quasi.end",
+                regex : /`/,
+                next  : "pop"
+            }, {
+                defaultToken: "string.quasi"
+            }]
+        });
+    }
+    
     this.embedRules(DocCommentHighlightRules, "doc-",
         [ DocCommentHighlightRules.getEndRule("no_regex") ]);
+    
+    this.normalizeRules();
 };
 
 oop.inherits(JavaScriptHighlightRules, TextHighlightRules);

lib/ace/mode/sh_highlight_rules.js

@@ -99,8 +99,12 @@ var ShHighlightRules = function() {
             }, {
                 defaultToken: "string"
             }]
-        },  {
+        }, {
+            regex : "<<<",
+            token : "keyword.operator"
+        }, {
             stateName: "heredoc",
+            regex : "(<<)(\\s*)(['\"`]?)([\\w\\-]+)(['\"`]?)",
             onMatch : function(value, currentState, stack) {
                 var next = value[2] == '-' ? "indentedHeredoc" : "heredoc";
                 var tokens = value.split(this.splitRegex);
@@ -113,7 +117,6 @@ var ShHighlightRules = function() {
                     {type:"string", value: tokens[5]}
                 ];
             },
-            regex : "(<<-?)(\\s*)(['\"`]?)([\\w\-]+)(['\"`]?)",
             rules: {
                 heredoc: [{
                     onMatch:  function(value, currentState, stack) {
@@ -131,7 +134,7 @@ var ShHighlightRules = function() {
                 }],
                 indentedHeredoc: [{
                     token: "string",
-                    regex: "^ +"
+                    regex: "^\t+"
                 }, {
                     onMatch:  function(value, currentState, stack) {
                         if (value === stack[1]) {
@@ -178,7 +181,7 @@ var ShHighlightRules = function() {
             regex : integer + "\\b"
         }, {
             token : keywordMapper,
-            regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            regex : "[a-zA-Z_][a-zA-Z0-9_]*\\b"
         }, {
             token : "keyword.operator",
             regex : "\\+|\\-|\\*|\\*\\*|\\/|\\/\\/|~|<|>|<=|=>|=|!="

lib/ace/mode/sjs_highlight_rules.js

@@ -36,7 +36,7 @@ var JavaScriptHighlightRules = require("./javascript_highlight_rules").JavaScrip
 var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
 
 var SJSHighlightRules = function() {
-    var parent = new JavaScriptHighlightRules();
+    var parent = new JavaScriptHighlightRules({noES6: true});
     var escapedRe = "\\\\(?:x[0-9a-fA-F]{2}|" + // hex
         "u[0-9a-fA-F]{4}|" + // unicode
         "[0-2][0-7]{0,2}|" + // oct
@@ -177,10 +177,10 @@ var SJSHighlightRules = function() {
     // for use within interpreted contexts
     // without interfering with context nesting
     var embeddableRules = [];
-    for (var i=0; i<this.$rules.no_regex.length; i++) {
+    for (var i=0; i < this.$rules.no_regex.length; i++) {
         var rule = this.$rules.no_regex[i];
         var token = String(rule.token);
-        if(token.indexOf('paren') == -1 && (!rule.next || rule.next.isContextAware)) {
+        if (token.indexOf('paren') == -1 && (!rule.next || rule.next.isContextAware)) {
             embeddableRules.push(rule);
         }
     };

lib/ace/mode/text_highlight_rules.js

@@ -60,7 +60,7 @@ var TextHighlightRules = function() {
             var state = rules[key];
             for (var i = 0; i < state.length; i++) {
                 var rule = state[i];
-                if (rule.next) {
+                if (rule.next || rule.onMatch) {
                     if (typeof rule.next != "string") {
                         if (rule.nextState && rule.nextState.indexOf(prefix) !== 0)
                             rule.nextState = prefix + rule.nextState;
@@ -68,7 +68,6 @@ var TextHighlightRules = function() {
                         if (rule.next.indexOf(prefix) !== 0)
                             rule.next = prefix + rule.next;
                     }
-
                 }
             }
             this.$rules[prefix + key] = state;

lib/ace/snippets/io.js

@@ -0,0 +1,69 @@
+define(function(require, exports, module) {
+"use strict";
+
+exports.snippets = [
+    {
+        "content": "assertEquals(${1:expected}, ${2:expr})",
+        "name": "assertEquals",
+        "scope": "io",
+        "tabTrigger": "ae"
+    },
+    {
+        "content": "${1:${2:newValue} := ${3:Object} }clone do(\n\t$0\n)",
+        "name": "clone do",
+        "scope": "io",
+        "tabTrigger": "cdo"
+    },
+    {
+        "content": "docSlot(\"${1:slotName}\", \"${2:documentation}\")",
+        "name": "docSlot",
+        "scope": "io",
+        "tabTrigger": "ds"
+    },
+    {
+        "content": "(${1:header,}\n\t${2:body}\n)$0",
+        "keyEquivalent": "@(",
+        "name": "Indented Bracketed Line",
+        "scope": "io",
+        "tabTrigger": "("
+    },
+    {
+        "content": "\n\t$0\n",
+        "keyEquivalent": "\r",
+        "name": "Special: Return Inside Empty Parenthesis",
+        "scope": "io meta.empty-parenthesis.io, io meta.comma-parenthesis.io"
+    },
+    {
+        "content": "${1:methodName} := method(${2:args,}\n\t$0\n)",
+        "name": "method",
+        "scope": "io",
+        "tabTrigger": "m"
+    },
+    {
+        "content": "newSlot(\"${1:slotName}\", ${2:defaultValue}, \"${3:docString}\")$0",
+        "name": "newSlot",
+        "scope": "io",
+        "tabTrigger": "ns"
+    },
+    {
+        "content": "${1:name} := Object clone do(\n\t$0\n)",
+        "name": "Object clone do",
+        "scope": "io",
+        "tabTrigger": "ocdo"
+    },
+    {
+        "content": "test${1:SomeFeature} := method(\n\t$0\n)",
+        "name": "testMethod",
+        "scope": "io",
+        "tabTrigger": "ts"
+    },
+    {
+        "content": "${1:Something}Test := ${2:UnitTest} clone do(\n\t$0\n)",
+        "name": "UnitTest",
+        "scope": "io",
+        "tabTrigger": "ut"
+    }
+];
+exports.scope = "io";
+
+});

lib/ace/tokenizer.js

@@ -221,6 +221,10 @@ var Tokenizer = function(rules) {
         if (startState && typeof startState != "string") {
             var stack = startState.slice(0);
             startState = stack[0];
+            if (startState === "#tmp") {
+                stack.shift()
+                startState = stack.shift()
+            }
         } else
             var stack = [];
 
@@ -269,8 +273,6 @@ var Tokenizer = function(rules) {
 
                 if (rule.next) {
                     if (typeof rule.next == "string") {
-                        if (stack.length && stack[0] == currentState && stack[1] == rule.next)
-                            stack.shift();
                         currentState = rule.next;
                     } else {
                         currentState = rule.next(currentState, stack);
@@ -334,7 +336,7 @@ var Tokenizer = function(rules) {
         
         if (stack.length > 1) {
             if (stack[0] !== currentState)
-                stack.unshift(currentState);
+                stack.unshift("#tmp", currentState);
         }
         return {
             tokens : tokens,