check for valid start state in tokenizer

This commit is contained in:
nightwing 2013-11-24 16:10:40 +04:00
commit 1704cbdcc2

View file

@ -95,7 +95,7 @@ var Tokenizer = function(rules) {
if (matchcount > 1) {
if (/\\\d/.test(rule.regex)) {
// Replace any backreferences and offset appropriately.
adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) {
adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function(match, digit) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1);
});
} else {
@ -222,6 +222,10 @@ var Tokenizer = function(rules) {
var currentState = startState || "start";
var state = this.states[currentState];
if (!state) {
currentState = "start";
state = this.states[currentState];
}
var mapping = this.matchMappings[currentState];
var re = this.regExps[currentState];
re.lastIndex = 0;