reaadd tokenizer check
This commit is contained in:
parent
faec023507
commit
4b3d183ba7
2 changed files with 8 additions and 4 deletions
|
|
@ -107,10 +107,14 @@ var Tokenizer = function(rules) {
|
||||||
} else {
|
} else {
|
||||||
token.value += value;
|
token.value += value;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (lastIndex == line.length) {
|
if (lastIndex == line.length) {
|
||||||
break;
|
break;
|
||||||
}
|
} else if (re.lastIndex == lastIndex) {
|
||||||
lastIndex = re.lastIndex;
|
throw new Error("tokenizer error");
|
||||||
|
}
|
||||||
|
|
||||||
|
lastIndex = re.lastIndex;
|
||||||
};
|
};
|
||||||
|
|
||||||
if (token.type) {
|
if (token.type) {
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
Subproject commit c0cab19cbb5d44df98bae0060c782a6f30464afe
|
Subproject commit 67a380309e5b139a9603334ad9d9f917659f04bc
|
||||||
Loading…
Add table
Add a link
Reference in a new issue