removed the 'match[i + 1].length' clause in order to support tokenizing on an empty line (all the test cases pass without it)
This commit is contained in:
parent
a34b2c6e82
commit
e1f5a64327
1 changed files with 1 additions and 1 deletions
|
|
@ -75,7 +75,7 @@ var Tokenizer = function(rules) {
|
||||||
var value = match[0];
|
var value = match[0];
|
||||||
|
|
||||||
for ( var i = 0; i < state.length; i++) {
|
for ( var i = 0; i < state.length; i++) {
|
||||||
if (match[i + 1] !== undefined && match[i + 1].length) {
|
if (match[i + 1] !== undefined) {
|
||||||
if (typeof state[i].token == "function") {
|
if (typeof state[i].token == "function") {
|
||||||
type = state[i].token(match[0]);
|
type = state[i].token(match[0]);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue