removed the 'match[i + 1].length' clause in order to support tokenizing on an empty line (all the test cases pass without it)
This commit is contained in:
parent
a34b2c6e82
commit
e1f5a64327
1 changed files with 1 additions and 1 deletions
|
|
@ -75,7 +75,7 @@ var Tokenizer = function(rules) {
|
|||
var value = match[0];
|
||||
|
||||
for ( var i = 0; i < state.length; i++) {
|
||||
if (match[i + 1] !== undefined && match[i + 1].length) {
|
||||
if (match[i + 1] !== undefined) {
|
||||
if (typeof state[i].token == "function") {
|
||||
type = state[i].token(match[0]);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue