removed the 'match[i + 1].length' clause in order to support tokenizing on an empty line (all the test cases pass without it)

This commit is contained in:
Daniel Krech 2011-02-10 12:20:27 +08:00 committed by Fabian Jakobs
commit e1f5a64327

View file

@ -75,7 +75,7 @@ var Tokenizer = function(rules) {
var value = match[0];
for ( var i = 0; i < state.length; i++) {
if (match[i + 1] !== undefined && match[i + 1].length) {
if (match[i + 1] !== undefined) {
if (typeof state[i].token == "function") {
type = state[i].token(match[0]);
}