allow tokenizer states with only defaultToken

This commit is contained in:
nightwing 2014-05-21 20:38:07 +04:00
commit 7e2dda1e18
2 changed files with 6 additions and 1 deletions

View file

@ -62,7 +62,7 @@ var Mode = function() {
this.getTokenizer = function() {
if (!this.$tokenizer) {
this.$highlightRules = new this.HighlightRules();
this.$highlightRules = this.$highlightRules || new this.HighlightRules();
this.$tokenizer = new Tokenizer(this.$highlightRules.getRules());
}
return this.$tokenizer;

View file

@ -116,6 +116,11 @@ var Tokenizer = function(rules) {
rule.onMatch = null;
}
if (!ruleRegExps.length) {
mapping[0] = 0;
ruleRegExps.push("$");
}
splitterRurles.forEach(function(rule) {
rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
}, this);