allow tokenizer states with only defaultToken
This commit is contained in:
parent
72a509e542
commit
7e2dda1e18
2 changed files with 6 additions and 1 deletions
|
|
@ -62,7 +62,7 @@ var Mode = function() {
|
|||
|
||||
this.getTokenizer = function() {
|
||||
if (!this.$tokenizer) {
|
||||
this.$highlightRules = new this.HighlightRules();
|
||||
this.$highlightRules = this.$highlightRules || new this.HighlightRules();
|
||||
this.$tokenizer = new Tokenizer(this.$highlightRules.getRules());
|
||||
}
|
||||
return this.$tokenizer;
|
||||
|
|
|
|||
|
|
@ -116,6 +116,11 @@ var Tokenizer = function(rules) {
|
|||
rule.onMatch = null;
|
||||
}
|
||||
|
||||
if (!ruleRegExps.length) {
|
||||
mapping[0] = 0;
|
||||
ruleRegExps.push("$");
|
||||
}
|
||||
|
||||
splitterRurles.forEach(function(rule) {
|
||||
rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
|
||||
}, this);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue