allow tokenizer states with only defaultToken
This commit is contained in:
parent
72a509e542
commit
7e2dda1e18
2 changed files with 6 additions and 1 deletions
|
|
@ -62,7 +62,7 @@ var Mode = function() {
|
||||||
|
|
||||||
this.getTokenizer = function() {
|
this.getTokenizer = function() {
|
||||||
if (!this.$tokenizer) {
|
if (!this.$tokenizer) {
|
||||||
this.$highlightRules = new this.HighlightRules();
|
this.$highlightRules = this.$highlightRules || new this.HighlightRules();
|
||||||
this.$tokenizer = new Tokenizer(this.$highlightRules.getRules());
|
this.$tokenizer = new Tokenizer(this.$highlightRules.getRules());
|
||||||
}
|
}
|
||||||
return this.$tokenizer;
|
return this.$tokenizer;
|
||||||
|
|
|
||||||
|
|
@ -116,6 +116,11 @@ var Tokenizer = function(rules) {
|
||||||
rule.onMatch = null;
|
rule.onMatch = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!ruleRegExps.length) {
|
||||||
|
mapping[0] = 0;
|
||||||
|
ruleRegExps.push("$");
|
||||||
|
}
|
||||||
|
|
||||||
splitterRurles.forEach(function(rule) {
|
splitterRurles.forEach(function(rule) {
|
||||||
rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
|
rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
|
||||||
}, this);
|
}, this);
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue