jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

allow array of states in tokenizer

Browse source
This commit is contained in:
nightwing 2012-12-28 19:13:30 +04:00
commit 08e621b443
2 changed files with 13 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

2
lib/ace/background_tokenizer.js
View file

@ -236,7 +236,7 @@ var BackgroundTokenizer = function(tokenizer, editor) {
data.state = "start";
}
if (this.states[row] !== data.state) {
if (this.states[row] + "" !== data.state + "") {
this.states[row] = data.state;
this.lines[row + 1] = null;
if (this.currentLine > row + 1)

19
lib/ace/tokenizer.js
View file

@ -54,15 +54,14 @@ var Tokenizer = function(rules, flag) {
this.regExps = {};
this.matchMappings = {};
for ( var key in this.rules) {
for (var key in this.rules) {
var rule = this.rules[key];
var state = rule;
var ruleRegExps = [];
var matchTotal = 0;
var mapping = this.matchMappings[key] = {};
for ( var i = 0; i < state.length; i++) {
for (var i = 0; i < state.length; i++) {
if (state[i].regex instanceof RegExp)
state[i].regex = state[i].regex.toString().slice(1, -1);
@ -75,7 +74,7 @@ var Tokenizer = function(rules, flag) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1);
});
if (matchcount > 1 && state[i].token.length !== matchcount-1)
if (matchcount > 1 && typeof state[i].token == "string" && state[i].token.length !== matchcount-1)
throw new Error("For " + state[i].regex + " the matching groups (" +(matchcount-1) + ") and length of the token array (" + state[i].token.length + ") don't match (rule #" + i + " of state " + key + ")");
mapping[matchTotal] = {
@ -98,6 +97,12 @@ var Tokenizer = function(rules, flag) {
* @returns {Object}
**/
this.getLineTokens = function(line, startState) {
if (startState && typeof startState != "string") {
var stack = startState.slice(0);
startState = stack[0];
} else {
var stack = []
}
var currentState = startState || "start";
var state = this.rules[currentState];
var mapping = this.matchMappings[currentState];
@ -129,7 +134,7 @@ var Tokenizer = function(rules, flag) {
// compute token type
if (typeof rule.token == "function")
type = rule.token.apply(this, value);
type = rule.token(value.length == 1 ? value[0] : value, currentState, stack);
else
type = rule.token;
@ -142,7 +147,7 @@ var Tokenizer = function(rules, flag) {
re = this.regExps[currentState];
if (re === undefined) {
throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
}
re.lastIndex = lastIndex;
@ -184,7 +189,7 @@ var Tokenizer = function(rules, flag) {
return {
tokens : tokens,
state : currentState
state : stack.length ? stack : currentState
};
};