jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add unit test to match regular expression literals (still fialing)

Browse source
This commit is contained in:
Fabian Jakobs 2011-02-22 23:39:01 +01:00
commit 90f57f682b
2 changed files with 25 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

5
lib/ace/test/mode/javascript_tokenizer_test.js
View file

@ -98,6 +98,11 @@ var Test = {
assert.equal(1, tokens.length); assert.equal(1, tokens.length);
assert.equal("rparen", tokens[0].type); assert.equal("rparen", tokens[0].type);
},
"test tokenize regular expressions": function() {
var tokens = this.tokenizer.getLineTokens("a/b/c", "start").tokens;
assert.equal(5, tokens.length);
} }
}; };

41
lib/ace/tokenizer.js
View file

@ -42,14 +42,15 @@ var Tokenizer = function(rules) {
this.regExps = {}; this.regExps = {};
for ( var key in this.rules) { for ( var key in this.rules) {
var state = this.rules[key]; var rule = this.rules[key];
var state = rule;
var ruleRegExps = []; var ruleRegExps = [];
for ( var i = 0; i < state.length; i++) { for ( var i = 0; i < state.length; i++)
ruleRegExps.push(state[i].regex); ruleRegExps.push(state[i].regex);
};
this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g"); this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
} }
}; };
@ -76,19 +77,19 @@ var Tokenizer = function(rules) {
for ( var i = 0; i < state.length; i++) { for ( var i = 0; i < state.length; i++) {
if (match[i + 1]) { if (match[i + 1]) {
if (typeof state[i].token == "function") { var rule = state[i];
type = state[i].token(match[0]);
}
else {
type = state[i].token;
}
if (state[i].next && state[i].next !== currentState) { if (typeof rule.token == "function")
currentState = state[i].next; type = rule.token(match[0]);
var state = this.rules[currentState]; else
var lastIndex = re.lastIndex; type = rule.token;
var re = this.regExps[currentState]; if (rule.next && rule.next !== currentState) {
currentState = rule.next;
state = this.rules[currentState];
lastIndex = re.lastIndex;
re = this.regExps[currentState];
re.lastIndex = lastIndex; re.lastIndex = lastIndex;
} }
break; break;
@ -97,9 +98,9 @@ var Tokenizer = function(rules) {
if (token.type !== type) { if (token.type !== type) {
if (token.type) { if (token.type)
tokens.push(token); tokens.push(token);
}
token = { token = {
type: type, type: type,
value: value value: value
@ -108,16 +109,14 @@ var Tokenizer = function(rules) {
token.value += value; token.value += value;
} }
if (lastIndex == line.length) { if (lastIndex == line.length)
break; break;
}
lastIndex = re.lastIndex; lastIndex = re.lastIndex;
}; };
if (token.type) { if (token.type)
tokens.push(token); tokens.push(token);
}
return { return {
tokens : tokens, tokens : tokens,