jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add unit test to match regular expression literals (still fialing)

Browse source
This commit is contained in:
Fabian Jakobs 2011-02-22 23:39:01 +01:00
commit 90f57f682b
2 changed files with 25 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

5
lib/ace/test/mode/javascript_tokenizer_test.js
View file

@ -98,6 +98,11 @@ var Test = {
assert.equal(1, tokens.length);
assert.equal("rparen", tokens[0].type);
},
"test tokenize regular expressions": function() {
var tokens = this.tokenizer.getLineTokens("a/b/c", "start").tokens;
assert.equal(5, tokens.length);
}
};

41
lib/ace/tokenizer.js
View file

@ -42,14 +42,15 @@ var Tokenizer = function(rules) {
this.regExps = {};
for ( var key in this.rules) {
var state = this.rules[key];
var rule = this.rules[key];
var state = rule;
var ruleRegExps = [];
for ( var i = 0; i < state.length; i++) {
for ( var i = 0; i < state.length; i++)
ruleRegExps.push(state[i].regex);
};
this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
}
};
@ -76,19 +77,19 @@ var Tokenizer = function(rules) {
for ( var i = 0; i < state.length; i++) {
if (match[i + 1]) {
if (typeof state[i].token == "function") {
type = state[i].token(match[0]);
}
else {
type = state[i].token;
}
var rule = state[i];
if (typeof rule.token == "function")
type = rule.token(match[0]);
else
type = rule.token;
if (state[i].next && state[i].next !== currentState) {
currentState = state[i].next;
var state = this.rules[currentState];
var lastIndex = re.lastIndex;
if (rule.next && rule.next !== currentState) {
currentState = rule.next;
state = this.rules[currentState];
lastIndex = re.lastIndex;
var re = this.regExps[currentState];
re = this.regExps[currentState];
re.lastIndex = lastIndex;
}
break;
@ -97,9 +98,9 @@ var Tokenizer = function(rules) {
if (token.type !== type) {
if (token.type) {
if (token.type)
tokens.push(token);
}
token = {
type: type,
value: value
@ -108,16 +109,14 @@ var Tokenizer = function(rules) {
token.value += value;
}
if (lastIndex == line.length) {
break;
}
if (lastIndex == line.length)
break;
lastIndex = re.lastIndex;
};
if (token.type) {
if (token.type)
tokens.push(token);
}
return {
tokens : tokens,