jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #253 from fivesixty/master

Browse source 
Add matching group and back reference support to tokenizer rules.
This commit is contained in:
Fabian Jakobs 2011-05-20 00:57:51 -07:00
commit 993f2a17ee
3 changed files with 61 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

8
lib/ace/mode/css_tokenizer_test.js
View file

@ -51,21 +51,21 @@ module.exports = {
"test: tokenize pixel number" : function() { "test: tokenize pixel number" : function() {
var line = "-12px"; var line = "-12px";
var tokens = this.tokenizer.getLineTokens(line, "start").tokens; var tokens = this.tokenizer.getLineTokens(line, "ruleset").tokens;
assert.equal(1, tokens.length); assert.equal(1, tokens.length);
assert.equal("constant.numeric", tokens[0].type); assert.equal("constant.numeric", tokens[0].type);
}, },
"test: tokenize hex3 color" : function() { "test: tokenize hex3 color" : function() {
var tokens = this.tokenizer.getLineTokens("#abc", "start").tokens; var tokens = this.tokenizer.getLineTokens("#abc", "ruleset").tokens;
assert.equal(1, tokens.length); assert.equal(1, tokens.length);
assert.equal("constant.numeric", tokens[0].type); assert.equal("constant.numeric", tokens[0].type);
}, },
"test: tokenize hex6 color" : function() { "test: tokenize hex6 color" : function() {
var tokens = this.tokenizer.getLineTokens("#abc012", "start").tokens; var tokens = this.tokenizer.getLineTokens("#abc012", "ruleset").tokens;
assert.equal(1, tokens.length); assert.equal(1, tokens.length);
assert.equal("constant.numeric", tokens[0].type); assert.equal("constant.numeric", tokens[0].type);
@ -81,7 +81,7 @@ module.exports = {
}, },
"test for last rule in ruleset to catch capturing group bugs" : function() { "test for last rule in ruleset to catch capturing group bugs" : function() {
var tokens = this.tokenizer.getLineTokens("top", "start").tokens; var tokens = this.tokenizer.getLineTokens("top", "ruleset").tokens;
assert.equal(1, tokens.length); assert.equal(1, tokens.length);
assert.equal("support.type", tokens[0].type); assert.equal("support.type", tokens[0].type);

4
lib/ace/test/mockdom.js
View file

@ -1,5 +1,5 @@
var dom = require('jsdom/level2/html').dom.level2.html; var dom = require('jsdom/lib/jsdom/level2/html').dom.level2.html;
var browser = require('jsdom/browser/index').windowAugmentation(dom); var browser = require('jsdom/lib/jsdom/browser/index').windowAugmentation(dom);
global.document = browser.document; global.document = browser.document;
global.window = browser.window; global.window = browser.window;

76
lib/ace/tokenizer.js
View file

@ -41,13 +41,32 @@ var Tokenizer = function(rules) {
this.rules = rules; this.rules = rules;
this.regExps = {}; this.regExps = {};
this.matchMappings = {};
for ( var key in this.rules) { for ( var key in this.rules) {
var rule = this.rules[key]; var rule = this.rules[key];
var state = rule; var state = rule;
var ruleRegExps = []; var ruleRegExps = [];
var matchTotal = 0;
for ( var i = 0; i < state.length; i++) var mapping = this.matchMappings[key] = {};
ruleRegExps.push(state[i].regex);
for ( var i = 0; i < state.length; i++) {
// Count number of matching groups. 2 extra groups from the full match
// And the catch-all on the end (used to force a match);
var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
// Replace any backreferences and offset appropriately.
var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1);
});
mapping[matchTotal] = {
rule: i,
len: matchcount
};
matchTotal += matchcount;
ruleRegExps.push(adjustedregex);
}
this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g"); this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
@ -59,34 +78,40 @@ var Tokenizer = function(rules) {
this.getLineTokens = function(line, startState) { this.getLineTokens = function(line, startState) {
var currentState = startState; var currentState = startState;
var state = this.rules[currentState]; var state = this.rules[currentState];
var mapping = this.matchMappings[currentState];
var re = this.regExps[currentState]; var re = this.regExps[currentState];
re.lastIndex = 0; re.lastIndex = 0;
var match, tokens = []; var match, tokens = [];
var lastIndex = 0; var lastIndex = 0;
var token = { var token = {
type: null, type: null,
value: "" value: ""
}; };
while (match = re.exec(line)) { while (match = re.exec(line)) {
var type = "text"; var type = "text";
var value = match[0]; var value = [match[0]];
for ( var i = 0; i < state.length; i++) { for ( var i = 0; i < match.length-2; i++) {
if (match[i + 1] !== undefined) { if (match[i + 1] !== undefined) {
var rule = state[i]; var rule = state[mapping[i].rule];
if (mapping[i].len > 1) {
value = match.slice(i+2, i+1+mapping[i].len);
}
if (typeof rule.token == "function") if (typeof rule.token == "function")
type = rule.token(match[0]); type = rule.token.apply(this, value);
else else
type = rule.token; type = rule.token;
if (rule.next && rule.next !== currentState) { if (rule.next && rule.next !== currentState) {
currentState = rule.next; currentState = rule.next;
state = this.rules[currentState]; state = this.rules[currentState];
mapping = this.matchMappings[currentState];
lastIndex = re.lastIndex; lastIndex = re.lastIndex;
re = this.regExps[currentState]; re = this.regExps[currentState];
@ -96,17 +121,26 @@ var Tokenizer = function(rules) {
} }
}; };
if (typeof type == "string") {
if (token.type !== type) { if (typeof value != "string") {
if (token.type) value = [value.join("")];
tokens.push(token); }
type = [type];
}
for ( var i = 0; i < value.length; i++) {
if (token.type !== type[i]) {
if (token.type) {
tokens.push(token);
}
token = { token = {
type: type, type: type[i],
value: value value: value[i]
}; }
} else { } else {
token.value += value; token.value += value[i];
}
} }
if (lastIndex == line.length) if (lastIndex == line.length)