jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add {defaultToken: "..."} rules for skipped text

Browse source
This commit is contained in:
nightwing 2013-01-04 16:41:54 +04:00
commit ea857f23c2
1 changed files with 52 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

96
lib/ace/tokenizer.js
View file

@ -50,19 +50,22 @@ define(function(require, exports, module) {
**/ **/
var Tokenizer = function(rules, flag) { var Tokenizer = function(rules, flag) {
flag = flag ? "g" + flag : "g"; flag = flag ? "g" + flag : "g";
this.rules = rules; this.states = rules;
this.regExps = {}; this.regExps = {};
this.matchMappings = {}; this.matchMappings = {};
for (var key in this.rules) { for (var key in this.states) {
var rule = this.rules[key]; var state = this.states[key];
var state = rule;
var ruleRegExps = []; var ruleRegExps = [];
var matchTotal = 0; var matchTotal = 0;
var mapping = this.matchMappings[key] = {}; var mapping = this.matchMappings[key] = {defaultToken: "text"};
for (var i = 0; i < state.length; i++) { for (var i = 0; i < state.length; i++) {
var rule = state[i]; var rule = state[i];
if (rule.defaultToken) {
mapping.defaultToken = rule.defaultToken;
continue;
}
if (rule.regex instanceof RegExp) if (rule.regex instanceof RegExp)
rule.regex = rule.regex.toString().slice(1, -1); rule.regex = rule.regex.toString().slice(1, -1);
@ -71,17 +74,12 @@ var Tokenizer = function(rules, flag) {
var adjustedregex = rule.regex var adjustedregex = rule.regex
var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2; var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
if (Array.isArray(rule.token)) { if (Array.isArray(rule.token)) {
if (rule.token.length != matchcount - 1) { if (rule.token.length == 1) {
throw new Error( rule.token = rule.token[0];
"For " + rule.regex + } else {
" the matching groups (" +(matchcount-1) + rule.tokenArray = rule.token;
") and length of the token array (" + rule.token.length + rule.token = this.$arrayTokens;
") don't match (rule #" + i + " of state " + key + ")"
);
} }
rule.tokenArray = rule.token;
rule.token = this.$splitHelper;
} }
if (matchcount > 1) { if (matchcount > 1) {
@ -89,19 +87,16 @@ var Tokenizer = function(rules, flag) {
// Replace any backreferences and offset appropriately. // Replace any backreferences and offset appropriately.
adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) { adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1); return "\\" + (parseInt(digit, 10) + matchTotal + 1);
}); });
} else { } else {
matchcount = 1; matchcount = 1;
adjustedregex = this.nonCapturingRegexp(rule.regex); adjustedregex = this.removeCapturingGroups(rule.regex);
} }
if (!rule.splitRegex) if (!rule.splitRegex)
rule.splitRegex = new RegExp(rule.regex) rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
} }
mapping[matchTotal] = { mapping[matchTotal] = i;
rule: i,
len: matchcount
};
matchTotal += matchcount; matchTotal += matchcount;
ruleRegExps.push(adjustedregex); ruleRegExps.push(adjustedregex);
@ -112,10 +107,14 @@ var Tokenizer = function(rules, flag) {
}; };
(function() { (function() {
this.$splitHelper = function(str) { this.$arrayTokens = function(str) {
var values = str.split(this.splitRegex) var values = str.split(this.splitRegex)
var tokens = []; var tokens = [];
var types = this.tokenArray; var types = this.tokenArray;
if (types.length != values.length - 2) {
console.log(types.length , values.length - 2, str, this.splitRegex)
return [{type: "error.invalid", value: str}];
}
for (var i = 0; i < types.length; i++) { for (var i = 0; i < types.length; i++) {
if (values[i + 1]) { if (values[i + 1]) {
tokens[tokens.length] = { tokens[tokens.length] = {
@ -125,12 +124,19 @@ var Tokenizer = function(rules, flag) {
} }
} }
return tokens; return tokens;
} };
this.nonCapturingRegexp = function(src) {
return src.replace( this.removeCapturingGroups = function(src) {
/\[(?:\\.|[^\]])*?\]|\\.|\((?:\?:|\?=|!=)|(\()/g, var r = src.replace(
/\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
function(x, y) {return y ? "(?:" : x;} function(x, y) {return y ? "(?:" : x;}
); );
return r;
};
this.createSplitterRegexp = function(src, flag) {
src = src.replace(/\(\?=([^()]|\\.)*?\)$/, "");
return new RegExp(src, flag);
}; };
/** /**
@ -145,7 +151,7 @@ var Tokenizer = function(rules, flag) {
var stack = []; var stack = [];
var currentState = startState || "start"; var currentState = startState || "start";
var state = this.rules[currentState]; var state = this.states[currentState];
var mapping = this.matchMappings[currentState]; var mapping = this.matchMappings[currentState];
var re = this.regExps[currentState]; var re = this.regExps[currentState];
re.lastIndex = 0; re.lastIndex = 0;
@ -156,38 +162,40 @@ var Tokenizer = function(rules, flag) {
var token = {type: null, value: ""}; var token = {type: null, value: ""};
while (match = re.exec(line)) { while (match = re.exec(line)) {
var type = "text"; var type = mapping.defaultToken;
var rule = null; var rule = null;
var value = match[0]; var value = match[0];
var index = re.lastIndex; var index = re.lastIndex;
if (index - value.length > lastIndex) { if (index - value.length > lastIndex) {
if (token.type) var skipped = line.substring(lastIndex, index - value.length);
tokens.push(token); if (token.type == type) {
token = { token.value += skipped;
type: "regex", } else {
value: line.substring(lastIndex, index - value.length) if (token.type)
}; tokens.push(token);
token = {type: type, value: skipped};
}
} }
for (var i = 0; i < match.length-2; i++) { for (var i = 0; i < match.length-2; i++) {
if (match[i + 1] === undefined) if (match[i + 1] === undefined)
continue; continue;
rule = state[mapping[i].rule]; rule = state[mapping[i]];
// compute token type // compute token type
if (typeof rule.token == "function") type = typeof rule.token == "function"
type = rule.token(value, currentState, stack); ? rule.token(value, currentState, stack)
else : rule.token;
type = rule.token;
if (rule.next) { if (rule.next) {
currentState = rule.next; currentState = rule.next;
state = this.rules[currentState]; state = this.states[currentState];
if (!state) { if (!state) {
window.console && console.error && console.error(currentState, "doesn't exist"); window.console && console.error && console.error(currentState, "doesn't exist");
currentState = "start"; currentState = "start";
state = this.rules[currentState]; state = this.states[currentState];
} }
mapping = this.matchMappings[currentState]; mapping = this.matchMappings[currentState];
lastIndex = index; lastIndex = index;