jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update more highlighters

Browse source
This commit is contained in:
nightwing 2013-01-12 20:31:42 +04:00
commit c477184f2a
7 changed files with 258 additions and 213 deletions
Download patch file Download diff file Expand all files Collapse all files

197
lib/ace/tokenizer_dev.js
View file

@ -37,51 +37,111 @@ define(function(require, exports, module) {
**/
var Tokenizer = function(rules, flag) {
flag = flag ? "g" + flag : "g";
this.rules = rules;
this.states = rules;
this.regExps = {};
this.matchMappings = {};
for ( var key in this.rules) {
var rule = this.rules[key];
var state = rule;
for (var key in this.states) {
var state = this.states[key];
var ruleRegExps = [];
var matchTotal = 0;
var mapping = this.matchMappings[key] = {};
var mapping = this.matchMappings[key] = {defaultToken: "default.text"};
for ( var i = 0; i < state.length; i++) {
if (state[i].regex instanceof RegExp)
state[i].regex = state[i].regex.toString().slice(1, -1);
for (var i = 0; i < state.length; i++) {
var rule = state[i];
if (rule.defaultToken) {
mapping.defaultToken = rule.defaultToken;
continue;
}
if (rule.regex instanceof RegExp)
rule.regex = rule.regex.toString().slice(1, -1);
// Count number of matching groups. 2 extra groups from the full match
// And the catch-all on the end (used to force a match);
var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
var adjustedregex = rule.regex;
var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
if (Array.isArray(rule.token)) {
if (rule.token.length == 1) {
rule.token = rule.token[0];
} else {
rule.tokenArray = rule.token;
rule.token = this.$arrayTokens;
}
}
// Replace any backreferences and offset appropriately.
var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1);
});
if (matchcount > 1) {
if (/\\\d/.test(rule.regex)) {
// Replace any backreferences and offset appropriately.
adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1);
});
} else {
matchcount = 1;
adjustedregex = this.removeCapturingGroups(rule.regex);
}
if (!rule.splitRegex)
rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
}
if (matchcount > 1 && state[i].token.length !== matchcount-1)
throw new Error("For " + state[i].regex + " the matching groups and length of the token array don't match (rule #" + i + " of state " + key + ")");
mapping[matchTotal] = {
rule: i,
len: matchcount
};
mapping[matchTotal] = i;
matchTotal += matchcount;
ruleRegExps.push(adjustedregex);
}
this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
}
};
(function() {
this.$arrayTokens = function(str) {
if (!str)
return [];
var values = str.split(this.splitRegex)
var tokens = [];
var types = this.tokenArray;
if (types.length != values.length - 2) {
if (window.console)
console.error(types.length , values.length - 2, str, this.splitRegex);
return [{type: "error.invalid", value: str}];
}
for (var i = 0; i < types.length; i++) {
if (values[i + 1]) {
tokens[tokens.length] = {
type: types[i],
value: values[i + 1]
};
}
}
return tokens;
};
this.removeCapturingGroups = function(src) {
var r = src.replace(
/\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
function(x, y) {return y ? "(?:" : x;}
);
return r;
};
this.createSplitterRegexp = function(src, flag) {
src = src.replace(/\(\?=([^()]|\\.)*?\)$/, "");
return new RegExp(src, flag);
};
/**
* Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
* @returns {Object}
**/
this.getLineTokens = function(line, startState) {
if (startState && typeof startState != "string") {
var stack = startState.slice(0);
startState = stack[0];
} else
var stack = [];
var currentState = startState || "start";
var state = this.rules[currentState];
var state = this.states[currentState];
var mapping = this.matchMappings[currentState];
var re = this.regExps[currentState];
re.lastIndex = 0;
@ -110,92 +170,85 @@ var Tokenizer = function(rules, flag) {
var maxRecur = 10000;
while (match = re.exec(line)) {
var type = "default.text";
var type = mapping.defaultToken;
var rule = null;
var value = [match[0]];
var value = match[0];
var index = re.lastIndex;
if (index - value.length > lastIndex) {
var skipped = line.substring(lastIndex, index - value.length);
if (token.type == type) {
token.value += skipped;
} else {
if (token.type)
tokens.push(token);
token = {type: type, value: skipped};
}
}
for (var i = 0; i < match.length-2; i++) {
if (match[i + 1] === undefined)
continue;
if (!maxRecur--) {
throw "infinite" + mapping[i].rule + currentState
throw "infinite" + state[mapping[i]] + currentState
}
rule = state[mapping[i].rule];
if (mapping[i].len > 1)
value = match.slice(i+2, i+1+mapping[i].len);
rule = state[mapping[i]];
// compute token type
if (typeof rule.token == "function")
type = rule.token.apply(this, value);
else
type = rule.token;
type = typeof rule.token == "function"
? rule.token(value, currentState, stack)
: rule.token;
if (rule.next) {
currentState = rule.next;
state = this.rules[currentState];
mapping = this.matchMappings[currentState];
lastIndex = re.lastIndex;
re = this.regExps[currentState];
if (re === undefined) {
throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
state = this.states[currentState];
if (!state) {
window.console && console.error && console.error(currentState, "doesn't exist");
currentState = "start";
state = this.states[currentState];
}
re.lastIndex = lastIndex;
mapping = this.matchMappings[currentState];
lastIndex = index;
re = this.regExps[currentState];
re.lastIndex = index;
onStateChange();
}
break;
}
if (value[0]) {
if (value) {
if (typeof type == "string") {
value = [value.join("")];
type = [type];
}
for (var i = 0; i < value.length; i++) {
if (!value[i])
continue;
var mergeable = (!rule || rule.merge || type[i] === "text") && token.type === type[i];
if (false && mergeable) {
token.value += value[i];
if ((!rule || rule.merge !== false) && token.type === type) {
token.value += value;
} else {
if (token.type) {
token.stateTransitions = stateTransitions;
if (token.type)
tokens.push(token);
initState()
}
token = {
type: type[i],
value: value[i],
state: currentState,
mergeable: mergeable
};
token = {type: type, value: value};
}
} else {
if (token.type)
tokens.push(token);
token = {type: null, value: ""};
for (var i = 0; i < type.length; i++)
tokens.push(type[i]);
}
}
if (lastIndex == line.length)
break;
lastIndex = re.lastIndex;
lastIndex = index;
}
if (token.type) {
token.stateTransitions = stateTransitions;
if (token.type)
tokens.push(token);
}
return {
tokens : tokens,
state : currentState
state : stack.length ? stack : currentState
};
};