jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

minimize number of capturing groups

Browse source
This commit is contained in:
nightwing 2013-01-03 18:31:28 +04:00
commit 903b7a1951
1 changed files with 73 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

116
lib/ace/tokenizer.js
View file

@ -3,7 +3,7 @@
*
* Copyright (c) 2010, Ajax.org B.V.
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
@ -14,7 +14,7 @@
* * Neither the name of Ajax.org B.V. nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@ -32,7 +32,7 @@ define(function(require, exports, module) {
"use strict";
/**
*
*
*
* This class takes a set of highlighting rules, and creates a tokenizer out of them. For more information, see [the wiki on extending highlighters](https://github.com/ajaxorg/ace/wiki/Creating-or-Extending-an-Edit-Mode#wiki-extendingTheHighlighter).
* @class Tokenizer
@ -43,9 +43,9 @@ define(function(require, exports, module) {
* @param {Object} rules The highlighting rules
* @param {String} flag Any additional regular expression flags to pass (like "i" for case insensitive)
*
*
*
*
*
*
* @constructor
**/
var Tokenizer = function(rules, flag) {
@ -62,21 +62,42 @@ var Tokenizer = function(rules, flag) {
var mapping = this.matchMappings[key] = {};
for (var i = 0; i < state.length; i++) {
if (state[i].regex instanceof RegExp)
state[i].regex = state[i].regex.toString().slice(1, -1);
var rule = state[i];
if (rule.regex instanceof RegExp)
rule.regex = rule.regex.toString().slice(1, -1);
// Count number of matching groups. 2 extra groups from the full match
// And the catch-all on the end (used to force a match);
var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
// Replace any backreferences and offset appropriately.
var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1);
});
if (matchcount > 1 && typeof state[i].token == "string" && state[i].token.length !== matchcount-1)
throw new Error("For " + state[i].regex + " the matching groups (" +(matchcount-1) + ") and length of the token array (" + state[i].token.length + ") don't match (rule #" + i + " of state " + key + ")");
var adjustedregex = rule.regex
var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
if (Array.isArray(rule.token)) {
if (rule.token.length != matchcount - 1) {
throw new Error(
"For " + rule.regex +
" the matching groups (" +(matchcount-1) +
") and length of the token array (" + rule.token.length +
") don't match (rule #" + i + " of state " + key + ")"
);
}
rule.tokenArray = rule.token;
rule.token = this.$splitHelper;
}
if (matchcount > 1) {
if (/\\\d/.test(rule.regex)) {
// Replace any backreferences and offset appropriately.
adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1);
});
} else {
matchcount = 1;
adjustedregex = this.nonCapturingRegexp(rule.regex);
}
if (!rule.splitRegex)
rule.splitRegex = new RegExp(rule.regex)
}
mapping[matchTotal] = {
rule: i,
len: matchcount
@ -85,12 +106,33 @@ var Tokenizer = function(rules, flag) {
ruleRegExps.push(adjustedregex);
}
console.log(key, ruleRegExps)
this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
}
};
(function() {
this.$splitHelper = function(str) {
var values = str.split(this.splitRegex)
var tokens = [];
var types = this.tokenArray;
for (var i = 0; i < types.length; i++) {
if (values[i + 1]) {
tokens[tokens.length] = {
type: types[i],
value: values[i + 1]
};
}
}
return tokens;
}
this.nonCapturingRegexp = function(src) {
return src.replace(
/\[(?:\\.|[^\]])*?\]|\\.|\((?:\?:|\?=|!=)|(\()/g,
function(x, y) {return y ? "(?:" : x;}
);
};
/**
* Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
@ -100,9 +142,9 @@ var Tokenizer = function(rules, flag) {
if (startState && typeof startState != "string") {
var stack = startState.slice(0);
startState = stack[0];
} else {
} else
var stack = []
}
var currentState = startState || "start";
var state = this.rules[currentState];
var mapping = this.matchMappings[currentState];
@ -110,18 +152,14 @@ var Tokenizer = function(rules, flag) {
re.lastIndex = 0;
var match, tokens = [];
var lastIndex = 0;
var token = {
type: null,
value: ""
};
var token = {type: null, value: ""};
while (match = re.exec(line)) {
var type = "text";
var rule = null;
var value = [match[0]];
var value = match[0];
for (var i = 0; i < match.length-2; i++) {
if (match[i + 1] === undefined)
@ -129,12 +167,9 @@ var Tokenizer = function(rules, flag) {
rule = state[mapping[i].rule];
if (mapping[i].len > 1)
value = match.slice(i+2, i+1+mapping[i].len);
// compute token type
if (typeof rule.token == "function")
type = rule.token(value.length == 1 ? value[0] : value, currentState, stack);
type = rule.token(value, currentState, stack);
else
type = rule.token;
@ -154,26 +189,21 @@ var Tokenizer = function(rules, flag) {
break;
}
if (value[0]) {
if (value) {
if (typeof type == "string") {
value = [value.join("")];
type = [type];
}
for (var i = 0; i < value.length; i++) {
if (!value[i])
continue;
if ((!rule || rule.merge !== false) && token.type === type[i]) {
token.value += value[i];
if ((!rule || rule.merge !== false) && token.type === type) {
token.value += value;
} else {
if (token.type)
tokens.push(token);
token = {
type: type[i],
value: value[i]
};
token = {type: type, value: value};
}
} else {
if (token.type)
tokens.push(token);
token = {type: null, value: ""};
for (var i = 0; i < type.length; i++)
tokens.push(type[i]);
}
}