jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix Tokenizer.createSplitterRegexp and add tests

Browse source 
fixes #1317
This commit is contained in:
nightwing 2013-03-20 11:44:32 +04:00
commit a3de845615
3 changed files with 81 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

1
lib/ace/test/all_browser.js
View file

@ -47,6 +47,7 @@ var testNames = [
"ace/search_test",
"ace/selection_test",
"ace/token_iterator_test",
"ace/tokenizer_test",
"ace/virtual_renderer_test"
];

20
lib/ace/tokenizer.js
View file

@ -66,7 +66,7 @@ var Tokenizer = function(rules) {
flag = "gi";
if (rule.regex == null)
continue;
if (rule.regex instanceof RegExp)
rule.regex = rule.regex.toString().slice(1, -1);
@ -87,7 +87,7 @@ var Tokenizer = function(rules) {
else
rule.onMatch = rule.token;
}
if (matchcount > 1) {
if (/\\\d/.test(rule.regex)) {
// Replace any backreferences and offset appropriately.
@ -106,7 +106,7 @@ var Tokenizer = function(rules) {
matchTotal += matchcount;
ruleRegExps.push(adjustedregex);
// makes property access faster
if (!rule.onMatch)
rule.onMatch = null;
@ -121,7 +121,7 @@ var Tokenizer = function(rules) {
this.$applyToken = function(str) {
var values = this.splitRegex.exec(str).slice(1);
var types = this.token.apply(this, values);
// required for compatibility with old modes
if (typeof types === "string")
return [{type: types, value: str}];
@ -157,7 +157,7 @@ var Tokenizer = function(rules) {
}
return tokens;
};
this.removeCapturingGroups = function(src) {
var r = src.replace(
/\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
@ -165,13 +165,13 @@ var Tokenizer = function(rules) {
);
return r;
};
this.createSplitterRegexp = function(src, flag) {
if (src.indexOf("(?=") != -1) {
var stack = 0;
var inChClass = false;
var lastCapture = {};
src.replace(/(\\.)|(\((?:\?[=!])?)|(\))|([])/g, function(
src.replace(/(\\.)|(\((?:\?[=!])?)|(\))|([\[\]])/g, function(
m, esc, parenOpen, parenClose, square, index
) {
if (inChClass) {
@ -179,8 +179,10 @@ var Tokenizer = function(rules) {
} else if (square) {
inChClass = true;
} else if (parenClose) {
if (stack == lastCapture.stack)
lastCapture.end = index+1
if (stack == lastCapture.stack) {
lastCapture.end = index+1;
lastCapture.stack = -1;
}
stack--;
} else if (parenOpen) {
stack++;

69
lib/ace/tokenizer_test.js Normal file
View file

@ -0,0 +1,69 @@
/* ***** BEGIN LICENSE BLOCK *****
* Distributed under the BSD license:
*
* Copyright (c) 2010, Ajax.org B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Ajax.org B.V. nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ***** END LICENSE BLOCK ***** */
if (typeof process !== "undefined") {
require("amd-loader");
}
define(function(require, exports, module) {
"use strict";
var Tokenizer = require("./tokenizer").Tokenizer;
var assert = require("./test/assertions");
module.exports = {
"test: createSplitterRegexp" : function() {
var t = new Tokenizer({});
var re = t.createSplitterRegexp("(a)(b)(?=[x)(])");
assert.equal(re.source, "(a)(b)");
var re = t.createSplitterRegexp("xc(?=([x)(]))");
assert.equal(re.source, "xc");
var re = t.createSplitterRegexp("(xc(?=([x)(])))");
assert.equal(re.source, "(xc)");
var re = t.createSplitterRegexp("(?=r)[(?=)](?=([x)(]))");
assert.equal(re.source, "(?=r)[(?=)]");
var re = t.createSplitterRegexp("(?=r)[(?=)](\\?=t)");
assert.equal(re.source, "(?=r)[(?=)](\\?=t)");
var re = t.createSplitterRegexp("[(?=)](\\?=t)");
assert.equal(re.source, "[(?=)](\\?=t)");
},
"test: removeCapturingGroups" : function() {
var t = new Tokenizer({});
var re = t.removeCapturingGroups("(ax(by))[()]");
assert.equal(re, "(?:ax(?:by))[()]");
}
};
});
if (typeof module !== "undefined" && module === require.main) {
require("asyncjs").test.testcase(module.exports).exec()
}