fix Tokenizer.createSplitterRegexp and add tests

fixes #1317

lib/ace/test/all_browser.js

@@ -47,6 +47,7 @@ var testNames = [
     "ace/search_test",
     "ace/selection_test",
     "ace/token_iterator_test",
+    "ace/tokenizer_test",
     "ace/virtual_renderer_test"
 ];
 

lib/ace/tokenizer.js

@@ -66,7 +66,7 @@ var Tokenizer = function(rules) {
                 flag = "gi";
             if (rule.regex == null)
                 continue;
-            
+
             if (rule.regex instanceof RegExp)
                 rule.regex = rule.regex.toString().slice(1, -1);
 
@@ -87,7 +87,7 @@ var Tokenizer = function(rules) {
                 else
                     rule.onMatch = rule.token;
             }
-            
+
             if (matchcount > 1) {
                 if (/\\\d/.test(rule.regex)) {
                     // Replace any backreferences and offset appropriately.
@@ -106,7 +106,7 @@ var Tokenizer = function(rules) {
             matchTotal += matchcount;
 
             ruleRegExps.push(adjustedregex);
-            
+
             // makes property access faster
             if (!rule.onMatch)
                 rule.onMatch = null;
@@ -121,7 +121,7 @@ var Tokenizer = function(rules) {
     this.$applyToken = function(str) {
         var values = this.splitRegex.exec(str).slice(1);
         var types = this.token.apply(this, values);
-        
+
         // required for compatibility with old modes
         if (typeof types === "string")
             return [{type: types, value: str}];
@@ -157,7 +157,7 @@ var Tokenizer = function(rules) {
         }
         return tokens;
     };
-    
+
     this.removeCapturingGroups = function(src) {
         var r = src.replace(
             /\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
@@ -165,13 +165,13 @@ var Tokenizer = function(rules) {
         );
         return r;
     };
-    
+
     this.createSplitterRegexp = function(src, flag) {
         if (src.indexOf("(?=") != -1) {
             var stack = 0;
             var inChClass = false;
             var lastCapture = {};
-            src.replace(/(\\.)|(\((?:\?[=!])?)|(\))|([])/g, function(
+            src.replace(/(\\.)|(\((?:\?[=!])?)|(\))|([\[\]])/g, function(
                 m, esc, parenOpen, parenClose, square, index
             ) {
                 if (inChClass) {
@@ -179,8 +179,10 @@ var Tokenizer = function(rules) {
                 } else if (square) {
                     inChClass = true;
                 } else if (parenClose) {
-                    if (stack == lastCapture.stack)
+                    if (stack == lastCapture.stack) {
-                        lastCapture.end = index+1
+                        lastCapture.end = index+1;
+                        lastCapture.stack = -1;
+                    }
                     stack--;
                 } else if (parenOpen) {
                     stack++;

lib/ace/tokenizer_test.js

@@ -0,0 +1,69 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2010, Ajax.org B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+ if (typeof process !== "undefined") {
+     require("amd-loader");
+ }
+
+define(function(require, exports, module) {
+"use strict";
+
+var Tokenizer = require("./tokenizer").Tokenizer;
+var assert = require("./test/assertions");
+
+module.exports = {
+    "test: createSplitterRegexp" : function() {
+        var t = new Tokenizer({});
+        var re = t.createSplitterRegexp("(a)(b)(?=[x)(])");
+        assert.equal(re.source, "(a)(b)");
+        var re = t.createSplitterRegexp("xc(?=([x)(]))");
+        assert.equal(re.source, "xc");
+        var re = t.createSplitterRegexp("(xc(?=([x)(])))");
+        assert.equal(re.source, "(xc)");
+        var re = t.createSplitterRegexp("(?=r)[(?=)](?=([x)(]))");
+        assert.equal(re.source, "(?=r)[(?=)]");
+        var re = t.createSplitterRegexp("(?=r)[(?=)](\\?=t)");
+        assert.equal(re.source, "(?=r)[(?=)](\\?=t)");
+        var re = t.createSplitterRegexp("[(?=)](\\?=t)");
+        assert.equal(re.source, "[(?=)](\\?=t)");
+    },
+
+    "test: removeCapturingGroups" : function() {
+        var t = new Tokenizer({});
+        var re = t.removeCapturingGroups("(ax(by))[()]");
+        assert.equal(re, "(?:ax(?:by))[()]");
+    }
+};
+
+});
+
+if (typeof module !== "undefined" && module === require.main) {
+    require("asyncjs").test.testcase(module.exports).exec()
+}