diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js index 8d161d74..af476878 100644 --- a/lib/ace/tokenizer.js +++ b/lib/ace/tokenizer.js @@ -3,7 +3,7 @@ * * Copyright (c) 2010, Ajax.org B.V. * All rights reserved. - * + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright @@ -14,7 +14,7 @@ * * Neither the name of Ajax.org B.V. nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -32,7 +32,7 @@ define(function(require, exports, module) { "use strict"; /** - * + * * * This class takes a set of highlighting rules, and creates a tokenizer out of them. For more information, see [the wiki on extending highlighters](https://github.com/ajaxorg/ace/wiki/Creating-or-Extending-an-Edit-Mode#wiki-extendingTheHighlighter). * @class Tokenizer @@ -43,9 +43,9 @@ define(function(require, exports, module) { * @param {Object} rules The highlighting rules * @param {String} flag Any additional regular expression flags to pass (like "i" for case insensitive) * - * * - * + * + * * @constructor **/ var Tokenizer = function(rules, flag) { @@ -62,21 +62,42 @@ var Tokenizer = function(rules, flag) { var mapping = this.matchMappings[key] = {}; for (var i = 0; i < state.length; i++) { - if (state[i].regex instanceof RegExp) - state[i].regex = state[i].regex.toString().slice(1, -1); + var rule = state[i]; + if (rule.regex instanceof RegExp) + rule.regex = rule.regex.toString().slice(1, -1); // Count number of matching groups. 2 extra groups from the full match // And the catch-all on the end (used to force a match); - var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2; - - // Replace any backreferences and offset appropriately. - var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) { - return "\\" + (parseInt(digit, 10) + matchTotal + 1); - }); - - if (matchcount > 1 && typeof state[i].token == "string" && state[i].token.length !== matchcount-1) - throw new Error("For " + state[i].regex + " the matching groups (" +(matchcount-1) + ") and length of the token array (" + state[i].token.length + ") don't match (rule #" + i + " of state " + key + ")"); + var adjustedregex = rule.regex + var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2; + if (Array.isArray(rule.token)) { + if (rule.token.length != matchcount - 1) { + throw new Error( + "For " + rule.regex + + " the matching groups (" +(matchcount-1) + + ") and length of the token array (" + rule.token.length + + ") don't match (rule #" + i + " of state " + key + ")" + ); + } + + rule.tokenArray = rule.token; + rule.token = this.$splitHelper; + } + if (matchcount > 1) { + if (/\\\d/.test(rule.regex)) { + // Replace any backreferences and offset appropriately. + adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) { + return "\\" + (parseInt(digit, 10) + matchTotal + 1); + }); + } else { + matchcount = 1; + adjustedregex = this.nonCapturingRegexp(rule.regex); + } + if (!rule.splitRegex) + rule.splitRegex = new RegExp(rule.regex) + } + mapping[matchTotal] = { rule: i, len: matchcount @@ -85,12 +106,33 @@ var Tokenizer = function(rules, flag) { ruleRegExps.push(adjustedregex); } + console.log(key, ruleRegExps) this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag); } }; (function() { + this.$splitHelper = function(str) { + var values = str.split(this.splitRegex) + var tokens = []; + var types = this.tokenArray; + for (var i = 0; i < types.length; i++) { + if (values[i + 1]) { + tokens[tokens.length] = { + type: types[i], + value: values[i + 1] + }; + } + } + return tokens; + } + this.nonCapturingRegexp = function(src) { + return src.replace( + /\[(?:\\.|[^\]])*?\]|\\.|\((?:\?:|\?=|!=)|(\()/g, + function(x, y) {return y ? "(?:" : x;} + ); + }; /** * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state. @@ -100,9 +142,9 @@ var Tokenizer = function(rules, flag) { if (startState && typeof startState != "string") { var stack = startState.slice(0); startState = stack[0]; - } else { + } else var stack = [] - } + var currentState = startState || "start"; var state = this.rules[currentState]; var mapping = this.matchMappings[currentState]; @@ -110,18 +152,14 @@ var Tokenizer = function(rules, flag) { re.lastIndex = 0; var match, tokens = []; - var lastIndex = 0; - var token = { - type: null, - value: "" - }; + var token = {type: null, value: ""}; while (match = re.exec(line)) { var type = "text"; var rule = null; - var value = [match[0]]; + var value = match[0]; for (var i = 0; i < match.length-2; i++) { if (match[i + 1] === undefined) @@ -129,12 +167,9 @@ var Tokenizer = function(rules, flag) { rule = state[mapping[i].rule]; - if (mapping[i].len > 1) - value = match.slice(i+2, i+1+mapping[i].len); - // compute token type if (typeof rule.token == "function") - type = rule.token(value.length == 1 ? value[0] : value, currentState, stack); + type = rule.token(value, currentState, stack); else type = rule.token; @@ -154,26 +189,21 @@ var Tokenizer = function(rules, flag) { break; } - if (value[0]) { + if (value) { if (typeof type == "string") { - value = [value.join("")]; - type = [type]; - } - for (var i = 0; i < value.length; i++) { - if (!value[i]) - continue; - - if ((!rule || rule.merge !== false) && token.type === type[i]) { - token.value += value[i]; + if ((!rule || rule.merge !== false) && token.type === type) { + token.value += value; } else { if (token.type) tokens.push(token); - - token = { - type: type[i], - value: value[i] - }; + token = {type: type, value: value}; } + } else { + if (token.type) + tokens.push(token); + token = {type: null, value: ""}; + for (var i = 0; i < type.length; i++) + tokens.push(type[i]); } }