diff --git a/lib/ace/mode/_test/tokens_jade.json b/lib/ace/mode/_test/tokens_jade.json index 502c6c1d..5bf45968 100644 --- a/lib/ace/mode/_test/tokens_jade.json +++ b/lib/ace/mode/_test/tokens_jade.json @@ -27,28 +27,29 @@ ],[ "start" ],[ - "comment_block", - ["punctuation.section.comment","// "] + ["comment_block",0,"start"], + ["comment","// "] ],[ - "comment_block", - ["comment.block.jade"," here it is. a block comment!"] + ["comment_block",0,"start"], + ["comment"," here it is. a block comment!"] ],[ - "comment_block", - ["comment.block.jade"," and another row!"] + ["comment_block",0,"start"], + ["comment"," and another row!"] ],[ "start", - ["text","but not here."] + ["meta.tag.any.jade","but"], + ["text"," not here."] ],[ "start" ],[ - "comment_block", - ["punctuation.section.comment"," // "] + ["comment_block",5,"start"], + ["comment"," // "] ],[ - "comment_block", - ["comment.block.jade"," a far spaced"] + ["comment_block",5,"start"], + ["comment"," a far spaced"] ],[ - "comment_block", - ["comment.block.jade"," should be lack of block"] + "start", + ["text"," should be lack of block"] ],[ "start" ],[ diff --git a/lib/ace/mode/jade_highlight_rules.js b/lib/ace/mode/jade_highlight_rules.js index ea7e7dbd..95650bb1 100644 --- a/lib/ace/mode/jade_highlight_rules.js +++ b/lib/ace/mode/jade_highlight_rules.js @@ -85,10 +85,11 @@ var JadeHighlightRules = function() { regex : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)" }, { - token : function(space, text) { - return "punctuation.section.comment"; + onMatch: function(value, currentState, stack) { + stack.unshift(this.next, value.length - 2, currentState); + return "comment"; }, - regex : "^((\\s*)\/\/)(?:\\s*$)", + regex: /^\s*\/\//, next: "comment_block" }, mixin_embed("markdown", "markdown-"), @@ -155,18 +156,18 @@ var JadeHighlightRules = function() { } ], "comment_block": [ - { - token: function(text) { + {regex: /^\s*/, onMatch: function(value, currentState, stack) { + if (value.length <= stack[1]) { + stack.shift(); + stack.shift(); + this.next = stack.shift(); return "text"; - }, - regex: "^(\\1\\S|$)", - "captures": "1", - next: "start" - }, - { - token: "comment.block.jade", - regex : ".+" - } + } else { + this.next = ""; + return "comment"; + } + }, next: "start"}, + {defaultToken: "comment"} ], /* diff --git a/lib/ace/mode/lua_highlight_rules.js b/lib/ace/mode/lua_highlight_rules.js index 4a762ab4..e9768d6a 100644 --- a/lib/ace/mode/lua_highlight_rules.js +++ b/lib/ace/mode/lua_highlight_rules.js @@ -98,14 +98,14 @@ var LuaHighlightRules = function() { this.$rules = { "start" : [{ stateName: "bracketedComment", - token : function(value, currentState, stack){ + onMatch : function(value, currentState, stack){ stack.unshift(this.next, value.length, currentState); return "comment"; }, regex : /\-\-\[=*\[/, next : [ { - token : function(value, currentState, stack) { + onMatch : function(value, currentState, stack) { if (value.length == stack[1]) { stack.shift(); stack.shift(); @@ -129,14 +129,14 @@ var LuaHighlightRules = function() { }, { stateName: "bracketedString", - token : function(value, currentState, stack){ + onMatch : function(value, currentState, stack){ stack.unshift(this.next, value.length, currentState); return "comment"; }, regex : /\[=*\[/, next : [ { - token : function(value, currentState, stack) { + onMatch : function(value, currentState, stack) { if (value.length == stack[1]) { stack.shift(); stack.shift(); diff --git a/lib/ace/mode/php_highlight_rules.js b/lib/ace/mode/php_highlight_rules.js index 7b1dce7a..3be84f56 100644 --- a/lib/ace/mode/php_highlight_rules.js +++ b/lib/ace/mode/php_highlight_rules.js @@ -971,7 +971,7 @@ var PhpLangHighlightRules = function() { // TODO: Unicode identifiers regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b" }, { - token : function(value, currentSate, state) { + onMatch : function(value, currentSate, state) { value = value.substr(3); if (value[0] == "'" || value[0] == '"') value = value.slice(1, -1); @@ -996,7 +996,7 @@ var PhpLangHighlightRules = function() { ], "heredoc" : [ { - token : function(value, currentSate, stack) { + onMatch : function(value, currentSate, stack) { if (stack[1] + ";" != value) return "string"; stack.shift(); diff --git a/lib/ace/mode/r_highlight_rules.js b/lib/ace/mode/r_highlight_rules.js index 106eeadf..99a74376 100644 --- a/lib/ace/mode/r_highlight_rules.js +++ b/lib/ace/mode/r_highlight_rules.js @@ -91,8 +91,7 @@ define(function(require, exports, module) regex : "`.*?`" }, { - token : function(value) - { + onMatch : function(value) { if (keywords[value]) return "keyword"; else if (buildinConstants[value]) diff --git a/lib/ace/mode/ruby_highlight_rules.js b/lib/ace/mode/ruby_highlight_rules.js index 2fd64a85..99841aaf 100644 --- a/lib/ace/mode/ruby_highlight_rules.js +++ b/lib/ace/mode/ruby_highlight_rules.js @@ -173,7 +173,7 @@ var RubyHighlightRules = function() { regex : "=>" }, { stateName: "heredoc", - token : function(value, currentState, stack) { + onMatch : function(value, currentState, stack) { var next = value[2] == '-' ? "indentedHeredoc" : "heredoc"; var tokens = value.split(this.splitRegex); stack.push(next, tokens[3]); @@ -187,7 +187,7 @@ var RubyHighlightRules = function() { regex : "(<<-?)(['\"`]?)([\\w]+)(['\"`]?)", rules: { heredoc: [{ - token: function(value, currentState, stack) { + onMatch: function(value, currentState, stack) { if (value == stack[1]) { stack.shift(); stack.shift(); @@ -202,7 +202,7 @@ var RubyHighlightRules = function() { token: "string", regex: "^ +" }, { - token: function(value, currentState, stack) { + onMatch: function(value, currentState, stack) { if (value == stack[1]) { stack.shift(); stack.shift(); diff --git a/lib/ace/mode/sass_highlight_rules.js b/lib/ace/mode/sass_highlight_rules.js index c98f07f2..09f30262 100644 --- a/lib/ace/mode/sass_highlight_rules.js +++ b/lib/ace/mode/sass_highlight_rules.js @@ -40,8 +40,8 @@ var SassHighlightRules = function() { var start = this.$rules.start; if (start[1].token == "comment") { start.splice(1, 1, { - token: function(value, currentState, stack) { - stack.unshift(this.next, value.length - 2, currentState); + onMatch: function(value, currentState, stack) { + stack.unshift(this.next, -1, value.length - 2, currentState); return "comment"; }, regex: /^\s*\/\*/, @@ -55,14 +55,14 @@ var SassHighlightRules = function() { }); this.$rules.comment = [ - {regex: /^\s*/, token: function(value, currentState, stack) { - if (value.length < stack[1]) { - stack.shift(); - stack.shift(); + {regex: /^\s*/, onMatch: function(value, currentState, stack) { + if (stack[1] === -1) + stack[1] = Math.max(stack[2], value.length - 1); + if (value.length <= stack[1]) { + /*shift3x*/stack.shift();stack.shift();stack.shift(); this.next = stack.shift(); return "text"; } else { - stack[1] = value.length; this.next = ""; return "comment"; } diff --git a/lib/ace/mode/tmsnippet.js b/lib/ace/mode/tmsnippet.js index 68638e56..8f777719 100644 --- a/lib/ace/mode/tmsnippet.js +++ b/lib/ace/mode/tmsnippet.js @@ -16,14 +16,14 @@ var SnippetHighlightRules = function() { {token:"constant.language.escape", regex: /\\[\$}`\\]/}, {token:"keyword", regex: "\\$(?:TM_)?(?:" + builtins + ")\\b"}, {token:"variable", regex: "\\$\\w+"}, - {token: function(value, state, stack) { + {onMatch: function(value, state, stack) { if (stack[1]) stack[1]++; else stack.unshift("start", 1); return this.tokenName; }, tokenName: "markup.list", regex: "\\${", next: "varDecl"}, - {token: function(value, state, stack) { + {onMatch: function(value, state, stack) { if (!stack[1]) return "text"; stack[1]--; diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js index 6ed271b1..602b521e 100644 --- a/lib/ace/tokenizer.js +++ b/lib/ace/tokenizer.js @@ -79,10 +79,15 @@ var Tokenizer = function(rules) { rule.token = rule.token[0]; } else { rule.tokenArray = rule.token; - rule.token = this.$arrayTokens; + rule.onMatch = this.$arrayTokens; } + } else if (typeof rule.token == "function" && !rule.onMatch) { + if (matchcount > 1) + rule.onMatch = this.$applyToken; + else + rule.onMatch = rule.token; } - + if (matchcount > 1) { if (/\\\d/.test(rule.regex)) { // Replace any backreferences and offset appropriately. @@ -101,6 +106,11 @@ var Tokenizer = function(rules) { matchTotal += matchcount; ruleRegExps.push(adjustedregex); + + // makes property access faster + if (!rule.onMatch) + rule.onMatch = null; + rule.__proto__ = null; } this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag); @@ -108,6 +118,25 @@ var Tokenizer = function(rules) { }; (function() { + this.$applyToken = function(str) { + var values = this.splitRegex.exec(str).slice(1); + var types = this.token.apply(this, values); + + // required for compatibility with old modes + if (typeof types === "string") + return [{type: types, value: str}]; + + var tokens = []; + for (var i = 0, l = types.length; i < l; i++) { + if (values[i]) + tokens[tokens.length] = { + type: types[i], + value: values[i] + }; + } + return tokens; + }, + this.$arrayTokens = function(str) { if (!str) return []; @@ -119,13 +148,12 @@ var Tokenizer = function(rules) { console.error(types , values, str, this.splitRegex, this); return [{type: "error.invalid", value: str}]; } - for (var i = 0; i < types.length; i++) { - if (values[i + 1]) { + for (var i = 0, l = types.length; i < l; i++) { + if (values[i + 1]) tokens[tokens.length] = { type: types[i], value: values[i + 1] }; - } } return tokens; }; @@ -188,10 +216,10 @@ var Tokenizer = function(rules) { rule = state[mapping[i]]; - // compute token type - type = typeof rule.token == "function" - ? rule.token(value, currentState, stack) - : rule.token; + if (rule.onMatch) + type = rule.onMatch(value, currentState, stack); + else + type = rule.token; if (rule.next) { if (typeof rule.next == "string")