jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

restore old behavior of token and add onMatch. fixes #1269

Browse source
This commit is contained in:
nightwing 2013-03-01 21:26:31 +04:00
commit 7960652010
9 changed files with 85 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

27
lib/ace/mode/_test/tokens_jade.json
View file

@ -27,28 +27,29 @@
],[ ],[
"start" "start"
],[ ],[
"comment_block", ["comment_block",0,"start"],
["punctuation.section.comment","// "] ["comment","// "]
],[ ],[
"comment_block", ["comment_block",0,"start"],
["comment.block.jade"," here it is. a block comment!"] ["comment"," here it is. a block comment!"]
],[ ],[
"comment_block", ["comment_block",0,"start"],
["comment.block.jade"," and another row!"] ["comment"," and another row!"]
],[ ],[
"start", "start",
["text","but not here."] ["meta.tag.any.jade","but"],
["text"," not here."]
],[ ],[
"start" "start"
],[ ],[
"comment_block", ["comment_block",5,"start"],
["punctuation.section.comment"," // "] ["comment"," // "]
],[ ],[
"comment_block", ["comment_block",5,"start"],
["comment.block.jade"," a far spaced"] ["comment"," a far spaced"]
],[ ],[
"comment_block", "start",
["comment.block.jade"," should be lack of block"] ["text"," should be lack of block"]
],[ ],[
"start" "start"
],[ ],[

29
lib/ace/mode/jade_highlight_rules.js
View file

@ -85,10 +85,11 @@ var JadeHighlightRules = function() {
regex : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)" regex : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)"
}, },
{ {
token : function(space, text) { onMatch: function(value, currentState, stack) {
return "punctuation.section.comment"; stack.unshift(this.next, value.length - 2, currentState);
return "comment";
}, },
regex : "^((\\s*)\/\/)(?:\\s*$)", regex: /^\s*\/\//,
next: "comment_block" next: "comment_block"
}, },
mixin_embed("markdown", "markdown-"), mixin_embed("markdown", "markdown-"),
@ -155,18 +156,18 @@ var JadeHighlightRules = function() {
} }
], ],
"comment_block": [ "comment_block": [
{ {regex: /^\s*/, onMatch: function(value, currentState, stack) {
token: function(text) { if (value.length <= stack[1]) {
stack.shift();
stack.shift();
this.next = stack.shift();
return "text"; return "text";
}, } else {
regex: "^(\\1\\S|$)", this.next = "";
"captures": "1", return "comment";
next: "start" }
}, }, next: "start"},
{ {defaultToken: "comment"}
token: "comment.block.jade",
regex : ".+"
}
], ],
/* /*

8
lib/ace/mode/lua_highlight_rules.js
View file

@ -98,14 +98,14 @@ var LuaHighlightRules = function() {
this.$rules = { this.$rules = {
"start" : [{ "start" : [{
stateName: "bracketedComment", stateName: "bracketedComment",
token : function(value, currentState, stack){ onMatch : function(value, currentState, stack){
stack.unshift(this.next, value.length, currentState); stack.unshift(this.next, value.length, currentState);
return "comment"; return "comment";
}, },
regex : /\-\-\[=*\[/, regex : /\-\-\[=*\[/,
next : [ next : [
{ {
token : function(value, currentState, stack) { onMatch : function(value, currentState, stack) {
if (value.length == stack[1]) { if (value.length == stack[1]) {
stack.shift(); stack.shift();
stack.shift(); stack.shift();
@ -129,14 +129,14 @@ var LuaHighlightRules = function() {
}, },
{ {
stateName: "bracketedString", stateName: "bracketedString",
token : function(value, currentState, stack){ onMatch : function(value, currentState, stack){
stack.unshift(this.next, value.length, currentState); stack.unshift(this.next, value.length, currentState);
return "comment"; return "comment";
}, },
regex : /\[=*\[/, regex : /\[=*\[/,
next : [ next : [
{ {
token : function(value, currentState, stack) { onMatch : function(value, currentState, stack) {
if (value.length == stack[1]) { if (value.length == stack[1]) {
stack.shift(); stack.shift();
stack.shift(); stack.shift();

4
lib/ace/mode/php_highlight_rules.js
View file

@ -971,7 +971,7 @@ var PhpLangHighlightRules = function() {
// TODO: Unicode identifiers // TODO: Unicode identifiers
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b" regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, { }, {
token : function(value, currentSate, state) { onMatch : function(value, currentSate, state) {
value = value.substr(3); value = value.substr(3);
if (value[0] == "'" || value[0] == '"') if (value[0] == "'" || value[0] == '"')
value = value.slice(1, -1); value = value.slice(1, -1);
@ -996,7 +996,7 @@ var PhpLangHighlightRules = function() {
], ],
"heredoc" : [ "heredoc" : [
{ {
token : function(value, currentSate, stack) { onMatch : function(value, currentSate, stack) {
if (stack[1] + ";" != value) if (stack[1] + ";" != value)
return "string"; return "string";
stack.shift(); stack.shift();

3
lib/ace/mode/r_highlight_rules.js
View file

@ -91,8 +91,7 @@ define(function(require, exports, module)
regex : "`.*?`" regex : "`.*?`"
}, },
{ {
token : function(value) onMatch : function(value) {
{
if (keywords[value]) if (keywords[value])
return "keyword"; return "keyword";
else if (buildinConstants[value]) else if (buildinConstants[value])

6
lib/ace/mode/ruby_highlight_rules.js
View file

@ -173,7 +173,7 @@ var RubyHighlightRules = function() {
regex : "=>" regex : "=>"
}, { }, {
stateName: "heredoc", stateName: "heredoc",
token : function(value, currentState, stack) { onMatch : function(value, currentState, stack) {
var next = value[2] == '-' ? "indentedHeredoc" : "heredoc"; var next = value[2] == '-' ? "indentedHeredoc" : "heredoc";
var tokens = value.split(this.splitRegex); var tokens = value.split(this.splitRegex);
stack.push(next, tokens[3]); stack.push(next, tokens[3]);
@ -187,7 +187,7 @@ var RubyHighlightRules = function() {
regex : "(<<-?)(['\"`]?)([\\w]+)(['\"`]?)", regex : "(<<-?)(['\"`]?)([\\w]+)(['\"`]?)",
rules: { rules: {
heredoc: [{ heredoc: [{
token: function(value, currentState, stack) { onMatch: function(value, currentState, stack) {
if (value == stack[1]) { if (value == stack[1]) {
stack.shift(); stack.shift();
stack.shift(); stack.shift();
@ -202,7 +202,7 @@ var RubyHighlightRules = function() {
token: "string", token: "string",
regex: "^ +" regex: "^ +"
}, { }, {
token: function(value, currentState, stack) { onMatch: function(value, currentState, stack) {
if (value == stack[1]) { if (value == stack[1]) {
stack.shift(); stack.shift();
stack.shift(); stack.shift();

14
lib/ace/mode/sass_highlight_rules.js
View file

@ -40,8 +40,8 @@ var SassHighlightRules = function() {
var start = this.$rules.start; var start = this.$rules.start;
if (start[1].token == "comment") { if (start[1].token == "comment") {
start.splice(1, 1, { start.splice(1, 1, {
token: function(value, currentState, stack) { onMatch: function(value, currentState, stack) {
stack.unshift(this.next, value.length - 2, currentState); stack.unshift(this.next, -1, value.length - 2, currentState);
return "comment"; return "comment";
}, },
regex: /^\s*\/\*/, regex: /^\s*\/\*/,
@ -55,14 +55,14 @@ var SassHighlightRules = function() {
}); });
this.$rules.comment = [ this.$rules.comment = [
{regex: /^\s*/, token: function(value, currentState, stack) { {regex: /^\s*/, onMatch: function(value, currentState, stack) {
if (value.length < stack[1]) { if (stack[1] === -1)
stack.shift(); stack[1] = Math.max(stack[2], value.length - 1);
stack.shift(); if (value.length <= stack[1]) {
/*shift3x*/stack.shift();stack.shift();stack.shift();
this.next = stack.shift(); this.next = stack.shift();
return "text"; return "text";
} else { } else {
stack[1] = value.length;
this.next = ""; this.next = "";
return "comment"; return "comment";
} }

4
lib/ace/mode/tmsnippet.js
View file

@ -16,14 +16,14 @@ var SnippetHighlightRules = function() {
{token:"constant.language.escape", regex: /\\[\$}`\\]/}, {token:"constant.language.escape", regex: /\\[\$}`\\]/},
{token:"keyword", regex: "\\$(?:TM_)?(?:" + builtins + ")\\b"}, {token:"keyword", regex: "\\$(?:TM_)?(?:" + builtins + ")\\b"},
{token:"variable", regex: "\\$\\w+"}, {token:"variable", regex: "\\$\\w+"},
{token: function(value, state, stack) { {onMatch: function(value, state, stack) {
if (stack[1]) if (stack[1])
stack[1]++; stack[1]++;
else else
stack.unshift("start", 1); stack.unshift("start", 1);
return this.tokenName; return this.tokenName;
}, tokenName: "markup.list", regex: "\\${", next: "varDecl"}, }, tokenName: "markup.list", regex: "\\${", next: "varDecl"},
{token: function(value, state, stack) { {onMatch: function(value, state, stack) {
if (!stack[1]) if (!stack[1])
return "text"; return "text";
stack[1]--; stack[1]--;

44
lib/ace/tokenizer.js
View file

@ -79,8 +79,13 @@ var Tokenizer = function(rules) {
rule.token = rule.token[0]; rule.token = rule.token[0];
} else { } else {
rule.tokenArray = rule.token; rule.tokenArray = rule.token;
rule.token = this.$arrayTokens; rule.onMatch = this.$arrayTokens;
} }
} else if (typeof rule.token == "function" && !rule.onMatch) {
if (matchcount > 1)
rule.onMatch = this.$applyToken;
else
rule.onMatch = rule.token;
} }
if (matchcount > 1) { if (matchcount > 1) {
@ -101,6 +106,11 @@ var Tokenizer = function(rules) {
matchTotal += matchcount; matchTotal += matchcount;
ruleRegExps.push(adjustedregex); ruleRegExps.push(adjustedregex);
// makes property access faster
if (!rule.onMatch)
rule.onMatch = null;
rule.__proto__ = null;
} }
this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag); this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
@ -108,6 +118,25 @@ var Tokenizer = function(rules) {
}; };
(function() { (function() {
this.$applyToken = function(str) {
var values = this.splitRegex.exec(str).slice(1);
var types = this.token.apply(this, values);
// required for compatibility with old modes
if (typeof types === "string")
return [{type: types, value: str}];
var tokens = [];
for (var i = 0, l = types.length; i < l; i++) {
if (values[i])
tokens[tokens.length] = {
type: types[i],
value: values[i]
};
}
return tokens;
},
this.$arrayTokens = function(str) { this.$arrayTokens = function(str) {
if (!str) if (!str)
return []; return [];
@ -119,13 +148,12 @@ var Tokenizer = function(rules) {
console.error(types , values, str, this.splitRegex, this); console.error(types , values, str, this.splitRegex, this);
return [{type: "error.invalid", value: str}]; return [{type: "error.invalid", value: str}];
} }
for (var i = 0; i < types.length; i++) { for (var i = 0, l = types.length; i < l; i++) {
if (values[i + 1]) { if (values[i + 1])
tokens[tokens.length] = { tokens[tokens.length] = {
type: types[i], type: types[i],
value: values[i + 1] value: values[i + 1]
}; };
}
} }
return tokens; return tokens;
}; };
@ -188,10 +216,10 @@ var Tokenizer = function(rules) {
rule = state[mapping[i]]; rule = state[mapping[i]];
// compute token type if (rule.onMatch)
type = typeof rule.token == "function" type = rule.onMatch(value, currentState, stack);
? rule.token(value, currentState, stack) else
: rule.token; type = rule.token;
if (rule.next) { if (rule.next) {
if (typeof rule.next == "string") if (typeof rule.next == "string")