jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

restore old behavior of token and add onMatch. fixes #1269

Browse source
This commit is contained in:
nightwing 2013-03-01 21:26:31 +04:00
commit 7960652010
9 changed files with 85 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

27
lib/ace/mode/_test/tokens_jade.json
View file

@ -27,28 +27,29 @@
],[
"start"
],[
"comment_block",
["punctuation.section.comment","// "]
["comment_block",0,"start"],
["comment","// "]
],[
"comment_block",
["comment.block.jade"," here it is. a block comment!"]
["comment_block",0,"start"],
["comment"," here it is. a block comment!"]
],[
"comment_block",
["comment.block.jade"," and another row!"]
["comment_block",0,"start"],
["comment"," and another row!"]
],[
"start",
["text","but not here."]
["meta.tag.any.jade","but"],
["text"," not here."]
],[
"start"
],[
"comment_block",
["punctuation.section.comment"," // "]
["comment_block",5,"start"],
["comment"," // "]
],[
"comment_block",
["comment.block.jade"," a far spaced"]
["comment_block",5,"start"],
["comment"," a far spaced"]
],[
"comment_block",
["comment.block.jade"," should be lack of block"]
"start",
["text"," should be lack of block"]
],[
"start"
],[

29
lib/ace/mode/jade_highlight_rules.js
View file

@ -85,10 +85,11 @@ var JadeHighlightRules = function() {
regex : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)"
},
{
token : function(space, text) {
return "punctuation.section.comment";
onMatch: function(value, currentState, stack) {
stack.unshift(this.next, value.length - 2, currentState);
return "comment";
},
regex : "^((\\s*)\/\/)(?:\\s*$)",
regex: /^\s*\/\//,
next: "comment_block"
},
mixin_embed("markdown", "markdown-"),
@ -155,18 +156,18 @@ var JadeHighlightRules = function() {
}
],
"comment_block": [
{
token: function(text) {
{regex: /^\s*/, onMatch: function(value, currentState, stack) {
if (value.length <= stack[1]) {
stack.shift();
stack.shift();
this.next = stack.shift();
return "text";
},
regex: "^(\\1\\S|$)",
"captures": "1",
next: "start"
},
{
token: "comment.block.jade",
regex : ".+"
}
} else {
this.next = "";
return "comment";
}
}, next: "start"},
{defaultToken: "comment"}
],
/*

8
lib/ace/mode/lua_highlight_rules.js
View file

@ -98,14 +98,14 @@ var LuaHighlightRules = function() {
this.$rules = {
"start" : [{
stateName: "bracketedComment",
token : function(value, currentState, stack){
onMatch : function(value, currentState, stack){
stack.unshift(this.next, value.length, currentState);
return "comment";
},
regex : /\-\-\[=*\[/,
next : [
{
token : function(value, currentState, stack) {
onMatch : function(value, currentState, stack) {
if (value.length == stack[1]) {
stack.shift();
stack.shift();
@ -129,14 +129,14 @@ var LuaHighlightRules = function() {
},
{
stateName: "bracketedString",
token : function(value, currentState, stack){
onMatch : function(value, currentState, stack){
stack.unshift(this.next, value.length, currentState);
return "comment";
},
regex : /\[=*\[/,
next : [
{
token : function(value, currentState, stack) {
onMatch : function(value, currentState, stack) {
if (value.length == stack[1]) {
stack.shift();
stack.shift();

4
lib/ace/mode/php_highlight_rules.js
View file

@ -971,7 +971,7 @@ var PhpLangHighlightRules = function() {
// TODO: Unicode identifiers
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, {
token : function(value, currentSate, state) {
onMatch : function(value, currentSate, state) {
value = value.substr(3);
if (value[0] == "'" || value[0] == '"')
value = value.slice(1, -1);
@ -996,7 +996,7 @@ var PhpLangHighlightRules = function() {
],
"heredoc" : [
{
token : function(value, currentSate, stack) {
onMatch : function(value, currentSate, stack) {
if (stack[1] + ";" != value)
return "string";
stack.shift();

3
lib/ace/mode/r_highlight_rules.js
View file

@ -91,8 +91,7 @@ define(function(require, exports, module)
regex : "`.*?`"
},
{
token : function(value)
{
onMatch : function(value) {
if (keywords[value])
return "keyword";
else if (buildinConstants[value])

6
lib/ace/mode/ruby_highlight_rules.js
View file

@ -173,7 +173,7 @@ var RubyHighlightRules = function() {
regex : "=>"
}, {
stateName: "heredoc",
token : function(value, currentState, stack) {
onMatch : function(value, currentState, stack) {
var next = value[2] == '-' ? "indentedHeredoc" : "heredoc";
var tokens = value.split(this.splitRegex);
stack.push(next, tokens[3]);
@ -187,7 +187,7 @@ var RubyHighlightRules = function() {
regex : "(<<-?)(['\"`]?)([\\w]+)(['\"`]?)",
rules: {
heredoc: [{
token: function(value, currentState, stack) {
onMatch: function(value, currentState, stack) {
if (value == stack[1]) {
stack.shift();
stack.shift();
@ -202,7 +202,7 @@ var RubyHighlightRules = function() {
token: "string",
regex: "^ +"
}, {
token: function(value, currentState, stack) {
onMatch: function(value, currentState, stack) {
if (value == stack[1]) {
stack.shift();
stack.shift();

14
lib/ace/mode/sass_highlight_rules.js
View file

@ -40,8 +40,8 @@ var SassHighlightRules = function() {
var start = this.$rules.start;
if (start[1].token == "comment") {
start.splice(1, 1, {
token: function(value, currentState, stack) {
stack.unshift(this.next, value.length - 2, currentState);
onMatch: function(value, currentState, stack) {
stack.unshift(this.next, -1, value.length - 2, currentState);
return "comment";
},
regex: /^\s*\/\*/,
@ -55,14 +55,14 @@ var SassHighlightRules = function() {
});
this.$rules.comment = [
{regex: /^\s*/, token: function(value, currentState, stack) {
if (value.length < stack[1]) {
stack.shift();
stack.shift();
{regex: /^\s*/, onMatch: function(value, currentState, stack) {
if (stack[1] === -1)
stack[1] = Math.max(stack[2], value.length - 1);
if (value.length <= stack[1]) {
/*shift3x*/stack.shift();stack.shift();stack.shift();
this.next = stack.shift();
return "text";
} else {
stack[1] = value.length;
this.next = "";
return "comment";
}

4
lib/ace/mode/tmsnippet.js
View file

@ -16,14 +16,14 @@ var SnippetHighlightRules = function() {
{token:"constant.language.escape", regex: /\\[\$}`\\]/},
{token:"keyword", regex: "\\$(?:TM_)?(?:" + builtins + ")\\b"},
{token:"variable", regex: "\\$\\w+"},
{token: function(value, state, stack) {
{onMatch: function(value, state, stack) {
if (stack[1])
stack[1]++;
else
stack.unshift("start", 1);
return this.tokenName;
}, tokenName: "markup.list", regex: "\\${", next: "varDecl"},
{token: function(value, state, stack) {
{onMatch: function(value, state, stack) {
if (!stack[1])
return "text";
stack[1]--;

46
lib/ace/tokenizer.js
View file

@ -79,10 +79,15 @@ var Tokenizer = function(rules) {
rule.token = rule.token[0];
} else {
rule.tokenArray = rule.token;
rule.token = this.$arrayTokens;
rule.onMatch = this.$arrayTokens;
}
} else if (typeof rule.token == "function" && !rule.onMatch) {
if (matchcount > 1)
rule.onMatch = this.$applyToken;
else
rule.onMatch = rule.token;
}
if (matchcount > 1) {
if (/\\\d/.test(rule.regex)) {
// Replace any backreferences and offset appropriately.
@ -101,6 +106,11 @@ var Tokenizer = function(rules) {
matchTotal += matchcount;
ruleRegExps.push(adjustedregex);
// makes property access faster
if (!rule.onMatch)
rule.onMatch = null;
rule.__proto__ = null;
}
this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
@ -108,6 +118,25 @@ var Tokenizer = function(rules) {
};
(function() {
this.$applyToken = function(str) {
var values = this.splitRegex.exec(str).slice(1);
var types = this.token.apply(this, values);
// required for compatibility with old modes
if (typeof types === "string")
return [{type: types, value: str}];
var tokens = [];
for (var i = 0, l = types.length; i < l; i++) {
if (values[i])
tokens[tokens.length] = {
type: types[i],
value: values[i]
};
}
return tokens;
},
this.$arrayTokens = function(str) {
if (!str)
return [];
@ -119,13 +148,12 @@ var Tokenizer = function(rules) {
console.error(types , values, str, this.splitRegex, this);
return [{type: "error.invalid", value: str}];
}
for (var i = 0; i < types.length; i++) {
if (values[i + 1]) {
for (var i = 0, l = types.length; i < l; i++) {
if (values[i + 1])
tokens[tokens.length] = {
type: types[i],
value: values[i + 1]
};
}
}
return tokens;
};
@ -188,10 +216,10 @@ var Tokenizer = function(rules) {
rule = state[mapping[i]];
// compute token type
type = typeof rule.token == "function"
? rule.token(value, currentState, stack)
: rule.token;
if (rule.onMatch)
type = rule.onMatch(value, currentState, stack);
else
type = rule.token;
if (rule.next) {
if (typeof rule.next == "string")