Merge pull request #1276 from ajaxorg/higlighting/onMatch

restore old behavior of token and add onMatch. fixes #1269
This commit is contained in:
Lennart Kats 2013-03-08 03:27:51 -08:00
commit c05016d7b7
14 changed files with 2425 additions and 58 deletions

View file

@ -27,28 +27,29 @@
],[
"start"
],[
"comment_block",
["punctuation.section.comment","// "]
["comment_block",0,"start"],
["comment","// "]
],[
"comment_block",
["comment.block.jade"," here it is. a block comment!"]
["comment_block",0,"start"],
["comment"," here it is. a block comment!"]
],[
"comment_block",
["comment.block.jade"," and another row!"]
["comment_block",0,"start"],
["comment"," and another row!"]
],[
"start",
["text","but not here."]
["meta.tag.any.jade","but"],
["text"," not here."]
],[
"start"
],[
"comment_block",
["punctuation.section.comment"," // "]
["comment_block",5,"start"],
["comment"," // "]
],[
"comment_block",
["comment.block.jade"," a far spaced"]
["comment_block",5,"start"],
["comment"," a far spaced"]
],[
"comment_block",
["comment.block.jade"," should be lack of block"]
"start",
["text"," should be lack of block"]
],[
"start"
],[

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,297 @@
[[
"punctuation.definition.comment.pascal2",
["punctuation.definition.comment.pascal","(*"],
["comment.block.pascal.one","****************************************************************************"]
],[
"punctuation.definition.comment.pascal2",
["comment.block.pascal.one"," * A simple bubble sort program. Reads integers, one per line, and prints *"]
],[
"punctuation.definition.comment.pascal2",
["comment.block.pascal.one"," * them out in sorted order. Blows up if there are more than 49. *"]
],[
"start",
["comment.block.pascal.one"," ****************************************************************************"],
["punctuation.definition.comment.pascal","*)"]
],[
"start",
["keyword.control.pascal","PROGRAM"],
["text"," Sort(input"],
["keyword.operator",","],
["text"," output)"],
["keyword.operator",";"]
],[
"start",
["text"," "],
["keyword.control.pascal","CONST"]
],[
"start",
["text"," "],
["punctuation.definition.comment.pascal","(*"],
["comment.block.pascal.one"," Max array size. "],
["punctuation.definition.comment.pascal","*)"]
],[
"start",
["text"," MaxElts "],
["keyword.operator","="],
["text"," "],
["constant.numeric.pascal","50"],
["keyword.operator",";"]
],[
"start",
["text"," "],
["keyword.control.pascal","TYPE"],
["text"," "]
],[
"start",
["text"," "],
["punctuation.definition.comment.pascal","(*"],
["comment.block.pascal.one"," Type of the element array. "],
["punctuation.definition.comment.pascal","*)"]
],[
"start",
["text"," IntArrType "],
["keyword.operator","="],
["text"," "],
["keyword.control.pascal","ARRAY"],
["text"," ["],
["constant.numeric.pascal","1"],
["text","..MaxElts] "],
["keyword.control.pascal","OF"],
["text"," Integer"],
["keyword.operator",";"]
],[
"start"
],[
"start",
["text"," "],
["keyword.control.pascal","VAR"]
],[
"start",
["text"," "],
["punctuation.definition.comment.pascal","(*"],
["comment.block.pascal.one"," Indexes, exchange temp, array size. "],
["punctuation.definition.comment.pascal","*)"]
],[
"start",
["text"," i"],
["keyword.operator",","],
["text"," j"],
["keyword.operator",","],
["text"," tmp"],
["keyword.operator",","],
["text"," size: integer"],
["keyword.operator",";"]
],[
"start"
],[
"start",
["text"," "],
["punctuation.definition.comment.pascal","(*"],
["comment.block.pascal.one"," Array of ints "],
["punctuation.definition.comment.pascal","*)"]
],[
"start",
["text"," arr: IntArrType"],
["keyword.operator",";"]
],[
"start"
],[
"start",
["text"," "],
["punctuation.definition.comment.pascal","(*"],
["comment.block.pascal.one"," Read in the integers. "],
["punctuation.definition.comment.pascal","*)"]
],[
"start",
["text"," "],
["variable.pascal","PROCEDURE"],
["text"," "],
["storage.type.function.pascal","ReadArr"],
["text","("],
["keyword.control.pascal","VAR"],
["text"," size: Integer"],
["keyword.operator",";"],
["text"," "],
["keyword.control.pascal","VAR"],
["text"," a: IntArrType)"],
["keyword.operator",";"],
["text"," "]
],[
"start",
["text"," "],
["keyword.control.pascal","BEGIN"]
],[
"start",
["text"," size "],
["keyword.operator",":="],
["text"," "],
["constant.numeric.pascal","1"],
["keyword.operator",";"]
],[
"start",
["text"," "],
["keyword.control.pascal","WHILE"],
["text"," "],
["keyword.control.pascal","NOT"],
["text"," eof "],
["keyword.control.pascal","DO"],
["text"," "],
["keyword.control.pascal","BEGIN"]
],[
"start",
["text"," readln(a[size])"],
["keyword.operator",";"]
],[
"start",
["text"," "],
["keyword.control.pascal","IF"],
["text"," "],
["keyword.control.pascal","NOT"],
["text"," eof "],
["keyword.control.pascal","THEN"],
["text"," "]
],[
"start",
["text"," size "],
["keyword.operator",":="],
["text"," size "],
["keyword.operator","+"],
["text"," "],
["constant.numeric.pascal","1"]
],[
"start",
["text"," "],
["keyword.control.pascal","END"]
],[
"start",
["text"," "],
["keyword.control.pascal","END"],
["keyword.operator",";"]
],[
"start"
],[
"start",
["text"," "],
["keyword.control.pascal","BEGIN"]
],[
"start",
["text"," "],
["punctuation.definition.comment.pascal","(*"],
["comment.block.pascal.one"," Read "],
["punctuation.definition.comment.pascal","*)"]
],[
"start",
["text"," ReadArr(size"],
["keyword.operator",","],
["text"," arr)"],
["keyword.operator",";"]
],[
"start"
],[
"start",
["text"," "],
["punctuation.definition.comment.pascal","(*"],
["comment.block.pascal.one"," Sort using bubble sort. "],
["punctuation.definition.comment.pascal","*)"]
],[
"start",
["text"," "],
["keyword.control.pascal","FOR"],
["text"," i "],
["keyword.operator",":="],
["text"," size "],
["keyword.operator","-"],
["text"," "],
["constant.numeric.pascal","1"],
["text"," DOWNTO "],
["constant.numeric.pascal","1"],
["text"," "],
["keyword.control.pascal","DO"]
],[
"start",
["text"," "],
["keyword.control.pascal","FOR"],
["text"," j "],
["keyword.operator",":="],
["text"," "],
["constant.numeric.pascal","1"],
["text"," "],
["keyword.control.pascal","TO"],
["text"," i "],
["keyword.control.pascal","DO"],
["text"," "]
],[
"start",
["text"," "],
["keyword.control.pascal","IF"],
["text"," arr[j] > arr[j "],
["keyword.operator","+"],
["text"," "],
["constant.numeric.pascal","1"],
["text","] "],
["keyword.control.pascal","THEN"],
["text"," "],
["keyword.control.pascal","BEGIN"]
],[
"start",
["text"," tmp "],
["keyword.operator",":="],
["text"," arr[j]"],
["keyword.operator",";"]
],[
"start",
["text"," arr[j] "],
["keyword.operator",":="],
["text"," arr[j "],
["keyword.operator","+"],
["text"," "],
["constant.numeric.pascal","1"],
["text","]"],
["keyword.operator",";"]
],[
"start",
["text"," arr[j "],
["keyword.operator","+"],
["text"," "],
["constant.numeric.pascal","1"],
["text","] "],
["keyword.operator",":="],
["text"," tmp"],
["keyword.operator",";"]
],[
"start",
["text"," "],
["keyword.control.pascal","END"],
["keyword.operator",";"]
],[
"start"
],[
"start",
["text"," "],
["punctuation.definition.comment.pascal","(*"],
["comment.block.pascal.one"," Print. "],
["punctuation.definition.comment.pascal","*)"]
],[
"start",
["text"," "],
["keyword.control.pascal","FOR"],
["text"," i "],
["keyword.operator",":="],
["text"," "],
["constant.numeric.pascal","1"],
["text"," "],
["keyword.control.pascal","TO"],
["text"," size "],
["keyword.control.pascal","DO"]
],[
"start",
["text"," writeln(arr[i])"]
],[
"start",
["text"," "],
["keyword.control.pascal","END"],
["text","."]
],[
"start",
["text"," "]
]]

View file

@ -0,0 +1,229 @@
[[
"start",
["comment","// sass ace mode;"]
],[
"start"
],[
"start",
["keyword","@import"],
["text"," "],
["support.function","url("],
["string","http://fonts.googleapis.com/css?family=Ace:700"],
["support.function",")"]
],[
"start"
],[
"start",
["variable.language","html"],
["text",", "],
["variable.language","body"]
],[
"start",
["support.type"," :background-color "],
["constant.numeric","#ace"]
],[
"start",
["text"," "],
["support.type","text-align"],
["text",": "],
["constant.language","center"]
],[
"start",
["text"," "],
["support.type","height"],
["text",": "],
["constant.numeric","100%"]
],[
["comment",-1,2,"start"],
["comment"," /*;*********;"]
],[
["comment",3,2,"start"],
["comment"," ;comment ;"]
],[
["comment",3,2,"start"],
["comment"," ;*********;"]
],[
"start"
],[
"start",
["variable.language",".toggle"]
],[
"start",
["text"," "],
["variable","$size"],
["text",": "],
["constant.numeric","14px"]
],[
"start"
],[
"start",
["support.type"," :background "],
["support.function","url("],
["string","http://subtlepatterns.com/patterns/dark_stripes.png"],
["support.function",")"]
],[
"start",
["text"," "],
["support.type","border-radius"],
["text",": "],
["constant.numeric","8px"]
],[
"start",
["text"," "],
["support.type","height"],
["text",": "],
["variable","$size"]
],[
"start"
],[
"start",
["text"," &"],
["variable.language",":before"]
],[
"start",
["text"," "],
["variable","$radius"],
["text",": "],
["variable","$size"],
["text"," "],
["keyword.operator","*"],
["text"," "],
["constant.numeric","0.845"]
],[
"start",
["text"," "],
["variable","$glow"],
["text",": "],
["variable","$size"],
["text"," "],
["keyword.operator","*"],
["text"," "],
["constant.numeric","0.125"]
],[
"start"
],[
"start",
["text"," "],
["support.type","box-shadow"],
["text",": "],
["constant.numeric","0"],
["text"," "],
["constant.numeric","0"],
["text"," "],
["variable","$glow"],
["text"," "],
["variable","$glow"],
["text"," / "],
["constant.numeric","2"],
["text"," "],
["constant.numeric","#fff"]
],[
"start",
["text"," "],
["support.type","border-radius"],
["text",": "],
["variable","$radius"]
],[
"start",
["text"," "]
],[
"start",
["text"," &"],
["variable.language",":active"]
],[
"start",
["text"," ~ "],
["variable.language",".button"]
],[
"start",
["text"," "],
["support.type","box-shadow"],
["text",": "],
["constant.numeric","0"],
["text"," "],
["constant.numeric","15px"],
["text"," "],
["constant.numeric","25px"],
["text"," "],
["constant.numeric","-4px"],
["text"," "],
["support.function","rgba"],
["paren.lparen","("],
["constant.numeric","0"],
["text",","],
["constant.numeric","0"],
["text",","],
["constant.numeric","0"],
["text",","],
["constant.numeric","0.4"],
["paren.rparen",")"],
["text"," "]
],[
"start",
["text"," ~ "],
["variable.language",".label"]
],[
"start",
["text"," "],
["support.type","font-size"],
["text",": "],
["constant.numeric","40px"]
],[
"start",
["text"," "],
["support.type","color"],
["text",": "],
["support.function","rgba"],
["paren.lparen","("],
["constant.numeric","0"],
["text",","],
["constant.numeric","0"],
["text",","],
["constant.numeric","0"],
["text",","],
["constant.numeric","0.45"],
["paren.rparen",")"]
],[
"start"
],[
"start",
["text"," &"],
["variable.language",":checked"],
["text"," "]
],[
"start",
["text"," ~ "],
["variable.language",".button"]
],[
"start",
["text"," "],
["support.type","box-shadow"],
["text",": "],
["constant.numeric","0"],
["text"," "],
["constant.numeric","15px"],
["text"," "],
["constant.numeric","25px"],
["text"," "],
["constant.numeric","-4px"],
["text"," "],
["constant.numeric","#ace"]
],[
"start",
["text"," ~ "],
["variable.language",".label"]
],[
"start",
["text"," "],
["support.type","font-size"],
["text",": "],
["constant.numeric","40px"]
],[
"start",
["text"," "],
["support.type","color"],
["text",": "],
["constant.numeric","#c9c9c9"]
],[
"start"
]]

View file

@ -0,0 +1,132 @@
[[
"start",
["doc,comment","$$------------------------------------"]
],[
"start"
],[
"start",
["text","tabTrigger: t"]
],[
"start"
],[
"start",
["text","name: Heading 3"]
],[
"start"
],[
"start",
["text","scope: language"]
],[
"start"
],[
"start",
["text","content: -----------------------------"]
],[
"start"
],[
"start",
["text","\\begin{"],
["markup.list","${"],
["constant.numeric","1"],
["punctuation.operator",":"],
["text","documnet"],
["markup.list","}"],
["text","}"]
],[
"start"
],[
"start",
["text"," "],
["markup.list","${"],
["constant.numeric","2"],
["punctuation.operator",":"],
["keyword","$TM_SELECTED_TEXT"],
["text",":some latex"],
["markup.list","}"]
],[
"start"
],[
"start",
["text"," "],
["markup.list","${"],
["constant.numeric","3"],
["punctuation.operator",":"],
["keyword","$TM_SELECTED_TEXT"],
["text","/a/b/c"],
["markup.list","}"]
],[
"start"
],[
"start",
["text"," "],
["markup.list","${"],
["constant.numeric","4"],
["punctuation.operator",":"],
["markup.list","${"],
["keyword","TM_SELECTED_TEXT"],
["string.regex","/(.)/"],
["string","\\"],
["keyword","\\u"],
["variable","$1"],
["string.regex","/g:"],
["text","7"],
["markup.list","}}"]
],[
"start"
],[
"start",
["text","\\end{"],
["variable","$1"],
["text","}"]
],[
"start"
],[
"start",
["variable","$0"],
["constant.language.escape","\\\\"],
["text","$$"]
],[
"start"
],[
"start",
["doc,comment","$$------------------------------------"]
],[
"start"
],[
"start",
["text","tabTrigger: ^3"]
],[
"start"
],[
"start",
["text","name: Heading 3"]
],[
"start"
],[
"start",
["text","scope: language"]
],[
"start"
],[
"start",
["text","content: -----------------------------"]
],[
"start"
],[
"start"
],[
"start"
],[
"start",
["markup.list","${"],
["keyword","TM_CURRENT_LINE"],
["string.regex","/./"],
["string","^"],
["string.regex","/g"],
["markup.list","}"]
],[
"start"
],[
"start",
["doc,comment","$$------------------------------------"]
]]

View file

@ -85,10 +85,11 @@ var JadeHighlightRules = function() {
regex : "^\\s*\/\/(?:\\s*[^-\\s]|\\s+\\S)(?:.*$)"
},
{
token : function(space, text) {
return "punctuation.section.comment";
onMatch: function(value, currentState, stack) {
stack.unshift(this.next, value.length - 2, currentState);
return "comment";
},
regex : "^((\\s*)\/\/)(?:\\s*$)",
regex: /^\s*\/\//,
next: "comment_block"
},
mixin_embed("markdown", "markdown-"),
@ -155,18 +156,18 @@ var JadeHighlightRules = function() {
}
],
"comment_block": [
{
token: function(text) {
{regex: /^\s*/, onMatch: function(value, currentState, stack) {
if (value.length <= stack[1]) {
stack.shift();
stack.shift();
this.next = stack.shift();
return "text";
},
regex: "^(\\1\\S|$)",
"captures": "1",
next: "start"
},
{
token: "comment.block.jade",
regex : ".+"
}
} else {
this.next = "";
return "comment";
}
}, next: "start"},
{defaultToken: "comment"}
],
/*

View file

@ -246,11 +246,11 @@ var JavaScriptHighlightRules = function() {
}, {
// invalid operators
token : "invalid",
regex: /\{\d+,?(?:\d+)?}[+*]|[+*$^?][+*]|[$^][?]|\?{3,}/
regex: /\{\d+\b,?\d*\}[+*]|[+*$^?][+*]|[$^][?]|\?{3,}/
}, {
// operators
token : "constant.language.escape",
regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[()$^+*?]/
regex: /\(\?[:=!]|\)|\{\d+\b,?\d*\}|[+*]\?|[()$^+*?]/
}, {
token : "constant.language.delimiter",
regex: /\|/

View file

@ -98,14 +98,14 @@ var LuaHighlightRules = function() {
this.$rules = {
"start" : [{
stateName: "bracketedComment",
token : function(value, currentState, stack){
onMatch : function(value, currentState, stack){
stack.unshift(this.next, value.length, currentState);
return "comment";
},
regex : /\-\-\[=*\[/,
next : [
{
token : function(value, currentState, stack) {
onMatch : function(value, currentState, stack) {
if (value.length == stack[1]) {
stack.shift();
stack.shift();
@ -129,14 +129,14 @@ var LuaHighlightRules = function() {
},
{
stateName: "bracketedString",
token : function(value, currentState, stack){
onMatch : function(value, currentState, stack){
stack.unshift(this.next, value.length, currentState);
return "comment";
},
regex : /\[=*\[/,
next : [
{
token : function(value, currentState, stack) {
onMatch : function(value, currentState, stack) {
if (value.length == stack[1]) {
stack.shift();
stack.shift();

View file

@ -971,7 +971,7 @@ var PhpLangHighlightRules = function() {
// TODO: Unicode identifiers
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, {
token : function(value, currentSate, state) {
onMatch : function(value, currentSate, state) {
value = value.substr(3);
if (value[0] == "'" || value[0] == '"')
value = value.slice(1, -1);
@ -996,7 +996,7 @@ var PhpLangHighlightRules = function() {
],
"heredoc" : [
{
token : function(value, currentSate, stack) {
onMatch : function(value, currentSate, stack) {
if (stack[1] + ";" != value)
return "string";
stack.shift();

View file

@ -91,8 +91,7 @@ define(function(require, exports, module)
regex : "`.*?`"
},
{
token : function(value)
{
onMatch : function(value) {
if (keywords[value])
return "keyword";
else if (buildinConstants[value])

View file

@ -173,7 +173,7 @@ var RubyHighlightRules = function() {
regex : "=>"
}, {
stateName: "heredoc",
token : function(value, currentState, stack) {
onMatch : function(value, currentState, stack) {
var next = value[2] == '-' ? "indentedHeredoc" : "heredoc";
var tokens = value.split(this.splitRegex);
stack.push(next, tokens[3]);
@ -187,7 +187,7 @@ var RubyHighlightRules = function() {
regex : "(<<-?)(['\"`]?)([\\w]+)(['\"`]?)",
rules: {
heredoc: [{
token: function(value, currentState, stack) {
onMatch: function(value, currentState, stack) {
if (value == stack[1]) {
stack.shift();
stack.shift();
@ -202,7 +202,7 @@ var RubyHighlightRules = function() {
token: "string",
regex: "^ +"
}, {
token: function(value, currentState, stack) {
onMatch: function(value, currentState, stack) {
if (value == stack[1]) {
stack.shift();
stack.shift();

View file

@ -40,8 +40,8 @@ var SassHighlightRules = function() {
var start = this.$rules.start;
if (start[1].token == "comment") {
start.splice(1, 1, {
token: function(value, currentState, stack) {
stack.unshift(this.next, value.length - 2, currentState);
onMatch: function(value, currentState, stack) {
stack.unshift(this.next, -1, value.length - 2, currentState);
return "comment";
},
regex: /^\s*\/\*/,
@ -55,14 +55,14 @@ var SassHighlightRules = function() {
});
this.$rules.comment = [
{regex: /^\s*/, token: function(value, currentState, stack) {
if (value.length < stack[1]) {
stack.shift();
stack.shift();
{regex: /^\s*/, onMatch: function(value, currentState, stack) {
if (stack[1] === -1)
stack[1] = Math.max(stack[2], value.length - 1);
if (value.length <= stack[1]) {
/*shift3x*/stack.shift();stack.shift();stack.shift();
this.next = stack.shift();
return "text";
} else {
stack[1] = value.length;
this.next = "";
return "comment";
}

View file

@ -16,14 +16,14 @@ var SnippetHighlightRules = function() {
{token:"constant.language.escape", regex: /\\[\$}`\\]/},
{token:"keyword", regex: "\\$(?:TM_)?(?:" + builtins + ")\\b"},
{token:"variable", regex: "\\$\\w+"},
{token: function(value, state, stack) {
{onMatch: function(value, state, stack) {
if (stack[1])
stack[1]++;
else
stack.unshift("start", 1);
return this.tokenName;
}, tokenName: "markup.list", regex: "\\${", next: "varDecl"},
{token: function(value, state, stack) {
{onMatch: function(value, state, stack) {
if (!stack[1])
return "text";
stack[1]--;

View file

@ -79,10 +79,15 @@ var Tokenizer = function(rules) {
rule.token = rule.token[0];
} else {
rule.tokenArray = rule.token;
rule.token = this.$arrayTokens;
rule.onMatch = this.$arrayTokens;
}
} else if (typeof rule.token == "function" && !rule.onMatch) {
if (matchcount > 1)
rule.onMatch = this.$applyToken;
else
rule.onMatch = rule.token;
}
if (matchcount > 1) {
if (/\\\d/.test(rule.regex)) {
// Replace any backreferences and offset appropriately.
@ -101,6 +106,11 @@ var Tokenizer = function(rules) {
matchTotal += matchcount;
ruleRegExps.push(adjustedregex);
// makes property access faster
if (!rule.onMatch)
rule.onMatch = null;
rule.__proto__ = null;
}
this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
@ -108,6 +118,25 @@ var Tokenizer = function(rules) {
};
(function() {
this.$applyToken = function(str) {
var values = this.splitRegex.exec(str).slice(1);
var types = this.token.apply(this, values);
// required for compatibility with old modes
if (typeof types === "string")
return [{type: types, value: str}];
var tokens = [];
for (var i = 0, l = types.length; i < l; i++) {
if (values[i])
tokens[tokens.length] = {
type: types[i],
value: values[i]
};
}
return tokens;
},
this.$arrayTokens = function(str) {
if (!str)
return [];
@ -119,13 +148,12 @@ var Tokenizer = function(rules) {
console.error(types , values, str, this.splitRegex, this);
return [{type: "error.invalid", value: str}];
}
for (var i = 0; i < types.length; i++) {
if (values[i + 1]) {
for (var i = 0, l = types.length; i < l; i++) {
if (values[i + 1])
tokens[tokens.length] = {
type: types[i],
value: values[i + 1]
};
}
}
return tokens;
};
@ -188,10 +216,10 @@ var Tokenizer = function(rules) {
rule = state[mapping[i]];
// compute token type
type = typeof rule.token == "function"
? rule.token(value, currentState, stack)
: rule.token;
if (rule.onMatch)
type = rule.onMatch(value, currentState, stack);
else
type = rule.token;
if (rule.next) {
if (typeof rule.next == "string")