jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use new tokenizer features

Browse source
This commit is contained in:
nightwing 2013-01-04 16:43:53 +04:00
commit b06e57b4c2
11 changed files with 50 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

2
lib/ace/mode/abap_highlight_rules.js
View file

@ -113,7 +113,7 @@ var AbapHighlightRules = function() {
{token : "variable.parameter", regex : /sy|pa?\d\d\d\d\|t\d\d\d\.|innnn/}, {token : "variable.parameter", regex : /sy|pa?\d\d\d\d\|t\d\d\d\.|innnn/},
{token : "keyword", regex : compoundKeywords}, {token : "keyword", regex : compoundKeywords},
{token : "variable.parameter", regex : /\w+-\w+(?:-\w+)*/}, {token : "variable.parameter", regex : /\w+-\w+(?:-\w+)*/},
{token : keywordMapper, regex : "\\w+\\b"}, {token : keywordMapper, regex : "\\b\\w+\\b"},
], ],
"qstring" : [ "qstring" : [
{token : "constant.language.escape", regex : "''"}, {token : "constant.language.escape", regex : "''"},

2
lib/ace/mode/dart_highlight_rules.js
View file

@ -51,7 +51,7 @@ var DartHighlightRules = function() {
"regex": "^(#!.*)$" "regex": "^(#!.*)$"
}, },
{ {
"token": [ "keyword.other.import.dart", "meta.declaration.dart"], "token": "keyword.other.import.dart",
"regex": "#(?:\\b)(?:library|import|source|resource)(?:\\b)" "regex": "#(?:\\b)(?:library|import|source|resource)(?:\\b)"
}, },
{ {

8
lib/ace/mode/diff_highlight_rules.js
View file

@ -63,7 +63,7 @@ var DiffHighlightRules = function() {
], ],
"name": "meta." "name": "meta."
}, { }, {
"regex": "^(?:(\\-{3}|\\+{3}|\\*{3})( .+))$", "regex": "^(\\-{3}|\\+{3}|\\*{3})( .+)$",
"token": [ "token": [
"constant.numeric", "constant.numeric",
"meta.tag" "meta.tag"
@ -89,8 +89,10 @@ var DiffHighlightRules = function() {
"regex": "^Index.+$", "regex": "^Index.+$",
"token": "variable" "token": "variable"
}, { }, {
"regex": "^(.*?)(\\s*)$", "regex": "\\s*$",
"token": ["invisible", "invalid"] "token": "invalid"
}, {
"defaultToken": "invisible"
} }
] ]
}; };

13
lib/ace/mode/doc_comment_highlight_rules.js
View file

@ -41,17 +41,10 @@ var DocCommentHighlightRules = function() {
token : "comment.doc.tag", token : "comment.doc.tag",
regex : "@[\\w\\d_]+" // TODO: fix email addresses regex : "@[\\w\\d_]+" // TODO: fix email addresses
}, { }, {
token : "comment.doc", token : "comment.doc.tag",
regex : "\\s+" regex : "\\bTODO\\b"
}, { }, {
token : "comment.doc", defaultToken : "comment.doc"
regex : "TODO"
}, {
token : "comment.doc",
regex : "[^@\\*]+"
}, {
token : "comment.doc",
regex : "."
}] }]
}; };
}; };

4
lib/ace/mode/haml_highlight_rules.js
View file

@ -75,8 +75,8 @@ var HamlHighlightRules = function() {
"next": "start" "next": "start"
}, },
{ {
"token": ["text", "punctuation"], "token": "empty",
"regex": "($)|((?!\\.|#|\\{|\\[|=|-|~|\\/))", "regex": "$|(?!\\.|#|\\{|\\[|=|-|~|\\/)",
"next": "start" "next": "start"
} }
], ],

2
lib/ace/mode/jade_highlight_rules.js
View file

@ -136,7 +136,7 @@ var JadeHighlightRules = function() {
}, },
// Match any tag, id or class. skip AST filters // Match any tag, id or class. skip AST filters
{ {
"token": ["meta.tag.any.jade", "entity.variable.tag.jade"], "token": "meta.tag.any.jade",
"regex": /^\s*(?!\w+\:)(?:[\w]+|(?=\.|#)])/, "regex": /^\s*(?!\w+\:)(?:[\w]+|(?=\.|#)])/,
"next": "tag_single" "next": "tag_single"
}, },

52
lib/ace/mode/javascript_highlight_rules.js
View file

@ -61,7 +61,8 @@ var JavaScriptHighlightRules = function() {
"constant.language": "constant.language":
"null|Infinity|NaN|undefined", "null|Infinity|NaN|undefined",
"support.function": "support.function":
"alert" "alert",
"constant.language.boolean": "true|false"
}, "identifier"); }, "identifier");
// keywords which can be followed by regular expressions // keywords which can be followed by regular expressions
@ -161,9 +162,6 @@ var JavaScriptHighlightRules = function() {
], ],
regex : "(:)(\\s*)(function)(\\s*)(\\()", regex : "(:)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments" next: "function_arguments"
}, {
token : "constant.language.boolean",
regex : /(?:true|false)\b/
}, { }, {
token : "keyword", token : "keyword",
regex : "(?:" + kwBeforeRe + ")\\b", regex : "(?:" + kwBeforeRe + ")\\b",
@ -205,9 +203,6 @@ var JavaScriptHighlightRules = function() {
}, { }, {
token: "comment", token: "comment",
regex: /^#!.*$/ regex: /^#!.*$/
}, {
token : "text",
regex : /\s+/
} }
], ],
// regular expressions are only allowed after certain tokens. This // regular expressions are only allowed after certain tokens. This
@ -253,18 +248,20 @@ var JavaScriptHighlightRules = function() {
}, { }, {
// operators // operators
token : "constant.language.escape", token : "constant.language.escape",
regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[(|)$^+*?]/ regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[()$^+*?]/
}, { }, {
token: "string.regexp", token : "constant.language.delimiter",
regex: /{|[^{\[\/\\(|)$^+*?]+/, regex: /\|/
}, { }, {
token: "constant.language.escape", token: "constant.language.escape",
regex: /\[\^?/, regex: /\[\^?/,
next: "regex_character_class", next: "regex_character_class",
}, { }, {
token: "empty", token: "empty",
regex: "", regex: "$",
next: "start" next: "start"
}, {
defaultToken: "string.regexp"
} }
], ],
"regex_character_class": [ "regex_character_class": [
@ -278,13 +275,12 @@ var JavaScriptHighlightRules = function() {
}, { }, {
token: "constant.language.escape", token: "constant.language.escape",
regex: "-" regex: "-"
}, {
token: "string.regexp.charachterclass",
regex: /[^\]\-\\]+/,
}, { }, {
token: "empty", token: "empty",
regex: "", regex: "$",
next: "start" next: "start"
}, {
defaultToken: "string.regexp.charachterclass"
} }
], ],
"function_arguments": [ "function_arguments": [
@ -304,24 +300,12 @@ var JavaScriptHighlightRules = function() {
} }
], ],
"comment_regex_allowed" : [ "comment_regex_allowed" : [
{ {token : "comment", regex : "\\*\\/", next : "regex_allowed"},
token : "comment", // closing comment {defaultToken : "comment"}
regex : ".*?\\*\\/",
next : "regex_allowed"
}, {
token : "comment", // comment spanning whole line
regex : ".+"
}
], ],
"comment" : [ "comment" : [
{ {token : "comment", regex : "\\*\\/", next : "start"},
token : "comment", // closing comment {defaultToken : "comment"}
regex : ".*?\\*\\/",
next : "start"
}, {
token : "comment", // comment spanning whole line
regex : ".+"
}
], ],
"qqstring" : [ "qqstring" : [
{ {
@ -336,8 +320,7 @@ var JavaScriptHighlightRules = function() {
regex : '"|$', regex : '"|$',
next : "start", next : "start",
}, { }, {
token : "string", defaultToken: "string"
regex : '.|\\w+|\\s+',
} }
], ],
"qstring" : [ "qstring" : [
@ -353,8 +336,7 @@ var JavaScriptHighlightRules = function() {
regex : "'|$", regex : "'|$",
next : "start", next : "start",
}, { }, {
token : "string", defaultToken: "string"
regex : '.|\\w+|\\s+',
} }
] ]
}; };

14
lib/ace/mode/lua_highlight_rules.js
View file

@ -99,7 +99,7 @@ var LuaHighlightRules = function() {
"start" : [{ "start" : [{
stateName: "bracketedComment", stateName: "bracketedComment",
token : function(value, currentState, stack){ token : function(value, currentState, stack){
stack.unshift("bracketedComment", value.length); stack.unshift(this.next, value.length, currentState);
return "comment"; return "comment";
}, },
regex : /\-\-\[=*\[/, regex : /\-\-\[=*\[/,
@ -109,7 +109,7 @@ var LuaHighlightRules = function() {
if (value.length == stack[1]) { if (value.length == stack[1]) {
stack.shift(); stack.shift();
stack.shift(); stack.shift();
this.next = "start"; this.next = stack.shift();
} else { } else {
this.next = ""; this.next = "";
} }
@ -118,8 +118,7 @@ var LuaHighlightRules = function() {
regex : /(?:[^\\]|\\.)*?\]=*\]/, regex : /(?:[^\\]|\\.)*?\]=*\]/,
next : "start" next : "start"
}, { }, {
token : "comment", defaultToken : "comment"
regex : '.+'
} }
] ]
}, },
@ -131,7 +130,7 @@ var LuaHighlightRules = function() {
{ {
stateName: "bracketedString", stateName: "bracketedString",
token : function(value, currentState, stack){ token : function(value, currentState, stack){
stack.unshift("bracketedString", value.length); stack.unshift(this.next, value.length, currentState);
return "comment"; return "comment";
}, },
regex : /\[=*\[/, regex : /\[=*\[/,
@ -141,7 +140,7 @@ var LuaHighlightRules = function() {
if (value.length == stack[1]) { if (value.length == stack[1]) {
stack.shift(); stack.shift();
stack.shift(); stack.shift();
this.next = "start"; this.next = stack.shift();
} else { } else {
this.next = ""; this.next = "";
} }
@ -151,8 +150,7 @@ var LuaHighlightRules = function() {
regex : /(?:[^\\]|\\.)*?\]=*\]/, regex : /(?:[^\\]|\\.)*?\]=*\]/,
next : "start" next : "start"
}, { }, {
token : "comment", defaultToken : "comment"
regex : '.+'
} }
] ]
}, },

12
lib/ace/mode/markdown_highlight_rules.js
View file

@ -56,8 +56,8 @@ var MarkdownHighlightRules = function() {
token : "empty_line", token : "empty_line",
regex : '^$' regex : '^$'
}, { // code span ` }, { // code span `
token : ["support.function", "support.function", "support.function"], token : "support.function",
regex : "(`+)([^\\r]*?[^`])(\\1)" regex : "(`+)(.*?[^`])(\\1)"
}, { // code block }, { // code block
token : "support.function", token : "support.function",
regex : "^[ ]{4}.+" regex : "^[ ]{4}.+"
@ -112,11 +112,11 @@ var MarkdownHighlightRules = function() {
regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+", regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
next : "listblock" next : "listblock"
}, { // strong ** __ }, { // strong ** __
token : ["string", "string", "string"], token : "string",
regex : "([*]{2}|[_]{2}(?=\\S))([^\\r]*?\\S[*_]*)(\\1)" regex : "([*]{2}|[_]{2}(?=\\S))(.*?\\S[*_]*)(\\1)"
}, { // emphasis * _ }, { // emphasis * _
token : ["string", "string", "string"], token : "string",
regex : "([*]|[_](?=\\S))([^\\r]*?\\S[*_]*)(\\1)" regex : "([*]|[_](?=\\S))(.*?\\S[*_]*)(\\1)"
}, { // }, { //
token : ["text", "url", "text"], token : ["text", "url", "text"],
regex : "(<)("+ regex : "(<)("+

6
lib/ace/mode/objectivec_highlight_rules.js
View file

@ -92,8 +92,8 @@ var ObjectiveCHighlightRules = function() {
"regex": "(@)(class|protocol)\\b" "regex": "(@)(class|protocol)\\b"
}, },
{ {
"token": [ "punctuation.definition.storage.type.objc", "punctuation.definition.storage.type.objc", "punctuation"], "token": [ "punctuation.definition.storage.type.objc", "punctuation"],
"regex": "(@)(selector)(\\s*\\()", "regex": "(@selector)(\\s*\\()",
"next": "selectors" "next": "selectors"
}, },
{ {
@ -231,7 +231,7 @@ var ObjectiveCHighlightRules = function() {
"next": "start" "next": "start"
}, },
{ {
"token": ["support.function.any-method.objc", "punctuation.separator.arguments.objc"], "token": "support.function.any-method.objc",
"regex": "\\w+(?::|(?=\]))", "regex": "\\w+(?::|(?=\]))",
"next": "start" "next": "start"
} }

7
lib/ace/tokenizer.js
View file

@ -71,7 +71,7 @@ var Tokenizer = function(rules, flag) {
// Count number of matching groups. 2 extra groups from the full match // Count number of matching groups. 2 extra groups from the full match
// And the catch-all on the end (used to force a match); // And the catch-all on the end (used to force a match);
var adjustedregex = rule.regex var adjustedregex = rule.regex;
var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2; var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
if (Array.isArray(rule.token)) { if (Array.isArray(rule.token)) {
if (rule.token.length == 1) { if (rule.token.length == 1) {
@ -108,11 +108,14 @@ var Tokenizer = function(rules, flag) {
(function() { (function() {
this.$arrayTokens = function(str) { this.$arrayTokens = function(str) {
if (!str)
return [];
var values = str.split(this.splitRegex) var values = str.split(this.splitRegex)
var tokens = []; var tokens = [];
var types = this.tokenArray; var types = this.tokenArray;
if (types.length != values.length - 2) { if (types.length != values.length - 2) {
console.log(types.length , values.length - 2, str, this.splitRegex) if (window.console)
console.error(types.length , values.length - 2, str, this.splitRegex);
return [{type: "error.invalid", value: str}]; return [{type: "error.invalid", value: str}];
} }
for (var i = 0; i < types.length; i++) { for (var i = 0; i < types.length; i++) {