jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use new tokenizer features

Browse source
This commit is contained in:
nightwing 2013-01-04 16:43:53 +04:00
commit b06e57b4c2
11 changed files with 50 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

2
lib/ace/mode/abap_highlight_rules.js
View file

@ -113,7 +113,7 @@ var AbapHighlightRules = function() {
{token : "variable.parameter", regex : /sy|pa?\d\d\d\d\|t\d\d\d\.|innnn/},
{token : "keyword", regex : compoundKeywords},
{token : "variable.parameter", regex : /\w+-\w+(?:-\w+)*/},
{token : keywordMapper, regex : "\\w+\\b"},
{token : keywordMapper, regex : "\\b\\w+\\b"},
],
"qstring" : [
{token : "constant.language.escape", regex : "''"},

2
lib/ace/mode/dart_highlight_rules.js
View file

@ -51,7 +51,7 @@ var DartHighlightRules = function() {
"regex": "^(#!.*)$"
},
{
"token": [ "keyword.other.import.dart", "meta.declaration.dart"],
"token": "keyword.other.import.dart",
"regex": "#(?:\\b)(?:library|import|source|resource)(?:\\b)"
},
{

8
lib/ace/mode/diff_highlight_rules.js
View file

@ -63,7 +63,7 @@ var DiffHighlightRules = function() {
],
"name": "meta."
}, {
"regex": "^(?:(\\-{3}|\\+{3}|\\*{3})( .+))$",
"regex": "^(\\-{3}|\\+{3}|\\*{3})( .+)$",
"token": [
"constant.numeric",
"meta.tag"
@ -89,8 +89,10 @@ var DiffHighlightRules = function() {
"regex": "^Index.+$",
"token": "variable"
}, {
"regex": "^(.*?)(\\s*)$",
"token": ["invisible", "invalid"]
"regex": "\\s*$",
"token": "invalid"
}, {
"defaultToken": "invisible"
}
]
};

13
lib/ace/mode/doc_comment_highlight_rules.js
View file

@ -41,17 +41,10 @@ var DocCommentHighlightRules = function() {
token : "comment.doc.tag",
regex : "@[\\w\\d_]+" // TODO: fix email addresses
}, {
token : "comment.doc",
regex : "\\s+"
token : "comment.doc.tag",
regex : "\\bTODO\\b"
}, {
token : "comment.doc",
regex : "TODO"
}, {
token : "comment.doc",
regex : "[^@\\*]+"
}, {
token : "comment.doc",
regex : "."
defaultToken : "comment.doc"
}]
};
};

4
lib/ace/mode/haml_highlight_rules.js
View file

@ -75,8 +75,8 @@ var HamlHighlightRules = function() {
"next": "start"
},
{
"token": ["text", "punctuation"],
"regex": "($)|((?!\\.|#|\\{|\\[|=|-|~|\\/))",
"token": "empty",
"regex": "$|(?!\\.|#|\\{|\\[|=|-|~|\\/)",
"next": "start"
}
],

2
lib/ace/mode/jade_highlight_rules.js
View file

@ -136,7 +136,7 @@ var JadeHighlightRules = function() {
},
// Match any tag, id or class. skip AST filters
{
"token": ["meta.tag.any.jade", "entity.variable.tag.jade"],
"token": "meta.tag.any.jade",
"regex": /^\s*(?!\w+\:)(?:[\w]+|(?=\.|#)])/,
"next": "tag_single"
},

52
lib/ace/mode/javascript_highlight_rules.js
View file

@ -61,7 +61,8 @@ var JavaScriptHighlightRules = function() {
"constant.language":
"null|Infinity|NaN|undefined",
"support.function":
"alert"
"alert",
"constant.language.boolean": "true|false"
}, "identifier");
// keywords which can be followed by regular expressions
@ -161,9 +162,6 @@ var JavaScriptHighlightRules = function() {
],
regex : "(:)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments"
}, {
token : "constant.language.boolean",
regex : /(?:true|false)\b/
}, {
token : "keyword",
regex : "(?:" + kwBeforeRe + ")\\b",
@ -205,9 +203,6 @@ var JavaScriptHighlightRules = function() {
}, {
token: "comment",
regex: /^#!.*$/
}, {
token : "text",
regex : /\s+/
}
],
// regular expressions are only allowed after certain tokens. This
@ -253,18 +248,20 @@ var JavaScriptHighlightRules = function() {
}, {
// operators
token : "constant.language.escape",
regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[(|)$^+*?]/
regex: /\(\?[:=!]|\)|{\d+,?(?:\d+)?}|{,\d+}|[+*]\?|[()$^+*?]/
}, {
token: "string.regexp",
regex: /{|[^{\[\/\\(|)$^+*?]+/,
token : "constant.language.delimiter",
regex: /\|/
}, {
token: "constant.language.escape",
regex: /\[\^?/,
next: "regex_character_class",
}, {
token: "empty",
regex: "",
regex: "$",
next: "start"
}, {
defaultToken: "string.regexp"
}
],
"regex_character_class": [
@ -278,13 +275,12 @@ var JavaScriptHighlightRules = function() {
}, {
token: "constant.language.escape",
regex: "-"
}, {
token: "string.regexp.charachterclass",
regex: /[^\]\-\\]+/,
}, {
token: "empty",
regex: "",
regex: "$",
next: "start"
}, {
defaultToken: "string.regexp.charachterclass"
}
],
"function_arguments": [
@ -304,24 +300,12 @@ var JavaScriptHighlightRules = function() {
}
],
"comment_regex_allowed" : [
{
token : "comment", // closing comment
regex : ".*?\\*\\/",
next : "regex_allowed"
}, {
token : "comment", // comment spanning whole line
regex : ".+"
}
{token : "comment", regex : "\\*\\/", next : "regex_allowed"},
{defaultToken : "comment"}
],
"comment" : [
{
token : "comment", // closing comment
regex : ".*?\\*\\/",
next : "start"
}, {
token : "comment", // comment spanning whole line
regex : ".+"
}
{token : "comment", regex : "\\*\\/", next : "start"},
{defaultToken : "comment"}
],
"qqstring" : [
{
@ -336,8 +320,7 @@ var JavaScriptHighlightRules = function() {
regex : '"|$',
next : "start",
}, {
token : "string",
regex : '.|\\w+|\\s+',
defaultToken: "string"
}
],
"qstring" : [
@ -353,8 +336,7 @@ var JavaScriptHighlightRules = function() {
regex : "'|$",
next : "start",
}, {
token : "string",
regex : '.|\\w+|\\s+',
defaultToken: "string"
}
]
};

14
lib/ace/mode/lua_highlight_rules.js
View file

@ -99,7 +99,7 @@ var LuaHighlightRules = function() {
"start" : [{
stateName: "bracketedComment",
token : function(value, currentState, stack){
stack.unshift("bracketedComment", value.length);
stack.unshift(this.next, value.length, currentState);
return "comment";
},
regex : /\-\-\[=*\[/,
@ -109,7 +109,7 @@ var LuaHighlightRules = function() {
if (value.length == stack[1]) {
stack.shift();
stack.shift();
this.next = "start";
this.next = stack.shift();
} else {
this.next = "";
}
@ -118,8 +118,7 @@ var LuaHighlightRules = function() {
regex : /(?:[^\\]|\\.)*?\]=*\]/,
next : "start"
}, {
token : "comment",
regex : '.+'
defaultToken : "comment"
}
]
},
@ -131,7 +130,7 @@ var LuaHighlightRules = function() {
{
stateName: "bracketedString",
token : function(value, currentState, stack){
stack.unshift("bracketedString", value.length);
stack.unshift(this.next, value.length, currentState);
return "comment";
},
regex : /\[=*\[/,
@ -141,7 +140,7 @@ var LuaHighlightRules = function() {
if (value.length == stack[1]) {
stack.shift();
stack.shift();
this.next = "start";
this.next = stack.shift();
} else {
this.next = "";
}
@ -151,8 +150,7 @@ var LuaHighlightRules = function() {
regex : /(?:[^\\]|\\.)*?\]=*\]/,
next : "start"
}, {
token : "comment",
regex : '.+'
defaultToken : "comment"
}
]
},

12
lib/ace/mode/markdown_highlight_rules.js
View file

@ -56,8 +56,8 @@ var MarkdownHighlightRules = function() {
token : "empty_line",
regex : '^$'
}, { // code span `
token : ["support.function", "support.function", "support.function"],
regex : "(`+)([^\\r]*?[^`])(\\1)"
token : "support.function",
regex : "(`+)(.*?[^`])(\\1)"
}, { // code block
token : "support.function",
regex : "^[ ]{4}.+"
@ -112,11 +112,11 @@ var MarkdownHighlightRules = function() {
regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
next : "listblock"
}, { // strong ** __
token : ["string", "string", "string"],
regex : "([*]{2}|[_]{2}(?=\\S))([^\\r]*?\\S[*_]*)(\\1)"
token : "string",
regex : "([*]{2}|[_]{2}(?=\\S))(.*?\\S[*_]*)(\\1)"
}, { // emphasis * _
token : ["string", "string", "string"],
regex : "([*]|[_](?=\\S))([^\\r]*?\\S[*_]*)(\\1)"
token : "string",
regex : "([*]|[_](?=\\S))(.*?\\S[*_]*)(\\1)"
}, { //
token : ["text", "url", "text"],
regex : "(<)("+

6
lib/ace/mode/objectivec_highlight_rules.js
View file

@ -92,8 +92,8 @@ var ObjectiveCHighlightRules = function() {
"regex": "(@)(class|protocol)\\b"
},
{
"token": [ "punctuation.definition.storage.type.objc", "punctuation.definition.storage.type.objc", "punctuation"],
"regex": "(@)(selector)(\\s*\\()",
"token": [ "punctuation.definition.storage.type.objc", "punctuation"],
"regex": "(@selector)(\\s*\\()",
"next": "selectors"
},
{
@ -231,7 +231,7 @@ var ObjectiveCHighlightRules = function() {
"next": "start"
},
{
"token": ["support.function.any-method.objc", "punctuation.separator.arguments.objc"],
"token": "support.function.any-method.objc",
"regex": "\\w+(?::|(?=\]))",
"next": "start"
}

7
lib/ace/tokenizer.js
View file

@ -71,7 +71,7 @@ var Tokenizer = function(rules, flag) {
// Count number of matching groups. 2 extra groups from the full match
// And the catch-all on the end (used to force a match);
var adjustedregex = rule.regex
var adjustedregex = rule.regex;
var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
if (Array.isArray(rule.token)) {
if (rule.token.length == 1) {
@ -108,11 +108,14 @@ var Tokenizer = function(rules, flag) {
(function() {
this.$arrayTokens = function(str) {
if (!str)
return [];
var values = str.split(this.splitRegex)
var tokens = [];
var types = this.tokenArray;
if (types.length != values.length - 2) {
console.log(types.length , values.length - 2, str, this.splitRegex)
if (window.console)
console.error(types.length , values.length - 2, str, this.splitRegex);
return [{type: "error.invalid", value: str}];
}
for (var i = 0; i < types.length; i++) {