jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

better way of handling keywords

Browse source
This commit is contained in:
nightwing 2012-08-31 16:48:44 +04:00
commit 8d054b9afe
4 changed files with 125 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

68
lib/ace/mode/_test/test_highlight_rules.js
View file

@ -8,7 +8,7 @@ require("amd-loader");
function generateTestData() { function generateTestData() {
var root = Array(5).join("../") + "/demo/kitchen-sink/docs" var root = Array(5).join("../") + "/demo/kitchen-sink/docs"
var docs = fs.readdirSync(root) var docs = fs.readdirSync(root)
var specialDocs = fs.readdirSync(".") var specialDocs = fs.readdirSync(".")
var modes = fs.readdirSync("../").filter(function(x){ var modes = fs.readdirSync("../").filter(function(x){
return /(\w+)_highlight_rules.js$/.test(x) return /(\w+)_highlight_rules.js$/.test(x)
}).map(function(x) { }).map(function(x) {
@ -31,12 +31,12 @@ function generateTestData() {
if (!modeName) { if (!modeName) {
return console.error(p) return console.error(p)
} }
var filePath = "text_" + modeName + ".txt" var filePath = "text_" + modeName + ".txt"
if (specialDocs.indexOf(filePath) == -1) { if (specialDocs.indexOf(filePath) == -1) {
filePath = root + "/" + docName filePath = root + "/" + docName
} }
var text = fs.readFileSync(filePath, "utf8") var text = fs.readFileSync(filePath, "utf8")
var Mode = require("../" + modeName).Mode; var Mode = require("../" + modeName).Mode;
var tokenizer = new Mode().getTokenizer(); var tokenizer = new Mode().getTokenizer();
@ -57,14 +57,16 @@ function generateTestData() {
}) })
} }
function test() { function test(startAt) {
var docs = fs.readdirSync(".").filter(function(x){ var docs = fs.readdirSync(".").filter(function(x){
return /^tokens_\w+.json/.test(x) return /^tokens_\w+.json/.test(x)
}); });
if (startAt && startAt > 1)
docs = docs.slice(startAt - 1)
docs.forEach(function(x, i){ docs.forEach(function(x, i){
var modeName = x.match(/tokens_(.*).json/)[1] var modeName = x.match(/tokens_(.*).json/)[1]
console.log((" " + (i+1)).slice(-3) + ") testing: \u001b[33m" + modeName + "\u001b[0m") console.log(padNumber(i+1, 3) + ") testing: \u001b[33m" + modeName + "\u001b[0m")
var text = fs.readFileSync("./" + x, "utf8") var text = fs.readFileSync("./" + x, "utf8")
var data = JSON.parse(text) var data = JSON.parse(text)
@ -79,9 +81,12 @@ function test() {
var values = tokens.tokens.map(function(x) {return x.value}) var values = tokens.tokens.map(function(x) {return x.value})
var types = tokens.tokens.map(function(x) {return x.type}) var types = tokens.tokens.map(function(x) {return x.type})
testEqual(lineData.state, tokens.state) var success = true
testEqual(lineData.types, types) var report =
testEqual(lineData.values, values) testEqual([
lineData.state, tokens.state,
lineData.types, types,
lineData.values, values])
state = lineData.state state = lineData.state
}) })
@ -89,14 +94,45 @@ function test() {
console.log("\u001b[32m" + "all ok" + "\u001b[0m") console.log("\u001b[32m" + "all ok" + "\u001b[0m")
function testEqual(a, b) { function testEqual(a) {
if (a + "" !== b + "") var err
throw a + "!=" + b if (a[0] !== a[1]) {
console.log(a[0],a[1])
err = 1
}
if ( a[2] + "" !== a[3] + ""
|| a[4] + "" !== a[5] + ""
) {
arrayDiff(a[2],a[3])
arrayDiff(a[4],a[5])
err = 1
}
if (err) {
throw "error"
}
}
function arrayDiff(a1, a2) {
var l = Math.max(a1.length, a2.length);
var out = []
for (var i = 0; i < l; i++) {
out.push("\n", padNumber(i+1, 3), ") ")
if (a1[i] !== a2[i]) {
out.push("\u001b[31m", a1[i], " != ", a2[i], "\u001b[0m")
} else {
out.push(a1[i])
}
}
console.log(out.join(""))
}
function padNumber(num, digits) {
return (" " + num).slice(-digits)
} }
} }
if (process.argv[2] == "gen") { if (process.argv[2] == "gen") {
generateTestData(process.argv.splice(3)) generateTestData(process.argv.splice(3))
} else { } else {
test() test(parseInt(process.argv[2],10) || 0)
} }

190
lib/ace/mode/javascript_highlight_rules.js
View file

@ -40,58 +40,43 @@ define(function(require, exports, module) {
"use strict"; "use strict";
var oop = require("../lib/oop"); var oop = require("../lib/oop");
var lang = require("../lib/lang");
var unicode = require("../unicode"); var unicode = require("../unicode");
var DocCommentHighlightRules = require("./doc_comment_highlight_rules").DocCommentHighlightRules; var DocCommentHighlightRules = require("./doc_comment_highlight_rules").DocCommentHighlightRules;
var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules; var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
var JavaScriptHighlightRules = function() { var JavaScriptHighlightRules = function() {
// see: https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects // see: https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects
var globals = lang.arrayToMap( var keywordMapper = this.createKeywordMapper({
// Constructors "variable.language":
("Array|Boolean|Date|Function|Iterator|Number|Object|RegExp|String|Proxy|" + "Array|Boolean|Date|Function|Iterator|Number|Object|RegExp|String|Proxy|" + // Constructors
// E4X "Namespace|QName|XML|XMLList|" + // E4X
"Namespace|QName|XML|XMLList|" + "ArrayBuffer|Float32Array|Float64Array|Int16Array|Int32Array|Int8Array|" +
"ArrayBuffer|Float32Array|Float64Array|Int16Array|Int32Array|Int8Array|" + "Uint16Array|Uint32Array|Uint8Array|Uint8ClampedArray|" +
"Uint16Array|Uint32Array|Uint8Array|Uint8ClampedArray|" + "Error|EvalError|InternalError|RangeError|ReferenceError|StopIteration|" + // Errors
// Errors "SyntaxError|TypeError|URIError|" +
"Error|EvalError|InternalError|RangeError|ReferenceError|StopIteration|" + "decodeURI|decodeURIComponent|encodeURI|encodeURIComponent|eval|isFinite|" + // Non-constructor functions
"SyntaxError|TypeError|URIError|" + "isNaN|parseFloat|parseInt|" +
// Non-constructor functions "JSON|Math|" + // Other
"decodeURI|decodeURIComponent|encodeURI|encodeURIComponent|eval|isFinite|" + "this|arguments|prototype|window|document" , // Pseudo
"isNaN|parseFloat|parseInt|" + "invalid.deprecated":
// Other "__parent__|__count__|escape|unescape|with|__proto__|debugger",
"JSON|Math|" + "keyword":
// Pseudo "const|yield|import|get|set" +
"this|arguments|prototype|window|document" "break|case|catch|continue|default|delete|do|else|finally|for|function|" +
).split("|") "if|in|instanceof|new|return|switch|throw|try|typeof|let|var|while|with|",
); "storage.type":
"const|let|var|function",
"invalid.illegal":
"class|enum|extends|super|export|implements|private|" +
"public|interface|package|protected|static",
"constant.language":
"null|Infinity|NaN|undefined",
}, "identifier");
var keywords = lang.arrayToMap(
("break|case|catch|continue|default|delete|do|else|finally|for|function|" +
"if|in|instanceof|new|return|switch|throw|try|typeof|let|var|while|with|" +
"const|yield|import|get|set").split("|")
);
// keywords which can be followed by regular expressions // keywords which can be followed by regular expressions
var kwBeforeRe = "case|do|else|finally|in|instanceof|return|throw|try|typeof|yield"; var kwBeforeRe = "case|do|else|finally|in|instanceof|return|throw|try|typeof|yield";
var deprecated = lang.arrayToMap(
("__parent__|__count__|escape|unescape|with|__proto__").split("|")
);
var definitions = lang.arrayToMap(("const|let|var|function").split("|"));
var buildinConstants = lang.arrayToMap(
("null|Infinity|NaN|undefined").split("|")
);
var futureReserved = lang.arrayToMap(
("class|enum|extends|super|export|implements|private|" +
"public|interface|package|protected|static").split("|")
);
// TODO: Unicode escape sequences // TODO: Unicode escape sequences
var identifierRe = "[a-zA-Z\\$_\u00a1-\uffff][a-zA-Z\d\\$_\u00a1-\uffff]*\\b"; var identifierRe = "[a-zA-Z\\$_\u00a1-\uffff][a-zA-Z\d\\$_\u00a1-\uffff]*\\b";
@ -132,106 +117,58 @@ var JavaScriptHighlightRules = function() {
}, { }, {
token : "constant.numeric", // float token : "constant.numeric", // float
regex : /[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?\b/ regex : /[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?\b/
}, { // match stuff like: Sound.prototype.play = function() { } }, {
// Sound.prototype.play =
token : [ token : [
"storage.type", "storage.type", "punctuation.operator", "support.function",
"punctuation.operator", "punctuation.operator", "entity.name.function", "text","keyword.operator"
"support.function",
"punctuation.operator",
"entity.name.function",
"text",
"keyword.operator",
"text",
"storage.type",
"text",
"paren.lparen"
], ],
regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)",
next: "function_arguments" next: "function_arguments"
}, { // match stuff like: Sound.prototype.play = myfunc }, {
// Sound.play = function() { }
token : [ token : [
"storage.type", "storage.type", "punctuation.operator", "entity.name.function", "text",
"punctuation.operator", "keyword.operator", "text", "storage.type", "text", "paren.lparen"
"support.function",
"punctuation.operator",
"entity.name.function",
"text",
"keyword.operator",
"text"
],
regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)",
next: "function_arguments"
}, { // match stuff like: Sound.play = function() { }
token : [
"storage.type",
"punctuation.operator",
"entity.name.function",
"text",
"keyword.operator",
"text",
"storage.type",
"text",
"paren.lparen"
], ],
regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments" next: "function_arguments"
}, { // match stuff like: play = function() { } }, {
// play = function() { }
token : [ token : [
"entity.name.function", "entity.name.function", "text", "keyword.operator", "text", "storage.type",
"text", "text", "paren.lparen"
"keyword.operator",
"text",
"storage.type",
"text",
"paren.lparen"
], ],
regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments" next: "function_arguments"
}, { // match stuff like: Sound.play = function play() { } }, {
// Sound.play = function play() { }
token : [ token : [
"storage.type", "storage.type", "punctuation.operator", "entity.name.function", "text",
"punctuation.operator", "keyword.operator", "text",
"entity.name.function", "storage.type", "text", "entity.name.function", "text", "paren.lparen"
"text",
"keyword.operator",
"text",
"storage.type",
"text",
"entity.name.function",
"text",
"paren.lparen"
], ],
regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s+)(\\w+)(\\s*)(\\()", regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s+)(\\w+)(\\s*)(\\()",
next: "function_arguments" next: "function_arguments"
}, { // match regular function like: function myFunc(arg) { } }, {
// function myFunc(arg) { }
token : [ token : [
"storage.type", "storage.type", "text", "entity.name.function", "text", "paren.lparen"
"text",
"entity.name.function",
"text",
"paren.lparen"
], ],
regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()", regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()",
next: "function_arguments" next: "function_arguments"
}, { // match stuff like: foobar: function() { } }, {
// foobar: function() { }
token : [ token : [
"entity.name.function", "entity.name.function", "text", "punctuation.operator",
"text", "text", "storage.type", "text", "paren.lparen"
"punctuation.operator",
"text",
"storage.type",
"text",
"paren.lparen"
], ],
regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()", regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments" next: "function_arguments"
}, { // Attempt to match : function() { } (this is for issues with 'foo': function() { }) }, {
// : function() { } (this is for issues with 'foo': function() { })
token : [ token : [
"text", "text", "text", "storage.type", "text", "paren.lparen"
"text",
"storage.type",
"text",
"paren.lparen"
], ],
regex : "(:)(\\s*)(function)(\\s*)(\\()", regex : "(:)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments" next: "function_arguments"
@ -255,24 +192,7 @@ var JavaScriptHighlightRules = function() {
token : ["storage.type", "punctuation.operator", "support.function.firebug"], token : ["storage.type", "punctuation.operator", "support.function.firebug"],
regex : /(console)(\.)(warn|info|log|error|time|timeEnd|assert)\b/ regex : /(console)(\.)(warn|info|log|error|time|timeEnd|assert)\b/
}, { }, {
token : function(value) { token : keywordMapper,
if (globals.hasOwnProperty(value))
return "variable.language";
else if (deprecated.hasOwnProperty(value))
return "invalid.deprecated";
else if (definitions.hasOwnProperty(value))
return "storage.type";
else if (keywords.hasOwnProperty(value))
return "keyword";
else if (buildinConstants.hasOwnProperty(value))
return "constant.language";
else if (futureReserved.hasOwnProperty(value))
return "invalid.illegal";
else if (value == "debugger")
return "invalid.deprecated";
else
return "identifier";
},
regex : identifierRe regex : identifierRe
}, { }, {
token : "keyword.operator", token : "keyword.operator",

21
lib/ace/mode/text_highlight_rules.js
View file

@ -74,7 +74,7 @@ var TextHighlightRules = function() {
this.getRules = function() { this.getRules = function() {
return this.$rules; return this.$rules;
}; };
this.embedRules = function (HighlightRules, prefix, escapeRules, states) { this.embedRules = function (HighlightRules, prefix, escapeRules, states) {
var embedRules = new HighlightRules().getRules(); var embedRules = new HighlightRules().getRules();
if (states) { if (states) {
@ -88,21 +88,34 @@ var TextHighlightRules = function() {
} }
} }
this.addRules(embedRules, prefix); this.addRules(embedRules, prefix);
for (var i = 0; i < states.length; i++) { for (var i = 0; i < states.length; i++) {
Array.prototype.unshift.apply(this.$rules[states[i]], lang.deepCopy(escapeRules)); Array.prototype.unshift.apply(this.$rules[states[i]], lang.deepCopy(escapeRules));
} }
if (!this.$embeds) { if (!this.$embeds) {
this.$embeds = []; this.$embeds = [];
} }
this.$embeds.push(prefix); this.$embeds.push(prefix);
} }
this.getEmbeds = function() { this.getEmbeds = function() {
return this.$embeds; return this.$embeds;
} }
this.createKeywordMapper = function(map, defaultToken, ignoreCase, splitChar) {
var keywords = Object.create(null);
Object.keys(map).forEach(function(className) {
var list = map[className].split(splitChar || "|");
for (var i = list.length; i--; )
keywords[list[i]] = className;
});
map = null;
return ignoreCase
? function(value) {return keywords[value.toLowerCase()] || defaultToken }
: function(value) {return keywords[value] || defaultToken };
}
}).call(TextHighlightRules.prototype); }).call(TextHighlightRules.prototype);
exports.TextHighlightRules = TextHighlightRules; exports.TextHighlightRules = TextHighlightRules;

2
lib/ace/mode/textile_highlight_rules.js
View file

@ -46,7 +46,7 @@ var TextileHighlightRules = function() {
"start" : [ "start" : [
{ {
token : function(value) { token : function(value) {
if (value.match(/^h\d$/)) if (value.charAt(0) == "h")
return "markup.heading." + value.charAt(1); return "markup.heading." + value.charAt(1);
else else
return "markup.heading"; return "markup.heading";