jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

better way of handling keywords

Browse source
This commit is contained in:
nightwing 2012-08-31 16:48:44 +04:00
commit 8d054b9afe
4 changed files with 125 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

68
lib/ace/mode/_test/test_highlight_rules.js
View file

@ -8,7 +8,7 @@ require("amd-loader");
function generateTestData() {
var root = Array(5).join("../") + "/demo/kitchen-sink/docs"
var docs = fs.readdirSync(root)
var specialDocs = fs.readdirSync(".")
var specialDocs = fs.readdirSync(".")
var modes = fs.readdirSync("../").filter(function(x){
return /(\w+)_highlight_rules.js$/.test(x)
}).map(function(x) {
@ -31,12 +31,12 @@ function generateTestData() {
if (!modeName) {
return console.error(p)
}
var filePath = "text_" + modeName + ".txt"
if (specialDocs.indexOf(filePath) == -1) {
filePath = root + "/" + docName
}
var filePath = "text_" + modeName + ".txt"
if (specialDocs.indexOf(filePath) == -1) {
filePath = root + "/" + docName
}
var text = fs.readFileSync(filePath, "utf8")
var Mode = require("../" + modeName).Mode;
var tokenizer = new Mode().getTokenizer();
@ -57,14 +57,16 @@ function generateTestData() {
})
}
function test() {
function test(startAt) {
var docs = fs.readdirSync(".").filter(function(x){
return /^tokens_\w+.json/.test(x)
});
if (startAt && startAt > 1)
docs = docs.slice(startAt - 1)
docs.forEach(function(x, i){
var modeName = x.match(/tokens_(.*).json/)[1]
console.log((" " + (i+1)).slice(-3) + ") testing: \u001b[33m" + modeName + "\u001b[0m")
console.log(padNumber(i+1, 3) + ") testing: \u001b[33m" + modeName + "\u001b[0m")
var text = fs.readFileSync("./" + x, "utf8")
var data = JSON.parse(text)
@ -79,9 +81,12 @@ function test() {
var values = tokens.tokens.map(function(x) {return x.value})
var types = tokens.tokens.map(function(x) {return x.type})
testEqual(lineData.state, tokens.state)
testEqual(lineData.types, types)
testEqual(lineData.values, values)
var success = true
var report =
testEqual([
lineData.state, tokens.state,
lineData.types, types,
lineData.values, values])
state = lineData.state
})
@ -89,14 +94,45 @@ function test() {
console.log("\u001b[32m" + "all ok" + "\u001b[0m")
function testEqual(a, b) {
if (a + "" !== b + "")
throw a + "!=" + b
function testEqual(a) {
var err
if (a[0] !== a[1]) {
console.log(a[0],a[1])
err = 1
}
if ( a[2] + "" !== a[3] + ""
|| a[4] + "" !== a[5] + ""
) {
arrayDiff(a[2],a[3])
arrayDiff(a[4],a[5])
err = 1
}
if (err) {
throw "error"
}
}
function arrayDiff(a1, a2) {
var l = Math.max(a1.length, a2.length);
var out = []
for (var i = 0; i < l; i++) {
out.push("\n", padNumber(i+1, 3), ") ")
if (a1[i] !== a2[i]) {
out.push("\u001b[31m", a1[i], " != ", a2[i], "\u001b[0m")
} else {
out.push(a1[i])
}
}
console.log(out.join(""))
}
function padNumber(num, digits) {
return (" " + num).slice(-digits)
}
}
if (process.argv[2] == "gen") {
generateTestData(process.argv.splice(3))
} else {
test()
test(parseInt(process.argv[2],10) || 0)
}

190
lib/ace/mode/javascript_highlight_rules.js
View file

@ -40,58 +40,43 @@ define(function(require, exports, module) {
"use strict";
var oop = require("../lib/oop");
var lang = require("../lib/lang");
var unicode = require("../unicode");
var DocCommentHighlightRules = require("./doc_comment_highlight_rules").DocCommentHighlightRules;
var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
var JavaScriptHighlightRules = function() {
// see: https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects
var globals = lang.arrayToMap(
// Constructors
("Array|Boolean|Date|Function|Iterator|Number|Object|RegExp|String|Proxy|" +
// E4X
"Namespace|QName|XML|XMLList|" +
"ArrayBuffer|Float32Array|Float64Array|Int16Array|Int32Array|Int8Array|" +
"Uint16Array|Uint32Array|Uint8Array|Uint8ClampedArray|" +
// Errors
"Error|EvalError|InternalError|RangeError|ReferenceError|StopIteration|" +
"SyntaxError|TypeError|URIError|" +
// Non-constructor functions
"decodeURI|decodeURIComponent|encodeURI|encodeURIComponent|eval|isFinite|" +
"isNaN|parseFloat|parseInt|" +
// Other
"JSON|Math|" +
// Pseudo
"this|arguments|prototype|window|document"
).split("|")
);
var keywordMapper = this.createKeywordMapper({
"variable.language":
"Array|Boolean|Date|Function|Iterator|Number|Object|RegExp|String|Proxy|" + // Constructors
"Namespace|QName|XML|XMLList|" + // E4X
"ArrayBuffer|Float32Array|Float64Array|Int16Array|Int32Array|Int8Array|" +
"Uint16Array|Uint32Array|Uint8Array|Uint8ClampedArray|" +
"Error|EvalError|InternalError|RangeError|ReferenceError|StopIteration|" + // Errors
"SyntaxError|TypeError|URIError|" +
"decodeURI|decodeURIComponent|encodeURI|encodeURIComponent|eval|isFinite|" + // Non-constructor functions
"isNaN|parseFloat|parseInt|" +
"JSON|Math|" + // Other
"this|arguments|prototype|window|document" , // Pseudo
"invalid.deprecated":
"__parent__|__count__|escape|unescape|with|__proto__|debugger",
"keyword":
"const|yield|import|get|set" +
"break|case|catch|continue|default|delete|do|else|finally|for|function|" +
"if|in|instanceof|new|return|switch|throw|try|typeof|let|var|while|with|",
"storage.type":
"const|let|var|function",
"invalid.illegal":
"class|enum|extends|super|export|implements|private|" +
"public|interface|package|protected|static",
"constant.language":
"null|Infinity|NaN|undefined",
}, "identifier");
var keywords = lang.arrayToMap(
("break|case|catch|continue|default|delete|do|else|finally|for|function|" +
"if|in|instanceof|new|return|switch|throw|try|typeof|let|var|while|with|" +
"const|yield|import|get|set").split("|")
);
// keywords which can be followed by regular expressions
var kwBeforeRe = "case|do|else|finally|in|instanceof|return|throw|try|typeof|yield";
var deprecated = lang.arrayToMap(
("__parent__|__count__|escape|unescape|with|__proto__").split("|")
);
var definitions = lang.arrayToMap(("const|let|var|function").split("|"));
var buildinConstants = lang.arrayToMap(
("null|Infinity|NaN|undefined").split("|")
);
var futureReserved = lang.arrayToMap(
("class|enum|extends|super|export|implements|private|" +
"public|interface|package|protected|static").split("|")
);
// TODO: Unicode escape sequences
var identifierRe = "[a-zA-Z\\$_\u00a1-\uffff][a-zA-Z\d\\$_\u00a1-\uffff]*\\b";
@ -132,106 +117,58 @@ var JavaScriptHighlightRules = function() {
}, {
token : "constant.numeric", // float
regex : /[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?\b/
}, { // match stuff like: Sound.prototype.play = function() { }
}, {
// Sound.prototype.play =
token : [
"storage.type",
"punctuation.operator",
"support.function",
"punctuation.operator",
"entity.name.function",
"text",
"keyword.operator",
"text",
"storage.type",
"text",
"paren.lparen"
"storage.type", "punctuation.operator", "support.function",
"punctuation.operator", "entity.name.function", "text","keyword.operator"
],
regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()",
regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)",
next: "function_arguments"
}, { // match stuff like: Sound.prototype.play = myfunc
}, {
// Sound.play = function() { }
token : [
"storage.type",
"punctuation.operator",
"support.function",
"punctuation.operator",
"entity.name.function",
"text",
"keyword.operator",
"text"
],
regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)",
next: "function_arguments"
}, { // match stuff like: Sound.play = function() { }
token : [
"storage.type",
"punctuation.operator",
"entity.name.function",
"text",
"keyword.operator",
"text",
"storage.type",
"text",
"paren.lparen"
"storage.type", "punctuation.operator", "entity.name.function", "text",
"keyword.operator", "text", "storage.type", "text", "paren.lparen"
],
regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments"
}, { // match stuff like: play = function() { }
}, {
// play = function() { }
token : [
"entity.name.function",
"text",
"keyword.operator",
"text",
"storage.type",
"text",
"paren.lparen"
"entity.name.function", "text", "keyword.operator", "text", "storage.type",
"text", "paren.lparen"
],
regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments"
}, { // match stuff like: Sound.play = function play() { }
}, {
// Sound.play = function play() { }
token : [
"storage.type",
"punctuation.operator",
"entity.name.function",
"text",
"keyword.operator",
"text",
"storage.type",
"text",
"entity.name.function",
"text",
"paren.lparen"
"storage.type", "punctuation.operator", "entity.name.function", "text",
"keyword.operator", "text",
"storage.type", "text", "entity.name.function", "text", "paren.lparen"
],
regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s+)(\\w+)(\\s*)(\\()",
next: "function_arguments"
}, { // match regular function like: function myFunc(arg) { }
}, {
// function myFunc(arg) { }
token : [
"storage.type",
"text",
"entity.name.function",
"text",
"paren.lparen"
"storage.type", "text", "entity.name.function", "text", "paren.lparen"
],
regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()",
next: "function_arguments"
}, { // match stuff like: foobar: function() { }
}, {
// foobar: function() { }
token : [
"entity.name.function",
"text",
"punctuation.operator",
"text",
"storage.type",
"text",
"paren.lparen"
"entity.name.function", "text", "punctuation.operator",
"text", "storage.type", "text", "paren.lparen"
],
regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments"
}, { // Attempt to match : function() { } (this is for issues with 'foo': function() { })
}, {
// : function() { } (this is for issues with 'foo': function() { })
token : [
"text",
"text",
"storage.type",
"text",
"paren.lparen"
"text", "text", "storage.type", "text", "paren.lparen"
],
regex : "(:)(\\s*)(function)(\\s*)(\\()",
next: "function_arguments"
@ -255,24 +192,7 @@ var JavaScriptHighlightRules = function() {
token : ["storage.type", "punctuation.operator", "support.function.firebug"],
regex : /(console)(\.)(warn|info|log|error|time|timeEnd|assert)\b/
}, {
token : function(value) {
if (globals.hasOwnProperty(value))
return "variable.language";
else if (deprecated.hasOwnProperty(value))
return "invalid.deprecated";
else if (definitions.hasOwnProperty(value))
return "storage.type";
else if (keywords.hasOwnProperty(value))
return "keyword";
else if (buildinConstants.hasOwnProperty(value))
return "constant.language";
else if (futureReserved.hasOwnProperty(value))
return "invalid.illegal";
else if (value == "debugger")
return "invalid.deprecated";
else
return "identifier";
},
token : keywordMapper,
regex : identifierRe
}, {
token : "keyword.operator",

21
lib/ace/mode/text_highlight_rules.js
View file

@ -74,7 +74,7 @@ var TextHighlightRules = function() {
this.getRules = function() {
return this.$rules;
};
this.embedRules = function (HighlightRules, prefix, escapeRules, states) {
var embedRules = new HighlightRules().getRules();
if (states) {
@ -88,21 +88,34 @@ var TextHighlightRules = function() {
}
}
this.addRules(embedRules, prefix);
for (var i = 0; i < states.length; i++) {
Array.prototype.unshift.apply(this.$rules[states[i]], lang.deepCopy(escapeRules));
}
if (!this.$embeds) {
this.$embeds = [];
}
this.$embeds.push(prefix);
}
this.getEmbeds = function() {
return this.$embeds;
}
this.createKeywordMapper = function(map, defaultToken, ignoreCase, splitChar) {
var keywords = Object.create(null);
Object.keys(map).forEach(function(className) {
var list = map[className].split(splitChar || "|");
for (var i = list.length; i--; )
keywords[list[i]] = className;
});
map = null;
return ignoreCase
? function(value) {return keywords[value.toLowerCase()] || defaultToken }
: function(value) {return keywords[value] || defaultToken };
}
}).call(TextHighlightRules.prototype);
exports.TextHighlightRules = TextHighlightRules;

2
lib/ace/mode/textile_highlight_rules.js
View file

@ -46,7 +46,7 @@ var TextileHighlightRules = function() {
"start" : [
{
token : function(value) {
if (value.match(/^h\d$/))
if (value.charAt(0) == "h")
return "markup.heading." + value.charAt(1);
else
return "markup.heading";