diff --git a/lib/ace/mode/_test/test_highlight_rules.js b/lib/ace/mode/_test/test_highlight_rules.js index 544438e6..2057e9df 100644 --- a/lib/ace/mode/_test/test_highlight_rules.js +++ b/lib/ace/mode/_test/test_highlight_rules.js @@ -8,7 +8,7 @@ require("amd-loader"); function generateTestData() { var root = Array(5).join("../") + "/demo/kitchen-sink/docs" var docs = fs.readdirSync(root) - var specialDocs = fs.readdirSync(".") + var specialDocs = fs.readdirSync(".") var modes = fs.readdirSync("../").filter(function(x){ return /(\w+)_highlight_rules.js$/.test(x) }).map(function(x) { @@ -31,12 +31,12 @@ function generateTestData() { if (!modeName) { return console.error(p) } - - var filePath = "text_" + modeName + ".txt" - if (specialDocs.indexOf(filePath) == -1) { - filePath = root + "/" + docName - } - + + var filePath = "text_" + modeName + ".txt" + if (specialDocs.indexOf(filePath) == -1) { + filePath = root + "/" + docName + } + var text = fs.readFileSync(filePath, "utf8") var Mode = require("../" + modeName).Mode; var tokenizer = new Mode().getTokenizer(); @@ -57,14 +57,16 @@ function generateTestData() { }) } -function test() { +function test(startAt) { var docs = fs.readdirSync(".").filter(function(x){ return /^tokens_\w+.json/.test(x) }); + if (startAt && startAt > 1) + docs = docs.slice(startAt - 1) docs.forEach(function(x, i){ var modeName = x.match(/tokens_(.*).json/)[1] - console.log((" " + (i+1)).slice(-3) + ") testing: \u001b[33m" + modeName + "\u001b[0m") + console.log(padNumber(i+1, 3) + ") testing: \u001b[33m" + modeName + "\u001b[0m") var text = fs.readFileSync("./" + x, "utf8") var data = JSON.parse(text) @@ -79,9 +81,12 @@ function test() { var values = tokens.tokens.map(function(x) {return x.value}) var types = tokens.tokens.map(function(x) {return x.type}) - testEqual(lineData.state, tokens.state) - testEqual(lineData.types, types) - testEqual(lineData.values, values) + var success = true + var report = + testEqual([ + lineData.state, tokens.state, + lineData.types, types, + lineData.values, values]) state = lineData.state }) @@ -89,14 +94,45 @@ function test() { console.log("\u001b[32m" + "all ok" + "\u001b[0m") - function testEqual(a, b) { - if (a + "" !== b + "") - throw a + "!=" + b + function testEqual(a) { + var err + if (a[0] !== a[1]) { + console.log(a[0],a[1]) + err = 1 + } + + if ( a[2] + "" !== a[3] + "" + || a[4] + "" !== a[5] + "" + ) { + arrayDiff(a[2],a[3]) + arrayDiff(a[4],a[5]) + err = 1 + } + if (err) { + throw "error" + } + } + + function arrayDiff(a1, a2) { + var l = Math.max(a1.length, a2.length); + var out = [] + for (var i = 0; i < l; i++) { + out.push("\n", padNumber(i+1, 3), ") ") + if (a1[i] !== a2[i]) { + out.push("\u001b[31m", a1[i], " != ", a2[i], "\u001b[0m") + } else { + out.push(a1[i]) + } + } + console.log(out.join("")) + } + function padNumber(num, digits) { + return (" " + num).slice(-digits) } } if (process.argv[2] == "gen") { generateTestData(process.argv.splice(3)) } else { - test() + test(parseInt(process.argv[2],10) || 0) } diff --git a/lib/ace/mode/javascript_highlight_rules.js b/lib/ace/mode/javascript_highlight_rules.js index c45186c5..93994a44 100644 --- a/lib/ace/mode/javascript_highlight_rules.js +++ b/lib/ace/mode/javascript_highlight_rules.js @@ -40,58 +40,43 @@ define(function(require, exports, module) { "use strict"; var oop = require("../lib/oop"); -var lang = require("../lib/lang"); var unicode = require("../unicode"); var DocCommentHighlightRules = require("./doc_comment_highlight_rules").DocCommentHighlightRules; var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules; var JavaScriptHighlightRules = function() { - // see: https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects - var globals = lang.arrayToMap( - // Constructors - ("Array|Boolean|Date|Function|Iterator|Number|Object|RegExp|String|Proxy|" + - // E4X - "Namespace|QName|XML|XMLList|" + - "ArrayBuffer|Float32Array|Float64Array|Int16Array|Int32Array|Int8Array|" + - "Uint16Array|Uint32Array|Uint8Array|Uint8ClampedArray|" + - // Errors - "Error|EvalError|InternalError|RangeError|ReferenceError|StopIteration|" + - "SyntaxError|TypeError|URIError|" + - // Non-constructor functions - "decodeURI|decodeURIComponent|encodeURI|encodeURIComponent|eval|isFinite|" + - "isNaN|parseFloat|parseInt|" + - // Other - "JSON|Math|" + - // Pseudo - "this|arguments|prototype|window|document" - ).split("|") - ); + var keywordMapper = this.createKeywordMapper({ + "variable.language": + "Array|Boolean|Date|Function|Iterator|Number|Object|RegExp|String|Proxy|" + // Constructors + "Namespace|QName|XML|XMLList|" + // E4X + "ArrayBuffer|Float32Array|Float64Array|Int16Array|Int32Array|Int8Array|" + + "Uint16Array|Uint32Array|Uint8Array|Uint8ClampedArray|" + + "Error|EvalError|InternalError|RangeError|ReferenceError|StopIteration|" + // Errors + "SyntaxError|TypeError|URIError|" + + "decodeURI|decodeURIComponent|encodeURI|encodeURIComponent|eval|isFinite|" + // Non-constructor functions + "isNaN|parseFloat|parseInt|" + + "JSON|Math|" + // Other + "this|arguments|prototype|window|document" , // Pseudo + "invalid.deprecated": + "__parent__|__count__|escape|unescape|with|__proto__|debugger", + "keyword": + "const|yield|import|get|set" + + "break|case|catch|continue|default|delete|do|else|finally|for|function|" + + "if|in|instanceof|new|return|switch|throw|try|typeof|let|var|while|with|", + "storage.type": + "const|let|var|function", + "invalid.illegal": + "class|enum|extends|super|export|implements|private|" + + "public|interface|package|protected|static", + "constant.language": + "null|Infinity|NaN|undefined", + }, "identifier"); - var keywords = lang.arrayToMap( - ("break|case|catch|continue|default|delete|do|else|finally|for|function|" + - "if|in|instanceof|new|return|switch|throw|try|typeof|let|var|while|with|" + - "const|yield|import|get|set").split("|") - ); // keywords which can be followed by regular expressions var kwBeforeRe = "case|do|else|finally|in|instanceof|return|throw|try|typeof|yield"; - var deprecated = lang.arrayToMap( - ("__parent__|__count__|escape|unescape|with|__proto__").split("|") - ); - - var definitions = lang.arrayToMap(("const|let|var|function").split("|")); - - var buildinConstants = lang.arrayToMap( - ("null|Infinity|NaN|undefined").split("|") - ); - - var futureReserved = lang.arrayToMap( - ("class|enum|extends|super|export|implements|private|" + - "public|interface|package|protected|static").split("|") - ); - // TODO: Unicode escape sequences var identifierRe = "[a-zA-Z\\$_\u00a1-\uffff][a-zA-Z\d\\$_\u00a1-\uffff]*\\b"; @@ -132,106 +117,58 @@ var JavaScriptHighlightRules = function() { }, { token : "constant.numeric", // float regex : /[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?\b/ - }, { // match stuff like: Sound.prototype.play = function() { } + }, { + // Sound.prototype.play = token : [ - "storage.type", - "punctuation.operator", - "support.function", - "punctuation.operator", - "entity.name.function", - "text", - "keyword.operator", - "text", - "storage.type", - "text", - "paren.lparen" + "storage.type", "punctuation.operator", "support.function", + "punctuation.operator", "entity.name.function", "text","keyword.operator" ], - regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", + regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)", next: "function_arguments" - }, { // match stuff like: Sound.prototype.play = myfunc + }, { + // Sound.play = function() { } token : [ - "storage.type", - "punctuation.operator", - "support.function", - "punctuation.operator", - "entity.name.function", - "text", - "keyword.operator", - "text" - ], - regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)", - next: "function_arguments" - }, { // match stuff like: Sound.play = function() { } - token : [ - "storage.type", - "punctuation.operator", - "entity.name.function", - "text", - "keyword.operator", - "text", - "storage.type", - "text", - "paren.lparen" + "storage.type", "punctuation.operator", "entity.name.function", "text", + "keyword.operator", "text", "storage.type", "text", "paren.lparen" ], regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", next: "function_arguments" - }, { // match stuff like: play = function() { } + }, { + // play = function() { } token : [ - "entity.name.function", - "text", - "keyword.operator", - "text", - "storage.type", - "text", - "paren.lparen" + "entity.name.function", "text", "keyword.operator", "text", "storage.type", + "text", "paren.lparen" ], regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", next: "function_arguments" - }, { // match stuff like: Sound.play = function play() { } + }, { + // Sound.play = function play() { } token : [ - "storage.type", - "punctuation.operator", - "entity.name.function", - "text", - "keyword.operator", - "text", - "storage.type", - "text", - "entity.name.function", - "text", - "paren.lparen" + "storage.type", "punctuation.operator", "entity.name.function", "text", + "keyword.operator", "text", + "storage.type", "text", "entity.name.function", "text", "paren.lparen" ], regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s+)(\\w+)(\\s*)(\\()", next: "function_arguments" - }, { // match regular function like: function myFunc(arg) { } + }, { + // function myFunc(arg) { } token : [ - "storage.type", - "text", - "entity.name.function", - "text", - "paren.lparen" + "storage.type", "text", "entity.name.function", "text", "paren.lparen" ], regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()", next: "function_arguments" - }, { // match stuff like: foobar: function() { } + }, { + // foobar: function() { } token : [ - "entity.name.function", - "text", - "punctuation.operator", - "text", - "storage.type", - "text", - "paren.lparen" + "entity.name.function", "text", "punctuation.operator", + "text", "storage.type", "text", "paren.lparen" ], regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()", next: "function_arguments" - }, { // Attempt to match : function() { } (this is for issues with 'foo': function() { }) + }, { + // : function() { } (this is for issues with 'foo': function() { }) token : [ - "text", - "text", - "storage.type", - "text", - "paren.lparen" + "text", "text", "storage.type", "text", "paren.lparen" ], regex : "(:)(\\s*)(function)(\\s*)(\\()", next: "function_arguments" @@ -255,24 +192,7 @@ var JavaScriptHighlightRules = function() { token : ["storage.type", "punctuation.operator", "support.function.firebug"], regex : /(console)(\.)(warn|info|log|error|time|timeEnd|assert)\b/ }, { - token : function(value) { - if (globals.hasOwnProperty(value)) - return "variable.language"; - else if (deprecated.hasOwnProperty(value)) - return "invalid.deprecated"; - else if (definitions.hasOwnProperty(value)) - return "storage.type"; - else if (keywords.hasOwnProperty(value)) - return "keyword"; - else if (buildinConstants.hasOwnProperty(value)) - return "constant.language"; - else if (futureReserved.hasOwnProperty(value)) - return "invalid.illegal"; - else if (value == "debugger") - return "invalid.deprecated"; - else - return "identifier"; - }, + token : keywordMapper, regex : identifierRe }, { token : "keyword.operator", diff --git a/lib/ace/mode/text_highlight_rules.js b/lib/ace/mode/text_highlight_rules.js index 4af6cb20..fb656fc8 100644 --- a/lib/ace/mode/text_highlight_rules.js +++ b/lib/ace/mode/text_highlight_rules.js @@ -74,7 +74,7 @@ var TextHighlightRules = function() { this.getRules = function() { return this.$rules; }; - + this.embedRules = function (HighlightRules, prefix, escapeRules, states) { var embedRules = new HighlightRules().getRules(); if (states) { @@ -88,21 +88,34 @@ var TextHighlightRules = function() { } } this.addRules(embedRules, prefix); - + for (var i = 0; i < states.length; i++) { Array.prototype.unshift.apply(this.$rules[states[i]], lang.deepCopy(escapeRules)); } - + if (!this.$embeds) { this.$embeds = []; } this.$embeds.push(prefix); } - + this.getEmbeds = function() { return this.$embeds; } + this.createKeywordMapper = function(map, defaultToken, ignoreCase, splitChar) { + var keywords = Object.create(null); + Object.keys(map).forEach(function(className) { + var list = map[className].split(splitChar || "|"); + for (var i = list.length; i--; ) + keywords[list[i]] = className; + }); + map = null; + return ignoreCase + ? function(value) {return keywords[value.toLowerCase()] || defaultToken } + : function(value) {return keywords[value] || defaultToken }; + } + }).call(TextHighlightRules.prototype); exports.TextHighlightRules = TextHighlightRules; diff --git a/lib/ace/mode/textile_highlight_rules.js b/lib/ace/mode/textile_highlight_rules.js index 87aa86fe..f44650cf 100644 --- a/lib/ace/mode/textile_highlight_rules.js +++ b/lib/ace/mode/textile_highlight_rules.js @@ -46,7 +46,7 @@ var TextileHighlightRules = function() { "start" : [ { token : function(value) { - if (value.match(/^h\d$/)) + if (value.charAt(0) == "h") return "markup.heading." + value.charAt(1); else return "markup.heading";