From e8a5c0dc2fcac27d69da5b0fe81b0a9c8efc3f9a Mon Sep 17 00:00:00 2001 From: nightwing Date: Sun, 15 Apr 2012 14:59:10 +0400 Subject: [PATCH] fix js_highlight_rules --- lib/ace/mode/javascript_highlight_rules.js | 103 +++++++++++------- .../mode/javascript_highlight_rules_test.js | 58 +++++++--- lib/ace/mode/text_highlight_rules.js | 2 - lib/ace/tokenizer.js | 5 +- 4 files changed, 107 insertions(+), 61 deletions(-) diff --git a/lib/ace/mode/javascript_highlight_rules.js b/lib/ace/mode/javascript_highlight_rules.js index 68919458..a375d054 100644 --- a/lib/ace/mode/javascript_highlight_rules.js +++ b/lib/ace/mode/javascript_highlight_rules.js @@ -124,11 +124,11 @@ var JavaScriptHighlightRules = function() { next : "comment" }, { token : "string", - regex : "'", + regex : "'(?=.)", next : "qstring" }, { token : "string", - regex : '"', + regex : '"(?=.)', next : "qqstring" }, { token : "constant.numeric", // hex @@ -148,11 +148,10 @@ var JavaScriptHighlightRules = function() { "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" + regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { // match stuff like: Sound.prototype.play = myfunc token : [ "storage.type", @@ -164,7 +163,8 @@ var JavaScriptHighlightRules = function() { "keyword.operator", "text" ], - regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)" + regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)", + next: "function_arguments" }, { // match stuff like: Sound.play = function() { } token : [ "storage.type", @@ -175,11 +175,10 @@ var JavaScriptHighlightRules = function() { "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" + regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { // match stuff like: play = function() { } token : [ "entity.name.function", @@ -188,22 +187,20 @@ var JavaScriptHighlightRules = function() { "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" + regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { // match regular function like: function myFunc(arg) { } token : [ "storage.type", "text", "entity.name.function", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()(.*?)(\\))" + regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()", + next: "function_arguments" }, { // match stuff like: foobar: function() { } token : [ "entity.name.function", @@ -212,22 +209,20 @@ var JavaScriptHighlightRules = function() { "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" + regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { // Attempt to match : function() { } (this is for issues with 'foo': function() { }) token : [ "text", "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(:)(\\s*)(function)?(\\s*)(\\()([^)]*)(\\))" + regex : "(:)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { token : "constant.language.boolean", regex : /(?:true|false)\b/ @@ -324,10 +319,9 @@ var JavaScriptHighlightRules = function() { "regex": [ { token: "regexp.keyword.operator", - regex: "\\\\(?:u[\\da-fA-F]{4}|x[\\da-fA-F]{2}|.)", - next: "regex" + regex: "\\\\(?:u[\\da-fA-F]{4}|x[\\da-fA-F]{2}|.)" }, { - // flag + // flag token: "string.regexp", regex: "/\\w*", next: "start", @@ -335,7 +329,6 @@ var JavaScriptHighlightRules = function() { }, { token: "string.regexp", regex: "[^\\\\/\\[]+", - next: "regex", merge: true }, { token: "string.regexp.charachterclass", @@ -351,8 +344,7 @@ var JavaScriptHighlightRules = function() { "regex_character_class": [ { token: "regexp.keyword.operator", - regex: "\\\\(?:u[\\da-fA-F]{4}|x[\\da-fA-F]{2}|.)", - next: "regex_character_class" + regex: "\\\\(?:u[\\da-fA-F]{4}|x[\\da-fA-F]{2}|.)" }, { token: "string.regexp.charachterclass", regex: "]", @@ -361,7 +353,24 @@ var JavaScriptHighlightRules = function() { }, { token: "string.regexp.charachterclass", regex: "[^\\\\\\]]+", - next: "regex_character_class", + merge: true + }, { + token: "empty", + regex: "", + next: "start" + } + ], + "function_arguments": [ + { + token: "variable.parameter", + regex: identifierRe, + }, { + token: "punctuation.operator", + regex: "[, ]+", + merge: true + }, { + token: "punctuation.operator", + regex: "$", merge: true }, { token: "empty", @@ -399,11 +408,18 @@ var JavaScriptHighlightRules = function() { regex : escapedRe }, { token : "string", - regex : '[^"\\\\]+' + regex : '[^"\\\\]+', + merge : true }, { token : "string", - regex : '"', - next : "start" + regex : "\\\\$", + next : "qqstring", + merge : true + }, { + token : "string", + regex : '"|$', + next : "start", + merge : true } ], "qstring" : [ @@ -412,11 +428,18 @@ var JavaScriptHighlightRules = function() { regex : escapedRe }, { token : "string", - regex : "[^'\\\\]+" + regex : "[^'\\\\]+", + merge : true }, { token : "string", - regex : "'", - next : "start" + regex : "\\\\$", + next : "qstring", + merge : true + }, { + token : "string", + regex : "'|$", + next : "start", + merge : true } ] }; diff --git a/lib/ace/mode/javascript_highlight_rules_test.js b/lib/ace/mode/javascript_highlight_rules_test.js index 123d376c..48fbbc15 100644 --- a/lib/ace/mode/javascript_highlight_rules_test.js +++ b/lib/ace/mode/javascript_highlight_rules_test.js @@ -46,9 +46,9 @@ var JavaScriptMode = require("./javascript").Mode; var assert = require("../test/assertions"); module.exports = { - + name: "JavaScript Tokenizer", - + setUp : function() { this.tokenizer = new JavaScriptMode().getTokenizer(); }, @@ -71,12 +71,12 @@ module.exports = { var tokens = this.tokenizer.getLineTokens(line, "start").tokens; - assert.equal(27, tokens.length); + assert.equal(23, tokens.length); assert.equal("support.function", tokens[2].type); // charCodeAt assert.equal("support.function.dom", tokens[10].type); // getElementById - assert.equal("support.function.firebug", tokens[20].type); // log + assert.equal("support.function.firebug", tokens[18].type); // log }, - + "test: tokenize doc comment" : function() { var line = "abc /** de */ fg"; @@ -135,11 +135,11 @@ module.exports = { var tokens = this.tokenizer.getLineTokens("a=/b/g", "start").tokens; assert.equal(3, tokens.length); assert.equal("string.regexp", tokens[2].type); - + var tokens = this.tokenizer.getLineTokens("a+/b/g", "start").tokens; assert.equal(3, tokens.length); assert.equal("string.regexp", tokens[2].type); - + var tokens = this.tokenizer.getLineTokens("a = 1 + /2 + 1/b", "start").tokens; assert.equal(9, tokens.length); assert.equal("string.regexp", tokens[8].type); @@ -148,17 +148,17 @@ module.exports = { assert.equal(7, tokens.length); assert.equal("string.regexp", tokens[2].type); assert.equal("string.regexp", tokens[6].type); - + var tokens = this.tokenizer.getLineTokens("case /a/.test(c)", "start").tokens; assert.equal(8, tokens.length); assert.equal("string.regexp", tokens[2].type); }, - + "test tokenize multi-line comment containing a single line comment" : function() { var tokens = this.tokenizer.getLineTokens("/* foo // bar */", "start").tokens; assert.equal(1, tokens.length); assert.equal("comment", tokens[0].type); - + var tokens = this.tokenizer.getLineTokens("/* foo // bar */", "regex_allowed").tokens; assert.equal(1, tokens.length); assert.equal("comment", tokens[0].type); @@ -168,7 +168,7 @@ module.exports = { var tokens = this.tokenizer.getLineTokens("füße", "start").tokens; assert.equal(1, tokens.length); }, - + "test // is not a regexp": function() { var tokens = this.tokenizer.getLineTokens("{ // 123", "start").tokens; assert.equal(3, tokens.length); @@ -176,19 +176,45 @@ module.exports = { assert.equal("text", tokens[1].type); assert.equal("comment", tokens[2].type); }, - + "test skipping escaped chars": function() { var line = "console.log('Meh\\nNeh');" var tokens = this.tokenizer.getLineTokens(line, "start").tokens; - assert.equal(11, tokens.length); - assert.equal("constant.language.escape", tokens[6].type); - + assert.equal(9, tokens.length); + assert.equal("constant.language.escape", tokens[5].type); + line = "console.log('\\u1232Feh');"; tokens = this.tokenizer.getLineTokens(line, "start").tokens; - assert.equal(10, tokens.length); + assert.equal(9, tokens.length); assert.equal("constant.language.escape", tokens[5].type); + }, + + "test multiline strings": function() { + var line = "console.log('Meh\\" + var data = this.tokenizer.getLineTokens(line, "start") + + assert.equal(5, data.tokens.length); + assert.equal(data.state, "qstring"); + + line = "console.log('Meh\\ " + data = this.tokenizer.getLineTokens(line, "start") + + assert.equal(6, data.tokens.length); + assert.equal(data.state, "start"); + + line = 'console.log("\\' + data = this.tokenizer.getLineTokens(line, "start") + + assert.equal(5, data.tokens.length); + assert.equal(data.state, "qqstring"); + + line = 'a="' + data = this.tokenizer.getLineTokens(line, "start") + + assert.equal(3, data.tokens.length); + assert.equal(data.state, "start"); } }; diff --git a/lib/ace/mode/text_highlight_rules.js b/lib/ace/mode/text_highlight_rules.js index 862035d6..4af6cb20 100644 --- a/lib/ace/mode/text_highlight_rules.js +++ b/lib/ace/mode/text_highlight_rules.js @@ -65,8 +65,6 @@ var TextHighlightRules = function() { var rule = state[i]; if (rule.next) { rule.next = prefix + rule.next; - } else { - rule.next = prefix + key; } } this.$rules[prefix + key] = state; diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js index 674e30ed..f24d2191 100644 --- a/lib/ace/tokenizer.js +++ b/lib/ace/tokenizer.js @@ -119,9 +119,8 @@ var Tokenizer = function(rules, flag) { else type = rule.token; - var next = rule.next; - if (next && next !== currentState) { - currentState = next; + if (rule.next) { + currentState = rule.next; state = this.rules[currentState]; mapping = this.matchMappings[currentState]; lastIndex = re.lastIndex;