From e8a5c0dc2fcac27d69da5b0fe81b0a9c8efc3f9a Mon Sep 17 00:00:00 2001 From: nightwing Date: Sun, 15 Apr 2012 14:59:10 +0400 Subject: [PATCH 1/3] fix js_highlight_rules --- lib/ace/mode/javascript_highlight_rules.js | 103 +++++++++++------- .../mode/javascript_highlight_rules_test.js | 58 +++++++--- lib/ace/mode/text_highlight_rules.js | 2 - lib/ace/tokenizer.js | 5 +- 4 files changed, 107 insertions(+), 61 deletions(-) diff --git a/lib/ace/mode/javascript_highlight_rules.js b/lib/ace/mode/javascript_highlight_rules.js index 68919458..a375d054 100644 --- a/lib/ace/mode/javascript_highlight_rules.js +++ b/lib/ace/mode/javascript_highlight_rules.js @@ -124,11 +124,11 @@ var JavaScriptHighlightRules = function() { next : "comment" }, { token : "string", - regex : "'", + regex : "'(?=.)", next : "qstring" }, { token : "string", - regex : '"', + regex : '"(?=.)', next : "qqstring" }, { token : "constant.numeric", // hex @@ -148,11 +148,10 @@ var JavaScriptHighlightRules = function() { "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" + regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { // match stuff like: Sound.prototype.play = myfunc token : [ "storage.type", @@ -164,7 +163,8 @@ var JavaScriptHighlightRules = function() { "keyword.operator", "text" ], - regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)" + regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)", + next: "function_arguments" }, { // match stuff like: Sound.play = function() { } token : [ "storage.type", @@ -175,11 +175,10 @@ var JavaScriptHighlightRules = function() { "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" + regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { // match stuff like: play = function() { } token : [ "entity.name.function", @@ -188,22 +187,20 @@ var JavaScriptHighlightRules = function() { "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" + regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { // match regular function like: function myFunc(arg) { } token : [ "storage.type", "text", "entity.name.function", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()(.*?)(\\))" + regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()", + next: "function_arguments" }, { // match stuff like: foobar: function() { } token : [ "entity.name.function", @@ -212,22 +209,20 @@ var JavaScriptHighlightRules = function() { "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" + regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { // Attempt to match : function() { } (this is for issues with 'foo': function() { }) token : [ "text", "text", "storage.type", "text", - "paren.lparen", - "variable.parameter", - "paren.rparen" + "paren.lparen" ], - regex : "(:)(\\s*)(function)?(\\s*)(\\()([^)]*)(\\))" + regex : "(:)(\\s*)(function)(\\s*)(\\()", + next: "function_arguments" }, { token : "constant.language.boolean", regex : /(?:true|false)\b/ @@ -324,10 +319,9 @@ var JavaScriptHighlightRules = function() { "regex": [ { token: "regexp.keyword.operator", - regex: "\\\\(?:u[\\da-fA-F]{4}|x[\\da-fA-F]{2}|.)", - next: "regex" + regex: "\\\\(?:u[\\da-fA-F]{4}|x[\\da-fA-F]{2}|.)" }, { - // flag + // flag token: "string.regexp", regex: "/\\w*", next: "start", @@ -335,7 +329,6 @@ var JavaScriptHighlightRules = function() { }, { token: "string.regexp", regex: "[^\\\\/\\[]+", - next: "regex", merge: true }, { token: "string.regexp.charachterclass", @@ -351,8 +344,7 @@ var JavaScriptHighlightRules = function() { "regex_character_class": [ { token: "regexp.keyword.operator", - regex: "\\\\(?:u[\\da-fA-F]{4}|x[\\da-fA-F]{2}|.)", - next: "regex_character_class" + regex: "\\\\(?:u[\\da-fA-F]{4}|x[\\da-fA-F]{2}|.)" }, { token: "string.regexp.charachterclass", regex: "]", @@ -361,7 +353,24 @@ var JavaScriptHighlightRules = function() { }, { token: "string.regexp.charachterclass", regex: "[^\\\\\\]]+", - next: "regex_character_class", + merge: true + }, { + token: "empty", + regex: "", + next: "start" + } + ], + "function_arguments": [ + { + token: "variable.parameter", + regex: identifierRe, + }, { + token: "punctuation.operator", + regex: "[, ]+", + merge: true + }, { + token: "punctuation.operator", + regex: "$", merge: true }, { token: "empty", @@ -399,11 +408,18 @@ var JavaScriptHighlightRules = function() { regex : escapedRe }, { token : "string", - regex : '[^"\\\\]+' + regex : '[^"\\\\]+', + merge : true }, { token : "string", - regex : '"', - next : "start" + regex : "\\\\$", + next : "qqstring", + merge : true + }, { + token : "string", + regex : '"|$', + next : "start", + merge : true } ], "qstring" : [ @@ -412,11 +428,18 @@ var JavaScriptHighlightRules = function() { regex : escapedRe }, { token : "string", - regex : "[^'\\\\]+" + regex : "[^'\\\\]+", + merge : true }, { token : "string", - regex : "'", - next : "start" + regex : "\\\\$", + next : "qstring", + merge : true + }, { + token : "string", + regex : "'|$", + next : "start", + merge : true } ] }; diff --git a/lib/ace/mode/javascript_highlight_rules_test.js b/lib/ace/mode/javascript_highlight_rules_test.js index 123d376c..48fbbc15 100644 --- a/lib/ace/mode/javascript_highlight_rules_test.js +++ b/lib/ace/mode/javascript_highlight_rules_test.js @@ -46,9 +46,9 @@ var JavaScriptMode = require("./javascript").Mode; var assert = require("../test/assertions"); module.exports = { - + name: "JavaScript Tokenizer", - + setUp : function() { this.tokenizer = new JavaScriptMode().getTokenizer(); }, @@ -71,12 +71,12 @@ module.exports = { var tokens = this.tokenizer.getLineTokens(line, "start").tokens; - assert.equal(27, tokens.length); + assert.equal(23, tokens.length); assert.equal("support.function", tokens[2].type); // charCodeAt assert.equal("support.function.dom", tokens[10].type); // getElementById - assert.equal("support.function.firebug", tokens[20].type); // log + assert.equal("support.function.firebug", tokens[18].type); // log }, - + "test: tokenize doc comment" : function() { var line = "abc /** de */ fg"; @@ -135,11 +135,11 @@ module.exports = { var tokens = this.tokenizer.getLineTokens("a=/b/g", "start").tokens; assert.equal(3, tokens.length); assert.equal("string.regexp", tokens[2].type); - + var tokens = this.tokenizer.getLineTokens("a+/b/g", "start").tokens; assert.equal(3, tokens.length); assert.equal("string.regexp", tokens[2].type); - + var tokens = this.tokenizer.getLineTokens("a = 1 + /2 + 1/b", "start").tokens; assert.equal(9, tokens.length); assert.equal("string.regexp", tokens[8].type); @@ -148,17 +148,17 @@ module.exports = { assert.equal(7, tokens.length); assert.equal("string.regexp", tokens[2].type); assert.equal("string.regexp", tokens[6].type); - + var tokens = this.tokenizer.getLineTokens("case /a/.test(c)", "start").tokens; assert.equal(8, tokens.length); assert.equal("string.regexp", tokens[2].type); }, - + "test tokenize multi-line comment containing a single line comment" : function() { var tokens = this.tokenizer.getLineTokens("/* foo // bar */", "start").tokens; assert.equal(1, tokens.length); assert.equal("comment", tokens[0].type); - + var tokens = this.tokenizer.getLineTokens("/* foo // bar */", "regex_allowed").tokens; assert.equal(1, tokens.length); assert.equal("comment", tokens[0].type); @@ -168,7 +168,7 @@ module.exports = { var tokens = this.tokenizer.getLineTokens("füße", "start").tokens; assert.equal(1, tokens.length); }, - + "test // is not a regexp": function() { var tokens = this.tokenizer.getLineTokens("{ // 123", "start").tokens; assert.equal(3, tokens.length); @@ -176,19 +176,45 @@ module.exports = { assert.equal("text", tokens[1].type); assert.equal("comment", tokens[2].type); }, - + "test skipping escaped chars": function() { var line = "console.log('Meh\\nNeh');" var tokens = this.tokenizer.getLineTokens(line, "start").tokens; - assert.equal(11, tokens.length); - assert.equal("constant.language.escape", tokens[6].type); - + assert.equal(9, tokens.length); + assert.equal("constant.language.escape", tokens[5].type); + line = "console.log('\\u1232Feh');"; tokens = this.tokenizer.getLineTokens(line, "start").tokens; - assert.equal(10, tokens.length); + assert.equal(9, tokens.length); assert.equal("constant.language.escape", tokens[5].type); + }, + + "test multiline strings": function() { + var line = "console.log('Meh\\" + var data = this.tokenizer.getLineTokens(line, "start") + + assert.equal(5, data.tokens.length); + assert.equal(data.state, "qstring"); + + line = "console.log('Meh\\ " + data = this.tokenizer.getLineTokens(line, "start") + + assert.equal(6, data.tokens.length); + assert.equal(data.state, "start"); + + line = 'console.log("\\' + data = this.tokenizer.getLineTokens(line, "start") + + assert.equal(5, data.tokens.length); + assert.equal(data.state, "qqstring"); + + line = 'a="' + data = this.tokenizer.getLineTokens(line, "start") + + assert.equal(3, data.tokens.length); + assert.equal(data.state, "start"); } }; diff --git a/lib/ace/mode/text_highlight_rules.js b/lib/ace/mode/text_highlight_rules.js index 862035d6..4af6cb20 100644 --- a/lib/ace/mode/text_highlight_rules.js +++ b/lib/ace/mode/text_highlight_rules.js @@ -65,8 +65,6 @@ var TextHighlightRules = function() { var rule = state[i]; if (rule.next) { rule.next = prefix + rule.next; - } else { - rule.next = prefix + key; } } this.$rules[prefix + key] = state; diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js index 674e30ed..f24d2191 100644 --- a/lib/ace/tokenizer.js +++ b/lib/ace/tokenizer.js @@ -119,9 +119,8 @@ var Tokenizer = function(rules, flag) { else type = rule.token; - var next = rule.next; - if (next && next !== currentState) { - currentState = next; + if (rule.next) { + currentState = rule.next; state = this.rules[currentState]; mapping = this.matchMappings[currentState]; lastIndex = re.lastIndex; From 68d0799554d76a640fcab1c859034fdc3a561a96 Mon Sep 17 00:00:00 2001 From: nightwing Date: Sun, 15 Apr 2012 19:02:18 +0400 Subject: [PATCH 2/3] make getStartRule getEndRule static --- lib/ace/mode/c_cpp_highlight_rules.js | 4 +-- lib/ace/mode/doc_comment_highlight_rules.js | 35 ++++++++++----------- lib/ace/mode/groovy_highlight_rules.js | 4 +-- lib/ace/mode/haxe_highlight_rules.js | 4 +-- lib/ace/mode/java_highlight_rules.js | 4 +-- lib/ace/mode/javascript_highlight_rules.js | 4 +-- lib/ace/mode/pgsql_highlight_rules.js | 4 +-- lib/ace/mode/php_highlight_rules.js | 4 +-- lib/ace/mode/scad_highlight_rules.js | 4 +-- lib/ace/mode/scala_highlight_rules.js | 4 +-- 10 files changed, 34 insertions(+), 37 deletions(-) diff --git a/lib/ace/mode/c_cpp_highlight_rules.js b/lib/ace/mode/c_cpp_highlight_rules.js index f2192803..236b7284 100644 --- a/lib/ace/mode/c_cpp_highlight_rules.js +++ b/lib/ace/mode/c_cpp_highlight_rules.js @@ -72,7 +72,7 @@ var c_cppHighlightRules = function() { token : "comment", regex : "\\/\\/.*$" }, - new DocCommentHighlightRules().getStartRule("doc-start"), + DocCommentHighlightRules.getStartRule("doc-start"), { token : "comment", // multi line comment merge : true, @@ -171,7 +171,7 @@ var c_cppHighlightRules = function() { }; this.embedRules(DocCommentHighlightRules, "doc-", - [ new DocCommentHighlightRules().getEndRule("start") ]); + [ DocCommentHighlightRules.getEndRule("start") ]); }; oop.inherits(c_cppHighlightRules, TextHighlightRules); diff --git a/lib/ace/mode/doc_comment_highlight_rules.js b/lib/ace/mode/doc_comment_highlight_rules.js index 101cfa78..b176ef01 100644 --- a/lib/ace/mode/doc_comment_highlight_rules.js +++ b/lib/ace/mode/doc_comment_highlight_rules.js @@ -69,27 +69,24 @@ var DocCommentHighlightRules = function() { oop.inherits(DocCommentHighlightRules, TextHighlightRules); -(function() { - - this.getStartRule = function(start) { - return { - token : "comment.doc", // doc comment - merge : true, - regex : "\\/\\*(?=\\*)", - next : start - }; - }; - - this.getEndRule = function (start) { - return { - token : "comment.doc", // closing comment - merge : true, - regex : "\\*\\/", - next : start - }; +DocCommentHighlightRules.getStartRule = function(start) { + return { + token : "comment.doc", // doc comment + merge : true, + regex : "\\/\\*(?=\\*)", + next : start }; +}; + +DocCommentHighlightRules.getEndRule = function (start) { + return { + token : "comment.doc", // closing comment + merge : true, + regex : "\\*\\/", + next : start + }; +}; -}).call(DocCommentHighlightRules.prototype); exports.DocCommentHighlightRules = DocCommentHighlightRules; diff --git a/lib/ace/mode/groovy_highlight_rules.js b/lib/ace/mode/groovy_highlight_rules.js index 6e3802a2..64ed0ab1 100644 --- a/lib/ace/mode/groovy_highlight_rules.js +++ b/lib/ace/mode/groovy_highlight_rules.js @@ -64,7 +64,7 @@ var GroovyHighlightRules = function() { token : "comment", regex : "\\/\\/.*$" }, - new DocCommentHighlightRules().getStartRule("doc-start"), + DocCommentHighlightRules.getStartRule("doc-start"), { token : "comment", // multi line comment merge : true, @@ -134,7 +134,7 @@ var GroovyHighlightRules = function() { }; this.embedRules(DocCommentHighlightRules, "doc-", - [ new DocCommentHighlightRules().getEndRule("start") ]); + [ DocCommentHighlightRules.getEndRule("start") ]); }; oop.inherits(GroovyHighlightRules, TextHighlightRules); diff --git a/lib/ace/mode/haxe_highlight_rules.js b/lib/ace/mode/haxe_highlight_rules.js index afd76659..5b17c650 100644 --- a/lib/ace/mode/haxe_highlight_rules.js +++ b/lib/ace/mode/haxe_highlight_rules.js @@ -26,7 +26,7 @@ var HaxeHighlightRules = function() { token : "comment", regex : "\\/\\/.*$" }, - new DocCommentHighlightRules().getStartRule("doc-start"), + DocCommentHighlightRules.getStartRule("doc-start"), { token : "comment", // multi line comment regex : "\\/\\*", @@ -95,7 +95,7 @@ var HaxeHighlightRules = function() { }; this.embedRules(DocCommentHighlightRules, "doc-", - [ new DocCommentHighlightRules().getEndRule("start") ]); + [ DocCommentHighlightRules.getEndRule("start") ]); }; oop.inherits(HaxeHighlightRules, TextHighlightRules); diff --git a/lib/ace/mode/java_highlight_rules.js b/lib/ace/mode/java_highlight_rules.js index 34aab3bd..77efd672 100644 --- a/lib/ace/mode/java_highlight_rules.js +++ b/lib/ace/mode/java_highlight_rules.js @@ -65,7 +65,7 @@ var JavaHighlightRules = function() { token : "comment", regex : "\\/\\/.*$" }, - new DocCommentHighlightRules().getStartRule("doc-start"), + DocCommentHighlightRules.getStartRule("doc-start"), { token : "comment", // multi line comment merge : true, @@ -135,7 +135,7 @@ var JavaHighlightRules = function() { }; this.embedRules(DocCommentHighlightRules, "doc-", - [ new DocCommentHighlightRules().getEndRule("start") ]); + [ DocCommentHighlightRules.getEndRule("start") ]); }; oop.inherits(JavaHighlightRules, TextHighlightRules); diff --git a/lib/ace/mode/javascript_highlight_rules.js b/lib/ace/mode/javascript_highlight_rules.js index a375d054..416dc3c4 100644 --- a/lib/ace/mode/javascript_highlight_rules.js +++ b/lib/ace/mode/javascript_highlight_rules.js @@ -116,7 +116,7 @@ var JavaScriptHighlightRules = function() { token : "comment", regex : /\/\/.*$/ }, - new DocCommentHighlightRules().getStartRule("doc-start"), + DocCommentHighlightRules.getStartRule("doc-start"), { token : "comment", // multi line comment merge : true, @@ -445,7 +445,7 @@ var JavaScriptHighlightRules = function() { }; this.embedRules(DocCommentHighlightRules, "doc-", - [ new DocCommentHighlightRules().getEndRule("start") ]); + [ DocCommentHighlightRules.getEndRule("start") ]); }; oop.inherits(JavaScriptHighlightRules, TextHighlightRules); diff --git a/lib/ace/mode/pgsql_highlight_rules.js b/lib/ace/mode/pgsql_highlight_rules.js index c5b6849f..d15c94da 100755 --- a/lib/ace/mode/pgsql_highlight_rules.js +++ b/lib/ace/mode/pgsql_highlight_rules.js @@ -442,7 +442,7 @@ var PgsqlHighlightRules = function() { token : "comment", regex : "--.*$" }, - new DocCommentHighlightRules().getStartRule("doc-start"), + DocCommentHighlightRules.getStartRule("doc-start"), { token : "comment", // multi-line comment merge : true, @@ -571,7 +571,7 @@ var PgsqlHighlightRules = function() { ] }; - this.embedRules(DocCommentHighlightRules, "doc-", [ new DocCommentHighlightRules().getEndRule("start") ]); + this.embedRules(DocCommentHighlightRules, "doc-", [ DocCommentHighlightRules.getEndRule("start") ]); this.embedRules(PerlHighlightRules, "perl-", [{token : "string", regex : "\\$perl\\$", next : "statement"}]); this.embedRules(PythonHighlightRules, "python-", [{token : "string", regex : "\\$python\\$", next : "statement"}]); }; diff --git a/lib/ace/mode/php_highlight_rules.js b/lib/ace/mode/php_highlight_rules.js index d2a76f5c..e25dddc8 100644 --- a/lib/ace/mode/php_highlight_rules.js +++ b/lib/ace/mode/php_highlight_rules.js @@ -45,7 +45,7 @@ var DocCommentHighlightRules = require("./doc_comment_highlight_rules").DocComme var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules; var PhpHighlightRules = function() { - var docComment = new DocCommentHighlightRules(); + var docComment = DocCommentHighlightRules; // http://php.net/quickref.php var builtinFunctions = lang.arrayToMap( ('abs|acos|acosh|addcslashes|addslashes|aggregate|aggregate_info|aggregate_methods|aggregate_methods_by_list|aggregate_methods_by_regexp|' + @@ -1150,7 +1150,7 @@ var PhpHighlightRules = function() { }; this.embedRules(DocCommentHighlightRules, "doc-", - [ new DocCommentHighlightRules().getEndRule("start") ]); + [ DocCommentHighlightRules.getEndRule("start") ]); }; oop.inherits(PhpHighlightRules, TextHighlightRules); diff --git a/lib/ace/mode/scad_highlight_rules.js b/lib/ace/mode/scad_highlight_rules.js index 79995250..82234750 100644 --- a/lib/ace/mode/scad_highlight_rules.js +++ b/lib/ace/mode/scad_highlight_rules.js @@ -65,7 +65,7 @@ var scadHighlightRules = function() { token : "comment", regex : "\\/\\/.*$" }, - new DocCommentHighlightRules().getStartRule("start"), + DocCommentHighlightRules.getStartRule("start"), { token : "comment", // multi line comment merge : true, @@ -159,7 +159,7 @@ var scadHighlightRules = function() { }; this.embedRules(DocCommentHighlightRules, "doc-", - [ new DocCommentHighlightRules().getEndRule("start") ]); + [ DocCommentHighlightRules.getEndRule("start") ]); }; oop.inherits(scadHighlightRules, TextHighlightRules); diff --git a/lib/ace/mode/scala_highlight_rules.js b/lib/ace/mode/scala_highlight_rules.js index 3664d67e..67f8e1b2 100644 --- a/lib/ace/mode/scala_highlight_rules.js +++ b/lib/ace/mode/scala_highlight_rules.js @@ -65,7 +65,7 @@ var ScalaHighlightRules = function() { token : "comment", regex : "\\/\\/.*$" }, - new DocCommentHighlightRules().getStartRule("doc-start"), + DocCommentHighlightRules.getStartRule("doc-start"), { token : "comment", // multi line comment merge : true, @@ -135,7 +135,7 @@ var ScalaHighlightRules = function() { }; this.embedRules(DocCommentHighlightRules, "doc-", - [ new DocCommentHighlightRules().getEndRule("start") ]); + [ DocCommentHighlightRules.getEndRule("start") ]); }; oop.inherits(ScalaHighlightRules, TextHighlightRules); From 0e21a9354ec15ac38d09168b095fc08684616aa3 Mon Sep 17 00:00:00 2001 From: nightwing Date: Sun, 15 Apr 2012 19:03:08 +0400 Subject: [PATCH 3/3] fix Block-comments after `function(){` --- lib/ace/mode/javascript_highlight_rules.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ace/mode/javascript_highlight_rules.js b/lib/ace/mode/javascript_highlight_rules.js index 416dc3c4..18100423 100644 --- a/lib/ace/mode/javascript_highlight_rules.js +++ b/lib/ace/mode/javascript_highlight_rules.js @@ -292,6 +292,7 @@ var JavaScriptHighlightRules = function() { // regular expressions are only allowed after certain tokens. This // makes sure we don't mix up regexps with the divison operator "regex_allowed": [ + DocCommentHighlightRules.getStartRule("doc-start"), { token : "comment", // multi line comment merge : true,