fix several highlighter issues

lib/ace/mode/_test/highlight_rules_test.js

@@ -1,20 +1,66 @@
 var fs = require("fs");
+var path = require("path");
 if (!fs.existsSync)
-    fs.existsSync = require("path").existsSync;
+    fs.existsSync = path.existsSync;
 
 require("amd-loader");
 
 var cwd = __dirname + "/";
+var root = path.normalize(cwd + Array(5).join("../"));
+
+function jsFileList(path, filter) {
+    if (!filter) filter = /_test/;
+    return fs.readdirSync(path).map(function(x) {
+        if (x.slice(-3) == ".js" && !filter.test(x) && !/\s/.test(x))
+            return x.slice(0, -3);
+    }).filter(Boolean);
+}
+
+function modeList() {
+    return jsFileList(cwd + "../", /_highlight_rules|_test|_worker|xml_util|_outdent|behaviour|completions/);
+}
+
+function checkModes() {
+    modeList().forEach(function(modeName) {
+        try {
+            var Mode = require("../" + modeName).Mode;
+        } catch(e) {
+            console.warn("Can't load mode :" + modeName, e);
+            return;
+        }
+        var m = new Mode();
+        if (!m.lineCommentStart && !m.blockComment)
+            console.warn("missing comment in " + modeName);
+        if (!m.$id)
+            console.warn("missing id in " + modeName);
+        var tokenizer = (new Mode).getTokenizer();
+        if (m.lineCommentStart) {
+            if (Array.isArray(m.lineCommentStart)) {
+                m.lineCommentStart.forEach(function(x) {
+                    testLineComment(tokenizer, x, modeName)
+                });
+            } else {
+                testLineComment(tokenizer, m.lineCommentStart, modeName)
+            }
+        }
+        // if (m.blockComment) {
+        //     var tokens = tok.getLineTokens(m.lineCommentStart, "start");
+        //     if (!/comment/.test(tokens[0]))
+        //         console.warn("broken lineCommentStart in " + modeName);
+        // }
+    });
+    
+    function testLineComment(tokenizer, commentStart, modeName) {
+        var tokens = tokenizer.getLineTokens(commentStart + " ", "start").tokens;
+        if (!/comment/.test(tokens[0].type))
+            console.warn("broken lineCommentStart in " + modeName);
+    }
+}
 
 function generateTestData() {
-    var root = Array(5).join("../") + "/demo/kitchen-sink/docs";
+    var docs = jsFileList(cwd + root);
-    var docs = fs.readdirSync(cwd + root);
     var specialDocs = fs.readdirSync(cwd);
-    var modes = fs.readdirSync(cwd + "../").filter(function(x){
+    var modes = modeList();
-        return !/(_highlight_rules|behaviour|worker)\.js$/.test(x) && /\.js$/.test(x);
-    }).map(function(x) {
-        return x.replace(/\.js$/, "");
-    });
 
     console.log("Docs:", docs);
     console.log("Modes:", modes);
@@ -61,7 +107,7 @@ function generateTestData() {
             return tmp.join(",\n  ");
         });
         
-        jsonStr = "[[\n   " + data.join("\n],[\n   ") + "\n]]";
+        var jsonStr = "[[\n   " + data.join("\n],[\n   ") + "\n]]";
         fs.writeFileSync(cwd + "tokens_" + modeName + ".json", jsonStr, "utf8");
     });
 }
@@ -91,7 +137,7 @@ function testMode(modeName, i) {
         lineData.state = lineData.shift();
         var line = null;
         if (typeof lineData[lineData.length - 1] == "string")
-            line = lineData.pop()
+            line = lineData.pop();
         lineData.forEach(function(x) {
             lineData.types.push(x[0]);
             lineData.values.push(x[1]);
@@ -103,14 +149,13 @@ function testMode(modeName, i) {
         var values = tokens.tokens.map(function(x) {return x.value;});
         var types = tokens.tokens.map(function(x) {return x.type;});
 
-        var success = true;
         var err = testEqual([
             JSON.stringify(lineData.state), JSON.stringify(tokens.state),
             lineData.types, types,
             lineData.values, values]);
         
         if (err) {
-            console.log(line)
+            console.log(line);
             throw "error";
         }
 
@@ -150,10 +195,12 @@ function padNumber(num, digits) {
 // cli
 var arg = process.argv[2];
 if (!arg)
-    test()
+    test();
 else if (/--?g(en)?/.test(arg))
     generateTestData(process.argv.splice(3));
+else if (/--?c(heck)?/.test(arg))
+    checkModes(process.argv.splice(3));
 else if (/\d+/.test(arg))
     test(parseInt(process.argv[2],10) || 0);
 else
-    testMode(arg, -1)
+    testMode(arg, -1);

lib/ace/mode/actionscript.js

@@ -27,10 +27,6 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  *
- * Contributor(s):
- *
- *
- *
  * ***** END LICENSE BLOCK ***** */
 
 /*

lib/ace/mode/autohotkey.js

@@ -26,11 +26,6 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- *
- * Contributor(s):
- *
- *
- *
  * ***** END LICENSE BLOCK ***** */
 
 /*
@@ -53,7 +48,7 @@ var Mode = function() {
 oop.inherits(Mode, TextMode);
 
 (function() {
-    this.lineCommentStart = "/\\*";
+    this.lineCommentStart = ";";
     this.blockComment = {start: "/*", end: "*/"};
     this.$id = "ace/mode/autohotkey";
 }).call(Mode.prototype);

lib/ace/mode/c_cpp_highlight_rules.js

@@ -50,11 +50,12 @@ var c_cppHighlightRules = function() {
     // regexp must not have capturing parentheses. Use (?:) instead.
     // regexps are ordered -> the first match is used
 
-    this.$rules = {
+    this.$rules = { 
         "start" : [
             {
                 token : "comment",
-                regex : "\\/\\/.*$"
+                regex : "//",
+                next : "singleLineComment"
             },
             DocCommentHighlightRules.getStartRule("doc-start"),
             {
@@ -121,14 +122,26 @@ var c_cppHighlightRules = function() {
                 regex : ".+"
             }
         ],
+        "singleLineComment" : [
+            {
+                token : "comment",
+                regex : /\\$/,
+                next : "singleLineComment"
+            }, {
+                token : "comment",
+                regex : /$/,
+                next : "start"
+            }, {
+                defaultToken: "comment"
+            }
+        ],
         "qqstring" : [
             {
                 token : "string",
                 regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
                 next : "start"
             }, {
-                token : "string",
+                defaultToken : "string"
-                regex : '.+'
             }
         ],
         "qstring" : [
@@ -137,8 +150,7 @@ var c_cppHighlightRules = function() {
                 regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
                 next : "start"
             }, {
-                token : "string",
+                defaultToken : "string"
-                regex : '.+'
             }
         ],
         "directive" : [

lib/ace/mode/coffee_highlight_rules.js

@@ -142,7 +142,7 @@ define(function(require, exports, module) {
                         }
                         if (val == "}" && stack.length) {
                             stack.shift();
-                            this.next = stack.shift();
+                            this.next = stack.shift() || "";
                             if (this.next.indexOf("string") != -1)
                                 return "paren.string";
                         }

lib/ace/mode/d.js

@@ -47,7 +47,7 @@ var Mode = function() {
 oop.inherits(Mode, TextMode);
 
 (function() {
-    this.lineCommentStart = "/\\+";
+    this.lineCommentStart = "//";
     this.blockComment = {start: "/*", end: "*/"};
     this.$id = "ace/mode/d";
 }).call(Mode.prototype);

lib/ace/mode/forth.js

@@ -27,10 +27,6 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  *
- * Contributor(s):
- *
- *
- *
  * ***** END LICENSE BLOCK ***** */
 
 /*
@@ -53,7 +49,7 @@ var Mode = function() {
 oop.inherits(Mode, TextMode);
 
 (function() {
-    this.lineCommentStart = "(?<=^|\\s)\\.?\\( [^)]*\\)";
+    this.lineCommentStart = "--";
     this.blockComment = {start: "/*", end: "*/"};
     this.$id = "ace/mode/forth";
 }).call(Mode.prototype);

lib/ace/mode/gitignore.js

@@ -12,6 +12,7 @@ var Mode = function() {
 oop.inherits(Mode, TextMode);
 
 (function() {
+    this.lineCommentStart = "#";
     this.$id = "ace/mode/gitignore";
 }).call(Mode.prototype);
 

lib/ace/mode/makefile.js

@@ -26,11 +26,6 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- *
- * Contributor(s):
- * 
- *
- *
  * ***** END LICENSE BLOCK ***** */
 
 /*

lib/ace/mode/rust.js

@@ -27,10 +27,6 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  *
- * Contributor(s):
- *
- *
- *
  * ***** END LICENSE BLOCK ***** */
 
 /*

lib/ace/mode/stylus.js

@@ -51,7 +51,10 @@ var Mode = function() {
 };
 oop.inherits(Mode, TextMode);
 
-(function() { 
+(function() {
+    this.lineCommentStart = "//";
+    this.blockComment = {start: "/*", end: "*/"};
+    
     this.$id = "ace/mode/stylus";
 }).call(Mode.prototype);
 

lib/ace/mode/textile.js

@@ -43,6 +43,7 @@ var Mode = function() {
 oop.inherits(Mode, TextMode);
 
 (function() {
+    this.type = "text";
     this.getNextLineIndent = function(state, line, tab) {
         if (state == "intag")
             return tab;