remove flag parameter from tokenizer

lib/ace/mode/abap.js

@@ -39,7 +39,7 @@ var TextMode = require("./text").Mode;
 var oop = require("../lib/oop");
 
 function Mode() {
-    this.$tokenizer = new Tokenizer(new Rules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new Rules().getRules());
     this.foldingRules = new FoldMode();
 }
 

lib/ace/mode/abap_highlight_rules.js

@@ -113,17 +113,18 @@ var AbapHighlightRules = function() {
             {token : "variable.parameter", regex : /sy|pa?\d\d\d\d\|t\d\d\d\.|innnn/}, 
             {token : "keyword", regex : compoundKeywords}, 
             {token : "variable.parameter", regex : /\w+-\w+(?:-\w+)*/}, 
-            {token : keywordMapper, regex : "\\b\\w+\\b"}, 
+            {token : keywordMapper, regex : "\\b\\w+\\b"},
+            {caseInsensitive: true}
         ],
         "qstring" : [
             {token : "constant.language.escape",   regex : "''"},
             {token : "string", regex : "'",     next  : "start"},
-            {token : "string", regex : ".|\w+"}
+            {defaultToken : "string"}
         ],
         "string" : [
             {token : "constant.language.escape",   regex : "``"},
             {token : "string", regex : "`",     next  : "start"},
-            {token : "string", regex : ".|\w+"}
+            {defaultToken : "string"}
         ]
     }
 };

lib/ace/mode/c9search.js

@@ -39,7 +39,7 @@ var MatchingBraceOutdent = require("./matching_brace_outdent").MatchingBraceOutd
 var C9StyleFoldMode = require("./folding/c9search").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new C9SearchHighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new C9SearchHighlightRules().getRules());
     this.$outdent = new MatchingBraceOutdent();
     this.foldingRules = new C9StyleFoldMode();
 };

lib/ace/mode/css.js

@@ -41,7 +41,7 @@ var CssBehaviour = require("./behaviour/css").CssBehaviour;
 var CStyleFoldMode = require("./folding/cstyle").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new CssHighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new CssHighlightRules().getRules());
     this.$outdent = new MatchingBraceOutdent();
     this.$behaviour = new CssBehaviour();
     this.foldingRules = new CStyleFoldMode();

lib/ace/mode/css_highlight_rules.js

@@ -92,6 +92,8 @@ var CssHighlightRules = function() {
         }, {
             token : keywordMapper,
             regex : "\\-?[a-zA-Z_][a-zA-Z0-9_\\-]*"
+        }, {
+            caseInsensitive: true
         }
       ];
 
@@ -160,6 +162,8 @@ var CssHighlightRules = function() {
         },{
             token: "constant",
             regex: "[a-z0-9-_]+"
+        },{
+            caseInsensitive: true
         }],
 
         "media" : [ {
@@ -186,6 +190,8 @@ var CssHighlightRules = function() {
         },{
             token: "constant",
             regex: "[a-z0-9-_]+"
+        },{
+            caseInsensitive: true
         }],
 
         "comment" : comment,

lib/ace/mode/diff.js

@@ -38,8 +38,8 @@ var HighlightRules = require("./diff_highlight_rules").DiffHighlightRules;
 var FoldMode = require("./folding/diff").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new HighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new HighlightRules().getRules());
-    this.foldingRules = new FoldMode(["diff", "index", "\\+{3}", "@@|\\*{5}"], "i");
+    this.foldingRules = new FoldMode(["diff", "index", "\\+{3}", "@@|\\*{5}"], "i");
 };
 oop.inherits(Mode, TextMode);
 

lib/ace/mode/diff_highlight_rules.js

@@ -92,7 +92,8 @@ var DiffHighlightRules = function() {
                 regex: "\\s*$",
                 token: "invalid"
             }, {
-                defaultToken: "invisible"
+                defaultToken: "invisible",
+                caseInsensitive: true
             }
         ]
     };

lib/ace/mode/less.js

@@ -40,7 +40,7 @@ var CssBehaviour = require("./behaviour/css").CssBehaviour;
 var CStyleFoldMode = require("./folding/cstyle").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new LessHighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new LessHighlightRules().getRules());
     this.$outdent = new MatchingBraceOutdent();
     this.$behaviour = new CssBehaviour();
     this.foldingRules = new CStyleFoldMode();

lib/ace/mode/less_highlight_rules.js

@@ -247,6 +247,8 @@ var LessHighlightRules = function() {
             }, {
                 token : "text",
                 regex : "\\s+"
+            }, {
+                caseInsensitive: true
             }
         ],
         "comment" : [

lib/ace/mode/scss.js

@@ -40,7 +40,7 @@ var CssBehaviour = require("./behaviour/css").CssBehaviour;
 var CStyleFoldMode = require("./folding/cstyle").FoldMode;
 
 var Mode = function() {
-    this.$tokenizer = new Tokenizer(new ScssHighlightRules().getRules(), "i");
+    this.$tokenizer = new Tokenizer(new ScssHighlightRules().getRules());
     this.$outdent = new MatchingBraceOutdent();
     this.$behaviour = new CssBehaviour();
     this.foldingRules = new CStyleFoldMode();

lib/ace/mode/scss_highlight_rules.js

@@ -249,6 +249,8 @@ var ScssHighlightRules = function() {
             }, {
                 token : "text",
                 regex : "\\s+"
+            }, {
+                caseInsensitive: true
             }
         ],
         "comment" : [

lib/ace/tokenizer.js

@@ -43,15 +43,10 @@ var MAX_TOKEN_COUNT = 1000;
 /**
  * Constructs a new tokenizer based on the given rules and flags.
  * @param {Object} rules The highlighting rules
- * @param {String} flag Any additional regular expression flags to pass (like "i" for case insensitive)
- *
- *
- *
  *
  * @constructor
  **/
-var Tokenizer = function(rules, flag) {
+var Tokenizer = function(rules) {
-    flag = flag ? "g" + flag : "g";
     this.states = rules;
 
     this.regExps = {};
@@ -61,13 +56,17 @@ var Tokenizer = function(rules, flag) {
         var ruleRegExps = [];
         var matchTotal = 0;
         var mapping = this.matchMappings[key] = {defaultToken: "text"};
+        var flag = "g";
 
         for (var i = 0; i < state.length; i++) {
             var rule = state[i];
-            if (rule.defaultToken) {
+            if (rule.defaultToken)
                 mapping.defaultToken = rule.defaultToken;
+            if (rule.caseInsensitive)
+                flag = "gi";
+            if (rule.regex == null)
                 continue;
-            }
+            
             if (rule.regex instanceof RegExp)
                 rule.regex = rule.regex.toString().slice(1, -1);