jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove flag parameter from tokenizer

Browse source
This commit is contained in:
nightwing 2013-02-07 23:13:11 +04:00
commit 382eb6ab55
12 changed files with 30 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

2
lib/ace/mode/abap.js
View file

@ -39,7 +39,7 @@ var TextMode = require("./text").Mode;
var oop = require("../lib/oop");
function Mode() {
this.$tokenizer = new Tokenizer(new Rules().getRules(), "i");
this.$tokenizer = new Tokenizer(new Rules().getRules());
this.foldingRules = new FoldMode();
}

7
lib/ace/mode/abap_highlight_rules.js
View file

@ -113,17 +113,18 @@ var AbapHighlightRules = function() {
{token : "variable.parameter", regex : /sy|pa?\d\d\d\d\|t\d\d\d\.|innnn/},
{token : "keyword", regex : compoundKeywords},
{token : "variable.parameter", regex : /\w+-\w+(?:-\w+)*/},
{token : keywordMapper, regex : "\\b\\w+\\b"},
{token : keywordMapper, regex : "\\b\\w+\\b"},
{caseInsensitive: true}
],
"qstring" : [
{token : "constant.language.escape", regex : "''"},
{token : "string", regex : "'", next : "start"},
{token : "string", regex : ".|\w+"}
{defaultToken : "string"}
],
"string" : [
{token : "constant.language.escape", regex : "``"},
{token : "string", regex : "`", next : "start"},
{token : "string", regex : ".|\w+"}
{defaultToken : "string"}
]
}
};

2
lib/ace/mode/c9search.js
View file

@ -39,7 +39,7 @@ var MatchingBraceOutdent = require("./matching_brace_outdent").MatchingBraceOutd
var C9StyleFoldMode = require("./folding/c9search").FoldMode;
var Mode = function() {
this.$tokenizer = new Tokenizer(new C9SearchHighlightRules().getRules(), "i");
this.$tokenizer = new Tokenizer(new C9SearchHighlightRules().getRules());
this.$outdent = new MatchingBraceOutdent();
this.foldingRules = new C9StyleFoldMode();
};

2
lib/ace/mode/css.js
View file

@ -41,7 +41,7 @@ var CssBehaviour = require("./behaviour/css").CssBehaviour;
var CStyleFoldMode = require("./folding/cstyle").FoldMode;
var Mode = function() {
this.$tokenizer = new Tokenizer(new CssHighlightRules().getRules(), "i");
this.$tokenizer = new Tokenizer(new CssHighlightRules().getRules());
this.$outdent = new MatchingBraceOutdent();
this.$behaviour = new CssBehaviour();
this.foldingRules = new CStyleFoldMode();

6
lib/ace/mode/css_highlight_rules.js
View file

@ -92,6 +92,8 @@ var CssHighlightRules = function() {
}, {
token : keywordMapper,
regex : "\\-?[a-zA-Z_][a-zA-Z0-9_\\-]*"
}, {
caseInsensitive: true
}
];
@ -160,6 +162,8 @@ var CssHighlightRules = function() {
},{
token: "constant",
regex: "[a-z0-9-_]+"
},{
caseInsensitive: true
}],
"media" : [ {
@ -186,6 +190,8 @@ var CssHighlightRules = function() {
},{
token: "constant",
regex: "[a-z0-9-_]+"
},{
caseInsensitive: true
}],
"comment" : comment,

4
lib/ace/mode/diff.js
View file

@ -38,8 +38,8 @@ var HighlightRules = require("./diff_highlight_rules").DiffHighlightRules;
var FoldMode = require("./folding/diff").FoldMode;
var Mode = function() {
this.$tokenizer = new Tokenizer(new HighlightRules().getRules(), "i");
this.foldingRules = new FoldMode(["diff", "index", "\\+{3}", "@@|\\*{5}"], "i");
this.$tokenizer = new Tokenizer(new HighlightRules().getRules());
this.foldingRules = new FoldMode(["diff", "index", "\\+{3}", "@@|\\*{5}"], "i");
};
oop.inherits(Mode, TextMode);

3
lib/ace/mode/diff_highlight_rules.js
View file

@ -92,7 +92,8 @@ var DiffHighlightRules = function() {
regex: "\\s*$",
token: "invalid"
}, {
defaultToken: "invisible"
defaultToken: "invisible",
caseInsensitive: true
}
]
};

2
lib/ace/mode/less.js
View file

@ -40,7 +40,7 @@ var CssBehaviour = require("./behaviour/css").CssBehaviour;
var CStyleFoldMode = require("./folding/cstyle").FoldMode;
var Mode = function() {
this.$tokenizer = new Tokenizer(new LessHighlightRules().getRules(), "i");
this.$tokenizer = new Tokenizer(new LessHighlightRules().getRules());
this.$outdent = new MatchingBraceOutdent();
this.$behaviour = new CssBehaviour();
this.foldingRules = new CStyleFoldMode();

2
lib/ace/mode/less_highlight_rules.js
View file

@ -247,6 +247,8 @@ var LessHighlightRules = function() {
}, {
token : "text",
regex : "\\s+"
}, {
caseInsensitive: true
}
],
"comment" : [

2
lib/ace/mode/scss.js
View file

@ -40,7 +40,7 @@ var CssBehaviour = require("./behaviour/css").CssBehaviour;
var CStyleFoldMode = require("./folding/cstyle").FoldMode;
var Mode = function() {
this.$tokenizer = new Tokenizer(new ScssHighlightRules().getRules(), "i");
this.$tokenizer = new Tokenizer(new ScssHighlightRules().getRules());
this.$outdent = new MatchingBraceOutdent();
this.$behaviour = new CssBehaviour();
this.foldingRules = new CStyleFoldMode();

2
lib/ace/mode/scss_highlight_rules.js
View file

@ -249,6 +249,8 @@ var ScssHighlightRules = function() {
}, {
token : "text",
regex : "\\s+"
}, {
caseInsensitive: true
}
],
"comment" : [

15
lib/ace/tokenizer.js
View file

@ -43,15 +43,10 @@ var MAX_TOKEN_COUNT = 1000;
/**
* Constructs a new tokenizer based on the given rules and flags.
* @param {Object} rules The highlighting rules
* @param {String} flag Any additional regular expression flags to pass (like "i" for case insensitive)
*
*
*
*
* @constructor
**/
var Tokenizer = function(rules, flag) {
flag = flag ? "g" + flag : "g";
var Tokenizer = function(rules) {
this.states = rules;
this.regExps = {};
@ -61,13 +56,17 @@ var Tokenizer = function(rules, flag) {
var ruleRegExps = [];
var matchTotal = 0;
var mapping = this.matchMappings[key] = {defaultToken: "text"};
var flag = "g";
for (var i = 0; i < state.length; i++) {
var rule = state[i];
if (rule.defaultToken) {
if (rule.defaultToken)
mapping.defaultToken = rule.defaultToken;
if (rule.caseInsensitive)
flag = "gi";
if (rule.regex == null)
continue;
}
if (rule.regex instanceof RegExp)
rule.regex = rule.regex.toString().slice(1, -1);