From 88363fe91170fb0bb164e0e55c8f36acf7554f1c Mon Sep 17 00:00:00 2001 From: Fabian Jakobs Date: Fri, 6 Apr 2012 13:40:43 +0200 Subject: [PATCH] allow real regular expressions in highlighters --- lib/ace/mode/javascript_highlight_rules.js | 157 ++++++++++----------- lib/ace/tokenizer.js | 40 +++--- 2 files changed, 100 insertions(+), 97 deletions(-) diff --git a/lib/ace/mode/javascript_highlight_rules.js b/lib/ace/mode/javascript_highlight_rules.js index 9d8782f7..0fb9b616 100644 --- a/lib/ace/mode/javascript_highlight_rules.js +++ b/lib/ace/mode/javascript_highlight_rules.js @@ -93,20 +93,20 @@ var JavaScriptHighlightRules = function() { ); // TODO: Unicode escape sequences - var identifierRe = "[" + unicode.packages.L + "\\$_][" + var identifierRe = "[" + unicode.packages.L + "\\$_][" + unicode.packages.L + unicode.packages.Mn + unicode.packages.Mc + unicode.packages.Nd + unicode.packages.Pc + "\\$_]*\\b"; - + var escapedRe = "\\\\(?:x[0-9a-fA-F]{2}|" + // hex "u[0-9a-fA-F]{4}|" + // unicode "[0-2][0-7]{0,2}|" + // oct "3[0-6][0-7]?|" + // oct - "37[0-7]?|" + // oct + "37[0-7]?|" + // oct "[4-7][0-7]?|" + //oct ".)"; - + // regexp must not have capturing parentheses. Use (?:) instead. // regexps are ordered -> the first match is used @@ -114,13 +114,13 @@ var JavaScriptHighlightRules = function() { "start" : [ { token : "comment", - regex : "\\/\\/.*$" + regex : /\/\/.*$/ }, new DocCommentHighlightRules().getStartRule("doc-start"), { token : "comment", // multi line comment merge : true, - regex : "\\/\\*", + regex : /\/\*/, next : "comment" }, { token : "string", @@ -132,49 +132,49 @@ var JavaScriptHighlightRules = function() { next : "qqstring" }, { token : "constant.numeric", // hex - regex : "0[xX][0-9a-fA-F]+\\b" + regex : /0[xX][0-9a-fA-F]+\b/ }, { token : "constant.numeric", // float - regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" + regex : /[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?\b/ }, { // match stuff like: Sound.prototype.play = function() { } token : [ "storage.type", - "punctuation.operator", - "support.function", - "punctuation.operator", - "entity.name.function", - "text", - "keyword.operator", - "text", - "storage.type", - "text", - "paren.lparen", - "variable.parameter", + "punctuation.operator", + "support.function", + "punctuation.operator", + "entity.name.function", + "text", + "keyword.operator", + "text", + "storage.type", + "text", + "paren.lparen", + "variable.parameter", "paren.rparen" ], regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" }, { // match stuff like: Sound.prototype.play = myfunc token : [ - "storage.type", - "punctuation.operator", - "support.function", - "punctuation.operator", - "entity.name.function", - "text", - "keyword.operator", + "storage.type", + "punctuation.operator", + "support.function", + "punctuation.operator", + "entity.name.function", + "text", + "keyword.operator", "text" ], regex : "(" + identifierRe + ")(\\.)(prototype)(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)" }, { // match stuff like: Sound.play = function() { } token : [ - "storage.type", - "punctuation.operator", - "entity.name.function", - "text", - "keyword.operator", - "text", "storage.type", - "text", + "punctuation.operator", + "entity.name.function", + "text", + "keyword.operator", + "text", + "storage.type", + "text", "paren.lparen", "variable.parameter", "paren.rparen" @@ -182,23 +182,23 @@ var JavaScriptHighlightRules = function() { regex : "(" + identifierRe + ")(\\.)(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" }, { // match stuff like: play = function() { } token : [ - "entity.name.function", - "text", - "keyword.operator", - "text", - "storage.type", - "text", - "paren.lparen", - "variable.parameter", + "entity.name.function", + "text", + "keyword.operator", + "text", + "storage.type", + "text", + "paren.lparen", + "variable.parameter", "paren.rparen" ], regex : "(" + identifierRe +")(\\s*)(=)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" }, { // match regular function like: function myFunc(arg) { } token : [ - "storage.type", + "storage.type", "text", "entity.name.function", - "text", + "text", "paren.lparen", "variable.parameter", "paren.rparen" @@ -206,48 +206,47 @@ var JavaScriptHighlightRules = function() { regex : "(function)(\\s+)(" + identifierRe + ")(\\s*)(\\()(.*?)(\\))" }, { // match stuff like: foobar: function() { } token : [ - "entity.name.function", - "text", - "punctuation.operator", - "text", - "storage.type", - "text", - "paren.lparen", - "variable.parameter", + "entity.name.function", + "text", + "punctuation.operator", + "text", + "storage.type", + "text", + "paren.lparen", + "variable.parameter", "paren.rparen" ], regex : "(" + identifierRe + ")(\\s*)(:)(\\s*)(function)(\\s*)(\\()(.*?)(\\))" }, { // Attempt to match : function() { } (this is for issues with 'foo': function() { }) token : [ - "text", - "text", - "storage.type", - "text", - "paren.lparen", - "variable.parameter", + "text", + "text", + "storage.type", + "text", + "paren.lparen", + "variable.parameter", "paren.rparen" ], regex : "(:)(\\s*)(function)?(\\s*)(\\()([^)]*)(\\))" }, { token : "constant.language.boolean", - regex : "(?:true|false)\\b" + regex : /(?:true|false)\b/ }, { token : "keyword", regex : "(?:" + kwBeforeRe + ")\\b", next : "regex_allowed" }, { token : "support.function", - regex : "\\b(?:s(?:h(?:ift|ow(?:Mod(?:elessDialog|alDialog)|Help))|croll(?:X|By(?:Pages|Lines)?|Y|To)?|t(?:op|rike)|i(?:n|zeToContent|debar|gnText)|ort|u(?:p|b(?:str(?:ing)?)?)|pli(?:ce|t)|e(?:nd|t(?:Re(?:sizable|questHeader)|M(?:i(?:nutes|lliseconds)|onth)|Seconds|Ho(?:tKeys|urs)|Year|Cursor|Time(?:out)?|Interval|ZOptions|Date|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Date|FullYear)|FullYear|Active)|arch)|qrt|lice|avePreferences|mall)|h(?:ome|andleEvent)|navigate|c(?:har(?:CodeAt|At)|o(?:s|n(?:cat|textual|firm)|mpile)|eil|lear(?:Timeout|Interval)?|a(?:ptureEvents|ll)|reate(?:StyleSheet|Popup|EventObject))|t(?:o(?:GMTString|S(?:tring|ource)|U(?:TCString|pperCase)|Lo(?:caleString|werCase))|est|a(?:n|int(?:Enabled)?))|i(?:s(?:NaN|Finite)|ndexOf|talics)|d(?:isableExternalCapture|ump|etachEvent)|u(?:n(?:shift|taint|escape|watch)|pdateCommands)|j(?:oin|avaEnabled)|p(?:o(?:p|w)|ush|lugins.refresh|a(?:ddings|rse(?:Int|Float)?)|r(?:int|ompt|eference))|e(?:scape|nableExternalCapture|val|lementFromPoint|x(?:p|ec(?:Script|Command)?))|valueOf|UTC|queryCommand(?:State|Indeterm|Enabled|Value)|f(?:i(?:nd|le(?:ModifiedDate|Size|CreatedDate|UpdatedDate)|xed)|o(?:nt(?:size|color)|rward)|loor|romCharCode)|watch|l(?:ink|o(?:ad|g)|astIndexOf)|a(?:sin|nchor|cos|t(?:tachEvent|ob|an(?:2)?)|pply|lert|b(?:s|ort))|r(?:ou(?:nd|teEvents)|e(?:size(?:By|To)|calc|turnValue|place|verse|l(?:oad|ease(?:Capture|Events)))|andom)|g(?:o|et(?:ResponseHeader|M(?:i(?:nutes|lliseconds)|onth)|Se(?:conds|lection)|Hours|Year|Time(?:zoneOffset)?|Da(?:y|te)|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Da(?:y|te)|FullYear)|FullYear|A(?:ttention|llResponseHeaders)))|m(?:in|ove(?:B(?:y|elow)|To(?:Absolute)?|Above)|ergeAttributes|a(?:tch|rgins|x))|b(?:toa|ig|o(?:ld|rderWidths)|link|ack))\\b(?=\\()" + regex : /\b(?:s(?:h(?:ift|ow(?:Mod(?:elessDialog|alDialog)|Help))|croll(?:X|By(?:Pages|Lines)?|Y|To)?|t(?:op|rike)|i(?:n|zeToContent|debar|gnText)|ort|u(?:p|b(?:str(?:ing)?)?)|pli(?:ce|t)|e(?:nd|t(?:Re(?:sizable|questHeader)|M(?:i(?:nutes|lliseconds)|onth)|Seconds|Ho(?:tKeys|urs)|Year|Cursor|Time(?:out)?|Interval|ZOptions|Date|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Date|FullYear)|FullYear|Active)|arch)|qrt|lice|avePreferences|mall)|h(?:ome|andleEvent)|navigate|c(?:har(?:CodeAt|At)|o(?:s|n(?:cat|textual|firm)|mpile)|eil|lear(?:Timeout|Interval)?|a(?:ptureEvents|ll)|reate(?:StyleSheet|Popup|EventObject))|t(?:o(?:GMTString|S(?:tring|ource)|U(?:TCString|pperCase)|Lo(?:caleString|werCase))|est|a(?:n|int(?:Enabled)?))|i(?:s(?:NaN|Finite)|ndexOf|talics)|d(?:isableExternalCapture|ump|etachEvent)|u(?:n(?:shift|taint|escape|watch)|pdateCommands)|j(?:oin|avaEnabled)|p(?:o(?:p|w)|ush|lugins.refresh|a(?:ddings|rse(?:Int|Float)?)|r(?:int|ompt|eference))|e(?:scape|nableExternalCapture|val|lementFromPoint|x(?:p|ec(?:Script|Command)?))|valueOf|UTC|queryCommand(?:State|Indeterm|Enabled|Value)|f(?:i(?:nd|le(?:ModifiedDate|Size|CreatedDate|UpdatedDate)|xed)|o(?:nt(?:size|color)|rward)|loor|romCharCode)|watch|l(?:ink|o(?:ad|g)|astIndexOf)|a(?:sin|nchor|cos|t(?:tachEvent|ob|an(?:2)?)|pply|lert|b(?:s|ort))|r(?:ou(?:nd|teEvents)|e(?:size(?:By|To)|calc|turnValue|place|verse|l(?:oad|ease(?:Capture|Events)))|andom)|g(?:o|et(?:ResponseHeader|M(?:i(?:nutes|lliseconds)|onth)|Se(?:conds|lection)|Hours|Year|Time(?:zoneOffset)?|Da(?:y|te)|UTC(?:M(?:i(?:nutes|lliseconds)|onth)|Seconds|Hours|Da(?:y|te)|FullYear)|FullYear|A(?:ttention|llResponseHeaders)))|m(?:in|ove(?:B(?:y|elow)|To(?:Absolute)?|Above)|ergeAttributes|a(?:tch|rgins|x))|b(?:toa|ig|o(?:ld|rderWidths)|link|ack))\b(?=\()/ }, { token : "support.function.dom", - regex : "\\b(?:s(?:ub(?:stringData|mit)|plitText|e(?:t(?:NamedItem|Attribute(?:Node)?)|lect))|has(?:ChildNodes|Feature)|namedItem|c(?:l(?:ick|o(?:se|neNode))|reate(?:C(?:omment|DATASection|aption)|T(?:Head|extNode|Foot)|DocumentFragment|ProcessingInstruction|E(?:ntityReference|lement)|Attribute))|tabIndex|i(?:nsert(?:Row|Before|Cell|Data)|tem)|open|delete(?:Row|C(?:ell|aption)|T(?:Head|Foot)|Data)|focus|write(?:ln)?|a(?:dd|ppend(?:Child|Data))|re(?:set|place(?:Child|Data)|move(?:NamedItem|Child|Attribute(?:Node)?)?)|get(?:NamedItem|Element(?:sBy(?:Name|TagName)|ById)|Attribute(?:Node)?)|blur)\\b(?=\\()" + regex : /\b(?:s(?:ub(?:stringData|mit)|plitText|e(?:t(?:NamedItem|Attribute(?:Node)?)|lect))|has(?:ChildNodes|Feature)|namedItem|c(?:l(?:ick|o(?:se|neNode))|reate(?:C(?:omment|DATASection|aption)|T(?:Head|extNode|Foot)|DocumentFragment|ProcessingInstruction|E(?:ntityReference|lement)|Attribute))|tabIndex|i(?:nsert(?:Row|Before|Cell|Data)|tem)|open|delete(?:Row|C(?:ell|aption)|T(?:Head|Foot)|Data)|focus|write(?:ln)?|a(?:dd|ppend(?:Child|Data))|re(?:set|place(?:Child|Data)|move(?:NamedItem|Child|Attribute(?:Node)?)?)|get(?:NamedItem|Element(?:sBy(?:Name|TagName)|ById)|Attribute(?:Node)?)|blur)\b(?=\()/ }, { token : "support.constant", - regex : "\\b(?:s(?:ystemLanguage|cr(?:ipts|ollbars|een(?:X|Y|Top|Left))|t(?:yle(?:Sheets)?|atus(?:Text|bar)?)|ibling(?:Below|Above)|ource|uffixes|e(?:curity(?:Policy)?|l(?:ection|f)))|h(?:istory|ost(?:name)?|as(?:h|Focus))|y|X(?:MLDocument|SLDocument)|n(?:ext|ame(?:space(?:s|URI)|Prop))|M(?:IN_VALUE|AX_VALUE)|c(?:haracterSet|o(?:n(?:structor|trollers)|okieEnabled|lorDepth|mp(?:onents|lete))|urrent|puClass|l(?:i(?:p(?:boardData)?|entInformation)|osed|asses)|alle(?:e|r)|rypto)|t(?:o(?:olbar|p)|ext(?:Transform|Indent|Decoration|Align)|ags)|SQRT(?:1_2|2)|i(?:n(?:ner(?:Height|Width)|put)|ds|gnoreCase)|zIndex|o(?:scpu|n(?:readystatechange|Line)|uter(?:Height|Width)|p(?:sProfile|ener)|ffscreenBuffering)|NEGATIVE_INFINITY|d(?:i(?:splay|alog(?:Height|Top|Width|Left|Arguments)|rectories)|e(?:scription|fault(?:Status|Ch(?:ecked|arset)|View)))|u(?:ser(?:Profile|Language|Agent)|n(?:iqueID|defined)|pdateInterval)|_content|p(?:ixelDepth|ort|ersonalbar|kcs11|l(?:ugins|atform)|a(?:thname|dding(?:Right|Bottom|Top|Left)|rent(?:Window|Layer)?|ge(?:X(?:Offset)?|Y(?:Offset)?))|r(?:o(?:to(?:col|type)|duct(?:Sub)?|mpter)|e(?:vious|fix)))|e(?:n(?:coding|abledPlugin)|x(?:ternal|pando)|mbeds)|v(?:isibility|endor(?:Sub)?|Linkcolor)|URLUnencoded|P(?:I|OSITIVE_INFINITY)|f(?:ilename|o(?:nt(?:Size|Family|Weight)|rmName)|rame(?:s|Element)|gColor)|E|whiteSpace|l(?:i(?:stStyleType|n(?:eHeight|kColor))|o(?:ca(?:tion(?:bar)?|lName)|wsrc)|e(?:ngth|ft(?:Context)?)|a(?:st(?:M(?:odified|atch)|Index|Paren)|yer(?:s|X)|nguage))|a(?:pp(?:MinorVersion|Name|Co(?:deName|re)|Version)|vail(?:Height|Top|Width|Left)|ll|r(?:ity|guments)|Linkcolor|bove)|r(?:ight(?:Context)?|e(?:sponse(?:XML|Text)|adyState))|global|x|m(?:imeTypes|ultiline|enubar|argin(?:Right|Bottom|Top|Left))|L(?:N(?:10|2)|OG(?:10E|2E))|b(?:o(?:ttom|rder(?:Width|RightWidth|BottomWidth|Style|Color|TopWidth|LeftWidth))|ufferDepth|elow|ackground(?:Color|Image)))\\b" + regex : /\b(?:s(?:ystemLanguage|cr(?:ipts|ollbars|een(?:X|Y|Top|Left))|t(?:yle(?:Sheets)?|atus(?:Text|bar)?)|ibling(?:Below|Above)|ource|uffixes|e(?:curity(?:Policy)?|l(?:ection|f)))|h(?:istory|ost(?:name)?|as(?:h|Focus))|y|X(?:MLDocument|SLDocument)|n(?:ext|ame(?:space(?:s|URI)|Prop))|M(?:IN_VALUE|AX_VALUE)|c(?:haracterSet|o(?:n(?:structor|trollers)|okieEnabled|lorDepth|mp(?:onents|lete))|urrent|puClass|l(?:i(?:p(?:boardData)?|entInformation)|osed|asses)|alle(?:e|r)|rypto)|t(?:o(?:olbar|p)|ext(?:Transform|Indent|Decoration|Align)|ags)|SQRT(?:1_2|2)|i(?:n(?:ner(?:Height|Width)|put)|ds|gnoreCase)|zIndex|o(?:scpu|n(?:readystatechange|Line)|uter(?:Height|Width)|p(?:sProfile|ener)|ffscreenBuffering)|NEGATIVE_INFINITY|d(?:i(?:splay|alog(?:Height|Top|Width|Left|Arguments)|rectories)|e(?:scription|fault(?:Status|Ch(?:ecked|arset)|View)))|u(?:ser(?:Profile|Language|Agent)|n(?:iqueID|defined)|pdateInterval)|_content|p(?:ixelDepth|ort|ersonalbar|kcs11|l(?:ugins|atform)|a(?:thname|dding(?:Right|Bottom|Top|Left)|rent(?:Window|Layer)?|ge(?:X(?:Offset)?|Y(?:Offset)?))|r(?:o(?:to(?:col|type)|duct(?:Sub)?|mpter)|e(?:vious|fix)))|e(?:n(?:coding|abledPlugin)|x(?:ternal|pando)|mbeds)|v(?:isibility|endor(?:Sub)?|Linkcolor)|URLUnencoded|P(?:I|OSITIVE_INFINITY)|f(?:ilename|o(?:nt(?:Size|Family|Weight)|rmName)|rame(?:s|Element)|gColor)|E|whiteSpace|l(?:i(?:stStyleType|n(?:eHeight|kColor))|o(?:ca(?:tion(?:bar)?|lName)|wsrc)|e(?:ngth|ft(?:Context)?)|a(?:st(?:M(?:odified|atch)|Index|Paren)|yer(?:s|X)|nguage))|a(?:pp(?:MinorVersion|Name|Co(?:deName|re)|Version)|vail(?:Height|Top|Width|Left)|ll|r(?:ity|guments)|Linkcolor|bove)|r(?:ight(?:Context)?|e(?:sponse(?:XML|Text)|adyState))|global|x|m(?:imeTypes|ultiline|enubar|argin(?:Right|Bottom|Top|Left))|L(?:N(?:10|2)|OG(?:10E|2E))|b(?:o(?:ttom|rder(?:Width|RightWidth|BottomWidth|Style|Color|TopWidth|LeftWidth))|ufferDepth|elow|ackground(?:Color|Image)))\b/ }, { - token : ["punctuation.operator", "support.function.firebug"], - regex : "(\\.)(warn|info|log|error|time|timeEnd|assert)\\b" - + token : ["storage.type", "punctuation.operator", "support.function.firebug"], + regex : /\b(console)(\.)(warn|info|log|error|time|timeEnd|assert)\b/ }, { token : function(value) { if (globals.hasOwnProperty(value)) @@ -270,29 +269,29 @@ var JavaScriptHighlightRules = function() { regex : identifierRe }, { token : "keyword.operator", - regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)", + regex : /!|\$|%|&|\*|\-\-|\-|\+\+|\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\|\||\?\:|\*=|%=|\+=|\-=|&=|\^=|\b(?:in|instanceof|new|delete|typeof|void)/, next : "regex_allowed" }, { token : "punctuation.operator", - regex : "\\?|\\:|\\,|\\;|\\.", + regex : /\?|\:|\,|\;|\./, next : "regex_allowed" }, { token : "paren.lparen", - regex : "[[({]", + regex : /[\[({]/, next : "regex_allowed" }, { token : "paren.rparen", - regex : "[\\])}]" + regex : /[\])}]/ }, { token : "keyword.operator", - regex : "\\/=?", + regex : /\/=?/, next : "regex_allowed" }, { token: "comment", - regex: "^#!.*$" + regex: /^#!.*$/ }, { token : "text", - regex : "\\s+" + regex : /\s+/ } ], // regular expressions are only allowed after certain tokens. This @@ -317,7 +316,7 @@ var JavaScriptHighlightRules = function() { }, { // immediately return to the start mode without matching // anything - token: "empty", + token: "empty", regex: "", next: "start" } @@ -329,10 +328,10 @@ var JavaScriptHighlightRules = function() { next: "regex" }, { // flag - token: "string.regexp", + token: "string.regexp", regex: "/\\w*", next: "start", - merge: true + merge: true }, { token: "string.regexp", regex: "[^\\\\/\\[]+", @@ -344,9 +343,9 @@ var JavaScriptHighlightRules = function() { next: "regex_character_class", merge: true }, { - token: "empty", + token: "empty", regex: "", - next: "start" + next: "start" } ], "regex_character_class": [ @@ -365,9 +364,9 @@ var JavaScriptHighlightRules = function() { next: "regex_character_class", merge: true }, { - token: "empty", + token: "empty", regex: "", - next: "start" + next: "start" } ], "comment_regex_allowed" : [ @@ -421,7 +420,7 @@ var JavaScriptHighlightRules = function() { } ] }; - + this.embedRules(DocCommentHighlightRules, "doc-", [ new DocCommentHighlightRules().getEndRule("start") ]); }; diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js index afd50b76..674e30ed 100644 --- a/lib/ace/tokenizer.js +++ b/lib/ace/tokenizer.js @@ -50,26 +50,30 @@ var Tokenizer = function(rules, flag) { var ruleRegExps = []; var matchTotal = 0; var mapping = this.matchMappings[key] = {}; - + for ( var i = 0; i < state.length; i++) { + + if (state[i].regex instanceof RegExp) + state[i].regex = state[i].regex.toString().slice(1, -1); + // Count number of matching groups. 2 extra groups from the full match // And the catch-all on the end (used to force a match); var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2; - + // Replace any backreferences and offset appropriately. var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) { return "\\" + (parseInt(digit, 10) + matchTotal + 1); }); - + if (matchcount > 1 && state[i].token.length !== matchcount-1) throw new Error("Matching groups and length of the token array don't match in rule #" + i + " of state " + key); - + mapping[matchTotal] = { rule: i, len: matchcount }; matchTotal += matchcount; - + ruleRegExps.push(adjustedregex); } @@ -85,16 +89,16 @@ var Tokenizer = function(rules, flag) { var mapping = this.matchMappings[currentState]; var re = this.regExps[currentState]; re.lastIndex = 0; - + var match, tokens = []; - + var lastIndex = 0; - + var token = { type: null, value: "" }; - + while (match = re.exec(line)) { var type = "text"; var rule = null; @@ -103,19 +107,19 @@ var Tokenizer = function(rules, flag) { for (var i = 0; i < match.length-2; i++) { if (match[i + 1] === undefined) continue; - + rule = state[mapping[i].rule]; - + if (mapping[i].len > 1) value = match.slice(i+2, i+1+mapping[i].len); - + // compute token type if (typeof rule.token == "function") type = rule.token.apply(this, value); else type = rule.token; - var next = rule.next; + var next = rule.next; if (next && next !== currentState) { currentState = next; state = this.rules[currentState]; @@ -127,7 +131,7 @@ var Tokenizer = function(rules, flag) { } break; } - + if (value[0]) { if (typeof type == "string") { value = [value.join("")]; @@ -136,13 +140,13 @@ var Tokenizer = function(rules, flag) { for (var i = 0; i < value.length; i++) { if (!value[i]) continue; - + if ((!rule || rule.merge || type[i] === "text") && token.type === type[i]) { token.value += value[i]; } else { if (token.type) tokens.push(token); - + token = { type: type[i], value: value[i] @@ -150,10 +154,10 @@ var Tokenizer = function(rules, flag) { } } } - + if (lastIndex == line.length) break; - + lastIndex = re.lastIndex; }