diff --git a/BackgroundTokenizer.js b/BackgroundTokenizer.js index ead891c4..fe190a30 100644 --- a/BackgroundTokenizer.js +++ b/BackgroundTokenizer.js @@ -1,9 +1,10 @@ -function BackgroundTokenizer(onUpdate, onComplete) +function BackgroundTokenizer(tokenizer, onUpdate, onComplete) { this.running = false; this.textLines = []; this.lines = []; this.currentLine = 0; + this.tokenizer = tokenizer; this.onUpdate = onUpdate || function(firstLine, lastLine) {}; this.onComplete = onComplete || function() {}; @@ -24,7 +25,7 @@ function BackgroundTokenizer(onUpdate, onComplete) var line = textLines[self.currentLine]; var state = self.currentLine == 0 ? "start" : self.lines[self.currentLine-1].state; - self.lines[self.currentLine] = getLineTokens(line, state); + self.lines[self.currentLine] = self.tokenizer.getLineTokens(line, state); if ((new Date()-workerStart) > 80) { @@ -58,7 +59,7 @@ BackgroundTokenizer.prototype.start = function(startRow) if (!this.running) { this.running = true; - setTimeout(this._worker); + setTimeout(this._worker, 50); } }; @@ -75,211 +76,6 @@ BackgroundTokenizer.prototype.getTokens = function(row) if (row > 0 && this.lines[row-1]) { state = this.lines[row-1].state; } - return getLineTokens(this.textLines[row] || "", state).tokens; - } -}; - -var keywords = { - "break" : 1, - "case" : 1, - "catch" : 1, - "continue" : 1, - "default" : 1, - "delete" : 1, - "do" : 1, - "else" : 1, - "finally" : 1, - "for" : 1, - "function" : 1, - "if" : 1, - "in" : 1, - "instanceof" : 1, - "new" : 1, - "return" : 1, - "switch" : 1, - "throw" : 1, - "try" : 1, - "typeof" : 1, - "var" : 1, - "while" : 1, - "with" : 1 -}; - -getLineTokens = function(line, startState) -{ - // regexp must not have capturing parentheses - // regexps are ordered -> the first match is used - - var rules = { - start : - [ - { - token: "comment", - regex: "\\/\\/.*$" - }, - { - token: "comment", // multi line comment in one line - regex: "\\/\\*.*?\\*\\/" - }, - { - token: "comment", // multi line comment start - regex: "\\/\\*.*$", - next: "comment" - }, - { - token: "string", // single line - regex: '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' - }, - { - token: "string", // multi line string start - regex: '["].*\\\\$', - next: "qqstring" - }, - { - token: "string", // single line - regex: "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" - }, - { - token: "string", // multi line string start - regex: "['].*\\\\$", - next: "qstring" - }, - { - token: "number", // hex - regex: "0[xX][0-9a-fA-F]+\\b" - }, - { - token: "number", // float - regex: "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" - }, - { - token: function(value) - { - if (keywords[value]) { - return "keyword"; - } else { - return "identifier" - } - }, - regex: "[a-zA-Z_][a-zA-Z0-9_]*\\b" - }, - { - token: function(value) { - //return parens[value]; - return "text"; - }, - regex: "[\\[\\]\\(\\)\\{\\}]" - }, - { - token: "text", - regex: "\\s+" - } - ], - "comment": - [ - { - token: "comment", // closing comment - regex: ".*?\\*\\/", - next: "start" - }, - { - token: "comment", // comment spanning whole line - regex: ".+" - } - ], - "qqstring": - [ - { - token: "string", - regex: '(?:(?:\\\\.)|(?:[^"\\\\]))*?"', - next: "start" - }, - { - token: "string", - regex: '.+' - } - ], - "qstring": - [ - { - token: "string", - regex: "(?:(?:\\\\.)|(?:[^'\\\\]))*?'", - next: "start" - }, - { - token: "string", - regex: '.+' - } - ] - }; - - var regExps = {}; - for (var key in rules) - { - var state = rules[key]; - var ruleRegExps = []; - - for (var i=0; i < state.length; i++) { - ruleRegExps.push(state[i].regex); - }; - - regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g"); - } - - - var currentState = startState; - var state = rules[currentState]; - var re = regExps[currentState]; - re.lastIndex = 0; - - var match, tokens = []; - - var lastIndex = 0; - - while (match = re.exec(line)) - { - var token = { - type: "text", - value: match[0] - } - - if (re.lastIndex == lastIndex) { - throw new Error("tokenizer error") - } - lastIndex = re.lastIndex; - - //console.log(match); - - for (var i=0; i < state.length; i++) - { - if (match[i+1]) - { - if (typeof state[i].token == "function") { - token.type = state[i].token(match[0]); - } else { - token.type = state[i].token; - } - - if (state[i].next && state[i].next !== currentState) - { - currentState = state[i].next; - var state = rules[currentState]; - var lastIndex = re.lastIndex; - - var re = regExps[currentState]; - re.lastIndex = lastIndex; - } - break; - } - }; - - tokens.push(token); - }; - - //console.log(tokens, currentState) - - return { - tokens: tokens, - state: currentState + return this.tokenizer.getLineTokens(this.textLines[row] || "", state).tokens; } }; \ No newline at end of file diff --git a/Editor.js b/Editor.js index fe8f7ff9..a77136bd 100644 --- a/Editor.js +++ b/Editor.js @@ -115,7 +115,11 @@ function Editor(doc, renderer) doc.addChangeListener(lib.bind(this.onDocumentChange, this)); renderer.setDocument(doc); - this.tokenizer = new BackgroundTokenizer(lib.bind(this.onTokenizerUpdate, this)); + this.tokenizer = new BackgroundTokenizer( + new Tokenizer(XML.RULES), + lib.bind(this.onTokenizerUpdate, this) + ); + this.tokenizer.setLines(doc.lines); renderer.setTokenizer(this.tokenizer); diff --git a/JavaScript.js b/JavaScript.js new file mode 100644 index 00000000..4816b7b1 --- /dev/null +++ b/JavaScript.js @@ -0,0 +1,137 @@ +(function() { + +window.JavaScript = {}; + +var keywords = { + "break" : 1, + "case" : 1, + "catch" : 1, + "continue" : 1, + "default" : 1, + "delete" : 1, + "do" : 1, + "else" : 1, + "finally" : 1, + "for" : 1, + "function" : 1, + "if" : 1, + "in" : 1, + "instanceof" : 1, + "new" : 1, + "return" : 1, + "switch" : 1, + "throw" : 1, + "try" : 1, + "typeof" : 1, + "var" : 1, + "while" : 1, + "with" : 1 +}; + +// regexp must not have capturing parentheses +// regexps are ordered -> the first match is used + +JavaScript.RULES = { + start : + [ + { + token: "comment", + regex: "\\/\\/.*$" + }, + { + token: "comment", // multi line comment in one line + regex: "\\/\\*.*?\\*\\/" + }, + { + token: "comment", // multi line comment start + regex: "\\/\\*.*$", + next: "comment" + }, + { + token: "string", // single line + regex: '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' + }, + { + token: "string", // multi line string start + regex: '["].*\\\\$', + next: "qqstring" + }, + { + token: "string", // single line + regex: "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" + }, + { + token: "string", // multi line string start + regex: "['].*\\\\$", + next: "qstring" + }, + { + token: "number", // hex + regex: "0[xX][0-9a-fA-F]+\\b" + }, + { + token: "number", // float + regex: "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" + }, + { + token: function(value) + { + if (keywords[value]) { + return "keyword"; + } else { + return "identifier" + } + }, + regex: "[a-zA-Z_][a-zA-Z0-9_]*\\b" + }, + { + token: function(value) { + //return parens[value]; + return "text"; + }, + regex: "[\\[\\]\\(\\)\\{\\}]" + }, + { + token: "text", + regex: "\\s+" + } + ], + "comment": + [ + { + token: "comment", // closing comment + regex: ".*?\\*\\/", + next: "start" + }, + { + token: "comment", // comment spanning whole line + regex: ".+" + } + ], + "qqstring": + [ + { + token: "string", + regex: '(?:(?:\\\\.)|(?:[^"\\\\]))*?"', + next: "start" + }, + { + token: "string", + regex: '.+' + } + ], + "qstring": + [ + { + token: "string", + regex: "(?:(?:\\\\.)|(?:[^'\\\\]))*?'", + next: "start" + }, + { + token: "string", + regex: '.+' + } + ] +}; + +})(); \ No newline at end of file diff --git a/Tokenizer.js b/Tokenizer.js new file mode 100644 index 00000000..f6cfe194 --- /dev/null +++ b/Tokenizer.js @@ -0,0 +1,77 @@ + +function Tokenizer(rules) +{ + this.rules = rules; + + this.regExps = {}; + for (var key in this.rules) + { + var state = this.rules[key]; + var ruleRegExps = []; + + for (var i=0; i < state.length; i++) { + ruleRegExps.push(state[i].regex); + }; + + this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g"); + } +}; + +Tokenizer.prototype.getLineTokens = function(line, startState) +{ + var currentState = startState; + var state = this.rules[currentState]; + var re = this.regExps[currentState]; + re.lastIndex = 0; + + var match, tokens = []; + + var lastIndex = 0; + + while (match = re.exec(line)) + { + var token = { + type: "text", + value: match[0] + } + + if (re.lastIndex == lastIndex) { + throw new Error("tokenizer error") + } + lastIndex = re.lastIndex; + + //console.log(match); + + for (var i=0; i < state.length; i++) + { + if (match[i+1]) + { + if (typeof state[i].token == "function") { + token.type = state[i].token(match[0]); + } else { + token.type = state[i].token; + } + + if (state[i].next && state[i].next !== currentState) + { + currentState = state[i].next; + var state = this.rules[currentState]; + var lastIndex = re.lastIndex; + + var re = this.regExps[currentState]; + re.lastIndex = lastIndex; + } + break; + } + }; + + tokens.push(token); + }; + + //console.log(tokens, currentState) + + return { + tokens: tokens, + state: currentState + } +}; \ No newline at end of file diff --git a/XML.js b/XML.js new file mode 100644 index 00000000..cd6b2b02 --- /dev/null +++ b/XML.js @@ -0,0 +1,78 @@ +(function() { + +window.XML = {}; + +// regexp must not have capturing parentheses +// regexps are ordered -> the first match is used + +XML.RULES = { + start : + [ + { + token: "text", + regex: "<\\!\\[CDATA\\[", + next: "cdata" + }, + { + token: "xml_pe", + regex: "<\\?.*?\\?>" + }, + { + token: "text", // opening tag + regex: "<", + next: "tag" + }, + { + token: "text", + regex: "\\s+" + }, + { + token: "text", + regex: ".+" + } + ], + + tag: + [ + { + token: "text", + regex: ">", + next: "start" + }, + { + token: "keyword", + regex: "[-_a-zA-Z0-9:]+" + }, + { + token: "text", + regex: "\\s+" + }, + { + token: "string", + regex: '".*?"' + }, + { + token: "string", + regex: "'.*?'" + } + ], + + cdata: + [ + { + token: "text", + regex: "\\]\\]>", + next: "start" + }, + { + token: "string", + regex: "\\s+" + }, + { + token: "text", + regex: ".+" + } + ] +}; + +})(); \ No newline at end of file diff --git a/editor.css b/editor.css index 0b6f789f..2cf1701a 100644 --- a/editor.css +++ b/editor.css @@ -85,6 +85,10 @@ color: rgb(0, 0, 205); } +.line .xml_pe { + color: rgb(104, 104, 91); +} + .marker-layer .selection { position: absolute; background: rgba(77, 151, 255, 0.33); diff --git a/editor.html b/editor.html index 87c93655..dcd024fd 100644 --- a/editor.html +++ b/editor.html @@ -21,7 +21,10 @@ - + + + +