diff --git a/BackgroundTokenizer.js b/BackgroundTokenizer.js
index ead891c4..fe190a30 100644
--- a/BackgroundTokenizer.js
+++ b/BackgroundTokenizer.js
@@ -1,9 +1,10 @@
-function BackgroundTokenizer(onUpdate, onComplete)
+function BackgroundTokenizer(tokenizer, onUpdate, onComplete)
{
this.running = false;
this.textLines = [];
this.lines = [];
this.currentLine = 0;
+ this.tokenizer = tokenizer;
this.onUpdate = onUpdate || function(firstLine, lastLine) {};
this.onComplete = onComplete || function() {};
@@ -24,7 +25,7 @@ function BackgroundTokenizer(onUpdate, onComplete)
var line = textLines[self.currentLine];
var state = self.currentLine == 0 ? "start" : self.lines[self.currentLine-1].state;
- self.lines[self.currentLine] = getLineTokens(line, state);
+ self.lines[self.currentLine] = self.tokenizer.getLineTokens(line, state);
if ((new Date()-workerStart) > 80)
{
@@ -58,7 +59,7 @@ BackgroundTokenizer.prototype.start = function(startRow)
if (!this.running)
{
this.running = true;
- setTimeout(this._worker);
+ setTimeout(this._worker, 50);
}
};
@@ -75,211 +76,6 @@ BackgroundTokenizer.prototype.getTokens = function(row)
if (row > 0 && this.lines[row-1]) {
state = this.lines[row-1].state;
}
- return getLineTokens(this.textLines[row] || "", state).tokens;
- }
-};
-
-var keywords = {
- "break" : 1,
- "case" : 1,
- "catch" : 1,
- "continue" : 1,
- "default" : 1,
- "delete" : 1,
- "do" : 1,
- "else" : 1,
- "finally" : 1,
- "for" : 1,
- "function" : 1,
- "if" : 1,
- "in" : 1,
- "instanceof" : 1,
- "new" : 1,
- "return" : 1,
- "switch" : 1,
- "throw" : 1,
- "try" : 1,
- "typeof" : 1,
- "var" : 1,
- "while" : 1,
- "with" : 1
-};
-
-getLineTokens = function(line, startState)
-{
- // regexp must not have capturing parentheses
- // regexps are ordered -> the first match is used
-
- var rules = {
- start :
- [
- {
- token: "comment",
- regex: "\\/\\/.*$"
- },
- {
- token: "comment", // multi line comment in one line
- regex: "\\/\\*.*?\\*\\/"
- },
- {
- token: "comment", // multi line comment start
- regex: "\\/\\*.*$",
- next: "comment"
- },
- {
- token: "string", // single line
- regex: '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
- },
- {
- token: "string", // multi line string start
- regex: '["].*\\\\$',
- next: "qqstring"
- },
- {
- token: "string", // single line
- regex: "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
- },
- {
- token: "string", // multi line string start
- regex: "['].*\\\\$",
- next: "qstring"
- },
- {
- token: "number", // hex
- regex: "0[xX][0-9a-fA-F]+\\b"
- },
- {
- token: "number", // float
- regex: "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
- },
- {
- token: function(value)
- {
- if (keywords[value]) {
- return "keyword";
- } else {
- return "identifier"
- }
- },
- regex: "[a-zA-Z_][a-zA-Z0-9_]*\\b"
- },
- {
- token: function(value) {
- //return parens[value];
- return "text";
- },
- regex: "[\\[\\]\\(\\)\\{\\}]"
- },
- {
- token: "text",
- regex: "\\s+"
- }
- ],
- "comment":
- [
- {
- token: "comment", // closing comment
- regex: ".*?\\*\\/",
- next: "start"
- },
- {
- token: "comment", // comment spanning whole line
- regex: ".+"
- }
- ],
- "qqstring":
- [
- {
- token: "string",
- regex: '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
- next: "start"
- },
- {
- token: "string",
- regex: '.+'
- }
- ],
- "qstring":
- [
- {
- token: "string",
- regex: "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
- next: "start"
- },
- {
- token: "string",
- regex: '.+'
- }
- ]
- };
-
- var regExps = {};
- for (var key in rules)
- {
- var state = rules[key];
- var ruleRegExps = [];
-
- for (var i=0; i < state.length; i++) {
- ruleRegExps.push(state[i].regex);
- };
-
- regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
- }
-
-
- var currentState = startState;
- var state = rules[currentState];
- var re = regExps[currentState];
- re.lastIndex = 0;
-
- var match, tokens = [];
-
- var lastIndex = 0;
-
- while (match = re.exec(line))
- {
- var token = {
- type: "text",
- value: match[0]
- }
-
- if (re.lastIndex == lastIndex) {
- throw new Error("tokenizer error")
- }
- lastIndex = re.lastIndex;
-
- //console.log(match);
-
- for (var i=0; i < state.length; i++)
- {
- if (match[i+1])
- {
- if (typeof state[i].token == "function") {
- token.type = state[i].token(match[0]);
- } else {
- token.type = state[i].token;
- }
-
- if (state[i].next && state[i].next !== currentState)
- {
- currentState = state[i].next;
- var state = rules[currentState];
- var lastIndex = re.lastIndex;
-
- var re = regExps[currentState];
- re.lastIndex = lastIndex;
- }
- break;
- }
- };
-
- tokens.push(token);
- };
-
- //console.log(tokens, currentState)
-
- return {
- tokens: tokens,
- state: currentState
+ return this.tokenizer.getLineTokens(this.textLines[row] || "", state).tokens;
}
};
\ No newline at end of file
diff --git a/Editor.js b/Editor.js
index fe8f7ff9..a77136bd 100644
--- a/Editor.js
+++ b/Editor.js
@@ -115,7 +115,11 @@ function Editor(doc, renderer)
doc.addChangeListener(lib.bind(this.onDocumentChange, this));
renderer.setDocument(doc);
- this.tokenizer = new BackgroundTokenizer(lib.bind(this.onTokenizerUpdate, this));
+ this.tokenizer = new BackgroundTokenizer(
+ new Tokenizer(XML.RULES),
+ lib.bind(this.onTokenizerUpdate, this)
+ );
+
this.tokenizer.setLines(doc.lines);
renderer.setTokenizer(this.tokenizer);
diff --git a/JavaScript.js b/JavaScript.js
new file mode 100644
index 00000000..4816b7b1
--- /dev/null
+++ b/JavaScript.js
@@ -0,0 +1,137 @@
+(function() {
+
+window.JavaScript = {};
+
+var keywords = {
+ "break" : 1,
+ "case" : 1,
+ "catch" : 1,
+ "continue" : 1,
+ "default" : 1,
+ "delete" : 1,
+ "do" : 1,
+ "else" : 1,
+ "finally" : 1,
+ "for" : 1,
+ "function" : 1,
+ "if" : 1,
+ "in" : 1,
+ "instanceof" : 1,
+ "new" : 1,
+ "return" : 1,
+ "switch" : 1,
+ "throw" : 1,
+ "try" : 1,
+ "typeof" : 1,
+ "var" : 1,
+ "while" : 1,
+ "with" : 1
+};
+
+// regexp must not have capturing parentheses
+// regexps are ordered -> the first match is used
+
+JavaScript.RULES = {
+ start :
+ [
+ {
+ token: "comment",
+ regex: "\\/\\/.*$"
+ },
+ {
+ token: "comment", // multi line comment in one line
+ regex: "\\/\\*.*?\\*\\/"
+ },
+ {
+ token: "comment", // multi line comment start
+ regex: "\\/\\*.*$",
+ next: "comment"
+ },
+ {
+ token: "string", // single line
+ regex: '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
+ },
+ {
+ token: "string", // multi line string start
+ regex: '["].*\\\\$',
+ next: "qqstring"
+ },
+ {
+ token: "string", // single line
+ regex: "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
+ },
+ {
+ token: "string", // multi line string start
+ regex: "['].*\\\\$",
+ next: "qstring"
+ },
+ {
+ token: "number", // hex
+ regex: "0[xX][0-9a-fA-F]+\\b"
+ },
+ {
+ token: "number", // float
+ regex: "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
+ },
+ {
+ token: function(value)
+ {
+ if (keywords[value]) {
+ return "keyword";
+ } else {
+ return "identifier"
+ }
+ },
+ regex: "[a-zA-Z_][a-zA-Z0-9_]*\\b"
+ },
+ {
+ token: function(value) {
+ //return parens[value];
+ return "text";
+ },
+ regex: "[\\[\\]\\(\\)\\{\\}]"
+ },
+ {
+ token: "text",
+ regex: "\\s+"
+ }
+ ],
+ "comment":
+ [
+ {
+ token: "comment", // closing comment
+ regex: ".*?\\*\\/",
+ next: "start"
+ },
+ {
+ token: "comment", // comment spanning whole line
+ regex: ".+"
+ }
+ ],
+ "qqstring":
+ [
+ {
+ token: "string",
+ regex: '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
+ next: "start"
+ },
+ {
+ token: "string",
+ regex: '.+'
+ }
+ ],
+ "qstring":
+ [
+ {
+ token: "string",
+ regex: "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
+ next: "start"
+ },
+ {
+ token: "string",
+ regex: '.+'
+ }
+ ]
+};
+
+})();
\ No newline at end of file
diff --git a/Tokenizer.js b/Tokenizer.js
new file mode 100644
index 00000000..f6cfe194
--- /dev/null
+++ b/Tokenizer.js
@@ -0,0 +1,77 @@
+
+function Tokenizer(rules)
+{
+ this.rules = rules;
+
+ this.regExps = {};
+ for (var key in this.rules)
+ {
+ var state = this.rules[key];
+ var ruleRegExps = [];
+
+ for (var i=0; i < state.length; i++) {
+ ruleRegExps.push(state[i].regex);
+ };
+
+ this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
+ }
+};
+
+Tokenizer.prototype.getLineTokens = function(line, startState)
+{
+ var currentState = startState;
+ var state = this.rules[currentState];
+ var re = this.regExps[currentState];
+ re.lastIndex = 0;
+
+ var match, tokens = [];
+
+ var lastIndex = 0;
+
+ while (match = re.exec(line))
+ {
+ var token = {
+ type: "text",
+ value: match[0]
+ }
+
+ if (re.lastIndex == lastIndex) {
+ throw new Error("tokenizer error")
+ }
+ lastIndex = re.lastIndex;
+
+ //console.log(match);
+
+ for (var i=0; i < state.length; i++)
+ {
+ if (match[i+1])
+ {
+ if (typeof state[i].token == "function") {
+ token.type = state[i].token(match[0]);
+ } else {
+ token.type = state[i].token;
+ }
+
+ if (state[i].next && state[i].next !== currentState)
+ {
+ currentState = state[i].next;
+ var state = this.rules[currentState];
+ var lastIndex = re.lastIndex;
+
+ var re = this.regExps[currentState];
+ re.lastIndex = lastIndex;
+ }
+ break;
+ }
+ };
+
+ tokens.push(token);
+ };
+
+ //console.log(tokens, currentState)
+
+ return {
+ tokens: tokens,
+ state: currentState
+ }
+};
\ No newline at end of file
diff --git a/XML.js b/XML.js
new file mode 100644
index 00000000..cd6b2b02
--- /dev/null
+++ b/XML.js
@@ -0,0 +1,78 @@
+(function() {
+
+window.XML = {};
+
+// regexp must not have capturing parentheses
+// regexps are ordered -> the first match is used
+
+XML.RULES = {
+ start :
+ [
+ {
+ token: "text",
+ regex: "<\\!\\[CDATA\\[",
+ next: "cdata"
+ },
+ {
+ token: "xml_pe",
+ regex: "<\\?.*?\\?>"
+ },
+ {
+ token: "text", // opening tag
+ regex: "<",
+ next: "tag"
+ },
+ {
+ token: "text",
+ regex: "\\s+"
+ },
+ {
+ token: "text",
+ regex: ".+"
+ }
+ ],
+
+ tag:
+ [
+ {
+ token: "text",
+ regex: ">",
+ next: "start"
+ },
+ {
+ token: "keyword",
+ regex: "[-_a-zA-Z0-9:]+"
+ },
+ {
+ token: "text",
+ regex: "\\s+"
+ },
+ {
+ token: "string",
+ regex: '".*?"'
+ },
+ {
+ token: "string",
+ regex: "'.*?'"
+ }
+ ],
+
+ cdata:
+ [
+ {
+ token: "text",
+ regex: "\\]\\]>",
+ next: "start"
+ },
+ {
+ token: "string",
+ regex: "\\s+"
+ },
+ {
+ token: "text",
+ regex: ".+"
+ }
+ ]
+};
+
+})();
\ No newline at end of file
diff --git a/editor.css b/editor.css
index 0b6f789f..2cf1701a 100644
--- a/editor.css
+++ b/editor.css
@@ -85,6 +85,10 @@
color: rgb(0, 0, 205);
}
+.line .xml_pe {
+ color: rgb(104, 104, 91);
+}
+
.marker-layer .selection {
position: absolute;
background: rgba(77, 151, 255, 0.33);
diff --git a/editor.html b/editor.html
index 87c93655..dcd024fd 100644
--- a/editor.html
+++ b/editor.html
@@ -21,7 +21,10 @@
-
+
+
+
+