externalize tokenizer and add XML support

BackgroundTokenizer.js

@@ -1,9 +1,10 @@
-function BackgroundTokenizer(onUpdate, onComplete)
+function BackgroundTokenizer(tokenizer, onUpdate, onComplete)
 {
   this.running = false;
   this.textLines = [];
   this.lines = [];
   this.currentLine = 0;
+  this.tokenizer = tokenizer;
   
   this.onUpdate = onUpdate || function(firstLine, lastLine) {};
   this.onComplete = onComplete || function() {};
@@ -24,7 +25,7 @@ function BackgroundTokenizer(onUpdate, onComplete)
       var line = textLines[self.currentLine];
       
       var state = self.currentLine == 0 ? "start" : self.lines[self.currentLine-1].state;      
-      self.lines[self.currentLine] = getLineTokens(line, state);
+      self.lines[self.currentLine] = self.tokenizer.getLineTokens(line, state);
 
       if ((new Date()-workerStart) > 80)
       {
@@ -58,7 +59,7 @@ BackgroundTokenizer.prototype.start = function(startRow)
   if (!this.running)
   {
     this.running = true;
-    setTimeout(this._worker);
+    setTimeout(this._worker, 50);
   }
 };
 
@@ -75,211 +76,6 @@ BackgroundTokenizer.prototype.getTokens = function(row)
     if (row > 0 && this.lines[row-1]) {
       state = this.lines[row-1].state;
     }
-    return getLineTokens(this.textLines[row] || "", state).tokens;
+    return this.tokenizer.getLineTokens(this.textLines[row] || "", state).tokens;
-  }
-};
-
-var keywords = {
-  "break" : 1,
-  "case" : 1,
-  "catch" : 1,
-  "continue" : 1,
-  "default" : 1,
-  "delete" : 1,
-  "do" : 1,
-  "else" : 1,
-  "finally" : 1,
-  "for" : 1,
-  "function" : 1,
-  "if" : 1,
-  "in" : 1,
-  "instanceof" : 1,
-  "new" : 1,
-  "return" : 1,
-  "switch" : 1,
-  "throw" : 1,
-  "try" : 1,
-  "typeof" : 1,
-  "var" : 1,
-  "while" : 1,
-  "with" : 1
-};
-
-getLineTokens = function(line, startState)
-{
-  // regexp must not have capturing parentheses
-  // regexps are ordered -> the first match is used
-  
-  var rules = {
-    start :
-    [
-      {
-        token: "comment",
-        regex: "\\/\\/.*$"
-      },
-      {
-        token: "comment", // multi line comment in one line
-        regex: "\\/\\*.*?\\*\\/"
-      },      
-      {
-        token: "comment", // multi line comment start
-        regex: "\\/\\*.*$",
-        next: "comment"
-      },
-      {
-        token: "string", // single line
-        regex: '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
-      },
-      {
-        token: "string", // multi line string start
-        regex: '["].*\\\\$',
-        next: "qqstring"
-      },      
-      {
-        token: "string", // single line
-        regex: "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
-      },
-      {
-        token: "string", // multi line string start
-        regex: "['].*\\\\$",
-        next: "qstring"
-      },    
-      {
-        token: "number", // hex
-        regex: "0[xX][0-9a-fA-F]+\\b"
-      },      
-      {
-        token: "number", // float
-        regex: "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
-      },
-      {
-        token: function(value)
-        {
-          if (keywords[value]) {
-            return "keyword";
-          } else {
-            return "identifier"
-          }
-        },
-        regex: "[a-zA-Z_][a-zA-Z0-9_]*\\b"
-      },
-      {
-        token: function(value) {
-          //return parens[value];
-          return "text";
-        },
-        regex: "[\\[\\]\\(\\)\\{\\}]"
-      },
-      {
-        token: "text",
-        regex: "\\s+"
-      }
-    ],
-    "comment":
-    [
-      {
-        token: "comment", // closing comment
-        regex: ".*?\\*\\/",
-        next: "start"
-      },
-      {
-        token: "comment", // comment spanning whole line
-        regex: ".+"
-      }
-    ],
-    "qqstring":
-    [
-      {
-        token: "string",
-        regex: '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
-        next: "start"
-      },
-      {
-        token: "string",
-        regex: '.+'
-      }
-    ],
-    "qstring":
-    [
-      {
-        token: "string",
-        regex: "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
-        next: "start"
-      },
-      {
-        token: "string",
-        regex: '.+'
-      }
-    ]    
-  };
-  
-  var regExps = {};
-  for (var key in rules)
-  {
-    var state = rules[key];
-    var ruleRegExps = [];
-    
-    for (var i=0; i < state.length; i++) {
-      ruleRegExps.push(state[i].regex);
-    };
-    
-    regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
-  }
-
-  
-  var currentState = startState;
-  var state = rules[currentState];  
-  var re = regExps[currentState];
-  re.lastIndex = 0;
-  
-  var match, tokens = [];
-  
-  var lastIndex = 0;
-  
-  while (match = re.exec(line))
-  {
-    var token = {
-      type: "text",
-      value: match[0]
-    }
-    
-    if (re.lastIndex == lastIndex) {
-      throw new Error("tokenizer error")
-    }
-    lastIndex = re.lastIndex;
-    
-    //console.log(match);
-    
-    for (var i=0; i < state.length; i++) 
-    {
-      if (match[i+1])
-      {
-        if (typeof state[i].token == "function") {
-          token.type = state[i].token(match[0]);
-        } else {
-          token.type = state[i].token;
-        }
-        
-        if (state[i].next && state[i].next !== currentState) 
-        {
-          currentState = state[i].next;
-          var state = rules[currentState];  
-          var lastIndex = re.lastIndex;
-          
-          var re = regExps[currentState];
-          re.lastIndex = lastIndex;
-        }
-        break;
-      }
-    };
-    
-    tokens.push(token);
-  };
-  
-  //console.log(tokens, currentState)
-  
-  return {
-    tokens: tokens,
-    state: currentState
   }
 };

Editor.js

@@ -115,7 +115,11 @@ function Editor(doc, renderer)
   doc.addChangeListener(lib.bind(this.onDocumentChange, this));
   renderer.setDocument(doc);
   
-  this.tokenizer = new BackgroundTokenizer(lib.bind(this.onTokenizerUpdate, this));
+  this.tokenizer = new BackgroundTokenizer(
+    new Tokenizer(XML.RULES),
+    lib.bind(this.onTokenizerUpdate, this)
+  );
+  
   this.tokenizer.setLines(doc.lines);
   renderer.setTokenizer(this.tokenizer);
   

JavaScript.js

@@ -0,0 +1,137 @@
+(function() {
+
+window.JavaScript = {};
+
+var keywords = {
+  "break" : 1,
+  "case" : 1,
+  "catch" : 1,
+  "continue" : 1,
+  "default" : 1,
+  "delete" : 1,
+  "do" : 1,
+  "else" : 1,
+  "finally" : 1,
+  "for" : 1,
+  "function" : 1,
+  "if" : 1,
+  "in" : 1,
+  "instanceof" : 1,
+  "new" : 1,
+  "return" : 1,
+  "switch" : 1,
+  "throw" : 1,
+  "try" : 1,
+  "typeof" : 1,
+  "var" : 1,
+  "while" : 1,
+  "with" : 1
+};
+
+// regexp must not have capturing parentheses
+// regexps are ordered -> the first match is used
+
+JavaScript.RULES = {
+  start :
+  [
+    {
+      token: "comment",
+      regex: "\\/\\/.*$"
+    },
+    {
+      token: "comment", // multi line comment in one line
+      regex: "\\/\\*.*?\\*\\/"
+    },      
+    {
+      token: "comment", // multi line comment start
+      regex: "\\/\\*.*$",
+      next: "comment"
+    },
+    {
+      token: "string", // single line
+      regex: '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
+    },
+    {
+      token: "string", // multi line string start
+      regex: '["].*\\\\$',
+      next: "qqstring"
+    },      
+    {
+      token: "string", // single line
+      regex: "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
+    },
+    {
+      token: "string", // multi line string start
+      regex: "['].*\\\\$",
+      next: "qstring"
+    },    
+    {
+      token: "number", // hex
+      regex: "0[xX][0-9a-fA-F]+\\b"
+    },      
+    {
+      token: "number", // float
+      regex: "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
+    },
+    {
+      token: function(value)
+      {
+        if (keywords[value]) {
+          return "keyword";
+        } else {
+          return "identifier"
+        }
+      },
+      regex: "[a-zA-Z_][a-zA-Z0-9_]*\\b"
+    },
+    {
+      token: function(value) {
+        //return parens[value];
+        return "text";
+      },
+      regex: "[\\[\\]\\(\\)\\{\\}]"
+    },
+    {
+      token: "text",
+      regex: "\\s+"
+    }
+  ],
+  "comment":
+  [
+    {
+      token: "comment", // closing comment
+      regex: ".*?\\*\\/",
+      next: "start"
+    },
+    {
+      token: "comment", // comment spanning whole line
+      regex: ".+"
+    }
+  ],
+  "qqstring":
+  [
+    {
+      token: "string",
+      regex: '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
+      next: "start"
+    },
+    {
+      token: "string",
+      regex: '.+'
+    }
+  ],
+  "qstring":
+  [
+    {
+      token: "string",
+      regex: "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
+      next: "start"
+    },
+    {
+      token: "string",
+      regex: '.+'
+    }
+  ]    
+};
+
+})();

Tokenizer.js

@@ -0,0 +1,77 @@
+
+function Tokenizer(rules)
+{
+  this.rules = rules;
+  
+  this.regExps = {};
+  for (var key in this.rules)
+  {
+    var state = this.rules[key];
+    var ruleRegExps = [];
+    
+    for (var i=0; i < state.length; i++) {
+      ruleRegExps.push(state[i].regex);
+    };
+    
+    this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
+  }
+};
+
+Tokenizer.prototype.getLineTokens = function(line, startState)
+{
+  var currentState = startState;
+  var state = this.rules[currentState];  
+  var re = this.regExps[currentState];
+  re.lastIndex = 0;
+  
+  var match, tokens = [];
+  
+  var lastIndex = 0;
+  
+  while (match = re.exec(line))
+  {
+    var token = {
+      type: "text",
+      value: match[0]
+    }
+    
+    if (re.lastIndex == lastIndex) {
+      throw new Error("tokenizer error")
+    }
+    lastIndex = re.lastIndex;
+    
+    //console.log(match);
+    
+    for (var i=0; i < state.length; i++) 
+    {
+      if (match[i+1])
+      {
+        if (typeof state[i].token == "function") {
+          token.type = state[i].token(match[0]);
+        } else {
+          token.type = state[i].token;
+        }
+        
+        if (state[i].next && state[i].next !== currentState) 
+        {
+          currentState = state[i].next;
+          var state = this.rules[currentState];  
+          var lastIndex = re.lastIndex;
+          
+          var re = this.regExps[currentState];
+          re.lastIndex = lastIndex;
+        }
+        break;
+      }
+    };
+    
+    tokens.push(token);
+  };
+  
+  //console.log(tokens, currentState)
+  
+  return {
+    tokens: tokens,
+    state: currentState
+  }
+};

XML.js

@@ -0,0 +1,78 @@
+(function() {
+
+window.XML = {};
+
+// regexp must not have capturing parentheses
+// regexps are ordered -> the first match is used
+
+XML.RULES = {
+  start :
+  [
+    {
+      token: "text",
+      regex: "<\\!\\[CDATA\\[",
+      next: "cdata"
+    },
+    {
+      token: "xml_pe",
+      regex: "<\\?.*?\\?>"
+    },
+    {
+      token: "text", // opening tag
+      regex: "<",
+      next: "tag"
+    },
+    {
+      token: "text",
+      regex: "\\s+"
+    },
+    {
+      token: "text",
+      regex: ".+"
+    }    
+  ],
+  
+  tag:
+  [
+    {
+      token: "text",
+      regex: ">",
+      next: "start"
+    },
+    {
+      token: "keyword",
+      regex: "[-_a-zA-Z0-9:]+"
+    },    
+    {
+      token: "text",
+      regex: "\\s+"
+    },
+    {
+      token: "string",
+      regex: '".*?"'
+    },
+    {
+      token: "string",
+      regex: "'.*?'"
+    }
+  ],
+  
+  cdata:
+  [
+    {
+      token: "text",
+      regex: "\\]\\]>",
+      next: "start"
+    },
+    {
+      token: "string",
+      regex: "\\s+"
+    },
+    {
+      token: "text",
+      regex: ".+"
+    }
+  ]
+};
+
+})();

editor.css

@@ -85,6 +85,10 @@
   color: rgb(0, 0, 205);
 }
 
+.line .xml_pe {
+  color: rgb(104, 104, 91);
+}
+
 .marker-layer .selection {
   position: absolute;
   background: rgba(77, 151, 255, 0.33);

editor.html

@@ -21,7 +21,10 @@
 	
 	<script src="lib.js" type="text/javascript" charset="utf-8"></script>
 	<script src="TextDocument.js" type="text/javascript" charset="utf-8"></script>
-	<script src="BackgroundTokenizer.js" type="text/javascript" charset="utf-8"></script>		
+	<script src="JavaScript.js" type="text/javascript" charset="utf-8"></script>
+	<script src="XML.js" type="text/javascript" charset="utf-8"></script>
+	<script src="Tokenizer.js" type="text/javascript" charset="utf-8"></script>
+	<script src="BackgroundTokenizer.js" type="text/javascript" charset="utf-8"></script>
 	<script src="CursorLayer.js" type="text/javascript" charset="utf-8"></script>	
 	<script src="GutterLayer.js" type="text/javascript" charset="utf-8"></script>	
 	<script src="TextLayer.js" type="text/javascript" charset="utf-8"></script>