From 1683f3e90f977f6bd31fdfaf673f892c2f83ed85 Mon Sep 17 00:00:00 2001
From: Fabian Jakobs <fabian.jakobs@web.de>
Date: Thu, 8 Apr 2010 18:46:43 +0200
Subject: [PATCH] Add support for background tokenizing

---
 BackgroundTokenizer.js     | 134 +++++++++++++++++++++++++++++++++++++
 Editor.js                  |  15 ++++-
 TextLayer.js               |   9 ++-
 VirtualRenderer.js         |   5 +-
 editor.html                |   3 +-
 experiments/tokenizer.html |  39 +++++++++++
 6 files changed, 196 insertions(+), 9 deletions(-)
 create mode 100644 BackgroundTokenizer.js
 create mode 100644 experiments/tokenizer.html

diff --git a/BackgroundTokenizer.js b/BackgroundTokenizer.js
new file mode 100644
index 00000000..91d89aec
--- /dev/null
+++ b/BackgroundTokenizer.js
@@ -0,0 +1,134 @@
+function BackgroundTokenizer(onUpdate, onComplete)
+{
+  this.running = false;
+  this.textLines = [];
+  this.lines = [];
+  this.currentLine = 0;
+  
+  this.onUpdate = onUpdate || function(firstLine, lastLine) {};
+  this.onComplete = onComplete || function() {};
+  
+  var self = this;
+  this._worker = function() 
+  {    
+    if (!self.running) {
+      return;
+    }
+
+    var workerStart = new Date();
+    var startLine = self.currentLine;
+    var textLines = self.textLines;
+
+    while (self.currentLine < textLines.length)
+    {
+      var line = textLines[self.currentLine];
+      
+      var state = self.currentLine == 0 ? "start" : self.lines[self.currentLine-1].state;      
+      self.lines[self.currentLine] = getLineTokens(line, state);
+
+      if ((new Date()-workerStart) > 80)
+      {
+        self.onUpdate(startLine, self.currentLine);
+        return setTimeout(self._worker, 20);
+      }
+      
+      self.currentLine++;      
+    }
+
+    self.running = false;
+    
+    self.onUpdate(startLine, textLines.length-1);
+    self.onComplete();    
+  }
+};
+
+BackgroundTokenizer.prototype.setLines = function(textLines)
+{
+  this.textLines = textLines;
+  this.lines = [];
+  
+  this.stop();    
+};
+
+BackgroundTokenizer.prototype.start = function(startRow)
+{
+  this.currentLine = Math.min(startRow || 0, this.currentLine, this.textLines.length);
+  this.lines.splice(startRow, this.lines.length);
+
+  if (!this.running)
+  {
+    this.running = true;
+    setTimeout(this._worker);
+  }
+};
+
+BackgroundTokenizer.prototype.stop = function() {
+  this.running = false;
+};
+
+BackgroundTokenizer.prototype.getTokens = function(row)
+{
+  if (this.lines[row]) {
+    return this.lines[row].tokens;
+  } else {
+    return getLineTokens(this.textLines[row] || "", "start").tokens;
+  }
+};
+
+var keywords = {
+  "break" : 1,
+  "case" : 1,
+  "catch" : 1,
+  "continue" : 1,
+  "default" : 1,
+  "delete" : 1,
+  "do" : 1,
+  "else" : 1,
+  "finally" : 1,
+  "for" : 1,
+  "function" : 1,
+  "if" : 1,
+  "in" : 1,
+  "instanceof" : 1,
+  "new" : 1,
+  "return" : 1,
+  "switch" : 1,
+  "throw" : 1,
+  "try" : 1,
+  "typeof" : 1,
+  "var" : 1,
+  "while" : 1,
+  "with" : 1
+};
+
+getLineTokens = function(line, state)
+{
+  var tokens = [];
+  
+  var re = /(?:(\s+)|("[^"]*")|('[^']*')|([\[\]\(\)\{\}])|([a-zA-Z_][a-zA-Z0-9_]*)|(\/\/.*)|(.))/g
+  re.lastIndex = 0;
+
+  var match;
+  while (match = re.exec(line))
+  {
+    var token = {
+      type: "text",
+      value: match[0]
+    }
+    
+    if (match[2] || match[3]) {
+      token.type = "string";
+    } else if (match[5] && keywords[match[5]]) {
+      token.type = "keyword";
+    } else if (match[6]) {
+      token.type = "comment";
+    }
+    
+    tokens.push(token);
+  };
+  
+  return {
+    tokens: tokens,
+    state: "start"
+  }
+};
\ No newline at end of file
diff --git a/Editor.js b/Editor.js
index 3ffa7444..32e97451 100644
--- a/Editor.js
+++ b/Editor.js
@@ -115,6 +115,10 @@ function Editor(doc, renderer)
   doc.addChangeListener(lib.bind(this.onDocumentChange, this));
   renderer.setDocument(doc);
   
+  this.tokenizer = new BackgroundTokenizer(lib.bind(this.onTokenizerUpdate, this));
+  this.tokenizer.setLines(doc.lines);
+  renderer.setTokenizer(this.tokenizer);
+  
   this.cursor = {
     row: 0,
     column: 0
@@ -147,9 +151,16 @@ function Editor(doc, renderer)
     this.renderer.visualizeBlur();
   },
   
-  onDocumentChange : function(startRow, endRow) {
+  onDocumentChange : function(startRow, endRow)
+  {
+    this.tokenizer.start(startRow);
     this.renderer.updateLines(startRow, endRow);
-  },  
+  },
+  
+  onTokenizerUpdate : function(startRow, endRow) {
+    console.log("token update", startRow, endRow);
+    this.renderer.updateLines(startRow, endRow);
+  },
   
   onMouseDown : function(e) 
   {
diff --git a/TextLayer.js b/TextLayer.js
index 918b20cd..dd4727bc 100644
--- a/TextLayer.js
+++ b/TextLayer.js
@@ -7,9 +7,8 @@ function TextLayer(parentEl)
   this._measureSizes();
 }
 
-TextLayer.prototype.setDocument = function(doc) {
-  this.lines = doc.lines;
-  this.doc = doc;
+TextLayer.prototype.setTokenizer = function(tokenizer) {
+  this.tokenizer = tokenizer;
 };
 
 TextLayer.prototype.getLineHeight = function() {
@@ -62,7 +61,7 @@ TextLayer.prototype.updateLines = function(layerConfig, firstRow, lastRow)
 TextLayer.prototype.update = function(config)
 {
   var html = [];
-  for (var i=config.firstRow; i<config.lastRow; i++)
+  for (var i=config.firstRow; i<=config.lastRow; i++)
   {
     html.push(
       "<div class='line ",
@@ -79,7 +78,7 @@ TextLayer.prototype.update = function(config)
 
 TextLayer.prototype.renderLine = function(stringBuilder, row)
 {
-  var tokens = this.doc.getLineTokens(row);
+  var tokens = this.tokenizer.getTokens(row);
   for (var i=0; i < tokens.length; i++)
   {
     var token = tokens[i];
diff --git a/VirtualRenderer.js b/VirtualRenderer.js
index 594f7d5e..385fcecd 100644
--- a/VirtualRenderer.js
+++ b/VirtualRenderer.js
@@ -36,7 +36,10 @@ VirtualRenderer.prototype.setDocument = function(doc)
 {
   this.lines = doc.lines;
   this.doc = doc;
-  this.textLayer.setDocument(doc);
+};
+
+VirtualRenderer.prototype.setTokenizer = function(tokenizer) {
+  this.textLayer.setTokenizer(tokenizer);
 };
 
 VirtualRenderer.prototype.getContainerElement = function() {
diff --git a/editor.html b/editor.html
index 58ea2c92..87c93655 100644
--- a/editor.html
+++ b/editor.html
@@ -20,7 +20,8 @@
 	<link rel="stylesheet" href="editor.css" type="text/css" charset="utf-8">
 	
 	<script src="lib.js" type="text/javascript" charset="utf-8"></script>
-	<script src="TextDocument.js" type="text/javascript" charset="utf-8"></script>	
+	<script src="TextDocument.js" type="text/javascript" charset="utf-8"></script>
+	<script src="BackgroundTokenizer.js" type="text/javascript" charset="utf-8"></script>		
 	<script src="CursorLayer.js" type="text/javascript" charset="utf-8"></script>	
 	<script src="GutterLayer.js" type="text/javascript" charset="utf-8"></script>	
 	<script src="TextLayer.js" type="text/javascript" charset="utf-8"></script>	
diff --git a/experiments/tokenizer.html b/experiments/tokenizer.html
new file mode 100644
index 00000000..adcac8f2
--- /dev/null
+++ b/experiments/tokenizer.html
@@ -0,0 +1,39 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+   "http://www.w3.org/TR/html4/strict.dtd">
+
+<html lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+	<title>BackgroundTokenizer</title>
+	<meta name="author" content="Fabian Jakobs">
+</head>
+<body>
+
+<textarea id="text"></textarea>
+<input type="button" value="tokenize" id="go">
+
+<script src="../BackgroundTokenizer.js" type="text/javascript" charset="utf-8"></script>
+<script type="text/javascript" charset="utf-8">
+
+var button = document.getElementById("go");
+var text = document.getElementById("text");
+
+button.onclick = function() 
+{
+  var onComplete = function() {
+    console.log("complete");
+  };
+
+  var onUpdate = function(firstLine, lastLine) {
+    console.log("update", firstLine, lastLine);
+  };
+  
+  var tokenizer = new BackgroundTokenizer(onUpdate, onComplete);
+  tokenizer.setLines(text.value.split(/[\n\r]/));
+  tokenizer.start();
+}
+
+</script>
+
+</body>
+</html>