jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve bgTokenizer

Browse source
This commit is contained in:
nightwing 2012-05-24 22:07:27 +04:00
commit 3adcd1e5f8
10 changed files with 103 additions and 108 deletions
Download patch file Download diff file Expand all files Collapse all files

124
lib/ace/background_tokenizer.js
View file

@ -41,6 +41,9 @@ define(function(require, exports, module) {
var oop = require("./lib/oop"); var oop = require("./lib/oop");
var EventEmitter = require("./lib/event_emitter").EventEmitter; var EventEmitter = require("./lib/event_emitter").EventEmitter;
// tokenizing lines longer than this makes editor very slow
var MAX_LINE_LENGTH = 5000;
/** /**
* class BackgroundTokenizer * class BackgroundTokenizer
* *
@ -61,6 +64,7 @@ var EventEmitter = require("./lib/event_emitter").EventEmitter;
var BackgroundTokenizer = function(tokenizer, editor) { var BackgroundTokenizer = function(tokenizer, editor) {
this.running = false; this.running = false;
this.lines = []; this.lines = [];
this.states = [];
this.currentLine = 0; this.currentLine = 0;
this.tokenizer = tokenizer; this.tokenizer = tokenizer;
@ -77,11 +81,12 @@ var BackgroundTokenizer = function(tokenizer, editor) {
var len = doc.getLength(); var len = doc.getLength();
while (self.currentLine < len) { while (self.currentLine < len) {
self.lines[self.currentLine] = self.$tokenizeRows(self.currentLine, self.currentLine)[0]; self.$tokenizeRow(self.currentLine);
self.currentLine++; while (self.lines[self.currentLine])
self.currentLine++;
// only check every 5 lines // only check every 5 lines
processedLines += 1; processedLines ++;
if ((processedLines % 5 == 0) && (new Date() - workerStart) > 20) { if ((processedLines % 5 == 0) && (new Date() - workerStart) > 20) {
self.fireUpdateEvent(startLine, self.currentLine-1); self.fireUpdateEvent(startLine, self.currentLine-1);
self.running = setTimeout(self.$worker, 20); self.running = setTimeout(self.$worker, 20);
@ -106,10 +111,10 @@ var BackgroundTokenizer = function(tokenizer, editor) {
* Sets a new tokenizer for this object. * Sets a new tokenizer for this object.
* *
**/ **/
this.setTokenizer = function(tokenizer) { this.setTokenizer = function(tokenizer) {
this.tokenizer = tokenizer; this.tokenizer = tokenizer;
this.lines = []; this.lines = [];
this.states = [];
this.start(0); this.start(0);
}; };
@ -121,10 +126,10 @@ var BackgroundTokenizer = function(tokenizer, editor) {
* Sets a new document to associate with this object. * Sets a new document to associate with this object.
* *
**/ **/
this.setDocument = function(doc) { this.setDocument = function(doc) {
this.doc = doc; this.doc = doc;
this.lines = []; this.lines = [];
this.states = [];
this.stop(); this.stop();
}; };
@ -137,7 +142,6 @@ var BackgroundTokenizer = function(tokenizer, editor) {
* Emits the `'update'` event. `firstRow` and `lastRow` are used to define the boundaries of the region to be updated. * Emits the `'update'` event. `firstRow` and `lastRow` are used to define the boundaries of the region to be updated.
* *
**/ **/
this.fireUpdateEvent = function(firstRow, lastRow) { this.fireUpdateEvent = function(firstRow, lastRow) {
var data = { var data = {
first: firstRow, first: firstRow,
@ -153,13 +157,36 @@ var BackgroundTokenizer = function(tokenizer, editor) {
* Starts tokenizing at the row indicated. * Starts tokenizing at the row indicated.
* *
**/ **/
this.start = function(startRow) { this.start = function(startRow) {
this.currentLine = Math.min(startRow || 0, this.currentLine, this.currentLine = Math.min(startRow || 0, this.currentLine, this.doc.getLength());
this.doc.getLength());
// remove all cached items below this line // remove all cached items below this line
this.lines.splice(this.currentLine, this.lines.length); this.lines.splice(this.currentLine, this.lines.length);
this.states.splice(this.currentLine, this.states.length);
this.stop();
// pretty long delay to prevent the tokenizer from interfering with the user
this.running = setTimeout(this.$worker, 700);
};
this.$updateOnChange = function(delta) {
var range = delta.range;
var startRow = range.start.row;
var len = range.end.row - startRow;
if (len === 0) {
this.lines[startRow] = null;
} else if (delta.action == "removeText" || delta.action == "removeLines") {
this.lines.splice(startRow, len + 1, null);
this.states.splice(startRow, len + 1, null);
} else {
var args = Array(len + 1);
args.unshift(startRow, 1);
this.lines.splice.apply(this.lines, args);
this.states.splice.apply(this.states, args);
}
this.currentLine = Math.min(startRow, this.currentLine, this.doc.getLength());
this.stop(); this.stop();
// pretty long delay to prevent the tokenizer from interfering with the user // pretty long delay to prevent the tokenizer from interfering with the user
@ -172,7 +199,6 @@ var BackgroundTokenizer = function(tokenizer, editor) {
* Stops tokenizing. * Stops tokenizing.
* *
**/ **/
this.stop = function() { this.stop = function() {
if (this.running) if (this.running)
clearTimeout(this.running); clearTimeout(this.running);
@ -187,71 +213,49 @@ var BackgroundTokenizer = function(tokenizer, editor) {
* Starts tokenizing at the row indicated. Returns a list of objects of the tokenized rows. * Starts tokenizing at the row indicated. Returns a list of objects of the tokenized rows.
* *
**/ **/
this.getTokens = function(row) {
this.getTokens = function(firstRow, lastRow) { return this.lines[row] || this.$tokenizeRow(row);
return this.$tokenizeRows(firstRow, lastRow);
}; };
/** /**
* BackgroundTokenizer.getState(row) -> String * BackgroundTokenizer.getState(row) -> String
* - row (Number): The row to start at * - row (Number): The row to start at
* *
* [Returns the state of tokenization for a row.]{: #BackgroundTokenizer.getState} * [Returns the state of tokenization at the end of a row.]{: #BackgroundTokenizer.getState}
*
**/ **/
this.getState = function(row) { this.getState = function(row) {
return this.$tokenizeRows(row, row)[0].state; if (this.currentLine == row)
this.$tokenizeRow(row);
return this.states[row] || "start";
}; };
/** this.$tokenizeRow = function(row) {
* BackgroundTokenizer.$tokenizeRows(firstRow, lastRow) -> [Object] var line = this.doc.getLine(row);
* - startRow (Number): The row to start at var state = this.states[row - 1];
* - lastRow (Number): The row to finish at
* + ([Object]): A list of the tokenized rows. Each item in the list is an object with two properties, `state` and `start`.
*
* Tokenizes all the rows within the specified region.
*
*
**/
this.$tokenizeRows = function(firstRow, lastRow) {
if (!this.doc || isNaN(firstRow) || isNaN(lastRow))
return [{'state':'start','tokens':[]}];
var rows = []; if (line.length > MAX_LINE_LENGTH) {
var overflow = {value: line.substr(MAX_LINE_LENGTH), type: "text"};
// determine start state line = line.slice(0, MAX_LINE_LENGTH);
var state = "start"; }
var doCache = false; var data = this.tokenizer.getLineTokens(line, state);
if (firstRow > 0 && this.lines[firstRow - 1]) { if (overflow) {
state = this.lines[firstRow - 1].state; data.tokens.push(overflow);
doCache = true; data.state = null;
} else if (firstRow == 0) {
state = "start";
doCache = true;
} else if (this.lines.length > 0) {
// Guess that we haven't changed state.
state = this.lines[this.lines.length-1].state;
} }
var lines = this.doc.getLines(firstRow, lastRow); if (data.state == "start" && this.states[row] == null)
for (var row=firstRow; row<=lastRow; row++) { this.states[row] = "start";
if (!this.lines[row]) {
var tokens = this.tokenizer.getLineTokens(lines[row-firstRow] || "", state);
var state = tokens.state;
rows.push(tokens);
if (doCache) { if (this.states[row] !== data.state) {
this.lines[row] = tokens; this.states[row] = data.state;
} this.lines[row + 1] = null;
} if (this.currentLine > row + 1)
else { this.currentLine = row + 1;
var tokens = this.lines[row]; } else if (this.currentLine == row) {
state = tokens.state; this.currentLine = row + 1;
rows.push(tokens);
}
} }
return rows;
return this.lines[row] = data.tokens;
}; };
}).call(BackgroundTokenizer.prototype); }).call(BackgroundTokenizer.prototype);

13
lib/ace/edit_session.js
View file

@ -188,7 +188,7 @@ var EditSession = function(text, mode) {
this.$informUndoManager.schedule(); this.$informUndoManager.schedule();
} }
this.bgTokenizer.start(delta.range.start.row); this.bgTokenizer.$updateOnChange(delta);
this._emit("change", e); this._emit("change", e);
}; };
@ -249,15 +249,14 @@ var EditSession = function(text, mode) {
}; };
/** related to: BackgroundTokenizer.getTokens /** related to: BackgroundTokenizer.getTokens
* EditSession.getTokens(firstRow, lastRow) -> Array * EditSession.getTokens(row) -> Array
* - firstRow (Number): The row to start at * - row (Number): The row to start at
* - lastRow (Number): The row to finish at
* *
* Starts tokenizing at the row indicated. Returns a list of objects of the tokenized rows. * Starts tokenizing at the row indicated. Returns a list of objects of the tokenized rows.
* *
**/ **/
this.getTokens = function(firstRow, lastRow) { this.getTokens = function(row) {
return this.bgTokenizer.getTokens(firstRow, lastRow); return this.bgTokenizer.getTokens(row);
}; };
/** /**
@ -268,7 +267,7 @@ var EditSession = function(text, mode) {
* Returns an array of tokens at the indicated row and column. * Returns an array of tokens at the indicated row and column.
**/ **/
this.getTokenAt = function(row, column) { this.getTokenAt = function(row, column) {
var tokens = this.bgTokenizer.getTokens(row, row)[0].tokens; var tokens = this.bgTokenizer.getTokens(row);
var token, c = 0; var token, c = 0;
if (column == null) { if (column == null) {
i = tokens.length - 1; i = tokens.length - 1;

3
lib/ace/ext/static_highlight.js
View file

@ -70,10 +70,9 @@ exports.render = function(input, mode, theme, lineStart) {
var stringBuilder = []; var stringBuilder = [];
var length = session.getLength(); var length = session.getLength();
var tokens = session.getTokens(0, length - 1);
for(var ix = 0; ix < length; ix++) { for(var ix = 0; ix < length; ix++) {
var lineTokens = tokens[ix].tokens; var lineTokens = session.getTokens(ix);
stringBuilder.push("<div class='ace_line'>"); stringBuilder.push("<div class='ace_line'>");
stringBuilder.push("<span class='ace_gutter ace_gutter-cell' unselectable='on'>" + (ix + lineStart) + "</span>"); stringBuilder.push("<span class='ace_gutter ace_gutter-cell' unselectable='on'>" + (ix + lineStart) + "</span>");
textLayer.$renderLine(stringBuilder, 0, lineTokens, true); textLayer.$renderLine(stringBuilder, 0, lineTokens, true);

27
lib/ace/layer/text.js
View file

@ -259,8 +259,8 @@ var Text = function(parentEl) {
continue; continue;
var html = []; var html = [];
var tokens = this.session.getTokens(i, i); var tokens = this.session.getTokens(i);
this.$renderLine(html, i, tokens[0].tokens, !this.$useLineGroups()); this.$renderLine(html, i, tokens, !this.$useLineGroups());
lineElement = dom.setInnerHtml(lineElement, html.join("")); lineElement = dom.setInnerHtml(lineElement, html.join(""));
i = this.session.getRowFoldEnd(i); i = this.session.getRowFoldEnd(i);
@ -321,11 +321,8 @@ var Text = function(parentEl) {
var html = []; var html = [];
// Get the tokens per line as there might be some lines in between // Get the tokens per line as there might be some lines in between
// beeing folded. // beeing folded.
// OPTIMIZE: If there is a long block of unfolded lines, just make var tokens = this.session.getTokens(row);
// this call once for that big block of unfolded lines. this.$renderLine(html, row, tokens, false);
var tokens = this.session.getTokens(row, row);
if (tokens.length == 1)
this.$renderLine(html, row, tokens[0].tokens, false);
// don't use setInnerHtml since we are working with an empty DIV // don't use setInnerHtml since we are working with an empty DIV
container.innerHTML = html.join(""); container.innerHTML = html.join("");
@ -368,11 +365,8 @@ var Text = function(parentEl) {
// Get the tokens per line as there might be some lines in between // Get the tokens per line as there might be some lines in between
// beeing folded. // beeing folded.
// OPTIMIZE: If there is a long block of unfolded lines, just make var tokens = this.session.getTokens(row);
// this call once for that big block of unfolded lines. this.$renderLine(html, row, tokens, false);
var tokens = this.session.getTokens(row, row);
if (tokens.length == 1)
this.$renderLine(html, row, tokens[0].tokens, false);
if (this.$useLineGroups()) if (this.$useLineGroups())
html.push("</div>"); // end the line group html.push("</div>"); // end the line group
@ -567,12 +561,11 @@ var Text = function(parentEl) {
value: placeholder value: placeholder
}); });
} else { } else {
if (isNewRow) { if (isNewRow)
tokens = this.session.getTokens(row, row)[0].tokens; tokens = this.session.getTokens(row);
}
if (tokens.length != 0) { if (tokens.length)
addTokens(tokens, lastColumn, column); addTokens(tokens, lastColumn, column);
}
} }
}.bind(this), foldLine.end.row, this.session.getLine(foldLine.end.row).length); }.bind(this), foldLine.end.row, this.session.getLine(foldLine.end.row).length);

6
lib/ace/layer/text_test.js
View file

@ -68,13 +68,13 @@ module.exports = {
// row with hard tabs // row with hard tabs
var row = 0; var row = 0;
var tokens = this.session.getTokens(row, row)[0].tokens; var tokens = this.session.getTokens(row);
var stringBuilder = []; var stringBuilder = [];
this.textLayer.$renderLine(stringBuilder, row, tokens); this.textLayer.$renderLine(stringBuilder, row, tokens);
// row with soft tabs // row with soft tabs
row = 1; row = 1;
tokens = this.session.getTokens(row, row)[0].tokens; tokens = this.session.getTokens(row);
var stringBuilder2 = []; var stringBuilder2 = [];
this.textLayer.$renderLine(stringBuilder2, row, tokens); this.textLayer.$renderLine(stringBuilder2, row, tokens);
assert.equal(stringBuilder.join(""), stringBuilder2.join("")); assert.equal(stringBuilder.join(""), stringBuilder2.join(""));
@ -83,7 +83,7 @@ module.exports = {
"test rendering width of ideographic space (U+3000)" : function() { "test rendering width of ideographic space (U+3000)" : function() {
this.session.setValue("\u3000"); this.session.setValue("\u3000");
var tokens = this.session.getTokens(0, 0)[0].tokens; var tokens = this.session.getTokens(0);
var stringBuilder = []; var stringBuilder = [];
this.textLayer.$renderLine(stringBuilder, 0, tokens, true); this.textLayer.$renderLine(stringBuilder, 0, tokens, true);
assert.equal(stringBuilder.join(""), "<span class='ace_cjk' style='width:20px'></span>"); assert.equal(stringBuilder.join(""), "<span class='ace_cjk' style='width:20px'></span>");

2
lib/ace/mode/behaviour/cstyle.js
View file

@ -168,7 +168,7 @@ var CstyleBehaviour = function () {
} }
// Find what token we're inside. // Find what token we're inside.
var tokens = session.getTokens(selection.start.row, selection.start.row)[0].tokens; var tokens = session.getTokens(selection.start.row);
var col = 0, token; var col = 0, token;
var quotepos = -1; // Track whether we're inside an open quote. var quotepos = -1; // Track whether we're inside an open quote.

2
lib/ace/mode/folding/xml.js
View file

@ -71,7 +71,7 @@ oop.inherits(FoldMode, BaseFoldMode);
}; };
this._getFirstTagInLine = function(session, row) { this._getFirstTagInLine = function(session, row) {
var tokens = session.getTokens(row, row)[0].tokens; var tokens = session.getTokens(row);
var value = ""; var value = "";
for (var i = 0; i < tokens.length; i++) { for (var i = 0; i < tokens.length; i++) {
var token = tokens[i]; var token = tokens[i];

6
lib/ace/token_iterator.js
View file

@ -58,7 +58,7 @@ define(function(require, exports, module) {
var TokenIterator = function(session, initialRow, initialColumn) { var TokenIterator = function(session, initialRow, initialColumn) {
this.$session = session; this.$session = session;
this.$row = initialRow; this.$row = initialRow;
this.$rowTokens = session.getTokens(initialRow, initialRow)[0].tokens; this.$rowTokens = session.getTokens(initialRow);
var token = session.getTokenAt(initialRow, initialColumn); var token = session.getTokenAt(initialRow, initialColumn);
this.$tokenIndex = token ? token.index : -1; this.$tokenIndex = token ? token.index : -1;
@ -82,7 +82,7 @@ var TokenIterator = function(session, initialRow, initialColumn) {
return null; return null;
} }
this.$rowTokens = this.$session.getTokens(this.$row, this.$row)[0].tokens; this.$rowTokens = this.$session.getTokens(this.$row);
this.$tokenIndex = this.$rowTokens.length - 1; this.$tokenIndex = this.$rowTokens.length - 1;
} }
@ -105,7 +105,7 @@ var TokenIterator = function(session, initialRow, initialColumn) {
return null; return null;
} }
this.$rowTokens = this.$session.getTokens(this.$row, this.$row)[0].tokens; this.$rowTokens = this.$session.getTokens(this.$row);
this.$tokenIndex = 0; this.$tokenIndex = 0;
} }

12
lib/ace/token_iterator_test.js
View file

@ -145,10 +145,10 @@ module.exports = {
]; ];
var session = new EditSession(lines.join("\n"), new JavaScriptMode()); var session = new EditSession(lines.join("\n"), new JavaScriptMode());
var rows = session.getTokens(0, lines.length-1);
var tokens = []; var tokens = [];
for (var i = 0; i < rows.length; i++) var len = session.getLength();
tokens = tokens.concat(rows[i].tokens); for (var i = 0; i < len; i++)
tokens = tokens.concat(session.getTokens(i));
var iterator = new TokenIterator(session, 0, 0); var iterator = new TokenIterator(session, 0, 0);
for (var i = 1; i < tokens.length; i++) for (var i = 1; i < tokens.length; i++)
@ -167,10 +167,10 @@ module.exports = {
]; ];
var session = new EditSession(lines.join("\n"), new JavaScriptMode()); var session = new EditSession(lines.join("\n"), new JavaScriptMode());
var rows = session.getTokens(0, lines.length-1);
var tokens = []; var tokens = [];
for (var i = 0; i < rows.length; i++) var len = session.getLength();
tokens = tokens.concat(rows[i].tokens); for (var i = 0; i < len; i++)
tokens = tokens.concat(session.getTokens(i));
var iterator = new TokenIterator(session, 4, 0); var iterator = new TokenIterator(session, 4, 0);
for (var i = tokens.length-2; i >= 0; i--) for (var i = tokens.length-2; i >= 0; i--)

2
lib/ace/tokenizer.js
View file

@ -104,7 +104,7 @@ var Tokenizer = function(rules, flag) {
* Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state. * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
**/ **/
this.getLineTokens = function(line, startState) { this.getLineTokens = function(line, startState) {
var currentState = startState; var currentState = startState || "start";
var state = this.rules[currentState]; var state = this.rules[currentState];
var mapping = this.matchMappings[currentState]; var mapping = this.matchMappings[currentState];
var re = this.regExps[currentState]; var re = this.regExps[currentState];