jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

externalize tokenizer and add XML support

Browse source
This commit is contained in:
Fabian Jakobs 2010-04-09 14:10:31 +02:00
commit 07dc92addb
7 changed files with 310 additions and 211 deletions
Download patch file Download diff file Expand all files Collapse all files

214
BackgroundTokenizer.js
View file

@ -1,9 +1,10 @@
function BackgroundTokenizer(onUpdate, onComplete)
function BackgroundTokenizer(tokenizer, onUpdate, onComplete)
{
this.running = false;
this.textLines = [];
this.lines = [];
this.currentLine = 0;
this.tokenizer = tokenizer;
this.onUpdate = onUpdate || function(firstLine, lastLine) {};
this.onComplete = onComplete || function() {};
@ -24,7 +25,7 @@ function BackgroundTokenizer(onUpdate, onComplete)
var line = textLines[self.currentLine];
var state = self.currentLine == 0 ? "start" : self.lines[self.currentLine-1].state;
self.lines[self.currentLine] = getLineTokens(line, state);
self.lines[self.currentLine] = self.tokenizer.getLineTokens(line, state);
if ((new Date()-workerStart) > 80)
{
@ -58,7 +59,7 @@ BackgroundTokenizer.prototype.start = function(startRow)
if (!this.running)
{
this.running = true;
setTimeout(this._worker);
setTimeout(this._worker, 50);
}
};
@ -75,211 +76,6 @@ BackgroundTokenizer.prototype.getTokens = function(row)
if (row > 0 && this.lines[row-1]) {
state = this.lines[row-1].state;
}
return getLineTokens(this.textLines[row] || "", state).tokens;
}
};
var keywords = {
"break" : 1,
"case" : 1,
"catch" : 1,
"continue" : 1,
"default" : 1,
"delete" : 1,
"do" : 1,
"else" : 1,
"finally" : 1,
"for" : 1,
"function" : 1,
"if" : 1,
"in" : 1,
"instanceof" : 1,
"new" : 1,
"return" : 1,
"switch" : 1,
"throw" : 1,
"try" : 1,
"typeof" : 1,
"var" : 1,
"while" : 1,
"with" : 1
};
getLineTokens = function(line, startState)
{
// regexp must not have capturing parentheses
// regexps are ordered -> the first match is used
var rules = {
start :
[
{
token: "comment",
regex: "\\/\\/.*$"
},
{
token: "comment", // multi line comment in one line
regex: "\\/\\*.*?\\*\\/"
},
{
token: "comment", // multi line comment start
regex: "\\/\\*.*$",
next: "comment"
},
{
token: "string", // single line
regex: '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
},
{
token: "string", // multi line string start
regex: '["].*\\\\$',
next: "qqstring"
},
{
token: "string", // single line
regex: "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
},
{
token: "string", // multi line string start
regex: "['].*\\\\$",
next: "qstring"
},
{
token: "number", // hex
regex: "0[xX][0-9a-fA-F]+\\b"
},
{
token: "number", // float
regex: "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
},
{
token: function(value)
{
if (keywords[value]) {
return "keyword";
} else {
return "identifier"
}
},
regex: "[a-zA-Z_][a-zA-Z0-9_]*\\b"
},
{
token: function(value) {
//return parens[value];
return "text";
},
regex: "[\\[\\]\\(\\)\\{\\}]"
},
{
token: "text",
regex: "\\s+"
}
],
"comment":
[
{
token: "comment", // closing comment
regex: ".*?\\*\\/",
next: "start"
},
{
token: "comment", // comment spanning whole line
regex: ".+"
}
],
"qqstring":
[
{
token: "string",
regex: '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
next: "start"
},
{
token: "string",
regex: '.+'
}
],
"qstring":
[
{
token: "string",
regex: "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
next: "start"
},
{
token: "string",
regex: '.+'
}
]
};
var regExps = {};
for (var key in rules)
{
var state = rules[key];
var ruleRegExps = [];
for (var i=0; i < state.length; i++) {
ruleRegExps.push(state[i].regex);
};
regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
}
var currentState = startState;
var state = rules[currentState];
var re = regExps[currentState];
re.lastIndex = 0;
var match, tokens = [];
var lastIndex = 0;
while (match = re.exec(line))
{
var token = {
type: "text",
value: match[0]
}
if (re.lastIndex == lastIndex) {
throw new Error("tokenizer error")
}
lastIndex = re.lastIndex;
//console.log(match);
for (var i=0; i < state.length; i++)
{
if (match[i+1])
{
if (typeof state[i].token == "function") {
token.type = state[i].token(match[0]);
} else {
token.type = state[i].token;
}
if (state[i].next && state[i].next !== currentState)
{
currentState = state[i].next;
var state = rules[currentState];
var lastIndex = re.lastIndex;
var re = regExps[currentState];
re.lastIndex = lastIndex;
}
break;
}
};
tokens.push(token);
};
//console.log(tokens, currentState)
return {
tokens: tokens,
state: currentState
return this.tokenizer.getLineTokens(this.textLines[row] || "", state).tokens;
}
};

6
Editor.js
View file

@ -115,7 +115,11 @@ function Editor(doc, renderer)
doc.addChangeListener(lib.bind(this.onDocumentChange, this));
renderer.setDocument(doc);
this.tokenizer = new BackgroundTokenizer(lib.bind(this.onTokenizerUpdate, this));
this.tokenizer = new BackgroundTokenizer(
new Tokenizer(XML.RULES),
lib.bind(this.onTokenizerUpdate, this)
);
this.tokenizer.setLines(doc.lines);
renderer.setTokenizer(this.tokenizer);

137
JavaScript.js Normal file
View file

@ -0,0 +1,137 @@
(function() {
window.JavaScript = {};
var keywords = {
"break" : 1,
"case" : 1,
"catch" : 1,
"continue" : 1,
"default" : 1,
"delete" : 1,
"do" : 1,
"else" : 1,
"finally" : 1,
"for" : 1,
"function" : 1,
"if" : 1,
"in" : 1,
"instanceof" : 1,
"new" : 1,
"return" : 1,
"switch" : 1,
"throw" : 1,
"try" : 1,
"typeof" : 1,
"var" : 1,
"while" : 1,
"with" : 1
};
// regexp must not have capturing parentheses
// regexps are ordered -> the first match is used
JavaScript.RULES = {
start :
[
{
token: "comment",
regex: "\\/\\/.*$"
},
{
token: "comment", // multi line comment in one line
regex: "\\/\\*.*?\\*\\/"
},
{
token: "comment", // multi line comment start
regex: "\\/\\*.*$",
next: "comment"
},
{
token: "string", // single line
regex: '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
},
{
token: "string", // multi line string start
regex: '["].*\\\\$',
next: "qqstring"
},
{
token: "string", // single line
regex: "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
},
{
token: "string", // multi line string start
regex: "['].*\\\\$",
next: "qstring"
},
{
token: "number", // hex
regex: "0[xX][0-9a-fA-F]+\\b"
},
{
token: "number", // float
regex: "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
},
{
token: function(value)
{
if (keywords[value]) {
return "keyword";
} else {
return "identifier"
}
},
regex: "[a-zA-Z_][a-zA-Z0-9_]*\\b"
},
{
token: function(value) {
//return parens[value];
return "text";
},
regex: "[\\[\\]\\(\\)\\{\\}]"
},
{
token: "text",
regex: "\\s+"
}
],
"comment":
[
{
token: "comment", // closing comment
regex: ".*?\\*\\/",
next: "start"
},
{
token: "comment", // comment spanning whole line
regex: ".+"
}
],
"qqstring":
[
{
token: "string",
regex: '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
next: "start"
},
{
token: "string",
regex: '.+'
}
],
"qstring":
[
{
token: "string",
regex: "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
next: "start"
},
{
token: "string",
regex: '.+'
}
]
};
})();

77
Tokenizer.js Normal file
View file

@ -0,0 +1,77 @@
function Tokenizer(rules)
{
this.rules = rules;
this.regExps = {};
for (var key in this.rules)
{
var state = this.rules[key];
var ruleRegExps = [];
for (var i=0; i < state.length; i++) {
ruleRegExps.push(state[i].regex);
};
this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
}
};
Tokenizer.prototype.getLineTokens = function(line, startState)
{
var currentState = startState;
var state = this.rules[currentState];
var re = this.regExps[currentState];
re.lastIndex = 0;
var match, tokens = [];
var lastIndex = 0;
while (match = re.exec(line))
{
var token = {
type: "text",
value: match[0]
}
if (re.lastIndex == lastIndex) {
throw new Error("tokenizer error")
}
lastIndex = re.lastIndex;
//console.log(match);
for (var i=0; i < state.length; i++)
{
if (match[i+1])
{
if (typeof state[i].token == "function") {
token.type = state[i].token(match[0]);
} else {
token.type = state[i].token;
}
if (state[i].next && state[i].next !== currentState)
{
currentState = state[i].next;
var state = this.rules[currentState];
var lastIndex = re.lastIndex;
var re = this.regExps[currentState];
re.lastIndex = lastIndex;
}
break;
}
};
tokens.push(token);
};
//console.log(tokens, currentState)
return {
tokens: tokens,
state: currentState
}
};

78
XML.js Normal file
View file

@ -0,0 +1,78 @@
(function() {
window.XML = {};
// regexp must not have capturing parentheses
// regexps are ordered -> the first match is used
XML.RULES = {
start :
[
{
token: "text",
regex: "<\\!\\[CDATA\\[",
next: "cdata"
},
{
token: "xml_pe",
regex: "<\\?.*?\\?>"
},
{
token: "text", // opening tag
regex: "<",
next: "tag"
},
{
token: "text",
regex: "\\s+"
},
{
token: "text",
regex: ".+"
}
],
tag:
[
{
token: "text",
regex: ">",
next: "start"
},
{
token: "keyword",
regex: "[-_a-zA-Z0-9:]+"
},
{
token: "text",
regex: "\\s+"
},
{
token: "string",
regex: '".*?"'
},
{
token: "string",
regex: "'.*?'"
}
],
cdata:
[
{
token: "text",
regex: "\\]\\]>",
next: "start"
},
{
token: "string",
regex: "\\s+"
},
{
token: "text",
regex: ".+"
}
]
};
})();

4
editor.css
View file

@ -85,6 +85,10 @@
color: rgb(0, 0, 205);
}
.line .xml_pe {
color: rgb(104, 104, 91);
}
.marker-layer .selection {
position: absolute;
background: rgba(77, 151, 255, 0.33);

5
editor.html
View file

@ -21,7 +21,10 @@
<script src="lib.js" type="text/javascript" charset="utf-8"></script>
<script src="TextDocument.js" type="text/javascript" charset="utf-8"></script>
<script src="BackgroundTokenizer.js" type="text/javascript" charset="utf-8"></script>
<script src="JavaScript.js" type="text/javascript" charset="utf-8"></script>
<script src="XML.js" type="text/javascript" charset="utf-8"></script>
<script src="Tokenizer.js" type="text/javascript" charset="utf-8"></script>
<script src="BackgroundTokenizer.js" type="text/javascript" charset="utf-8"></script>
<script src="CursorLayer.js" type="text/javascript" charset="utf-8"></script>
<script src="GutterLayer.js" type="text/javascript" charset="utf-8"></script>
<script src="TextLayer.js" type="text/javascript" charset="utf-8"></script>