update narcissus. fix #365
This commit is contained in:
Fabian Jakobs
parent
69d8e19314
commit
1b02f2895e
4 changed files with 2817 additions and 1948 deletions
|
|
@ -87,7 +87,7 @@ var Document = function(text) {
|
|||
|
||||
|
||||
this.$detectNewLine = function(text) {
|
||||
var match = text.match(/^.*?(\r?\n)/m);
|
||||
var match = text.match(/^.*?(\r\n|\r|\n)/m);
|
||||
if (match) {
|
||||
this.$autoNewLine = match[1];
|
||||
} else {
|
||||
|
|
@ -111,7 +111,8 @@ var Document = function(text) {
|
|||
this.$autoNewLine = "\n";
|
||||
this.$newLineMode = "auto";
|
||||
this.setNewLineMode = function(newLineMode) {
|
||||
if (this.$newLineMode === newLineMode) return;
|
||||
if (this.$newLineMode === newLineMode)
|
||||
return;
|
||||
|
||||
this.$newLineMode = newLineMode;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -52,15 +52,26 @@
|
|||
|
||||
define(function(require, exports, module) {
|
||||
|
||||
exports.options = {
|
||||
var narcissus = {
|
||||
options: {
|
||||
version: 185,
|
||||
};
|
||||
// Global variables to hide from the interpreter
|
||||
hiddenHostGlobals: { Narcissus: true },
|
||||
// Desugar SpiderMonkey language extensions?
|
||||
desugarExtensions: false
|
||||
},
|
||||
hostSupportsEvalConst: (function() {
|
||||
try {
|
||||
return eval("(function(s) { eval(s); return x })('const x = true;')");
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
})(),
|
||||
hostGlobal: this
|
||||
};
|
||||
Narcissus = narcissus;
|
||||
|
||||
(function() {
|
||||
exports.hostGlobal = this
|
||||
})();
|
||||
|
||||
var tokens = [
|
||||
var tokens = [
|
||||
// End of source.
|
||||
"END",
|
||||
|
||||
|
|
@ -99,10 +110,10 @@ var tokens = [
|
|||
"break",
|
||||
"case", "catch", "const", "continue",
|
||||
"debugger", "default", "delete", "do",
|
||||
"else",
|
||||
"else", "export",
|
||||
"false", "finally", "for", "function",
|
||||
"if", "in", "instanceof",
|
||||
"let",
|
||||
"if", "import", "in", "instanceof",
|
||||
"let", "module",
|
||||
"new", "null",
|
||||
"return",
|
||||
"switch",
|
||||
|
|
@ -110,9 +121,9 @@ var tokens = [
|
|||
"var", "void",
|
||||
"yield",
|
||||
"while", "with",
|
||||
];
|
||||
];
|
||||
|
||||
var statementStartTokens = [
|
||||
var statementStartTokens = [
|
||||
"break",
|
||||
"const", "continue",
|
||||
"debugger", "do",
|
||||
|
|
@ -124,13 +135,30 @@ var statementStartTokens = [
|
|||
"var",
|
||||
"yield",
|
||||
"while", "with",
|
||||
];
|
||||
];
|
||||
|
||||
// Operator and punctuator mapping from token to tree node type name.
|
||||
// NB: because the lexer doesn't backtrack, all token prefixes must themselves
|
||||
// be valid tokens (e.g. !== is acceptable because its prefixes are the valid
|
||||
// tokens != and !).
|
||||
var opTypeNames = {
|
||||
// Whitespace characters (see ECMA-262 7.2)
|
||||
var whitespaceChars = [
|
||||
// normal whitespace:
|
||||
"\u0009", "\u000B", "\u000C", "\u0020", "\u00A0", "\uFEFF",
|
||||
|
||||
// high-Unicode whitespace:
|
||||
"\u1680", "\u180E",
|
||||
"\u2000", "\u2001", "\u2002", "\u2003", "\u2004", "\u2005", "\u2006",
|
||||
"\u2007", "\u2008", "\u2009", "\u200A",
|
||||
"\u202F", "\u205F", "\u3000"
|
||||
];
|
||||
|
||||
var whitespace = {};
|
||||
for (var i = 0; i < whitespaceChars.length; i++) {
|
||||
whitespace[whitespaceChars[i]] = true;
|
||||
}
|
||||
|
||||
// Operator and punctuator mapping from token to tree node type name.
|
||||
// NB: because the lexer doesn't backtrack, all token prefixes must themselves
|
||||
// be valid tokens (e.g. !== is acceptable because its prefixes are the valid
|
||||
// tokens != and !).
|
||||
var opTypeNames = {
|
||||
'\n': "NEWLINE",
|
||||
';': "SEMICOLON",
|
||||
',': "COMMA",
|
||||
|
|
@ -169,18 +197,18 @@ var opTypeNames = {
|
|||
'}': "RIGHT_CURLY",
|
||||
'(': "LEFT_PAREN",
|
||||
')': "RIGHT_PAREN"
|
||||
};
|
||||
};
|
||||
|
||||
// Hash of keyword identifier to tokens index. NB: we must null __proto__ to
|
||||
// avoid toString, etc. namespace pollution.
|
||||
var keywords = {__proto__: null};
|
||||
// Hash of keyword identifier to tokens index. NB: we must null __proto__ to
|
||||
// avoid toString, etc. namespace pollution.
|
||||
var keywords = {__proto__: null};
|
||||
|
||||
// Define const END, etc., based on the token names. Also map name to index.
|
||||
var tokenIds = {};
|
||||
// Define const END, etc., based on the token names. Also map name to index.
|
||||
var tokenIds = {};
|
||||
|
||||
// Building up a string to be eval'd in different contexts.
|
||||
var consts = "const ";
|
||||
for (var i = 0, j = tokens.length; i < j; i++) {
|
||||
// Building up a string to be eval'd in different contexts.
|
||||
var consts = Narcissus.hostSupportsEvalConst ? "const " : "var ";
|
||||
for (var i = 0, j = tokens.length; i < j; i++) {
|
||||
if (i > 0)
|
||||
consts += ", ";
|
||||
var t = tokens[i];
|
||||
|
|
@ -194,54 +222,205 @@ for (var i = 0, j = tokens.length; i < j; i++) {
|
|||
consts += name + " = " + i;
|
||||
tokenIds[name] = i;
|
||||
tokens[t] = i;
|
||||
}
|
||||
consts += ";";
|
||||
}
|
||||
consts += ";";
|
||||
|
||||
var isStatementStartCode = {__proto__: null};
|
||||
for (i = 0, j = statementStartTokens.length; i < j; i++)
|
||||
var isStatementStartCode = {__proto__: null};
|
||||
for (i = 0, j = statementStartTokens.length; i < j; i++)
|
||||
isStatementStartCode[keywords[statementStartTokens[i]]] = true;
|
||||
|
||||
// Map assignment operators to their indexes in the tokens array.
|
||||
var assignOps = ['|', '^', '&', '<<', '>>', '>>>', '+', '-', '*', '/', '%'];
|
||||
// Map assignment operators to their indexes in the tokens array.
|
||||
var assignOps = ['|', '^', '&', '<<', '>>', '>>>', '+', '-', '*', '/', '%'];
|
||||
|
||||
for (i = 0, j = assignOps.length; i < j; i++) {
|
||||
for (i = 0, j = assignOps.length; i < j; i++) {
|
||||
t = assignOps[i];
|
||||
assignOps[t] = tokens[t];
|
||||
}
|
||||
}
|
||||
|
||||
function defineGetter(obj, prop, fn, dontDelete, dontEnum) {
|
||||
function defineGetter(obj, prop, fn, dontDelete, dontEnum) {
|
||||
Object.defineProperty(obj, prop,
|
||||
{ get: fn, configurable: !dontDelete, enumerable: !dontEnum });
|
||||
}
|
||||
}
|
||||
|
||||
function defineProperty(obj, prop, val, dontDelete, readOnly, dontEnum) {
|
||||
function defineGetterSetter(obj, prop, getter, setter, dontDelete, dontEnum) {
|
||||
Object.defineProperty(obj, prop, {
|
||||
get: getter,
|
||||
set: setter,
|
||||
configurable: !dontDelete,
|
||||
enumerable: !dontEnum
|
||||
});
|
||||
}
|
||||
|
||||
function defineMemoGetter(obj, prop, fn, dontDelete, dontEnum) {
|
||||
Object.defineProperty(obj, prop, {
|
||||
get: function() {
|
||||
var val = fn();
|
||||
defineProperty(obj, prop, val, dontDelete, true, dontEnum);
|
||||
return val;
|
||||
},
|
||||
configurable: true,
|
||||
enumerable: !dontEnum
|
||||
});
|
||||
}
|
||||
|
||||
function defineProperty(obj, prop, val, dontDelete, readOnly, dontEnum) {
|
||||
Object.defineProperty(obj, prop,
|
||||
{ value: val, writable: !readOnly, configurable: !dontDelete,
|
||||
enumerable: !dontEnum });
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if fn is a native function. (Note: SpiderMonkey specific.)
|
||||
function isNativeCode(fn) {
|
||||
// Returns true if fn is a native function. (Note: SpiderMonkey specific.)
|
||||
function isNativeCode(fn) {
|
||||
// Relies on the toString method to identify native code.
|
||||
return ((typeof fn) === "function") && fn.toString().match(/\[native code\]/);
|
||||
}
|
||||
}
|
||||
|
||||
function getPropertyDescriptor(obj, name) {
|
||||
function getPropertyDescriptor(obj, name) {
|
||||
while (obj) {
|
||||
if (({}).hasOwnProperty.call(obj, name))
|
||||
return Object.getOwnPropertyDescriptor(obj, name);
|
||||
obj = Object.getPrototypeOf(obj);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getOwnProperties(obj) {
|
||||
function getPropertyNames(obj) {
|
||||
var table = Object.create(null, {});
|
||||
while (obj) {
|
||||
var names = Object.getOwnPropertyNames(obj);
|
||||
for (var i = 0, n = names.length; i < n; i++)
|
||||
table[names[i]] = true;
|
||||
obj = Object.getPrototypeOf(obj);
|
||||
}
|
||||
return Object.keys(table);
|
||||
}
|
||||
|
||||
function getOwnProperties(obj) {
|
||||
var map = {};
|
||||
for (var name in Object.getOwnPropertyNames(obj))
|
||||
map[name] = Object.getOwnPropertyDescriptor(obj, name);
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
function makePassthruHandler(obj) {
|
||||
function blacklistHandler(target, blacklist) {
|
||||
var mask = Object.create(null, {});
|
||||
var redirect = StringMap.create(blacklist).mapObject(function(name) { return mask; });
|
||||
return mixinHandler(redirect, target);
|
||||
}
|
||||
|
||||
function whitelistHandler(target, whitelist) {
|
||||
var catchall = Object.create(null, {});
|
||||
var redirect = StringMap.create(whitelist).mapObject(function(name) { return target; });
|
||||
return mixinHandler(redirect, catchall);
|
||||
}
|
||||
|
||||
function mirrorHandler(target, writable) {
|
||||
var handler = makePassthruHandler(target);
|
||||
|
||||
var defineProperty = handler.defineProperty;
|
||||
handler.defineProperty = function(name, desc) {
|
||||
if (!desc.enumerable)
|
||||
throw new Error("mirror property must be enumerable");
|
||||
if (!desc.configurable)
|
||||
throw new Error("mirror property must be configurable");
|
||||
if (desc.writable !== writable)
|
||||
throw new Error("mirror property must " + (writable ? "" : "not ") + "be writable");
|
||||
defineProperty(name, desc);
|
||||
};
|
||||
|
||||
handler.fix = function() { };
|
||||
handler.getOwnPropertyDescriptor = handler.getPropertyDescriptor;
|
||||
handler.getOwnPropertyNames = getPropertyNames.bind(handler, target);
|
||||
handler.keys = handler.enumerate;
|
||||
handler["delete"] = function() { return false; };
|
||||
handler.hasOwn = handler.has;
|
||||
return handler;
|
||||
}
|
||||
|
||||
/*
|
||||
* Mixin proxies break the single-inheritance model of prototypes, so
|
||||
* the handler treats all properties as own-properties:
|
||||
*
|
||||
* X
|
||||
* |
|
||||
* +------------+------------+
|
||||
* | O |
|
||||
* | | |
|
||||
* | O O O |
|
||||
* | | | | |
|
||||
* | O O O O |
|
||||
* | | | | | |
|
||||
* | O O O O O |
|
||||
* | | | | | | |
|
||||
* +-(*)--(w)--(x)--(y)--(z)-+
|
||||
*/
|
||||
|
||||
function mixinHandler(redirect, catchall) {
|
||||
function targetFor(name) {
|
||||
return hasOwn(redirect, name) ? redirect[name] : catchall;
|
||||
}
|
||||
|
||||
function getMuxPropertyDescriptor(name) {
|
||||
var desc = getPropertyDescriptor(targetFor(name), name);
|
||||
if (desc)
|
||||
desc.configurable = true;
|
||||
return desc;
|
||||
}
|
||||
|
||||
function getMuxPropertyNames() {
|
||||
var names1 = Object.getOwnPropertyNames(redirect).filter(function(name) {
|
||||
return name in redirect[name];
|
||||
});
|
||||
var names2 = getPropertyNames(catchall).filter(function(name) {
|
||||
return !hasOwn(redirect, name);
|
||||
});
|
||||
return names1.concat(names2);
|
||||
}
|
||||
|
||||
function enumerateMux() {
|
||||
var result = Object.getOwnPropertyNames(redirect).filter(function(name) {
|
||||
return name in redirect[name];
|
||||
});
|
||||
for (name in catchall) {
|
||||
if (!hasOwn(redirect, name))
|
||||
result.push(name);
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
function hasMux(name) {
|
||||
return name in targetFor(name);
|
||||
}
|
||||
|
||||
return {
|
||||
getOwnPropertyDescriptor: getMuxPropertyDescriptor,
|
||||
getPropertyDescriptor: getMuxPropertyDescriptor,
|
||||
getOwnPropertyNames: getMuxPropertyNames,
|
||||
defineProperty: function(name, desc) {
|
||||
Object.defineProperty(targetFor(name), name, desc);
|
||||
},
|
||||
"delete": function(name) {
|
||||
var target = targetFor(name);
|
||||
return delete target[name];
|
||||
},
|
||||
// FIXME: ha ha ha
|
||||
fix: function() { },
|
||||
has: hasMux,
|
||||
hasOwn: hasMux,
|
||||
get: function(receiver, name) {
|
||||
var target = targetFor(name);
|
||||
return target[name];
|
||||
},
|
||||
set: function(receiver, name, val) {
|
||||
var target = targetFor(name);
|
||||
target[name] = val;
|
||||
return true;
|
||||
},
|
||||
enumerate: enumerateMux,
|
||||
keys: enumerateMux
|
||||
};
|
||||
}
|
||||
|
||||
function makePassthruHandler(obj) {
|
||||
// Handler copied from
|
||||
// http://wiki.ecmascript.org/doku.php?id=harmony:proxies&s=proxy%20object#examplea_no-op_forwarding_proxy
|
||||
return {
|
||||
|
|
@ -288,19 +467,32 @@ function makePassthruHandler(obj) {
|
|||
},
|
||||
keys: function() { return Object.keys(obj); }
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// default function used when looking for a property in the global object
|
||||
function noPropFound() { return undefined; }
|
||||
var hasOwnProperty = ({}).hasOwnProperty;
|
||||
|
||||
var hasOwnProperty = ({}).hasOwnProperty;
|
||||
function hasOwn(obj, name) {
|
||||
return hasOwnProperty.call(obj, name);
|
||||
}
|
||||
|
||||
function StringMap() {
|
||||
this.table = Object.create(null, {});
|
||||
this.size = 0;
|
||||
}
|
||||
function StringMap(table, size) {
|
||||
this.table = table || Object.create(null, {});
|
||||
this.size = size || 0;
|
||||
}
|
||||
|
||||
StringMap.prototype = {
|
||||
StringMap.create = function(table) {
|
||||
var init = Object.create(null, {});
|
||||
var size = 0;
|
||||
var names = Object.getOwnPropertyNames(table);
|
||||
for (var i = 0, n = names.length; i < n; i++) {
|
||||
var name = names[i];
|
||||
init[name] = table[name];
|
||||
size++;
|
||||
}
|
||||
return new StringMap(init, size);
|
||||
};
|
||||
|
||||
StringMap.prototype = {
|
||||
has: function(x) { return hasOwnProperty.call(this.table, x); },
|
||||
set: function(x, v) {
|
||||
if (!hasOwnProperty.call(this.table, x))
|
||||
|
|
@ -320,15 +512,119 @@ StringMap.prototype = {
|
|||
for (var key in table)
|
||||
f.call(this, key, table[key]);
|
||||
},
|
||||
map: function(f) {
|
||||
var table1 = this.table;
|
||||
var table2 = Object.create(null, {});
|
||||
this.forEach(function(key, val) {
|
||||
table2[key] = f.call(this, val, key);
|
||||
});
|
||||
return new StringMap(table2, this.size);
|
||||
},
|
||||
mapObject: function(f) {
|
||||
var table1 = this.table;
|
||||
var table2 = Object.create(null, {});
|
||||
this.forEach(function(key, val) {
|
||||
table2[key] = f.call(this, val, key);
|
||||
});
|
||||
return table2;
|
||||
},
|
||||
toObject: function() {
|
||||
return this.mapObject(function(val) { return val; });
|
||||
},
|
||||
choose: function() {
|
||||
return Object.getOwnPropertyNames(this.table)[0];
|
||||
},
|
||||
remove: function(x) {
|
||||
if (hasOwnProperty.call(this.table, x)) {
|
||||
this.size--;
|
||||
delete this.table[x];
|
||||
}
|
||||
},
|
||||
copy: function() {
|
||||
var table = Object.create(null, {});
|
||||
for (var key in this.table)
|
||||
table[key] = this.table[key];
|
||||
return new StringMap(table, this.size);
|
||||
},
|
||||
toString: function() { return "[object StringMap]" }
|
||||
};
|
||||
};
|
||||
|
||||
// non-destructive stack
|
||||
function Stack(elts) {
|
||||
// an object-key table with poor asymptotics (replace with WeakMap when possible)
|
||||
function ObjectMap(array) {
|
||||
this.array = array || [];
|
||||
}
|
||||
|
||||
function searchMap(map, key, found, notFound) {
|
||||
var a = map.array;
|
||||
for (var i = 0, n = a.length; i < n; i++) {
|
||||
var pair = a[i];
|
||||
if (pair.key === key)
|
||||
return found(pair, i);
|
||||
}
|
||||
return notFound();
|
||||
}
|
||||
|
||||
ObjectMap.prototype = {
|
||||
has: function(x) {
|
||||
return searchMap(this, x, function() { return true }, function() { return false });
|
||||
},
|
||||
set: function(x, v) {
|
||||
var a = this.array;
|
||||
searchMap(this, x,
|
||||
function(pair) { pair.value = v },
|
||||
function() { a.push({ key: x, value: v }) });
|
||||
},
|
||||
get: function(x) {
|
||||
return searchMap(this, x,
|
||||
function(pair) { return pair.value },
|
||||
function() { return null });
|
||||
},
|
||||
getDef: function(x, thunk) {
|
||||
var a = this.array;
|
||||
return searchMap(this, x,
|
||||
function(pair) { return pair.value },
|
||||
function() {
|
||||
var v = thunk();
|
||||
a.push({ key: x, value: v });
|
||||
return v;
|
||||
});
|
||||
},
|
||||
forEach: function(f) {
|
||||
var a = this.array;
|
||||
for (var i = 0, n = a.length; i < n; i++) {
|
||||
var pair = a[i];
|
||||
f.call(this, pair.key, pair.value);
|
||||
}
|
||||
},
|
||||
choose: function() {
|
||||
return this.array[0].key;
|
||||
},
|
||||
get size() {
|
||||
return this.array.length;
|
||||
},
|
||||
remove: function(x) {
|
||||
var a = this.array;
|
||||
searchMap(this, x,
|
||||
function(pair, i) { a.splice(i, 1) },
|
||||
function() { });
|
||||
},
|
||||
copy: function() {
|
||||
return new ObjectMap(this.array.map(function(pair) {
|
||||
return { key: pair.key, value: pair.value }
|
||||
}));
|
||||
},
|
||||
clear: function() {
|
||||
this.array = [];
|
||||
},
|
||||
toString: function() { return "[object ObjectMap]" }
|
||||
};
|
||||
|
||||
// non-destructive stack
|
||||
function Stack(elts) {
|
||||
this.elts = elts || null;
|
||||
}
|
||||
}
|
||||
|
||||
Stack.prototype = {
|
||||
Stack.prototype = {
|
||||
push: function(x) {
|
||||
return new Stack({ top: x, rest: this.elts });
|
||||
},
|
||||
|
|
@ -355,21 +651,29 @@ Stack.prototype = {
|
|||
f(elts.top);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
exports.tokens = tokens;
|
||||
exports.opTypeNames = opTypeNames;
|
||||
exports.keywords = keywords;
|
||||
exports.isStatementStartCode = isStatementStartCode;
|
||||
exports.tokenIds = tokenIds;
|
||||
exports.consts = consts;
|
||||
exports.assignOps = assignOps;
|
||||
exports.defineGetter = defineGetter;
|
||||
exports.defineProperty = defineProperty;
|
||||
exports.isNativeCode = isNativeCode;
|
||||
exports.makePassthruHandler = makePassthruHandler;
|
||||
exports.noPropFound = noPropFound;
|
||||
exports.StringMap = StringMap;
|
||||
exports.Stack = Stack;
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
tokens: tokens,
|
||||
whitespace: whitespace,
|
||||
opTypeNames: opTypeNames,
|
||||
keywords: keywords,
|
||||
isStatementStartCode: isStatementStartCode,
|
||||
tokenIds: tokenIds,
|
||||
consts: consts,
|
||||
assignOps: assignOps,
|
||||
defineGetter: defineGetter,
|
||||
defineGetterSetter: defineGetterSetter,
|
||||
defineMemoGetter: defineMemoGetter,
|
||||
defineProperty: defineProperty,
|
||||
isNativeCode: isNativeCode,
|
||||
mirrorHandler: mirrorHandler,
|
||||
mixinHandler: mixinHandler,
|
||||
whitelistHandler: whitelistHandler,
|
||||
blacklistHandler: blacklistHandler,
|
||||
makePassthruHandler: makePassthruHandler,
|
||||
StringMap: StringMap,
|
||||
ObjectMap: ObjectMap,
|
||||
Stack: Stack
|
||||
};
|
||||
});
|
||||
|
|
@ -47,16 +47,23 @@
|
|||
* Lexical scanner.
|
||||
*/
|
||||
|
||||
define(function(require, exports, module) {
|
||||
define(function(require, exports, module) {
|
||||
|
||||
var definitions = require("ace/narcissus/jsdefs");
|
||||
var definitions = require("ace/narcissus/jsdefs");
|
||||
|
||||
// Set constants in the local scope.
|
||||
eval(definitions.consts);
|
||||
// Set constants in the local scope.
|
||||
eval(definitions.consts);
|
||||
|
||||
// Build up a trie of operator tokens.
|
||||
var opTokens = {};
|
||||
for (var op in definitions.opTypeNames) {
|
||||
// Banned keywords by language version
|
||||
const blackLists = { 160: {}, 185: {}, harmony: {} };
|
||||
blackLists[160][LET] = true;
|
||||
blackLists[160][MODULE] = true;
|
||||
blackLists[160][YIELD] = true;
|
||||
blackLists[185][MODULE] = true;
|
||||
|
||||
// Build up a trie of operator tokens.
|
||||
var opTokens = {};
|
||||
for (var op in definitions.opTypeNames) {
|
||||
if (op === '\n' || op === '.')
|
||||
continue;
|
||||
|
||||
|
|
@ -68,12 +75,12 @@ for (var op in definitions.opTypeNames) {
|
|||
node = node[ch];
|
||||
node.op = op;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
/*
|
||||
* Tokenizer :: (source, filename, line number) -> Tokenizer
|
||||
*/
|
||||
function Tokenizer(s, f, l) {
|
||||
function Tokenizer(s, f, l) {
|
||||
this.cursor = 0;
|
||||
this.source = String(s);
|
||||
this.tokens = [];
|
||||
|
|
@ -83,9 +90,11 @@ function Tokenizer(s, f, l) {
|
|||
this.unexpectedEOF = false;
|
||||
this.filename = f || "";
|
||||
this.lineno = l || 1;
|
||||
}
|
||||
this.blackList = blackLists[Narcissus.options.version];
|
||||
this.blockComments = null;
|
||||
}
|
||||
|
||||
Tokenizer.prototype = {
|
||||
Tokenizer.prototype = {
|
||||
get done() {
|
||||
// We need to set scanOperand to true here because the first thing
|
||||
// might be a regexp.
|
||||
|
|
@ -129,16 +138,31 @@ Tokenizer.prototype = {
|
|||
return tt;
|
||||
},
|
||||
|
||||
lastBlockComment: function() {
|
||||
var length = this.blockComments.length;
|
||||
return length ? this.blockComments[length - 1] : null;
|
||||
},
|
||||
|
||||
// Eat comments and whitespace.
|
||||
skip: function () {
|
||||
var input = this.source;
|
||||
this.blockComments = [];
|
||||
for (;;) {
|
||||
var ch = input[this.cursor++];
|
||||
var next = input[this.cursor];
|
||||
// handle \r, \r\n and (always preferable) \n
|
||||
if (ch === '\r') {
|
||||
// if the next character is \n, we don't care about this at all
|
||||
if (next === '\n') continue;
|
||||
|
||||
// otherwise, we want to consider this as a newline
|
||||
ch = '\n';
|
||||
}
|
||||
|
||||
if (ch === '\n' && !this.scanNewlines) {
|
||||
this.lineno++;
|
||||
} else if (ch === '/' && next === '*') {
|
||||
this.cursor++;
|
||||
var commentStart = ++this.cursor;
|
||||
for (;;) {
|
||||
ch = input[this.cursor++];
|
||||
if (ch === undefined)
|
||||
|
|
@ -147,6 +171,7 @@ Tokenizer.prototype = {
|
|||
if (ch === '*') {
|
||||
next = input[this.cursor];
|
||||
if (next === '/') {
|
||||
var commentEnd = this.cursor - 1;
|
||||
this.cursor++;
|
||||
break;
|
||||
}
|
||||
|
|
@ -154,19 +179,30 @@ Tokenizer.prototype = {
|
|||
this.lineno++;
|
||||
}
|
||||
}
|
||||
this.blockComments.push(input.substring(commentStart, commentEnd));
|
||||
} else if (ch === '/' && next === '/') {
|
||||
this.cursor++;
|
||||
for (;;) {
|
||||
ch = input[this.cursor++];
|
||||
next = input[this.cursor];
|
||||
if (ch === undefined)
|
||||
return;
|
||||
|
||||
if (ch === '\r') {
|
||||
// check for \r\n
|
||||
if (next !== '\n') ch = '\n';
|
||||
}
|
||||
|
||||
if (ch === '\n') {
|
||||
if (this.scanNewlines) {
|
||||
this.cursor--;
|
||||
} else {
|
||||
this.lineno++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (ch !== ' ' && ch !== '\t') {
|
||||
} else if (!(ch in definitions.whitespace)) {
|
||||
this.cursor--;
|
||||
return;
|
||||
}
|
||||
|
|
@ -363,19 +399,19 @@ Tokenizer.prototype = {
|
|||
},
|
||||
|
||||
// FIXME: Unicode escape sequences
|
||||
// FIXME: Unicode identifiers
|
||||
lexIdent: function (ch) {
|
||||
var token = this.token, input = this.source;
|
||||
var token = this.token;
|
||||
var id = ch;
|
||||
|
||||
do {
|
||||
ch = input[this.cursor++];
|
||||
} while ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
|
||||
(ch >= '0' && ch <= '9') || ch === '$' || ch === '_');
|
||||
while ((ch = this.getValidIdentifierChar(false)) !== null) {
|
||||
id += ch;
|
||||
}
|
||||
|
||||
this.cursor--; // Put the non-word character back.
|
||||
|
||||
var id = input.substring(token.start, this.cursor);
|
||||
token.type = definitions.keywords[id] || IDENTIFIER;
|
||||
if (token.type in this.blackList) {
|
||||
// banned keyword, this is an identifier
|
||||
token.type = IDENTIFIER;
|
||||
}
|
||||
token.value = id;
|
||||
},
|
||||
|
||||
|
|
@ -403,15 +439,16 @@ Tokenizer.prototype = {
|
|||
this.tokens[this.tokenIndex] = token = {};
|
||||
|
||||
var input = this.source;
|
||||
if (this.cursor === input.length)
|
||||
if (this.cursor >= input.length)
|
||||
return token.type = END;
|
||||
|
||||
token.start = this.cursor;
|
||||
token.lineno = this.lineno;
|
||||
|
||||
var ch = input[this.cursor++];
|
||||
if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '$' || ch === '_') {
|
||||
this.lexIdent(ch);
|
||||
var ich = this.getValidIdentifierChar(true);
|
||||
var ch = (ich === null) ? input[this.cursor++] : null;
|
||||
if (ich !== null) {
|
||||
this.lexIdent(ich);
|
||||
} else if (scanOperand && ch === '/') {
|
||||
this.lexRegExp(ch);
|
||||
} else if (ch in opTokens) {
|
||||
|
|
@ -424,7 +461,9 @@ Tokenizer.prototype = {
|
|||
this.lexZeroNumber(ch);
|
||||
} else if (ch === '"' || ch === "'") {
|
||||
this.lexString(ch);
|
||||
} else if (this.scanNewlines && ch === '\n') {
|
||||
} else if (this.scanNewlines && (ch === '\n' || ch === '\r')) {
|
||||
// if this was a \r, look for \r\n
|
||||
if (ch === '\r' && input[this.cursor] === '\n') this.cursor++;
|
||||
token.type = NEWLINE;
|
||||
token.value = '\n';
|
||||
this.lineno++;
|
||||
|
|
@ -447,16 +486,64 @@ Tokenizer.prototype = {
|
|||
},
|
||||
|
||||
newSyntaxError: function (m) {
|
||||
m = (this.filename ? this.filename + ":" : "") + this.lineno + ": " + m;
|
||||
var e = new SyntaxError(m, this.filename, this.lineno);
|
||||
e.source = this.source;
|
||||
e.lineno = this.lineno;
|
||||
e.cursor = this.lookahead
|
||||
? this.tokens[(this.tokenIndex + this.lookahead) & 3].start
|
||||
: this.cursor;
|
||||
return e;
|
||||
},
|
||||
};
|
||||
|
||||
exports.Tokenizer = Tokenizer;
|
||||
/* Gets a single valid identifier char from the input stream, or null
|
||||
* if there is none.
|
||||
* Since JavaScript provides no convenient way to determine if a
|
||||
* character is in a particular Unicode category, we use
|
||||
* metacircularity to accomplish this (oh yeaaaah!) */
|
||||
getValidIdentifierChar: function(first) {
|
||||
var input = this.source;
|
||||
if (this.cursor >= input.length) return null;
|
||||
var ch = input[this.cursor];
|
||||
|
||||
// first check for \u escapes
|
||||
if (ch === '\\' && input[this.cursor+1] === 'u') {
|
||||
// get the character value
|
||||
try {
|
||||
ch = String.fromCharCode(parseInt(
|
||||
input.substring(this.cursor + 2, this.cursor + 6),
|
||||
16));
|
||||
} catch (ex) {
|
||||
return null;
|
||||
}
|
||||
this.cursor += 5;
|
||||
}
|
||||
|
||||
// check directly for ASCII
|
||||
if (ch <= "\u007F") {
|
||||
if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '$' || ch === '_' ||
|
||||
(!first && (ch >= '0' && ch <= '9'))) {
|
||||
this.cursor++;
|
||||
return ch;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// create an object to test this in
|
||||
var x = {};
|
||||
x["x"+ch] = true;
|
||||
x[ch] = true;
|
||||
|
||||
// then use eval to determine if it's a valid character
|
||||
var valid = false;
|
||||
try {
|
||||
valid = (Function("x", "return (x." + (first?"":"x") + ch + ");")(x) === true);
|
||||
} catch (ex) {}
|
||||
if (valid) this.cursor++;
|
||||
return (valid ? ch : null);
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
return { Tokenizer: Tokenizer };
|
||||
|
||||
});
|
||||
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue