update narcissus. fix #365

lib/ace/document.js

@@ -87,7 +87,7 @@ var Document = function(text) {
 
 
     this.$detectNewLine = function(text) {
-        var match = text.match(/^.*?(\r?\n)/m);
+        var match = text.match(/^.*?(\r\n|\r|\n)/m);
         if (match) {
             this.$autoNewLine = match[1];
         } else {
@@ -111,7 +111,8 @@ var Document = function(text) {
     this.$autoNewLine = "\n";
     this.$newLineMode = "auto";
     this.setNewLineMode = function(newLineMode) {
-        if (this.$newLineMode === newLineMode) return;
+        if (this.$newLineMode === newLineMode)
+            return;
 
         this.$newLineMode = newLineMode;
     };

lib/ace/narcissus/jsdefs.js

@@ -52,15 +52,26 @@
 
 define(function(require, exports, module) {
 
-exports.options =  {
+    var narcissus = {
+        options: {
             version: 185,
-};
+            // Global variables to hide from the interpreter
+            hiddenHostGlobals: { Narcissus: true },
+            // Desugar SpiderMonkey language extensions?
+            desugarExtensions: false
+        },
+        hostSupportsEvalConst: (function() {
+            try {
+                return eval("(function(s) { eval(s); return x })('const x = true;')");
+            } catch (e) {
+                return false;
+            }
+        })(),
+        hostGlobal: this
+    };
+    Narcissus = narcissus;
 
-(function() {
+    var tokens = [
-    exports.hostGlobal = this
-})();
-
-var tokens = [
         // End of source.
         "END",
 
@@ -99,10 +110,10 @@ var tokens = [
         "break",
         "case", "catch", "const", "continue",
         "debugger", "default", "delete", "do",
-    "else",
+        "else", "export",
         "false", "finally", "for", "function",
-    "if", "in", "instanceof",
+        "if", "import", "in", "instanceof",
-    "let",
+        "let", "module",
         "new", "null",
         "return",
         "switch",
@@ -110,9 +121,9 @@ var tokens = [
         "var", "void",
         "yield",
         "while", "with",
-];
+    ];
 
-var statementStartTokens = [
+    var statementStartTokens = [
         "break",
         "const", "continue",
         "debugger", "do",
@@ -124,13 +135,30 @@ var statementStartTokens = [
         "var",
         "yield",
         "while", "with",
-];
+    ];
 
-// Operator and punctuator mapping from token to tree node type name.
+    // Whitespace characters (see ECMA-262 7.2)
-// NB: because the lexer doesn't backtrack, all token prefixes must themselves
+    var whitespaceChars = [
-// be valid tokens (e.g. !== is acceptable because its prefixes are the valid
+        // normal whitespace:
-// tokens != and !).
+        "\u0009", "\u000B", "\u000C", "\u0020", "\u00A0", "\uFEFF", 
-var opTypeNames = {
+
+        // high-Unicode whitespace:
+        "\u1680", "\u180E",
+        "\u2000", "\u2001", "\u2002", "\u2003", "\u2004", "\u2005", "\u2006",
+        "\u2007", "\u2008", "\u2009", "\u200A",
+        "\u202F", "\u205F", "\u3000"
+    ];
+
+    var whitespace = {};
+    for (var i = 0; i < whitespaceChars.length; i++) {
+        whitespace[whitespaceChars[i]] = true;
+    }
+
+    // Operator and punctuator mapping from token to tree node type name.
+    // NB: because the lexer doesn't backtrack, all token prefixes must themselves
+    // be valid tokens (e.g. !== is acceptable because its prefixes are the valid
+    // tokens != and !).
+    var opTypeNames = {
         '\n':   "NEWLINE",
         ';':    "SEMICOLON",
         ',':    "COMMA",
@@ -169,18 +197,18 @@ var opTypeNames = {
         '}':    "RIGHT_CURLY",
         '(':    "LEFT_PAREN",
         ')':    "RIGHT_PAREN"
-};
+    };
 
-// Hash of keyword identifier to tokens index.  NB: we must null __proto__ to
+    // Hash of keyword identifier to tokens index.  NB: we must null __proto__ to
-// avoid toString, etc. namespace pollution.
+    // avoid toString, etc. namespace pollution.
-var keywords = {__proto__: null};
+    var keywords = {__proto__: null};
 
-// Define const END, etc., based on the token names.  Also map name to index.
+    // Define const END, etc., based on the token names.  Also map name to index.
-var tokenIds = {};
+    var tokenIds = {};
 
-// Building up a string to be eval'd in different contexts.
+    // Building up a string to be eval'd in different contexts.
-var consts = "const ";
+    var consts = Narcissus.hostSupportsEvalConst ? "const " : "var ";
-for (var i = 0, j = tokens.length; i < j; i++) {
+    for (var i = 0, j = tokens.length; i < j; i++) {
         if (i > 0)
             consts += ", ";
         var t = tokens[i];
@@ -194,54 +222,205 @@ for (var i = 0, j = tokens.length; i < j; i++) {
         consts += name + " = " + i;
         tokenIds[name] = i;
         tokens[t] = i;
-}
+    }
-consts += ";";
+    consts += ";";
 
-var isStatementStartCode = {__proto__: null};
+    var isStatementStartCode = {__proto__: null};
-for (i = 0, j = statementStartTokens.length; i < j; i++)
+    for (i = 0, j = statementStartTokens.length; i < j; i++)
         isStatementStartCode[keywords[statementStartTokens[i]]] = true;
 
-// Map assignment operators to their indexes in the tokens array.
+    // Map assignment operators to their indexes in the tokens array.
-var assignOps = ['|', '^', '&', '<<', '>>', '>>>', '+', '-', '*', '/', '%'];
+    var assignOps = ['|', '^', '&', '<<', '>>', '>>>', '+', '-', '*', '/', '%'];
 
-for (i = 0, j = assignOps.length; i < j; i++) {
+    for (i = 0, j = assignOps.length; i < j; i++) {
         t = assignOps[i];
         assignOps[t] = tokens[t];
-}
+    }
 
-function defineGetter(obj, prop, fn, dontDelete, dontEnum) {
+    function defineGetter(obj, prop, fn, dontDelete, dontEnum) {
         Object.defineProperty(obj, prop,
                               { get: fn, configurable: !dontDelete, enumerable: !dontEnum });
-}
+    }
 
-function defineProperty(obj, prop, val, dontDelete, readOnly, dontEnum) {
+    function defineGetterSetter(obj, prop, getter, setter, dontDelete, dontEnum) {
+        Object.defineProperty(obj, prop, {
+            get: getter,
+            set: setter,
+            configurable: !dontDelete,
+            enumerable: !dontEnum
+        });
+    }
+
+    function defineMemoGetter(obj, prop, fn, dontDelete, dontEnum) {
+        Object.defineProperty(obj, prop, {
+            get: function() {
+                var val = fn();
+                defineProperty(obj, prop, val, dontDelete, true, dontEnum);
+                return val;
+            },
+            configurable: true,
+            enumerable: !dontEnum
+        });
+    }
+
+    function defineProperty(obj, prop, val, dontDelete, readOnly, dontEnum) {
         Object.defineProperty(obj, prop,
                               { value: val, writable: !readOnly, configurable: !dontDelete,
                                 enumerable: !dontEnum });
-}
+    }
 
-// Returns true if fn is a native function.  (Note: SpiderMonkey specific.)
+    // Returns true if fn is a native function.  (Note: SpiderMonkey specific.)
-function isNativeCode(fn) {
+    function isNativeCode(fn) {
         // Relies on the toString method to identify native code.
         return ((typeof fn) === "function") && fn.toString().match(/\[native code\]/);
-}
+    }
 
-function getPropertyDescriptor(obj, name) {
+    function getPropertyDescriptor(obj, name) {
         while (obj) {
             if (({}).hasOwnProperty.call(obj, name))
                 return Object.getOwnPropertyDescriptor(obj, name);
             obj = Object.getPrototypeOf(obj);
         }
-}
+    }
 
-function getOwnProperties(obj) {
+    function getPropertyNames(obj) {
+        var table = Object.create(null, {});
+        while (obj) {
+            var names = Object.getOwnPropertyNames(obj);
+            for (var i = 0, n = names.length; i < n; i++)
+                table[names[i]] = true;
+            obj = Object.getPrototypeOf(obj);
+        }
+        return Object.keys(table);
+    }
+
+    function getOwnProperties(obj) {
         var map = {};
         for (var name in Object.getOwnPropertyNames(obj))
             map[name] = Object.getOwnPropertyDescriptor(obj, name);
         return map;
-}
+    }
 
-function makePassthruHandler(obj) {
+    function blacklistHandler(target, blacklist) {
+        var mask = Object.create(null, {});
+        var redirect = StringMap.create(blacklist).mapObject(function(name) { return mask; });
+        return mixinHandler(redirect, target);
+    }
+
+    function whitelistHandler(target, whitelist) {
+        var catchall = Object.create(null, {});
+        var redirect = StringMap.create(whitelist).mapObject(function(name) { return target; });
+        return mixinHandler(redirect, catchall);
+    }
+
+    function mirrorHandler(target, writable) {
+        var handler = makePassthruHandler(target);
+
+        var defineProperty = handler.defineProperty;
+        handler.defineProperty = function(name, desc) {
+            if (!desc.enumerable)
+                throw new Error("mirror property must be enumerable");
+            if (!desc.configurable)
+                throw new Error("mirror property must be configurable");
+            if (desc.writable !== writable)
+                throw new Error("mirror property must " + (writable ? "" : "not ") + "be writable");
+            defineProperty(name, desc);
+        };
+
+        handler.fix = function() { };
+        handler.getOwnPropertyDescriptor = handler.getPropertyDescriptor;
+        handler.getOwnPropertyNames = getPropertyNames.bind(handler, target);
+        handler.keys = handler.enumerate;
+        handler["delete"] = function() { return false; };
+        handler.hasOwn = handler.has;
+        return handler;
+    }
+
+    /*
+     * Mixin proxies break the single-inheritance model of prototypes, so
+     * the handler treats all properties as own-properties:
+     *
+     *                  X
+     *                  |
+     *     +------------+------------+
+     *     |                 O       |
+     *     |                 |       |
+     *     |  O         O    O       |
+     *     |  |         |    |       |
+     *     |  O    O    O    O       |
+     *     |  |    |    |    |       |
+     *     |  O    O    O    O    O  |
+     *     |  |    |    |    |    |  |
+     *     +-(*)--(w)--(x)--(y)--(z)-+
+     */
+
+    function mixinHandler(redirect, catchall) {
+        function targetFor(name) {
+            return hasOwn(redirect, name) ? redirect[name] : catchall;
+        }
+
+        function getMuxPropertyDescriptor(name) {
+            var desc = getPropertyDescriptor(targetFor(name), name);
+            if (desc)
+                desc.configurable = true;
+            return desc;
+        }
+
+        function getMuxPropertyNames() {
+            var names1 = Object.getOwnPropertyNames(redirect).filter(function(name) {
+                return name in redirect[name];
+            });
+            var names2 = getPropertyNames(catchall).filter(function(name) {
+                return !hasOwn(redirect, name);
+            });
+            return names1.concat(names2);
+        }
+
+        function enumerateMux() {
+            var result = Object.getOwnPropertyNames(redirect).filter(function(name) {
+                return name in redirect[name];
+            });
+            for (name in catchall) {
+                if (!hasOwn(redirect, name))
+                    result.push(name);
+            };
+            return result;
+        }
+
+        function hasMux(name) {
+            return name in targetFor(name);
+        }
+
+        return {
+            getOwnPropertyDescriptor: getMuxPropertyDescriptor,
+            getPropertyDescriptor: getMuxPropertyDescriptor,
+            getOwnPropertyNames: getMuxPropertyNames,
+            defineProperty: function(name, desc) {
+                Object.defineProperty(targetFor(name), name, desc);
+            },
+            "delete": function(name) {
+                var target = targetFor(name);
+                return delete target[name];
+            },
+            // FIXME: ha ha ha
+            fix: function() { },
+            has: hasMux,
+            hasOwn: hasMux,
+            get: function(receiver, name) {
+                var target = targetFor(name);
+                return target[name];
+            },
+            set: function(receiver, name, val) {
+                var target = targetFor(name);
+                target[name] = val;
+                return true;
+            },
+            enumerate: enumerateMux,
+            keys: enumerateMux
+        };
+    }
+
+    function makePassthruHandler(obj) {
         // Handler copied from
         // http://wiki.ecmascript.org/doku.php?id=harmony:proxies&s=proxy%20object#examplea_no-op_forwarding_proxy
         return {
@@ -288,19 +467,32 @@ function makePassthruHandler(obj) {
             },
             keys: function() { return Object.keys(obj); }
         };
-}
+    }
 
-// default function used when looking for a property in the global object
+    var hasOwnProperty = ({}).hasOwnProperty;
-function noPropFound() { return undefined; }
 
-var hasOwnProperty = ({}).hasOwnProperty;
+    function hasOwn(obj, name) {
+        return hasOwnProperty.call(obj, name);
+    }
 
-function StringMap() {
+    function StringMap(table, size) {
-    this.table = Object.create(null, {});
+        this.table = table || Object.create(null, {});
-    this.size = 0;
+        this.size = size || 0;
-}
+    }
 
-StringMap.prototype = {
+    StringMap.create = function(table) {
+        var init = Object.create(null, {});
+        var size = 0;
+        var names = Object.getOwnPropertyNames(table);
+        for (var i = 0, n = names.length; i < n; i++) {
+            var name = names[i];
+            init[name] = table[name];
+            size++;
+        }
+        return new StringMap(init, size);
+    };
+
+    StringMap.prototype = {
         has: function(x) { return hasOwnProperty.call(this.table, x); },
         set: function(x, v) {
             if (!hasOwnProperty.call(this.table, x))
@@ -320,15 +512,119 @@ StringMap.prototype = {
             for (var key in table)
                 f.call(this, key, table[key]);
         },
+        map: function(f) {
+            var table1 = this.table;
+            var table2 = Object.create(null, {});
+            this.forEach(function(key, val) {
+                table2[key] = f.call(this, val, key);
+            });
+            return new StringMap(table2, this.size);
+        },
+        mapObject: function(f) {
+            var table1 = this.table;
+            var table2 = Object.create(null, {});
+            this.forEach(function(key, val) {
+                table2[key] = f.call(this, val, key);
+            });
+            return table2;
+        },
+        toObject: function() {
+            return this.mapObject(function(val) { return val; });
+        },
+        choose: function() {
+            return Object.getOwnPropertyNames(this.table)[0];
+        },
+        remove: function(x) {
+            if (hasOwnProperty.call(this.table, x)) {
+                this.size--;
+                delete this.table[x];
+            }
+        },
+        copy: function() {
+            var table = Object.create(null, {});
+            for (var key in this.table)
+                table[key] = this.table[key];
+            return new StringMap(table, this.size);
+        },
         toString: function() { return "[object StringMap]" }
-};
+    };
 
-// non-destructive stack
+    // an object-key table with poor asymptotics (replace with WeakMap when possible)
-function Stack(elts) {
+    function ObjectMap(array) {
+        this.array = array || [];
+    }
+
+    function searchMap(map, key, found, notFound) {
+        var a = map.array;
+        for (var i = 0, n = a.length; i < n; i++) {
+            var pair = a[i];
+            if (pair.key === key)
+                return found(pair, i);
+        }
+        return notFound();
+    }
+
+    ObjectMap.prototype = {
+        has: function(x) {
+            return searchMap(this, x, function() { return true }, function() { return false });
+        },
+        set: function(x, v) {
+            var a = this.array;
+            searchMap(this, x,
+                      function(pair) { pair.value = v },
+                      function() { a.push({ key: x, value: v }) });
+        },
+        get: function(x) {
+            return searchMap(this, x,
+                             function(pair) { return pair.value },
+                             function() { return null });
+        },
+        getDef: function(x, thunk) {
+            var a = this.array;
+            return searchMap(this, x,
+                             function(pair) { return pair.value },
+                             function() {
+                                 var v = thunk();
+                                 a.push({ key: x, value: v });
+                                 return v;
+                             });
+        },
+        forEach: function(f) {
+            var a = this.array;
+            for (var i = 0, n = a.length; i < n; i++) {
+                var pair = a[i];
+                f.call(this, pair.key, pair.value);
+            }
+        },
+        choose: function() {
+            return this.array[0].key;
+        },
+        get size() {
+            return this.array.length;
+        },
+        remove: function(x) {
+            var a = this.array;
+            searchMap(this, x,
+                      function(pair, i) { a.splice(i, 1) },
+                      function() { });
+        },
+        copy: function() {
+            return new ObjectMap(this.array.map(function(pair) {
+                return { key: pair.key, value: pair.value }
+            }));
+        },
+        clear: function() {
+            this.array = [];
+        },
+        toString: function() { return "[object ObjectMap]" }
+    };
+
+    // non-destructive stack
+    function Stack(elts) {
         this.elts = elts || null;
-}
+    }
 
-Stack.prototype = {
+    Stack.prototype = {
         push: function(x) {
             return new Stack({ top: x, rest: this.elts });
         },
@@ -355,21 +651,29 @@ Stack.prototype = {
                 f(elts.top);
             }
         }
-};
+    };
-
-exports.tokens = tokens;
-exports.opTypeNames = opTypeNames;
-exports.keywords = keywords;
-exports.isStatementStartCode = isStatementStartCode;
-exports.tokenIds = tokenIds;
-exports.consts = consts;
-exports.assignOps = assignOps;
-exports.defineGetter = defineGetter;
-exports.defineProperty = defineProperty;
-exports.isNativeCode = isNativeCode;
-exports.makePassthruHandler = makePassthruHandler;
-exports.noPropFound = noPropFound;
-exports.StringMap = StringMap;
-exports.Stack = Stack;
 
+    module.exports = {
+        tokens: tokens,
+        whitespace: whitespace,
+        opTypeNames: opTypeNames,
+        keywords: keywords,
+        isStatementStartCode: isStatementStartCode,
+        tokenIds: tokenIds,
+        consts: consts,
+        assignOps: assignOps,
+        defineGetter: defineGetter,
+        defineGetterSetter: defineGetterSetter,
+        defineMemoGetter: defineMemoGetter,
+        defineProperty: defineProperty,
+        isNativeCode: isNativeCode,
+        mirrorHandler: mirrorHandler,
+        mixinHandler: mixinHandler,
+        whitelistHandler: whitelistHandler,
+        blacklistHandler: blacklistHandler,
+        makePassthruHandler: makePassthruHandler,
+        StringMap: StringMap,
+        ObjectMap: ObjectMap,
+        Stack: Stack
+    };
 });

lib/ace/narcissus/jslex.js

@@ -47,16 +47,23 @@
  * Lexical scanner.
  */
 
-define(function(require, exports, module) {
+ define(function(require, exports, module) {
 
-var definitions = require("ace/narcissus/jsdefs");
+    var definitions = require("ace/narcissus/jsdefs");
 
-// Set constants in the local scope.
+    // Set constants in the local scope.
-eval(definitions.consts);
+    eval(definitions.consts);
 
-// Build up a trie of operator tokens.
+    // Banned keywords by language version
-var opTokens = {};
+    const blackLists = { 160: {}, 185: {}, harmony: {} };
-for (var op in definitions.opTypeNames) {
+    blackLists[160][LET] = true;
+    blackLists[160][MODULE] = true;
+    blackLists[160][YIELD] = true;
+    blackLists[185][MODULE] = true;
+
+    // Build up a trie of operator tokens.
+    var opTokens = {};
+    for (var op in definitions.opTypeNames) {
         if (op === '\n' || op === '.')
             continue;
 
@@ -68,12 +75,12 @@ for (var op in definitions.opTypeNames) {
             node = node[ch];
             node.op = op;
         }
-}
+    }
 
-/*
+    /*
      * Tokenizer :: (source, filename, line number) -> Tokenizer
      */
-function Tokenizer(s, f, l) {
+    function Tokenizer(s, f, l) {
         this.cursor = 0;
         this.source = String(s);
         this.tokens = [];
@@ -83,9 +90,11 @@ function Tokenizer(s, f, l) {
         this.unexpectedEOF = false;
         this.filename = f || "";
         this.lineno = l || 1;
-}
+        this.blackList = blackLists[Narcissus.options.version];
+        this.blockComments = null;
+    }
 
-Tokenizer.prototype = {
+    Tokenizer.prototype = {
         get done() {
             // We need to set scanOperand to true here because the first thing
             // might be a regexp.
@@ -129,16 +138,31 @@ Tokenizer.prototype = {
             return tt;
         },
 
+        lastBlockComment: function() {
+            var length = this.blockComments.length;
+            return length ? this.blockComments[length - 1] : null;
+        },
+
         // Eat comments and whitespace.
         skip: function () {
             var input = this.source;
+            this.blockComments = [];
             for (;;) {
                 var ch = input[this.cursor++];
                 var next = input[this.cursor];
+                // handle \r, \r\n and (always preferable) \n
+                if (ch === '\r') {
+                    // if the next character is \n, we don't care about this at all
+                    if (next === '\n') continue;
+
+                    // otherwise, we want to consider this as a newline
+                    ch = '\n';
+                }
+
                 if (ch === '\n' && !this.scanNewlines) {
                     this.lineno++;
                 } else if (ch === '/' && next === '*') {
-                this.cursor++;
+                    var commentStart = ++this.cursor;
                     for (;;) {
                         ch = input[this.cursor++];
                         if (ch === undefined)
@@ -147,6 +171,7 @@ Tokenizer.prototype = {
                         if (ch === '*') {
                             next = input[this.cursor];
                             if (next === '/') {
+                                var commentEnd = this.cursor - 1;
                                 this.cursor++;
                                 break;
                             }
@@ -154,19 +179,30 @@ Tokenizer.prototype = {
                             this.lineno++;
                         }
                     }
+                    this.blockComments.push(input.substring(commentStart, commentEnd));
                 } else if (ch === '/' && next === '/') {
                     this.cursor++;
                     for (;;) {
                         ch = input[this.cursor++];
+                        next = input[this.cursor];
                         if (ch === undefined)
                             return;
 
+                        if (ch === '\r') {
+                            // check for \r\n
+                            if (next !== '\n') ch = '\n';
+                        }
+
                         if (ch === '\n') {
+                            if (this.scanNewlines) {
+                                this.cursor--;
+                            } else {
                                 this.lineno++;
+                            }
                             break;
                         }
                     }
-            } else if (ch !== ' ' && ch !== '\t') {
+                } else if (!(ch in definitions.whitespace)) {
                     this.cursor--;
                     return;
                 }
@@ -363,19 +399,19 @@ Tokenizer.prototype = {
         },
 
         // FIXME: Unicode escape sequences
-    // FIXME: Unicode identifiers
         lexIdent: function (ch) {
-        var token = this.token, input = this.source;
+            var token = this.token;
+            var id = ch;
 
-        do {
+            while ((ch = this.getValidIdentifierChar(false)) !== null) {
-            ch = input[this.cursor++];
+                id += ch;
-        } while ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
+            }
-                 (ch >= '0' && ch <= '9') || ch === '$' || ch === '_');
 
-        this.cursor--;  // Put the non-word character back.
-
-        var id = input.substring(token.start, this.cursor);
             token.type = definitions.keywords[id] || IDENTIFIER;
+            if (token.type in this.blackList) {
+                // banned keyword, this is an identifier
+                token.type = IDENTIFIER;
+            }
             token.value = id;
         },
 
@@ -403,15 +439,16 @@ Tokenizer.prototype = {
                 this.tokens[this.tokenIndex] = token = {};
 
             var input = this.source;
-        if (this.cursor === input.length)
+            if (this.cursor >= input.length)
                 return token.type = END;
 
             token.start = this.cursor;
             token.lineno = this.lineno;
 
-        var ch = input[this.cursor++];
+            var ich = this.getValidIdentifierChar(true);
-        if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '$' || ch === '_') {
+            var ch = (ich === null) ? input[this.cursor++] : null;
-            this.lexIdent(ch);
+            if (ich !== null) {
+                this.lexIdent(ich);
             } else if (scanOperand && ch === '/') {
                 this.lexRegExp(ch);
             } else if (ch in opTokens) {
@@ -424,7 +461,9 @@ Tokenizer.prototype = {
                 this.lexZeroNumber(ch);
             } else if (ch === '"' || ch === "'") {
                 this.lexString(ch);
-        } else if (this.scanNewlines && ch === '\n') {
+            } else if (this.scanNewlines && (ch === '\n' || ch === '\r')) {
+                // if this was a \r, look for \r\n
+                if (ch === '\r' && input[this.cursor] === '\n') this.cursor++;
                 token.type = NEWLINE;
                 token.value = '\n';
                 this.lineno++;
@@ -447,16 +486,64 @@ Tokenizer.prototype = {
         },
 
         newSyntaxError: function (m) {
+            m = (this.filename ? this.filename + ":" : "") + this.lineno + ": " + m;
             var e = new SyntaxError(m, this.filename, this.lineno);
             e.source = this.source;
-        e.lineno = this.lineno;
             e.cursor = this.lookahead
                        ? this.tokens[(this.tokenIndex + this.lookahead) & 3].start
                        : this.cursor;
             return e;
         },
-};
 
-exports.Tokenizer = Tokenizer;
+        /* Gets a single valid identifier char from the input stream, or null
+         * if there is none.
+         * Since JavaScript provides no convenient way to determine if a
+         * character is in a particular Unicode category, we use
+         * metacircularity to accomplish this (oh yeaaaah!) */
+        getValidIdentifierChar: function(first) {
+            var input = this.source;
+            if (this.cursor >= input.length) return null;
+            var ch = input[this.cursor];
+
+            // first check for \u escapes
+            if (ch === '\\' && input[this.cursor+1] === 'u') {
+                // get the character value
+                try {
+                    ch = String.fromCharCode(parseInt(
+                        input.substring(this.cursor + 2, this.cursor + 6),
+                        16));
+                } catch (ex) {
+                    return null;
+                }
+                this.cursor += 5;
+            }
+
+            // check directly for ASCII
+            if (ch <= "\u007F") {
+                if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '$' || ch === '_' ||
+                    (!first && (ch >= '0' && ch <= '9'))) {
+                    this.cursor++;
+                    return ch;
+                }
+                return null;
+            }
+    
+            // create an object to test this in
+            var x = {};
+            x["x"+ch] = true;
+            x[ch] = true;
+    
+            // then use eval to determine if it's a valid character
+            var valid = false;
+            try {
+                valid = (Function("x", "return (x." + (first?"":"x") + ch + ");")(x) === true);
+            } catch (ex) {}
+            if (valid) this.cursor++;
+            return (valid ? ch : null);
+        },
+    };
+
+
+    return { Tokenizer: Tokenizer };
 
 });