apply Kevin's refactoring

- lower case file names
- use define(function(require, exports, module) {..})

lib/ace/_tokenizer.js

@@ -0,0 +1,132 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Ajax.org Code Editor (ACE).
+ *
+ * The Initial Developer of the Original Code is
+ * Ajax.org Services B.V.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *      Fabian Jakobs <fabian AT ajax DOT org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+
+var Tokenizer = function(rules) {
+    this.rules = rules;
+
+    this.regExps = {};
+    for ( var key in this.rules) {
+        var state = this.rules[key];
+        var ruleRegExps = [];
+
+        for ( var i = 0; i < state.length; i++) {
+            ruleRegExps.push(state[i].regex);
+        };
+
+        this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
+    }
+};
+
+(function() {
+
+    this.getLineTokens = function(line, startState) {
+        var currentState = startState;
+        var state = this.rules[currentState];
+        var re = this.regExps[currentState];
+        re.lastIndex = 0;
+
+        var match, tokens = [];
+
+        var lastIndex = 0;
+
+        var token = {
+            type: null,
+            value: ""
+        };
+
+        while (match = re.exec(line)) {
+            var type = "text";
+            var value = match[0];
+
+            if (re.lastIndex == lastIndex) { throw new Error("tokenizer error"); }
+            lastIndex = re.lastIndex;
+
+            window.LOG && console.log(currentState, match);
+
+            for ( var i = 0; i < state.length; i++) {
+                if (match[i + 1]) {
+                    if (typeof state[i].token == "function") {
+                        type = state[i].token(match[0]);
+                    }
+                    else {
+                        type = state[i].token;
+                    }
+
+                    if (state[i].next && state[i].next !== currentState) {
+                        currentState = state[i].next;
+                        var state = this.rules[currentState];
+                        var lastIndex = re.lastIndex;
+
+                        var re = this.regExps[currentState];
+                        re.lastIndex = lastIndex;
+                    }
+                    break;
+                }
+            };
+            
+                  
+            if (token.type !== type) {
+                if (token.type) {
+                    tokens.push(token);
+                }
+                token = {
+                    type: type,
+                    value: value
+                };
+            } else {
+                token.value += value;
+            }
+        };
+
+        if (token.type) {
+            tokens.push(token);
+        }
+
+        window.LOG && console.log(tokens, currentState);
+
+        return {
+            tokens : tokens,
+            state : currentState
+        };
+    };
+
+}).call(Tokenizer.prototype);
+
+return Tokenizer;
+});