ace.provide("ace.Tokenizer");

ace.Tokenizer = function(rules) {
    this.rules = rules;

    this.regExps = {};
    for ( var key in this.rules) {
        var state = this.rules[key];
        var ruleRegExps = [];

        for ( var i = 0; i < state.length; i++) {
            ruleRegExps.push(state[i].regex);
        };

        this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(")
                + ")|(.))", "g");
    }
};

ace.Tokenizer.prototype.getLineTokens = function(line, startState) {
    var currentState = startState;
    var state = this.rules[currentState];
    var re = this.regExps[currentState];
    re.lastIndex = 0;

    var match, tokens = [];

    var lastIndex = 0;

    while (match = re.exec(line)) {
        var token = {
            type : "text",
            value : match[0]
        };

        if (re.lastIndex == lastIndex) { throw new Error("tokenizer error"); }
        lastIndex = re.lastIndex;

//        window.LOG && console.log(match);

        for ( var i = 0; i < state.length; i++) {
            if (match[i + 1]) {
                if (typeof state[i].token == "function") {
                    token.type = state[i].token(match[0]);
                }
                else {
                    token.type = state[i].token;
                }

                if (state[i].next && state[i].next !== currentState) {
                    currentState = state[i].next;
                    var state = this.rules[currentState];
                    var lastIndex = re.lastIndex;

                    var re = this.regExps[currentState];
                    re.lastIndex = lastIndex;
                }
                break;
            }
        };

        tokens.push(token);
    };

//    window.LOG && console.log(tokens, currentState);

    return {
        tokens : tokens,
        state : currentState
    };
};