Merge branch 'match-on-tokens' of https://github.com/dgeorge/ace into dgeorge-match-on-tokens

Conflicts:
	lib/ace/edit_session.js
	lib/ace/edit_session_test.js
	lib/ace/test/all_browser.js

lib/ace/edit_session.js

@@ -612,98 +612,6 @@ var EditSession = function(text, mode) {
         return this.doc.getTextRange(range);
     };
 
-    this.findMatchingBracket = function(position) {
-        if (position.column == 0) return null;
-
-        var charBeforeCursor = this.getLine(position.row).charAt(position.column-1);
-        if (charBeforeCursor == "") return null;
-
-        var match = charBeforeCursor.match(/([\(\[\{])|([\)\]\}])/);
-        if (!match) {
-            return null;
-        }
-
-        if (match[1]) {
-            return this.$findClosingBracket(match[1], position);
-        } else {
-            return this.$findOpeningBracket(match[2], position);
-        }
-    };
-
-    this.$brackets = {
-        ")": "(",
-        "(": ")",
-        "]": "[",
-        "[": "]",
-        "{": "}",
-        "}": "{"
-    };
-
-    this.$findOpeningBracket = function(bracket, position) {
-        var openBracket = this.$brackets[bracket];
-
-        var column = position.column - 2;
-        var row = position.row;
-        var depth = 1;
-
-        var line = this.getLine(row);
-
-        while (true) {
-            while(column >= 0) {
-                var ch = line.charAt(column);
-                if (ch == openBracket) {
-                    depth -= 1;
-                    if (depth == 0) {
-                        return {row: row, column: column};
-                    }
-                }
-                else if (ch == bracket) {
-                    depth +=1;
-                }
-                column -= 1;
-            }
-            row -=1;
-            if (row < 0) break;
-
-            var line = this.getLine(row);
-            var column = line.length-1;
-        }
-        return null;
-    };
-
-    this.$findClosingBracket = function(bracket, position) {
-        var closingBracket = this.$brackets[bracket];
-
-        var column = position.column;
-        var row = position.row;
-        var depth = 1;
-
-        var line = this.getLine(row);
-        var lineCount = this.getLength();
-
-        while (true) {
-            while(column < line.length) {
-                var ch = line.charAt(column);
-                if (ch == closingBracket) {
-                    depth -= 1;
-                    if (depth == 0) {
-                        return {row: row, column: column};
-                    }
-                }
-                else if (ch == bracket) {
-                    depth +=1;
-                }
-                column += 1;
-            }
-            row +=1;
-            if (row >= lineCount) break;
-
-            var line = this.getLine(row);
-            var column = 0;
-        }
-        return null;
-    };
-
     this.insert = function(position, text) {
         return this.doc.insert(position, text);
     };
@@ -1736,6 +1644,7 @@ var EditSession = function(text, mode) {
 }).call(EditSession.prototype);
 
 require("./edit_session/folding").Folding.call(EditSession.prototype);
+require("./edit_session/bracket_match").BracketMatch.call(EditSession.prototype);
 
 exports.EditSession = EditSession;
 });

lib/ace/edit_session/bracket_match.js

@@ -0,0 +1,180 @@
+/* vim:ts=4:sts=4:sw=4:
+ * ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Ajax.org Code Editor (ACE).
+ *
+ * The Initial Developer of the Original Code is
+ * Ajax.org B.V.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *      Fabian Jakobs <fabian AT ajax DOT org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+
+var TokenIterator = require("ace/token_iterator").TokenIterator;
+
+function BracketMatch() {
+
+    this.findMatchingBracket = function(position) {
+        if (position.column == 0) return null;
+
+        var charBeforeCursor = this.getLine(position.row).charAt(position.column-1);
+        if (charBeforeCursor == "") return null;
+
+        var match = charBeforeCursor.match(/([\(\[\{])|([\)\]\}])/);
+        if (!match) {
+            return null;
+        }
+
+        if (match[1]) {
+            return this.$findClosingBracket(match[1], position);
+        } else {
+            return this.$findOpeningBracket(match[2], position);
+        }
+    };
+
+    this.$brackets = {
+        ")": "(",
+        "(": ")",
+        "]": "[",
+        "[": "]",
+        "{": "}",
+        "}": "{"
+    };
+
+    this.$findOpeningBracket = function(bracket, position) {
+        var openBracket = this.$brackets[bracket];
+        var depth = 1;
+
+        var iterator = new TokenIterator(this, position.row, position.column);
+        var token = iterator.getCurrentToken();
+        if (!token) return null;
+        
+        // token.type contains a period-delimited list of token identifiers
+        // (e.g.: "constant.numeric" or "paren.lparen").  Create a pattern that
+        // matches any token containing the same identifiers or a subset.  In
+        // addition, if token.type includes "rparen", then also match "lparen".
+        // So if type.token is "paren.rparen", then typeRe will match "lparen.paren".
+        var typeRe = new RegExp("(\\.?" +
+            token.type.replace(".", "|").replace("rparen", "lparen|rparen") + ")+");
+        
+        // Start searching in token, just before the character at position.column
+        var valueIndex = position.column - iterator.getCurrentTokenColumn() - 2;
+        var value = token.value;
+        
+        while (true) {
+        
+            while (valueIndex >= 0) {
+                var char = value.charAt(valueIndex);
+                if (char == openBracket) {
+                    depth -= 1;
+                    if (depth == 0) {
+                        return {row: iterator.getCurrentTokenRow(),
+                            column: valueIndex + iterator.getCurrentTokenColumn()};
+                    }
+                }
+                else if (char == bracket) {
+                    depth += 1;
+                }
+                valueIndex -= 1;
+            }
+
+            // Scan backward through the document, looking for the next token
+            // whose type matches typeRe
+            do {
+                token = iterator.stepBackward();
+            } while (token && !typeRe.test(token.type));
+
+            if (token == null)
+                break;
+                
+            value = token.value;
+            valueIndex = value.length - 1;
+        }
+        
+        return null;
+    };
+
+    this.$findClosingBracket = function(bracket, position) {
+        var closingBracket = this.$brackets[bracket];
+        var depth = 1;
+
+        var iterator = new TokenIterator(this, position.row, position.column);
+        var token = iterator.getCurrentToken();
+        if (!token) return null;
+
+        // token.type contains a period-delimited list of token identifiers
+        // (e.g.: "constant.numeric" or "paren.lparen").  Create a pattern that
+        // matches any token containing the same identifiers or a subset.  In
+        // addition, if token.type includes "lparen", then also match "rparen".
+        // So if type.token is "lparen.paren", then typeRe will match "paren.rparen".
+        var typeRe = new RegExp("(\\.?" +
+            token.type.replace(".", "|").replace("lparen", "lparen|rparen") + ")+");
+
+        // Start searching in token, after the character at position.column
+        var valueIndex = position.column - iterator.getCurrentTokenColumn();
+
+        while (true) {
+
+            var value = token.value;
+            var valueLength = value.length;
+            while (valueIndex < valueLength) {
+                var char = value.charAt(valueIndex);
+                if (char == closingBracket) {
+                    depth -= 1;
+                    if (depth == 0) {
+                        return {row: iterator.getCurrentTokenRow(),
+                            column: valueIndex + iterator.getCurrentTokenColumn()};
+                    }
+                }
+                else if (char == bracket) {
+                    depth += 1;
+                }
+                valueIndex += 1;
+            }
+
+            // Scan forward through the document, looking for the next token
+            // whose type matches typeRe
+            do {
+                token = iterator.stepForward();
+            } while (token && !typeRe.test(token.type));
+
+            if (token == null)
+                break;
+
+            valueIndex = 0;
+        }
+        
+        return null;
+    };
+}
+exports.BracketMatch = BracketMatch;
+
+});

lib/ace/edit_session_test.js

@@ -50,6 +50,7 @@ var UndoManager = require("./undomanager").UndoManager;
 var MockRenderer = require("./test/mockrenderer").MockRenderer;
 var Range = require("./range").Range;
 var assert = require("./test/assertions");
+var JavaScriptMode = require("./mode/javascript").Mode;
 
 function createFoldTestSession() {
     var lines = [
@@ -69,7 +70,7 @@ function createFoldTestSession() {
 
 module.exports = {
 
-   "test: find matching opening bracket" : function() {
+   "test: find matching opening bracket in Text mode" : function() {
         var session = new EditSession(["(()(", "())))"]);
 
         assert.position(session.findMatchingBracket({row: 0, column: 3}), 0, 1);
@@ -79,7 +80,7 @@ module.exports = {
         assert.equal(session.findMatchingBracket({row: 1, column: 5}), null);
     },
 
-    "test: find matching closing bracket" : function() {
+    "test: find matching closing bracket in Text mode" : function() {
         var session = new EditSession(["(()(", "())))"]);
 
         assert.position(session.findMatchingBracket({row: 1, column: 1}), 1, 1);
@@ -90,6 +91,67 @@ module.exports = {
         assert.equal(session.findMatchingBracket({row: 0, column: 0}), null);
     },
 
+    "test: find matching opening bracket in JavaScript mode" : function() {
+        var lines = [
+            "function foo() {",
+            "    var str = \"{ foo()\";",
+            "    if (debug) {",
+            "        // write str (a string) to the console",
+            "        console.log(str);",
+            "    }",
+            "    str += \" bar() }\";",
+            "}"
+        ];
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());
+
+        assert.position(session.findMatchingBracket({row: 0, column: 14}), 0, 12);
+        assert.position(session.findMatchingBracket({row: 7, column: 1}), 0, 15);
+        assert.position(session.findMatchingBracket({row: 6, column: 20}), 1, 15);
+        assert.position(session.findMatchingBracket({row: 1, column: 22}), 1, 20);
+        assert.position(session.findMatchingBracket({row: 3, column: 31}), 3, 21);
+        assert.position(session.findMatchingBracket({row: 4, column: 24}), 4, 19);        
+        assert.equal(session.findMatchingBracket({row: 0, column: 1}), null);
+    },
+    
+    "test: find matching closing bracket in JavaScript mode" : function() {
+        var lines = [
+            "function foo() {",
+            "    var str = \"{ foo()\";",
+            "    if (debug) {",
+            "        // write str (a string) to the console",
+            "        console.log(str);",
+            "    }",
+            "    str += \" bar() }\";",
+            "}"
+        ];
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());
+
+        assert.position(session.findMatchingBracket({row: 0, column: 13}), 0, 13);
+        assert.position(session.findMatchingBracket({row: 0, column: 16}), 7, 0);
+        assert.position(session.findMatchingBracket({row: 1, column: 16}), 6, 19);
+        assert.position(session.findMatchingBracket({row: 1, column: 21}), 1, 21);
+        assert.position(session.findMatchingBracket({row: 3, column: 22}), 3, 30);
+        assert.position(session.findMatchingBracket({row: 4, column: 20}), 4, 23);        
+    },
+
+    "test: handle unbalanced brackets in JavaScript mode" : function() {
+        var lines = [
+            "function foo() {",
+            "    var str = \"{ foo()\";",
+            "    if (debug) {",
+            "        // write str a string) to the console",
+            "        console.log(str);",
+            "    ",
+            "    str += \" bar() \";",
+            "}"
+        ];
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());
+
+        assert.equal(session.findMatchingBracket({row: 0, column: 16}), null);
+        assert.equal(session.findMatchingBracket({row: 3, column: 30}), null);
+        assert.equal(session.findMatchingBracket({row: 1, column: 16}), null);
+    },
+
     "test: match different bracket types" : function() {
         var session = new EditSession(["({[", ")]}"]);
 

lib/ace/mode/javascript_test.js

@@ -141,7 +141,7 @@ module.exports = {
     },
 
     "test: auto outdent should indent the line with the same indent as the line with the matching opening brace" : function() {
-        var session = new EditSession(["  function foo() {", "    bla", "    }"]);
+        var session = new EditSession(["  function foo() {", "    bla", "    }"], new JavaScriptMode());
         this.mode.autoOutdent("start", session, 2);
         assert.equal("  }", session.getLine(2));
     },

lib/ace/test/all_browser.js

@@ -39,6 +39,7 @@ var tests = [
      require("ace/range_test"),
      require("ace/search_test"),
      require("ace/selection_test"),
+     require("ace/token_iterator_test"),
      require("ace/virtual_renderer_test")
 ]
 

lib/ace/token_iterator.js

@@ -0,0 +1,112 @@
+/* vim:ts=4:sts=4:sw=4:
+ * ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Ajax.org Code Editor (ACE).
+ *
+ * The Initial Developer of the Original Code is
+ * Ajax.org B.V.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *      Fabian Jakobs <fabian AT ajax DOT org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+
+var TokenIterator = function(session, initialRow, initialColumn) {
+    this.$session = session;
+    this.$row = initialRow;
+    this.$rowTokens = session.getTokens(initialRow, initialRow)[0].tokens;
+
+    var token = session.getTokenAt(initialRow, initialColumn);
+    this.$tokenIndex = token ? token.index : -1;
+};
+
+(function() {
+    
+    this.stepBackward = function() {
+        this.$tokenIndex -= 1;
+        
+        while (this.$tokenIndex < 0) {
+            this.$row -= 1;
+            if (this.$row < 0)
+                return null;
+                
+            this.$rowTokens = this.$session.getTokens(this.$row, this.$row)[0].tokens;
+            this.$tokenIndex = this.$rowTokens.length - 1;
+        }
+            
+        return this.$rowTokens[this.$tokenIndex];
+    }
+    
+    this.stepForward = function() {
+        var rowCount = this.$session.getLength();
+        this.$tokenIndex += 1;
+        
+        while (this.$tokenIndex >= this.$rowTokens.length) {
+            this.$row += 1;
+            if (this.$row >= rowCount)
+                return null;
+
+            this.$rowTokens = this.$session.getTokens(this.$row, this.$row)[0].tokens;
+            this.$tokenIndex = 0;
+        }
+            
+        return this.$rowTokens[this.$tokenIndex];
+    }
+    
+    this.getCurrentToken = function () {
+        return this.$rowTokens[this.$tokenIndex];
+    }
+    
+    this.getCurrentTokenRow = function () {
+        return this.$row;
+    }
+    
+    this.getCurrentTokenColumn = function() {
+        var rowTokens = this.$rowTokens;
+        var tokenIndex = this.$tokenIndex;
+        
+        // If a column was cached by EditSession.getTokenAt, then use it
+        var column = rowTokens[tokenIndex].start;
+        if (column !== undefined)
+            return column;
+            
+        column = 0;
+        while (tokenIndex > 0) {
+            tokenIndex -= 1;
+            column += rowTokens[tokenIndex].value.length;
+        }
+        
+        return column;  
+    }
+            
+}).call(TokenIterator.prototype);
+
+exports.TokenIterator = TokenIterator;
+});

lib/ace/token_iterator_test.js

@@ -0,0 +1,218 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Ajax.org Code Editor (ACE).
+ *
+ * The Initial Developer of the Original Code is
+ * Ajax.org B.V.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *      Fabian Jakobs <fabian AT ajax DOT org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+if (typeof process !== "undefined") {
+    require("../../support/paths");
+}
+
+define(function(require, exports, module) {
+
+var EditSession = require("ace/edit_session").EditSession;
+var JavaScriptMode = require("ace/mode/javascript").Mode;
+var TokenIterator = require("ace/token_iterator").TokenIterator;
+var assert = require("ace/test/assertions");
+
+module.exports = {
+    "test: token iterator initialization in JavaScript document" : function() {
+        var lines = [
+            "function foo(items) {",
+            "    for (var i=0; i<items.length; i++) {",
+            "        alert(items[i] + \"juhu\");",
+            "    } // Real Tab.",
+            "}"
+        ];
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());      
+
+        var iterator = new TokenIterator(session, 0, 0);
+        assert.equal(iterator.getCurrentToken().value, "function");
+        assert.equal(iterator.getCurrentTokenRow(), 0);
+        assert.equal(iterator.getCurrentTokenColumn(), 0);
+        
+        iterator.stepForward();
+        assert.equal(iterator.getCurrentToken().value, " ");
+        assert.equal(iterator.getCurrentTokenRow(), 0);
+        assert.equal(iterator.getCurrentTokenColumn(), 8);      
+
+        var iterator = new TokenIterator(session, 0, 4);
+        assert.equal(iterator.getCurrentToken().value, "function");
+        assert.equal(iterator.getCurrentTokenRow(), 0);
+        assert.equal(iterator.getCurrentTokenColumn(), 0);
+
+        iterator.stepForward();
+        assert.equal(iterator.getCurrentToken().value, " ");
+        assert.equal(iterator.getCurrentTokenRow(), 0);
+        assert.equal(iterator.getCurrentTokenColumn(), 8);      
+
+        var iterator = new TokenIterator(session, 2, 18);
+        assert.equal(iterator.getCurrentToken().value, "items");
+        assert.equal(iterator.getCurrentTokenRow(), 2);
+        assert.equal(iterator.getCurrentTokenColumn(), 14);
+
+        iterator.stepForward();
+        assert.equal(iterator.getCurrentToken().value, "[");
+        assert.equal(iterator.getCurrentTokenRow(), 2);
+        assert.equal(iterator.getCurrentTokenColumn(), 19);     
+        
+        var iterator = new TokenIterator(session, 4, 0);
+        assert.equal(iterator.getCurrentToken().value, "}");
+        assert.equal(iterator.getCurrentTokenRow(), 4);
+        assert.equal(iterator.getCurrentTokenColumn(), 0);
+
+        iterator.stepBackward();
+        assert.equal(iterator.getCurrentToken().value, "// Real Tab.");
+        assert.equal(iterator.getCurrentTokenRow(), 3);
+        assert.equal(iterator.getCurrentTokenColumn(), 6);      
+       
+        var iterator = new TokenIterator(session, 5, 0);
+        assert.equal(iterator.getCurrentToken(), null);
+    },
+ 
+    "test: token iterator initialization in text document" : function() {
+        var lines = [
+            "Lorem ipsum dolor sit amet, consectetur adipisicing elit,", 
+            "sed do eiusmod tempor incididunt ut labore et dolore magna",
+            "aliqua. Ut enim ad minim veniam, quis nostrud exercitation", 
+            "ullamco laboris nisi ut aliquip ex ea commodo consequat."
+        ];
+        var session = new EditSession(lines.join("\n"));
+        
+        var iterator = new TokenIterator(session, 0, 0);
+        assert.equal(iterator.getCurrentToken().value, lines[0]);
+        assert.equal(iterator.getCurrentTokenRow(), 0);
+        assert.equal(iterator.getCurrentTokenColumn(), 0);
+
+        var iterator = new TokenIterator(session, 0, 4);
+        assert.equal(iterator.getCurrentToken().value, lines[0]);
+        assert.equal(iterator.getCurrentTokenRow(), 0);
+        assert.equal(iterator.getCurrentTokenColumn(), 0);
+
+        var iterator = new TokenIterator(session, 2, 18);
+        assert.equal(iterator.getCurrentToken().value, lines[2]);
+        assert.equal(iterator.getCurrentTokenRow(), 2);
+        assert.equal(iterator.getCurrentTokenColumn(), 0);
+        
+        var iterator = new TokenIterator(session, 3, lines[3].length-1);
+        assert.equal(iterator.getCurrentToken().value, lines[3]);
+        assert.equal(iterator.getCurrentTokenRow(), 3);
+        assert.equal(iterator.getCurrentTokenColumn(), 0);
+       
+        var iterator = new TokenIterator(session, 4, 0);
+        assert.equal(iterator.getCurrentToken(), null);
+    }, 
+    
+    "test: token iterator step forward in JavaScript document" : function() {
+        var lines = [
+            "function foo(items) {",
+            "    for (var i=0; i<items.length; i++) {",
+            "        alert(items[i] + \"juhu\");",
+            "    } // Real Tab.",
+            "}"
+        ];
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());      
+    
+        var rows = session.getTokens(0, lines.length-1);
+        var tokens = [];    
+        for (var i = 0; i < rows.length; i++)
+            tokens = tokens.concat(rows[i].tokens);
+
+        var iterator = new TokenIterator(session, 0, 0);
+        for (var i = 1; i < tokens.length; i++)
+            assert.equal(iterator.stepForward(), tokens[i]);
+        assert.equal(iterator.stepForward(), null);
+        assert.equal(iterator.getCurrentToken(), null);
+    },
+    
+    "test: token iterator step backward in JavaScript document" : function() {
+        var lines = [
+            "function foo(items) {",
+            "     for (var i=0; i<items.length; i++) {",
+            "         alert(items[i] + \"juhu\");",
+            "     } // Real Tab.",
+            "}"
+        ];
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());      
+
+        var rows = session.getTokens(0, lines.length-1);
+        var tokens = [];    
+        for (var i = 0; i < rows.length; i++)
+            tokens = tokens.concat(rows[i].tokens);
+    
+        var iterator = new TokenIterator(session, 4, 0);
+        for (var i = tokens.length-2; i >= 0; i--)
+            assert.equal(iterator.stepBackward(), tokens[i]);
+        assert.equal(iterator.stepBackward(), null);
+        assert.equal(iterator.getCurrentToken(), null);
+    },
+
+    "test: token iterator reports correct row and column" : function() {
+        var lines = [
+            "function foo(items) {",
+            "    for (var i=0; i<items.length; i++) {",
+            "        alert(items[i] + \"juhu\");",
+            "    } // Real Tab.",
+            "}"
+        ];
+        var session = new EditSession(lines.join("\n"), new JavaScriptMode());      
+
+        var iterator = new TokenIterator(session, 0, 0);
+        
+        iterator.stepForward();
+        iterator.stepForward();
+        
+        assert.equal(iterator.getCurrentToken().value, "foo");
+        assert.equal(iterator.getCurrentTokenRow(), 0);
+        assert.equal(iterator.getCurrentTokenColumn(), 9);
+
+        iterator.stepForward();
+        iterator.stepForward();
+        iterator.stepForward();
+        iterator.stepForward();
+        iterator.stepForward();
+        iterator.stepForward();
+        iterator.stepForward();
+
+        assert.equal(iterator.getCurrentToken().value, "for");
+        assert.equal(iterator.getCurrentTokenRow(), 1);
+        assert.equal(iterator.getCurrentTokenColumn(), 4);
+    },
+};
+
+});
+
+if (typeof module !== "undefined" && module === require.main) {
+    require("asyncjs").test.testcase(module.exports).exec()
+}