Add tests for most of lexer

tests/test_lexer.py

@@ -2,9 +2,44 @@ import pytest
 from atheris.lexer import Lexer
 from atheris.token import TokenType
 
-@pytest.fixture(scope="module")
-def lexer():
-    return Lexer(open('example.ath'))
+from .config import TEST_PATH
+
+def test_large_file():
+    lexer = Lexer(open(TEST_PATH+'/example.ath'))
+    tokens = list(lexer.tokens())
+
+def test_idents():
+    lexer = Lexer('junk flop')
+    tokens = list(lexer.tokens())
+    assert tokens[0].token_type == TokenType.IDENT
+
+def test_comments():
+    lexer = Lexer('# balls and stuff')
+    tokens = list(lexer.tokens())
+    assert tokens[0].token_type == TokenType.COMMENT
+    assert tokens[0].content == ' balls and stuff'
+
+def test_keywords():
+    lexer = Lexer('def let var int float for in class')
+    tokens = list(lexer.tokens())
+
+    assert tokens[0].token_type == TokenType.DEF
+    assert tokens[1].token_type == TokenType.LET
+    assert tokens[2].token_type == TokenType.VAR
+    assert tokens[3].token_type == TokenType.INT
+    assert tokens[4].token_type == TokenType.FLOAT
+    assert tokens[5].token_type == TokenType.FOR
+    assert tokens[6].token_type == TokenType.IN
+    assert tokens[7].token_type == TokenType.CLASS
+
+def test_simple_codeblock():
+    lexer = Lexer('def stuff():\n    pass')
+    tokens = list(lexer.tokens())
+
+    assert tokens[5].token_type == TokenType.INDENT
+    assert tokens[5].content == 4
+    assert tokens[7].token_type == TokenType.DEDENT
+    assert tokens[7].content == 4
 
 def test_hanging_indents():
     lexer = Lexer('x\n    \n        \n')