Add tests for most of lexer

tests/config.py

@@ -0,0 +1,3 @@
+import os
+
+TEST_PATH = os.path.dirname(__file__)

tests/example.ath

@@ -3,4 +3,4 @@ def stuff(vars: int): string =
     var
       t: int = 4
       junk: ClassName = ClassName()
-
+    a and b or c not d and e is f

tests/test_lexer.py

@@ -2,9 +2,44 @@ import pytest
 from atheris.lexer import Lexer
 from atheris.token import TokenType
 
-@pytest.fixture(scope="module")
+from .config import TEST_PATH
-def lexer():
+
-    return Lexer(open('example.ath'))
+def test_large_file():
+    lexer = Lexer(open(TEST_PATH+'/example.ath'))
+    tokens = list(lexer.tokens())
+
+def test_idents():
+    lexer = Lexer('junk flop')
+    tokens = list(lexer.tokens())
+    assert tokens[0].token_type == TokenType.IDENT
+
+def test_comments():
+    lexer = Lexer('# balls and stuff')
+    tokens = list(lexer.tokens())
+    assert tokens[0].token_type == TokenType.COMMENT
+    assert tokens[0].content == ' balls and stuff'
+
+def test_keywords():
+    lexer = Lexer('def let var int float for in class')
+    tokens = list(lexer.tokens())
+
+    assert tokens[0].token_type == TokenType.DEF
+    assert tokens[1].token_type == TokenType.LET
+    assert tokens[2].token_type == TokenType.VAR
+    assert tokens[3].token_type == TokenType.INT
+    assert tokens[4].token_type == TokenType.FLOAT
+    assert tokens[5].token_type == TokenType.FOR
+    assert tokens[6].token_type == TokenType.IN
+    assert tokens[7].token_type == TokenType.CLASS
+
+def test_simple_codeblock():
+    lexer = Lexer('def stuff():\n    pass')
+    tokens = list(lexer.tokens())
+
+    assert tokens[5].token_type == TokenType.INDENT
+    assert tokens[5].content == 4
+    assert tokens[7].token_type == TokenType.DEDENT
+    assert tokens[7].content == 4
 
 def test_hanging_indents():
     lexer = Lexer('x\n    \n        \n')