From 63437014b9a5236a1085e62718c6fb5551ffe684 Mon Sep 17 00:00:00 2001
From: Joey Payne <jyapayne@gmail.com>
Date: Sun, 22 Jan 2017 21:15:52 -0700
Subject: [PATCH] Add tests for most of lexer

---
 tests/config.py     |  3 +++
 tests/example.ath   |  2 +-
 tests/test_lexer.py | 41 ++++++++++++++++++++++++++++++++++++++---
 3 files changed, 42 insertions(+), 4 deletions(-)
 create mode 100644 tests/config.py

diff --git a/tests/config.py b/tests/config.py
new file mode 100644
index 0000000..bf52d09
--- /dev/null
+++ b/tests/config.py
@@ -0,0 +1,3 @@
+import os
+
+TEST_PATH = os.path.dirname(__file__)
diff --git a/tests/example.ath b/tests/example.ath
index 009d70d..394d630 100755
--- a/tests/example.ath
+++ b/tests/example.ath
@@ -3,4 +3,4 @@ def stuff(vars: int): string =
     var
       t: int = 4
       junk: ClassName = ClassName()
-
+    a and b or c not d and e is f
diff --git a/tests/test_lexer.py b/tests/test_lexer.py
index 7d390a7..bec68a3 100755
--- a/tests/test_lexer.py
+++ b/tests/test_lexer.py
@@ -2,9 +2,44 @@ import pytest
 from atheris.lexer import Lexer
 from atheris.token import TokenType
 
-@pytest.fixture(scope="module")
-def lexer():
-    return Lexer(open('example.ath'))
+from .config import TEST_PATH
+
+def test_large_file():
+    lexer = Lexer(open(TEST_PATH+'/example.ath'))
+    tokens = list(lexer.tokens())
+
+def test_idents():
+    lexer = Lexer('junk flop')
+    tokens = list(lexer.tokens())
+    assert tokens[0].token_type == TokenType.IDENT
+
+def test_comments():
+    lexer = Lexer('# balls and stuff')
+    tokens = list(lexer.tokens())
+    assert tokens[0].token_type == TokenType.COMMENT
+    assert tokens[0].content == ' balls and stuff'
+
+def test_keywords():
+    lexer = Lexer('def let var int float for in class')
+    tokens = list(lexer.tokens())
+
+    assert tokens[0].token_type == TokenType.DEF
+    assert tokens[1].token_type == TokenType.LET
+    assert tokens[2].token_type == TokenType.VAR
+    assert tokens[3].token_type == TokenType.INT
+    assert tokens[4].token_type == TokenType.FLOAT
+    assert tokens[5].token_type == TokenType.FOR
+    assert tokens[6].token_type == TokenType.IN
+    assert tokens[7].token_type == TokenType.CLASS
+
+def test_simple_codeblock():
+    lexer = Lexer('def stuff():\n    pass')
+    tokens = list(lexer.tokens())
+
+    assert tokens[5].token_type == TokenType.INDENT
+    assert tokens[5].content == 4
+    assert tokens[7].token_type == TokenType.DEDENT
+    assert tokens[7].content == 4
 
 def test_hanging_indents():
     lexer = Lexer('x\n    \n        \n')