Add initial lexer and test

tests/__init__.py

@@ -0,0 +1,3 @@
+import sys, os
+myPath = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, myPath + '/../')

tests/example.ath

@@ -0,0 +1,6 @@
+def stuff(vars: int): string =
+    let x: float = 3.4
+    var
+      t: int = 4
+      junk: ClassName = ClassName()
+

tests/indents.ath

@@ -0,0 +1,3 @@
+x
+    
+        

tests/test_lexer.py

@@ -0,0 +1,41 @@
+import pytest
+from atheris.lexer import Lexer
+from atheris.token import TokenType
+
+@pytest.fixture(scope="module")
+def lexer():
+    return Lexer(open('example.ath'))
+
+def test_hanging_indents():
+    lexer = Lexer('x\n    \n        \n')
+    tokens = list(lexer.tokens())
+
+    assert tokens[0].token_type == TokenType.IDENT
+    assert tokens[0].content == 'x'
+    assert tokens[0].line == 1
+    assert tokens[0].col == 1
+
+    assert tokens[1].token_type == TokenType.INDENT
+    assert tokens[1].content == 4
+    assert tokens[1].line == 2
+    assert tokens[1].col == 1
+
+    assert tokens[2].token_type == TokenType.INDENT
+    assert tokens[2].content == 4
+    assert tokens[2].line == 3
+    assert tokens[2].col == 4
+
+    assert tokens[3].token_type == TokenType.DEDENT
+    assert tokens[3].content == 4
+    assert tokens[3].line == 4
+    assert tokens[3].col == 1
+
+    assert tokens[4].token_type == TokenType.DEDENT
+    assert tokens[4].content == 4
+    assert tokens[4].line == 4
+    assert tokens[4].col == 1
+
+    assert tokens[5].token_type == TokenType.EOF
+    assert tokens[5].content == ''
+    assert tokens[5].line == 4
+    assert tokens[5].col == 1