76 lines
2.3 KiB
Python
76 lines
2.3 KiB
Python
import pytest
|
|
from atheris.lexer import Lexer
|
|
from atheris.token import TokenType
|
|
|
|
from .config import TEST_PATH
|
|
|
|
def test_large_file():
|
|
lexer = Lexer(open(TEST_PATH+'/example.ath'))
|
|
tokens = list(lexer.tokens())
|
|
|
|
def test_idents():
|
|
lexer = Lexer('junk flop')
|
|
tokens = list(lexer.tokens())
|
|
assert tokens[0].token_type == TokenType.IDENT
|
|
|
|
def test_comments():
|
|
lexer = Lexer('# balls and stuff')
|
|
tokens = list(lexer.tokens())
|
|
assert tokens[0].token_type == TokenType.COMMENT
|
|
assert tokens[0].content == ' balls and stuff'
|
|
|
|
def test_keywords():
|
|
lexer = Lexer('def let var int float for in class')
|
|
tokens = list(lexer.tokens())
|
|
|
|
assert tokens[0].token_type == TokenType.DEF
|
|
assert tokens[1].token_type == TokenType.LET
|
|
assert tokens[2].token_type == TokenType.VAR
|
|
assert tokens[3].token_type == TokenType.INT
|
|
assert tokens[4].token_type == TokenType.FLOAT
|
|
assert tokens[5].token_type == TokenType.FOR
|
|
assert tokens[6].token_type == TokenType.IN
|
|
assert tokens[7].token_type == TokenType.CLASS
|
|
|
|
def test_simple_codeblock():
|
|
lexer = Lexer('def stuff():\n pass')
|
|
tokens = list(lexer.tokens())
|
|
|
|
assert tokens[5].token_type == TokenType.INDENT
|
|
assert tokens[5].content == 4
|
|
assert tokens[7].token_type == TokenType.DEDENT
|
|
assert tokens[7].content == 4
|
|
|
|
def test_hanging_indents():
|
|
lexer = Lexer('x\n \n \n')
|
|
tokens = list(lexer.tokens())
|
|
|
|
assert tokens[0].token_type == TokenType.IDENT
|
|
assert tokens[0].content == 'x'
|
|
assert tokens[0].line == 1
|
|
assert tokens[0].col == 1
|
|
|
|
assert tokens[1].token_type == TokenType.INDENT
|
|
assert tokens[1].content == 4
|
|
assert tokens[1].line == 2
|
|
assert tokens[1].col == 1
|
|
|
|
assert tokens[2].token_type == TokenType.INDENT
|
|
assert tokens[2].content == 4
|
|
assert tokens[2].line == 3
|
|
assert tokens[2].col == 4
|
|
|
|
assert tokens[3].token_type == TokenType.DEDENT
|
|
assert tokens[3].content == 4
|
|
assert tokens[3].line == 4
|
|
assert tokens[3].col == 1
|
|
|
|
assert tokens[4].token_type == TokenType.DEDENT
|
|
assert tokens[4].content == 4
|
|
assert tokens[4].line == 4
|
|
assert tokens[4].col == 1
|
|
|
|
assert tokens[5].token_type == TokenType.EOF
|
|
assert tokens[5].content == ''
|
|
assert tokens[5].line == 4
|
|
assert tokens[5].col == 1
|