Cleared up headers and code spacing on PythonLangImpl2.rst
This commit is contained in:
parent
f01d92df74
commit
5bb9471e1e
1 changed files with 422 additions and 327 deletions
|
|
@ -5,7 +5,7 @@ Chapter 2: Implementing a Parser and AST
|
||||||
Written by `Chris Lattner <mailto:sabre@nondot.org>`_ and `Max
|
Written by `Chris Lattner <mailto:sabre@nondot.org>`_ and `Max
|
||||||
Shawabkeh <http://max99x.com>`_
|
Shawabkeh <http://max99x.com>`_
|
||||||
|
|
||||||
Introduction # {#intro}
|
Introduction
|
||||||
=======================
|
=======================
|
||||||
|
|
||||||
Welcome to Chapter 2 of the `Implementing a language with
|
Welcome to Chapter 2 of the `Implementing a language with
|
||||||
|
|
@ -26,7 +26,7 @@ talk about the output of the parser: the Abstract Syntax Tree.
|
||||||
|
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
The Abstract Syntax Tree (AST) # {#ast}
|
The Abstract Syntax Tree (AST)
|
||||||
=======================================
|
=======================================
|
||||||
|
|
||||||
The AST for a program captures its behavior in such a way that it is
|
The AST for a program captures its behavior in such a way that it is
|
||||||
|
|
@ -43,7 +43,8 @@ We'll start with expressions first:
|
||||||
ExpressionNode(object): pass
|
ExpressionNode(object): pass
|
||||||
|
|
||||||
# Expression class for numeric literals like "1.0".
|
# Expression class for numeric literals like "1.0".
|
||||||
class NumberExpressionNode(ExpressionNode): def **init**\ (self, value):
|
class NumberExpressionNode(ExpressionNode):
|
||||||
|
def __init__(self, value):
|
||||||
self.value = value
|
self.value = value
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -65,17 +66,23 @@ that we'll use in the basic form of the Kaleidoscope language:
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# Expression class for referencing a variable,
|
# Expression class for referencing a variable,
|
||||||
like "a". class VariableExpressionNode(ExpressionNode): def
|
like "a".
|
||||||
**init**\ (self, name): self.name = name
|
class VariableExpressionNode(ExpressionNode):
|
||||||
|
def __init__(self, name):
|
||||||
|
self.name = name
|
||||||
|
|
||||||
# Expression class for a binary operator.
|
# Expression class for a binary operator.
|
||||||
class BinaryOperatorExpressionNode(ExpressionNode): def **init**\ (self,
|
class BinaryOperatorExpressionNode(ExpressionNode):
|
||||||
operator, left, right): self.operator = operator self.left = left
|
def __init__(self, operator, left, right):
|
||||||
|
self.operator = operator
|
||||||
|
self.left = left
|
||||||
self.right = right
|
self.right = right
|
||||||
|
|
||||||
# Expression class for function calls.
|
# Expression class for function calls.
|
||||||
class CallExpressionNode(ExpressionNode): def **init**\ (self, callee,
|
class CallExpressionNode(ExpressionNode):
|
||||||
args): self.callee = callee self.args = args
|
def __init__self, callee, args):
|
||||||
|
self.callee = callee
|
||||||
|
self.args = args
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -98,13 +105,17 @@ way to talk about functions themselves:
|
||||||
|
|
||||||
# This class represents the "prototype" for a
|
# This class represents the "prototype" for a
|
||||||
function, which captures its name, # and its argument names (thus
|
function, which captures its name, # and its argument names (thus
|
||||||
implicitly the number of arguments the function # takes). class
|
implicitly the number of arguments the function # takes).
|
||||||
PrototypeNode(object): def **init**\ (self, name, args): self.name =
|
class PrototypeNode(object):
|
||||||
name self.args = args
|
def __init__(self, name, args):
|
||||||
|
self.name = name
|
||||||
|
self.args = args
|
||||||
|
|
||||||
# This class represents a function definition itself.
|
# This class represents a function definition itself.
|
||||||
class FunctionNode(object): def **init**\ (self, prototype, body):
|
class FunctionNode(object):
|
||||||
self.prototype = prototype self.body = body
|
def __init__(self, prototype, body):
|
||||||
|
self.prototype = prototype
|
||||||
|
self.body = body
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -119,7 +130,7 @@ function bodies in Kaleidoscope.
|
||||||
|
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
Parser Basics # {#parserbasics}
|
Parser Basics
|
||||||
===============================
|
===============================
|
||||||
|
|
||||||
Now that we have an AST to build, we need to define the parser code to
|
Now that we have an AST to build, we need to define the parser code to
|
||||||
|
|
@ -130,9 +141,9 @@ that could be generated with calls like this:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
x = VariableExpressionNode('x') y =
|
x = VariableExpressionNode('x')
|
||||||
VariableExpressionNode('y') result = BinaryOperatorExpressionNode('+',
|
y = VariableExpressionNode('y')
|
||||||
x, y)
|
result = BinaryOperatorExpressionNode('+', x, y)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -144,12 +155,15 @@ class with some basic helper routines:
|
||||||
|
|
||||||
class Parser(object):
|
class Parser(object):
|
||||||
|
|
||||||
def **init**\ (self, tokens, binop_precedence): self.tokens = tokens
|
def __init__ (self, tokens, binop_precedence):
|
||||||
self.binop_precedence = binop_precedence self.Next()
|
self.tokens = tokens
|
||||||
|
self.binop_precedence = binop_precedence
|
||||||
|
self.Next()
|
||||||
|
|
||||||
# Provide a simple token buffer. Parser.current is the current token the
|
# Provide a simple token buffer. Parser.current is the current token the
|
||||||
# parser is looking at. Parser.Next() reads another token from the lexer
|
# parser is looking at. Parser.Next() reads another token from the lexer
|
||||||
and # updates Parser.current with its results. def Next(self):
|
and # updates Parser.current with its results.
|
||||||
|
def Next(self):
|
||||||
self.current = self.tokens.next()
|
self.current = self.tokens.next()
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -167,7 +181,7 @@ our grammar: numeric literals.
|
||||||
|
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
Basic Expression Parsing # {#parserprimexprs}
|
Basic Expression Parsing
|
||||||
=============================================
|
=============================================
|
||||||
|
|
||||||
We start with numeric literals, because they are the simplest to
|
We start with numeric literals, because they are the simplest to
|
||||||
|
|
@ -177,9 +191,11 @@ which parses that production. For numeric literals, we have:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# numberexpr ::= number def
|
# numberexpr ::= number
|
||||||
ParseNumberExpr(self): result = NumberExpressionNode(self.current.value)
|
def ParseNumberExpr(self):
|
||||||
self.Next() # consume the number. return result
|
result = NumberExpressionNode(self.current.value)
|
||||||
|
self.Next() # consume the number.
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -198,10 +214,10 @@ the parenthesis operator is defined like this:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# parenexpr ::= '(' expression ')' def
|
# parenexpr ::= '(' expression ')'
|
||||||
ParseParenExpr(self): self.Next() # eat '('.
|
def ParseParenExpr(self):
|
||||||
|
self.Next() # eat '('.
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
contents = self.ParseExpression()
|
contents = self.ParseExpression()
|
||||||
|
|
||||||
|
|
@ -232,10 +248,11 @@ function calls:
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# identifierexpr ::= identifier \| identifier '('
|
# identifierexpr ::= identifier \| identifier '('
|
||||||
expression\* ')' def ParseIdentifierExpr(self): identifier_name =
|
expression\* ')'
|
||||||
self.current.name self.Next() # eat identifier.
|
def ParseIdentifierExpr(self):
|
||||||
|
identifier_name = self.current.name
|
||||||
|
self.Next() # eat identifier.
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
if self.current != CharacterToken('('): # Simple variable reference.
|
if self.current != CharacterToken('('): # Simple variable reference.
|
||||||
return VariableExpressionNode(identifier_name);
|
return VariableExpressionNode(identifier_name);
|
||||||
|
|
@ -278,12 +295,17 @@ primary expression, we need to determine what sort of expression it is:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# primary ::= identifierexpr \| numberexpr \|
|
# primary ::= identifierexpr | numberexpr |
|
||||||
parenexpr def ParsePrimary(self): if isinstance(self.current,
|
parenexpr
|
||||||
IdentifierToken): return self.ParseIdentifierExpr() elif
|
def ParsePrimary(self):
|
||||||
isinstance(self.current, NumberToken): return self.ParseNumberExpr();
|
if isinstance(self.current, IdentifierToken):
|
||||||
elif self.current == CharacterToken('('): return self.ParseParenExpr()
|
return self.ParseIdentifierExpr()
|
||||||
else: raise RuntimeError('Unknown token when expecting an expression.')
|
elif isinstance(self.current, NumberToken):
|
||||||
|
return self.ParseNumberExpr();
|
||||||
|
elif self.current == CharacterToken('('):
|
||||||
|
return self.ParseParenExpr()
|
||||||
|
else:
|
||||||
|
raise RuntimeError('Unknown token when expecting an expression.')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -298,7 +320,7 @@ expressions. They are a bit more complex.
|
||||||
|
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
Binary Expression Parsing # {#parserbinops}
|
Binary Expression Parsing
|
||||||
===========================================
|
===========================================
|
||||||
|
|
||||||
Binary expressions are significantly harder to parse because they are
|
Binary expressions are significantly harder to parse because they are
|
||||||
|
|
@ -320,11 +342,15 @@ Now is the time to use it:
|
||||||
|
|
||||||
def main(): # Install standard binary operators.
|
def main(): # Install standard binary operators.
|
||||||
# 1 is lowest possible precedence. 40 is the highest.
|
# 1 is lowest possible precedence. 40 is the highest.
|
||||||
operator_precedence = { '<': 10, '+': 20, '-': 20, '\*': 40 }
|
operator_precedence = {
|
||||||
|
'<': 10,
|
||||||
|
'+': 20,
|
||||||
|
'-': 20,
|
||||||
|
'*': 40
|
||||||
|
}
|
||||||
|
|
||||||
# Run the main ``interpreter loop``. while True:
|
# Run the main ``interpreter loop``.
|
||||||
|
while True:
|
||||||
::
|
|
||||||
|
|
||||||
...
|
...
|
||||||
|
|
||||||
|
|
@ -348,10 +374,12 @@ token, or -1 if the token is not a binary operator:
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# Gets the precedence of the current token, or -1
|
# Gets the precedence of the current token, or -1
|
||||||
if the token is not a binary # operator. def
|
if the token is not a binary # operator.
|
||||||
GetCurrentTokenPrecedence(self): if isinstance(self.current,
|
def GetCurrentTokenPrecedence(self):
|
||||||
CharacterToken): return self.binop_precedence.get(self.current.char,
|
if isinstance(self.current, CharacterToken):
|
||||||
-1) else: return -1
|
return self.binop_precedence.get(self.current.char, -1)
|
||||||
|
else:
|
||||||
|
return -1
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -372,9 +400,10 @@ a sequence of ``[binop,primaryexpr]`` pairs:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# expression ::= primary binoprhs def
|
# expression ::= primary binoprhs
|
||||||
ParseExpression(self): left = self.ParsePrimary() return
|
def ParseExpression(self):
|
||||||
self.ParseBinOpRHS(left, 0)
|
left = self.ParsePrimary()
|
||||||
|
return self.ParseBinOpRHS(left, 0)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -396,12 +425,11 @@ starts with:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# binoprhs ::= (operator primary)\* def
|
# binoprhs ::= (operator primary)*
|
||||||
ParseBinOpRHS(self, left, left_precedence): # If this is a binary
|
def ParseBinOpRHS(self, left, left_precedence):
|
||||||
operator, find its precedence. while True: precedence =
|
# If this is a binary operator, find its precedence.
|
||||||
self.GetCurrentTokenPrecedence()
|
while True:
|
||||||
|
precedence = self.GetCurrentTokenPrecedence()
|
||||||
::
|
|
||||||
|
|
||||||
# If this is a binary operator that binds at least as tightly as the
|
# If this is a binary operator that binds at least as tightly as the
|
||||||
# current one, consume it; otherwise we are done.
|
# current one, consume it; otherwise we are done.
|
||||||
|
|
@ -422,10 +450,9 @@ expression:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
binary_operator = self.current.char self.Next()
|
binary_operator = self.current.char
|
||||||
# eat the operator.
|
self.Next() # eat the operator.
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
# Parse the primary expression after the binary operator.
|
# Parse the primary expression after the binary operator.
|
||||||
right = self.ParsePrimary()
|
right = self.ParsePrimary()
|
||||||
|
|
@ -450,7 +477,8 @@ precedence (which is '+' in this case):
|
||||||
|
|
||||||
# If binary_operator binds less tightly with
|
# If binary_operator binds less tightly with
|
||||||
right than the operator after # right, let the pending operator take
|
right than the operator after # right, let the pending operator take
|
||||||
right as its left. next_precedence = self.GetCurrentTokenPrecedence()
|
right as its left.
|
||||||
|
next_precedence = self.GetCurrentTokenPrecedence()
|
||||||
if precedence < next_precedence:
|
if precedence < next_precedence:
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -465,10 +493,8 @@ for ``a+b``, and then continue parsing:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
if precedence < next_precedence: ... if body
|
if precedence < next_precedence:
|
||||||
omitted ...
|
... if body omitted ...
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
# Merge left/right.
|
# Merge left/right.
|
||||||
left = BinaryOperatorExpressionNode(binary_operator, left, right);
|
left = BinaryOperatorExpressionNode(binary_operator, left, right);
|
||||||
|
|
@ -496,12 +522,11 @@ duplicated for context):
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# If binary_operator binds less tightly with
|
# If binary_operator binds less tightly with
|
||||||
right than the operator after # right, let the pending operator take
|
right than the operator after # right, let the pending operator take right as its left.
|
||||||
right as its left. next_precedence = self.GetCurrentTokenPrecedence()
|
next_precedence = self.GetCurrentTokenPrecedence()
|
||||||
if precedence < next_precedence: right = self.ParseBinOpRHS(right,
|
if precedence < next_precedence:
|
||||||
precedence + 1)
|
right = self.ParseBinOpRHS(right, precedence + 1)
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
# Merge left/right.
|
# Merge left/right.
|
||||||
left = BinaryOperatorExpressionNode(binary_operator, left, right)
|
left = BinaryOperatorExpressionNode(binary_operator, left, right)
|
||||||
|
|
@ -534,7 +559,7 @@ we need to handle function definitions, etc.
|
||||||
|
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
Parsing the Rest # {#parsertop}
|
Parsing the Rest
|
||||||
===============================
|
===============================
|
||||||
|
|
||||||
The next thing missing is handling of function prototypes. In
|
The next thing missing is handling of function prototypes. In
|
||||||
|
|
@ -546,11 +571,11 @@ expressions):
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# prototype ::= id '(' id\* ')' def
|
# prototype ::= id '(' id* ')'
|
||||||
ParsePrototype(self): if not isinstance(self.current, IdentifierToken):
|
def ParsePrototype(self):
|
||||||
|
if not isinstance(self.current, IdentifierToken):
|
||||||
raise RuntimeError('Expected function name in prototype.')
|
raise RuntimeError('Expected function name in prototype.')
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
function_name = self.current.name
|
function_name = self.current.name
|
||||||
self.Next() # eat function name.
|
self.Next() # eat function name.
|
||||||
|
|
@ -582,10 +607,12 @@ an expression to implement the body:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# definition ::= 'def' prototype expression def
|
# definition ::= 'def' prototype expression
|
||||||
ParseDefinition(self): self.Next() # eat def. proto =
|
def ParseDefinition(self):
|
||||||
self.ParsePrototype() body = self.ParseExpression() return
|
self.Next() # eat def.
|
||||||
FunctionNode(proto, body)
|
proto = self.ParsePrototype()
|
||||||
|
body = self.ParseExpression()
|
||||||
|
return FunctionNode(proto, body)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -596,9 +623,10 @@ In addition, we support 'extern' to declare functions like 'sin' and
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# external ::= 'extern' prototype def
|
# external ::= 'extern' prototype
|
||||||
ParseExtern(self): self.Next() # eat extern. return
|
def ParseExtern(self):
|
||||||
self.ParsePrototype()
|
self.Next() # eat extern.
|
||||||
|
return self.ParsePrototype()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -609,9 +637,10 @@ nullary (zero argument) functions for them:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# toplevelexpr ::= expression def
|
# toplevelexpr ::= expression
|
||||||
ParseTopLevelExpr(self): proto = PrototypeNode('', []) return
|
def ParseTopLevelExpr(self):
|
||||||
FunctionNode(proto, self.ParseExpression())
|
proto = PrototypeNode('', [])
|
||||||
|
return FunctionNode(proto, self.ParseExpression())
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -620,7 +649,7 @@ let us actually *execute* this code we've built!
|
||||||
|
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
The Driver # {#driver}
|
The Driver
|
||||||
======================
|
======================
|
||||||
|
|
||||||
The driver for this simply invokes all of the parsing pieces with a
|
The driver for this simply invokes all of the parsing pieces with a
|
||||||
|
|
@ -630,10 +659,14 @@ include the top-level loop. See `below <#code>`_ for full code.
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
# Run the main "interpreter loop". while True:
|
# Run the main "interpreter loop".
|
||||||
print 'ready>', try: raw = raw_input() except KeyboardInterrupt: return
|
while True:
|
||||||
|
print 'ready>',
|
||||||
|
try:
|
||||||
|
raw = raw_input()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
return
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
parser = Parser(Tokenize(raw), operator_precedence)
|
parser = Parser(Tokenize(raw), operator_precedence)
|
||||||
while True:
|
while True:
|
||||||
|
|
@ -657,7 +690,7 @@ allow the user to quit using Ctrl+C.
|
||||||
|
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
Conclusions # {#conclusions}
|
Conclusions
|
||||||
============================
|
============================
|
||||||
|
|
||||||
With just under 330 lines of commented code (200 lines of non-comment,
|
With just under 330 lines of commented code (200 lines of non-comment,
|
||||||
|
|
@ -667,14 +700,21 @@ validate Kaleidoscope code and tell us if it is grammatically invalid.
|
||||||
For example, here is a sample interaction:
|
For example, here is a sample interaction:
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: bash
|
||||||
|
|
||||||
$ python kaleidoscope.py ready> def foo(x y)
|
$ python kaleidoscope.py
|
||||||
x+foo(y, 4.0) Parsed a function definition. ready> def foo(x y) x+y y
|
ready> def foo(x y) x+foo(y, 4.0)
|
||||||
Parsed a function definition. Parsed a top-level expression. ready> def
|
Parsed a function definition.
|
||||||
foo(x y) x+y ) Parsed a function definition. Error: Unknown token when
|
ready> def foo(x y) x+y y
|
||||||
expecting an expression. ready> extern sin(a); Parsed an extern. ready>
|
Parsed a function definition.
|
||||||
^C $
|
Parsed a top-level expression.
|
||||||
|
ready> def foo(x y) x+y )
|
||||||
|
Parsed a function definition.
|
||||||
|
Error: Unknown token when expecting an expression.
|
||||||
|
ready> extern sin(a);
|
||||||
|
Parsed an extern.
|
||||||
|
ready> ^C
|
||||||
|
$
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -685,7 +725,7 @@ LLVM Intermediate Representation (IR) from the AST.
|
||||||
|
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
Full Code Listing # {#code}
|
Full Code Listing
|
||||||
===========================
|
===========================
|
||||||
|
|
||||||
Here is the complete code listing for this and the previous chapter.
|
Here is the complete code listing for this and the previous chapter.
|
||||||
|
|
@ -703,22 +743,30 @@ external libraries at all for this.
|
||||||
-----
|
-----
|
||||||
|
|
||||||
# The lexer yields one of these types for each token.
|
# The lexer yields one of these types for each token.
|
||||||
class EOFToken(object): pass
|
class EOFToken(object):
|
||||||
|
pass
|
||||||
|
|
||||||
class DefToken(object): pass
|
class DefToken(object):
|
||||||
|
pass
|
||||||
|
|
||||||
class ExternToken(object): pass
|
class ExternToken(object):
|
||||||
|
pass
|
||||||
|
|
||||||
class IdentifierToken(object): def **init**\ (self, name): self.name =
|
class IdentifierToken(object):
|
||||||
name
|
def **init**\ (self, name):
|
||||||
|
self.name = name
|
||||||
|
|
||||||
class NumberToken(object): def **init**\ (self, value): self.value =
|
class NumberToken(object):
|
||||||
value
|
def **init**\ (self, value):
|
||||||
|
self.value = value
|
||||||
|
|
||||||
class CharacterToken(object): def **init**\ (self, char): self.char =
|
class CharacterToken(object):
|
||||||
char def **eq**\ (self, other): return isinstance(other, CharacterToken)
|
def **init**\ (self, char):
|
||||||
and self.char == other.char def **ne**\ (self, other): return not self
|
self.char = char
|
||||||
== other
|
def **eq**\ (self, other):
|
||||||
|
return isinstance(other, CharacterToken) and self.char == other.char
|
||||||
|
def **ne**\ (self, other):
|
||||||
|
return not self == other
|
||||||
|
|
||||||
# Regular expressions that tokens and comments of our language.
|
# Regular expressions that tokens and comments of our language.
|
||||||
REGEX_NUMBER = re.compile('[0-9]+(?:.[0-9]+)?') REGEX_IDENTIFIER =
|
REGEX_NUMBER = re.compile('[0-9]+(?:.[0-9]+)?') REGEX_IDENTIFIER =
|
||||||
|
|
@ -727,7 +775,6 @@ external libraries at all for this.
|
||||||
def Tokenize(string): while string: # Skip whitespace. if
|
def Tokenize(string): while string: # Skip whitespace. if
|
||||||
string[0].isspace(): string = string[1:] continue
|
string[0].isspace(): string = string[1:] continue
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
# Run regexes.
|
# Run regexes.
|
||||||
comment_match = REGEX_COMMENT.match(string)
|
comment_match = REGEX_COMMENT.match(string)
|
||||||
|
|
@ -763,30 +810,39 @@ external libraries at all for this.
|
||||||
-------------------------------------
|
-------------------------------------
|
||||||
|
|
||||||
# Base class for all expression nodes.
|
# Base class for all expression nodes.
|
||||||
class ExpressionNode(object): pass
|
class ExpressionNode(object):
|
||||||
|
pass
|
||||||
|
|
||||||
# Expression class for numeric literals like "1.0".
|
# Expression class for numeric literals like "1.0".
|
||||||
class NumberExpressionNode(ExpressionNode): def **init**\ (self, value):
|
class NumberExpressionNode(ExpressionNode):
|
||||||
|
def **init**\ (self, value):
|
||||||
self.value = value
|
self.value = value
|
||||||
|
|
||||||
# Expression class for referencing a variable, like "a".
|
# Expression class for referencing a variable, like "a".
|
||||||
class VariableExpressionNode(ExpressionNode): def **init**\ (self,
|
class VariableExpressionNode(ExpressionNode):
|
||||||
name): self.name = name
|
def **init**\ (self, name):
|
||||||
|
self.name = name
|
||||||
|
|
||||||
# Expression class for a binary operator.
|
# Expression class for a binary operator.
|
||||||
class BinaryOperatorExpressionNode(ExpressionNode): def **init**\ (self,
|
class BinaryOperatorExpressionNode(ExpressionNode):
|
||||||
operator, left, right): self.operator = operator self.left = left
|
def **init**\ (self, operator, left, right):
|
||||||
|
self.operator = operator
|
||||||
|
self.left = left
|
||||||
self.right = right
|
self.right = right
|
||||||
|
|
||||||
# Expression class for function calls.
|
# Expression class for function calls.
|
||||||
class CallExpressionNode(ExpressionNode): def **init**\ (self, callee,
|
class CallExpressionNode(ExpressionNode):
|
||||||
args): self.callee = callee self.args = args
|
def **init**\ (self, callee, args):
|
||||||
|
self.callee = callee
|
||||||
|
self.args = args
|
||||||
|
|
||||||
# This class represents the "prototype" for a function, which captures its name,
|
# This class represents the "prototype" for a function, which captures its name,
|
||||||
# and its argument names (thus implicitly the number of arguments the function
|
# and its argument names (thus implicitly the number of arguments the function
|
||||||
# takes).
|
# takes).
|
||||||
class PrototypeNode(object): def **init**\ (self, name, args): self.name
|
class PrototypeNode(object):
|
||||||
= name self.args = args
|
def **init**\ (self, name, args):
|
||||||
|
self.name = name
|
||||||
|
self.args = args
|
||||||
|
|
||||||
# This class represents a function definition itself.
|
# This class represents a function definition itself.
|
||||||
class FunctionNode(object): def **init**\ (self, prototype, body):
|
class FunctionNode(object): def **init**\ (self, prototype, body):
|
||||||
|
|
@ -797,24 +853,30 @@ external libraries at all for this.
|
||||||
|
|
||||||
class Parser(object):
|
class Parser(object):
|
||||||
|
|
||||||
def **init**\ (self, tokens, binop_precedence): self.tokens = tokens
|
def __init__(self, tokens, binop_precedence):
|
||||||
self.binop_precedence = binop_precedence self.Next()
|
self.tokens = tokens
|
||||||
|
self.binop_precedence = binop_precedence
|
||||||
|
self.Next()
|
||||||
|
|
||||||
# Provide a simple token buffer. Parser.current is the current token the
|
# Provide a simple token buffer. Parser.current is the current token the
|
||||||
# parser is looking at. Parser.Next() reads another token from the lexer
|
# parser is looking at. Parser.Next() reads another token from the lexer
|
||||||
and # updates Parser.current with its results. def Next(self):
|
and # updates Parser.current with its results.
|
||||||
|
def Next(self):
|
||||||
self.current = self.tokens.next()
|
self.current = self.tokens.next()
|
||||||
|
|
||||||
# Gets the precedence of the current token, or -1 if the token is not a
|
# Gets the precedence of the current token, or -1 if the token is not a
|
||||||
binary # operator. def GetCurrentTokenPrecedence(self): if
|
binary # operator.
|
||||||
isinstance(self.current, CharacterToken): return
|
def GetCurrentTokenPrecedence(self):
|
||||||
self.binop_precedence.get(self.current.char, -1) else: return -1
|
if isinstance(self.current, CharacterToken):
|
||||||
|
return self.binop_precedence.get(self.current.char, -1)
|
||||||
|
else:
|
||||||
|
return -1
|
||||||
|
|
||||||
# identifierexpr ::= identifier \| identifier '(' expression\* ')' def
|
# identifierexpr ::= identifier \| identifier '(' expression\* ')'
|
||||||
ParseIdentifierExpr(self): identifier_name = self.current.name
|
def ParseIdentifierExpr(self):
|
||||||
|
identifier_name = self.current.name
|
||||||
self.Next() # eat identifier.
|
self.Next() # eat identifier.
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
if self.current != CharacterToken('('): # Simple variable reference.
|
if self.current != CharacterToken('('): # Simple variable reference.
|
||||||
return VariableExpressionNode(identifier_name)
|
return VariableExpressionNode(identifier_name)
|
||||||
|
|
@ -834,14 +896,15 @@ external libraries at all for this.
|
||||||
self.Next() # eat ')'.
|
self.Next() # eat ')'.
|
||||||
return CallExpressionNode(identifier_name, args)
|
return CallExpressionNode(identifier_name, args)
|
||||||
|
|
||||||
# numberexpr ::= number def ParseNumberExpr(self): result =
|
# numberexpr ::= number
|
||||||
NumberExpressionNode(self.current.value) self.Next() # consume the
|
def ParseNumberExpr(self):
|
||||||
number. return result
|
result = NumberExpressionNode(self.current.value)
|
||||||
|
self.Next() # consume the number.
|
||||||
|
return result
|
||||||
|
|
||||||
# parenexpr ::= '(' expression ')' def ParseParenExpr(self): self.Next()
|
# parenexpr ::= '(' expression ')'
|
||||||
# eat '('.
|
def ParseParenExpr(self):
|
||||||
|
self.Next() # eat '('.
|
||||||
::
|
|
||||||
|
|
||||||
contents = self.ParseExpression()
|
contents = self.ParseExpression()
|
||||||
|
|
||||||
|
|
@ -851,18 +914,23 @@ external libraries at all for this.
|
||||||
|
|
||||||
return contents
|
return contents
|
||||||
|
|
||||||
# primary ::= identifierexpr \| numberexpr \| parenexpr def
|
# primary ::= identifierexpr | numberexpr | parenexpr
|
||||||
ParsePrimary(self): if isinstance(self.current, IdentifierToken): return
|
def ParsePrimary(self):
|
||||||
self.ParseIdentifierExpr() elif isinstance(self.current, NumberToken):
|
if isinstance(self.current, IdentifierToken):
|
||||||
return self.ParseNumberExpr() elif self.current == CharacterToken('('):
|
return self.ParseIdentifierExpr()
|
||||||
return self.ParseParenExpr() else: raise RuntimeError('Unknown token
|
elif isinstance(self.current, NumberToken):
|
||||||
when expecting an expression.')
|
return self.ParseNumberExpr()
|
||||||
|
elif self.current == CharacterToken('('):
|
||||||
|
return self.ParseParenExpr()
|
||||||
|
else:
|
||||||
|
raise RuntimeError('Unknown token when expecting an expression.')
|
||||||
|
|
||||||
# binoprhs ::= (operator primary)\* def ParseBinOpRHS(self, left,
|
# binoprhs ::= (operator primary)*
|
||||||
left_precedence): # If this is a binary operator, find its precedence.
|
def ParseBinOpRHS(self, left, left_precedence):
|
||||||
while True: precedence = self.GetCurrentTokenPrecedence()
|
# If this is a binary operator, find its precedence.
|
||||||
|
while True:
|
||||||
|
precedence = self.GetCurrentTokenPrecedence()
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
# If this is a binary operator that binds at least as tightly as the
|
# If this is a binary operator that binds at least as tightly as the
|
||||||
# current one, consume it; otherwise we are done.
|
# current one, consume it; otherwise we are done.
|
||||||
|
|
@ -884,14 +952,16 @@ external libraries at all for this.
|
||||||
# Merge left/right.
|
# Merge left/right.
|
||||||
left = BinaryOperatorExpressionNode(binary_operator, left, right)
|
left = BinaryOperatorExpressionNode(binary_operator, left, right)
|
||||||
|
|
||||||
# expression ::= primary binoprhs def ParseExpression(self): left =
|
# expression ::= primary binoprhs
|
||||||
self.ParsePrimary() return self.ParseBinOpRHS(left, 0)
|
def ParseExpression(self):
|
||||||
|
left = self.ParsePrimary()
|
||||||
|
return self.ParseBinOpRHS(left, 0)
|
||||||
|
|
||||||
# prototype ::= id '(' id\* ')' def ParsePrototype(self): if not
|
# prototype ::= id '(' id\* ')'
|
||||||
isinstance(self.current, IdentifierToken): raise RuntimeError('Expected
|
def ParsePrototype(self):
|
||||||
function name in prototype.')
|
if not isinstance(self.current, IdentifierToken):
|
||||||
|
raise RuntimeError('Expected function name in prototype.')
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
function_name = self.current.name
|
function_name = self.current.name
|
||||||
self.Next() # eat function name.
|
self.Next() # eat function name.
|
||||||
|
|
@ -913,40 +983,64 @@ external libraries at all for this.
|
||||||
|
|
||||||
return PrototypeNode(function_name, arg_names)
|
return PrototypeNode(function_name, arg_names)
|
||||||
|
|
||||||
# definition ::= 'def' prototype expression def ParseDefinition(self):
|
# definition ::= 'def' prototype expression
|
||||||
self.Next() # eat def. proto = self.ParsePrototype() body =
|
def ParseDefinition(self):
|
||||||
self.ParseExpression() return FunctionNode(proto, body)
|
self.Next() # eat def.
|
||||||
|
proto = self.ParsePrototype()
|
||||||
|
body = self.ParseExpression()
|
||||||
|
return FunctionNode(proto, body)
|
||||||
|
|
||||||
# toplevelexpr ::= expression def ParseTopLevelExpr(self): proto =
|
# toplevelexpr ::= expression
|
||||||
PrototypeNode('', []) return FunctionNode(proto, self.ParseExpression())
|
def ParseTopLevelExpr(self):
|
||||||
|
proto = PrototypeNode('', [])
|
||||||
|
return FunctionNode(proto, self.ParseExpression())
|
||||||
|
|
||||||
# external ::= 'extern' prototype def ParseExtern(self): self.Next() #
|
# external ::= 'extern' prototype
|
||||||
eat extern. return self.ParsePrototype()
|
def ParseExtern(self):
|
||||||
|
self.Next() #eat extern.
|
||||||
|
return self.ParsePrototype()
|
||||||
|
|
||||||
# Top-Level parsing def HandleDefinition(self):
|
# Top-Level parsing
|
||||||
|
def HandleDefinition(self):
|
||||||
self.Handle(self.ParseDefinition, 'Parsed a function definition.')
|
self.Handle(self.ParseDefinition, 'Parsed a function definition.')
|
||||||
|
|
||||||
def HandleExtern(self): self.Handle(self.ParseExtern, 'Parsed an
|
def HandleExtern(self):
|
||||||
extern.')
|
self.Handle(self.ParseExtern, 'Parsed an extern.')
|
||||||
|
|
||||||
def HandleTopLevelExpression(self): self.Handle(self.ParseTopLevelExpr,
|
def HandleTopLevelExpression(self):
|
||||||
'Parsed a top-level expression.')
|
self.Handle(self.ParseTopLevelExpr, 'Parsed a top-level expression.')
|
||||||
|
|
||||||
def Handle(self, function, message): try: function() print message
|
def Handle(self, function, message):
|
||||||
except Exception, e: print 'Error:', e try: self.Next() # Skip for error
|
try:
|
||||||
recovery. except: pass
|
function()
|
||||||
|
print message
|
||||||
|
except Exception, e:
|
||||||
|
print 'Error:', e
|
||||||
|
try:
|
||||||
|
self.Next() # Skip for error recovery.
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
Main driver code.
|
Main driver code.
|
||||||
-----------------
|
-----------------
|
||||||
|
|
||||||
def main(): # Install standard binary operators. # 1 is lowest possible
|
def main():
|
||||||
precedence. 40 is the highest. operator_precedence = { '<': 10, '+':
|
# Install standard binary operators.
|
||||||
20, '-': 20, '\*': 40 }
|
# 1 is lowest possible precedence. 40 is the highest.
|
||||||
|
operator_precedence = {
|
||||||
|
'<': 10,
|
||||||
|
'+':20,
|
||||||
|
'-': 20,
|
||||||
|
'*': 40
|
||||||
|
}
|
||||||
|
|
||||||
# Run the main "interpreter loop". while True: print 'ready>', try: raw
|
# Run the main "interpreter loop".
|
||||||
= raw_input() except KeyboardInterrupt: return
|
while True:
|
||||||
|
print 'ready>',
|
||||||
::
|
try:
|
||||||
|
raw = raw_input()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
return
|
||||||
|
|
||||||
parser = Parser(Tokenize(raw), operator_precedence)
|
parser = Parser(Tokenize(raw), operator_precedence)
|
||||||
while True:
|
while True:
|
||||||
|
|
@ -960,4 +1054,5 @@ external libraries at all for this.
|
||||||
else:
|
else:
|
||||||
parser.HandleTopLevelExpression()
|
parser.HandleTopLevelExpression()
|
||||||
|
|
||||||
if **name** == '**main**\ ': main()
|
if ==name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue