Misc doc cleanup
This commit is contained in:
Maggie Mari
parent
258efb0518
commit
8e00cd7e66
3 changed files with 154 additions and 145 deletions
|
|
@ -160,7 +160,7 @@ The Lexer
|
|||
When it comes to implementing a language, the first thing needed is the
|
||||
ability to process a text file and recognize what it says. The
|
||||
traditional way to do this is to use a
|
||||
`lexer <http://en.wikipedia.org/wiki/Lexical_analysis>`_" (aka
|
||||
`lexer <http://en.wikipedia.org/wiki/Lexical_analysis>`_ (aka
|
||||
'scanner') to break the input up into "tokens". Each token returned by
|
||||
the lexer includes a token type and potentially some metadata (e.g. the
|
||||
numeric value of a number). First, we define the possibilities:
|
||||
|
|
@ -238,15 +238,11 @@ ignoring whitespace between tokens:
|
|||
.. code-block:: python
|
||||
|
||||
def Tokenize(string):
|
||||
while string: # Skip whitespace.
|
||||
while string: # Skip whitespace.
|
||||
if string[0].isspace():
|
||||
string = string[1:]
|
||||
continue
|
||||
|
||||
::
|
||||
|
||||
...
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -271,9 +267,9 @@ ignore the captured match:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
# Check if any of the regexes matched and yield
|
||||
# the appropriate result.
|
||||
if comment_match:
|
||||
# Check if any of the regexes matched and yield
|
||||
# the appropriate result.
|
||||
if comment_match:
|
||||
comment = comment_match.group(0)
|
||||
string = string[len(comment):]
|
||||
|
||||
|
|
@ -281,27 +277,27 @@ For numbers, we yield the captured match, converted to a float and
|
|||
tagged with the appropriate token type:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
elif number_match:
|
||||
number = number_match.group(0)
|
||||
yield NumberToken(float(number))
|
||||
string = string[len(number):]
|
||||
|
||||
elif number_match:
|
||||
number = number_match.group(0)
|
||||
yield NumberToken(float(number))
|
||||
string = string[len(number):]
|
||||
|
||||
The identifier case is a little more complex. We have to check for
|
||||
keywords to decide whether we have captured an identifier or a keyword:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
elif identifier_match:
|
||||
identifier = identifier_match.group(0)
|
||||
# Check if we matched a keyword.
|
||||
if identifier == 'def':
|
||||
yield DefToken()
|
||||
elif identifier == 'extern':
|
||||
yield ExternToken()
|
||||
else:
|
||||
yield IdentifierToken(identifier)
|
||||
string = string[len(identifier):]
|
||||
elif identifier_match:
|
||||
identifier = identifier_match.group(0)
|
||||
# Check if we matched a keyword.
|
||||
if identifier == 'def':
|
||||
yield DefToken()
|
||||
elif identifier == 'extern':
|
||||
yield ExternToken()
|
||||
else:
|
||||
yield IdentifierToken(identifier)
|
||||
string = string[len(identifier):]
|
||||
|
||||
|
||||
Finally, if we haven't recognized a comment, a number of an identifier,
|
||||
|
|
@ -311,9 +307,9 @@ used, for example, for operators like ``+`` or ``*``:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
else: # Yield the unknown character.
|
||||
yield CharacterToken(string[0])
|
||||
string = string[1:]
|
||||
else: # Yield the unknown character.
|
||||
yield CharacterToken(string[0])
|
||||
string = string[1:]
|
||||
|
||||
|
||||
Once we're done with the loop, we return a final end-of-file token:
|
||||
|
|
@ -321,5 +317,5 @@ Once we're done with the loop, we return a final end-of-file token:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
yield EOFToken()
|
||||
yield EOFToken()
|
||||
|
||||
|
|
|
|||
|
|
@ -439,7 +439,6 @@ at this point. We'll fix this in `Chapter 5 <PythonLangImpl5.html>`_ :).
|
|||
# Validate the generated code, checking for consistency.
|
||||
function.verify()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -642,8 +641,10 @@ need to `download <../download.html>`_ and
|
|||
import re
|
||||
from llvm.core import Module, Constant, Type, Function, Builder, FCMP_ULT
|
||||
|
||||
Globals
|
||||
-------
|
||||
Globals
|
||||
-------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# The LLVM module, which holds all the IR code.
|
||||
g_llvm_module = Module.new('my cool jit')
|
||||
|
|
@ -655,8 +656,10 @@ need to `download <../download.html>`_ and
|
|||
# and what their LLVM representation is.
|
||||
g_named_values = {}
|
||||
|
||||
Lexer
|
||||
-----
|
||||
Lexer
|
||||
-----
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# The lexer yields one of these types for each token.
|
||||
class EOFToken(object):
|
||||
|
|
@ -678,55 +681,58 @@ need to `download <../download.html>`_ and
|
|||
|
||||
class CharacterToken(object):
|
||||
def __init__(self, char):
|
||||
self.char = char def __eq__(self, other):
|
||||
self.char = char
|
||||
def __eq__(self, other):
|
||||
return isinstance(other, CharacterToken)and self.char == other.char
|
||||
def __ne__(self, other):
|
||||
return not self == other
|
||||
|
||||
# Regular expressions that tokens and comments of our language.
|
||||
REGEX_NUMBER = re.compile('[0-9]+(?:.[0-9]+)?')
|
||||
REGEX_IDENTIFIER = re.compile('[a-zA-Z][a-zA-Z0-9]\ *')
|
||||
REGEX_COMMENT = re.compile('#.*')
|
||||
|
||||
def Tokenize(string):
|
||||
while string:
|
||||
# Skip whitespace.
|
||||
if string[0].isspace():
|
||||
string = string[1:]
|
||||
continue
|
||||
|
||||
# Run regexes.
|
||||
comment_match = REGEX_COMMENT.match(string)
|
||||
number_match = REGEX_NUMBER.match(string)
|
||||
identifier_match = REGEX_IDENTIFIER.match(string)
|
||||
# Regular expressions that tokens and comments of our language.
|
||||
REGEX_NUMBER = re.compile('[0-9]+(?:.[0-9]+)?')
|
||||
REGEX_IDENTIFIER = re.compile('[a-zA-Z][a-zA-Z0-9]\ *')
|
||||
REGEX_COMMENT = re.compile('#.*')
|
||||
|
||||
# Check if any of the regexes matched and yield the appropriate result.
|
||||
if comment_match:
|
||||
comment = comment_match.group(0)
|
||||
string = string[len(comment):]
|
||||
elif number_match:
|
||||
number = number_match.group(0)
|
||||
yield NumberToken(float(number))
|
||||
string = string[len(number):]
|
||||
elif identifier_match:
|
||||
identifier = identifier_match.group(0)
|
||||
# Check if we matched a keyword.
|
||||
if identifier == 'def':
|
||||
yield DefToken()
|
||||
elif identifier == 'extern':
|
||||
yield ExternToken()
|
||||
else:
|
||||
yield IdentifierToken(identifier)
|
||||
string = string[len(identifier):]
|
||||
else:
|
||||
# Yield the ASCII value of the unknown character.
|
||||
yield CharacterToken(string[0])
|
||||
string = string[1:]
|
||||
def Tokenize(string):
|
||||
while string:
|
||||
# Skip whitespace.
|
||||
if string[0].isspace():
|
||||
string = string[1:]
|
||||
continue
|
||||
|
||||
# Run regexes.
|
||||
comment_match = REGEX_COMMENT.match(string)
|
||||
number_match = REGEX_NUMBER.match(string)
|
||||
identifier_match = REGEX_IDENTIFIER.match(string)
|
||||
|
||||
# Check if any of the regexes matched and yield the appropriate result.
|
||||
if comment_match:
|
||||
comment = comment_match.group(0)
|
||||
string = string[len(comment):]
|
||||
elif number_match:
|
||||
number = number_match.group(0)
|
||||
yield NumberToken(float(number))
|
||||
string = string[len(number):]
|
||||
elif identifier_match:
|
||||
identifier = identifier_match.group(0)
|
||||
# Check if we matched a keyword.
|
||||
if identifier == 'def':
|
||||
yield DefToken()
|
||||
elif identifier == 'extern':
|
||||
yield ExternToken()
|
||||
else:
|
||||
yield IdentifierToken(identifier)
|
||||
string = string[len(identifier):]
|
||||
else:
|
||||
# Yield the ASCII value of the unknown character.
|
||||
yield CharacterToken(string[0])
|
||||
string = string[1:]
|
||||
|
||||
yield EOFToken()
|
||||
|
||||
yield EOFToken()
|
||||
|
||||
Abstract Syntax Tree (aka Parse Tree)
|
||||
-------------------------------------
|
||||
Abstract Syntax Tree (aka Parse Tree)
|
||||
-------------------------------------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# Base class for all expression nodes.
|
||||
class ExpressionNode(object):
|
||||
|
|
@ -756,25 +762,27 @@ need to `download <../download.html>`_ and
|
|||
# Expression class for a binary operator.
|
||||
class BinaryOperatorExpressionNode(ExpressionNode):
|
||||
|
||||
def __init__(self, operator, left, right): self.operator = operator
|
||||
self.left = left self.right = right
|
||||
def __init__(self, operator, left, right):
|
||||
self.operator = operator
|
||||
self.left = left
|
||||
self.right = right
|
||||
|
||||
def CodeGen(self):
|
||||
left = self.left.CodeGen()
|
||||
right = self.right.CodeGen()
|
||||
|
||||
if self.operator == '+':
|
||||
return g_llvm_builder.fadd(left, right, 'addtmp')
|
||||
elif self.operator == '-':
|
||||
return g_llvm_builder.fsub(left, right, 'subtmp')
|
||||
elif self.operator == '*':
|
||||
return g_llvm_builder.fmul(left, right, 'multmp')
|
||||
elif self.operator == '<':
|
||||
result = g_llvm_builder.fcmp(FCMP_ULT, left, right, 'cmptmp')
|
||||
# Convert bool 0 or 1 to double 0.0 or 1.0.
|
||||
return g_llvm_builder.uitofp(result, Type.double(), 'booltmp')
|
||||
else:
|
||||
raise RuntimeError('Unknown binary operator.')
|
||||
if self.operator == '+':
|
||||
return g_llvm_builder.fadd(left, right, 'addtmp')
|
||||
elif self.operator == '-':
|
||||
return g_llvm_builder.fsub(left, right, 'subtmp')
|
||||
elif self.operator == '*':
|
||||
return g_llvm_builder.fmul(left, right, 'multmp')
|
||||
elif self.operator == '<':
|
||||
result = g_llvm_builder.fcmp(FCMP_ULT, left, right, 'cmptmp')
|
||||
# Convert bool 0 or 1 to double 0.0 or 1.0.
|
||||
return g_llvm_builder.uitofp(result, Type.double(), 'booltmp')
|
||||
else:
|
||||
raise RuntimeError('Unknown binary operator.')
|
||||
|
||||
# Expression class for function calls.
|
||||
class CallExpressionNode(ExpressionNode):
|
||||
|
|
@ -787,13 +795,13 @@ need to `download <../download.html>`_ and
|
|||
# Look up the name in the global module table.
|
||||
callee = g_llvm_module.get_function_named(self.callee)
|
||||
|
||||
# Check for argument mismatch error.
|
||||
if len(callee.args) != len(self.args):
|
||||
raise RuntimeError('Incorrect number of arguments passed.')
|
||||
|
||||
arg_values = [i.CodeGen() for i in self.args]
|
||||
|
||||
return g_llvm_builder.call(callee, arg_values, 'calltmp')
|
||||
# Check for argument mismatch error.
|
||||
if len(callee.args) != len(self.args):
|
||||
raise RuntimeError('Incorrect number of arguments passed.')
|
||||
|
||||
arg_values = [i.CodeGen() for i in self.args]
|
||||
|
||||
return g_llvm_builder.call(callee, arg_values, 'calltmp')
|
||||
|
||||
# This class represents the "prototype" for a function, which captures its name,
|
||||
# and its argument names (thus implicitly the number of arguments the function
|
||||
|
|
@ -805,9 +813,9 @@ need to `download <../download.html>`_ and
|
|||
self.args = args
|
||||
|
||||
def CodeGen(self):
|
||||
# Make the function type, eg. double(double,double).
|
||||
funct_type = Type.function(
|
||||
Type.double(), [Type.double()] * len(self.args), False)
|
||||
# Make the function type, eg. double(double,double).
|
||||
funct_type = Type.function(
|
||||
Type.double(), [Type.double()] * len(self.args), False)
|
||||
|
||||
function = Function.new(g_llvm_module, funct_type, self.name)
|
||||
|
||||
|
|
@ -866,8 +874,10 @@ need to `download <../download.html>`_ and
|
|||
|
||||
return function
|
||||
|
||||
Parser
|
||||
------
|
||||
Parser
|
||||
------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class Parser(object):
|
||||
|
||||
|
|
@ -877,13 +887,13 @@ need to `download <../download.html>`_ and
|
|||
self.Next()
|
||||
|
||||
# Provide a simple token buffer. Parser.current is the current token the
|
||||
# parser is looking at. Parser.Next() reads another token from the lexer
|
||||
and # updates Parser.current with its results.
|
||||
# parser is looking at. Parser.Next() reads another token from the lexer and
|
||||
# updates Parser.current with its results.
|
||||
def Next(self):
|
||||
self.current = self.tokens.next()
|
||||
|
||||
# Gets the precedence of the current token, or -1 if the token is not a
|
||||
binary # operator.
|
||||
# Gets the precedence of the current token, or -1 if the token is not a binary
|
||||
# operator.
|
||||
def GetCurrentTokenPrecedence(self):
|
||||
if isinstance(self.current, CharacterToken):
|
||||
return self.binop_precedence.get(self.current.char, -1)
|
||||
|
|
@ -893,7 +903,7 @@ need to `download <../download.html>`_ and
|
|||
# identifierexpr ::= identifier | identifier '(' expression* ')'
|
||||
def ParseIdentifierExpr(self):
|
||||
identifier_name = self.current.name
|
||||
self.Next() # eat identifier.
|
||||
self.Next() # eat identifier.
|
||||
|
||||
if self.current != CharacterToken('('): # Simple variable reference.
|
||||
return VariableExpressionNode(identifier_name)
|
||||
|
|
@ -916,7 +926,7 @@ need to `download <../download.html>`_ and
|
|||
# numberexpr ::= number
|
||||
def ParseNumberExpr(self):
|
||||
result = NumberExpressionNode(self.current.value)
|
||||
self.Next() # consume the number.
|
||||
self.Next() # consume the number.
|
||||
return result
|
||||
|
||||
# parenexpr ::= '(' expression ')'
|
||||
|
|
@ -948,25 +958,25 @@ need to `download <../download.html>`_ and
|
|||
while True:
|
||||
precedence = self.GetCurrentTokenPrecedence()
|
||||
|
||||
# If this is a binary operator that binds at least as tightly as the
|
||||
# current one, consume it; otherwise we are done.
|
||||
if precedence < left_precedence:
|
||||
return left
|
||||
|
||||
binary_operator = self.current.char
|
||||
self.Next() # eat the operator.
|
||||
|
||||
# Parse the primary expression after the binary operator.
|
||||
right = self.ParsePrimary()
|
||||
|
||||
# If binary_operator binds less tightly with right than the operator after
|
||||
# right, let the pending operator take right as its left.
|
||||
next_precedence = self.GetCurrentTokenPrecedence()
|
||||
if precedence < next_precedence:
|
||||
right = self.ParseBinOpRHS(right, precedence + 1)
|
||||
|
||||
# Merge left/right.
|
||||
left = BinaryOperatorExpressionNode(binary_operator, left, right)
|
||||
# If this is a binary operator that binds at least as tightly as the
|
||||
# current one, consume it; otherwise we are done.
|
||||
if precedence < left_precedence:
|
||||
return left
|
||||
|
||||
binary_operator = self.current.char
|
||||
self.Next() # eat the operator.
|
||||
|
||||
# Parse the primary expression after the binary operator.
|
||||
right = self.ParsePrimary()
|
||||
|
||||
# If binary_operator binds less tightly with right than the operator after
|
||||
# right, let the pending operator take right as its left.
|
||||
next_precedence = self.GetCurrentTokenPrecedence()
|
||||
if precedence < next_precedence:
|
||||
right = self.ParseBinOpRHS(right, precedence + 1)
|
||||
|
||||
# Merge left/right.
|
||||
left = BinaryOperatorExpressionNode(binary_operator, left, right)
|
||||
|
||||
# expression ::= primary binoprhs
|
||||
def ParseExpression(self):
|
||||
|
|
@ -1000,7 +1010,7 @@ need to `download <../download.html>`_ and
|
|||
|
||||
# definition ::= 'def' prototype expression
|
||||
def ParseDefinition(self):
|
||||
self.Next() # eat def.
|
||||
self.Next() # eat def.
|
||||
proto = self.ParsePrototype()
|
||||
body = self.ParseExpression()
|
||||
return FunctionNode(proto, body)
|
||||
|
|
@ -1011,7 +1021,7 @@ need to `download <../download.html>`_ and
|
|||
return FunctionNode(proto, self.ParseExpression())
|
||||
|
||||
# external ::= 'extern' prototype
|
||||
def ParseExtern(self):
|
||||
def ParseExtern(self):
|
||||
self.Next() # eat extern.
|
||||
return self.ParsePrototype()
|
||||
|
||||
|
|
@ -1026,17 +1036,19 @@ need to `download <../download.html>`_ and
|
|||
self.Handle(self.ParseTopLevelExpr, 'Read a top-level expression:')
|
||||
|
||||
def Handle(self, function, message):
|
||||
try:
|
||||
print message, function().CodeGen()
|
||||
except Exception, e:
|
||||
print 'Error:', e
|
||||
try:
|
||||
self.Next() # Skip for error recovery.
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
print message, function().CodeGen()
|
||||
except Exception, e:
|
||||
print 'Error:', e
|
||||
try:
|
||||
self.Next() # Skip for error recovery.
|
||||
except:
|
||||
pass
|
||||
|
||||
Main driver code.
|
||||
-----------------
|
||||
Main driver code.
|
||||
-----------------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def main():
|
||||
# Install standard binary operators.
|
||||
|
|
|
|||
|
|
@ -284,10 +284,10 @@ this:
|
|||
print 'Evaluated to:', result.as_real(Type.double())
|
||||
except Exception, e:
|
||||
print 'Error:', e
|
||||
try:
|
||||
self.Next() # Skip for error recovery.
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
self.Next() # Skip for error recovery.
|
||||
except:
|
||||
pass
|
||||
|
||||
Recall that we compile top-level expressions into a self-contained LLVM
|
||||
function that takes no arguments and returns the computed double.
|
||||
|
|
@ -400,9 +400,10 @@ example, we can create a C file with the following simple function:
|
|||
|
||||
.. code-block:: c
|
||||
|
||||
#include
|
||||
#include <stdio.h>
|
||||
|
||||
double putchard(double x) { putchar((char)x); return 0; } {%
|
||||
double putchard(double x) {
|
||||
putchar((char)x); return 0; } {%
|
||||
endhighlight %}
|
||||
|
||||
We can then compile this into a shared library with GCC:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue