jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

got rid of some arcane module names

Browse source
This commit is contained in:
Araq 2011-04-21 00:54:44 +02:00
commit 36c67455d4
34 changed files with 327 additions and 1258 deletions
Download patch file Download diff file Expand all files Collapse all files

768
compiler/scanner.nim
View file

@ -1,768 +0,0 @@
#
#
# The Nimrod Compiler
# (c) Copyright 2011 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
# This scanner is handwritten for efficiency. I used an elegant buffering
# scheme which I have not seen anywhere else:
# We guarantee that a whole line is in the buffer. Thus only when scanning
# the \n or \r character we have to check wether we need to read in the next
# chunk. (\n or \r already need special handling for incrementing the line
# counter; choosing both \n and \r allows the scanner to properly read Unix,
# DOS or Macintosh text files, even when it is not the native format.
import
nhashes, options, msgs, strutils, platform, idents, lexbase, llstream,
wordrecg
const
MaxLineLength* = 80 # lines longer than this lead to a warning
numChars*: TCharSet = {'0'..'9', 'a'..'z', 'A'..'Z'}
SymChars*: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'}
SymStartChars*: TCharSet = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
OpChars*: TCharSet = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
'|', '=', '%', '&', '$', '@', '~', ':', '\x80'..'\xFF'}
type
TTokType* = enum
tkInvalid, tkEof, # order is important here!
tkSymbol, # keywords:
tkAddr, tkAnd, tkAs, tkAsm, tkAtomic,
tkBind, tkBlock, tkBreak, tkCase, tkCast,
tkConst, tkContinue, tkConverter, tkDiscard, tkDistinct, tkDiv, tkElif,
tkElse, tkEnd, tkEnum, tkExcept, tkFinally, tkFor, tkFrom, tkGeneric, tkIf,
tkImplies, tkImport, tkIn, tkInclude, tkIs, tkIsnot, tkIterator,
tkLambda, tkLet,
tkMacro, tkMethod, tkMod, tkNil, tkNot, tkNotin, tkObject, tkOf, tkOr,
tkOut, tkProc, tkPtr, tkRaise, tkRef, tkReturn, tkShl, tkShr, tkTemplate,
tkTry, tkTuple, tkType, tkVar, tkWhen, tkWhile, tkWith, tkWithout, tkXor,
tkYield, # end of keywords
tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit, tkFloatLit,
tkFloat32Lit, tkFloat64Lit, tkStrLit, tkRStrLit, tkTripleStrLit,
tkGStrLit, tkGTripleStrLit, tkCharLit, tkParLe, tkParRi, tkBracketLe,
tkBracketRi, tkCurlyLe, tkCurlyRi,
tkBracketDotLe, tkBracketDotRi, # [. and .]
tkCurlyDotLe, tkCurlyDotRi, # {. and .}
tkParDotLe, tkParDotRi, # (. and .)
tkComma, tkSemiColon, tkColon, tkColonColon, tkEquals, tkDot, tkDotDot,
tkHat, tkOpr, tkComment, tkAccent, tkInd, tkSad,
tkDed, # pseudo token types used by the source renderers:
tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
TTokTypes* = set[TTokType]
const
tokKeywordLow* = succ(tkSymbol)
tokKeywordHigh* = pred(tkIntLit)
TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]",
"tkSymbol",
"addr", "and", "as", "asm", "atomic",
"bind", "block", "break", "case", "cast",
"const", "continue", "converter", "discard", "distinct", "div", "elif",
"else", "end", "enum", "except", "finally", "for", "from", "generic", "if",
"implies", "import", "in", "include", "is", "isnot", "iterator",
"lambda", "let",
"macro", "method", "mod", "nil", "not", "notin", "object", "of", "or",
"out", "proc", "ptr", "raise", "ref", "return", "shl", "shr", "template",
"try", "tuple", "type", "var", "when", "while", "with", "without", "xor",
"yield",
"tkIntLit", "tkInt8Lit", "tkInt16Lit", "tkInt32Lit", "tkInt64Lit",
"tkFloatLit", "tkFloat32Lit", "tkFloat64Lit", "tkStrLit", "tkRStrLit",
"tkTripleStrLit", "tkGStrLit", "tkGTripleStrLit", "tkCharLit", "(",
")", "[", "]", "{", "}", "[.", ".]", "{.", ".}", "(.", ".)", ",", ";",
":", "::",
"=", ".", "..", "^", "tkOpr", "tkComment", "`", "[new indentation]",
"[same indentation]", "[dedentation]", "tkSpaces", "tkInfixOpr",
"tkPrefixOpr", "tkPostfixOpr"]
type
TNumericalBase* = enum
base10, # base10 is listed as the first element,
# so that it is the correct default value
base2, base8, base16
PToken* = ref TToken
TToken* = object # a Nimrod token
tokType*: TTokType # the type of the token
indent*: int # the indentation; only valid if tokType = tkIndent
ident*: PIdent # the parsed identifier
iNumber*: BiggestInt # the parsed integer literal
fNumber*: BiggestFloat # the parsed floating point literal
base*: TNumericalBase # the numerical base; only valid for int
# or float literals
literal*: string # the parsed (string) literal; and
# documentation comments are here too
next*: PToken # next token; can be used for arbitrary look-ahead
PLexer* = ref TLexer
TLexer* = object of TBaseLexer
filename*: string
indentStack*: seq[int] # the indentation stack
dedent*: int # counter for DED token generation
indentAhead*: int # if > 0 an indendation has already been read
# this is needed because scanning comments
# needs so much look-ahead
var gLinesCompiled*: int # all lines that have been compiled
proc pushInd*(L: var TLexer, indent: int)
proc popInd*(L: var TLexer)
proc isKeyword*(kind: TTokType): bool
proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream)
proc rawGetTok*(L: var TLexer, tok: var TToken)
# reads in the next token into tok and skips it
proc getColumn*(L: TLexer): int
proc getLineInfo*(L: TLexer): TLineInfo
proc closeLexer*(lex: var TLexer)
proc PrintTok*(tok: PToken)
proc tokToStr*(tok: PToken): string
proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "")
# the Pascal scanner uses this too:
proc fillToken*(L: var TToken)
# implementation
proc isKeyword(kind: TTokType): bool =
result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
proc isNimrodIdentifier*(s: string): bool =
if s[0] in SymStartChars:
var i = 1
while i < s.len:
if s[i] == '_':
inc(i)
if s[i] notin SymChars: return
if s[i] notin SymChars: return
inc(i)
result = true
proc pushInd(L: var TLexer, indent: int) =
var length = len(L.indentStack)
setlen(L.indentStack, length + 1)
if (indent > L.indentStack[length - 1]):
L.indentstack[length] = indent
else:
InternalError("pushInd")
proc popInd(L: var TLexer) =
var length = len(L.indentStack)
setlen(L.indentStack, length - 1)
proc findIdent(L: TLexer, indent: int): bool =
for i in countdown(len(L.indentStack) - 1, 0):
if L.indentStack[i] == indent:
return true
proc tokToStr(tok: PToken): string =
case tok.tokType
of tkIntLit..tkInt64Lit: result = $tok.iNumber
of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
of tkInvalid, tkStrLit..tkCharLit, tkComment: result = tok.literal
of tkParLe..tkColon, tkEof, tkInd, tkSad, tkDed, tkAccent:
result = tokTypeToStr[tok.tokType]
else:
if (tok.ident != nil):
result = tok.ident.s
else:
InternalError("tokToStr")
result = ""
proc PrintTok(tok: PToken) =
write(stdout, TokTypeToStr[tok.tokType])
write(stdout, " ")
writeln(stdout, tokToStr(tok))
var dummyIdent: PIdent
proc fillToken(L: var TToken) =
L.TokType = tkInvalid
L.iNumber = 0
L.Indent = 0
L.literal = ""
L.fNumber = 0.0
L.base = base10
L.ident = dummyIdent # this prevents many bugs!
proc openLexer(lex: var TLexer, filename: string, inputstream: PLLStream) =
openBaseLexer(lex, inputstream)
lex.indentStack = @[0]
lex.filename = filename
lex.indentAhead = - 1
inc(lex.Linenumber, inputstream.lineOffset)
proc closeLexer(lex: var TLexer) =
inc(gLinesCompiled, lex.LineNumber)
closeBaseLexer(lex)
proc getColumn(L: TLexer): int =
result = getColNumber(L, L.bufPos)
proc getLineInfo(L: TLexer): TLineInfo =
result = newLineInfo(L.filename, L.linenumber, getColNumber(L, L.bufpos))
proc lexMessage(L: TLexer, msg: TMsgKind, arg = "") =
msgs.Message(getLineInfo(L), msg, arg)
proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
var info = newLineInfo(L.filename, L.linenumber, pos - L.lineStart)
msgs.Message(info, msg, arg)
proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: TCharSet) =
var pos = L.bufpos # use registers for pos, buf
var buf = L.buf
while true:
if buf[pos] in chars:
add(tok.literal, buf[pos])
Inc(pos)
else:
break
if buf[pos] == '_':
if buf[pos+1] notin chars:
lexMessage(L, errInvalidToken, "_")
break
add(tok.literal, '_')
Inc(pos)
L.bufPos = pos
proc matchTwoChars(L: TLexer, first: Char, second: TCharSet): bool =
result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in Second)
proc isFloatLiteral(s: string): bool =
for i in countup(0, len(s) + 0 - 1):
if s[i] in {'.', 'e', 'E'}:
return true
result = false
proc GetNumber(L: var TLexer): TToken =
var
pos, endpos: int
xi: biggestInt
# get the base:
result.tokType = tkIntLit # int literal until we know better
result.literal = ""
result.base = base10 # BUGFIX
pos = L.bufpos # make sure the literal is correct for error messages:
matchUnderscoreChars(L, result, {'A'..'Z', 'a'..'z', '0'..'9'})
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
add(result.literal, '.')
inc(L.bufpos)
#matchUnderscoreChars(L, result, ['A'..'Z', 'a'..'z', '0'..'9'])
matchUnderscoreChars(L, result, {'0'..'9'})
if L.buf[L.bufpos] in {'e', 'E'}:
add(result.literal, 'e')
inc(L.bufpos)
if L.buf[L.bufpos] in {'+', '-'}:
add(result.literal, L.buf[L.bufpos])
inc(L.bufpos)
matchUnderscoreChars(L, result, {'0'..'9'})
endpos = L.bufpos
if L.buf[endpos] == '\'':
#matchUnderscoreChars(L, result, ['''', 'f', 'F', 'i', 'I', '0'..'9']);
inc(endpos)
L.bufpos = pos # restore position
case L.buf[endpos]
of 'f', 'F':
inc(endpos)
if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'):
result.tokType = tkFloat64Lit
inc(endpos, 2)
elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'):
result.tokType = tkFloat32Lit
inc(endpos, 2)
else:
lexMessage(L, errInvalidNumber, result.literal)
of 'i', 'I':
inc(endpos)
if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'):
result.tokType = tkInt64Lit
inc(endpos, 2)
elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'):
result.tokType = tkInt32Lit
inc(endpos, 2)
elif (L.buf[endpos] == '1') and (L.buf[endpos + 1] == '6'):
result.tokType = tkInt16Lit
inc(endpos, 2)
elif (L.buf[endpos] == '8'):
result.tokType = tkInt8Lit
inc(endpos)
else:
lexMessage(L, errInvalidNumber, result.literal)
else: lexMessage(L, errInvalidNumber, result.literal)
else:
L.bufpos = pos # restore position
try:
if (L.buf[pos] == '0') and
(L.buf[pos + 1] in {'x', 'X', 'b', 'B', 'o', 'O', 'c', 'C'}):
inc(pos, 2)
xi = 0 # it may be a base prefix
case L.buf[pos - 1] # now look at the optional type suffix:
of 'b', 'B':
result.base = base2
while true:
case L.buf[pos]
of 'A'..'Z', 'a'..'z', '2'..'9', '.':
lexMessage(L, errInvalidNumber, result.literal)
inc(pos)
of '_':
if L.buf[pos+1] notin {'0'..'1'}:
lexMessage(L, errInvalidToken, "_")
break
inc(pos)
of '0', '1':
xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
inc(pos)
else: break
of 'o', 'c', 'C':
result.base = base8
while true:
case L.buf[pos]
of 'A'..'Z', 'a'..'z', '8'..'9', '.':
lexMessage(L, errInvalidNumber, result.literal)
inc(pos)
of '_':
if L.buf[pos+1] notin {'0'..'7'}:
lexMessage(L, errInvalidToken, "_")
break
inc(pos)
of '0'..'7':
xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
inc(pos)
else: break
of 'O':
lexMessage(L, errInvalidNumber, result.literal)
of 'x', 'X':
result.base = base16
while true:
case L.buf[pos]
of 'G'..'Z', 'g'..'z':
lexMessage(L, errInvalidNumber, result.literal)
inc(pos)
of '_':
if L.buf[pos+1] notin {'0'..'9', 'a'..'f', 'A'..'F'}:
lexMessage(L, errInvalidToken, "_")
break
inc(pos)
of '0'..'9':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
inc(pos)
of 'a'..'f':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10)
inc(pos)
of 'A'..'F':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
inc(pos)
else: break
else: InternalError(getLineInfo(L), "getNumber")
case result.tokType
of tkIntLit, tkInt64Lit: result.iNumber = xi
of tkInt8Lit: result.iNumber = biggestInt(int8(toU8(int(xi))))
of tkInt16Lit: result.iNumber = biggestInt(toU16(int(xi)))
of tkInt32Lit: result.iNumber = biggestInt(toU32(xi))
of tkFloat32Lit:
result.fNumber = (cast[PFloat32](addr(xi)))[]
# note: this code is endian neutral!
# XXX: Test this on big endian machine!
of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[]
else: InternalError(getLineInfo(L), "getNumber")
elif isFloatLiteral(result.literal) or (result.tokType == tkFloat32Lit) or
(result.tokType == tkFloat64Lit):
result.fnumber = parseFloat(result.literal)
if result.tokType == tkIntLit: result.tokType = tkFloatLit
else:
result.iNumber = ParseBiggestInt(result.literal)
if (result.iNumber < low(int32)) or (result.iNumber > high(int32)):
if result.tokType == tkIntLit:
result.tokType = tkInt64Lit
elif result.tokType != tkInt64Lit:
lexMessage(L, errInvalidNumber, result.literal)
except EInvalidValue: lexMessage(L, errInvalidNumber, result.literal)
except EOverflow: lexMessage(L, errNumberOutOfRange, result.literal)
except EOutOfRange: lexMessage(L, errNumberOutOfRange, result.literal)
L.bufpos = endpos
proc handleHexChar(L: var TLexer, xi: var int) =
case L.buf[L.bufpos]
of '0'..'9':
xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('0'))
inc(L.bufpos)
of 'a'..'f':
xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('a') + 10)
inc(L.bufpos)
of 'A'..'F':
xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('A') + 10)
inc(L.bufpos)
else: nil
proc handleDecChars(L: var TLexer, xi: var int) =
while L.buf[L.bufpos] in {'0'..'9'}:
xi = (xi * 10) + (ord(L.buf[L.bufpos]) - ord('0'))
inc(L.bufpos)
proc getEscapedChar(L: var TLexer, tok: var TToken) =
inc(L.bufpos) # skip '\'
case L.buf[L.bufpos]
of 'n', 'N':
if tok.toktype == tkCharLit: lexMessage(L, errNnotAllowedInCharacter)
add(tok.literal, tnl)
Inc(L.bufpos)
of 'r', 'R', 'c', 'C':
add(tok.literal, CR)
Inc(L.bufpos)
of 'l', 'L':
add(tok.literal, LF)
Inc(L.bufpos)
of 'f', 'F':
add(tok.literal, FF)
inc(L.bufpos)
of 'e', 'E':
add(tok.literal, ESC)
Inc(L.bufpos)
of 'a', 'A':
add(tok.literal, BEL)
Inc(L.bufpos)
of 'b', 'B':
add(tok.literal, BACKSPACE)
Inc(L.bufpos)
of 'v', 'V':
add(tok.literal, VT)
Inc(L.bufpos)
of 't', 'T':
add(tok.literal, Tabulator)
Inc(L.bufpos)
of '\'', '\"':
add(tok.literal, L.buf[L.bufpos])
Inc(L.bufpos)
of '\\':
add(tok.literal, '\\')
Inc(L.bufpos)
of 'x', 'X':
inc(L.bufpos)
var xi = 0
handleHexChar(L, xi)
handleHexChar(L, xi)
add(tok.literal, Chr(xi))
of '0'..'9':
if matchTwoChars(L, '0', {'0'..'9'}):
lexMessage(L, warnOctalEscape)
var xi = 0
handleDecChars(L, xi)
if (xi <= 255): add(tok.literal, Chr(xi))
else: lexMessage(L, errInvalidCharacterConstant)
else: lexMessage(L, errInvalidCharacterConstant)
proc HandleCRLF(L: var TLexer, pos: int): int =
case L.buf[pos]
of CR:
if getColNumber(L, pos) > MaxLineLength:
lexMessagePos(L, hintLineTooLong, pos)
result = lexbase.HandleCR(L, pos)
of LF:
if getColNumber(L, pos) > MaxLineLength:
lexMessagePos(L, hintLineTooLong, pos)
result = lexbase.HandleLF(L, pos)
else: result = pos
proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
var pos = L.bufPos + 1 # skip "
var buf = L.buf # put `buf` in a register
var line = L.linenumber # save linenumber for better error message
if buf[pos] == '\"' and buf[pos+1] == '\"':
tok.tokType = tkTripleStrLit # long string literal:
inc(pos, 2) # skip ""
# skip leading newline:
pos = HandleCRLF(L, pos)
buf = L.buf
while true:
case buf[pos]
of '\"':
if buf[pos+1] == '\"' and buf[pos+2] == '\"' and
buf[pos+3] != '\"':
L.bufpos = pos + 3 # skip the three """
break
add(tok.literal, '\"')
Inc(pos)
of CR, LF:
pos = HandleCRLF(L, pos)
buf = L.buf
add(tok.literal, tnl)
of lexbase.EndOfFile:
var line2 = L.linenumber
L.LineNumber = line
lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart)
L.LineNumber = line2
break
else:
add(tok.literal, buf[pos])
Inc(pos)
else:
# ordinary string literal
if rawMode: tok.tokType = tkRStrLit
else: tok.tokType = tkStrLit
while true:
var c = buf[pos]
if c == '\"':
if rawMode and buf[pos+1] == '\"':
inc(pos, 2)
add(tok.literal, '"')
else:
inc(pos) # skip '"'
break
elif c in {CR, LF, lexbase.EndOfFile}:
lexMessage(L, errClosingQuoteExpected)
break
elif (c == '\\') and not rawMode:
L.bufPos = pos
getEscapedChar(L, tok)
pos = L.bufPos
else:
add(tok.literal, c)
Inc(pos)
L.bufpos = pos
proc getCharacter(L: var TLexer, tok: var TToken) =
Inc(L.bufpos) # skip '
var c = L.buf[L.bufpos]
case c
of '\0'..Pred(' '), '\'': lexMessage(L, errInvalidCharacterConstant)
of '\\': getEscapedChar(L, tok)
else:
tok.literal = $c
Inc(L.bufpos)
if L.buf[L.bufpos] != '\'': lexMessage(L, errMissingFinalQuote)
inc(L.bufpos) # skip '
proc getSymbol(L: var TLexer, tok: var TToken) =
var h: THash = 0
var pos = L.bufpos
var buf = L.buf
while true:
var c = buf[pos]
case c
of 'a'..'z', '0'..'9', '\x80'..'\xFF':
h = concHash(h, ord(c))
of 'A'..'Z':
c = chr(ord(c) + (ord('a') - ord('A'))) # toLower()
h = concHash(h, ord(c))
of '_':
if buf[pos+1] notin SymChars:
lexMessage(L, errInvalidToken, "_")
break
else: break
Inc(pos)
h = finishHash(h)
tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
L.bufpos = pos
if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
(tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
tok.tokType = tkSymbol
else:
tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))
proc endOperator(L: var TLexer, tok: var TToken, pos: int,
hash: THash) {.inline.} =
var h = finishHash(hash)
tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
else: tok.tokType = TTokType(tok.ident.id - oprLow + ord(tkColon))
L.bufpos = pos
proc getOperator(L: var TLexer, tok: var TToken) =
var pos = L.bufpos
var buf = L.buf
var h: THash = 0
while true:
var c = buf[pos]
if c notin OpChars: break
h = concHash(h, Ord(c))
Inc(pos)
endOperator(L, tok, pos, h)
proc handleIndentation(L: var TLexer, tok: var TToken, indent: int) =
tok.indent = indent
var i = high(L.indentStack)
if indent > L.indentStack[i]:
tok.tokType = tkInd
elif indent == L.indentStack[i]:
tok.tokType = tkSad
else:
# check we have the indentation somewhere in the stack:
while (i >= 0) and (indent != L.indentStack[i]):
dec(i)
inc(L.dedent)
dec(L.dedent)
tok.tokType = tkDed
if i < 0:
tok.tokType = tkSad # for the parser it is better as SAD
lexMessage(L, errInvalidIndentation)
proc scanComment(L: var TLexer, tok: var TToken) =
var pos = L.bufpos
var buf = L.buf
# a comment ends if the next line does not start with the # on the same
# column after only whitespace
tok.tokType = tkComment
var col = getColNumber(L, pos)
while true:
while not (buf[pos] in {CR, LF, lexbase.EndOfFile}):
add(tok.literal, buf[pos])
inc(pos)
pos = handleCRLF(L, pos)
buf = L.buf
var indent = 0
while buf[pos] == ' ':
inc(pos)
inc(indent)
if (buf[pos] == '#') and (col == indent):
tok.literal = tok.literal & "\n"
else:
if buf[pos] > ' ':
L.indentAhead = indent
inc(L.dedent)
break
L.bufpos = pos
proc skip(L: var TLexer, tok: var TToken) =
var pos = L.bufpos
var buf = L.buf
while true:
case buf[pos]
of ' ':
Inc(pos)
of Tabulator:
lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
inc(pos) # BUGFIX
of CR, LF:
pos = HandleCRLF(L, pos)
buf = L.buf
var indent = 0
while buf[pos] == ' ':
Inc(pos)
Inc(indent)
if (buf[pos] > ' '):
handleIndentation(L, tok, indent)
break
else:
break # EndOfFile also leaves the loop
L.bufpos = pos
proc rawGetTok(L: var TLexer, tok: var TToken) =
fillToken(tok)
if L.dedent > 0:
dec(L.dedent)
if L.indentAhead >= 0:
handleIndentation(L, tok, L.indentAhead)
L.indentAhead = - 1
else:
tok.tokType = tkDed
return
skip(L, tok)
# got an documentation comment or tkIndent, return that:
if tok.toktype != tkInvalid: return
var c = L.buf[L.bufpos]
if c in SymStartChars - {'r', 'R', 'l'}:
getSymbol(L, tok)
elif c in {'0'..'9'}:
tok = getNumber(L)
else:
case c
of '#':
scanComment(L, tok)
of '*':
# '*:' is unfortunately a special case, because it is two tokens in
# 'var v*: int'.
if L.buf[L.bufpos+1] == ':' and L.buf[L.bufpos+2] notin OpChars:
var h = concHash(0, ord('*'))
endOperator(L, tok, L.bufpos+1, h)
else:
getOperator(L, tok)
of ',':
tok.toktype = tkComma
Inc(L.bufpos)
of 'l':
# if we parsed exactly one character and its a small L (l), this
# is treated as a warning because it may be confused with the number 1
if not (L.buf[L.bufpos + 1] in (SymChars + {'_'})):
lexMessage(L, warnSmallLshouldNotBeUsed)
getSymbol(L, tok)
of 'r', 'R':
if L.buf[L.bufPos + 1] == '\"':
Inc(L.bufPos)
getString(L, tok, true)
else:
getSymbol(L, tok)
of '(':
Inc(L.bufpos)
if (L.buf[L.bufPos] == '.') and (L.buf[L.bufPos + 1] != '.'):
tok.toktype = tkParDotLe
Inc(L.bufpos)
else:
tok.toktype = tkParLe
of ')':
tok.toktype = tkParRi
Inc(L.bufpos)
of '[':
Inc(L.bufpos)
if (L.buf[L.bufPos] == '.') and (L.buf[L.bufPos + 1] != '.'):
tok.toktype = tkBracketDotLe
Inc(L.bufpos)
else:
tok.toktype = tkBracketLe
of ']':
tok.toktype = tkBracketRi
Inc(L.bufpos)
of '.':
if L.buf[L.bufPos + 1] == ']':
tok.tokType = tkBracketDotRi
Inc(L.bufpos, 2)
elif L.buf[L.bufPos + 1] == '}':
tok.tokType = tkCurlyDotRi
Inc(L.bufpos, 2)
elif L.buf[L.bufPos + 1] == ')':
tok.tokType = tkParDotRi
Inc(L.bufpos, 2)
else:
getOperator(L, tok)
of '{':
Inc(L.bufpos)
if (L.buf[L.bufPos] == '.') and (L.buf[L.bufPos+1] != '.'):
tok.toktype = tkCurlyDotLe
Inc(L.bufpos)
else:
tok.toktype = tkCurlyLe
of '}':
tok.toktype = tkCurlyRi
Inc(L.bufpos)
of ';':
tok.toktype = tkSemiColon
Inc(L.bufpos)
of '`':
tok.tokType = tkAccent
Inc(L.bufpos)
of '\"':
# check for extended raw string literal:
var rawMode = L.bufpos > 0 and L.buf[L.bufpos-1] in SymChars
getString(L, tok, rawMode)
if rawMode:
# tkRStrLit -> tkGStrLit
# tkTripleStrLit -> tkGTripleStrLit
inc(tok.tokType, 2)
of '\'':
tok.tokType = tkCharLit
getCharacter(L, tok)
tok.tokType = tkCharLit
else:
if c in OpChars:
getOperator(L, tok)
elif c == lexbase.EndOfFile:
tok.toktype = tkEof
else:
tok.literal = c & ""
tok.tokType = tkInvalid
lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')')
Inc(L.bufpos)
dummyIdent = getIdent("")