jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #38 from keitahaga/pas2nim

Browse source 
pas2nim: nhashes library doesn't exist
This commit is contained in:
Araq 2011-07-25 10:53:19 -07:00
commit 4d753e128e
1 changed files with 181 additions and 181 deletions
Download patch file Download diff file Expand all files Collapse all files

362
compiler/pas2nim/paslex.nim
View file

@ -10,30 +10,30 @@
# This module implements a FreePascal scanner. This is an adaption from # This module implements a FreePascal scanner. This is an adaption from
# the scanner module. # the scanner module.
import import
nhashes, options, msgs, strutils, platform, idents, lexbase, llstream hashes, options, msgs, strutils, platform, idents, lexbase, llstream
const const
MaxLineLength* = 80 # lines longer than this lead to a warning MaxLineLength* = 80 # lines longer than this lead to a warning
numChars*: TCharSet = {'0'..'9', 'a'..'z', 'A'..'Z'} numChars*: TCharSet = {'0'..'9', 'a'..'z', 'A'..'Z'}
SymChars*: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'} SymChars*: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'}
SymStartChars*: TCharSet = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'} SymStartChars*: TCharSet = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
OpChars*: TCharSet = {'+', '-', '*', '/', '<', '>', '!', '?', '^', '.', '|', OpChars*: TCharSet = {'+', '-', '*', '/', '<', '>', '!', '?', '^', '.', '|',
'=', ':', '%', '&', '$', '@', '~', '\x80'..'\xFF'} '=', ':', '%', '&', '$', '@', '~', '\x80'..'\xFF'}
# keywords are sorted! # keywords are sorted!
type type
TTokKind* = enum TTokKind* = enum
pxInvalid, pxEof, pxInvalid, pxEof,
pxAnd, pxArray, pxAs, pxAsm, pxBegin, pxCase, pxClass, pxConst, pxAnd, pxArray, pxAs, pxAsm, pxBegin, pxCase, pxClass, pxConst,
pxConstructor, pxDestructor, pxDiv, pxDo, pxDownto, pxElse, pxEnd, pxExcept, pxConstructor, pxDestructor, pxDiv, pxDo, pxDownto, pxElse, pxEnd, pxExcept,
pxExports, pxFinalization, pxFinally, pxFor, pxFunction, pxGoto, pxIf, pxExports, pxFinalization, pxFinally, pxFor, pxFunction, pxGoto, pxIf,
pxImplementation, pxIn, pxInherited, pxInitialization, pxInline, pxImplementation, pxIn, pxInherited, pxInitialization, pxInline,
pxInterface, pxIs, pxLabel, pxLibrary, pxMod, pxNil, pxNot, pxObject, pxOf, pxInterface, pxIs, pxLabel, pxLibrary, pxMod, pxNil, pxNot, pxObject, pxOf,
pxOr, pxOut, pxPacked, pxProcedure, pxProgram, pxProperty, pxRaise, pxOr, pxOut, pxPacked, pxProcedure, pxProgram, pxProperty, pxRaise,
pxRecord, pxRepeat, pxResourcestring, pxSet, pxShl, pxShr, pxThen, pxRecord, pxRepeat, pxResourcestring, pxSet, pxShl, pxShr, pxThen,
pxThreadvar, pxTo, pxTry, pxType, pxUnit, pxUntil, pxUses, pxVar, pxWhile, pxThreadvar, pxTo, pxTry, pxType, pxUnit, pxUntil, pxUses, pxVar, pxWhile,
pxWith, pxXor, pxWith, pxXor,
pxComment, # ordinary comment pxComment, # ordinary comment
pxCommand, # {@} pxCommand, # {@}
@ -41,22 +41,22 @@ type
pxPer, # {%} pxPer, # {%}
pxStrLit, pxSymbol, # a symbol pxStrLit, pxSymbol, # a symbol
pxIntLit, pxInt64Lit, # long constant like 0x70fffffff or out of int range pxIntLit, pxInt64Lit, # long constant like 0x70fffffff or out of int range
pxFloatLit, pxParLe, pxParRi, pxBracketLe, pxBracketRi, pxComma, pxFloatLit, pxParLe, pxParRi, pxBracketLe, pxBracketRi, pxComma,
pxSemiColon, pxColon, # operators pxSemiColon, pxColon, # operators
pxAsgn, pxEquals, pxDot, pxDotDot, pxHat, pxPlus, pxMinus, pxStar, pxSlash, pxAsgn, pxEquals, pxDot, pxDotDot, pxHat, pxPlus, pxMinus, pxStar, pxSlash,
pxLe, pxLt, pxGe, pxGt, pxNeq, pxAt, pxStarDirLe, pxStarDirRi, pxCurlyDirLe, pxLe, pxLt, pxGe, pxGt, pxNeq, pxAt, pxStarDirLe, pxStarDirRi, pxCurlyDirLe,
pxCurlyDirRi pxCurlyDirRi
TTokKinds* = set[TTokKind] TTokKinds* = set[TTokKind]
const const
Keywords = ["and", "array", "as", "asm", "begin", "case", "class", "const", Keywords = ["and", "array", "as", "asm", "begin", "case", "class", "const",
"constructor", "destructor", "div", "do", "downto", "else", "end", "except", "constructor", "destructor", "div", "do", "downto", "else", "end", "except",
"exports", "finalization", "finally", "for", "function", "goto", "if", "exports", "finalization", "finally", "for", "function", "goto", "if",
"implementation", "in", "inherited", "initialization", "inline", "implementation", "in", "inherited", "initialization", "inline",
"interface", "is", "label", "library", "mod", "nil", "not", "object", "of", "interface", "is", "label", "library", "mod", "nil", "not", "object", "of",
"or", "out", "packed", "procedure", "program", "property", "raise", "or", "out", "packed", "procedure", "program", "property", "raise",
"record", "repeat", "resourcestring", "set", "shl", "shr", "then", "record", "repeat", "resourcestring", "set", "shl", "shr", "then",
"threadvar", "to", "try", "type", "unit", "until", "uses", "var", "while", "threadvar", "to", "try", "type", "unit", "until", "uses", "var", "while",
"with", "xor"] "with", "xor"]
firstKeyword = pxAnd firstKeyword = pxAnd
@ -72,10 +72,10 @@ type
base*: TNumericalBase # the numerical base; only valid for int base*: TNumericalBase # the numerical base; only valid for int
# or float literals # or float literals
literal*: string # the parsed (string) literal literal*: string # the parsed (string) literal
TLexer* = object of TBaseLexer TLexer* = object of TBaseLexer
filename*: string filename*: string
proc getTok*(L: var TLexer, tok: var TToken) proc getTok*(L: var TLexer, tok: var TToken)
proc PrintTok*(tok: TToken) proc PrintTok*(tok: TToken)
@ -86,32 +86,32 @@ var
dummyIdent: PIdent dummyIdent: PIdent
gLinesCompiled: int gLinesCompiled: int
proc fillToken(L: var TToken) = proc fillToken(L: var TToken) =
L.xkind = pxInvalid L.xkind = pxInvalid
L.iNumber = 0 L.iNumber = 0
L.literal = "" L.literal = ""
L.fNumber = 0.0 L.fNumber = 0.0
L.base = base10 L.base = base10
L.ident = dummyIdent # this prevents many bugs! L.ident = dummyIdent # this prevents many bugs!
proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream) = proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream) =
openBaseLexer(lex, inputstream) openBaseLexer(lex, inputstream)
lex.filename = filename lex.filename = filename
proc closeLexer*(lex: var TLexer) = proc closeLexer*(lex: var TLexer) =
inc(gLinesCompiled, lex.LineNumber) inc(gLinesCompiled, lex.LineNumber)
closeBaseLexer(lex) closeBaseLexer(lex)
proc getColumn(L: TLexer): int = proc getColumn(L: TLexer): int =
result = getColNumber(L, L.bufPos) result = getColNumber(L, L.bufPos)
proc getLineInfo*(L: TLexer): TLineInfo = proc getLineInfo*(L: TLexer): TLineInfo =
result = newLineInfo(L.filename, L.linenumber, getColNumber(L, L.bufpos)) result = newLineInfo(L.filename, L.linenumber, getColNumber(L, L.bufpos))
proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "") = proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "") =
msgs.GenericMessage(getLineInfo(L), msg, arg) msgs.GenericMessage(getLineInfo(L), msg, arg)
proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") = proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
var info = newLineInfo(L.filename, L.linenumber, pos - L.lineStart) var info = newLineInfo(L.filename, L.linenumber, pos - L.lineStart)
msgs.GenericMessage(info, msg, arg) msgs.GenericMessage(info, msg, arg)
@ -154,196 +154,196 @@ proc TokKindToStr*(k: TTokKind): string =
of pxCurlyDirLe: result = "{$" of pxCurlyDirLe: result = "{$"
of pxCurlyDirRi: result = "}" of pxCurlyDirRi: result = "}"
proc `$`(tok: TToken): string = proc `$`(tok: TToken): string =
case tok.xkind case tok.xkind
of pxInvalid, pxComment, pxStrLit: result = tok.literal of pxInvalid, pxComment, pxStrLit: result = tok.literal
of pxSymbol: result = tok.ident.s of pxSymbol: result = tok.ident.s
of pxIntLit, pxInt64Lit: result = $tok.iNumber of pxIntLit, pxInt64Lit: result = $tok.iNumber
of pxFloatLit: result = $tok.fNumber of pxFloatLit: result = $tok.fNumber
else: result = TokKindToStr(tok.xkind) else: result = TokKindToStr(tok.xkind)
proc PrintTok(tok: TToken) = proc PrintTok(tok: TToken) =
writeln(stdout, $tok) writeln(stdout, $tok)
proc setKeyword(L: var TLexer, tok: var TToken) = proc setKeyword(L: var TLexer, tok: var TToken) =
var x = binaryStrSearch(keywords, toLower(tok.ident.s)) var x = binaryStrSearch(keywords, toLower(tok.ident.s))
if x < 0: tok.xkind = pxSymbol if x < 0: tok.xkind = pxSymbol
else: tok.xKind = TTokKind(x + ord(firstKeyword)) else: tok.xKind = TTokKind(x + ord(firstKeyword))
proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: TCharSet) = proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: TCharSet) =
# matches ([chars]_)* # matches ([chars]_)*
var pos = L.bufpos # use registers for pos, buf var pos = L.bufpos # use registers for pos, buf
var buf = L.buf var buf = L.buf
while true: while true:
if buf[pos] in chars: if buf[pos] in chars:
add(tok.literal, buf[pos]) add(tok.literal, buf[pos])
Inc(pos) Inc(pos)
else: else:
break break
if buf[pos] == '_': if buf[pos] == '_':
add(tok.literal, '_') add(tok.literal, '_')
Inc(pos) Inc(pos)
L.bufPos = pos L.bufPos = pos
proc isFloatLiteral(s: string): bool = proc isFloatLiteral(s: string): bool =
for i in countup(0, len(s)-1): for i in countup(0, len(s)-1):
if s[i] in {'.', 'e', 'E'}: if s[i] in {'.', 'e', 'E'}:
return true return true
proc getNumber2(L: var TLexer, tok: var TToken) = proc getNumber2(L: var TLexer, tok: var TToken) =
var pos = L.bufpos + 1 # skip % var pos = L.bufpos + 1 # skip %
if not (L.buf[pos] in {'0'..'1'}): if not (L.buf[pos] in {'0'..'1'}):
# BUGFIX for %date% # BUGFIX for %date%
tok.xkind = pxInvalid tok.xkind = pxInvalid
add(tok.literal, '%') add(tok.literal, '%')
inc(L.bufpos) inc(L.bufpos)
return return
tok.base = base2 tok.base = base2
var xi: biggestInt = 0 var xi: biggestInt = 0
var bits = 0 var bits = 0
while true: while true:
case L.buf[pos] case L.buf[pos]
of 'A'..'Z', 'a'..'z', '2'..'9', '.': of 'A'..'Z', 'a'..'z', '2'..'9', '.':
lexMessage(L, errInvalidNumber) lexMessage(L, errInvalidNumber)
inc(pos) inc(pos)
of '_': of '_':
inc(pos) inc(pos)
of '0', '1': of '0', '1':
xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0')) xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
inc(pos) inc(pos)
inc(bits) inc(bits)
else: break else: break
tok.iNumber = xi tok.iNumber = xi
if (bits > 32): tok.xkind = pxInt64Lit if (bits > 32): tok.xkind = pxInt64Lit
else: tok.xkind = pxIntLit else: tok.xkind = pxIntLit
L.bufpos = pos L.bufpos = pos
proc getNumber16(L: var TLexer, tok: var TToken) = proc getNumber16(L: var TLexer, tok: var TToken) =
var pos = L.bufpos + 1 # skip $ var pos = L.bufpos + 1 # skip $
tok.base = base16 tok.base = base16
var xi: biggestInt = 0 var xi: biggestInt = 0
var bits = 0 var bits = 0
while true: while true:
case L.buf[pos] case L.buf[pos]
of 'G'..'Z', 'g'..'z', '.': of 'G'..'Z', 'g'..'z', '.':
lexMessage(L, errInvalidNumber) lexMessage(L, errInvalidNumber)
inc(pos) inc(pos)
of '_': inc(pos) of '_': inc(pos)
of '0'..'9': of '0'..'9':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0')) xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
inc(pos) inc(pos)
inc(bits, 4) inc(bits, 4)
of 'a'..'f': of 'a'..'f':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10) xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10)
inc(pos) inc(pos)
inc(bits, 4) inc(bits, 4)
of 'A'..'F': of 'A'..'F':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10) xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
inc(pos) inc(pos)
inc(bits, 4) inc(bits, 4)
else: break else: break
tok.iNumber = xi tok.iNumber = xi
if (bits > 32): if (bits > 32):
tok.xkind = pxInt64Lit tok.xkind = pxInt64Lit
else: else:
tok.xkind = pxIntLit tok.xkind = pxIntLit
L.bufpos = pos L.bufpos = pos
proc getNumber10(L: var TLexer, tok: var TToken) = proc getNumber10(L: var TLexer, tok: var TToken) =
tok.base = base10 tok.base = base10
matchUnderscoreChars(L, tok, {'0'..'9'}) matchUnderscoreChars(L, tok, {'0'..'9'})
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}): if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
add(tok.literal, '.') add(tok.literal, '.')
inc(L.bufpos) inc(L.bufpos)
matchUnderscoreChars(L, tok, {'e', 'E', '+', '-', '0'..'9'}) matchUnderscoreChars(L, tok, {'e', 'E', '+', '-', '0'..'9'})
try: try:
if isFloatLiteral(tok.literal): if isFloatLiteral(tok.literal):
tok.fnumber = parseFloat(tok.literal) tok.fnumber = parseFloat(tok.literal)
tok.xkind = pxFloatLit tok.xkind = pxFloatLit
else: else:
tok.iNumber = ParseInt(tok.literal) tok.iNumber = ParseInt(tok.literal)
if (tok.iNumber < low(int32)) or (tok.iNumber > high(int32)): if (tok.iNumber < low(int32)) or (tok.iNumber > high(int32)):
tok.xkind = pxInt64Lit tok.xkind = pxInt64Lit
else: else:
tok.xkind = pxIntLit tok.xkind = pxIntLit
except EInvalidValue: except EInvalidValue:
lexMessage(L, errInvalidNumber, tok.literal) lexMessage(L, errInvalidNumber, tok.literal)
except EOverflow: except EOverflow:
lexMessage(L, errNumberOutOfRange, tok.literal) lexMessage(L, errNumberOutOfRange, tok.literal)
proc HandleCRLF(L: var TLexer, pos: int): int = proc HandleCRLF(L: var TLexer, pos: int): int =
case L.buf[pos] case L.buf[pos]
of CR: result = lexbase.HandleCR(L, pos) of CR: result = lexbase.HandleCR(L, pos)
of LF: result = lexbase.HandleLF(L, pos) of LF: result = lexbase.HandleLF(L, pos)
else: result = pos else: result = pos
proc getString(L: var TLexer, tok: var TToken) = proc getString(L: var TLexer, tok: var TToken) =
var xi: int var xi: int
var pos = L.bufPos var pos = L.bufPos
var buf = L.buf var buf = L.buf
while true: while true:
if buf[pos] == '\'': if buf[pos] == '\'':
inc(pos) inc(pos)
while true: while true:
case buf[pos] case buf[pos]
of CR, LF, lexbase.EndOfFile: of CR, LF, lexbase.EndOfFile:
lexMessage(L, errClosingQuoteExpected) lexMessage(L, errClosingQuoteExpected)
break break
of '\'': of '\'':
inc(pos) inc(pos)
if buf[pos] == '\'': if buf[pos] == '\'':
inc(pos) inc(pos)
add(tok.literal, '\'') add(tok.literal, '\'')
else: else:
break break
else: else:
add(tok.literal, buf[pos]) add(tok.literal, buf[pos])
inc(pos) inc(pos)
elif buf[pos] == '#': elif buf[pos] == '#':
inc(pos) inc(pos)
xi = 0 xi = 0
case buf[pos] case buf[pos]
of '$': of '$':
inc(pos) inc(pos)
xi = 0 xi = 0
while true: while true:
case buf[pos] case buf[pos]
of '0'..'9': xi = (xi shl 4) or (ord(buf[pos]) - ord('0')) of '0'..'9': xi = (xi shl 4) or (ord(buf[pos]) - ord('0'))
of 'a'..'f': xi = (xi shl 4) or (ord(buf[pos]) - ord('a') + 10) of 'a'..'f': xi = (xi shl 4) or (ord(buf[pos]) - ord('a') + 10)
of 'A'..'F': xi = (xi shl 4) or (ord(buf[pos]) - ord('A') + 10) of 'A'..'F': xi = (xi shl 4) or (ord(buf[pos]) - ord('A') + 10)
else: break else: break
inc(pos) inc(pos)
of '0'..'9': of '0'..'9':
xi = 0 xi = 0
while buf[pos] in {'0'..'9'}: while buf[pos] in {'0'..'9'}:
xi = (xi * 10) + (ord(buf[pos]) - ord('0')) xi = (xi * 10) + (ord(buf[pos]) - ord('0'))
inc(pos) inc(pos)
else: lexMessage(L, errInvalidCharacterConstant) else: lexMessage(L, errInvalidCharacterConstant)
if (xi <= 255): add(tok.literal, Chr(xi)) if (xi <= 255): add(tok.literal, Chr(xi))
else: lexMessage(L, errInvalidCharacterConstant) else: lexMessage(L, errInvalidCharacterConstant)
else: else:
break break
tok.xkind = pxStrLit tok.xkind = pxStrLit
L.bufpos = pos L.bufpos = pos
proc getSymbol(L: var TLexer, tok: var TToken) = proc getSymbol(L: var TLexer, tok: var TToken) =
var h: THash = 0 var h: THash = 0
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf var buf = L.buf
while true: while true:
var c = buf[pos] var c = buf[pos]
case c case c
of 'a'..'z', '0'..'9', '\x80'..'\xFF': of 'a'..'z', '0'..'9', '\x80'..'\xFF':
h = h +% Ord(c) h = h +% Ord(c)
h = h +% h shl 10 h = h +% h shl 10
h = h xor (h shr 6) h = h xor (h shr 6)
of 'A'..'Z': of 'A'..'Z':
c = chr(ord(c) + (ord('a') - ord('A'))) # toLower() c = chr(ord(c) + (ord('a') - ord('A'))) # toLower()
h = h +% Ord(c) h = h +% Ord(c)
h = h +% h shl 10 h = h +% h shl 10
h = h xor (h shr 6) h = h xor (h shr 6)
of '_': nil of '_': nil
else: break else: break
Inc(pos) Inc(pos)
h = h +% h shl 3 h = h +% h shl 3
h = h xor (h shr 11) h = h xor (h shr 11)
@ -352,218 +352,218 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
L.bufpos = pos L.bufpos = pos
setKeyword(L, tok) setKeyword(L, tok)
proc scanLineComment(L: var TLexer, tok: var TToken) = proc scanLineComment(L: var TLexer, tok: var TToken) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf var buf = L.buf
# a comment ends if the next line does not start with the // on the same # a comment ends if the next line does not start with the // on the same
# column after only whitespace # column after only whitespace
tok.xkind = pxComment tok.xkind = pxComment
var col = getColNumber(L, pos) var col = getColNumber(L, pos)
while true: while true:
inc(pos, 2) # skip // inc(pos, 2) # skip //
add(tok.literal, '#') add(tok.literal, '#')
while not (buf[pos] in {CR, LF, lexbase.EndOfFile}): while not (buf[pos] in {CR, LF, lexbase.EndOfFile}):
add(tok.literal, buf[pos]) add(tok.literal, buf[pos])
inc(pos) inc(pos)
pos = handleCRLF(L, pos) pos = handleCRLF(L, pos)
buf = L.buf buf = L.buf
var indent = 0 var indent = 0
while buf[pos] == ' ': while buf[pos] == ' ':
inc(pos) inc(pos)
inc(indent) inc(indent)
if (col == indent) and (buf[pos] == '/') and (buf[pos + 1] == '/'): if (col == indent) and (buf[pos] == '/') and (buf[pos + 1] == '/'):
tok.literal = tok.literal & "\n" tok.literal = tok.literal & "\n"
else: else:
break break
L.bufpos = pos L.bufpos = pos
proc scanCurlyComment(L: var TLexer, tok: var TToken) = proc scanCurlyComment(L: var TLexer, tok: var TToken) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf var buf = L.buf
tok.literal = "#" tok.literal = "#"
tok.xkind = pxComment tok.xkind = pxComment
while true: while true:
case buf[pos] case buf[pos]
of CR, LF: of CR, LF:
pos = HandleCRLF(L, pos) pos = HandleCRLF(L, pos)
buf = L.buf buf = L.buf
add(tok.literal, "\n#") add(tok.literal, "\n#")
of '}': of '}':
inc(pos) inc(pos)
break break
of lexbase.EndOfFile: lexMessage(L, errTokenExpected, "}") of lexbase.EndOfFile: lexMessage(L, errTokenExpected, "}")
else: else:
add(tok.literal, buf[pos]) add(tok.literal, buf[pos])
inc(pos) inc(pos)
L.bufpos = pos L.bufpos = pos
proc scanStarComment(L: var TLexer, tok: var TToken) = proc scanStarComment(L: var TLexer, tok: var TToken) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf var buf = L.buf
tok.literal = "#" tok.literal = "#"
tok.xkind = pxComment tok.xkind = pxComment
while true: while true:
case buf[pos] case buf[pos]
of CR, LF: of CR, LF:
pos = HandleCRLF(L, pos) pos = HandleCRLF(L, pos)
buf = L.buf buf = L.buf
add(tok.literal, "\n#") add(tok.literal, "\n#")
of '*': of '*':
inc(pos) inc(pos)
if buf[pos] == ')': if buf[pos] == ')':
inc(pos) inc(pos)
break break
else: else:
add(tok.literal, '*') add(tok.literal, '*')
of lexbase.EndOfFile: of lexbase.EndOfFile:
lexMessage(L, errTokenExpected, "*)") lexMessage(L, errTokenExpected, "*)")
else: else:
add(tok.literal, buf[pos]) add(tok.literal, buf[pos])
inc(pos) inc(pos)
L.bufpos = pos L.bufpos = pos
proc skip(L: var TLexer, tok: var TToken) = proc skip(L: var TLexer, tok: var TToken) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf var buf = L.buf
while true: while true:
case buf[pos] case buf[pos]
of ' ', Tabulator: of ' ', Tabulator:
Inc(pos) # newline is special: Inc(pos) # newline is special:
of CR, LF: of CR, LF:
pos = HandleCRLF(L, pos) pos = HandleCRLF(L, pos)
buf = L.buf buf = L.buf
else: else:
break # EndOfFile also leaves the loop break # EndOfFile also leaves the loop
L.bufpos = pos L.bufpos = pos
proc getTok(L: var TLexer, tok: var TToken) = proc getTok(L: var TLexer, tok: var TToken) =
tok.xkind = pxInvalid tok.xkind = pxInvalid
fillToken(tok) fillToken(tok)
skip(L, tok) skip(L, tok)
var c = L.buf[L.bufpos] var c = L.buf[L.bufpos]
if c in SymStartChars: if c in SymStartChars:
getSymbol(L, tok) getSymbol(L, tok)
elif c in {'0'..'9'}: elif c in {'0'..'9'}:
getNumber10(L, tok) getNumber10(L, tok)
else: else:
case c case c
of ';': of ';':
tok.xkind = pxSemicolon tok.xkind = pxSemicolon
Inc(L.bufpos) Inc(L.bufpos)
of '/': of '/':
if L.buf[L.bufpos + 1] == '/': if L.buf[L.bufpos + 1] == '/':
scanLineComment(L, tok) scanLineComment(L, tok)
else: else:
tok.xkind = pxSlash tok.xkind = pxSlash
inc(L.bufpos) inc(L.bufpos)
of ',': of ',':
tok.xkind = pxComma tok.xkind = pxComma
Inc(L.bufpos) Inc(L.bufpos)
of '(': of '(':
Inc(L.bufpos) Inc(L.bufpos)
if (L.buf[L.bufPos] == '*'): if (L.buf[L.bufPos] == '*'):
if (L.buf[L.bufPos + 1] == '$'): if (L.buf[L.bufPos + 1] == '$'):
Inc(L.bufpos, 2) Inc(L.bufpos, 2)
skip(L, tok) skip(L, tok)
getSymbol(L, tok) getSymbol(L, tok)
tok.xkind = pxStarDirLe tok.xkind = pxStarDirLe
else: else:
inc(L.bufpos) inc(L.bufpos)
scanStarComment(L, tok) scanStarComment(L, tok)
else: else:
tok.xkind = pxParLe tok.xkind = pxParLe
of '*': of '*':
inc(L.bufpos) inc(L.bufpos)
if L.buf[L.bufpos] == ')': if L.buf[L.bufpos] == ')':
inc(L.bufpos) inc(L.bufpos)
tok.xkind = pxStarDirRi tok.xkind = pxStarDirRi
else: else:
tok.xkind = pxStar tok.xkind = pxStar
of ')': of ')':
tok.xkind = pxParRi tok.xkind = pxParRi
Inc(L.bufpos) Inc(L.bufpos)
of '[': of '[':
Inc(L.bufpos) Inc(L.bufpos)
tok.xkind = pxBracketLe tok.xkind = pxBracketLe
of ']': of ']':
Inc(L.bufpos) Inc(L.bufpos)
tok.xkind = pxBracketRi tok.xkind = pxBracketRi
of '.': of '.':
inc(L.bufpos) inc(L.bufpos)
if L.buf[L.bufpos] == '.': if L.buf[L.bufpos] == '.':
tok.xkind = pxDotDot tok.xkind = pxDotDot
inc(L.bufpos) inc(L.bufpos)
else: else:
tok.xkind = pxDot tok.xkind = pxDot
of '{': of '{':
Inc(L.bufpos) Inc(L.bufpos)
case L.buf[L.bufpos] case L.buf[L.bufpos]
of '$': of '$':
Inc(L.bufpos) Inc(L.bufpos)
skip(L, tok) skip(L, tok)
getSymbol(L, tok) getSymbol(L, tok)
tok.xkind = pxCurlyDirLe tok.xkind = pxCurlyDirLe
of '&': of '&':
Inc(L.bufpos) Inc(L.bufpos)
tok.xkind = pxAmp tok.xkind = pxAmp
of '%': of '%':
Inc(L.bufpos) Inc(L.bufpos)
tok.xkind = pxPer tok.xkind = pxPer
of '@': of '@':
Inc(L.bufpos) Inc(L.bufpos)
tok.xkind = pxCommand tok.xkind = pxCommand
else: scanCurlyComment(L, tok) else: scanCurlyComment(L, tok)
of '+': of '+':
tok.xkind = pxPlus tok.xkind = pxPlus
inc(L.bufpos) inc(L.bufpos)
of '-': of '-':
tok.xkind = pxMinus tok.xkind = pxMinus
inc(L.bufpos) inc(L.bufpos)
of ':': of ':':
inc(L.bufpos) inc(L.bufpos)
if L.buf[L.bufpos] == '=': if L.buf[L.bufpos] == '=':
inc(L.bufpos) inc(L.bufpos)
tok.xkind = pxAsgn tok.xkind = pxAsgn
else: else:
tok.xkind = pxColon tok.xkind = pxColon
of '<': of '<':
inc(L.bufpos) inc(L.bufpos)
if L.buf[L.bufpos] == '>': if L.buf[L.bufpos] == '>':
inc(L.bufpos) inc(L.bufpos)
tok.xkind = pxNeq tok.xkind = pxNeq
elif L.buf[L.bufpos] == '=': elif L.buf[L.bufpos] == '=':
inc(L.bufpos) inc(L.bufpos)
tok.xkind = pxLe tok.xkind = pxLe
else: else:
tok.xkind = pxLt tok.xkind = pxLt
of '>': of '>':
inc(L.bufpos) inc(L.bufpos)
if L.buf[L.bufpos] == '=': if L.buf[L.bufpos] == '=':
inc(L.bufpos) inc(L.bufpos)
tok.xkind = pxGe tok.xkind = pxGe
else: else:
tok.xkind = pxGt tok.xkind = pxGt
of '=': of '=':
tok.xkind = pxEquals tok.xkind = pxEquals
inc(L.bufpos) inc(L.bufpos)
of '@': of '@':
tok.xkind = pxAt tok.xkind = pxAt
inc(L.bufpos) inc(L.bufpos)
of '^': of '^':
tok.xkind = pxHat tok.xkind = pxHat
inc(L.bufpos) inc(L.bufpos)
of '}': of '}':
tok.xkind = pxCurlyDirRi tok.xkind = pxCurlyDirRi
Inc(L.bufpos) Inc(L.bufpos)
of '\'', '#': of '\'', '#':
getString(L, tok) getString(L, tok)
of '$': of '$':
getNumber16(L, tok) getNumber16(L, tok)
of '%': of '%':
getNumber2(L, tok) getNumber2(L, tok)
of lexbase.EndOfFile: of lexbase.EndOfFile:
tok.xkind = pxEof tok.xkind = pxEof
else: else:
tok.literal = c & "" tok.literal = c & ""
tok.xkind = pxInvalid tok.xkind = pxInvalid
lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')') lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')')