jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix #2523 - first commit. Needs some cleanup.

Browse source
This commit is contained in:
Oscar Campbell 2015-05-31 00:56:27 +02:00
commit 49b953533a
1 changed files with 204 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

302
compiler/lexer.nim
View file

@ -240,12 +240,43 @@ proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
break break
if buf[pos] == '_': if buf[pos] == '_':
if buf[pos+1] notin chars: if buf[pos+1] notin chars:
lexMessage(L, errInvalidToken, "_") lexMessage(L, errInvalidToken, "_") # TODO/ozra - lift out for better msg
break break
add(tok.literal, '_') add(tok.literal, '_')
inc(pos) inc(pos)
L.bufpos = pos L.bufpos = pos
# Used for getting human friendlier err messages. Also only used in getNumber
proc matchAllLiteralishForMessage(L: var TLexer, tok: var TToken, startpos: int) =
# Note: <the erroneous 'O' in the set is intentional
const literalishChars = { 'A'..'F', 'a'..'f', '0'..'9',
'X', 'x', 'o', 'O', 'c', 'C', 'b', 'B',
'_', '.', '\'', 'd', 'i', 'u'
}
tok.literal = "" # Start over fresh for the "message literal"
var msgPos = L.bufpos
L.bufpos = startpos # Use L.bufpos as positioner because of matchUnderscore...
# We use matchUnderscoreChars - but abuse it a little, passing '_'
matchUnderscoreChars(L, tok, literalishChars)
# We must verify +/- specifically so that we're not past the literal
if L.buf[L.bufpos] in {'+', '-'} and
L.buf[L.bufpos - 1] in {'e', 'E'}:
add(tok.literal, L.buf[L.bufpos])
inc(L.bufpos)
matchUnderscoreChars(L, tok, literalishChars)
if L.buf[L.bufpos] in {
'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'
}:
inc(L.bufpos)
add(tok.literal, L.buf[L.bufpos])
matchUnderscoreChars(L, tok, {'0'..'9'})
L.bufpos = msgPos
proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool = proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second) result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
@ -268,136 +299,203 @@ proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int =
result = i - start result = i - start
{.pop.} # overflowChecks {.pop.} # overflowChecks
template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
add(t.literal, replacementChar)
inc(L.bufpos)
template eatChar(L: var TLexer, t: var TToken) =
add(t.literal, L.buf[L.bufpos])
inc(L.bufpos)
# ###### ######## ######## ## ## ## ## ## ##
# ## ## ## ## ### ## ## ## ### ###
# ## ## ## #### ## ## ## #### ####
# ## #### ###### ## ## ## ## ## ## ## ### ##
# ## ## ## ## ## #### ## ## ## ##
# ## ## ## ## ## ### ## ## ## ##
# ###### ######## ## ## ## ####### ## ##
proc getNumber(L: var TLexer): TToken = proc getNumber(L: var TLexer): TToken =
var var
pos, endpos: int startpos, endpos: int
xi: BiggestInt xi: BiggestInt
const literalishChars = { 'A'..'F', 'a'..'f', '0'..'9',
'X', 'x', 'o', 'c', 'C', 'b', 'B',
'_', '.', '\''
}
const literalishCharsNoDot = { 'A'..'F', 'a'..'f', '0'..'9',
'X', 'x', 'o', 'c', 'C', 'b', 'B',
'_', '\''
}
proc lexLiteralNumberMessage(L: var TLexer, msg: TMsgKind, startpos: int) =
var t: TToken
matchAllLiteralishForMessage(L, t, startpos)
lexMessage(L, msg, t.literal)
# get the base: # get the base:
result.tokType = tkIntLit # int literal until we know better result.tokType = tkIntLit # int literal until we know better
result.literal = "" result.literal = ""
result.base = base10 # BUGFIX result.base = base10 # BUGFIX
pos = L.bufpos # make sure the literal is correct for error messages:
var eallowed = false startpos = L.bufpos # make sure the literal is correct for error messages:
if L.buf[pos] == '0' and L.buf[pos+1] in {'X', 'x'}:
matchUnderscoreChars(L, result, {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x'}) var isAFloatLiteral = false
# This first pass makes verifications and builds the literal string for
# the token, to keep the value extraction phase less error prone..
if L.buf[L.bufpos] == '0' and
L.buf[L.bufpos + 1] in {'X', 'x', 'o', 'O', 'c', 'C', 'b', 'B'}:
eatChar(L, result, '0')
case L.buf[L.bufpos]
of 'O':
# TODO/ozra - add nicer message: " (did you mean octal? Then use one of '0o', '0c' or '0C'.)"
lexLiteralNumberMessage(L, errInvalidNumber, startpos)
of 'x', 'X':
eatChar(L, result, 'x')
#echo "is hex: ", result.literal
matchUnderscoreChars(L, result, {'0'..'9', 'a'..'f', 'A'..'F'})
#echo "got hex: ", result.literal
of 'o', 'c', 'C':
eatChar(L, result, 'c')
#echo "is octal: ", result.literal
matchUnderscoreChars(L, result, {'0'..'7'})
of 'b', 'B':
eatChar(L, result, 'b')
#echo "is binary: ", result.literal
matchUnderscoreChars(L, result, {'0'..'1'})
else:
internalError(getLineInfo(L), "getNumber")
else: else:
matchUnderscoreChars(L, result, {'0'..'9', 'b', 'B', 'o', 'c', 'C'}) #echo "is decimal"
eallowed = true
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
add(result.literal, '.')
inc(L.bufpos)
matchUnderscoreChars(L, result, {'0'..'9'})
eallowed = true
if eallowed and L.buf[L.bufpos] in {'e', 'E'}:
add(result.literal, 'e')
inc(L.bufpos)
if L.buf[L.bufpos] in {'+', '-'}:
add(result.literal, L.buf[L.bufpos])
inc(L.bufpos)
matchUnderscoreChars(L, result, {'0'..'9'}) matchUnderscoreChars(L, result, {'0'..'9'})
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
isAFloatLiteral = true
eatChar(L, result, '.')
matchUnderscoreChars(L, result, {'0'..'9'})
if L.buf[L.bufpos] in {'e', 'E'}:
isAFloatLiteral = true
eatChar(L, result, 'e')
if L.buf[L.bufpos] in {'+', '-'}:
eatChar(L, result)
matchUnderscoreChars(L, result, {'0'..'9'})
endpos = L.bufpos endpos = L.bufpos
if L.buf[endpos] in {'\'', 'f', 'F', 'i', 'I', 'u', 'U'}: var postPos = endpos
if L.buf[endpos] == '\'': inc(endpos)
L.bufpos = pos # restore position #echo "At endpos is: ", L.buf[postPos]
case L.buf[endpos]
if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
if L.buf[postPos] == '\'':
inc(postPos)
#echo "is postfix: ", L.buf[postPos]
case L.buf[postPos]
of 'f', 'F': of 'f', 'F':
inc(endpos) inc(postPos)
if (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'): if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
result.tokType = tkFloat32Lit result.tokType = tkFloat32Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'): elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
result.tokType = tkFloat64Lit result.tokType = tkFloat64Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '1') and elif (L.buf[postPos] == '1') and
(L.buf[endpos + 1] == '2') and (L.buf[postPos + 1] == '2') and
(L.buf[endpos + 2] == '8'): (L.buf[postPos + 2] == '8'):
result.tokType = tkFloat128Lit result.tokType = tkFloat128Lit
inc(endpos, 3) inc(postPos, 3)
else: else: # "f" alone defaults to float32
lexMessage(L, errInvalidNumber, result.literal & "'f" & L.buf[endpos]) result.tokType = tkFloat32Lit
#lexMessage(L, errInvalidNumber, result.literal & "'f" & L.buf[postPos])
# 'd' is an ad hoc convenience shortcut for f64
of 'd', 'D':
inc(postPos)
result.tokType = tkFloat64Lit
of 'i', 'I': of 'i', 'I':
inc(endpos) inc(postPos)
if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'): if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
result.tokType = tkInt64Lit result.tokType = tkInt64Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'): elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
result.tokType = tkInt32Lit result.tokType = tkInt32Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '1') and (L.buf[endpos + 1] == '6'): elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
result.tokType = tkInt16Lit result.tokType = tkInt16Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '8'): elif (L.buf[postPos] == '8'):
result.tokType = tkInt8Lit result.tokType = tkInt8Lit
inc(endpos) inc(postPos)
else: else:
lexMessage(L, errInvalidNumber, result.literal & "'i" & L.buf[endpos]) lexLiteralNumberMessage(L, errInvalidNumber, startpos)
#lexMessage(L, errInvalidNumber, result.literal & "'i" & L.buf[postPos])
#echo "is intish: ", result.literal, ":", $ result.tokType
of 'u', 'U': of 'u', 'U':
inc(endpos) inc(postPos)
if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'): if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
result.tokType = tkUInt64Lit result.tokType = tkUInt64Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'): elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
result.tokType = tkUInt32Lit result.tokType = tkUInt32Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '1') and (L.buf[endpos + 1] == '6'): elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
result.tokType = tkUInt16Lit result.tokType = tkUInt16Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '8'): elif (L.buf[postPos] == '8'):
result.tokType = tkUInt8Lit result.tokType = tkUInt8Lit
inc(endpos) inc(postPos)
else: else:
result.tokType = tkUIntLit result.tokType = tkUIntLit
else: lexMessage(L, errInvalidNumber, result.literal & "'" & L.buf[endpos]) else:
else: lexLiteralNumberMessage(L, errInvalidNumber, startpos)
L.bufpos = pos # restore position #lexMessage(L, errInvalidNumber, result.literal & "'" & L.buf[postPos])
# Still a literalish char? Error!
if L.buf[postPos] in literalishCharsNoDot or
(L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
echo "likely literal error, additionally: '", L.buf[postPos], L.buf[postPos+1], L.buf[postPos+2], L.buf[postPos+3], "', ", postPos
lexLiteralNumberMessage(L, errInvalidNumber, startpos)
L.bufpos = startpos # restore position
var pos: int = startpos
try: try:
if (L.buf[pos] == '0') and if (L.buf[pos] == '0') and
(L.buf[pos + 1] in {'x', 'X', 'b', 'B', 'o', 'O', 'c', 'C'}): (L.buf[pos + 1] in {'x', 'X', 'b', 'B', 'o', 'O', 'c', 'C'}):
inc(pos, 2) inc(pos, 2)
xi = 0 # it may be a base prefix xi = 0 # it is a base prefix
case L.buf[pos - 1] # now look at the optional type suffix: case L.buf[pos - 1] # now look at the optional type suffix:
of 'b', 'B': of 'b', 'B':
result.base = base2 result.base = base2
while true: while pos < endpos:
case L.buf[pos] if L.buf[pos] != '_':
of '2'..'9', '.':
lexMessage(L, errInvalidNumber, result.literal)
inc(pos)
of '_':
if L.buf[pos+1] notin {'0'..'1'}:
lexMessage(L, errInvalidToken, "_")
break
inc(pos)
of '0', '1':
xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0')) xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
inc(pos) inc(pos)
else: break
of 'o', 'c', 'C': of 'o', 'c', 'C':
result.base = base8 result.base = base8
while true: while pos < endpos:
case L.buf[pos] if L.buf[pos] != '_':
of '8'..'9', '.':
lexMessage(L, errInvalidNumber, result.literal)
inc(pos)
of '_':
if L.buf[pos+1] notin {'0'..'7'}:
lexMessage(L, errInvalidToken, "_")
break
inc(pos)
of '0'..'7':
xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0')) xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
inc(pos) inc(pos)
else: break
of 'O':
lexMessage(L, errInvalidNumber, result.literal)
of 'x', 'X': of 'x', 'X':
result.base = base16 result.base = base16
while true: while pos < endpos:
case L.buf[pos] case L.buf[pos]
of '_': of '_':
if L.buf[pos+1] notin {'0'..'9', 'a'..'f', 'A'..'F'}:
lexMessage(L, errInvalidToken, "_")
break
inc(pos) inc(pos)
of '0'..'9': of '0'..'9':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0')) xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
@ -408,8 +506,11 @@ proc getNumber(L: var TLexer): TToken =
of 'A'..'F': of 'A'..'F':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10) xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
inc(pos) inc(pos)
else: break else:
else: internalError(getLineInfo(L), "getNumber") break
else:
internalError(getLineInfo(L), "getNumber")
case result.tokType case result.tokType
of tkIntLit, tkInt64Lit: result.iNumber = xi of tkIntLit, tkInt64Lit: result.iNumber = xi
of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi)))) of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
@ -425,34 +526,39 @@ proc getNumber(L: var TLexer): TToken =
# XXX: Test this on big endian machine! # XXX: Test this on big endian machine!
of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[] of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[]
else: internalError(getLineInfo(L), "getNumber") else: internalError(getLineInfo(L), "getNumber")
elif isFloatLiteral(result.literal) or (result.tokType == tkFloat32Lit) or
elif isAFloatLiteral or (result.tokType == tkFloat32Lit) or
(result.tokType == tkFloat64Lit): (result.tokType == tkFloat64Lit):
result.fNumber = parseFloat(result.literal) result.fNumber = parseFloat(result.literal)
if result.tokType == tkIntLit: result.tokType = tkFloatLit if result.tokType == tkIntLit: result.tokType = tkFloatLit
elif result.tokType == tkUint64Lit: elif result.tokType == tkUint64Lit:
xi = 0 xi = 0
let len = unsafeParseUInt(result.literal, xi) let len = unsafeParseUInt(result.literal, xi)
if len != result.literal.len or len == 0: if len != result.literal.len or len == 0:
raise newException(ValueError, "invalid integer: " & $xi) raise newException(ValueError, "invalid integer: " & $xi)
result.iNumber = xi result.iNumber = xi
else: else:
result.iNumber = parseBiggestInt(result.literal) result.iNumber = parseBiggestInt(result.literal)
if (result.iNumber < low(int32)) or (result.iNumber > high(int32)): if (result.iNumber < low(int32)) or (result.iNumber > high(int32)):
if result.tokType == tkIntLit: if result.tokType == tkIntLit:
result.tokType = tkInt64Lit result.tokType = tkInt64Lit
elif result.tokType in {tkInt8Lit, tkInt16Lit, tkInt32Lit}: elif result.tokType in {tkInt8Lit, tkInt16Lit, tkInt32Lit}:
lexMessage(L, errNumberOutOfRange, result.literal) lexLiteralNumberMessage(L, errNumberOutOfRange, startpos)
elif result.tokType == tkInt8Lit and elif result.tokType == tkInt8Lit and
(result.iNumber < int8.low or result.iNumber > int8.high): (result.iNumber < int8.low or result.iNumber > int8.high):
lexMessage(L, errNumberOutOfRange, result.literal) lexLiteralNumberMessage(L, errNumberOutOfRange, startpos)
elif result.tokType == tkInt16Lit and elif result.tokType == tkInt16Lit and
(result.iNumber < int16.low or result.iNumber > int16.high): (result.iNumber < int16.low or result.iNumber > int16.high):
lexMessage(L, errNumberOutOfRange, result.literal) lexLiteralNumberMessage(L, errNumberOutOfRange, startpos)
except ValueError: except ValueError:
lexMessage(L, errInvalidNumber, result.literal) lexLiteralNumberMessage(L, errInvalidNumber, startpos)
except OverflowError, RangeError: except OverflowError, RangeError:
lexMessage(L, errNumberOutOfRange, result.literal) lexLiteralNumberMessage(L, errNumberOutOfRange, startpos)
L.bufpos = endpos
L.bufpos = postPos
proc handleHexChar(L: var TLexer, xi: var int) = proc handleHexChar(L: var TLexer, xi: var int) =
case L.buf[L.bufpos] case L.buf[L.bufpos]