jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #2848 from ozra/bugfix-2523-number-literal-lexing

Browse source 
Bugfix #2523 number literal lexing
This commit is contained in:
Andreas Rumpf 2015-06-04 11:40:00 +02:00
commit 51488ebd12
2 changed files with 164 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

276
compiler/lexer.nim
View file

@ -231,23 +231,6 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart) var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart)
L.dispMessage(info, msg, arg) L.dispMessage(info, msg, arg)
proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
var pos = L.bufpos # use registers for pos, buf
var buf = L.buf
while true:
if buf[pos] in chars:
add(tok.literal, buf[pos])
inc(pos)
else:
break
if buf[pos] == '_':
if buf[pos+1] notin chars:
lexMessage(L, errInvalidToken, "_")
break
add(tok.literal, '_')
inc(pos)
L.bufpos = pos
proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool = proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second) result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
@ -270,136 +253,195 @@ proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int =
result = i - start result = i - start
{.pop.} # overflowChecks {.pop.} # overflowChecks
template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
add(t.literal, replacementChar)
inc(L.bufpos)
template eatChar(L: var TLexer, t: var TToken) =
add(t.literal, L.buf[L.bufpos])
inc(L.bufpos)
proc getNumber(L: var TLexer): TToken = proc getNumber(L: var TLexer): TToken =
var var
pos, endpos: int startpos, endpos: int
xi: BiggestInt xi: BiggestInt
# get the base: const literalishChars = { 'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'c',
'C', 'b', 'B', '_', '.', '\''}
const literalishCharsNoDot = literalishChars - {'.'}
proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
var pos = L.bufpos # use registers for pos, buf
var buf = L.buf
while true:
if buf[pos] in chars:
add(tok.literal, buf[pos])
inc(pos)
else:
break
if buf[pos] == '_':
if buf[pos+1] notin chars:
lexMessage(L, errInvalidToken, "_")
break
add(tok.literal, '_')
inc(pos)
L.bufpos = pos
proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
var pos = L.bufpos # use registers for pos, buf
var buf = L.buf
while buf[pos] in chars:
add(tok.literal, buf[pos])
inc(pos)
L.bufpos = pos
proc lexMessageLitNum(L: var TLexer, msg: TMsgKind, startpos: int) =
# Used to get slightly human friendlier err messages.
# Note: the erroneous 'O' char in the character set is intentional
const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
var msgPos = L.bufpos
var t: TToken
t.literal = ""
L.bufpos = startpos # Use L.bufpos as pos because of matchChars
matchChars(L, t, literalishChars)
# We must verify +/- specifically so that we're not past the literal
if L.buf[L.bufpos] in {'+', '-'} and
L.buf[L.bufpos - 1] in {'e', 'E'}:
add(t.literal, L.buf[L.bufpos])
inc(L.bufpos)
matchChars(L, t, literalishChars)
if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
inc(L.bufpos)
add(t.literal, L.buf[L.bufpos])
matchChars(L, t, {'0'..'9'})
L.bufpos = msgPos
lexMessage(L, msg, t.literal)
result.tokType = tkIntLit # int literal until we know better result.tokType = tkIntLit # int literal until we know better
result.literal = "" result.literal = ""
result.base = base10 # BUGFIX result.base = base10
pos = L.bufpos # make sure the literal is correct for error messages: startpos = L.bufpos
var eallowed = false var isAFloatLiteral = false
if L.buf[pos] == '0' and L.buf[pos+1] in {'X', 'x'}: # First stage: find out base, make verifications, build token literal string
matchUnderscoreChars(L, result, {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x'}) if L.buf[L.bufpos] == '0' and
L.buf[L.bufpos + 1] in {'X', 'x', 'o', 'O', 'c', 'C', 'b', 'B'}:
eatChar(L, result, '0')
case L.buf[L.bufpos]
of 'O':
lexMessageLitNum(L, errInvalidNumberOctalCode, startpos)
of 'x', 'X':
eatChar(L, result, 'x')
matchUnderscoreChars(L, result, {'0'..'9', 'a'..'f', 'A'..'F'})
of 'o', 'c', 'C':
eatChar(L, result, 'c')
matchUnderscoreChars(L, result, {'0'..'7'})
of 'b', 'B':
eatChar(L, result, 'b')
matchUnderscoreChars(L, result, {'0'..'1'})
else:
internalError(getLineInfo(L), "getNumber")
else: else:
matchUnderscoreChars(L, result, {'0'..'9', 'b', 'B', 'o', 'c', 'C'})
eallowed = true
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
add(result.literal, '.')
inc(L.bufpos)
matchUnderscoreChars(L, result, {'0'..'9'})
eallowed = true
if eallowed and L.buf[L.bufpos] in {'e', 'E'}:
add(result.literal, 'e')
inc(L.bufpos)
if L.buf[L.bufpos] in {'+', '-'}:
add(result.literal, L.buf[L.bufpos])
inc(L.bufpos)
matchUnderscoreChars(L, result, {'0'..'9'}) matchUnderscoreChars(L, result, {'0'..'9'})
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
isAFloatLiteral = true
eatChar(L, result, '.')
matchUnderscoreChars(L, result, {'0'..'9'})
if L.buf[L.bufpos] in {'e', 'E'}:
isAFloatLiteral = true
eatChar(L, result, 'e')
if L.buf[L.bufpos] in {'+', '-'}:
eatChar(L, result)
matchUnderscoreChars(L, result, {'0'..'9'})
endpos = L.bufpos endpos = L.bufpos
if L.buf[endpos] in {'\'', 'f', 'F', 'i', 'I', 'u', 'U'}: # Second stage, find out if there's a datatype postfix and handle it
if L.buf[endpos] == '\'': inc(endpos) var postPos = endpos
L.bufpos = pos # restore position if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
case L.buf[endpos] if L.buf[postPos] == '\'':
inc(postPos)
case L.buf[postPos]
of 'f', 'F': of 'f', 'F':
inc(endpos) inc(postPos)
if (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'): if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
result.tokType = tkFloat32Lit result.tokType = tkFloat32Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'): elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
result.tokType = tkFloat64Lit result.tokType = tkFloat64Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '1') and elif (L.buf[postPos] == '1') and
(L.buf[endpos + 1] == '2') and (L.buf[postPos + 1] == '2') and
(L.buf[endpos + 2] == '8'): (L.buf[postPos + 2] == '8'):
result.tokType = tkFloat128Lit result.tokType = tkFloat128Lit
inc(endpos, 3) inc(postPos, 3)
else: else: # "f" alone defaults to float32
lexMessage(L, errInvalidNumber, result.literal & "'f" & L.buf[endpos]) result.tokType = tkFloat32Lit
of 'd', 'D': # ad hoc convenience shortcut for f64
inc(postPos)
result.tokType = tkFloat64Lit
of 'i', 'I': of 'i', 'I':
inc(endpos) inc(postPos)
if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'): if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
result.tokType = tkInt64Lit result.tokType = tkInt64Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'): elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
result.tokType = tkInt32Lit result.tokType = tkInt32Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '1') and (L.buf[endpos + 1] == '6'): elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
result.tokType = tkInt16Lit result.tokType = tkInt16Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '8'): elif (L.buf[postPos] == '8'):
result.tokType = tkInt8Lit result.tokType = tkInt8Lit
inc(endpos) inc(postPos)
else: else:
lexMessage(L, errInvalidNumber, result.literal & "'i" & L.buf[endpos]) lexMessageLitNum(L, errInvalidNumber, startpos)
of 'u', 'U': of 'u', 'U':
inc(endpos) inc(postPos)
if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'): if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
result.tokType = tkUInt64Lit result.tokType = tkUInt64Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'): elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
result.tokType = tkUInt32Lit result.tokType = tkUInt32Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '1') and (L.buf[endpos + 1] == '6'): elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
result.tokType = tkUInt16Lit result.tokType = tkUInt16Lit
inc(endpos, 2) inc(postPos, 2)
elif (L.buf[endpos] == '8'): elif (L.buf[postPos] == '8'):
result.tokType = tkUInt8Lit result.tokType = tkUInt8Lit
inc(endpos) inc(postPos)
else: else:
result.tokType = tkUIntLit result.tokType = tkUIntLit
else: lexMessage(L, errInvalidNumber, result.literal & "'" & L.buf[endpos]) else:
else: lexMessageLitNum(L, errInvalidNumber, startpos)
L.bufpos = pos # restore position # Is there still a literalish char awaiting? Then it's an error!
if L.buf[postPos] in literalishCharsNoDot or
(L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
lexMessageLitNum(L, errInvalidNumber, startpos)
# Third stage, extract actual number
L.bufpos = startpos # restore position
var pos: int = startpos
try: try:
if (L.buf[pos] == '0') and if (L.buf[pos] == '0') and
(L.buf[pos + 1] in {'x', 'X', 'b', 'B', 'o', 'O', 'c', 'C'}): (L.buf[pos + 1] in {'x', 'X', 'b', 'B', 'o', 'O', 'c', 'C'}):
inc(pos, 2) inc(pos, 2)
xi = 0 # it may be a base prefix xi = 0 # it is a base prefix
case L.buf[pos - 1] # now look at the optional type suffix: case L.buf[pos - 1] # now look at the optional type suffix:
of 'b', 'B': of 'b', 'B':
result.base = base2 result.base = base2
while true: while pos < endpos:
case L.buf[pos] if L.buf[pos] != '_':
of '2'..'9', '.':
lexMessage(L, errInvalidNumber, result.literal)
inc(pos)
of '_':
if L.buf[pos+1] notin {'0'..'1'}:
lexMessage(L, errInvalidToken, "_")
break
inc(pos)
of '0', '1':
xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0')) xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
inc(pos) inc(pos)
else: break
of 'o', 'c', 'C': of 'o', 'c', 'C':
result.base = base8 result.base = base8
while true: while pos < endpos:
case L.buf[pos] if L.buf[pos] != '_':
of '8'..'9', '.':
lexMessage(L, errInvalidNumber, result.literal)
inc(pos)
of '_':
if L.buf[pos+1] notin {'0'..'7'}:
lexMessage(L, errInvalidToken, "_")
break
inc(pos)
of '0'..'7':
xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0')) xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
inc(pos) inc(pos)
else: break
of 'O':
lexMessage(L, errInvalidNumber, result.literal)
of 'x', 'X': of 'x', 'X':
result.base = base16 result.base = base16
while true: while pos < endpos:
case L.buf[pos] case L.buf[pos]
of '_': of '_':
if L.buf[pos+1] notin {'0'..'9', 'a'..'f', 'A'..'F'}:
lexMessage(L, errInvalidToken, "_")
break
inc(pos) inc(pos)
of '0'..'9': of '0'..'9':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0')) xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
@ -410,8 +452,10 @@ proc getNumber(L: var TLexer): TToken =
of 'A'..'F': of 'A'..'F':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10) xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
inc(pos) inc(pos)
else: break else:
else: internalError(getLineInfo(L), "getNumber") break
else:
internalError(getLineInfo(L), "getNumber")
case result.tokType case result.tokType
of tkIntLit, tkInt64Lit: result.iNumber = xi of tkIntLit, tkInt64Lit: result.iNumber = xi
of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi)))) of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
@ -427,7 +471,7 @@ proc getNumber(L: var TLexer): TToken =
# XXX: Test this on big endian machine! # XXX: Test this on big endian machine!
of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[] of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[]
else: internalError(getLineInfo(L), "getNumber") else: internalError(getLineInfo(L), "getNumber")
elif isFloatLiteral(result.literal) or (result.tokType == tkFloat32Lit) or elif isAFloatLiteral or (result.tokType == tkFloat32Lit) or
(result.tokType == tkFloat64Lit): (result.tokType == tkFloat64Lit):
result.fNumber = parseFloat(result.literal) result.fNumber = parseFloat(result.literal)
if result.tokType == tkIntLit: result.tokType = tkFloatLit if result.tokType == tkIntLit: result.tokType = tkFloatLit
@ -443,18 +487,18 @@ proc getNumber(L: var TLexer): TToken =
if result.tokType == tkIntLit: if result.tokType == tkIntLit:
result.tokType = tkInt64Lit result.tokType = tkInt64Lit
elif result.tokType in {tkInt8Lit, tkInt16Lit, tkInt32Lit}: elif result.tokType in {tkInt8Lit, tkInt16Lit, tkInt32Lit}:
lexMessage(L, errNumberOutOfRange, result.literal) lexMessageLitNum(L, errNumberOutOfRange, startpos)
elif result.tokType == tkInt8Lit and elif result.tokType == tkInt8Lit and
(result.iNumber < int8.low or result.iNumber > int8.high): (result.iNumber < int8.low or result.iNumber > int8.high):
lexMessage(L, errNumberOutOfRange, result.literal) lexMessageLitNum(L, errNumberOutOfRange, startpos)
elif result.tokType == tkInt16Lit and elif result.tokType == tkInt16Lit and
(result.iNumber < int16.low or result.iNumber > int16.high): (result.iNumber < int16.low or result.iNumber > int16.high):
lexMessage(L, errNumberOutOfRange, result.literal) lexMessageLitNum(L, errNumberOutOfRange, startpos)
except ValueError: except ValueError:
lexMessage(L, errInvalidNumber, result.literal) lexMessageLitNum(L, errInvalidNumber, startpos)
except OverflowError, RangeError: except OverflowError, RangeError:
lexMessage(L, errNumberOutOfRange, result.literal) lexMessageLitNum(L, errNumberOutOfRange, startpos)
L.bufpos = endpos L.bufpos = postPos
proc handleHexChar(L: var TLexer, xi: var int) = proc handleHexChar(L: var TLexer, xi: var int) =
case L.buf[L.bufpos] case L.buf[L.bufpos]

8
compiler/msgs.nim
View file

@ -17,10 +17,9 @@ type
errIntLiteralExpected, errInvalidCharacterConstant, errIntLiteralExpected, errInvalidCharacterConstant,
errClosingTripleQuoteExpected, errClosingQuoteExpected, errClosingTripleQuoteExpected, errClosingQuoteExpected,
errTabulatorsAreNotAllowed, errInvalidToken, errLineTooLong, errTabulatorsAreNotAllowed, errInvalidToken, errLineTooLong,
errInvalidNumber, errNumberOutOfRange, errNnotAllowedInCharacter, errInvalidNumber, errInvalidNumberOctalCode, errNumberOutOfRange,
errClosingBracketExpected, errMissingFinalQuote, errIdentifierExpected, errNnotAllowedInCharacter, errClosingBracketExpected, errMissingFinalQuote,
errNewlineExpected, errIdentifierExpected, errNewlineExpected, errInvalidModuleName,
errInvalidModuleName,
errOperatorExpected, errTokenExpected, errStringAfterIncludeExpected, errOperatorExpected, errTokenExpected, errStringAfterIncludeExpected,
errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected, errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected,
errInvalidPragma, errUnknownPragma, errInvalidDirectiveX, errInvalidPragma, errUnknownPragma, errInvalidDirectiveX,
@ -143,6 +142,7 @@ const
errInvalidToken: "invalid token: $1", errInvalidToken: "invalid token: $1",
errLineTooLong: "line too long", errLineTooLong: "line too long",
errInvalidNumber: "$1 is not a valid number", errInvalidNumber: "$1 is not a valid number",
errInvalidNumberOctalCode: "$1 is not a valid number; did you mean octal? Then use one of '0o', '0c' or '0C'.",
errNumberOutOfRange: "number $1 out of valid range", errNumberOutOfRange: "number $1 out of valid range",
errNnotAllowedInCharacter: "\\n not allowed in character literal", errNnotAllowedInCharacter: "\\n not allowed in character literal",
errClosingBracketExpected: "closing ']' expected, but end of file reached", errClosingBracketExpected: "closing ']' expected, but end of file reached",