jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Clean up to compiler style. Refine error-msg for illegal octal 'O'

Browse source
This commit is contained in:
Oscar Campbell 2015-05-31 02:55:06 +02:00
commit 6a43b0e817
2 changed files with 68 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

188
compiler/lexer.nim
View file

@ -229,54 +229,6 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart) var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart)
L.dispMessage(info, msg, arg) L.dispMessage(info, msg, arg)
proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
var pos = L.bufpos # use registers for pos, buf
var buf = L.buf
while true:
if buf[pos] in chars:
add(tok.literal, buf[pos])
inc(pos)
else:
break
if buf[pos] == '_':
if buf[pos+1] notin chars:
lexMessage(L, errInvalidToken, "_") # TODO/ozra - lift out for better msg
break
add(tok.literal, '_')
inc(pos)
L.bufpos = pos
# Used for getting human friendlier err messages. Also only used in getNumber
proc matchAllLiteralishForMessage(L: var TLexer, tok: var TToken, startpos: int) =
# Note: <the erroneous 'O' in the set is intentional
const literalishChars = { 'A'..'F', 'a'..'f', '0'..'9',
'X', 'x', 'o', 'O', 'c', 'C', 'b', 'B',
'_', '.', '\'', 'd', 'i', 'u'
}
tok.literal = "" # Start over fresh for the "message literal"
var msgPos = L.bufpos
L.bufpos = startpos # Use L.bufpos as positioner because of matchUnderscore...
# We use matchUnderscoreChars - but abuse it a little, passing '_'
matchUnderscoreChars(L, tok, literalishChars)
# We must verify +/- specifically so that we're not past the literal
if L.buf[L.bufpos] in {'+', '-'} and
L.buf[L.bufpos - 1] in {'e', 'E'}:
add(tok.literal, L.buf[L.bufpos])
inc(L.bufpos)
matchUnderscoreChars(L, tok, literalishChars)
if L.buf[L.bufpos] in {
'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'
}:
inc(L.bufpos)
add(tok.literal, L.buf[L.bufpos])
matchUnderscoreChars(L, tok, {'0'..'9'})
L.bufpos = msgPos
proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool = proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second) result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
@ -308,99 +260,104 @@ template eatChar(L: var TLexer, t: var TToken) =
add(t.literal, L.buf[L.bufpos]) add(t.literal, L.buf[L.bufpos])
inc(L.bufpos) inc(L.bufpos)
# ###### ######## ######## ## ## ## ## ## ##
# ## ## ## ## ### ## ## ## ### ###
# ## ## ## #### ## ## ## #### ####
# ## #### ###### ## ## ## ## ## ## ## ### ##
# ## ## ## ## ## #### ## ## ## ##
# ## ## ## ## ## ### ## ## ## ##
# ###### ######## ## ## ## ####### ## ##
proc getNumber(L: var TLexer): TToken = proc getNumber(L: var TLexer): TToken =
var var
startpos, endpos: int startpos, endpos: int
xi: BiggestInt xi: BiggestInt
const literalishChars = { 'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'c',
'C', 'b', 'B', '_', '.', '\''}
const literalishCharsNoDot = { 'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o',
'c', 'C', 'b', 'B', '_', '\''}
const literalishChars = { 'A'..'F', 'a'..'f', '0'..'9', proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
'X', 'x', 'o', 'c', 'C', 'b', 'B', var pos = L.bufpos # use registers for pos, buf
'_', '.', '\'' var buf = L.buf
} while true:
const literalishCharsNoDot = { 'A'..'F', 'a'..'f', '0'..'9', if buf[pos] in chars:
'X', 'x', 'o', 'c', 'C', 'b', 'B', add(tok.literal, buf[pos])
'_', '\'' inc(pos)
} else:
break
if buf[pos] == '_':
if buf[pos+1] notin chars:
lexMessage(L, errInvalidToken, "_")
break
add(tok.literal, '_')
inc(pos)
L.bufpos = pos
proc lexLiteralNumberMessage(L: var TLexer, msg: TMsgKind, startpos: int) = proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
var pos = L.bufpos # use registers for pos, buf
var buf = L.buf
while buf[pos] in chars:
add(tok.literal, buf[pos])
inc(pos)
L.bufpos = pos
proc lexMessageLitNum(L: var TLexer, msg: TMsgKind, startpos: int) =
# Used to get slightly human friendlier err messages.
# Note: the erroneous 'O' char in the character set is intentional
const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
var msgPos = L.bufpos
var t: TToken var t: TToken
matchAllLiteralishForMessage(L, t, startpos) t.literal = ""
L.bufpos = startpos # Use L.bufpos as pos because of matchChars
matchChars(L, t, literalishChars)
# We must verify +/- specifically so that we're not past the literal
if L.buf[L.bufpos] in {'+', '-'} and
L.buf[L.bufpos - 1] in {'e', 'E'}:
add(t.literal, L.buf[L.bufpos])
inc(L.bufpos)
matchChars(L, t, literalishChars)
if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
inc(L.bufpos)
add(t.literal, L.buf[L.bufpos])
matchChars(L, t, {'0'..'9'})
L.bufpos = msgPos
lexMessage(L, msg, t.literal) lexMessage(L, msg, t.literal)
# get the base:
result.tokType = tkIntLit # int literal until we know better result.tokType = tkIntLit # int literal until we know better
result.literal = "" result.literal = ""
result.base = base10 # BUGFIX result.base = base10
startpos = L.bufpos
startpos = L.bufpos # make sure the literal is correct for error messages:
var isAFloatLiteral = false var isAFloatLiteral = false
# First stage: find out base, make verifications, build token literal string
# This first pass makes verifications and builds the literal string for
# the token, to keep the value extraction phase less error prone..
if L.buf[L.bufpos] == '0' and if L.buf[L.bufpos] == '0' and
L.buf[L.bufpos + 1] in {'X', 'x', 'o', 'O', 'c', 'C', 'b', 'B'}: L.buf[L.bufpos + 1] in {'X', 'x', 'o', 'O', 'c', 'C', 'b', 'B'}:
eatChar(L, result, '0') eatChar(L, result, '0')
case L.buf[L.bufpos] case L.buf[L.bufpos]
of 'O': of 'O':
# TODO/ozra - add nicer message: " (did you mean octal? Then use one of '0o', '0c' or '0C'.)" lexMessageLitNum(L, errInvalidNumberOctalCode, startpos)
lexLiteralNumberMessage(L, errInvalidNumber, startpos)
of 'x', 'X': of 'x', 'X':
eatChar(L, result, 'x') eatChar(L, result, 'x')
#echo "is hex: ", result.literal
matchUnderscoreChars(L, result, {'0'..'9', 'a'..'f', 'A'..'F'}) matchUnderscoreChars(L, result, {'0'..'9', 'a'..'f', 'A'..'F'})
#echo "got hex: ", result.literal
of 'o', 'c', 'C': of 'o', 'c', 'C':
eatChar(L, result, 'c') eatChar(L, result, 'c')
#echo "is octal: ", result.literal
matchUnderscoreChars(L, result, {'0'..'7'}) matchUnderscoreChars(L, result, {'0'..'7'})
of 'b', 'B': of 'b', 'B':
eatChar(L, result, 'b') eatChar(L, result, 'b')
#echo "is binary: ", result.literal
matchUnderscoreChars(L, result, {'0'..'1'}) matchUnderscoreChars(L, result, {'0'..'1'})
else: else:
internalError(getLineInfo(L), "getNumber") internalError(getLineInfo(L), "getNumber")
else: else:
#echo "is decimal"
matchUnderscoreChars(L, result, {'0'..'9'}) matchUnderscoreChars(L, result, {'0'..'9'})
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}): if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
isAFloatLiteral = true isAFloatLiteral = true
eatChar(L, result, '.') eatChar(L, result, '.')
matchUnderscoreChars(L, result, {'0'..'9'}) matchUnderscoreChars(L, result, {'0'..'9'})
if L.buf[L.bufpos] in {'e', 'E'}: if L.buf[L.bufpos] in {'e', 'E'}:
isAFloatLiteral = true isAFloatLiteral = true
eatChar(L, result, 'e') eatChar(L, result, 'e')
if L.buf[L.bufpos] in {'+', '-'}: if L.buf[L.bufpos] in {'+', '-'}:
eatChar(L, result) eatChar(L, result)
matchUnderscoreChars(L, result, {'0'..'9'}) matchUnderscoreChars(L, result, {'0'..'9'})
endpos = L.bufpos endpos = L.bufpos
# Second stage, find out if there's a datatype postfix and handle it
var postPos = endpos var postPos = endpos
#echo "At endpos is: ", L.buf[postPos]
if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}: if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
if L.buf[postPos] == '\'': if L.buf[postPos] == '\'':
inc(postPos) inc(postPos)
#echo "is postfix: ", L.buf[postPos]
case L.buf[postPos] case L.buf[postPos]
of 'f', 'F': of 'f', 'F':
inc(postPos) inc(postPos)
@ -417,13 +374,9 @@ proc getNumber(L: var TLexer): TToken =
inc(postPos, 3) inc(postPos, 3)
else: # "f" alone defaults to float32 else: # "f" alone defaults to float32
result.tokType = tkFloat32Lit result.tokType = tkFloat32Lit
#lexMessage(L, errInvalidNumber, result.literal & "'f" & L.buf[postPos]) of 'd', 'D': # ad hoc convenience shortcut for f64
# 'd' is an ad hoc convenience shortcut for f64
of 'd', 'D':
inc(postPos) inc(postPos)
result.tokType = tkFloat64Lit result.tokType = tkFloat64Lit
of 'i', 'I': of 'i', 'I':
inc(postPos) inc(postPos)
if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'): if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
@ -439,11 +392,7 @@ proc getNumber(L: var TLexer): TToken =
result.tokType = tkInt8Lit result.tokType = tkInt8Lit
inc(postPos) inc(postPos)
else: else:
lexLiteralNumberMessage(L, errInvalidNumber, startpos) lexMessageLitNum(L, errInvalidNumber, startpos)
#lexMessage(L, errInvalidNumber, result.literal & "'i" & L.buf[postPos])
#echo "is intish: ", result.literal, ":", $ result.tokType
of 'u', 'U': of 'u', 'U':
inc(postPos) inc(postPos)
if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'): if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
@ -461,18 +410,15 @@ proc getNumber(L: var TLexer): TToken =
else: else:
result.tokType = tkUIntLit result.tokType = tkUIntLit
else: else:
lexLiteralNumberMessage(L, errInvalidNumber, startpos) lexMessageLitNum(L, errInvalidNumber, startpos)
#lexMessage(L, errInvalidNumber, result.literal & "'" & L.buf[postPos]) # Is there still a literalish char awaiting? Then it's an error!
# Still a literalish char? Error!
if L.buf[postPos] in literalishCharsNoDot or if L.buf[postPos] in literalishCharsNoDot or
(L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}): (L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
echo "likely literal error, additionally: '", L.buf[postPos], L.buf[postPos+1], L.buf[postPos+2], L.buf[postPos+3], "', ", postPos echo "likely literal error, additionally: '", L.buf[postPos], L.buf[postPos+1], L.buf[postPos+2], L.buf[postPos+3], "', ", postPos
lexLiteralNumberMessage(L, errInvalidNumber, startpos) lexMessageLitNum(L, errInvalidNumber, startpos)
# Third stage, extract actual number
L.bufpos = startpos # restore position L.bufpos = startpos # restore position
var pos: int = startpos var pos: int = startpos
try: try:
if (L.buf[pos] == '0') and if (L.buf[pos] == '0') and
(L.buf[pos + 1] in {'x', 'X', 'b', 'B', 'o', 'O', 'c', 'C'}): (L.buf[pos + 1] in {'x', 'X', 'b', 'B', 'o', 'O', 'c', 'C'}):
@ -510,7 +456,6 @@ proc getNumber(L: var TLexer): TToken =
break break
else: else:
internalError(getLineInfo(L), "getNumber") internalError(getLineInfo(L), "getNumber")
case result.tokType case result.tokType
of tkIntLit, tkInt64Lit: result.iNumber = xi of tkIntLit, tkInt64Lit: result.iNumber = xi
of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi)))) of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
@ -526,38 +471,33 @@ proc getNumber(L: var TLexer): TToken =
# XXX: Test this on big endian machine! # XXX: Test this on big endian machine!
of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[] of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[]
else: internalError(getLineInfo(L), "getNumber") else: internalError(getLineInfo(L), "getNumber")
elif isAFloatLiteral or (result.tokType == tkFloat32Lit) or elif isAFloatLiteral or (result.tokType == tkFloat32Lit) or
(result.tokType == tkFloat64Lit): (result.tokType == tkFloat64Lit):
result.fNumber = parseFloat(result.literal) result.fNumber = parseFloat(result.literal)
if result.tokType == tkIntLit: result.tokType = tkFloatLit if result.tokType == tkIntLit: result.tokType = tkFloatLit
elif result.tokType == tkUint64Lit: elif result.tokType == tkUint64Lit:
xi = 0 xi = 0
let len = unsafeParseUInt(result.literal, xi) let len = unsafeParseUInt(result.literal, xi)
if len != result.literal.len or len == 0: if len != result.literal.len or len == 0:
raise newException(ValueError, "invalid integer: " & $xi) raise newException(ValueError, "invalid integer: " & $xi)
result.iNumber = xi result.iNumber = xi
else: else:
result.iNumber = parseBiggestInt(result.literal) result.iNumber = parseBiggestInt(result.literal)
if (result.iNumber < low(int32)) or (result.iNumber > high(int32)): if (result.iNumber < low(int32)) or (result.iNumber > high(int32)):
if result.tokType == tkIntLit: if result.tokType == tkIntLit:
result.tokType = tkInt64Lit result.tokType = tkInt64Lit
elif result.tokType in {tkInt8Lit, tkInt16Lit, tkInt32Lit}: elif result.tokType in {tkInt8Lit, tkInt16Lit, tkInt32Lit}:
lexLiteralNumberMessage(L, errNumberOutOfRange, startpos) lexMessageLitNum(L, errNumberOutOfRange, startpos)
elif result.tokType == tkInt8Lit and elif result.tokType == tkInt8Lit and
(result.iNumber < int8.low or result.iNumber > int8.high): (result.iNumber < int8.low or result.iNumber > int8.high):
lexLiteralNumberMessage(L, errNumberOutOfRange, startpos) lexMessageLitNum(L, errNumberOutOfRange, startpos)
elif result.tokType == tkInt16Lit and elif result.tokType == tkInt16Lit and
(result.iNumber < int16.low or result.iNumber > int16.high): (result.iNumber < int16.low or result.iNumber > int16.high):
lexLiteralNumberMessage(L, errNumberOutOfRange, startpos) lexMessageLitNum(L, errNumberOutOfRange, startpos)
except ValueError: except ValueError:
lexLiteralNumberMessage(L, errInvalidNumber, startpos) lexMessageLitNum(L, errInvalidNumber, startpos)
except OverflowError, RangeError: except OverflowError, RangeError:
lexLiteralNumberMessage(L, errNumberOutOfRange, startpos) lexMessageLitNum(L, errNumberOutOfRange, startpos)
L.bufpos = postPos L.bufpos = postPos
proc handleHexChar(L: var TLexer, xi: var int) = proc handleHexChar(L: var TLexer, xi: var int) =

8
compiler/msgs.nim
View file

@ -17,10 +17,9 @@ type
errIntLiteralExpected, errInvalidCharacterConstant, errIntLiteralExpected, errInvalidCharacterConstant,
errClosingTripleQuoteExpected, errClosingQuoteExpected, errClosingTripleQuoteExpected, errClosingQuoteExpected,
errTabulatorsAreNotAllowed, errInvalidToken, errLineTooLong, errTabulatorsAreNotAllowed, errInvalidToken, errLineTooLong,
errInvalidNumber, errNumberOutOfRange, errNnotAllowedInCharacter, errInvalidNumber, errInvalidNumberOctalCode, errNumberOutOfRange,
errClosingBracketExpected, errMissingFinalQuote, errIdentifierExpected, errNnotAllowedInCharacter, errClosingBracketExpected, errMissingFinalQuote,
errNewlineExpected, errIdentifierExpected, errNewlineExpected, errInvalidModuleName,
errInvalidModuleName,
errOperatorExpected, errTokenExpected, errStringAfterIncludeExpected, errOperatorExpected, errTokenExpected, errStringAfterIncludeExpected,
errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected, errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected,
errInvalidPragma, errUnknownPragma, errInvalidDirectiveX, errInvalidPragma, errUnknownPragma, errInvalidDirectiveX,
@ -143,6 +142,7 @@ const
errInvalidToken: "invalid token: $1", errInvalidToken: "invalid token: $1",
errLineTooLong: "line too long", errLineTooLong: "line too long",
errInvalidNumber: "$1 is not a valid number", errInvalidNumber: "$1 is not a valid number",
errInvalidNumberOctalCode: "$1 is not a valid number; did you mean octal? Then use one of '0o', '0c' or '0C'.",
errNumberOutOfRange: "number $1 out of valid range", errNumberOutOfRange: "number $1 out of valid range",
errNnotAllowedInCharacter: "\\n not allowed in character literal", errNnotAllowedInCharacter: "\\n not allowed in character literal",
errClosingBracketExpected: "closing ']' expected, but end of file reached", errClosingBracketExpected: "closing ']' expected, but end of file reached",