jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

custom integer literals (#17489)

Browse source 
* user defined integer literals; refs #17020
* updated renderer.nim
* use mlexerutils helper
* imported all test cases from https://github.com/nim-lang/Nim/pull/17020
* final grammar updated
This commit is contained in:
Andreas Rumpf 2021-03-24 14:46:19 +01:00 • committed by GitHub
commit 5f5a92379f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 409 additions and 218 deletions
Download patch file Download diff file Expand all files Collapse all files

2
changelog.md
View file

@ -266,6 +266,8 @@
- The unary minus in `-1` is now part of the integer literal, it is now parsed as a single token. - The unary minus in `-1` is now part of the integer literal, it is now parsed as a single token.
This implies that edge cases like `-128'i8` finally work correctly. This implies that edge cases like `-128'i8` finally work correctly.
- Custom numeric literals are now supported.
## Compiler changes ## Compiler changes

2
compiler/docgen.nim
View file

@ -416,7 +416,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
of tkOpr: of tkOpr:
dispA(d.conf, result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}", dispA(d.conf, result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}",
[escLit]) [escLit])
of tkStrLit..tkTripleStrLit: of tkStrLit..tkTripleStrLit, tkCustomLit:
dispA(d.conf, result, "<span class=\"StringLit\">$1</span>", dispA(d.conf, result, "<span class=\"StringLit\">$1</span>",
"\\spanStringLit{$1}", [escLit]) "\\spanStringLit{$1}", [escLit])
of tkCharLit: of tkCharLit:

342
compiler/lexer.nim
View file

@ -60,6 +60,7 @@ type
tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit", tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit",
tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit", tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit",
tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit", tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit",
tkCustomLit = "tkCustomLit",
tkParLe = "(", tkParRi = ")", tkBracketLe = "[", tkParLe = "(", tkParRi = ")", tkBracketLe = "[",
tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}", tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}",
@ -313,8 +314,7 @@ proc getNumber(L: var Lexer, result: var Token) =
proc lexMessageLitNum(L: var Lexer, msg: string, startpos: int, msgKind = errGenerated) = proc lexMessageLitNum(L: var Lexer, msg: string, startpos: int, msgKind = errGenerated) =
# Used to get slightly human friendlier err messages. # Used to get slightly human friendlier err messages.
const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O', const literalishChars = {'A'..'Z', 'a'..'z', '0'..'9', '_', '.', '\''}
'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
var msgPos = L.bufpos var msgPos = L.bufpos
var t: Token var t: Token
t.literal = "" t.literal = ""
@ -326,15 +326,14 @@ proc getNumber(L: var Lexer, result: var Token) =
t.literal.add(L.buf[L.bufpos]) t.literal.add(L.buf[L.bufpos])
inc(L.bufpos) inc(L.bufpos)
matchChars(L, t, literalishChars) matchChars(L, t, literalishChars)
if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}: if L.buf[L.bufpos] in literalishChars:
inc(L.bufpos)
t.literal.add(L.buf[L.bufpos]) t.literal.add(L.buf[L.bufpos])
inc(L.bufpos)
matchChars(L, t, {'0'..'9'}) matchChars(L, t, {'0'..'9'})
L.bufpos = msgPos L.bufpos = msgPos
lexMessage(L, msgKind, msg % t.literal) lexMessage(L, msgKind, msg % t.literal)
var var
startpos, endpos: int
xi: BiggestInt xi: BiggestInt
isBase10 = true isBase10 = true
numDigits = 0 numDigits = 0
@ -346,7 +345,7 @@ proc getNumber(L: var Lexer, result: var Token) =
result.tokType = tkIntLit # int literal until we know better result.tokType = tkIntLit # int literal until we know better
result.literal = "" result.literal = ""
result.base = base10 result.base = base10
startpos = L.bufpos let startpos = L.bufpos
tokenBegin(result, startpos) tokenBegin(result, startpos)
var isPositive = true var isPositive = true
@ -395,201 +394,187 @@ proc getNumber(L: var Lexer, result: var Token) =
discard matchUnderscoreChars(L, result, {'0'..'9'}) discard matchUnderscoreChars(L, result, {'0'..'9'})
if L.buf[L.bufpos] in {'e', 'E'}: if L.buf[L.bufpos] in {'e', 'E'}:
result.tokType = tkFloatLit result.tokType = tkFloatLit
eatChar(L, result, 'e') eatChar(L, result)
if L.buf[L.bufpos] in {'+', '-'}: if L.buf[L.bufpos] in {'+', '-'}:
eatChar(L, result) eatChar(L, result)
discard matchUnderscoreChars(L, result, {'0'..'9'}) discard matchUnderscoreChars(L, result, {'0'..'9'})
endpos = L.bufpos let endpos = L.bufpos
# Second stage, find out if there's a datatype suffix and handle it # Second stage, find out if there's a datatype suffix and handle it
var postPos = endpos var postPos = endpos
if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}: if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
let errPos = postPos
var customLitPossible = false
if L.buf[postPos] == '\'': if L.buf[postPos] == '\'':
inc(postPos) inc(postPos)
customLitPossible = true
case L.buf[postPos] if L.buf[postPos] in SymChars:
of 'f', 'F': var suffixAsLower = newStringOfCap(10)
inc(postPos) var suffix = newStringOfCap(10)
if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'): while true:
result.tokType = tkFloat32Lit let c = L.buf[postPos]
inc(postPos, 2) suffix.add c
elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'): suffixAsLower.add toLowerAscii(c)
result.tokType = tkFloat64Lit inc postPos
inc(postPos, 2) if L.buf[postPos] notin SymChars+{'_'}: break
elif (L.buf[postPos] == '1') and case suffix
(L.buf[postPos + 1] == '2') and of "f", "f32": result.tokType = tkFloat32Lit
(L.buf[postPos + 2] == '8'): of "d", "f64": result.tokType = tkFloat64Lit
result.tokType = tkFloat128Lit of "f128": result.tokType = tkFloat128Lit
inc(postPos, 3) of "i8": result.tokType = tkInt8Lit
else: # "f" alone defaults to float32 of "i16": result.tokType = tkInt16Lit
result.tokType = tkFloat32Lit of "i32": result.tokType = tkInt32Lit
of 'd', 'D': # ad hoc convenience shortcut for f64 of "i64": result.tokType = tkInt64Lit
inc(postPos) of "u": result.tokType = tkUIntLit
result.tokType = tkFloat64Lit of "u8": result.tokType = tkUInt8Lit
of 'i', 'I': of "u16": result.tokType = tkUInt16Lit
inc(postPos) of "u32": result.tokType = tkUInt32Lit
if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'): of "u64": result.tokType = tkUInt64Lit
result.tokType = tkInt64Lit
inc(postPos, 2)
elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
result.tokType = tkInt32Lit
inc(postPos, 2)
elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
result.tokType = tkInt16Lit
inc(postPos, 2)
elif (L.buf[postPos] == '8'):
result.tokType = tkInt8Lit
inc(postPos)
else: else:
lexMessageLitNum(L, "invalid number: '$1'", startpos) if customLitPossible:
of 'u', 'U': # remember the position of the ``'`` so that the parser doesn't
inc(postPos) # have to reparse the custom literal:
if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'): result.iNumber = len(result.literal)
result.tokType = tkUInt64Lit result.literal.add '\''
inc(postPos, 2) result.literal.add suffix
elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'): result.tokType = tkCustomLit
result.tokType = tkUInt32Lit else:
inc(postPos, 2) lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
result.tokType = tkUInt16Lit
inc(postPos, 2)
elif (L.buf[postPos] == '8'):
result.tokType = tkUInt8Lit
inc(postPos)
else:
result.tokType = tkUIntLit
else: else:
lexMessageLitNum(L, "invalid number: '$1'", startpos) lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
# Is there still a literalish char awaiting? Then it's an error! # Is there still a literalish char awaiting? Then it's an error!
if L.buf[postPos] in literalishChars or if L.buf[postPos] in literalishChars or
(L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}): (L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
lexMessageLitNum(L, "invalid number: '$1'", startpos) lexMessageLitNum(L, "invalid number: '$1'", startpos)
# Third stage, extract actual number if result.tokType != tkCustomLit:
L.bufpos = startpos # restore position # Third stage, extract actual number
var pos = startpos L.bufpos = startpos # restore position
try: var pos = startpos
if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars): try:
inc(pos, 2) if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars):
xi = 0 # it is a base prefix inc(pos, 2)
xi = 0 # it is a base prefix
case L.buf[pos - 1] case L.buf[pos - 1]
of 'b', 'B': of 'b', 'B':
result.base = base2 result.base = base2
while pos < endpos: while pos < endpos:
if L.buf[pos] != '_': if L.buf[pos] != '_':
xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0')) xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
inc(pos)
# 'c', 'C' is deprecated
of 'o', 'c', 'C':
result.base = base8
while pos < endpos:
if L.buf[pos] != '_':
xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
inc(pos)
of 'x', 'X':
result.base = base16
while pos < endpos:
case L.buf[pos]
of '_':
inc(pos) inc(pos)
of '0'..'9': # 'c', 'C' is deprecated
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0')) of 'o', 'c', 'C':
result.base = base8
while pos < endpos:
if L.buf[pos] != '_':
xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
inc(pos) inc(pos)
of 'a'..'f': of 'x', 'X':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10) result.base = base16
inc(pos) while pos < endpos:
of 'A'..'F': case L.buf[pos]
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10) of '_':
inc(pos) inc(pos)
else: of '0'..'9':
break xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
inc(pos)
of 'a'..'f':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10)
inc(pos)
of 'A'..'F':
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
inc(pos)
else:
break
else:
internalError(L.config, getLineInfo(L), "getNumber")
case result.tokType
of tkIntLit, tkInt64Lit: setNumber result.iNumber, xi
of tkInt8Lit: setNumber result.iNumber, ashr(xi shl 56, 56)
of tkInt16Lit: setNumber result.iNumber, ashr(xi shl 48, 48)
of tkInt32Lit: setNumber result.iNumber, ashr(xi shl 32, 32)
of tkUIntLit, tkUInt64Lit: setNumber result.iNumber, xi
of tkUInt8Lit: setNumber result.iNumber, xi and 0xff
of tkUInt16Lit: setNumber result.iNumber, xi and 0xffff
of tkUInt32Lit: setNumber result.iNumber, xi and 0xffffffff
of tkFloat32Lit:
setNumber result.fNumber, (cast[PFloat32](addr(xi)))[]
# note: this code is endian neutral!
# XXX: Test this on big endian machine!
of tkFloat64Lit, tkFloatLit:
setNumber result.fNumber, (cast[PFloat64](addr(xi)))[]
else: internalError(L.config, getLineInfo(L), "getNumber")
# Bounds checks. Non decimal literals are allowed to overflow the range of
# the datatype as long as their pattern don't overflow _bitwise_, hence
# below checks of signed sizes against uint*.high is deliberate:
# (0x80'u8 = 128, 0x80'i8 = -128, etc == OK)
if result.tokType notin floatTypes:
let outOfRange =
case result.tokType
of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi
of tkInt8Lit: (xi > BiggestInt(uint8.high))
of tkInt16Lit: (xi > BiggestInt(uint16.high))
of tkInt32Lit: (xi > BiggestInt(uint32.high))
else: false
if outOfRange:
#echo "out of range num: ", result.iNumber, " vs ", xi
lexMessageLitNum(L, "number out of range: '$1'", startpos)
else: else:
internalError(L.config, getLineInfo(L), "getNumber") case result.tokType
of floatTypes:
result.fNumber = parseFloat(result.literal)
of tkUInt64Lit, tkUIntLit:
var iNumber: uint64
var len: int
try:
len = parseBiggestUInt(result.literal, iNumber)
except ValueError:
raise newException(OverflowDefect, "number out of range: " & $result.literal)
if len != result.literal.len:
raise newException(ValueError, "invalid integer: " & $result.literal)
result.iNumber = cast[int64](iNumber)
else:
var iNumber: int64
var len: int
try:
len = parseBiggestInt(result.literal, iNumber)
except ValueError:
raise newException(OverflowDefect, "number out of range: " & $result.literal)
if len != result.literal.len:
raise newException(ValueError, "invalid integer: " & $result.literal)
result.iNumber = iNumber
case result.tokType # Explicit bounds checks.
of tkIntLit, tkInt64Lit: setNumber result.iNumber, xi
of tkInt8Lit: setNumber result.iNumber, ashr(xi shl 56, 56)
of tkInt16Lit: setNumber result.iNumber, ashr(xi shl 48, 48)
of tkInt32Lit: setNumber result.iNumber, ashr(xi shl 32, 32)
of tkUIntLit, tkUInt64Lit: setNumber result.iNumber, xi
of tkUInt8Lit: setNumber result.iNumber, xi and 0xff
of tkUInt16Lit: setNumber result.iNumber, xi and 0xffff
of tkUInt32Lit: setNumber result.iNumber, xi and 0xffffffff
of tkFloat32Lit:
setNumber result.fNumber, (cast[PFloat32](addr(xi)))[]
# note: this code is endian neutral!
# XXX: Test this on big endian machine!
of tkFloat64Lit, tkFloatLit:
setNumber result.fNumber, (cast[PFloat64](addr(xi)))[]
else: internalError(L.config, getLineInfo(L), "getNumber")
# Bounds checks. Non decimal literals are allowed to overflow the range of
# the datatype as long as their pattern don't overflow _bitwise_, hence
# below checks of signed sizes against uint*.high is deliberate:
# (0x80'u8 = 128, 0x80'i8 = -128, etc == OK)
if result.tokType notin floatTypes:
let outOfRange = let outOfRange =
case result.tokType case result.tokType
of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi of tkInt8Lit: result.iNumber > int8.high or result.iNumber < int8.low
of tkInt8Lit: (xi > BiggestInt(uint8.high)) of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high) or result.iNumber < 0
of tkInt16Lit: (xi > BiggestInt(uint16.high)) of tkInt16Lit: result.iNumber > int16.high or result.iNumber < int16.low
of tkInt32Lit: (xi > BiggestInt(uint32.high)) of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high) or result.iNumber < 0
of tkInt32Lit: result.iNumber > int32.high or result.iNumber < int32.low
of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high) or result.iNumber < 0
else: false else: false
if outOfRange: if outOfRange:
#echo "out of range num: ", result.iNumber, " vs ", xi
lexMessageLitNum(L, "number out of range: '$1'", startpos) lexMessageLitNum(L, "number out of range: '$1'", startpos)
else: # Promote int literal to int64? Not always necessary, but more consistent
case result.tokType if result.tokType == tkIntLit:
of floatTypes: if result.iNumber > high(int32) or result.iNumber < low(int32):
result.fNumber = parseFloat(result.literal) result.tokType = tkInt64Lit
of tkUInt64Lit, tkUIntLit:
var iNumber: uint64
var len: int
try:
len = parseBiggestUInt(result.literal, iNumber)
except ValueError:
raise newException(OverflowDefect, "number out of range: " & $result.literal)
if len != result.literal.len:
raise newException(ValueError, "invalid integer: " & $result.literal)
result.iNumber = cast[int64](iNumber)
else:
var iNumber: int64
var len: int
try:
len = parseBiggestInt(result.literal, iNumber)
except ValueError:
raise newException(OverflowDefect, "number out of range: " & $result.literal)
if len != result.literal.len:
raise newException(ValueError, "invalid integer: " & $result.literal)
result.iNumber = iNumber
# Explicit bounds checks. except ValueError:
let outOfRange = lexMessageLitNum(L, "invalid number: '$1'", startpos)
case result.tokType except OverflowDefect, RangeDefect:
of tkInt8Lit: result.iNumber > int8.high or result.iNumber < int8.low lexMessageLitNum(L, "number out of range: '$1'", startpos)
of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high) or result.iNumber < 0
of tkInt16Lit: result.iNumber > int16.high or result.iNumber < int16.low
of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high) or result.iNumber < 0
of tkInt32Lit: result.iNumber > int32.high or result.iNumber < int32.low
of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high) or result.iNumber < 0
else: false
if outOfRange:
lexMessageLitNum(L, "number out of range: '$1'", startpos)
# Promote int literal to int64? Not always necessary, but more consistent
if result.tokType == tkIntLit:
if result.iNumber > high(int32) or result.iNumber < low(int32):
result.tokType = tkInt64Lit
except ValueError:
lexMessageLitNum(L, "invalid number: '$1'", startpos)
except OverflowDefect, RangeDefect:
lexMessageLitNum(L, "number out of range: '$1'", startpos)
tokenEnd(result, postPos-1) tokenEnd(result, postPos-1)
L.bufpos = postPos L.bufpos = postPos
@ -830,8 +815,9 @@ proc getString(L: var Lexer, tok: var Token, mode: StringMode) =
inc(pos) inc(pos)
L.bufpos = pos L.bufpos = pos
proc getCharacter(L: var Lexer, tok: var Token) = proc getCharacter(L: var Lexer; tok: var Token) =
tokenBegin(tok, L.bufpos) tokenBegin(tok, L.bufpos)
let startPos = L.bufpos
inc(L.bufpos) # skip ' inc(L.bufpos) # skip '
var c = L.buf[L.bufpos] var c = L.buf[L.bufpos]
case c case c
@ -842,10 +828,16 @@ proc getCharacter(L: var Lexer, tok: var Token) =
else: else:
tok.literal = $c tok.literal = $c
inc(L.bufpos) inc(L.bufpos)
if L.buf[L.bufpos] != '\'': if L.buf[L.bufpos] == '\'':
lexMessage(L, errGenerated, "missing closing ' for character literal") tokenEndIgnore(tok, L.bufpos)
tokenEndIgnore(tok, L.bufpos) inc(L.bufpos) # skip '
inc(L.bufpos) # skip ' else:
if startPos > 0 and L.buf[startPos-1] == '`':
tok.literal = "'"
L.bufpos = startPos+1
else:
lexMessage(L, errGenerated, "missing closing ' for character literal")
tokenEndIgnore(tok, L.bufpos)
proc getSymbol(L: var Lexer, tok: var Token) = proc getSymbol(L: var Lexer, tok: var Token) =
var h: Hash = 0 var h: Hash = 0

18
compiler/parser.nim
View file

@ -355,7 +355,7 @@ proc parseSymbol(p: var Parser, mode = smNormal): PNode =
let node = newNodeI(nkIdent, lineinfo) let node = newNodeI(nkIdent, lineinfo)
node.ident = p.lex.cache.getIdent(accm) node.ident = p.lex.cache.getIdent(accm)
result.add(node) result.add(node)
of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit: of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCustomLit:
result.add(newIdentNodeP(p.lex.cache.getIdent($p.tok), p)) result.add(newIdentNodeP(p.lex.cache.getIdent($p.tok), p))
getTok(p) getTok(p)
else: else:
@ -627,7 +627,7 @@ proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode =
#| | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT #| | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT
#| | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT #| | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT
#| | STR_LIT | RSTR_LIT | TRIPLESTR_LIT #| | STR_LIT | RSTR_LIT | TRIPLESTR_LIT
#| | CHAR_LIT #| | CHAR_LIT | CUSTOM_NUMERIC_LIT
#| | NIL #| | NIL
#| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT #| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT
#| identOrLiteral = generalizedLit | symbol | literal #| identOrLiteral = generalizedLit | symbol | literal
@ -710,6 +710,14 @@ proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode =
of tkCharLit: of tkCharLit:
result = newIntNodeP(nkCharLit, ord(p.tok.literal[0]), p) result = newIntNodeP(nkCharLit, ord(p.tok.literal[0]), p)
getTok(p) getTok(p)
of tkCustomLit:
let splitPos = p.tok.iNumber.int
let str = newStrNodeP(nkRStrLit, p.tok.literal.substr(0, splitPos-1), p)
let callee = newIdentNodeP(getIdent(p.lex.cache, p.tok.literal.substr(splitPos)), p)
result = newNodeP(nkDotExpr, p)
result.add str
result.add callee
getTok(p)
of tkNil: of tkNil:
result = newNodeP(nkNilLit, p) result = newNodeP(nkNilLit, p)
getTok(p) getTok(p)
@ -807,7 +815,7 @@ proc primarySuffix(p: var Parser, r: PNode,
result = commandExpr(p, result, mode) result = commandExpr(p, result, mode)
break break
result = namedParams(p, result, nkCurlyExpr, tkCurlyRi) result = namedParams(p, result, nkCurlyExpr, tkCurlyRi)
of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast, of tkSymbol, tkAccent, tkIntLit..tkCustomLit, tkNil, tkCast,
tkOpr, tkDotDot, tkVar, tkOut, tkStatic, tkType, tkEnum, tkTuple, tkOpr, tkDotDot, tkVar, tkOut, tkStatic, tkType, tkEnum, tkTuple,
tkObject, tkProc: tkObject, tkProc:
# XXX: In type sections we allow the free application of the # XXX: In type sections we allow the free application of the
@ -1097,7 +1105,7 @@ proc isExprStart(p: Parser): bool =
case p.tok.tokType case p.tok.tokType
of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor, of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor,
tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics, tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics,
tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr, tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCustomLit, tkVar, tkRef, tkPtr,
tkTuple, tkObject, tkWhen, tkCase, tkOut: tkTuple, tkObject, tkWhen, tkCase, tkOut:
result = true result = true
else: result = false else: result = false
@ -1498,7 +1506,7 @@ proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode =
#| yieldStmt = 'yield' optInd expr? #| yieldStmt = 'yield' optInd expr?
#| discardStmt = 'discard' optInd expr? #| discardStmt = 'discard' optInd expr?
#| breakStmt = 'break' optInd expr? #| breakStmt = 'break' optInd expr?
#| continueStmt = 'break' optInd expr? #| continueStmt = 'continue' optInd expr?
result = newNodeP(kind, p) result = newNodeP(kind, p)
getTok(p) getTok(p)
if p.tok.tokType == tkComment: if p.tok.tokType == tkComment:

18
compiler/renderer.nim
View file

@ -942,7 +942,7 @@ proc skipHiddenNodes(n: PNode): PNode =
else: break else: break
proc accentedName(g: var TSrcGen, n: PNode) = proc accentedName(g: var TSrcGen, n: PNode) =
const backticksNeeded = OpChars + {'[', '{'} const backticksNeeded = OpChars + {'[', '{', '\''}
if n == nil: return if n == nil: return
let isOperator = let isOperator =
if n.kind == nkIdent and n.ident.s.len > 0 and n.ident.s[0] in backticksNeeded: true if n.kind == nkIdent and n.ident.s.len > 0 and n.ident.s[0] in backticksNeeded: true
@ -976,6 +976,11 @@ proc infixArgument(g: var TSrcGen, n: PNode, i: int) =
if needsParenthesis: if needsParenthesis:
put(g, tkParRi, ")") put(g, tkParRi, ")")
proc isCustomLit(n: PNode): bool =
n.len == 2 and n[0].kind == nkRStrLit and
(n[1].kind == nkIdent and n[1].ident.s.startsWith('\'')) or
(n[1].kind == nkSym and n[1].sym.name.s.startsWith('\''))
proc gsub(g: var TSrcGen, n: PNode, c: TContext) = proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
if isNil(n): return if isNil(n): return
var var
@ -1195,9 +1200,14 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
gcomma(g, n, c) gcomma(g, n, c)
put(g, tkBracketRi, "]") put(g, tkBracketRi, "]")
of nkDotExpr: of nkDotExpr:
gsub(g, n, 0) if isCustomLit(n):
put(g, tkDot, ".") put(g, tkCustomLit, n[0].strVal)
gsub(g, n, 1) gsub(g, n, 1)
else:
gsub(g, n, 0)
put(g, tkDot, ".")
if n.len > 1:
accentedName(g, n[1])
of nkBind: of nkBind:
putWithSpace(g, tkBind, "bind") putWithSpace(g, tkBind, "bind")
gsub(g, n, 0) gsub(g, n, 0)

3
doc/grammar.txt
View file

@ -46,7 +46,7 @@ literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT
| UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT
| FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT
| STR_LIT | RSTR_LIT | TRIPLESTR_LIT | STR_LIT | RSTR_LIT | TRIPLESTR_LIT
| CHAR_LIT | CHAR_LIT | CUSTOM_NUMERIC_LIT
| NIL | NIL
generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT
identOrLiteral = generalizedLit | symbol | literal identOrLiteral = generalizedLit | symbol | literal
@ -100,6 +100,7 @@ postExprBlocks = ':' stmt? ( IND{=} doBlock
| IND{=} 'of' exprList ':' stmt | IND{=} 'of' exprList ':' stmt
| IND{=} 'elif' expr ':' stmt | IND{=} 'elif' expr ':' stmt
| IND{=} 'except' exprList ':' stmt | IND{=} 'except' exprList ':' stmt
| IND{=} 'finally' ':' stmt
| IND{=} 'else' ':' stmt )* | IND{=} 'else' ':' stmt )*
exprStmt = simpleExpr exprStmt = simpleExpr
(( '=' optInd expr colonBody? ) (( '=' optInd expr colonBody? )

54
doc/manual.rst
View file

@ -490,11 +490,17 @@ this. Another reason is that Nim can thus support `array[char, int]` or
type is used for Unicode characters, it can represent any Unicode character. type is used for Unicode characters, it can represent any Unicode character.
`Rune` is declared in the `unicode module <unicode.html>`_. `Rune` is declared in the `unicode module <unicode.html>`_.
A character literal that does not end in ``'`` interpreted as ``'`` if there
is a preceeding backtick token. There must be no whitespace between the preceeding
backtick token and the character literal. This special rule ensures that a declaration
like ``proc `'customLiteral`(s: string)`` is valid. See also
`Custom Numeric Literals <#custom-numeric-literals>`_.
Numerical constants
-------------------
Numerical constants are of a single type and have the form:: Numeric Literals
----------------
Numeric literals have the form::
hexdigit = digit | 'A'..'F' | 'a'..'f' hexdigit = digit | 'A'..'F' | 'a'..'f'
octdigit = '0'..'7' octdigit = '0'..'7'
@ -530,8 +536,10 @@ Numerical constants are of a single type and have the form::
FLOAT64_LIT = HEX_LIT '\'' FLOAT64_SUFFIX FLOAT64_LIT = HEX_LIT '\'' FLOAT64_SUFFIX
| (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT64_SUFFIX | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT64_SUFFIX
CUSTOM_NUMERIC_LIT = (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) '\'' CUSTOM_NUMERIC_SUFFIX
As can be seen in the productions, numerical constants can contain underscores
As can be seen in the productions, numeric literals can contain underscores
for readability. Integer and floating-point literals may be given in decimal (no for readability. Integer and floating-point literals may be given in decimal (no
prefix), binary (prefix `0b`), octal (prefix `0o`), and hexadecimal prefix), binary (prefix `0b`), octal (prefix `0o`), and hexadecimal
(prefix `0x`) notation. (prefix `0x`) notation.
@ -579,7 +587,7 @@ is optional if it is not ambiguous (only hexadecimal floating-point literals
with a type suffix can be ambiguous). with a type suffix can be ambiguous).
The type suffixes are: The pre-defined type suffixes are:
================= ========================= ================= =========================
Type Suffix Resulting type of literal Type Suffix Resulting type of literal
@ -611,6 +619,42 @@ the bit width of the datatype, it is accepted.
Hence: 0b10000000'u8 == 0x80'u8 == 128, but, 0b10000000'i8 == 0x80'i8 == -1 Hence: 0b10000000'u8 == 0x80'u8 == 128, but, 0b10000000'i8 == 0x80'i8 == -1
instead of causing an overflow error. instead of causing an overflow error.
Custom Numeric Literals
~~~~~~~~~~~~~~~~~~~~~~~
If the suffix is not predefined, then the suffix is assumed to be a call
to a proc, template, macro or other callable identifier that is passed the
string containing the literal. The callable identifier needs to be declared
with a special ``'`` prefix:
.. code-block:: nim
import strutils
type u4 = distinct uint8 # a 4-bit unsigned integer aka "nibble"
proc `'u4`(n: string): u4 =
# The leading ' is required.
result = (parseInt(n) and 0x0F).u4
var x = 5'u4
More formally, a custom numeric literal `123'custom` is transformed
to r"123".`'custom` in the parsing step. There is no AST node kind that
corresponds to this transformation. The transformation naturally handles
the case that additional parameters are passed to the callee:
.. code-block:: nim
import strutils
type u4 = distinct uint8 # a 4-bit unsigned integer aka "nibble"
proc `'u4`(n: string; moreData: int): u4 =
result = (parseInt(n) and 0x0F).u4
var x = 5'u4(123)
Custom numeric literals are covered by the grammar rule named `CUSTOM_NUMERIC_LIT`.
Operators Operators
--------- ---------

9
tests/lexer/mlexerutils.nim Normal file
View file

@ -0,0 +1,9 @@
import macros
macro lispReprStr*(a: untyped): untyped = newLit(a.lispRepr)
macro assertAST*(expected: string, struct: untyped): untyped =
var ast = newLit(struct.treeRepr)
result = quote do:
if `ast` != `expected`:
doAssert false, "\nGot:\n" & `ast`.indent(2) & "\nExpected:\n" & `expected`.indent(2)

150
tests/lexer/tcustom_numeric_literals.nim Normal file
View file

@ -0,0 +1,150 @@
discard """
targets: "c cpp js"
"""
# Test tkStrNumLit
import std/[macros, strutils]
import mlexerutils
# AST checks
assertAST dedent """
StmtList
ProcDef
AccQuoted
Ident "\'"
Ident "wrap"
Empty
Empty
FormalParams
Ident "string"
IdentDefs
Ident "number"
Ident "string"
Empty
Empty
Empty
StmtList
Asgn
Ident "result"
Infix
Ident "&"
Infix
Ident "&"
StrLit "[["
Ident "number"
StrLit "]]"""":
proc `'wrap`(number: string): string =
result = "[[" & number & "]]"
assertAST dedent """
StmtList
DotExpr
RStrLit "-38383839292839283928392839283928392839283.928493849385935898243e-50000"
Ident "\'wrap"""":
-38383839292839283928392839283928392839283.928493849385935898243e-50000'wrap
proc `'wrap`(number: string): string = "[[" & number & "]]"
doAssert lispReprStr(-1'wrap) == """(DotExpr (RStrLit "-1") (Ident "\'wrap"))"""
template main =
block: # basic suffix usage
template `'twrap`(number: string): untyped =
number.`'wrap`
proc extraContext(): string =
22.40'wrap
proc `*`(left, right: string): string =
result = left & "times" & right
proc `+`(left, right: string): string =
result = left & "plus" & right
doAssert 1'wrap == "[[1]]"
doAssert -1'wrap == "[[-1]]":
"unable to resolve a negative integer-suffix pattern"
doAssert 12345.67890'wrap == "[[12345.67890]]"
doAssert 1'wrap*1'wrap == "[[1]]times[[1]]":
"unable to resolve an operator between two suffixed numeric literals"
doAssert 1'wrap+ -1'wrap == "[[1]]plus[[-1]]": # will generate a compiler warning about inconsistent spacing
"unable to resolve a negative suffixed numeric literal following an operator"
doAssert 1'wrap + -1'wrap == "[[1]]plus[[-1]]"
doAssert 1'twrap == "[[1]]"
doAssert extraContext() == "[[22.40]]":
"unable to return a suffixed numeric literal by an implicit return"
doAssert 0x5a3a'wrap == "[[0x5a3a]]"
doAssert 0o5732'wrap == "[[0o5732]]"
doAssert 0b0101111010101'wrap == "[[0b0101111010101]]"
doAssert -38383839292839283928392839283928392839283.928493849385935898243e-50000'wrap == "[[-38383839292839283928392839283928392839283.928493849385935898243e-50000]]"
doAssert 1234.56'wrap == "[[1234.56]]":
"unable to properly account for context with suffixed numeric literals"
block: # verify that the i64, f32, etc builtin suffixes still parse correctly
const expectedF32: float32 = 123.125
proc `'f9`(number: string): string = # proc starts with 'f' just like 'f32'
"[[" & number & "]]"
proc `'f32a`(number: string): string = # looks even more like 'f32'
"[[" & number & "]]"
proc `'d9`(number: string): string = # proc starts with 'd' just like the d suffix
"[[" & number & "]]"
proc `'i9`(number: string): string = # proc starts with 'i' just like 'i64'
"[[" & number & "]]"
proc `'u9`(number: string): string = # proc starts with 'u' just like 'u8'
"[[" & number & "]]"
doAssert 123.125f32 == expectedF32:
"failing to support non-quoted legacy f32 floating point suffix"
doAssert 123.125'f32 == expectedF32
doAssert 123.125e0'f32 == expectedF32
doAssert 1234.56'wrap == 1234.56'f9
doAssert 1234.56'wrap == 1234.56'f32a
doAssert 1234.56'wrap == 1234.56'd9
doAssert 1234.56'wrap == 1234.56'i9
doAssert 1234.56'wrap == 1234.56'u9
doAssert lispReprStr(1234.56'u9) == """(DotExpr (RStrLit "1234.56") (Ident "\'u9"))""":
"failed to properly build AST for suffix that starts with u"
doAssert -128'i8 == (-128).int8
block: # case checks
doAssert 1E2 == 100:
"lexer not handling upper-case exponent"
doAssert 1.0E2 == 100.0
doAssert 1e2 == 100
doAssert 0xdeadBEEF'wrap == "[[0xdeadBEEF]]":
"lexer not maintaining original case"
doAssert 0.1E12'wrap == "[[0.1E12]]"
doAssert 0.0e12'wrap == "[[0.0e12]]"
doAssert 0.0e+12'wrap == "[[0.0e+12]]"
doAssert 0.0e-12'wrap == "[[0.0e-12]]"
doAssert 0e-12'wrap == "[[0e-12]]"
block: # macro and template usage
template `'foo`(a: string): untyped = (a, 2)
doAssert -12'foo == ("-12", 2)
template `'fooplus`(a: string, b: int): untyped = (a, b)
doAssert -12'fooplus(2) == ("-12", 2)
template `'fooplusopt`(a: string, b: int = 99): untyped = (a, b)
doAssert -12'fooplusopt(2) == ("-12", 2)
doAssert -12'fooplusopt() == ("-12", 99)
doAssert -12'fooplusopt == ("-12", 99)
macro `'bar`(a: static string): untyped =
var infix = newNimNode(nnkInfix)
infix.add newIdentNode("&")
infix.add newLit("got ")
infix.add newLit(a.repr)
result = newNimNode(nnkStmtList)
result.add infix
doAssert -12'bar == "got \"-12\""
macro deb(a): untyped = newLit(a.repr)
doAssert deb(-12'bar) == "-12'bar"
# macro metawrap(): untyped =
# func wrap1(a: string): string = "{" & a & "}"
# func `'wrap2`(a: string): string = "{" & a & "}"
# result = quote do:
# let a1 = wrap1"-128"
# let a2 = -128'wrap2
# metawrap()
# doAssert a1 == "{-128}"
# doAssert a2 == "{-128}"
static: main()
main()

19
tests/lexer/tstrlits.nim
View file

@ -1,19 +0,0 @@
discard """
output: "a\"\"long string\"\"\"\"\"abc\"def_'2'●𝌆𝌆A"
"""
# Test the new different string literals
const
tripleEmpty = """"long string"""""""" # "long string """""
rawQuote = r"a"""
raw = r"abc""def"
escaped = "\x5f'\50'\u25cf\u{1D306}\u{1d306}\u{41}"
stdout.write(rawQuote)
stdout.write(tripleEmpty)
stdout.write(raw)
stdout.writeLine(escaped)

8
tests/lexer/tunary_minus.nim
View file

@ -6,13 +6,7 @@ discard """
import std/[macros, strutils] import std/[macros, strutils]
macro lispReprStr*(a: untyped): untyped = newLit(a.lispRepr) import mlexerutils
macro assertAST*(expected: string, struct: untyped): untyped =
var ast = newLit(struct.treeRepr)
result = quote do:
if `ast` != `expected`:
doAssert false, "\nGot:\n" & `ast`.indent(2) & "\nExpected:\n" & `expected`.indent(2)
const one = 1 const one = 1
const minusOne = `-`(one) const minusOne = `-`(one)