custom integer literals (#17489)

* user defined integer literals; refs #17020
* updated renderer.nim
* use mlexerutils helper
* imported all test cases from https://github.com/nim-lang/Nim/pull/17020
* final grammar updated

changelog.md

@@ -266,6 +266,8 @@
 - The unary minus in `-1` is now part of the integer literal, it is now parsed as a single token.
   This implies that edge cases like `-128'i8` finally work correctly.
 
+- Custom numeric literals are now supported.
+
 
 ## Compiler changes
 

compiler/docgen.nim

@@ -416,7 +416,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
     of tkOpr:
       dispA(d.conf, result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}",
             [escLit])
-    of tkStrLit..tkTripleStrLit:
+    of tkStrLit..tkTripleStrLit, tkCustomLit:
       dispA(d.conf, result, "<span class=\"StringLit\">$1</span>",
             "\\spanStringLit{$1}", [escLit])
     of tkCharLit:

compiler/lexer.nim

@@ -60,6 +60,7 @@ type
     tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit",
     tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit",
     tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit",
+    tkCustomLit = "tkCustomLit",
 
     tkParLe = "(", tkParRi = ")", tkBracketLe = "[",
     tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}",
@@ -313,8 +314,7 @@ proc getNumber(L: var Lexer, result: var Token) =
 
   proc lexMessageLitNum(L: var Lexer, msg: string, startpos: int, msgKind = errGenerated) =
     # Used to get slightly human friendlier err messages.
-    const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
+    const literalishChars = {'A'..'Z', 'a'..'z', '0'..'9', '_', '.', '\''}
-      'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
     var msgPos = L.bufpos
     var t: Token
     t.literal = ""
@@ -326,15 +326,14 @@ proc getNumber(L: var Lexer, result: var Token) =
       t.literal.add(L.buf[L.bufpos])
       inc(L.bufpos)
       matchChars(L, t, literalishChars)
-    if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
+    if L.buf[L.bufpos] in literalishChars:
-      inc(L.bufpos)
       t.literal.add(L.buf[L.bufpos])
+      inc(L.bufpos)
       matchChars(L, t, {'0'..'9'})
     L.bufpos = msgPos
     lexMessage(L, msgKind, msg % t.literal)
 
   var
-    startpos, endpos: int
     xi: BiggestInt
     isBase10 = true
     numDigits = 0
@@ -346,7 +345,7 @@ proc getNumber(L: var Lexer, result: var Token) =
   result.tokType = tkIntLit   # int literal until we know better
   result.literal = ""
   result.base = base10
-  startpos = L.bufpos
+  let startpos = L.bufpos
   tokenBegin(result, startpos)
 
   var isPositive = true
@@ -395,77 +394,63 @@ proc getNumber(L: var Lexer, result: var Token) =
       discard matchUnderscoreChars(L, result, {'0'..'9'})
     if L.buf[L.bufpos] in {'e', 'E'}:
       result.tokType = tkFloatLit
-      eatChar(L, result, 'e')
+      eatChar(L, result)
       if L.buf[L.bufpos] in {'+', '-'}:
         eatChar(L, result)
       discard matchUnderscoreChars(L, result, {'0'..'9'})
-  endpos = L.bufpos
+  let endpos = L.bufpos
 
   # Second stage, find out if there's a datatype suffix and handle it
   var postPos = endpos
+
   if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
+    let errPos = postPos
+    var customLitPossible = false
     if L.buf[postPos] == '\'':
       inc(postPos)
+      customLitPossible = true
 
-    case L.buf[postPos]
+    if L.buf[postPos] in SymChars:
-    of 'f', 'F':
+      var suffixAsLower = newStringOfCap(10)
-      inc(postPos)
+      var suffix = newStringOfCap(10)
-      if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
+      while true:
-        result.tokType = tkFloat32Lit
+        let c = L.buf[postPos]
-        inc(postPos, 2)
+        suffix.add c
-      elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
+        suffixAsLower.add toLowerAscii(c)
-        result.tokType = tkFloat64Lit
+        inc postPos
-        inc(postPos, 2)
+        if L.buf[postPos] notin SymChars+{'_'}: break
-      elif (L.buf[postPos] == '1') and
+      case suffix
-           (L.buf[postPos + 1] == '2') and
+      of "f", "f32": result.tokType = tkFloat32Lit
-           (L.buf[postPos + 2] == '8'):
+      of "d", "f64": result.tokType = tkFloat64Lit
-        result.tokType = tkFloat128Lit
+      of "f128": result.tokType = tkFloat128Lit
-        inc(postPos, 3)
+      of "i8": result.tokType = tkInt8Lit
-      else:   # "f" alone defaults to float32
+      of "i16": result.tokType = tkInt16Lit
-        result.tokType = tkFloat32Lit
+      of "i32": result.tokType = tkInt32Lit
-    of 'd', 'D':  # ad hoc convenience shortcut for f64
+      of "i64": result.tokType = tkInt64Lit
-      inc(postPos)
+      of "u": result.tokType = tkUIntLit
-      result.tokType = tkFloat64Lit
+      of "u8": result.tokType = tkUInt8Lit
-    of 'i', 'I':
+      of "u16": result.tokType = tkUInt16Lit
-      inc(postPos)
+      of "u32": result.tokType = tkUInt32Lit
-      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
+      of "u64": result.tokType = tkUInt64Lit
-        result.tokType = tkInt64Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
-        result.tokType = tkInt32Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
-        result.tokType = tkInt16Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '8'):
-        result.tokType = tkInt8Lit
-        inc(postPos)
       else:
-        lexMessageLitNum(L, "invalid number: '$1'", startpos)
+        if customLitPossible:
-    of 'u', 'U':
+          # remember the position of the ``'`` so that the parser doesn't
-      inc(postPos)
+          # have to reparse the custom literal:
-      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
+          result.iNumber = len(result.literal)
-        result.tokType = tkUInt64Lit
+          result.literal.add '\''
-        inc(postPos, 2)
+          result.literal.add suffix
-      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
+          result.tokType = tkCustomLit
-        result.tokType = tkUInt32Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
-        result.tokType = tkUInt16Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '8'):
-        result.tokType = tkUInt8Lit
-        inc(postPos)
         else:
-        result.tokType = tkUIntLit
+          lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
     else:
-      lexMessageLitNum(L, "invalid number: '$1'", startpos)
+      lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
 
   # Is there still a literalish char awaiting? Then it's an error!
   if  L.buf[postPos] in literalishChars or
      (L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
     lexMessageLitNum(L, "invalid number: '$1'", startpos)
 
+  if result.tokType != tkCustomLit:
     # Third stage, extract actual number
     L.bufpos = startpos            # restore position
     var pos = startpos
@@ -830,8 +815,9 @@ proc getString(L: var Lexer, tok: var Token, mode: StringMode) =
         inc(pos)
     L.bufpos = pos
 
-proc getCharacter(L: var Lexer, tok: var Token) =
+proc getCharacter(L: var Lexer; tok: var Token) =
   tokenBegin(tok, L.bufpos)
+  let startPos = L.bufpos
   inc(L.bufpos)               # skip '
   var c = L.buf[L.bufpos]
   case c
@@ -842,10 +828,16 @@ proc getCharacter(L: var Lexer, tok: var Token) =
   else:
     tok.literal = $c
     inc(L.bufpos)
-  if L.buf[L.bufpos] != '\'':
+  if L.buf[L.bufpos] == '\'':
-    lexMessage(L, errGenerated, "missing closing ' for character literal")
     tokenEndIgnore(tok, L.bufpos)
     inc(L.bufpos)               # skip '
+  else:
+    if startPos > 0 and L.buf[startPos-1] == '`':
+      tok.literal = "'"
+      L.bufpos = startPos+1
+    else:
+      lexMessage(L, errGenerated, "missing closing ' for character literal")
+    tokenEndIgnore(tok, L.bufpos)
 
 proc getSymbol(L: var Lexer, tok: var Token) =
   var h: Hash = 0

compiler/parser.nim

@@ -355,7 +355,7 @@ proc parseSymbol(p: var Parser, mode = smNormal): PNode =
         let node = newNodeI(nkIdent, lineinfo)
         node.ident = p.lex.cache.getIdent(accm)
         result.add(node)
-      of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit:
+      of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCustomLit:
         result.add(newIdentNodeP(p.lex.cache.getIdent($p.tok), p))
         getTok(p)
       else:
@@ -627,7 +627,7 @@ proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode =
   #|           | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT
   #|           | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT
   #|           | STR_LIT | RSTR_LIT | TRIPLESTR_LIT
-  #|           | CHAR_LIT
+  #|           | CHAR_LIT | CUSTOM_NUMERIC_LIT
   #|           | NIL
   #| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT
   #| identOrLiteral = generalizedLit | symbol | literal
@@ -710,6 +710,14 @@ proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode =
   of tkCharLit:
     result = newIntNodeP(nkCharLit, ord(p.tok.literal[0]), p)
     getTok(p)
+  of tkCustomLit:
+    let splitPos = p.tok.iNumber.int
+    let str = newStrNodeP(nkRStrLit, p.tok.literal.substr(0, splitPos-1), p)
+    let callee = newIdentNodeP(getIdent(p.lex.cache, p.tok.literal.substr(splitPos)), p)
+    result = newNodeP(nkDotExpr, p)
+    result.add str
+    result.add callee
+    getTok(p)
   of tkNil:
     result = newNodeP(nkNilLit, p)
     getTok(p)
@@ -807,7 +815,7 @@ proc primarySuffix(p: var Parser, r: PNode,
         result = commandExpr(p, result, mode)
         break
       result = namedParams(p, result, nkCurlyExpr, tkCurlyRi)
-    of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast,
+    of tkSymbol, tkAccent, tkIntLit..tkCustomLit, tkNil, tkCast,
        tkOpr, tkDotDot, tkVar, tkOut, tkStatic, tkType, tkEnum, tkTuple,
        tkObject, tkProc:
       # XXX: In type sections we allow the free application of the
@@ -1097,7 +1105,7 @@ proc isExprStart(p: Parser): bool =
   case p.tok.tokType
   of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor,
      tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics,
-     tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr,
+     tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCustomLit, tkVar, tkRef, tkPtr,
      tkTuple, tkObject, tkWhen, tkCase, tkOut:
     result = true
   else: result = false
@@ -1498,7 +1506,7 @@ proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode =
   #| yieldStmt = 'yield' optInd expr?
   #| discardStmt = 'discard' optInd expr?
   #| breakStmt = 'break' optInd expr?
-  #| continueStmt = 'break' optInd expr?
+  #| continueStmt = 'continue' optInd expr?
   result = newNodeP(kind, p)
   getTok(p)
   if p.tok.tokType == tkComment:

compiler/renderer.nim

@@ -942,7 +942,7 @@ proc skipHiddenNodes(n: PNode): PNode =
     else: break
 
 proc accentedName(g: var TSrcGen, n: PNode) =
-  const backticksNeeded = OpChars + {'[', '{'}
+  const backticksNeeded = OpChars + {'[', '{', '\''}
   if n == nil: return
   let isOperator =
     if n.kind == nkIdent and n.ident.s.len > 0 and n.ident.s[0] in backticksNeeded: true
@@ -976,6 +976,11 @@ proc infixArgument(g: var TSrcGen, n: PNode, i: int) =
   if needsParenthesis:
     put(g, tkParRi, ")")
 
+proc isCustomLit(n: PNode): bool =
+  n.len == 2 and n[0].kind == nkRStrLit and
+    (n[1].kind == nkIdent and n[1].ident.s.startsWith('\'')) or
+    (n[1].kind == nkSym and n[1].sym.name.s.startsWith('\''))
+
 proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   if isNil(n): return
   var
@@ -1195,9 +1200,14 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     gcomma(g, n, c)
     put(g, tkBracketRi, "]")
   of nkDotExpr:
+    if isCustomLit(n):
+      put(g, tkCustomLit, n[0].strVal)
+      gsub(g, n, 1)
+    else:
       gsub(g, n, 0)
       put(g, tkDot, ".")
-    gsub(g, n, 1)
+      if n.len > 1:
+        accentedName(g, n[1])
   of nkBind:
     putWithSpace(g, tkBind, "bind")
     gsub(g, n, 0)

doc/grammar.txt

@@ -46,7 +46,7 @@ literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT
           | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT
           | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT
           | STR_LIT | RSTR_LIT | TRIPLESTR_LIT
-          | CHAR_LIT
+          | CHAR_LIT | CUSTOM_NUMERIC_LIT
           | NIL
 generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT
 identOrLiteral = generalizedLit | symbol | literal
@@ -100,6 +100,7 @@ postExprBlocks = ':' stmt? ( IND{=} doBlock
                            | IND{=} 'of' exprList ':' stmt
                            | IND{=} 'elif' expr ':' stmt
                            | IND{=} 'except' exprList ':' stmt
+                           | IND{=} 'finally' ':' stmt
                            | IND{=} 'else' ':' stmt )*
 exprStmt = simpleExpr
          (( '=' optInd expr colonBody? )

doc/manual.rst

@@ -490,11 +490,17 @@ this. Another reason is that Nim can thus support `array[char, int]` or
 type is used for Unicode characters, it can represent any Unicode character.
 `Rune` is declared in the `unicode module <unicode.html>`_.
 
+A character literal that  does not end in ``'`` interpreted as ``'`` if there
+is a preceeding backtick token. There must be no whitespace between the preceeding
+backtick token and the character literal. This special rule ensures that a declaration
+like ``proc `'customLiteral`(s: string)`` is valid. See also
+`Custom Numeric Literals <#custom-numeric-literals>`_.
 
-Numerical constants
--------------------
 
-Numerical constants are of a single type and have the form::
+Numeric Literals
+----------------
+
+Numeric literals have the form::
 
   hexdigit = digit | 'A'..'F' | 'a'..'f'
   octdigit = '0'..'7'
@@ -530,8 +536,10 @@ Numerical constants are of a single type and have the form::
   FLOAT64_LIT = HEX_LIT '\'' FLOAT64_SUFFIX
               | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT64_SUFFIX
 
+  CUSTOM_NUMERIC_LIT = (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) '\'' CUSTOM_NUMERIC_SUFFIX
 
-As can be seen in the productions, numerical constants can contain underscores
+
+As can be seen in the productions, numeric literals can contain underscores
 for readability. Integer and floating-point literals may be given in decimal (no
 prefix), binary (prefix `0b`), octal (prefix `0o`), and hexadecimal
 (prefix `0x`) notation.
@@ -579,7 +587,7 @@ is optional if it is not ambiguous (only hexadecimal floating-point literals
 with a type suffix can be ambiguous).
 
 
-The type suffixes are:
+The pre-defined type suffixes are:
 
 =================    =========================
   Type Suffix        Resulting type of literal
@@ -611,6 +619,42 @@ the bit width of the datatype, it is accepted.
 Hence: 0b10000000'u8 == 0x80'u8 == 128, but, 0b10000000'i8 == 0x80'i8 == -1
 instead of causing an overflow error.
 
+
+Custom Numeric Literals
+~~~~~~~~~~~~~~~~~~~~~~~
+
+If the suffix is not predefined, then the suffix is assumed to be a call
+to a proc, template, macro or other callable identifier that is passed the
+string containing the literal. The callable identifier needs to be declared
+with a special ``'`` prefix:
+
+.. code-block:: nim
+
+  import strutils
+  type u4 = distinct uint8 # a 4-bit unsigned integer aka "nibble"
+  proc `'u4`(n: string): u4 =
+    # The leading ' is required.
+    result = (parseInt(n) and 0x0F).u4
+
+  var x = 5'u4
+
+More formally, a custom numeric literal `123'custom` is transformed
+to r"123".`'custom` in the parsing step. There is no AST node kind that
+corresponds to this transformation. The transformation naturally handles
+the case that additional parameters are passed to the callee:
+
+.. code-block:: nim
+
+  import strutils
+  type u4 = distinct uint8 # a 4-bit unsigned integer aka "nibble"
+  proc `'u4`(n: string; moreData: int): u4 =
+    result = (parseInt(n) and 0x0F).u4
+
+  var x = 5'u4(123)
+
+Custom numeric literals are covered by the grammar rule named `CUSTOM_NUMERIC_LIT`.
+
+
 Operators
 ---------
 

tests/lexer/mlexerutils.nim

@@ -0,0 +1,9 @@
+import macros
+
+macro lispReprStr*(a: untyped): untyped = newLit(a.lispRepr)
+
+macro assertAST*(expected: string, struct: untyped): untyped =
+  var ast = newLit(struct.treeRepr)
+  result = quote do:
+    if `ast` != `expected`:
+      doAssert false, "\nGot:\n" & `ast`.indent(2) & "\nExpected:\n" & `expected`.indent(2)

tests/lexer/tcustom_numeric_literals.nim

@@ -0,0 +1,150 @@
+discard """
+  targets: "c cpp js"
+"""
+
+# Test tkStrNumLit
+
+import std/[macros, strutils]
+import mlexerutils
+
+# AST checks
+
+assertAST dedent """
+  StmtList
+    ProcDef
+      AccQuoted
+        Ident "\'"
+        Ident "wrap"
+      Empty
+      Empty
+      FormalParams
+        Ident "string"
+        IdentDefs
+          Ident "number"
+          Ident "string"
+          Empty
+      Empty
+      Empty
+      StmtList
+        Asgn
+          Ident "result"
+          Infix
+            Ident "&"
+            Infix
+              Ident "&"
+              StrLit "[["
+              Ident "number"
+            StrLit "]]"""":
+  proc `'wrap`(number: string): string =
+    result = "[[" & number & "]]"
+
+assertAST dedent """
+  StmtList
+    DotExpr
+      RStrLit "-38383839292839283928392839283928392839283.928493849385935898243e-50000"
+      Ident "\'wrap"""":
+  -38383839292839283928392839283928392839283.928493849385935898243e-50000'wrap
+
+proc `'wrap`(number: string): string = "[[" & number & "]]"
+doAssert lispReprStr(-1'wrap) == """(DotExpr (RStrLit "-1") (Ident "\'wrap"))"""
+
+template main =
+  block: # basic suffix usage
+    template `'twrap`(number: string): untyped =
+      number.`'wrap`
+    proc extraContext(): string =
+      22.40'wrap
+    proc `*`(left, right: string): string =
+      result = left & "times" & right
+    proc `+`(left, right: string): string =
+      result = left & "plus" & right
+
+    doAssert 1'wrap == "[[1]]"
+    doAssert -1'wrap == "[[-1]]":
+      "unable to resolve a negative integer-suffix pattern"
+    doAssert 12345.67890'wrap == "[[12345.67890]]"
+    doAssert 1'wrap*1'wrap == "[[1]]times[[1]]":
+      "unable to resolve an operator between two suffixed numeric literals"
+    doAssert 1'wrap+ -1'wrap == "[[1]]plus[[-1]]":  # will generate a compiler warning about inconsistent spacing
+      "unable to resolve a negative suffixed numeric literal following an operator"
+    doAssert 1'wrap + -1'wrap == "[[1]]plus[[-1]]"
+    doAssert 1'twrap == "[[1]]"
+    doAssert extraContext() == "[[22.40]]":
+      "unable to return a suffixed numeric literal by an implicit return"
+    doAssert 0x5a3a'wrap == "[[0x5a3a]]"
+    doAssert 0o5732'wrap == "[[0o5732]]"
+    doAssert 0b0101111010101'wrap == "[[0b0101111010101]]"
+    doAssert -38383839292839283928392839283928392839283.928493849385935898243e-50000'wrap == "[[-38383839292839283928392839283928392839283.928493849385935898243e-50000]]"
+    doAssert 1234.56'wrap == "[[1234.56]]":
+      "unable to properly account for context with suffixed numeric literals"
+
+  block: # verify that the i64, f32, etc builtin suffixes still parse correctly
+    const expectedF32: float32 = 123.125
+    proc `'f9`(number: string): string =   # proc starts with 'f' just like 'f32'
+      "[[" & number & "]]"
+    proc `'f32a`(number: string): string =   # looks even more like 'f32'
+      "[[" & number & "]]"
+    proc `'d9`(number: string): string =   # proc starts with 'd' just like the d suffix
+      "[[" & number & "]]"
+    proc `'i9`(number: string): string =   # proc starts with 'i' just like 'i64'
+      "[[" & number & "]]"
+    proc `'u9`(number: string): string =   # proc starts with 'u' just like 'u8'
+      "[[" & number & "]]"
+
+    doAssert 123.125f32 == expectedF32:
+      "failing to support non-quoted legacy f32 floating point suffix"
+    doAssert 123.125'f32 == expectedF32
+    doAssert 123.125e0'f32 == expectedF32
+    doAssert 1234.56'wrap == 1234.56'f9
+    doAssert 1234.56'wrap == 1234.56'f32a
+    doAssert 1234.56'wrap == 1234.56'd9
+    doAssert 1234.56'wrap == 1234.56'i9
+    doAssert 1234.56'wrap == 1234.56'u9
+    doAssert lispReprStr(1234.56'u9) == """(DotExpr (RStrLit "1234.56") (Ident "\'u9"))""":
+      "failed to properly build AST for suffix that starts with u"
+    doAssert -128'i8 == (-128).int8
+
+  block: # case checks
+    doAssert 1E2 == 100:
+      "lexer not handling upper-case exponent"
+    doAssert 1.0E2 == 100.0
+    doAssert 1e2 == 100
+    doAssert 0xdeadBEEF'wrap == "[[0xdeadBEEF]]":
+      "lexer not maintaining original case"
+    doAssert 0.1E12'wrap == "[[0.1E12]]"
+    doAssert 0.0e12'wrap == "[[0.0e12]]"
+    doAssert 0.0e+12'wrap == "[[0.0e+12]]"
+    doAssert 0.0e-12'wrap == "[[0.0e-12]]"
+    doAssert 0e-12'wrap == "[[0e-12]]"
+
+  block: # macro and template usage
+    template `'foo`(a: string): untyped = (a, 2)
+    doAssert -12'foo == ("-12", 2)
+    template `'fooplus`(a: string, b: int): untyped = (a, b)
+    doAssert -12'fooplus(2) == ("-12", 2)
+    template `'fooplusopt`(a: string, b: int = 99): untyped = (a, b)
+    doAssert -12'fooplusopt(2) == ("-12", 2)
+    doAssert -12'fooplusopt() == ("-12", 99)
+    doAssert -12'fooplusopt == ("-12", 99)
+    macro `'bar`(a: static string): untyped =
+      var infix = newNimNode(nnkInfix)
+      infix.add newIdentNode("&")
+      infix.add newLit("got ")
+      infix.add newLit(a.repr)
+      result = newNimNode(nnkStmtList)
+      result.add infix
+    doAssert -12'bar == "got \"-12\""
+    macro deb(a): untyped = newLit(a.repr)
+    doAssert deb(-12'bar) == "-12'bar"
+    # macro metawrap(): untyped =
+    #   func wrap1(a: string): string = "{" & a & "}"
+    #   func `'wrap2`(a: string): string = "{" & a & "}"
+    #   result = quote do:
+    #     let a1 = wrap1"-128"
+    #     let a2 = -128'wrap2
+    # metawrap()
+    # doAssert a1 == "{-128}"
+    # doAssert a2 == "{-128}"
+
+static: main()
+main()

tests/lexer/tstrlits.nim

@@ -1,19 +0,0 @@
-discard """
-  output: "a\"\"long string\"\"\"\"\"abc\"def_'2'●𝌆𝌆A"
-"""
-# Test the new different string literals
-
-const
-  tripleEmpty = """"long string"""""""" # "long string """""
-
-  rawQuote = r"a"""
-
-  raw = r"abc""def"
-
-  escaped = "\x5f'\50'\u25cf\u{1D306}\u{1d306}\u{41}"
-
-
-stdout.write(rawQuote)
-stdout.write(tripleEmpty)
-stdout.write(raw)
-stdout.writeLine(escaped)

tests/lexer/tunary_minus.nim

@@ -6,13 +6,7 @@ discard """
 
 import std/[macros, strutils]
 
-macro lispReprStr*(a: untyped): untyped = newLit(a.lispRepr)
+import mlexerutils
-
-macro assertAST*(expected: string, struct: untyped): untyped =
-  var ast = newLit(struct.treeRepr)
-  result = quote do:
-    if `ast` != `expected`:
-      doAssert false, "\nGot:\n" & `ast`.indent(2) & "\nExpected:\n" & `expected`.indent(2)
 
 const one = 1
 const minusOne = `-`(one)