followup custom literals (#17500)

compiler/condsyms.nim

@@ -130,3 +130,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasWarningAsError")
   defineSymbol("nimHasHintAsError")
   defineSymbol("nimHasSpellSuggest")
+  defineSymbol("nimHasCustomLiterals")

compiler/ic/packed_ast.nim

@@ -333,6 +333,7 @@ template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand
 proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
 
 when false:
+  # xxx `nkStrLit` or `nkStrLit..nkTripleStrLit:` below?
   proc strLit*(tree: PackedTree; n: NodePos): lent string =
     assert n.kind == nkStrLit
     result = tree.sh.strings[LitId tree.nodes[n.int].operand]

compiler/lexer.nim

@@ -412,14 +412,12 @@ proc getNumber(L: var Lexer, result: var Token) =
       customLitPossible = true
 
     if L.buf[postPos] in SymChars:
-      var suffixAsLower = newStringOfCap(10)
       var suffix = newStringOfCap(10)
       while true:
-        let c = L.buf[postPos]
+        suffix.add L.buf[postPos]
-        suffix.add c
-        suffixAsLower.add toLowerAscii(c)
         inc postPos
         if L.buf[postPos] notin SymChars+{'_'}: break
+      let suffixAsLower = suffix.toLowerAscii
       case suffixAsLower
       of "f", "f32": result.tokType = tkFloat32Lit
       of "d", "f64": result.tokType = tkFloat64Lit
@@ -433,16 +431,15 @@ proc getNumber(L: var Lexer, result: var Token) =
       of "u16": result.tokType = tkUInt16Lit
       of "u32": result.tokType = tkUInt32Lit
       of "u64": result.tokType = tkUInt64Lit
+      elif customLitPossible:
+        # remember the position of the `'` so that the parser doesn't
+        # have to reparse the custom literal:
+        result.iNumber = len(result.literal)
+        result.literal.add '\''
+        result.literal.add suffix
+        result.tokType = tkCustomLit
       else:
-        if customLitPossible:
+        lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
-          # remember the position of the ``'`` so that the parser doesn't
-          # have to reparse the custom literal:
-          result.iNumber = len(result.literal)
-          result.literal.add '\''
-          result.literal.add suffix
-          result.tokType = tkCustomLit
-        else:
-          lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
     else:
       lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
 
@@ -467,7 +464,7 @@ proc getNumber(L: var Lexer, result: var Token) =
             if L.buf[pos] != '_':
               xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
             inc(pos)
-        # 'c', 'C' is deprecated
+        # 'c', 'C' is deprecated (a warning is issued elsewhere)
         of 'o', 'c', 'C':
           result.base = base8
           while pos < endpos:

doc/manual.rst

@@ -536,7 +536,7 @@ Numeric literals have the form::
   FLOAT64_LIT = HEX_LIT '\'' FLOAT64_SUFFIX
               | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT64_SUFFIX
 
-  CUSTOM_NUMERIC_LIT = (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) '\'' CUSTOM_NUMERIC_SUFFIX
+  CUSTOM_NUMERIC_LIT = (FLOAT_LIT | INT_LIT) '\'' CUSTOM_NUMERIC_SUFFIX
 
   # CUSTOM_NUMERIC_SUFFIX is any Nim identifier that is not
   # a pre-defined type suffix.

tests/lexer/tcustom_numeric_literals.nim

@@ -46,6 +46,7 @@ assertAST dedent """
   -38383839292839283928392839283928392839283.928493849385935898243e-50000'wrap
 
 proc `'wrap`(number: string): string = "[[" & number & "]]"
+proc wrap2(number: string): string = "[[" & number & "]]"
 doAssert lispReprStr(-1'wrap) == """(DotExpr (RStrLit "-1") (Ident "\'wrap"))"""
 
 template main =
@@ -126,25 +127,59 @@ template main =
     doAssert -12'fooplusopt(2) == ("-12", 2)
     doAssert -12'fooplusopt() == ("-12", 99)
     doAssert -12'fooplusopt == ("-12", 99)
-    macro `'bar`(a: static string): untyped =
+    macro `'bar`(a: static string): untyped = newLit(a.repr)
-      var infix = newNimNode(nnkInfix)
+    doAssert -12'bar == "\"-12\""
-      infix.add newIdentNode("&")
-      infix.add newLit("got ")
-      infix.add newLit(a.repr)
-      result = newNimNode(nnkStmtList)
-      result.add infix
-    doAssert -12'bar == "got \"-12\""
     macro deb(a): untyped = newLit(a.repr)
     doAssert deb(-12'bar) == "-12'bar"
-    # macro metawrap(): untyped =
+
-    #   func wrap1(a: string): string = "{" & a & "}"
+  block: # bug 1 from https://github.com/nim-lang/Nim/pull/17020#issuecomment-803193947
-    #   func `'wrap2`(a: string): string = "{" & a & "}"
+    macro deb1(a): untyped = newLit a.repr
-    #   result = quote do:
+    macro deb2(a): untyped = newLit a.lispRepr
-    #     let a1 = wrap1"-128"
+    doAssert deb1(-12'wrap) == "-12'wrap"
-    #     let a2 = -128'wrap2
+    doAssert deb1(-12'nonexistant) == "-12'nonexistant"
-    # metawrap()
+    doAssert deb2(-12'nonexistant) == """(DotExpr (RStrLit "-12") (Ident "\'nonexistant"))"""
-    # doAssert a1 == "{-128}"
+    when false: # xxx bug:
-    # doAssert a2 == "{-128}"
+      # this holds:
+      doAssert deb2(-12.wrap2) == """(DotExpr (IntLit -12) (Sym "wrap2"))"""
+      doAssert deb2(-12'wrap) == """(DotExpr (RStrLit "-12") (Sym "\'wrap"))"""
+      # but instead this should hold:
+      doAssert deb2(-12.wrap2) == """(DotExpr (IntLit -12) (Ident "wrap2"))"""
+      doAssert deb2(-12'wrap) == """(DotExpr (RStrLit "-12") (Ident "\'wrap"))"""
+
+  block: # bug 2 from https://github.com/nim-lang/Nim/pull/17020#issuecomment-803193947
+    template toSuf(`'suf`): untyped =
+      let x = -12'suf
+      x
+    doAssert toSuf(`'wrap`) == "[[-12]]"
+
+  block: # bug 10 from https://github.com/nim-lang/Nim/pull/17020#issuecomment-803193947
+    proc `myecho`(a: auto): auto = a
+    template fn1(): untyped =
+      let a = "abc"
+      -12'wrap
+    template fn2(): untyped =
+      `myecho` -12'wrap
+    template fn3(): untyped =
+      -12'wrap
+    doAssert fn1() == "[[-12]]"
+    doAssert fn2() == "[[-12]]"
+    doAssert fn3() == "[[-12]]"
+
+    when false: # xxx this fails; bug 9 from https://github.com/nim-lang/Nim/pull/17020#issuecomment-803193947
+      #[
+      possible workaround: use `genAst` (https://github.com/nim-lang/Nim/pull/17426) and this:
+      let a3 = `'wrap3`("-128")
+      ]#
+      block:
+        macro metawrap(): untyped =
+          func wrap1(a: string): string = "{" & a & "}"
+          func `'wrap3`(a: string): string = "{" & a & "}"
+          result = quote do:
+            let a1 = wrap1"-128"
+            let a2 = -128'wrap3
+        metawrap()
+        doAssert a1 == "{-128}"
+        doAssert a2 == "{-128}"
 
 static: main()
 main()

tests/lexer/tstrlits.nim

@@ -0,0 +1,19 @@
+discard """
+  output: "a\"\"long string\"\"\"\"\"abc\"def_'2'●𝌆𝌆A"
+"""
+# Test the new different string literals
+
+const
+  tripleEmpty = """"long string"""""""" # "long string """""
+
+  rawQuote = r"a"""
+
+  raw = r"abc""def"
+
+  escaped = "\x5f'\50'\u25cf\u{1D306}\u{1d306}\u{41}"
+
+
+stdout.write(rawQuote)
+stdout.write(tripleEmpty)
+stdout.write(raw)
+stdout.writeLine(escaped)

tests/lexer/tunary_minus.nim

@@ -51,8 +51,17 @@ template main =
     doAssert x() == minusOne:
       "unable to handle negatives after semi-colon"
 
+  block:
     doAssert -0b111 == -7
     doAssert -0xff == -255
+    doAssert -128'i8 == (-128).int8
+    doAssert $(-128'i8) == "-128"
+    doAssert -32768'i16 == int16.low
+    doAssert -2147483648'i32 == int32.low
+    when int.sizeof > 4:
+      doAssert -9223372036854775808 == int.low
+    when not defined(js):
+      doAssert -9223372036854775808 == int64.low
 
   block: # check when a minus (-) is an unary op
     doAssert -one == minusOne:
@@ -68,6 +77,5 @@ template main =
     doAssert 4 - one == 3:
       "unable to handle subtraction with surrounding spaces with an identifier"
 
-
 static: main()
 main()