Named arguments in commands + many grammar fixes (#20994)

* Breaking parser changes, implement https://github.com/nim-lang/RFCs/issues/442

Types are separated from expressions and better reflected in the grammar.

* add test

* more accurate grammar

* fix keyword typedescs

* accept expressions in proc argument lists

* CI "fixes"

* fixes

* allow full ref expressions again, adapt old tests

* cleanup, fix some tests

* improve grammar, try and revert semtypes change

* restrict sigil binding to identOrLiteral

* fix, should have caught this immediately

* add changelog entry, fix double not nil bug

* correct grammar

* change section

* fix

* real fix hopefully

* fix test

* support LL(1) for tuples

* make grammar.txt too

tests/assert/tassert2.nim

@@ -24,7 +24,7 @@ except AssertionDefect as e:
 try:
   assert false # assert test with no msg
 except AssertionDefect as e:
-  assert e.msg.endsWith "tassert2.nim(25, 10) `false` "
+  assert e.msg.endsWith "tassert2.nim(25, 3) `false` "
 
 try:
   let a = 1
@@ -100,7 +100,7 @@ block: ## checks for issue https://github.com/nim-lang/Nim/issues/9301
     doAssert 1 + 1 == 3
   except AssertionDefect as e:
     # used to const fold as false
-    assert e.msg.endsWith "tassert2.nim(100, 14) `1 + 1 == 3` "
+    assert e.msg.endsWith "tassert2.nim(100, 5) `1 + 1 == 3` "
 
 block: ## checks AST isn't transformed as it used to
   let a = 1
@@ -108,4 +108,4 @@ block: ## checks AST isn't transformed as it used to
     doAssert a > 1
   except AssertionDefect as e:
     # used to rewrite as `1 < a`
-    assert e.msg.endsWith "tassert2.nim(108, 14) `a > 1` "
+    assert e.msg.endsWith "tassert2.nim(108, 5) `a > 1` "

tests/effects/tdiagnostic_messages.nim

@@ -12,7 +12,7 @@ tdiagnostic_messages.nim(36, 6) Error: 'a' can have side effects
 >>> tdiagnostic_messages.nim(32, 33) Hint: 'callWithSideEffects' calls `.sideEffect` 'indirectCallViaPointer'
 >>>> tdiagnostic_messages.nim(27, 6) Hint: 'indirectCallViaPointer' called by 'callWithSideEffects'
 >>>>> tdiagnostic_messages.nim(28, 32) Hint: 'indirectCallViaPointer' calls routine via pointer indirection
->>> tdiagnostic_messages.nim(33, 10) Hint: 'callWithSideEffects' calls `.sideEffect` 'myEcho'
+>>> tdiagnostic_messages.nim(33, 3) Hint: 'callWithSideEffects' calls `.sideEffect` 'myEcho'
 >>>> tdiagnostic_messages.nim(24, 6) Hint: 'myEcho' called by 'callWithSideEffects'
 >>> tdiagnostic_messages.nim(34, 3) Hint: 'callWithSideEffects' accesses global state 'globalVar'
 >>>> tdiagnostic_messages.nim(23, 5) Hint: 'globalVar' accessed by 'callWithSideEffects'

tests/effects/teffects1.nim

@@ -17,7 +17,7 @@ proc forw: int {. .}
 proc lier(): int {.raises: [IO2Error].} = #[tt.Hint
                             ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]#
   writeLine stdout, "arg" #[tt.Error
-            ^ writeLine stdout, ["arg"] can raise an unlisted exception: ref IOError ]#
+  ^ writeLine stdout, ["arg"] can raise an unlisted exception: ref IOError ]#
 
 proc forw: int =
   raise newException(IOError, "arg")

tests/errmsgs/t9768.nim

@@ -1,5 +1,5 @@
 discard """
-  errormsg: "unhandled exception: t9768.nim(24, 12) `a < 4`  [AssertionDefect]"
+  errormsg: "unhandled exception: t9768.nim(24, 3) `a < 4`  [AssertionDefect]"
   file: "std/assertions.nim"
   nimout: '''
 stack trace: (most recent call last)

tests/errmsgs/tsigmatch2.nim

@@ -14,7 +14,7 @@ proc foo(i: Foo): string
 
 expression: foo(1.2)
 tsigmatch2.nim(40, 14) Error: expression '' has no type (or is ambiguous)
-tsigmatch2.nim(46, 7) Error: type mismatch: got <int literal(1)>
+tsigmatch2.nim(46, 3) Error: type mismatch: got <int literal(1)>
 but expected one of:
 proc foo(args: varargs[string, myproc])
   first type mismatch at position: 1
@@ -44,4 +44,4 @@ block:
     let temp = 12.isNil
   proc foo(args: varargs[string, myproc]) = discard
   foo 1
-static: echo "done"
+static: echo "done"

tests/parser/tcommand_as_expr.nim

@@ -36,3 +36,12 @@ echo f -4
 
 echo int -1 # doesn't compile
 echo int `-` 1 # compiles
+
+var num = 1
+num += int 2
+doAssert num == 3
+
+import options
+var opt = some some none int
+opt = some some none int
+opt = some none Option[int]

tests/parser/tcommandequals.nim

@@ -0,0 +1,17 @@
+discard """
+  output: '''
+5
+'''
+"""
+
+proc foo(a, b: int) =
+  echo a + b
+
+foo a = 2, b = 3
+
+import macros
+
+macro bar(args: varargs[untyped]): untyped =
+  doAssert args[0].kind == nnkExprEqExpr
+
+bar "a" = 1

tests/parser/tcommandindent.nim

@@ -0,0 +1,16 @@
+when false: # parse the following
+  let foo = Obj(
+    field1: proc (src: pointer, srcLen: Natural)
+                    {.nimcall, gcsafe, raises: [IOError, Defect].} =
+      var file = FileOutputStream(s).file
+
+      implementWrites s.buffers, src, srcLen, "FILE",
+                      writeStartAddr, writeLen,
+        file.writeBuffer(writeStartAddr, writeLen)
+    ,
+    field2: proc {.nimcall, gcsafe, raises: [IOError, Defect].} =
+      flushFile FileOutputStream(s).file
+    ,
+    field3: proc () {.nimcall, gcsafe, raises: [IOError, Defect].} =
+      close FileOutputStream(s).file
+  )

tests/parser/tdoublenotnil.nim

@@ -0,0 +1,3 @@
+when false:
+  type Foo = Bar not nil not nil #[tt.Error
+                         ^ invalid indentation]#

tests/parser/ttypeexprobject.nim

@@ -0,0 +1,10 @@
+discard """
+  errormsg: "invalid indentation"
+  line: 10
+  column: 14
+"""
+
+type
+  A = (object | tuple | int)
+  B = int | object | tuple
+  C = object | tuple | int # issue #8846

tests/parser/ttypeexprs.nim

@@ -0,0 +1,25 @@
+proc foo[T: ptr int | ptr string](x: T) = discard
+var x = "abc"
+foo(addr x)
+
+let n = 3'u32
+type Double = (
+  when n.sizeof == 4: uint64
+  elif n.sizeof == 2: uint32
+  else: uint16
+)
+
+type
+  A = (ref | ptr | pointer)
+  B = pointer | ptr | ref
+  C = ref | ptr | pointer
+
+template `+`(a, b): untyped = (b, a)
+template `*`(a, b): untyped = (a, b)
+
+doAssert (ref int + ref float * ref string + ref bool) is
+  (ref bool, ((ref float, ref string), ref int))
+type X = ref int + ref float * ref string + ref bool
+doAssert X is (ref bool, ((ref float, ref string), ref int))
+
+type SomePointer = proc | ref | ptr | pointer

tests/system/tvarargslen.nim

@@ -1,8 +1,8 @@
 discard """
   output: '''
-tvarargslen.nim:35:9 (1, 2)
-tvarargslen.nim:36:9 12
-tvarargslen.nim:37:9 1
+tvarargslen.nim:35:2 (1, 2)
+tvarargslen.nim:36:2 12
+tvarargslen.nim:37:2 1
 tvarargslen.nim:38:8 
 done
 '''

tests/typerel/t7600_1.nim

@@ -1,6 +1,6 @@
 discard """
 errormsg: "type mismatch: got <Thin[system.int]>"
-nimout: '''t7600_1.nim(21, 6) Error: type mismatch: got <Thin[system.int]>
+nimout: '''t7600_1.nim(21, 1) Error: type mismatch: got <Thin[system.int]>
 but expected one of:
 proc test[T](x: Paper[T])
   first type mismatch at position: 1

tests/typerel/t7600_2.nim

@@ -1,6 +1,6 @@
 discard """
 errormsg: "type mismatch: got <Thin>"
-nimout: '''t7600_2.nim(20, 6) Error: type mismatch: got <Thin>
+nimout: '''t7600_2.nim(20, 1) Error: type mismatch: got <Thin>
 but expected one of:
 proc test(x: Paper)
   first type mismatch at position: 1

tests/varres/tprevent_forloopvar_mutations.nim

@@ -1,6 +1,6 @@
 discard """
   errormsg: "type mismatch: got <int>"
-  nimout: '''tprevent_forloopvar_mutations.nim(16, 7) Error: type mismatch: got <int>
+  nimout: '''tprevent_forloopvar_mutations.nim(16, 3) Error: type mismatch: got <int>
 but expected one of:
 proc inc[T, V: Ordinal](x: var T; y: V = 1)
   first type mismatch at position: 1