Named arguments in commands + many grammar fixes (#20994)

* Breaking parser changes, implement https://github.com/nim-lang/RFCs/issues/442

Types are separated from expressions and better reflected in the grammar.

* add test

* more accurate grammar

* fix keyword typedescs

* accept expressions in proc argument lists

* CI "fixes"

* fixes

* allow full ref expressions again, adapt old tests

* cleanup, fix some tests

* improve grammar, try and revert semtypes change

* restrict sigil binding to identOrLiteral

* fix, should have caught this immediately

* add changelog entry, fix double not nil bug

* correct grammar

* change section

* fix

* real fix hopefully

* fix test

* support LL(1) for tuples

* make grammar.txt too

changelog.md

@@ -80,6 +80,11 @@
 - Removed two type pragma syntaxes deprecated since 0.20, namely
   `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`.
 
+- `foo a = b` now means `foo(a = b)` rather than `foo(a) = b`. This is consistent
+  with the existing behavior of `foo a, b = c` meaning `foo(a, b = c)`.
+  This decision was made with the assumption that the old syntax was used rarely;
+  if your code used the old syntax, please be aware of this change.
+
 - [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators
   are no longer experimental.
 

compiler/parser.nim

@@ -79,7 +79,7 @@ type
     smNormal, smAllowNil, smAfterDot
 
   PrimaryMode = enum
-    pmNormal, pmTypeDesc, pmTypeDef, pmSkipSuffix
+    pmNormal, pmTypeDesc, pmTypeDef, pmTrySimple
 
 proc parseAll*(p: var Parser): PNode
 proc closeParser*(p: var Parser)
@@ -286,7 +286,8 @@ proc newIdentNodeP(ident: PIdent, p: Parser): PNode =
 
 proc parseExpr(p: var Parser): PNode
 proc parseStmt(p: var Parser): PNode
-proc parseTypeDesc(p: var Parser): PNode
+proc parseTypeDesc(p: var Parser, fullExpr = false): PNode
+proc parseTypeDefValue(p: var Parser): PNode
 proc parseParamList(p: var Parser, retColon = true): PNode
 
 proc isSigilLike(tok: Token): bool {.inline.} =
@@ -406,15 +407,8 @@ proc parseSymbol(p: var Parser, mode = smNormal): PNode =
     #if not isKeyword(p.tok.tokType): getTok(p)
     result = p.emptyNode
 
-proc colonOrEquals(p: var Parser, a: PNode): PNode =
+proc equals(p: var Parser, a: PNode): PNode =
-  if p.tok.tokType == tkColon:
+  if p.tok.tokType == tkEquals:
-    result = newNodeP(nkExprColonExpr, p)
-    getTok(p)
-    newlineWasSplitting(p)
-    #optInd(p, result)
-    result.add(a)
-    result.add(parseExpr(p))
-  elif p.tok.tokType == tkEquals:
     result = newNodeP(nkExprEqExpr, p)
     getTok(p)
     #optInd(p, result)
@@ -423,6 +417,17 @@ proc colonOrEquals(p: var Parser, a: PNode): PNode =
   else:
     result = a
 
+proc colonOrEquals(p: var Parser, a: PNode): PNode =
+  if p.tok.tokType == tkColon:
+    result = newNodeP(nkExprColonExpr, p)
+    getTok(p)
+    newlineWasSplitting(p)
+    #optInd(p, result)
+    result.add(a)
+    result.add(parseExpr(p))
+  else:
+    result = equals(p, a)
+
 proc exprColonEqExpr(p: var Parser): PNode =
   #| exprColonEqExpr = expr (':'|'=' expr)?
   var a = parseExpr(p)
@@ -431,6 +436,14 @@ proc exprColonEqExpr(p: var Parser): PNode =
   else:
     result = colonOrEquals(p, a)
 
+proc exprEqExpr(p: var Parser): PNode =
+  #| exprEqExpr = expr ('=' expr)?
+  var a = parseExpr(p)
+  if p.tok.tokType == tkDo:
+    result = postExprBlocks(p, a)
+  else:
+    result = equals(p, a)
+
 proc exprList(p: var Parser, endTok: TokType, result: PNode) =
   #| exprList = expr ^+ comma
   when defined(nimpretty):
@@ -802,25 +815,24 @@ proc namedParams(p: var Parser, callee: PNode,
 proc commandParam(p: var Parser, isFirstParam: var bool; mode: PrimaryMode): PNode =
   if mode == pmTypeDesc:
     result = simpleExpr(p, mode)
+  elif not isFirstParam:
+    result = exprEqExpr(p)
   else:
     result = parseExpr(p)
-  if p.tok.tokType == tkDo:
+    if p.tok.tokType == tkDo:
-    result = postExprBlocks(p, result)
+      result = postExprBlocks(p, result)
-  elif p.tok.tokType == tkEquals and not isFirstParam:
-    let lhs = result
-    result = newNodeP(nkExprEqExpr, p)
-    getTok(p)
-    result.add(lhs)
-    result.add(parseExpr(p))
   isFirstParam = false
 
 proc commandExpr(p: var Parser; r: PNode; mode: PrimaryMode): PNode =
-  result = newNodeP(nkCommand, p)
+  if mode == pmTrySimple:
-  result.add(r)
+    result = r
-  var isFirstParam = true
+  else:
-  # progress NOT guaranteed
+    result = newNodeP(nkCommand, p)
-  p.hasProgress = false
+    result.add(r)
-  result.add commandParam(p, isFirstParam, mode)
+    var isFirstParam = true
+    # progress NOT guaranteed
+    p.hasProgress = false
+    result.add commandParam(p, isFirstParam, mode)
 
 proc isDotLike(tok: Token): bool =
   result = tok.tokType == tkOpr and tok.ident.s.len > 1 and
@@ -833,7 +845,7 @@ proc primarySuffix(p: var Parser, r: PNode,
   #|       | DOTLIKEOP optInd symbol generalizedLit?
   #|       | '[' optInd exprColonEqExprList optPar ']'
   #|       | '{' optInd exprColonEqExprList optPar '}'
-  #|       | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr (comma expr)* # command syntax
+  # XXX strong spaces need to be reflected above
   result = r
 
   # progress guaranteed
@@ -844,13 +856,6 @@ proc primarySuffix(p: var Parser, r: PNode,
       # progress guaranteed
       if p.tok.strongSpaceA:
         result = commandExpr(p, result, mode)
-        # type sections allow full command syntax
-        if mode == pmTypeDef:
-          var isFirstParam = false
-          while p.tok.tokType == tkComma:
-            getTok(p)
-            optInd(p, result)
-            result.add(commandParam(p, isFirstParam, mode))
         break
       result = namedParams(p, result, nkCall, tkParRi)
       if result.len > 1 and result[1].kind == nkExprColonExpr:
@@ -891,18 +896,9 @@ proc primarySuffix(p: var Parser, r: PNode,
           # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet
           # solution, but pragmas.nim can't handle that
           result = commandExpr(p, result, mode)
-          if mode == pmTypeDef:
-            var isFirstParam = false
-            while p.tok.tokType == tkComma:
-              getTok(p)
-              optInd(p, result)
-              result.add(commandParam(p, isFirstParam, mode))
         break
     else:
       break
-  # type sections allow post-expr blocks
-  if mode == pmTypeDef:
-    result = postExprBlocks(p, result)
 
 proc parseOperators(p: var Parser, headNode: PNode,
                     limit: int, mode: PrimaryMode): PNode =
@@ -929,7 +925,10 @@ proc parseOperators(p: var Parser, headNode: PNode,
     opPrec = getPrecedence(p.tok)
 
 proc simpleExprAux(p: var Parser, limit: int, mode: PrimaryMode): PNode =
+  var mode = mode
   result = primary(p, mode)
+  if mode == pmTrySimple:
+    mode = pmNormal
   if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)) and
      mode == pmNormal:
     var pragmaExp = newNodeP(nkPragmaExpr, p)
@@ -1010,9 +1009,9 @@ type
 
 proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode =
   #| declColonEquals = identWithPragma (comma identWithPragma)* comma?
-  #|                   (':' optInd typeDesc)? ('=' optInd expr)?
+  #|                   (':' optInd typeDescExpr)? ('=' optInd expr)?
   #| identColonEquals = IDENT (comma IDENT)* comma?
-  #|      (':' optInd typeDesc)? ('=' optInd expr)?)
+  #|      (':' optInd typeDescExpr)? ('=' optInd expr)?)
   var a: PNode
   result = newNodeP(nkIdentDefs, p)
   # progress guaranteed
@@ -1030,7 +1029,7 @@ proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode =
   if p.tok.tokType == tkColon:
     getTok(p)
     optInd(p, result)
-    result.add(parseTypeDesc(p))
+    result.add(parseTypeDesc(p, fullExpr = true))
   else:
     result.add(newNodeP(nkEmpty, p))
     if p.tok.tokType != tkEquals and withBothOptional notin flags:
@@ -1043,9 +1042,10 @@ proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode =
     result.add(newNodeP(nkEmpty, p))
 
 proc parseTuple(p: var Parser, indentAllowed = false): PNode =
-  #| tupleDecl = 'tuple'
+  #| tupleTypeBracket = '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']'
-  #|     '[' optInd  (identColonEquals (comma/semicolon)?)*  optPar ']' |
+  #| tupleType = 'tuple' tupleTypeBracket
-  #|     COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?
+  #| tupleDecl = 'tuple' (tupleTypeBracket /
+  #|     COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?)
   result = newNodeP(nkTupleTy, p)
   getTok(p)
   if p.tok.tokType == tkBracketLe:
@@ -1155,6 +1155,7 @@ proc parseDoBlock(p: var Parser; info: TLineInfo): PNode =
 
 proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode =
   #| routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)?
+  #| routineType = ('proc' | 'iterator') paramListColon pragma?
   # either a proc type or a anonymous proc
   let info = parLineInfo(p)
   let hasSignature = p.tok.tokType in {tkParLe, tkColon} and p.tok.indent < 0
@@ -1179,7 +1180,7 @@ proc isExprStart(p: Parser): bool =
   of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor,
      tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics,
      tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCustomLit, tkVar, tkRef, tkPtr,
-     tkTuple, tkObject, tkWhen, tkCase, tkOut, tkTry, tkBlock:
+     tkEnum, tkTuple, tkObject, tkWhen, tkCase, tkOut, tkTry, tkBlock:
     result = true
   else: result = false
 
@@ -1199,8 +1200,12 @@ proc parseTypeDescKAux(p: var Parser, kind: TNodeKind,
   getTok(p)
   if p.tok.indent != -1 and p.tok.indent <= p.currInd: return
   optInd(p, result)
+  let isTypedef = mode == pmTypeDef and p.tok.tokType in {tkObject, tkTuple}
   if not isOperator(p.tok) and isExprStart(p):
-    result.add(primary(p, mode))
+    if isTypedef:
+      result.add(parseTypeDefValue(p))
+    else:
+      result.add(primary(p, mode))
   if kind == nkDistinctTy and p.tok.tokType == tkSymbol:
     # XXX document this feature!
     var nodeKind: TNodeKind
@@ -1214,11 +1219,13 @@ proc parseTypeDescKAux(p: var Parser, kind: TNodeKind,
     let list = newNodeP(nodeKind, p)
     result.add list
     parseSymbolList(p, list)
+  if mode == pmTypeDef and not isTypedef:
+    result = parseOperators(p, result, -1, mode)
 
 proc parseVarTuple(p: var Parser): PNode
 
 proc parseFor(p: var Parser): PNode =
-  #| forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt
+  #| forStmt = 'for' ((varTuple / identWithPragma) ^+ comma) 'in' expr colcom stmt
   #| forExpr = forStmt
   getTokNoInd(p)
   result = newNodeP(nkForStmt, p)
@@ -1283,11 +1290,18 @@ proc parseObject(p: var Parser): PNode
 proc parseTypeClass(p: var Parser): PNode
 
 proc primary(p: var Parser, mode: PrimaryMode): PNode =
-  #| primary = operatorB primary primarySuffix* |
+  #| simplePrimary = SIGILLIKEOP? identOrLiteral primarySuffix*
-  #|           tupleDecl | routineExpr | enumDecl
+  #| commandStart = &('`'|IDENT|literal|'cast'|'addr'|'type'|'var'|'out'|
-  #|           objectDecl | conceptDecl | ('bind' primary)
+  #|                  'static'|'enum'|'tuple'|'object'|'proc')
-  #|           ('var' | 'out' | 'ref' | 'ptr' | 'distinct') primary
+  #| primary = simplePrimary (commandStart expr)
-  #|         /  prefixOperator* identOrLiteral primarySuffix*
+  #|         / operatorB primary
+  #|         / routineExpr
+  #|         / rawTypeDesc
+  #|         / prefixOperator primary
+  # XXX strong spaces need to be reflected in commandStart
+  # command part is handled in the primarySuffix proc
+
+  # prefix operators:
   if isOperator(p.tok):
     # Note 'sigil like' operators are currently not reflected in the grammar
     # and should be removed for Nim 2.0, I don't think anybody uses them.
@@ -1297,60 +1311,39 @@ proc primary(p: var Parser, mode: PrimaryMode): PNode =
     result.add(a)
     getTok(p)
     optInd(p, a)
-    if isSigil:
+    const identOrLiteralKinds = tkBuiltInMagics + {tkSymbol, tkAccent, tkNil,
-      #XXX prefix operators
+      tkIntLit..tkCustomLit, tkCast, tkOut, tkParLe, tkBracketLe, tkCurlyLe}
+    if isSigil and p.tok.tokType in identOrLiteralKinds:
       let baseInd = p.lex.currLineIndent
-      result.add(primary(p, pmSkipSuffix))
+      result.add(identOrLiteral(p, mode))
       result = primarySuffix(p, result, baseInd, mode)
     else:
       result.add(primary(p, pmNormal))
     return
 
   case p.tok.tokType
-  of tkTuple: result = parseTuple(p, mode == pmTypeDef)
   of tkProc:
     getTok(p)
-    result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}, nkLambda)
+    result = parseProcExpr(p, mode != pmTypeDesc, nkLambda)
   of tkFunc:
     getTok(p)
-    result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}, nkFuncDef)
+    result = parseProcExpr(p, mode != pmTypeDesc, nkFuncDef)
   of tkIterator:
     getTok(p)
-    result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}, nkIteratorDef)
+    result = parseProcExpr(p, mode != pmTypeDesc, nkIteratorDef)
-  of tkEnum:
-    if mode == pmTypeDef:
-      prettySection:
-        result = parseEnum(p)
-    else:
-      result = newNodeP(nkEnumTy, p)
-      getTok(p)
-  of tkObject:
-    if mode == pmTypeDef:
-      prettySection:
-        result = parseObject(p)
-    else:
-      result = newNodeP(nkObjectTy, p)
-      getTok(p)
-  of tkConcept:
-    if mode == pmTypeDef:
-      result = parseTypeClass(p)
-    else:
-      parMessage(p, "the 'concept' keyword is only valid in 'type' sections")
   of tkBind:
+    # legacy syntax, no-op in current nim
     result = newNodeP(nkBind, p)
     getTok(p)
     optInd(p, result)
     result.add(primary(p, pmNormal))
-  of tkVar: result = parseTypeDescKAux(p, nkVarTy, mode)
+  of tkTuple, tkEnum, tkObject, tkConcept,
-  of tkOut: result = parseTypeDescKAux(p, nkOutTy, mode)
+    tkVar, tkOut, tkRef, tkPtr, tkDistinct:
-  of tkRef: result = parseTypeDescKAux(p, nkRefTy, mode)
+    result = parseTypeDesc(p)
-  of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, mode)
-  of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, mode)
   else:
     let baseInd = p.lex.currLineIndent
     result = identOrLiteral(p, mode)
-    if mode != pmSkipSuffix:
+    result = primarySuffix(p, result, baseInd, mode)
-      result = primarySuffix(p, result, baseInd, mode)
 
 proc binaryNot(p: var Parser; a: PNode): PNode =
   if p.tok.tokType == tkNot:
@@ -1365,16 +1358,70 @@ proc binaryNot(p: var Parser; a: PNode): PNode =
   else:
     result = a
 
-proc parseTypeDesc(p: var Parser): PNode =
+proc parseTypeDesc(p: var Parser, fullExpr = false): PNode =
-  #| typeDesc = simpleExpr ('not' expr)?
+  #| rawTypeDesc = (tupleType | routineType | 'enum' | 'object' |
+  #|                 ('var' | 'out' | 'ref' | 'ptr' | 'distinct') typeDesc?)
+  #|                 ('not' expr)?
+  #| typeDescExpr = (routineType / simpleExpr) ('not' expr)?
+  #| typeDesc = rawTypeDesc / typeDescExpr
   newlineWasSplitting(p)
-  result = simpleExpr(p, pmTypeDesc)
+  if fullExpr:
+    result = simpleExpr(p, pmTypeDesc)
+  else:
+    case p.tok.tokType
+    of tkTuple:
+      result = parseTuple(p, false)
+    of tkProc:
+      getTok(p)
+      result = parseProcExpr(p, false, nkLambda)
+    of tkIterator:
+      getTok(p)
+      result = parseProcExpr(p, false, nkIteratorDef)
+    of tkEnum:
+      result = newNodeP(nkEnumTy, p)
+      getTok(p)
+    of tkObject:
+      result = newNodeP(nkObjectTy, p)
+      getTok(p)
+    of tkConcept:
+      parMessage(p, "the 'concept' keyword is only valid in 'type' sections")
+    of tkVar: result = parseTypeDescKAux(p, nkVarTy, pmTypeDesc)
+    of tkOut: result = parseTypeDescKAux(p, nkOutTy, pmTypeDesc)
+    of tkRef: result = parseTypeDescKAux(p, nkRefTy, pmTypeDesc)
+    of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, pmTypeDesc)
+    of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, pmTypeDesc)
+    else:
+      result = simpleExpr(p, pmTypeDesc)
   result = binaryNot(p, result)
 
-proc parseTypeDefAux(p: var Parser): PNode =
+proc parseTypeDefValue(p: var Parser): PNode =
-  #| typeDefAux = simpleExpr ('not' expr
+  #| typeDefValue = ((tupleDecl | enumDecl | objectDecl | conceptDecl |
-  #|                         | postExprBlocks)?
+  #|                  ('ref' | 'ptr' | 'distinct') (tupleDecl | objectDecl))
-  result = simpleExpr(p, pmTypeDef)
+  #|                / (simpleExpr (exprEqExpr ^+ comma postExprBlocks)?))
+  #|                ('not' expr)?
+  case p.tok.tokType
+  of tkTuple: result = parseTuple(p, true)
+  of tkRef: result = parseTypeDescKAux(p, nkRefTy, pmTypeDef)
+  of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, pmTypeDef)
+  of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, pmTypeDef)
+  of tkEnum:
+    prettySection:
+      result = parseEnum(p)
+  of tkObject:
+    prettySection:
+      result = parseObject(p)
+  of tkConcept:
+    result = parseTypeClass(p)
+  else:
+    result = simpleExpr(p, pmTypeDef)
+    if p.tok.tokType != tkNot:
+      if result.kind == nkCommand:
+        var isFirstParam = false
+        while p.tok.tokType == tkComma:
+          getTok(p)
+          optInd(p, result)
+          result.add(commandParam(p, isFirstParam, pmTypeDef))
+      result = postExprBlocks(p, result)
   result = binaryNot(p, result)
 
 proc makeCall(n: PNode): PNode =
@@ -1467,12 +1514,10 @@ proc postExprBlocks(p: var Parser, x: PNode): PNode =
       parMessage(p, "expected ':'")
 
 proc parseExprStmt(p: var Parser): PNode =
-  #| exprStmt = simpleExpr
+  #| exprStmt = simpleExpr postExprBlocks?
-  #|          (( '=' optInd expr colonBody? )
+  #|          / simplePrimary (exprEqExpr ^+ comma) postExprBlocks?
-  #|          / ( expr ^+ comma
+  #|          / simpleExpr '=' optInd (expr postExprBlocks?)
-  #|              postExprBlocks
+  var a = simpleExpr(p, pmTrySimple)
-  #|            ))?
-  var a = simpleExpr(p)
   if p.tok.tokType == tkEquals:
     result = newNodeP(nkAsgn, p)
     getTok(p)
@@ -1482,20 +1527,17 @@ proc parseExprStmt(p: var Parser): PNode =
     result.add(a)
     result.add(b)
   else:
-    # simpleExpr parsed 'p a' from 'p a, b'?
     var isFirstParam = false
-    if p.tok.indent < 0 and p.tok.tokType == tkComma and a.kind == nkCommand:
+    # if an expression is starting here, a simplePrimary was parsed and
-      result = a
+    # this is the start of a command
-      while true:
+    if p.tok.indent < 0 and isExprStart(p):
-        getTok(p)
-        optInd(p, result)
-        result.add(commandParam(p, isFirstParam, pmNormal))
-        if p.tok.tokType != tkComma: break
-    elif p.tok.indent < 0 and isExprStart(p):
       result = newTreeI(nkCommand, a.info, a)
+      let baseIndent = p.currInd
       while true:
         result.add(commandParam(p, isFirstParam, pmNormal))
-        if p.tok.tokType != tkComma: break
+        if p.tok.tokType != tkComma or
+          (p.tok.indent >= 0 and p.tok.indent < baseIndent):
+          break
         getTok(p)
         optInd(p, result)
     else:
@@ -2132,7 +2174,7 @@ proc parseTypeClass(p: var Parser): PNode =
 
 proc parseTypeDef(p: var Parser): PNode =
   #|
-  #| typeDef = identVisDot genericParamList? pragma '=' optInd typeDefAux
+  #| typeDef = identVisDot genericParamList? pragma '=' optInd typeDefValue
   #|             indAndComment?
   result = newNodeP(nkTypeDef, p)
   var identifier = identVis(p, allowDot=true)
@@ -2158,7 +2200,7 @@ proc parseTypeDef(p: var Parser): PNode =
     result.info = parLineInfo(p)
     getTok(p)
     optInd(p, result)
-    result.add(parseTypeDefAux(p))
+    result.add(parseTypeDefValue(p))
   else:
     result.add(p.emptyNode)
   indAndComment(p, result)    # special extension!

doc/grammar.txt

@@ -28,6 +28,7 @@ operatorB = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 |
 symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`'
        | IDENT | KEYW
 exprColonEqExpr = expr (':'|'=' expr)?
+exprEqExpr = expr ('=' expr)?
 exprList = expr ^+ comma
 exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)?
 qualifiedIdent = symbol ('.' optInd symbol)?
@@ -60,25 +61,26 @@ primarySuffix = '(' (exprColonEqExpr comma?)* ')'
       | DOTLIKEOP optInd symbol generalizedLit?
       | '[' optInd exprColonEqExprList optPar ']'
       | '{' optInd exprColonEqExprList optPar '}'
-      | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr (comma expr)* # command syntax
 pragma = '{.' optInd (exprColonEqExpr comma?)* optPar ('.}' | '}')
 identVis = symbol OPR?  # postfix position
 identVisDot = symbol '.' optInd symbol OPR?
 identWithPragma = identVis pragma?
 identWithPragmaDot = identVisDot pragma?
 declColonEquals = identWithPragma (comma identWithPragma)* comma?
-                  (':' optInd typeDesc)? ('=' optInd expr)?
+                  (':' optInd typeDescExpr)? ('=' optInd expr)?
 identColonEquals = IDENT (comma IDENT)* comma?
-     (':' optInd typeDesc)? ('=' optInd expr)?)
+     (':' optInd typeDescExpr)? ('=' optInd expr)?)
-tupleDecl = 'tuple'
+tupleTypeBracket = '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']'
-    '[' optInd  (identColonEquals (comma/semicolon)?)*  optPar ']' |
+tupleType = 'tuple' tupleTypeBracket
-    COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?
+tupleDecl = 'tuple' (tupleTypeBracket /
+    COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?)
 paramList = '(' declColonEquals ^* (comma/semicolon) ')'
 paramListArrow = paramList? ('->' optInd typeDesc)?
 paramListColon = paramList? (':' optInd typeDesc)?
 doBlock = 'do' paramListArrow pragma? colcom stmt
 routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)?
-forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt
+routineType = ('proc' | 'iterator') paramListColon pragma?
+forStmt = 'for' ((varTuple / identWithPragma) ^+ comma) 'in' expr colcom stmt
 forExpr = forStmt
 expr = (blockExpr
       | ifExpr
@@ -87,25 +89,32 @@ expr = (blockExpr
       | forExpr
       | tryExpr)
       / simpleExpr
-primary = operatorB primary primarySuffix* |
+simplePrimary = SIGILLIKEOP? identOrLiteral primarySuffix*
-          tupleDecl | routineExpr | enumDecl
+commandStart = &('`'|IDENT|literal|'cast'|'addr'|'type'|'var'|'out'|
-          objectDecl | conceptDecl | ('bind' primary)
+                 'static'|'enum'|'tuple'|'object'|'proc')
-          ('var' | 'out' | 'ref' | 'ptr' | 'distinct') primary
+primary = simplePrimary (commandStart expr)
-        /  prefixOperator* identOrLiteral primarySuffix*
+        / operatorB primary
-typeDesc = simpleExpr ('not' expr)?
+        / routineExpr
-typeDefAux = simpleExpr ('not' expr
+        / rawTypeDesc
-                        | postExprBlocks)?
+        / prefixOperator primary
+rawTypeDesc = (tupleType | routineType | 'enum' | 'object' |
+                ('var' | 'out' | 'ref' | 'ptr' | 'distinct') typeDesc?)
+                ('not' expr)?
+typeDescExpr = (routineType / simpleExpr) ('not' expr)?
+typeDesc = rawTypeDesc / typeDescExpr
+typeDefValue = ((tupleDecl | enumDecl | objectDecl | conceptDecl |
+                 ('ref' | 'ptr' | 'distinct') (tupleDecl | objectDecl))
+               / (simpleExpr (exprEqExpr ^+ comma postExprBlocks)?))
+               ('not' expr)?
 postExprBlocks = ':' stmt? ( IND{=} doBlock
                            | IND{=} 'of' exprList ':' stmt
                            | IND{=} 'elif' expr ':' stmt
                            | IND{=} 'except' exprList ':' stmt
                            | IND{=} 'finally' ':' stmt
                            | IND{=} 'else' ':' stmt )*
-exprStmt = simpleExpr
+exprStmt = simpleExpr postExprBlocks?
-         (( '=' optInd expr colonBody? )
+         / simplePrimary (exprEqExpr ^+ comma) postExprBlocks?
-         / ( expr ^+ comma
+         / simpleExpr '=' optInd (expr postExprBlocks?)
-             postExprBlocks
-           ))?
 importStmt = 'import' optInd expr
               ((comma expr)*
               / 'except' optInd (expr ^+ comma))
@@ -175,7 +184,7 @@ objectDecl = 'object' ('of' typeDesc)? COMMENT? objectPart
 conceptParam = ('var' | 'out')? symbol
 conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
               &IND{>} stmt
-typeDef = identVisDot genericParamList? pragma '=' optInd typeDefAux
+typeDef = identVisDot genericParamList? pragma '=' optInd typeDefValue
             indAndComment?
 varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr
 colonBody = colcom stmt postExprBlocks?

tests/assert/tassert2.nim

@@ -24,7 +24,7 @@ except AssertionDefect as e:
 try:
   assert false # assert test with no msg
 except AssertionDefect as e:
-  assert e.msg.endsWith "tassert2.nim(25, 10) `false` "
+  assert e.msg.endsWith "tassert2.nim(25, 3) `false` "
 
 try:
   let a = 1
@@ -100,7 +100,7 @@ block: ## checks for issue https://github.com/nim-lang/Nim/issues/9301
     doAssert 1 + 1 == 3
   except AssertionDefect as e:
     # used to const fold as false
-    assert e.msg.endsWith "tassert2.nim(100, 14) `1 + 1 == 3` "
+    assert e.msg.endsWith "tassert2.nim(100, 5) `1 + 1 == 3` "
 
 block: ## checks AST isn't transformed as it used to
   let a = 1
@@ -108,4 +108,4 @@ block: ## checks AST isn't transformed as it used to
     doAssert a > 1
   except AssertionDefect as e:
     # used to rewrite as `1 < a`
-    assert e.msg.endsWith "tassert2.nim(108, 14) `a > 1` "
+    assert e.msg.endsWith "tassert2.nim(108, 5) `a > 1` "

tests/effects/tdiagnostic_messages.nim

@@ -12,7 +12,7 @@ tdiagnostic_messages.nim(36, 6) Error: 'a' can have side effects
 >>> tdiagnostic_messages.nim(32, 33) Hint: 'callWithSideEffects' calls `.sideEffect` 'indirectCallViaPointer'
 >>>> tdiagnostic_messages.nim(27, 6) Hint: 'indirectCallViaPointer' called by 'callWithSideEffects'
 >>>>> tdiagnostic_messages.nim(28, 32) Hint: 'indirectCallViaPointer' calls routine via pointer indirection
->>> tdiagnostic_messages.nim(33, 10) Hint: 'callWithSideEffects' calls `.sideEffect` 'myEcho'
+>>> tdiagnostic_messages.nim(33, 3) Hint: 'callWithSideEffects' calls `.sideEffect` 'myEcho'
 >>>> tdiagnostic_messages.nim(24, 6) Hint: 'myEcho' called by 'callWithSideEffects'
 >>> tdiagnostic_messages.nim(34, 3) Hint: 'callWithSideEffects' accesses global state 'globalVar'
 >>>> tdiagnostic_messages.nim(23, 5) Hint: 'globalVar' accessed by 'callWithSideEffects'

tests/effects/teffects1.nim

@@ -17,7 +17,7 @@ proc forw: int {. .}
 proc lier(): int {.raises: [IO2Error].} = #[tt.Hint
                             ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]#
   writeLine stdout, "arg" #[tt.Error
-            ^ writeLine stdout, ["arg"] can raise an unlisted exception: ref IOError ]#
+  ^ writeLine stdout, ["arg"] can raise an unlisted exception: ref IOError ]#
 
 proc forw: int =
   raise newException(IOError, "arg")

tests/errmsgs/t9768.nim

@@ -1,5 +1,5 @@
 discard """
-  errormsg: "unhandled exception: t9768.nim(24, 12) `a < 4`  [AssertionDefect]"
+  errormsg: "unhandled exception: t9768.nim(24, 3) `a < 4`  [AssertionDefect]"
   file: "std/assertions.nim"
   nimout: '''
 stack trace: (most recent call last)

tests/errmsgs/tsigmatch2.nim

@@ -14,7 +14,7 @@ proc foo(i: Foo): string
 
 expression: foo(1.2)
 tsigmatch2.nim(40, 14) Error: expression '' has no type (or is ambiguous)
-tsigmatch2.nim(46, 7) Error: type mismatch: got <int literal(1)>
+tsigmatch2.nim(46, 3) Error: type mismatch: got <int literal(1)>
 but expected one of:
 proc foo(args: varargs[string, myproc])
   first type mismatch at position: 1
@@ -44,4 +44,4 @@ block:
     let temp = 12.isNil
   proc foo(args: varargs[string, myproc]) = discard
   foo 1
-static: echo "done"
+static: echo "done"

tests/parser/tcommand_as_expr.nim

@@ -36,3 +36,12 @@ echo f -4
 
 echo int -1 # doesn't compile
 echo int `-` 1 # compiles
+
+var num = 1
+num += int 2
+doAssert num == 3
+
+import options
+var opt = some some none int
+opt = some some none int
+opt = some none Option[int]

tests/parser/tcommandequals.nim

@@ -0,0 +1,17 @@
+discard """
+  output: '''
+5
+'''
+"""
+
+proc foo(a, b: int) =
+  echo a + b
+
+foo a = 2, b = 3
+
+import macros
+
+macro bar(args: varargs[untyped]): untyped =
+  doAssert args[0].kind == nnkExprEqExpr
+
+bar "a" = 1

tests/parser/tcommandindent.nim

@@ -0,0 +1,16 @@
+when false: # parse the following
+  let foo = Obj(
+    field1: proc (src: pointer, srcLen: Natural)
+                    {.nimcall, gcsafe, raises: [IOError, Defect].} =
+      var file = FileOutputStream(s).file
+
+      implementWrites s.buffers, src, srcLen, "FILE",
+                      writeStartAddr, writeLen,
+        file.writeBuffer(writeStartAddr, writeLen)
+    ,
+    field2: proc {.nimcall, gcsafe, raises: [IOError, Defect].} =
+      flushFile FileOutputStream(s).file
+    ,
+    field3: proc () {.nimcall, gcsafe, raises: [IOError, Defect].} =
+      close FileOutputStream(s).file
+  )

tests/parser/tdoublenotnil.nim

@@ -0,0 +1,3 @@
+when false:
+  type Foo = Bar not nil not nil #[tt.Error
+                         ^ invalid indentation]#

tests/parser/ttypeexprobject.nim

@@ -0,0 +1,10 @@
+discard """
+  errormsg: "invalid indentation"
+  line: 10
+  column: 14
+"""
+
+type
+  A = (object | tuple | int)
+  B = int | object | tuple
+  C = object | tuple | int # issue #8846

tests/parser/ttypeexprs.nim

@@ -0,0 +1,25 @@
+proc foo[T: ptr int | ptr string](x: T) = discard
+var x = "abc"
+foo(addr x)
+
+let n = 3'u32
+type Double = (
+  when n.sizeof == 4: uint64
+  elif n.sizeof == 2: uint32
+  else: uint16
+)
+
+type
+  A = (ref | ptr | pointer)
+  B = pointer | ptr | ref
+  C = ref | ptr | pointer
+
+template `+`(a, b): untyped = (b, a)
+template `*`(a, b): untyped = (a, b)
+
+doAssert (ref int + ref float * ref string + ref bool) is
+  (ref bool, ((ref float, ref string), ref int))
+type X = ref int + ref float * ref string + ref bool
+doAssert X is (ref bool, ((ref float, ref string), ref int))
+
+type SomePointer = proc | ref | ptr | pointer

tests/system/tvarargslen.nim

@@ -1,8 +1,8 @@
 discard """
   output: '''
-tvarargslen.nim:35:9 (1, 2)
+tvarargslen.nim:35:2 (1, 2)
-tvarargslen.nim:36:9 12
+tvarargslen.nim:36:2 12
-tvarargslen.nim:37:9 1
+tvarargslen.nim:37:2 1
 tvarargslen.nim:38:8 
 done
 '''

tests/typerel/t7600_1.nim

@@ -1,6 +1,6 @@
 discard """
 errormsg: "type mismatch: got <Thin[system.int]>"
-nimout: '''t7600_1.nim(21, 6) Error: type mismatch: got <Thin[system.int]>
+nimout: '''t7600_1.nim(21, 1) Error: type mismatch: got <Thin[system.int]>
 but expected one of:
 proc test[T](x: Paper[T])
   first type mismatch at position: 1

tests/typerel/t7600_2.nim

@@ -1,6 +1,6 @@
 discard """
 errormsg: "type mismatch: got <Thin>"
-nimout: '''t7600_2.nim(20, 6) Error: type mismatch: got <Thin>
+nimout: '''t7600_2.nim(20, 1) Error: type mismatch: got <Thin>
 but expected one of:
 proc test(x: Paper)
   first type mismatch at position: 1

tests/varres/tprevent_forloopvar_mutations.nim

@@ -1,6 +1,6 @@
 discard """
   errormsg: "type mismatch: got <int>"
-  nimout: '''tprevent_forloopvar_mutations.nim(16, 7) Error: type mismatch: got <int>
+  nimout: '''tprevent_forloopvar_mutations.nim(16, 3) Error: type mismatch: got <int>
 but expected one of:
 proc inc[T, V: Ordinal](x: var T; y: V = 1)
   first type mismatch at position: 1