nimpretty: nimpretty now understands splitting newlines

compiler/layouter.nim

@@ -143,6 +143,8 @@ proc closeEmitter*(em: var Emitter) =
   var lineBegin = 0
   var i = 0
   while i <= em.tokens.high:
+    when defined(debug):
+      echo "i-th token ", em.kinds[i], " ", em.tokens[i]
     case em.kinds[i]
     of ltBeginSection:
       maxLhs = computeMax(em, lineBegin)
@@ -219,11 +221,24 @@ proc wr(em: var Emitter; x: string; lt: LayoutToken) =
   inc em.col, x.len
   assert em.tokens.len == em.kinds.len
 
-proc wrNewline(em: var Emitter) =
+proc wrNewline(em: var Emitter; kind = ltCrucialNewline) =
   em.tokens.add "\L"
-  em.kinds.add ltCrucialNewline
+  em.kinds.add kind
   em.col = 0
 
+proc newlineWasSplitting*(em: var Emitter) =
+  if em.kinds.len >= 3 and em.kinds[^3] == ltCrucialNewline:
+    em.kinds[^3] = ltSplittingNewline
+
+#[
+Splitting newlines can occur:
+- after commas, semicolon, '[', '('.
+- after binary operators, '='.
+- after ':' type
+
+We only need parser support for the "after type" case.
+]#
+
 proc wrSpaces(em: var Emitter; spaces: int) =
   if spaces > 0:
     wr(em, strutils.repeat(' ', spaces), ltSpaces)
@@ -261,7 +276,7 @@ const
               tkBracketRi, tkCurlyDotRi,
               tkCurlyRi}
 
-  splitters = openPars + {tkComma, tkSemicolon}
+  splitters = openPars + {tkComma, tkSemicolon} # do not add 'tkColon' here!
   oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
             tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
 
@@ -366,12 +381,14 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     em.fixedUntil = em.tokens.high
 
   elif tok.indent >= 0:
+    var newlineKind = ltCrucialNewline
     if em.keepIndents > 0:
       em.indentLevel = tok.indent
     elif (em.lastTok in (splitters + oprSet) and tok.tokType notin closedPars):
       # aka: we are in an expression context:
       let alignment = max(tok.indent - em.indentStack[^1], 0)
       em.indentLevel = alignment + em.indentStack.high * em.indWidth
+      newlineKind = ltSplittingNewline
     else:
       if tok.indent > em.indentStack[^1]:
         em.indentStack.add tok.indent
@@ -391,7 +408,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     ]#
     # remove trailing whitespace:
     removeSpaces em
-    wrNewline em
+    wrNewline em, newlineKind
     for i in 2..tok.line - em.lastLineNumber: wrNewline(em)
     wrSpaces em, em.indentLevel
     em.fixedUntil = em.tokens.high

compiler/parser.nim

@@ -147,10 +147,17 @@ template withInd(p, body: untyped) =
   body
   p.currInd = oldInd
 
+template newlineWasSplitting(p: var TParser) =
+  when defined(nimpretty):
+    layouter.newlineWasSplitting(p.em)
+
 template realInd(p): bool = p.tok.indent > p.currInd
 template sameInd(p): bool = p.tok.indent == p.currInd
 template sameOrNoInd(p): bool = p.tok.indent == p.currInd or p.tok.indent < 0
 
+proc validInd(p: var TParser): bool {.inline.} =
+  result = p.tok.indent < 0 or p.tok.indent > p.currInd
+
 proc rawSkipComment(p: var TParser, node: PNode) =
   if p.tok.tokType == tkComment:
     if node != nil:
@@ -367,6 +374,7 @@ proc colonOrEquals(p: var TParser, a: PNode): PNode =
   if p.tok.tokType == tkColon:
     result = newNodeP(nkExprColonExpr, p)
     getTok(p)
+    newlineWasSplitting(p)
     #optInd(p, result)
     addSon(result, a)
     addSon(result, parseExpr(p))
@@ -1295,6 +1303,7 @@ proc binaryNot(p: var TParser; a: PNode): PNode =
 
 proc parseTypeDesc(p: var TParser): PNode =
   #| typeDesc = simpleExpr ('not' expr)?
+  newlineWasSplitting(p)
   result = simpleExpr(p, pmTypeDesc)
   result = binaryNot(p, result)
 
@@ -1510,6 +1519,7 @@ proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode =
   elif p.tok.indent >= 0 and p.tok.indent <= p.currInd or not isExprStart(p):
     # NL terminates:
     addSon(result, p.emptyNode)
+    # nimpretty here!
   else:
     var e = parseExpr(p)
     e = postExprBlocks(p, e)
@@ -1722,9 +1732,6 @@ proc parsePattern(p: var TParser): PNode =
   result = parseStmt(p)
   eat(p, tkCurlyRi)
 
-proc validInd(p: var TParser): bool =
-  result = p.tok.indent < 0 or p.tok.indent > p.currInd
-
 proc parseRoutine(p: var TParser, kind: TNodeKind): PNode =
   #| indAndComment = (IND{>} COMMENT)? | COMMENT?
   #| routine = optInd identVis pattern? genericParamList?
@@ -1809,7 +1816,7 @@ proc parseEnum(p: var TParser): PNode =
       symPragma = newNodeP(nkPragmaExpr, p)
       addSon(symPragma, a)
       addSon(symPragma, pragma)
-
+    # nimpretty support here
     if p.tok.indent >= 0 and p.tok.indent <= p.currInd:
       add(result, symPragma)
       break
@@ -2219,6 +2226,7 @@ proc parseStmt(p: var TParser): PNode =
   #| stmt = (IND{>} complexOrSimpleStmt^+(IND{=} / ';') DED)
   #|      / simpleStmt ^+ ';'
   if p.tok.indent > p.currInd:
+    # nimpretty support here
     result = newNodeP(nkStmtList, p)
     withInd(p):
       while true:
@@ -2295,6 +2303,7 @@ proc parseTopLevelStmt(p: var TParser): PNode =
   result = p.emptyNode
   # progress guaranteed
   while true:
+    # nimpretty support here
     if p.tok.indent != 0:
       if p.firstTok and p.tok.indent < 0: discard
       elif p.tok.tokType != tkSemiColon: