[bugfix] fix #11469, new rules for a newline in nimpretty (#11512)

* [bugfix] fix #11469, new rules for a newline in nimpretty
* concatenate two lines if they have the same indentation level

compiler/layouter.nim

@@ -16,6 +16,7 @@ from sequtils import delete
 
 const
   MaxLineLen = 80
+  MinLineLen = 10
 
 type
   SplitKind = enum
@@ -25,7 +26,7 @@ type
     detectSemicolonKind, useSemicolon, dontTouch
 
   LayoutToken = enum
-    ltSpaces, ltNewline, ltTab,
+    ltSpaces, ltNewline, ltTab, ltOptionalNewline,
     ltComment, ltLit, ltKeyword, ltExportMarker, ltIdent,
     ltOther, ltOpr,
     ltBeginSection, ltEndSection
@@ -88,6 +89,46 @@ proc computeRhs(em: Emitter; pos: int): int =
     inc result, em.tokens[p].len
     inc p
 
+proc isLongEnough(lineLen, startPos, endPos: int): bool =
+  result = lineLen > MinLineLen and endPos > startPos + 4
+
+proc findNewline(em: Emitter; p, lineLen: var int) =
+  while p < em.tokens.len and em.kinds[p] != ltNewline:
+    inc lineLen, em.tokens[p].len
+    inc p
+
+proc optionalIsGood(em: var Emitter; pos: int): bool =
+  let ourIndent = em.tokens[pos].len
+  var p = pos+1
+  var lineLen = 0
+  em.findNewline(p, lineLen)
+  if p+1 < em.tokens.len and em.kinds[p+1] == ltSpaces and
+      em.kinds[p-1] == ltOptionalNewline:
+    if em.tokens[p+1].len == ourIndent:
+      # concatenate lines with the same indententation
+      var nlPos = p
+      var lineLenTotal = lineLen
+      inc p
+      em.findNewline(p, lineLenTotal)
+      if isLongEnough(lineLenTotal, nlPos, p):
+        em.kinds[nlPos] = ltOptionalNewline
+        if em.kinds[nlPos+1] == ltSpaces:
+          # inhibit extra spaces when concatenating two lines
+          em.tokens[nlPos+1] = if em.tokens[nlPos-2] == ",": " " else: ""
+      result = true
+    elif em.tokens[p+1].len < ourIndent:
+      result = isLongEnough(lineLen, pos, p)
+  elif em.kinds[pos+1] == ltOther: # note: pos+1, not p+1
+    result = false
+  else:
+    result = isLongEnough(lineLen, pos, p)
+
+proc lenOfNextTokens(em: Emitter; pos: int): int =
+  result = 0
+  for i in 1 ..< em.tokens.len-pos:
+    if em.kinds[pos+i] in {ltNewline, ltOptionalNewline}: break
+    inc result, em.tokens[pos+i].len
+
 proc closeEmitter*(em: var Emitter) =
   let outFile = em.config.absOutFile
 
@@ -95,7 +136,8 @@ proc closeEmitter*(em: var Emitter) =
   var maxLhs = 0
   var lineLen = 0
   var lineBegin = 0
-  for i in 0..em.tokens.high:
+  var i = 0
+  while i <= em.tokens.high:
     case em.kinds[i]
     of ltBeginSection:
       maxLhs = computeMax(em, lineBegin)
@@ -113,9 +155,24 @@ proc closeEmitter*(em: var Emitter) =
       content.add em.tokens[i]
       lineLen = 0
       lineBegin = i+1
+    of ltOptionalNewline:
+      let totalLineLen = lineLen + lenOfNextTokens(em, i)
+      if totalLineLen > MaxLineLen + MinLineLen or
+         totalLineLen > MaxLineLen and optionalIsGood(em, i):
+        if i-1 >= 0 and em.kinds[i-1] == ltSpaces:
+          let spaces = em.tokens[i-1].len
+          content.setLen(content.len - spaces)
+        content.add "\L"
+        content.add em.tokens[i]
+        lineLen = em.tokens[i].len
+        lineBegin = i+1
+        if i+1 < em.kinds.len and em.kinds[i+1] == ltSpaces:
+          # inhibit extra spaces at the start of a new line
+          inc i
     else:
       content.add em.tokens[i]
       inc lineLen, em.tokens[i].len
+    inc i
 
   if fileExists(outFile) and readFile(outFile.string) == content:
     discard "do nothing, see #9499"
@@ -170,7 +227,6 @@ proc removeSpaces(em: var Emitter) =
     setLen(em.kinds, em.kinds.len-1)
     dec em.col, tokenLen
 
-template goodCol(col): bool = col in 40..MaxLineLen
 
 const
   openPars = {tkParLe, tkParDotLe,
@@ -184,49 +240,17 @@ const
   oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
             tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
 
-template rememberSplit(kind) =
+template goodCol(col): bool = col >= MaxLineLen div 2
-  if goodCol(em.col):
-    em.altSplitPos[kind] = em.tokens.len
 
 template moreIndent(em): int =
-  (if em.doIndentMore > 0: em.indWidth*2 else: em.indWidth)
+  if em.doIndentMore > 0: em.indWidth*2 else: em.indWidth
 
-proc softLinebreak(em: var Emitter, lit: string) =
+template rememberSplit(kind) =
-  # XXX Use an algorithm that is outlined here:
+  if goodCol(em.col) and not em.inquote:
-  # https://llvm.org/devmtg/2013-04/jasper-slides.pdf
-  # +2 because we blindly assume a comma or ' &' might follow
-  if not em.inquote and em.col+lit.len+2 >= MaxLineLen:
-    if em.lastTok in splitters:
-      # bug #10295, check first if even more indentation would help:
     let spaces = em.indentLevel+moreIndent(em)
-      if spaces < em.col:
+    if spaces < em.col and spaces > 0:
-        removeSpaces em
+      wr(em, strutils.repeat(' ', spaces), ltOptionalNewline)
-        wrNewline(em)
+    #em.altSplitPos[kind] = em.tokens.len
-        em.col = 0
-        wrSpaces em, spaces
-    else:
-      # search backwards for a good split position:
-      for a in mitems(em.altSplitPos):
-        if a > em.fixedUntil:
-          var spaces = 0
-          while a+spaces < em.kinds.len and em.kinds[a+spaces] == ltSpaces:
-            inc spaces
-          if spaces > 0:
-            delete(em.tokens, a, a+spaces-1)
-            delete(em.kinds, a, a+spaces-1)
-          em.kinds.insert(ltNewline, a)
-          em.tokens.insert("\L", a)
-          em.kinds.insert(ltSpaces, a+1)
-          em.tokens.insert(repeat(' ', em.indentLevel+moreIndent(em)), a+1)
-          # recompute em.col:
-          var i = em.kinds.len-1
-          em.col = 0
-          while i >= 0 and em.kinds[i] != ltNewline:
-            inc em.col, em.tokens[i].len
-            dec i
-          # mark position as "already split here"
-          a = -1
-          break
 
 proc emitMultilineComment(em: var Emitter, lit: string, col: int) =
   # re-align every line in the multi-line comment:
@@ -358,14 +382,9 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
 
     if not em.inquote:
       wr(em, TokTypeToStr[tok.tokType], ltKeyword)
-
+      if tok.tokType in {tkAnd, tkOr, tkIn, tkNotin}:
-      case tok.tokType
-      of tkAnd: rememberSplit(splitAnd)
-      of tkOr: rememberSplit(splitOr)
-      of tkIn, tkNotin:
         rememberSplit(splitIn)
         wrSpace em
-      else: discard
     else:
       # keywords in backticks are not normalized:
       wr(em, tok.ident.s, ltIdent)
@@ -422,7 +441,6 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       emitComment(em, tok)
   of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
     let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
-    softLinebreak(em, lit)
     if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
     em.lineSpan = countNewlines(lit)
     if em.lineSpan > 0: calcCol(em, lit)
@@ -430,7 +448,6 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
   of tkEof: discard
   else:
     let lit = if tok.ident != nil: tok.ident.s else: tok.literal
-    softLinebreak(em, lit)
     if endsInAlpha(em): wrSpace(em)
     wr em, lit, ltIdent
 

compiler/lexer.nim

@@ -1334,10 +1334,13 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
   var tok: TToken
   initToken(tok)
   openLexer(lex, fileIdx, inputstream, cache, config)
-  while true:
+  var prevToken = tkEof
+  while tok.tokType != tkEof:
     rawGetTok(lex, tok)
+    if tok.indent > 0 and prevToken in {tkColon, tkEquals, tkType, tkConst, tkLet, tkVar, tkUsing}:
       result = tok.indent
-    if result > 0 or tok.tokType == tkEof: break
+      if result > 0: break
+    prevToken = tok.tokType
   closeLexer(lex)
 
 proc getPrecedence*(ident: PIdent): int =

compiler/parser.nim

@@ -904,6 +904,7 @@ proc parsePragma(p: var TParser): PNode =
   result = newNodeP(nkPragma, p)
   inc p.inPragma
   when defined(nimpretty):
+    inc p.em.doIndentMore
     inc p.em.keepIndents
   getTok(p)
   optInd(p, result)
@@ -924,6 +925,7 @@ proc parsePragma(p: var TParser): PNode =
     parMessage(p, "expected '.}'")
   dec p.inPragma
   when defined(nimpretty):
+    dec p.em.doIndentMore
     dec p.em.keepIndents
 
 proc identVis(p: var TParser; allowDot=false): PNode =

nimpretty/tests/exhaustive.nim

@@ -122,7 +122,7 @@ type
     fixedUntil: int # marks where we must not go in the content
     altSplitPos: array[SplitKind, int] # alternative split positions
 
-proc openEmitter*[T, S](em: var Emitter; config: ConfigRef, fileIdx: FileIndex) {.pragmaHereWrongCurlyEnd} =
+proc openEmitter*[T, S](em: var Emitter; config: ConfigRef; fileIdx: FileIndex) {.pragmaHereWrongCurlyEnd} =
   let outfile = changeFileExt(config.toFullPath(fileIdx), ".pretty.nim")
   em.f = llStreamOpen(outfile, fmWrite)
   em.config = config
@@ -607,3 +607,14 @@ type
     tagVar,        ## the HTML ``var`` element
     tagVideo,      ## the HTML ``video`` element
     tagWbr         ## the HTML ``wbr`` element
+
+
+# bug #11469
+const lookup: array[32, uint8] = [0'u8, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 16, 17,
+    25, 17, 4, 8, 31, 27, 13, 23]
+
+veryLongVariableName.createVar("future" & $node[1][0].toStrLit, node[1], futureValue1,
+    futureValue2, node)
+
+veryLongVariableName.createVar("future" & $node[1][0].toStrLit, node[1], futureValue1,
+                               futureValue2, node)

nimpretty/tests/expected/exhaustive.nim

@@ -200,8 +200,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     if em.lineSpan > 0: calcCol(em, lit)
     if not endsInWhite(em):
       wr(" ")
-      if em.lineSpan == 0 and max(em.col,
+      if em.lineSpan == 0 and max(em.col, LineCommentColumn) + lit.len <= MaxLineLen:
-          LineCommentColumn) + lit.len <= MaxLineLen:
         for i in 1 .. LineCommentColumn - em.col: wr(" ")
     wr lit
 
@@ -285,12 +284,10 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
   of tkComment:
     if not preventComment:
       emitComment(em, tok)
-  of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit,
+  of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
-      tkGTripleStrLit, tkCharLit:
     let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
     softLinebreak(em, lit)
-    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wr(
+    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wr(" ")
-        " ")
     em.lineSpan = countNewlines(lit)
     if em.lineSpan > 0: calcCol(em, lit)
     wr lit
@@ -383,8 +380,7 @@ import osproc
 let res = execProcess(
     "echo | openssl s_client -connect example.com:443 2>/dev/null | openssl x509 -noout -dates")
 
-let res = execProcess(
+let res = execProcess("echo | openssl s_client -connect example.com:443 2>/dev/null | openssl x509 -noout -dates")
-    "echo | openssl s_client -connect example.com:443 2>/dev/null | openssl x509 -noout -dates")
 
 
 # bug #10177
@@ -617,3 +613,14 @@ type
     tagVar,         ## the HTML ``var`` element
     tagVideo,       ## the HTML ``video`` element
     tagWbr          ## the HTML ``wbr`` element
+
+
+# bug #11469
+const lookup: array[32, uint8] = [0'u8, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15,
+    16, 17, 25, 17, 4, 8, 31, 27, 13, 23]
+
+veryLongVariableName.createVar("future" & $node[1][0].toStrLit, node[1],
+    futureValue1, futureValue2, node)
+
+veryLongVariableName.createVar("future" & $node[1][0].toStrLit, node[1], futureValue1,
+                               futureValue2, node)

nimpretty/tests/expected/simple3.nim

@@ -8,9 +8,9 @@ foo bar
 proc fun() =
   echo "ok1"
 
-proc fun2(a = "fooo" & "bar" & "bar" & "bar" & "bar" & (
+proc fun2(a = "fooo" & "bar" & "bar" & "bar" & "bar" & ("bar" & "bar" & "bar") &
-    "bar" & "bar" & "bar") & "bar" & "bar" & "bar" & "bar" & "bar" & "bar" &
+    "bar" & "bar" & "bar" & "bar" & "bar" & "bar" & "bar" & "bar" &
-    "bar" & "bar" & "bar"): auto =
+    "bar"): auto =
   discard
 
 fun2()