nimpretty: proper command line handling; added tests; travis ensures these stay green

.travis.yml

@@ -45,6 +45,8 @@ script:
   - nimble install niminst
   - nim c --taintMode:on -d:nimCoroutines tests/testament/tester
   - tests/testament/tester --pedantic all -d:nimCoroutines
+  - nim c -o:bin/nimpretty nimpretty/nimpretty.nim
+  - nim c -r nimpretty/tester.nim
   - ./koch web
   - ./koch csource
   - ./koch nimsuggest

compiler/layouter.nim

@@ -21,7 +21,6 @@ type
     splitComma, splitParLe, splitAnd, splitOr, splitIn, splitBinary
 
   Emitter* = object
-    f: PLLStream
     config: ConfigRef
     fid: FileIndex
     lastTok: TTokType
@@ -40,8 +39,6 @@ proc openEmitter*(em: var Emitter, cache: IdentCache;
   em.indWidth = getIndentWidth(fileIdx, llStreamOpen(fullPath, fmRead),
                                cache, config)
   if em.indWidth == 0: em.indWidth = 2
-  let outfile = changeFileExt(fullPath, ".pretty.nim")
-  em.f = llStreamOpen(outfile, fmWrite)
   em.config = config
   em.fid = fileIdx
   em.lastTok = tkInvalid
@@ -50,12 +47,13 @@ proc openEmitter*(em: var Emitter, cache: IdentCache;
   em.content = newStringOfCap(16_000)
   em.indentStack = newSeqOfCap[int](30)
   em.indentStack.add 0
-  if em.f == nil:
-    rawMessage(config, errGenerated, "cannot open file: " & outfile)
 
 proc closeEmitter*(em: var Emitter) =
-  em.f.llStreamWrite em.content
+  var f = llStreamOpen(em.config.outFile, fmWrite)
-  llStreamClose(em.f)
+  if f == nil:
+    rawMessage(em.config, errGenerated, "cannot open file: " & em.config.outFile)
+  f.llStreamWrite em.content
+  llStreamClose(f)
 
 proc countNewlines(s: string): int =
   result = 0

koch.nim

@@ -254,15 +254,13 @@ proc buildTool(toolname, args: string) =
   copyFile(dest="bin" / splitFile(toolname).name.exe, source=toolname.exe)
 
 proc buildTools(latest: bool) =
-  let nimsugExe = "bin/nimsuggest".exe
+  nimexec "c --noNimblePath -p:compiler -d:release -o:" & ("bin/nimsuggest".exe) &
-  nimexec "c --noNimblePath -p:compiler -d:release -o:" & nimsugExe &
       " nimsuggest/nimsuggest.nim"
 
-  let nimgrepExe = "bin/nimgrep".exe
+  nimexec "c -d:release -o:" & ("bin/nimgrep".exe) & " tools/nimgrep.nim"
-  nimexec "c -d:release -o:" & nimgrepExe & " tools/nimgrep.nim"
   when defined(windows): buildVccTool()
 
-  #nimexec "c -o:" & ("bin/nimresolve".exe) & " tools/nimresolve.nim"
+  nimexec "c -o:" & ("bin/nimpretty".exe) & " nimpretty/nimpretty.nim"
 
   buildNimble(latest)
 

nimpretty/nimpretty.nim

@@ -25,6 +25,7 @@ Usage:
   nimpretty [options] file.nim
 Options:
   --backup:on|off     create a backup file before overwritting (default: ON)
+  --output:file       set the output file (default: overwrite the .nim file)
   --version           show the version
   --help              show this help
 """
@@ -39,18 +40,18 @@ proc writeVersion() =
   stdout.flushFile()
   quit(0)
 
-proc prettyPrint(infile: string) =
+proc prettyPrint(infile, outfile: string) =
-  let conf = newConfigRef()
+  var conf = newConfigRef()
   let fileIdx = fileInfoIdx(conf, infile)
+  conf.outFile = outfile
   when defined(nimpretty2):
     discard parseFile(fileIdx, newIdentCache(), conf)
   else:
     let tree = parseFile(fileIdx, newIdentCache(), conf)
-    let outfile = changeFileExt(infile, ".pretty.nim")
     renderModule(tree, infile, outfile, {}, fileIdx, conf)
 
 proc main =
-  var infile: string
+  var infile, outfile: string
   var backup = true
   for kind, key, val in getopt():
     case kind
@@ -61,12 +62,14 @@ proc main =
       of "help", "h": writeHelp()
       of "version", "v": writeVersion()
       of "backup": backup = parseBool(val)
+      of "output", "o": outfile = val
       else: writeHelp()
     of cmdEnd: assert(false) # cannot happen
   if infile.len == 0:
     quit "[Error] no input file."
   if backup:
     os.copyFile(source=infile, dest=changeFileExt(infile, ".nim.backup"))
-  prettyPrint(infile)
+  if outfile.len == 0: outfile = infile
+  prettyPrint(infile, outfile)
 
 main()

nimpretty/tester.nim

@@ -0,0 +1,29 @@
+# Small program that runs the test cases
+
+import strutils, os
+
+const
+  dir = "nimpretty/tests/"
+
+var
+  failures = 0
+
+proc test(infile, outfile: string) =
+  if execShellCmd("nimpretty -o:$2 --backup:off $1" % [infile, outfile]) != 0:
+    quit("FAILURE")
+  let nimFile = splitFile(infile).name
+  let expected = dir / "expected" / nimFile & ".nim"
+  let produced = dir / nimFile & ".pretty"
+  if strip(readFile(expected)) != strip(readFile(produced)):
+    echo "FAILURE: files differ: ", nimFile
+    discard execShellCmd("diff -uNdr " & produced & " " & expected)
+    failures += 1
+  else:
+    echo "SUCCESS: files identical: ", nimFile
+
+for t in walkFiles(dir / "*.nim"):
+  let res = t.changeFileExt("pretty")
+  test(t, res)
+  removeFile(res)
+
+if failures > 0: quit($failures & " failures occurred.")

nimpretty/tests/exhaustive.nim

@@ -0,0 +1,284 @@
+discard """
+  outputsub: '''ObjectAssignmentError'''
+  exitcode: "1"
+"""
+
+import verylongnamehere,verylongnamehere,verylongnamehereverylongnamehereverylong,namehere,verylongnamehere
+
+type
+  GeneralTokenizer* = object of RootObj ## comment here
+    kind*: TokenClass ## and here
+    start*, length*: int ## you know how it goes...
+    buf: cstring
+    pos: int # other comment here.
+    state: TokenClass
+
+var x*: string
+var y: seq[string] #[ yay inline comments. So nice I have to care bout these. ]#
+
+echo "#", x, "##", y, "#"
+
+echo (tup, here)
+echo(argA, argB)
+
+import macros
+
+## A documentation comment here.
+## That spans multiple lines.
+## And is not to be touched.
+
+const numbers = [4u8, 5'u16, 89898_00]
+
+macro m(n): untyped =
+  result = foo"string literal"
+
+{.push m.}
+proc p() = echo "p", 1+4 * 5, if true: 5 else: 6
+proc q(param: var ref ptr string) =
+  p()
+  if true:
+    echo a and b or not c and not -d
+{.pop.}
+
+q()
+
+when false:
+  # bug #4766
+  type
+    Plain = ref object
+      discard
+
+    Wrapped[T] = object
+      value: T
+
+  converter toWrapped[T](value: T): Wrapped[T] =
+    Wrapped[T](value: value)
+
+  let result = Plain()
+  discard $result
+
+when false:
+  # bug #3670
+  template someTempl(someConst: bool) =
+    when someConst:
+      var a: int
+    if true:
+      when not someConst:
+        var a: int
+      a = 5
+
+  someTempl(true)
+
+
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2018 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Layouter for nimpretty. Still primitive but useful.
+
+import idents, lexer, lineinfos, llstream, options, msgs, strutils
+from os import changeFileExt
+
+const
+  MaxLineLen = 80
+  LineCommentColumn = 30
+
+type
+  SplitKind = enum
+    splitComma, splitParLe, splitAnd, splitOr, splitIn, splitBinary
+
+  Emitter* = object
+    f: PLLStream
+    config: ConfigRef
+    fid: FileIndex
+    lastTok: TTokType
+    inquote {.pragmaHereWrongCurlyEnd}: bool
+    col, lastLineNumber, lineSpan, indentLevel: int
+    content: string
+    fixedUntil: int # marks where we must not go in the content
+    altSplitPos: array[SplitKind, int] # alternative split positions
+
+proc openEmitter*[T, S](em: var Emitter, config: ConfigRef, fileIdx: FileIndex) {.pragmaHereWrongCurlyEnd} =
+  let outfile = changeFileExt(config.toFullPath(fileIdx), ".pretty.nim")
+  em.f = llStreamOpen(outfile, fmWrite)
+  em.config = config
+  em.fid = fileIdx
+  em.lastTok = tkInvalid
+  em.inquote = false
+  em.col = 0
+  em.content = newStringOfCap(16_000)
+  if em.f == nil:
+    rawMessage(config, errGenerated, "cannot open file: " & outfile)
+
+proc closeEmitter*(em: var Emitter) {.inline.} =
+  em.f.llStreamWrite em.content
+  llStreamClose(em.f)
+
+proc countNewlines(s: string): int =
+  result = 0
+  for i in 0..<s.len:
+    if s[i+1] == '\L': inc result
+
+proc calcCol(em: var Emitter; s: string) =
+  var i = s.len-1
+  em.col = 0
+  while i >= 0 and s[i] != '\L':
+    dec i
+    inc em.col
+
+template wr(x) =
+  em.content.add x
+  inc em.col, x.len
+
+template goodCol(col): bool = col in 40..MaxLineLen
+
+const splitters = {tkComma, tkSemicolon, tkParLe, tkParDotLe,
+                   tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
+                   tkCurlyLe}
+
+template rememberSplit(kind) =
+  if goodCol(em.col):
+    em.altSplitPos[kind] = em.content.len
+
+proc softLinebreak(em: var Emitter, lit: string) =
+  # XXX Use an algorithm that is outlined here:
+  # https://llvm.org/devmtg/2013-04/jasper-slides.pdf
+  # +2 because we blindly assume a comma or ' &' might follow
+  if not em.inquote and em.col+lit.len+2 >= MaxLineLen:
+    if em.lastTok in splitters:
+      wr("\L")
+      em.col = 0
+      for i in 1..em.indentLevel+2: wr(" ")
+    else:
+      # search backwards for a good split position:
+      for a in em.altSplitPos:
+        if a > em.fixedUntil:
+          let ws = "\L" & repeat(' ',em.indentLevel+2)
+          em.col = em.content.len - a
+          em.content.insert(ws, a)
+          break
+
+proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
+
+  template endsInWhite(em): bool =
+    em.content.len > 0 and em.content[em.content.high] in {' ', '\L'}
+  template endsInAlpha(em): bool =
+    em.content.len > 0 and em.content[em.content.high] in SymChars+{'_'}
+
+  proc emitComment(em: var Emitter; tok: TToken) =
+    let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    if not endsInWhite(em):
+      wr(" ")
+      if em.lineSpan == 0 and max(em.col, LineCommentColumn) + lit.len <= MaxLineLen:
+        for i in 1 .. LineCommentColumn - em.col: wr(" ")
+    wr lit
+
+  var preventComment = case tok.tokType
+                       of tokKeywordLow..tokKeywordHigh:
+                          if endsInAlpha(em): wr(" ")
+                          wr(TokTypeToStr[tok.tokType])
+
+                          case tok.tokType
+                          of tkAnd: rememberSplit(splitAnd)
+                          of tkOr: rememberSplit(splitOr)
+                          of tkIn: rememberSplit(splitIn)
+                          else: 90
+                       else:
+                         "case returns value"
+
+
+  if tok.tokType == tkComment and tok.line == em.lastLineNumber and tok.indent >= 0:
+    # we have an inline comment so handle it before the indentation token:
+    emitComment(em, tok)
+    preventComment = true
+    em.fixedUntil = em.content.high
+
+  elif tok.indent >= 0:
+        em.indentLevel = tok.indent
+        # remove trailing whitespace:
+        while em.content.len > 0 and em.content[em.content.high] == ' ':
+          setLen(em.content, em.content.len-1)
+        wr("\L")
+        for i in 2..tok.line - em.lastLineNumber: wr("\L")
+        em.col = 0
+        for i in 1..tok.indent:
+          wr(" ")
+        em.fixedUntil = em.content.high
+
+  case tok.tokType
+  of tokKeywordLow..tokKeywordHigh:
+    if endsInAlpha(em): wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+
+    case tok.tokType
+    of tkAnd: rememberSplit(splitAnd)
+    of tkOr: rememberSplit(splitOr)
+    of tkIn: rememberSplit(splitIn)
+    else: discard
+
+  of tkColon:
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+  of tkSemicolon,
+     tkComma:
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+    rememberSplit(splitComma)
+  of tkParLe, tkParRi, tkBracketLe,
+     tkBracketRi, tkCurlyLe, tkCurlyRi,
+     tkBracketDotLe, tkBracketDotRi,
+     tkCurlyDotLe, tkCurlyDotRi,
+     tkParDotLe, tkParDotRi,
+     tkColonColon, tkDot, tkBracketLeColon:
+    wr(TokTypeToStr[tok.tokType])
+    if tok.tokType in splitters:
+      rememberSplit(splitParLe)
+  of tkEquals:
+    if not em.endsInWhite: wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+  of tkOpr, tkDotDot:
+    if not em.endsInWhite: wr(" ")
+    wr(tok.ident.s)
+    template isUnary(tok): bool =
+      tok.strongSpaceB == 0 and tok.strongSpaceA > 0
+
+    if not isUnary(tok) or em.lastTok in {tkOpr, tkDotDot}:
+      wr(" ")
+      rememberSplit(splitBinary)
+  of tkAccent:
+    wr(TokTypeToStr[tok.tokType])
+    em.inquote = not em.inquote
+  of tkComment:
+    if not preventComment:
+      emitComment(em, tok)
+  of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
+    let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
+    softLinebreak(em, lit)
+    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wr(" ")
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    wr lit
+  of tkEof: discard
+  else:
+    let lit = if tok.ident != nil: tok.ident.s else: tok.literal
+    softLinebreak(em, lit)
+    if endsInAlpha(em): wr(" ")
+    wr lit
+
+  em.lastTok = tok.tokType
+  em.lastLineNumber = tok.line + em.lineSpan
+  em.lineSpan = 0
+
+proc starWasExportMarker*(em: var Emitter) =
+  if em.content.endsWith(" * "):
+    setLen(em.content, em.content.len-3)
+    em.content.add("*")
+    dec em.col, 2

nimpretty/tests/expected/exhaustive.nim

@@ -0,0 +1,291 @@
+discard """
+  outputsub: '''ObjectAssignmentError'''
+  exitcode: "1"
+"""
+
+import verylongnamehere, verylongnamehere, 
+  verylongnamehereverylongnamehereverylong, namehere, verylongnamehere
+
+type
+  GeneralTokenizer* = object of RootObj ## comment here
+    kind*: TokenClass         ## and here
+    start*, length*: int      ## you know how it goes...
+    buf: cstring
+    pos: int                  # other comment here.
+    state: TokenClass
+
+var x*: string
+var y: seq[string] #[ yay inline comments. So nice I have to care bout these. ]#
+
+echo "#", x, "##", y, "#"
+
+echo (tup, here)
+echo(argA, argB)
+
+import macros
+
+## A documentation comment here.
+## That spans multiple lines.
+## And is not to be touched.
+
+const numbers = [4u8, 5'u16, 89898_00]
+
+macro m(n): untyped =
+  result = foo"string literal"
+
+{.push m.}
+proc p() = echo "p", 1+4 * 5, if true: 5 else: 6
+proc q(param: var ref ptr string) =
+  p()
+  if true:
+    echo a and b or not c and not -d
+{.pop.}
+
+q()
+
+when false:
+  # bug #4766
+  type
+    Plain = ref object
+      discard
+
+    Wrapped[T] = object
+      value: T
+
+  converter toWrapped[T](value: T): Wrapped[T] =
+    Wrapped[T](value: value)
+
+  let result = Plain()
+  discard $result
+
+when false:
+  # bug #3670
+  template someTempl(someConst: bool) =
+    when someConst:
+      var a: int
+    if true:
+      when not someConst:
+        var a: int
+      a = 5
+
+  someTempl(true)
+
+
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2018 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Layouter for nimpretty. Still primitive but useful.
+
+import idents, lexer, lineinfos, llstream, options, msgs, strutils
+from os import changeFileExt
+
+const
+  MaxLineLen = 80
+  LineCommentColumn = 30
+
+type
+  SplitKind = enum
+    splitComma, splitParLe, splitAnd, splitOr, splitIn, splitBinary
+
+  Emitter* = object
+    f: PLLStream
+    config: ConfigRef
+    fid: FileIndex
+    lastTok: TTokType
+    inquote {.pragmaHereWrongCurlyEnd.}: bool
+    col, lastLineNumber, lineSpan, indentLevel: int
+    content: string
+    fixedUntil: int           # marks where we must not go in the content
+    altSplitPos: array[SplitKind, int] # alternative split positions
+
+proc openEmitter*[T, S](em: var Emitter; config: ConfigRef; 
+    fileIdx: FileIndex) {.pragmaHereWrongCurlyEnd.} =
+  let outfile = changeFileExt(config.toFullPath(fileIdx), ".pretty.nim")
+  em.f = llStreamOpen(outfile, fmWrite)
+  em.config = config
+  em.fid = fileIdx
+  em.lastTok = tkInvalid
+  em.inquote = false
+  em.col = 0
+  em.content = newStringOfCap(16_000)
+  if em.f == nil:
+    rawMessage(config, errGenerated, "cannot open file: " & outfile)
+
+proc closeEmitter*(em: var Emitter) {.inline.} =
+  em.f.llStreamWrite em.content
+  llStreamClose(em.f)
+
+proc countNewlines(s: string): int =
+  result = 0
+  for i in 0..<s.len:
+    if s[i+1] == '\L': inc result
+
+proc calcCol(em: var Emitter; s: string) =
+  var i = s.len-1
+  em.col = 0
+  while i >= 0 and s[i] != '\L':
+    dec i
+    inc em.col
+
+template wr(x) =
+  em.content.add x
+  inc em.col, x.len
+
+template goodCol(col): bool = col in 40..MaxLineLen
+
+const splitters = {tkComma, tkSemicolon, tkParLe, tkParDotLe,
+                   tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
+                   tkCurlyLe}
+
+template rememberSplit(kind) =
+  if goodCol(em.col):
+    em.altSplitPos[kind] = em.content.len
+
+proc softLinebreak(em: var Emitter; lit: string) =
+  # XXX Use an algorithm that is outlined here:
+  # https://llvm.org/devmtg/2013-04/jasper-slides.pdf
+  # +2 because we blindly assume a comma or ' &' might follow
+  if not em.inquote and em.col+lit.len+2 >= MaxLineLen:
+    if em.lastTok in splitters:
+      wr("\L")
+      em.col = 0
+      for i in 1..em.indentLevel+2: wr(" ")
+    else:
+      # search backwards for a good split position:
+      for a in em.altSplitPos:
+        if a > em.fixedUntil:
+          let ws = "\L" & repeat(' ', em.indentLevel+2)
+          em.col = em.content.len - a
+          em.content.insert(ws, a)
+          break
+
+proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
+
+  template endsInWhite(em): bool =
+    em.content.len > 0 and em.content[em.content.high] in {' ', '\L'}
+  template endsInAlpha(em): bool =
+    em.content.len > 0 and em.content[em.content.high] in SymChars+{'_'}
+
+  proc emitComment(em: var Emitter; tok: TToken) =
+    let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, 
+        tok.commentOffsetB)
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    if not endsInWhite(em):
+      wr(" ")
+      if em.lineSpan == 0 and max(em.col, 
+          LineCommentColumn) + lit.len <= MaxLineLen:
+        for i in 1 .. LineCommentColumn - em.col: wr(" ")
+    wr lit
+
+  var preventComment = case tok.tokType
+    of tokKeywordLow..tokKeywordHigh:
+      if endsInAlpha(em): wr(" ")
+      wr(TokTypeToStr[tok.tokType])
+
+      case tok.tokType
+      of tkAnd: rememberSplit(splitAnd)
+      of tkOr: rememberSplit(splitOr)
+      of tkIn: rememberSplit(splitIn)
+      else: 90
+    else:
+      "case returns value"
+
+
+  if tok.tokType == tkComment and tok.line == em.lastLineNumber and
+      tok.indent >= 0:
+    # we have an inline comment so handle it before the indentation token:
+    emitComment(em, tok)
+    preventComment = true
+    em.fixedUntil = em.content.high
+
+  elif tok.indent >= 0:
+    em.indentLevel = tok.indent
+    # remove trailing whitespace:
+    while em.content.len > 0 and em.content[em.content.high] == ' ':
+      setLen(em.content, em.content.len-1)
+    wr("\L")
+    for i in 2..tok.line - em.lastLineNumber: wr("\L")
+    em.col = 0
+    for i in 1..tok.indent:
+      wr(" ")
+    em.fixedUntil = em.content.high
+
+  case tok.tokType
+  of tokKeywordLow..tokKeywordHigh:
+    if endsInAlpha(em): wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+
+    case tok.tokType
+    of tkAnd: rememberSplit(splitAnd)
+    of tkOr: rememberSplit(splitOr)
+    of tkIn: rememberSplit(splitIn)
+    else: discard
+
+  of tkColon:
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+  of tkSemicolon,
+     tkComma:
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+    rememberSplit(splitComma)
+  of tkParLe, tkParRi, tkBracketLe,
+     tkBracketRi, tkCurlyLe, tkCurlyRi,
+     tkBracketDotLe, tkBracketDotRi,
+     tkCurlyDotLe, tkCurlyDotRi,
+     tkParDotLe, tkParDotRi,
+     tkColonColon, tkDot, tkBracketLeColon:
+    wr(TokTypeToStr[tok.tokType])
+    if tok.tokType in splitters:
+      rememberSplit(splitParLe)
+  of tkEquals:
+    if not em.endsInWhite: wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+  of tkOpr, tkDotDot:
+    if not em.endsInWhite: wr(" ")
+    wr(tok.ident.s)
+    template isUnary(tok): bool =
+      tok.strongSpaceB == 0 and tok.strongSpaceA > 0
+
+    if not isUnary(tok) or em.lastTok in {tkOpr, tkDotDot}:
+      wr(" ")
+      rememberSplit(splitBinary)
+  of tkAccent:
+    wr(TokTypeToStr[tok.tokType])
+    em.inquote = not em.inquote
+  of tkComment:
+    if not preventComment:
+      emitComment(em, tok)
+  of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, 
+      tkGTripleStrLit, tkCharLit:
+    let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
+    softLinebreak(em, lit)
+    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wr(
+        " ")
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    wr lit
+  of tkEof: discard
+  else:
+    let lit = if tok.ident != nil: tok.ident.s else: tok.literal
+    softLinebreak(em, lit)
+    if endsInAlpha(em): wr(" ")
+    wr lit
+
+  em.lastTok = tok.tokType
+  em.lastLineNumber = tok.line + em.lineSpan
+  em.lineSpan = 0
+
+proc starWasExportMarker*(em: var Emitter) =
+  if em.content.endsWith(" * "):
+    setLen(em.content, em.content.len-3)
+    em.content.add("*")
+    dec em.col, 2