Merge branch 'devel' into typedesc-reforms

.travis.yml

@@ -45,6 +45,8 @@ script:
   - nimble install niminst
   - nim c --taintMode:on -d:nimCoroutines tests/testament/tester
   - tests/testament/tester --pedantic all -d:nimCoroutines
+  - nim c -o:bin/nimpretty nimpretty/nimpretty.nim
+  - nim c -r nimpretty/tester.nim
   - ./koch web
   - ./koch csource
   - ./koch nimsuggest

appveyor.yml

@@ -47,6 +47,8 @@ build_script:
   - koch boot -d:release
   - koch nimble
   - nim e tests/test_nimscript.nims
+  - nim c -o:bin/nimpretty.exe nimpretty/nimpretty.nim
+  - nim c -r nimpretty/tester.nim
   - nimble install zip -y
   - nimble install opengl
   - nimble install sdl1

changelog.md

@@ -107,6 +107,9 @@
   use the Nim VM in a native Nim application.
 - Added the parameter ``val`` for the ``CritBitTree[T].incl`` proc.
 - The proc ``tgamma`` was renamed to ``gamma``. ``tgamma`` is deprecated.
+- The ``pegs`` module now exports getters for the fields of its ``Peg`` and ``NonTerminal``
+  object types. ``Peg``s with child nodes now have the standard ``items`` and ``pairs``
+  iterators.
 
 ### Language additions
 

compiler/ccgexprs.nim

@@ -59,7 +59,7 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope =
     else:
       result = rope("NIM_NIL")
   of nkStrLit..nkTripleStrLit:
-    case skipTypes(ty, abstractVarRange).kind
+    case skipTypes(ty, abstractVarRange + {tyStatic}).kind
     of tyNil:
       result = genNilStringLiteral(p.module, n.info)
     of tyString:
@@ -385,7 +385,7 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) =
     else:
       addrLoc(p.config, a)
 
-  var ty = skipTypes(dest.t, abstractVarRange)
+  var ty = skipTypes(dest.t, abstractVarRange + {tyStatic})
   case ty.kind
   of tyPtr, tyRef, tyProc, tyTuple, tyObject, tyArray:
     # XXX optimize this

compiler/ccgstmts.nim

@@ -978,12 +978,13 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
   if isAsmStmt and hasGnuAsm in CC[p.config.cCompiler].props:
     for x in splitLines(res):
       var j = 0
-      while x[j] in {' ', '\t'}: inc(j)
+      while j < x.len and x[j] in {' ', '\t'}: inc(j)
+      if j < x.len:
         if x[j] in {'"', ':'}:
           # don't modify the line if already in quotes or
           # some clobber register list:
           add(result, x); add(result, "\L")
-      elif x[j] != '\0':
+        else:
           # ignore empty lines
           add(result, "\"")
           add(result, x)

compiler/commands.nim

@@ -606,7 +606,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     incl(conf.globalOptions, optRun)
   of "verbosity":
     expectArg(conf, switch, arg, pass, info)
-    conf.verbosity = parseInt(arg)
+    let verbosity = parseInt(arg)
+    if verbosity notin {0..3}:
+      localError(conf, info, "invalid verbosity level: '$1'" % arg)
+    conf.verbosity = verbosity
     conf.notes = NotesVerbosity[conf.verbosity]
     incl(conf.notes, conf.enableNotes)
     excl(conf.notes, conf.disableNotes)

compiler/docgen.nim

@@ -582,7 +582,7 @@ proc traceDeps(d: PDoc, it: PNode) =
     if d.section[k] != nil: add(d.section[k], ", ")
     dispA(d.conf, d.section[k],
           "<a class=\"reference external\" href=\"$1.html\">$1</a>",
-          "$1", [rope(getModuleName(d.conf, it))])
+          "$1", [rope(splitFile(getModuleName(d.conf, it)).name)])
 
 proc generateDoc*(d: PDoc, n: PNode) =
   case n.kind
@@ -780,13 +780,11 @@ proc getOutFile2(conf: ConfigRef; filename, ext, dir: string): string =
 proc writeOutput*(d: PDoc, filename, outExt: string, useWarning = false) =
   var content = genOutFile(d)
   var success = true
-  var filename: string
   if optStdout in d.conf.globalOptions:
     writeRope(stdout, content)
-    filename = "<stdout>"
   else:
-    filename = getOutFile2(d.conf, filename, outExt, "htmldocs")
+    let outfile = getOutFile2(d.conf, filename, outExt, "htmldocs")
-    success = writeRope(content, filename)
+    success = writeRope(content, outfile)
   if not success:
     rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile, filename)
 

compiler/extccomp.nim

@@ -603,7 +603,7 @@ proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) =
 
 proc addExternalFileToCompile*(conf: ConfigRef; filename: string) =
   var c = Cfile(cname: filename,
-    obj: toObjFile(conf, completeCFilePath(conf, changeFileExt(filename, ""), false)),
+    obj: toObjFile(conf, completeCFilePath(conf, filename, false)),
     flags: {CfileFlag.External})
   addExternalFileToCompile(conf, c)
 

compiler/layouter.nim

@@ -7,11 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## Layouter for nimpretty. Still primitive but useful.
+## Layouter for nimpretty.
-## TODO
-## - Fix 'echo ()' vs 'echo()' difference!
-## - Make indentations consistent.
-## - Align 'if' and 'case' expressions properly.
 
 import idents, lexer, lineinfos, llstream, options, msgs, strutils
 from os import changeFileExt
@@ -24,32 +20,44 @@ type
   SplitKind = enum
     splitComma, splitParLe, splitAnd, splitOr, splitIn, splitBinary
 
+  SemicolonKind = enum
+    detectSemicolonKind, useSemicolon, dontTouch
+
   Emitter* = object
-    f: PLLStream
     config: ConfigRef
     fid: FileIndex
     lastTok: TTokType
     inquote: bool
-    col, lastLineNumber, lineSpan, indentLevel: int
+    semicolons: SemicolonKind
+    col, lastLineNumber, lineSpan, indentLevel, indWidth: int
+    nested: int
+    doIndentMore*: int
     content: string
+    indentStack: seq[int]
     fixedUntil: int # marks where we must not go in the content
     altSplitPos: array[SplitKind, int] # alternative split positions
 
-proc openEmitter*(em: var Emitter, config: ConfigRef, fileIdx: FileIndex) =
+proc openEmitter*(em: var Emitter, cache: IdentCache;
-  let outfile = changeFileExt(config.toFullPath(fileIdx), ".pretty.nim")
+                  config: ConfigRef, fileIdx: FileIndex) =
-  em.f = llStreamOpen(outfile, fmWrite)
+  let fullPath = config.toFullPath(fileIdx)
+  em.indWidth = getIndentWidth(fileIdx, llStreamOpen(fullPath, fmRead),
+                               cache, config)
+  if em.indWidth == 0: em.indWidth = 2
   em.config = config
   em.fid = fileIdx
   em.lastTok = tkInvalid
   em.inquote = false
   em.col = 0
   em.content = newStringOfCap(16_000)
-  if em.f == nil:
+  em.indentStack = newSeqOfCap[int](30)
-    rawMessage(config, errGenerated, "cannot open file: " & outfile)
+  em.indentStack.add 0
 
 proc closeEmitter*(em: var Emitter) =
-  em.f.llStreamWrite em.content
+  var f = llStreamOpen(em.config.outFile, fmWrite)
-  llStreamClose(em.f)
+  if f == nil:
+    rawMessage(em.config, errGenerated, "cannot open file: " & em.config.outFile)
+  f.llStreamWrite em.content
+  llStreamClose(f)
 
 proc countNewlines(s: string): int =
   result = 0
@@ -69,28 +77,41 @@ template wr(x) =
 
 template goodCol(col): bool = col in 40..MaxLineLen
 
-const splitters = {tkComma, tkSemicolon, tkParLe, tkParDotLe,
+const
+  openPars = {tkParLe, tkParDotLe,
               tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
               tkCurlyLe}
+  splitters = openPars + {tkComma, tkSemicolon}
+  oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
+            tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
 
 template rememberSplit(kind) =
   if goodCol(em.col):
     em.altSplitPos[kind] = em.content.len
 
+template moreIndent(em): int =
+  (if em.doIndentMore > 0: em.indWidth*2 else: em.indWidth)
+
 proc softLinebreak(em: var Emitter, lit: string) =
   # XXX Use an algorithm that is outlined here:
   # https://llvm.org/devmtg/2013-04/jasper-slides.pdf
   # +2 because we blindly assume a comma or ' &' might follow
   if not em.inquote and em.col+lit.len+2 >= MaxLineLen:
     if em.lastTok in splitters:
+      while em.content.len > 0 and em.content[em.content.high] == ' ':
+        setLen(em.content, em.content.len-1)
       wr("\L")
       em.col = 0
-      for i in 1..em.indentLevel+2: wr(" ")
+      for i in 1..em.indentLevel+moreIndent(em): wr(" ")
     else:
       # search backwards for a good split position:
       for a in em.altSplitPos:
         if a > em.fixedUntil:
-          let ws = "\L" & repeat(' ',em.indentLevel+2)
+          var spaces = 0
+          while a+spaces < em.content.len and em.content[a+spaces] == ' ':
+            inc spaces
+          if spaces > 0: delete(em.content, a, a+spaces-1)
+          let ws = "\L" & repeat(' ',em.indentLevel+moreIndent(em))
           em.col = em.content.len - a
           em.content.insert(ws, a)
           break
@@ -98,7 +119,7 @@ proc softLinebreak(em: var Emitter, lit: string) =
 proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
 
   template endsInWhite(em): bool =
-    em.content.len > 0 and em.content[em.content.high] in {' ', '\L'}
+    em.content.len == 0 or em.content[em.content.high] in {' ', '\L'}
   template endsInAlpha(em): bool =
     em.content.len > 0 and em.content[em.content.high] in SymChars+{'_'}
 
@@ -120,14 +141,32 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     em.fixedUntil = em.content.high
 
   elif tok.indent >= 0:
+    if em.lastTok in (splitters + oprSet):
       em.indentLevel = tok.indent
+    else:
+      if tok.indent > em.indentStack[^1]:
+        em.indentStack.add tok.indent
+      else:
+        # dedent?
+        while em.indentStack.len > 1 and em.indentStack[^1] > tok.indent:
+          discard em.indentStack.pop()
+      em.indentLevel = em.indentStack.high * em.indWidth
+    #[ we only correct the indentation if it is not in an expression context,
+       so that code like
+
+        const splitters = {tkComma, tkSemicolon, tkParLe, tkParDotLe,
+                          tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
+                          tkCurlyLe}
+
+       is not touched.
+    ]#
     # remove trailing whitespace:
     while em.content.len > 0 and em.content[em.content.high] == ' ':
       setLen(em.content, em.content.len-1)
     wr("\L")
     for i in 2..tok.line - em.lastLineNumber: wr("\L")
     em.col = 0
-    for i in 1..tok.indent:
+    for i in 1..em.indentLevel:
       wr(" ")
     em.fixedUntil = em.content.high
 
@@ -135,9 +174,12 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
   of tokKeywordLow..tokKeywordHigh:
     if endsInAlpha(em):
       wr(" ")
-    elif not em.inquote and not endsInWhite(em):
+    elif not em.inquote and not endsInWhite(em) and
+        em.lastTok notin openPars:
+      #and tok.tokType in oprSet
       wr(" ")
 
+    if not em.inquote:
       wr(TokTypeToStr[tok.tokType])
 
       case tok.tokType
@@ -147,37 +189,49 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
         rememberSplit(splitIn)
         wr(" ")
       else: discard
+    else:
+      # keywords in backticks are not normalized:
+      wr(tok.ident.s)
 
   of tkColon:
     wr(TokTypeToStr[tok.tokType])
     wr(" ")
   of tkSemicolon, tkComma:
     wr(TokTypeToStr[tok.tokType])
-    wr(" ")
     rememberSplit(splitComma)
-  of tkParLe, tkParRi, tkBracketLe,
-     tkBracketRi, tkCurlyLe, tkCurlyRi,
-     tkBracketDotLe, tkBracketDotRi,
-     tkCurlyDotLe, tkCurlyDotRi,
-     tkParDotLe, tkParDotRi,
-     tkColonColon, tkDot, tkBracketLeColon:
-    wr(TokTypeToStr[tok.tokType])
-    if tok.tokType in splitters:
-      rememberSplit(splitParLe)
-  of tkEquals:
-    if not em.endsInWhite: wr(" ")
-    wr(TokTypeToStr[tok.tokType])
     wr(" ")
+  of tkParDotLe, tkParLe, tkBracketDotLe, tkBracketLe,
+     tkCurlyLe, tkCurlyDotLe, tkBracketLeColon:
+    if tok.strongSpaceA > 0 and not em.endsInWhite:
+      wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+    rememberSplit(splitParLe)
+  of tkParRi,
+     tkBracketRi, tkCurlyRi,
+     tkBracketDotRi,
+     tkCurlyDotRi,
+     tkParDotRi,
+     tkColonColon, tkDot:
+    wr(TokTypeToStr[tok.tokType])
+  of tkEquals:
+    if not em.inquote and not em.endsInWhite: wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+    if not em.inquote: wr(" ")
   of tkOpr, tkDotDot:
+    if tok.strongSpaceA == 0 and tok.strongSpaceB == 0:
+      # if not surrounded by whitespace, don't produce any whitespace either:
+      wr(tok.ident.s)
+    else:
       if not em.endsInWhite: wr(" ")
       wr(tok.ident.s)
       template isUnary(tok): bool =
         tok.strongSpaceB == 0 and tok.strongSpaceA > 0
 
-    if not isUnary(tok) or em.lastTok in {tkOpr, tkDotDot}:
+      if not isUnary(tok):
         wr(" ")
         rememberSplit(splitBinary)
   of tkAccent:
+    if not em.inquote and endsInAlpha(em): wr(" ")
     wr(TokTypeToStr[tok.tokType])
     em.inquote = not em.inquote
   of tkComment:
@@ -206,3 +260,15 @@ proc starWasExportMarker*(em: var Emitter) =
     setLen(em.content, em.content.len-3)
     em.content.add("*")
     dec em.col, 2
+
+proc commaWasSemicolon*(em: var Emitter) =
+  if em.semicolons == detectSemicolonKind:
+    em.semicolons = if em.content.endsWith(", "): dontTouch else: useSemicolon
+  if em.semicolons == useSemicolon and em.content.endsWith(", "):
+    setLen(em.content, em.content.len-2)
+    em.content.add("; ")
+
+proc curlyRiWasPragma*(em: var Emitter) =
+  if em.content.endsWith("}"):
+    setLen(em.content, em.content.len-1)
+    em.content.add(".}")

compiler/lexer.nim

@@ -867,7 +867,7 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
-proc newlineFollows*(L: var TLexer): bool =
+proc newlineFollows*(L: TLexer): bool =
   var pos = L.bufpos
   var buf = L.buf
   while true:
@@ -1220,3 +1220,15 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
         lexMessage(L, errGenerated, "invalid token: " & c & " (\\" & $(ord(c)) & ')')
         inc(L.bufpos)
   atTokenEnd()
+
+proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
+                     cache: IdentCache; config: ConfigRef): int =
+  var lex: TLexer
+  var tok: TToken
+  initToken(tok)
+  openLexer(lex, fileIdx, inputstream, cache, config)
+  while true:
+    rawGetTok(lex, tok)
+    result = tok.indent
+    if result > 0 or tok.tokType == tkEof: break
+  closeLexer(lex)

compiler/parser.nim

@@ -108,7 +108,7 @@ proc openParser*(p: var TParser, fileIdx: FileIndex, inputStream: PLLStream,
   initToken(p.tok)
   openLexer(p.lex, fileIdx, inputStream, cache, config)
   when defined(nimpretty2):
-    openEmitter(p.em, config, fileIdx)
+    openEmitter(p.em, cache, config, fileIdx)
   getTok(p)                   # read the first token
   p.firstTok = true
   p.strongSpaces = strongSpaces
@@ -416,6 +416,8 @@ proc exprColonEqExpr(p: var TParser): PNode =
 
 proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
   #| exprList = expr ^+ comma
+  when defined(nimpretty2):
+    inc p.em.doIndentMore
   getTok(p)
   optInd(p, result)
   # progress guaranteed
@@ -425,6 +427,8 @@ proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
     if p.tok.tokType != tkComma: break
     getTok(p)
     optInd(p, a)
+  when defined(nimpretty2):
+    dec p.em.doIndentMore
 
 proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) =
   assert(endTok in {tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi})
@@ -856,7 +860,11 @@ proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode =
   result = parseOperators(p, result, limit, mode)
 
 proc simpleExpr(p: var TParser, mode = pmNormal): PNode =
+  when defined(nimpretty2):
+    inc p.em.doIndentMore
   result = simpleExprAux(p, -1, mode)
+  when defined(nimpretty2):
+    dec p.em.doIndentMore
 
 proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode =
   #| condExpr = expr colcom expr optInd
@@ -931,8 +939,12 @@ proc parsePragma(p: var TParser): PNode =
       getTok(p)
       skipComment(p, a)
   optPar(p)
-  if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}: getTok(p)
+  if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}:
-  else: parMessage(p, "expected '.}'")
+    when defined(nimpretty2):
+      if p.tok.tokType == tkCurlyRi: curlyRiWasPragma(p.em)
+    getTok(p)
+  else:
+    parMessage(p, "expected '.}'")
   dec p.inPragma
 
 proc identVis(p: var TParser; allowDot=false): PNode =
@@ -1019,6 +1031,8 @@ proc parseTuple(p: var TParser, indentAllowed = false): PNode =
       var a = parseIdentColonEquals(p, {})
       addSon(result, a)
       if p.tok.tokType notin {tkComma, tkSemiColon}: break
+      when defined(nimpretty2):
+        commaWasSemicolon(p.em)
       getTok(p)
       skipComment(p, a)
     optPar(p)
@@ -1053,6 +1067,8 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
   var a: PNode
   result = newNodeP(nkFormalParams, p)
   addSon(result, p.emptyNode) # return type
+  when defined(nimpretty2):
+    inc p.em.doIndentMore
   let hasParLe = p.tok.tokType == tkParLe and p.tok.indent < 0
   if hasParLe:
     getTok(p)
@@ -1072,6 +1088,8 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
         break
       addSon(result, a)
       if p.tok.tokType notin {tkComma, tkSemiColon}: break
+      when defined(nimpretty2):
+        commaWasSemicolon(p.em)
       getTok(p)
       skipComment(p, a)
     optPar(p)
@@ -1085,6 +1103,8 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
   elif not retColon and not hasParle:
     # Mark as "not there" in order to mark for deprecation in the semantic pass:
     result = p.emptyNode
+  when defined(nimpretty2):
+    dec p.em.doIndentMore
 
 proc optPragmas(p: var TParser): PNode =
   if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)):
@@ -1678,6 +1698,8 @@ proc parseGenericParamList(p: var TParser): PNode =
     var a = parseGenericParam(p)
     addSon(result, a)
     if p.tok.tokType notin {tkComma, tkSemiColon}: break
+    when defined(nimpretty2):
+      commaWasSemicolon(p.em)
     getTok(p)
     skipComment(p, a)
   optPar(p)

compiler/pragmas.nim

@@ -466,7 +466,7 @@ proc processCompile(c: PContext, n: PNode) =
       else:
         found = findFile(c.config, s)
         if found.len == 0: found = s
-    let obj = toObjFile(c.config, completeCFilePath(c.config, changeFileExt(found, ""), false))
+    let obj = toObjFile(c.config, completeCFilePath(c.config, found, false))
     docompile(c, it, found, obj)
 
 proc processCommonLink(c: PContext, n: PNode, feature: TLinkFeature) =

compiler/semexprs.nim

@@ -289,7 +289,7 @@ proc semLowHigh(c: PContext, n: PNode, m: TMagic): PNode =
     localError(c.config, n.info, errXExpectsTypeOrValue % opToStr[m])
   else:
     n.sons[1] = semExprWithType(c, n.sons[1], {efDetermineType})
-    var typ = skipTypes(n.sons[1].typ, abstractVarRange + {tyTypeDesc})
+    var typ = skipTypes(n.sons[1].typ, abstractVarRange + {tyTypeDesc, tyUserTypeClassInst})
     case typ.kind
     of tySequence, tyString, tyCString, tyOpenArray, tyVarargs:
       n.typ = getSysType(c.graph, n.info, tyInt)
@@ -1351,7 +1351,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
   # make sure we don't evaluate generic macros/templates
   n.sons[0] = semExprWithType(c, n.sons[0],
                               {efNoEvaluateGeneric})
-  var arr = skipTypes(n.sons[0].typ, {tyGenericInst,
+  var arr = skipTypes(n.sons[0].typ, {tyGenericInst, tyUserTypeClassInst,
                                       tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink})
   if arr.kind == tyStatic:
     if arr.base.kind == tyNone:

compiler/semfold.nim

@@ -211,7 +211,12 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode =
   of mUnaryMinusF64: result = newFloatNodeT(- getFloat(a), n, g)
   of mNot: result = newIntNodeT(1 - getInt(a), n, g)
   of mCard: result = newIntNodeT(nimsets.cardSet(g.config, a), n, g)
-  of mBitnotI: result = newIntNodeT(not getInt(a), n, g)
+  of mBitnotI:
+    case skipTypes(n.typ, abstractRange).kind
+    of tyUInt..tyUInt64:
+      result = newIntNodeT((not getInt(a)) and lastOrd(g.config, a.typ, fixedUnsigned=true), n, g)
+    else:
+      result = newIntNodeT(not getInt(a), n, g)
   of mLengthArray: result = newIntNodeT(lengthOrd(g.config, a.typ), n, g)
   of mLengthSeq, mLengthOpenArray, mXLenSeq, mLengthStr, mXLenStr:
     if a.kind == nkNilLit:
@@ -250,8 +255,10 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode =
     of tyInt8: result = newIntNodeT(int8(getInt(a)) shl int8(getInt(b)), n, g)
     of tyInt16: result = newIntNodeT(int16(getInt(a)) shl int16(getInt(b)), n, g)
     of tyInt32: result = newIntNodeT(int32(getInt(a)) shl int32(getInt(b)), n, g)
-    of tyInt64, tyInt, tyUInt..tyUInt64:
+    of tyInt64, tyInt:
       result = newIntNodeT(`shl`(getInt(a), getInt(b)), n, g)
+    of tyUInt..tyUInt64:
+      result = newIntNodeT(`shl`(getInt(a), getInt(b)) and lastOrd(g.config, a.typ, fixedUnsigned=true), n, g)
     else: internalError(g.config, n.info, "constant folding for shl")
   of mShrI:
     case skipTypes(n.typ, abstractRange).kind
@@ -612,7 +619,7 @@ proc getConstExpr(m: PSym, n: PNode; g: ModuleGraph): PNode =
       of mLow:
         result = newIntNodeT(firstOrd(g.config, n.sons[1].typ), n, g)
       of mHigh:
-        if skipTypes(n.sons[1].typ, abstractVar).kind notin
+        if skipTypes(n.sons[1].typ, abstractVar+{tyUserTypeClassInst}).kind notin
             {tySequence, tyString, tyCString, tyOpenArray, tyVarargs}:
           result = newIntNodeT(lastOrd(g.config, skipTypes(n[1].typ, abstractVar)), n, g)
         else:

compiler/semstmts.nim

@@ -519,7 +519,9 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         localError(c.config, a.info, errWrongNumberOfVariables)
       b = newNodeI(nkVarTuple, a.info)
       newSons(b, length)
-      b.sons[length-2] = a.sons[length-2] # keep type desc for doc generator
+      # keep type desc for doc generator
+      # NOTE: at the moment this is always ast.emptyNode, see parser.nim
+      b.sons[length-2] = a.sons[length-2]
       b.sons[length-1] = def
       addToVarSection(c, result, n, b)
     elif tup.kind == tyTuple and def.kind in {nkPar, nkTupleConstr} and
@@ -558,7 +560,12 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
           # keep documentation information:
           b.comment = a.comment
         addSon(b, newSymNode(v))
-        addSon(b, a.sons[length-2])      # keep type desc for doc generator
+        # keep type desc for doc generator, but only if the user explicitly
+        # added it
+        if a.sons[length-2].kind != nkEmpty:
+          addSon(b, newNodeIT(nkType, a.info, typ))
+        else:
+          addSon(b, a.sons[length-2])
         addSon(b, copyTree(def))
         addToVarSection(c, result, n, b)
       else:

compiler/sighashes.nim

@@ -189,18 +189,19 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
         c.hashTypeSym(t.sym)
       else:
         c.hashSym(t.sym)
-      if sfAnon in t.sym.flags:
+      if {sfAnon, sfGenSym} * t.sym.flags != {}:
         # generated object names can be identical, so we need to
         # disambiguate furthermore by hashing the field types and names:
         # mild hack to prevent endless recursions (makes nimforum compile again):
-        excl t.sym.flags, sfAnon
+        let oldFlags = t.sym.flags
+        t.sym.flags = t.sym.flags - {sfAnon, sfGenSym}
         let n = t.n
         for i in 0 ..< n.len:
           assert n[i].kind == nkSym
           let s = n[i].sym
           c.hashSym s
           c.hashType s.typ, flags
-        incl t.sym.flags, sfAnon
+        t.sym.flags = oldFlags
     else:
       c &= t.id
     if t.len > 0 and t.sons[0] != nil:

compiler/types.nim

@@ -618,7 +618,7 @@ proc firstOrd*(conf: ConfigRef; t: PType): BiggestInt =
     else:
       assert(t.n.sons[0].kind == nkSym)
       result = t.n.sons[0].sym.position
-  of tyGenericInst, tyDistinct, tyTypeDesc, tyAlias, tyStatic, tyInferred:
+  of tyGenericInst, tyDistinct, tyTypeDesc, tyAlias, tyStatic, tyInferred, tyUserTypeClassInst:
     result = firstOrd(conf, lastSon(t))
   of tyOrdinal:
     if t.len > 0: result = firstOrd(conf, lastSon(t))

doc/tut1.rst

@@ -208,7 +208,8 @@ Note that declaring multiple variables with a single assignment which calls a
 procedure can have unexpected results: the compiler will *unroll* the
 assignments and end up calling the procedure several times. If the result of
 the procedure depends on side effects, your variables may end up having
-different values! For safety use only constant values.
+different values! For safety use side-effect free procedures if making multiple
+assignments.
 
 
 Constants
@@ -642,7 +643,7 @@ initialisation.
 
 Parameters
 ----------
-Parameters are constant in the procedure body. By default, their value cannot be
+Parameters are immutable in the procedure body. By default, their value cannot be
 changed because this allows the compiler to implement parameter passing in the
 most efficient way. If a mutable variable is needed inside the procedure, it has
 to be declared with ``var`` in the procedure body. Shadowing the parameter name

koch.nim

@@ -254,15 +254,13 @@ proc buildTool(toolname, args: string) =
   copyFile(dest="bin" / splitFile(toolname).name.exe, source=toolname.exe)
 
 proc buildTools(latest: bool) =
-  let nimsugExe = "bin/nimsuggest".exe
+  nimexec "c --noNimblePath -p:compiler -d:release -o:" & ("bin/nimsuggest".exe) &
-  nimexec "c --noNimblePath -p:compiler -d:release -o:" & nimsugExe &
       " nimsuggest/nimsuggest.nim"
 
-  let nimgrepExe = "bin/nimgrep".exe
+  nimexec "c -d:release -o:" & ("bin/nimgrep".exe) & " tools/nimgrep.nim"
-  nimexec "c -d:release -o:" & nimgrepExe & " tools/nimgrep.nim"
   when defined(windows): buildVccTool()
 
-  #nimexec "c -o:" & ("bin/nimresolve".exe) & " tools/nimresolve.nim"
+  nimexec "c -o:" & ("bin/nimpretty".exe) & " nimpretty/nimpretty.nim"
 
   buildNimble(latest)
 

lib/pure/httpclient.nim

@@ -1184,7 +1184,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string,
   ## Connects to the hostname specified by the URL and performs a request
   ## using the custom method string specified by ``httpMethod``.
   ##
-  ## Connection will kept alive. Further requests on the same ``client`` to
+  ## Connection will be kept alive. Further requests on the same ``client`` to
   ## the same hostname will not require a new connection to be made. The
   ## connection can be closed by using the ``close`` procedure.
   ##

lib/pure/pegs.nim

@@ -32,7 +32,7 @@ const
                        ## can be captured. More subpatterns cannot be captured!
 
 type
-  PegKind = enum
+  PegKind* = enum
     pkEmpty,
     pkAny,              ## any character (.)
     pkAnyRune,          ## any Unicode character (_)
@@ -67,7 +67,7 @@ type
     pkRule,             ## a <- b
     pkList,             ## a, b
     pkStartAnchor       ## ^      --> Internal DSL: startAnchor()
-  NonTerminalFlag = enum
+  NonTerminalFlag* = enum
     ntDeclared, ntUsed
   NonTerminalObj = object         ## represents a non terminal symbol
     name: string                  ## the name of the symbol
@@ -86,6 +86,25 @@ type
     else: sons: seq[Peg]
   NonTerminal* = ref NonTerminalObj
 
+proc name*(nt: NonTerminal): string = nt.name
+proc line*(nt: NonTerminal): int = nt.line
+proc col*(nt: NonTerminal): int = nt.col
+proc flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags
+proc rule*(nt: NonTerminal): Peg = nt.rule
+
+proc kind*(p: Peg): PegKind = p.kind
+proc term*(p: Peg): string = p.term
+proc ch*(p: Peg): char = p.ch
+proc charChoice*(p: Peg): ref set[char] = p.charChoice
+proc nt*(p: Peg): NonTerminal = p.nt
+proc index*(p: Peg): range[0..MaxSubpatterns] = p.index
+iterator items*(p: Peg): Peg {.inline.} =
+  for s in p.sons:
+    yield s
+iterator pairs*(p: Peg): (int, Peg) {.inline.} =
+  for i in 0 ..< p.sons.len:
+    yield (i, p.sons[i])
+
 proc term*(t: string): Peg {.nosideEffect, rtl, extern: "npegs$1Str".} =
   ## constructs a PEG from a terminal string
   if t.len != 1:

lib/pure/strutils.nim

@@ -17,6 +17,10 @@ import parseutils
 from math import pow, round, floor, log10
 from algorithm import reverse
 
+when defined(nimVmExportFixed):
+  from unicode import toLower, toUpper
+  export toLower, toUpper
+
 {.deadCodeElim: on.}  # dce option deprecated
 
 {.push debugger:off .} # the user does not want to trace a part

lib/pure/terminal.nim

@@ -470,13 +470,15 @@ type
   Style* = enum          ## different styles for text output
     styleBright = 1,     ## bright text
     styleDim,            ## dim text
-    styleUnknown,        ## unknown
+    styleItalic,         ## italic (or reverse on terminals not supporting)
     styleUnderscore = 4, ## underscored text
     styleBlink,          ## blinking/bold text
-    styleReverse = 7,    ## unknown
+    styleReverse = 7,    ## reverse
     styleHidden          ## hidden text
+    styleStrikethrough,  ## strikethrough
 
 {.deprecated: [TStyle: Style].}
+{.deprecated: [styleUnknown: styleItalic].}
 
 when not defined(windows):
   var
@@ -540,7 +542,9 @@ type
     fgBlue,                ## blue
     fgMagenta,             ## magenta
     fgCyan,                ## cyan
-    fgWhite                ## white
+    fgWhite,               ## white
+    fg8Bit,                ## 256-color (not supported, see ``enableTrueColors`` instead.)
+    fgDefault              ## default terminal foreground color
 
   BackgroundColor* = enum  ## terminal's background colors
     bgBlack = 40,          ## black
@@ -550,27 +554,39 @@ type
     bgBlue,                ## blue
     bgMagenta,             ## magenta
     bgCyan,                ## cyan
-    bgWhite                ## white
+    bgWhite,               ## white
+    bg8Bit,                ## 256-color (not supported, see ``enableTrueColors`` instead.)
+    bgDefault              ## default terminal background color
 
 {.deprecated: [TForegroundColor: ForegroundColor,
                TBackgroundColor: BackgroundColor].}
 
+when defined(windows):
+  var defaultForegroundColor, defaultBackgroundColor: int16 = 0xFFFF'i16 # Default to an invalid value 0xFFFF
+
 proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) =
   ## Sets the terminal's foreground color.
   when defined(windows):
     let h = conHandle(f)
     var old = getAttributes(h) and not FOREGROUND_RGB
+    if defaultForegroundColor == 0xFFFF'i16:
+      defaultForegroundColor = old
     old = if bright: old or FOREGROUND_INTENSITY
           else:      old and not(FOREGROUND_INTENSITY)
     const lookup: array[ForegroundColor, int] = [
-      0,
+      0, # ForegroundColor enum with ordinal 30
       (FOREGROUND_RED),
       (FOREGROUND_GREEN),
       (FOREGROUND_RED or FOREGROUND_GREEN),
       (FOREGROUND_BLUE),
       (FOREGROUND_RED or FOREGROUND_BLUE),
       (FOREGROUND_BLUE or FOREGROUND_GREEN),
-      (FOREGROUND_BLUE or FOREGROUND_GREEN or FOREGROUND_RED)]
+      (FOREGROUND_BLUE or FOREGROUND_GREEN or FOREGROUND_RED),
+      0, # fg8Bit not supported, see ``enableTrueColors`` instead.
+      0] # unused
+    if fg == fgDefault:
+      discard setConsoleTextAttribute(h, toU16(old or defaultForegroundColor))
+    else:
       discard setConsoleTextAttribute(h, toU16(old or lookup[fg]))
   else:
     gFG = ord(fg)
@@ -582,17 +598,24 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) =
   when defined(windows):
     let h = conHandle(f)
     var old = getAttributes(h) and not BACKGROUND_RGB
+    if defaultBackgroundColor == 0xFFFF'i16:
+      defaultBackgroundColor = old
     old = if bright: old or BACKGROUND_INTENSITY
           else:      old and not(BACKGROUND_INTENSITY)
     const lookup: array[BackgroundColor, int] = [
-      0,
+      0, # BackgroundColor enum with ordinal 40
       (BACKGROUND_RED),
       (BACKGROUND_GREEN),
       (BACKGROUND_RED or BACKGROUND_GREEN),
       (BACKGROUND_BLUE),
       (BACKGROUND_RED or BACKGROUND_BLUE),
       (BACKGROUND_BLUE or BACKGROUND_GREEN),
-      (BACKGROUND_BLUE or BACKGROUND_GREEN or BACKGROUND_RED)]
+      (BACKGROUND_BLUE or BACKGROUND_GREEN or BACKGROUND_RED),
+      0, # bg8Bit not supported, see ``enableTrueColors`` instead.
+      0] # unused
+    if bg == bgDefault:
+      discard setConsoleTextAttribute(h, toU16(old or defaultBackgroundColor))
+    else:
       discard setConsoleTextAttribute(h, toU16(old or lookup[bg]))
   else:
     gBG = ord(bg)
@@ -690,8 +713,8 @@ template styledEchoProcessArg(f: File, cmd: TerminalCmd) =
   when cmd == bgColor:
     fgSetColor = false
 
-macro styledWriteLine*(f: File, m: varargs[typed]): untyped =
+macro styledWrite*(f: File, m: varargs[typed]): untyped =
-  ## Similar to ``writeLine``, but treating terminal style arguments specially.
+  ## Similar to ``write``, but treating terminal style arguments specially.
   ## When some argument is ``Style``, ``set[Style]``, ``ForegroundColor``,
   ## ``BackgroundColor`` or ``TerminalCmd`` then it is not sent directly to
   ## ``f``, but instead corresponding terminal style proc is called.
@@ -700,8 +723,8 @@ macro styledWriteLine*(f: File, m: varargs[typed]): untyped =
   ##
   ## .. code-block:: nim
   ##
-  ##   proc error(msg: string) =
+  ##   stdout.styledWrite(fgRed, "red text ")
-  ##     styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg)
+  ##   stdout.styledWrite(fgGreen, "green text")
   ##
   let m = callsite()
   var reset = false
@@ -712,8 +735,8 @@ macro styledWriteLine*(f: File, m: varargs[typed]): untyped =
     case item.kind
     of nnkStrLit..nnkTripleStrLit:
       if i == m.len - 1:
-        # optimize if string literal is last, just call writeLine
+        # optimize if string literal is last, just call write
-        result.add(newCall(bindSym"writeLine", f, item))
+        result.add(newCall(bindSym"write", f, item))
         if reset: result.add(newCall(bindSym"resetAttributes", f))
         return
       else:
@@ -722,16 +745,24 @@ macro styledWriteLine*(f: File, m: varargs[typed]): untyped =
     else:
       result.add(newCall(bindSym"styledEchoProcessArg", f, item))
       reset = true
-
-  result.add(newCall(bindSym"write", f, newStrLitNode("\n")))
   if reset: result.add(newCall(bindSym"resetAttributes", f))
 
-macro styledEcho*(args: varargs[untyped]): untyped =
+template styledWriteLine*(f: File, args: varargs[untyped]) =
+  ## Calls ``styledWrite`` and appends a newline at the end.
+  ##
+  ## Example:
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc error(msg: string) =
+  ##     styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg)
+  ##
+  styledWrite(f, args)
+  write(f, "\n")
+
+template styledEcho*(args: varargs[untyped]) =
   ## Echoes styles arguments to stdout using ``styledWriteLine``.
-  result = newCall(bindSym"styledWriteLine")
+  stdout.styledWriteLine(args)
-  result.add(bindSym"stdout")
-  for arg in children(args):
-    result.add(arg)
 
 proc getch*(): char =
   ## Read a single character from the terminal, blocking until it is entered.
@@ -838,16 +869,6 @@ proc resetAttributes*() {.noconv.} =
   ## ``system.addQuitProc(resetAttributes)``.
   resetAttributes(stdout)
 
-when not defined(testing) and isMainModule:
-  #system.addQuitProc(resetAttributes)
-  write(stdout, "never mind")
-  stdout.eraseLine()
-  stdout.styledWriteLine("styled text ", {styleBright, styleBlink, styleUnderscore})
-  stdout.setBackGroundColor(bgCyan, true)
-  stdout.setForeGroundColor(fgBlue)
-  stdout.writeLine("ordinary text")
-  stdout.resetAttributes()
-
 proc isTrueColorSupported*(): bool =
   ## Returns true if a terminal supports true color.
   return trueColorIsSupported
@@ -898,3 +919,43 @@ proc disableTrueColors*() =
       trueColorIsEnabled = false
   else:
     trueColorIsEnabled = false
+
+when not defined(testing) and isMainModule:
+  #system.addQuitProc(resetAttributes)
+  write(stdout, "never mind")
+  stdout.eraseLine()
+  stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ")
+  stdout.styledWriteLine("italic text ", {styleItalic})
+  stdout.setBackGroundColor(bgCyan, true)
+  stdout.setForeGroundColor(fgBlue)
+  stdout.write("blue text in cyan background")
+  stdout.resetAttributes()
+  echo ""
+  stdout.writeLine("ordinary text")
+  echo "more ordinary text"
+  styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!"
+  echo "ordinary text again"
+  styledEcho styleBright, fgRed, "[FAIL]", resetStyle, fgRed, " Nay :("
+  echo "ordinary text again"
+  setForeGroundColor(fgGreen)
+  echo "green text"
+  echo "more green text"
+  setForeGroundColor(fgBlue)
+  echo "blue text"
+  resetAttributes()
+  echo "ordinary text"
+
+  stdout.styledWriteLine(fgRed, "red text ")
+  stdout.styledWriteLine(fgWhite, bgRed, "white text in red background")
+  stdout.styledWriteLine(" ordinary text ")
+  stdout.styledWriteLine(fgGreen, "green text")
+
+  stdout.styledWrite(fgRed, "red text ")
+  stdout.styledWrite(fgWhite, bgRed, "white text in red background")
+  stdout.styledWrite(" ordinary text ")
+  stdout.styledWrite(fgGreen, "green text")
+  echo ""
+  echo "ordinary text"
+  stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text")
+  stdout.styledWriteLine(bgYellow, "text in yellow bg", styleBright, " bold text in yellow bg", bgDefault, " bold text")
+  echo "ordinary text"

lib/pure/times.nim

@@ -1243,7 +1243,7 @@ proc evaluateStaticInterval(interval: TimeInterval): Duration =
     minutes = interval.minutes,
     hours = interval.hours)
 
-proc between*(startDt, endDt:DateTime): TimeInterval =
+proc between*(startDt, endDt: DateTime): TimeInterval =
   ## Evaluate difference between two dates in ``TimeInterval`` format, so, it
   ## will be relative.
   ##
@@ -1526,7 +1526,6 @@ proc formatToken(dt: DateTime, token: string, buf: var string) =
   else:
     raise newException(ValueError, "Invalid format string: " & token)
 
-
 proc format*(dt: DateTime, f: string): string {.tags: [].}=
   ## This procedure formats `dt` as specified by `f`. The following format
   ## specifiers are available:
@@ -1601,18 +1600,14 @@ proc format*(dt: DateTime, f: string): string {.tags: [].}=
     inc(i)
   formatToken(dt, currentF, result)
 
-proc format*(time: Time, f: string, zone_info: proc(t: Time): DateTime): string {.tags: [].} =
+proc format*(time: Time, f: string, zone: Timezone = local()): string {.tags: [].} =
-  ## converts a `Time` value to a string representation. It will use format from
+  ## Converts a `Time` value to a string representation. It will use format from
   ## ``format(dt: DateTime, f: string)``.
   runnableExamples:
-    var dt = initDateTime(01, mJan, 1970, 00, 00, 00, local())
+    var dt = initDateTime(01, mJan, 1970, 00, 00, 00, utc())
     var tm = dt.toTime()
-    doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", local) == "1970-01-01T00:00:00"
+    doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc()) == "1970-01-01T00:00:00"
-    dt = initDateTime(01, mJan, 1970, 00, 00, 00, utc())
+  time.inZone(zone).format(f)
-    tm = dt.toTime()
-    doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc) == "1970-01-01T00:00:00"
-
-  zone_info(time).format(f)
 
 proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} =
   ## Converts a `DateTime` object to a string representation.
@@ -1984,16 +1979,12 @@ proc countYearsAndDays*(daySpan: int): tuple[years: int, days: int] =
 proc toTimeInterval*(time: Time): TimeInterval =
   ## Converts a Time to a TimeInterval.
   ##
-  ## To be used when diffing times.
+  ## To be used when diffing times. Consider using `between` instead.
   runnableExamples:
     let a = fromUnix(10)
-    let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local())
+    let b = fromUnix(1_500_000_000)
-    doAssert a.toTimeInterval() == initTimeInterval(
+    let ti = b.toTimeInterval() - a.toTimeInterval()
-      years=1970, days=1, seconds=10, hours=convert(
+    doAssert a + ti == b
-        Seconds, Hours, -dt.utcOffset
-      )
-    )
-
   var dt = time.local
   initTimeInterval(dt.nanosecond, 0, 0, dt.second, dt.minute, dt.hour,
     dt.monthday, 0, dt.month.ord - 1, dt.year)

lib/std/varints.nim

@@ -19,7 +19,7 @@ proc readVu64*(z: openArray[byte]; pResult: var uint64): int =
     return 1
   if z[0] <= 248:
     if z.len < 2: return 0
-    pResult = (uint64 z[0] - 241) * 256 + uint64 z[1] + 240
+    pResult = (uint64 z[0] - 241) * 256 + z[1].uint64 + 240
     return 2
   if z.len < int(z[0]-246): return 0
   if z[0] == 249:
@@ -135,6 +135,13 @@ when isMainModule:
     if encodeZigzag(decodeZigzag(test)) != test:
       echo "Failure for ", test, " ", encodeZigzag(decodeZigzag(test)), " ", decodeZigzag(test)
 
+  for test in 0u64..300u64:
+    let wrLen = writeVu64(dest, test)
+    let rdLen = readVu64(dest, got)
+    assert wrLen == rdLen
+    if got != test:
+      echo "BUG! expected: ", test, " got: ", got, " z0: ", dest[0]
+
   # check this also works for floats:
   for test in [0.0, 0.1, 2.0, +Inf, Nan, NegInf]:
     let t = cast[uint64](test)

nimpretty/nimpretty.nim

@@ -25,6 +25,7 @@ Usage:
   nimpretty [options] file.nim
 Options:
   --backup:on|off     create a backup file before overwritting (default: ON)
+  --output:file       set the output file (default: overwrite the .nim file)
   --version           show the version
   --help              show this help
 """
@@ -39,18 +40,18 @@ proc writeVersion() =
   stdout.flushFile()
   quit(0)
 
-proc prettyPrint(infile: string) =
+proc prettyPrint(infile, outfile: string) =
-  let conf = newConfigRef()
+  var conf = newConfigRef()
   let fileIdx = fileInfoIdx(conf, infile)
+  conf.outFile = outfile
   when defined(nimpretty2):
     discard parseFile(fileIdx, newIdentCache(), conf)
   else:
     let tree = parseFile(fileIdx, newIdentCache(), conf)
-    let outfile = changeFileExt(infile, ".pretty.nim")
     renderModule(tree, infile, outfile, {}, fileIdx, conf)
 
 proc main =
-  var infile: string
+  var infile, outfile: string
   var backup = true
   for kind, key, val in getopt():
     case kind
@@ -61,12 +62,14 @@ proc main =
       of "help", "h": writeHelp()
       of "version", "v": writeVersion()
       of "backup": backup = parseBool(val)
+      of "output", "o": outfile = val
       else: writeHelp()
     of cmdEnd: assert(false) # cannot happen
   if infile.len == 0:
     quit "[Error] no input file."
   if backup:
     os.copyFile(source=infile, dest=changeFileExt(infile, ".nim.backup"))
-  prettyPrint(infile)
+  if outfile.len == 0: outfile = infile
+  prettyPrint(infile, outfile)
 
 main()

nimpretty/tester.nim

@@ -0,0 +1,29 @@
+# Small program that runs the test cases
+
+import strutils, os
+
+const
+  dir = "nimpretty/tests/"
+
+var
+  failures = 0
+
+proc test(infile, outfile: string) =
+  if execShellCmd("nimpretty -o:$2 --backup:off $1" % [infile, outfile]) != 0:
+    quit("FAILURE")
+  let nimFile = splitFile(infile).name
+  let expected = dir / "expected" / nimFile & ".nim"
+  let produced = dir / nimFile & ".pretty"
+  if strip(readFile(expected)) != strip(readFile(produced)):
+    echo "FAILURE: files differ: ", nimFile
+    discard execShellCmd("diff -uNdr " & expected & " " & produced)
+    failures += 1
+  else:
+    echo "SUCCESS: files identical: ", nimFile
+
+for t in walkFiles(dir / "*.nim"):
+  let res = t.changeFileExt("pretty")
+  test(t, res)
+  removeFile(res)
+
+if failures > 0: quit($failures & " failures occurred.")

nimpretty/tests/exhaustive.nim

@@ -0,0 +1,316 @@
+discard """
+  outputsub: '''ObjectAssignmentError'''
+  exitcode: "1"
+"""
+
+import verylongnamehere,verylongnamehere,verylongnamehereverylongnamehereverylong,namehere,verylongnamehere
+
+proc `[]=`() = discard "index setter"
+proc `putter=`() = discard cast[pointer](cast[int](buffer) + size)
+
+(not false)
+
+let expr = if true: "true" else: "false"
+
+var body = newNimNode(nnkIfExpr).add(
+  newNimNode(nnkElifBranch).add(
+    infix(newDotExpr(ident("a"), ident("kind")), "==", newDotExpr(ident("b"), ident("kind"))),
+    condition
+  ),
+  newNimNode(nnkElse).add(newStmtList(newNimNode(nnkReturnStmt).add(ident("false"))))
+)
+
+# comment
+
+var x = 1
+
+type
+  GeneralTokenizer* = object of RootObj ## comment here
+    kind*: TokenClass ## and here
+    start*, length*: int ## you know how it goes...
+    buf: cstring
+    pos: int # other comment here.
+    state: TokenClass
+
+var x*: string
+var y: seq[string] #[ yay inline comments. So nice I have to care bout these. ]#
+
+echo "#", x, "##", y, "#" & "string" & $test
+
+echo (tup, here)
+echo(argA, argB)
+
+import macros
+
+## A documentation comment here.
+## That spans multiple lines.
+## And is not to be touched.
+
+const numbers = [4u8, 5'u16, 89898_00]
+
+macro m(n): untyped =
+  result = foo"string literal"
+
+{.push m.}
+proc p() = echo "p", 1+4 * 5, if true: 5 else: 6
+proc q(param: var ref ptr string) =
+  p()
+  if true:
+    echo a and b or not c and not -d
+{.pop.}
+
+q()
+
+when false:
+  # bug #4766
+  type
+    Plain = ref object
+      discard
+
+    Wrapped[T] = object
+      value: T
+
+  converter toWrapped[T](value: T): Wrapped[T] =
+    Wrapped[T](value: value)
+
+  let result = Plain()
+  discard $result
+
+when false:
+  # bug #3670
+  template someTempl(someConst: bool) =
+    when someConst:
+      var a: int
+    if true:
+      when not someConst:
+        var a: int
+      a = 5
+
+  someTempl(true)
+
+
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2018 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Layouter for nimpretty. Still primitive but useful.
+
+import idents, lexer, lineinfos, llstream, options, msgs, strutils
+from os import changeFileExt
+
+const
+  MaxLineLen = 80
+  LineCommentColumn = 30
+
+type
+  SplitKind = enum
+    splitComma, splitParLe, splitAnd, splitOr, splitIn, splitBinary
+
+  Emitter* = object
+    f: PLLStream
+    config: ConfigRef
+    fid: FileIndex
+    lastTok: TTokType
+    inquote {.pragmaHereWrongCurlyEnd}: bool
+    col, lastLineNumber, lineSpan, indentLevel: int
+    content: string
+    fixedUntil: int # marks where we must not go in the content
+    altSplitPos: array[SplitKind, int] # alternative split positions
+
+proc openEmitter*[T, S](em: var Emitter; config: ConfigRef, fileIdx: FileIndex) {.pragmaHereWrongCurlyEnd} =
+  let outfile = changeFileExt(config.toFullPath(fileIdx), ".pretty.nim")
+  em.f = llStreamOpen(outfile, fmWrite)
+  em.config = config
+  em.fid = fileIdx
+  em.lastTok = tkInvalid
+  em.inquote = false
+  em.col = 0
+  em.content = newStringOfCap(16_000)
+  if em.f == nil:
+    rawMessage(config, errGenerated, "cannot open file: " & outfile)
+
+proc closeEmitter*(em: var Emitter) {.inline.} =
+  em.f.llStreamWrite em.content
+  llStreamClose(em.f)
+
+proc countNewlines(s: string): int =
+  result = 0
+  for i in 0..<s.len:
+    if s[i+1] == '\L': inc result
+
+proc calcCol(em: var Emitter; s: string) =
+  var i = s.len-1
+  em.col = 0
+  while i >= 0 and s[i] != '\L':
+    dec i
+    inc em.col
+
+template wr(x) =
+  em.content.add x
+  inc em.col, x.len
+
+template goodCol(col): bool = col in 40..MaxLineLen
+
+const splitters = {tkComma, tkSemicolon, tkParLe, tkParDotLe,
+                   tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
+                   tkCurlyLe}
+
+template rememberSplit(kind) =
+  if goodCol(em.col):
+    em.altSplitPos[kind] = em.content.len
+
+proc softLinebreak(em: var Emitter, lit: string) =
+  # XXX Use an algorithm that is outlined here:
+  # https://llvm.org/devmtg/2013-04/jasper-slides.pdf
+  # +2 because we blindly assume a comma or ' &' might follow
+  if not em.inquote and em.col+lit.len+2 >= MaxLineLen:
+    if em.lastTok in splitters:
+      wr("\L")
+      em.col = 0
+      for i in 1..em.indentLevel+2: wr(" ")
+    else:
+      # search backwards for a good split position:
+      for a in em.altSplitPos:
+        if a > em.fixedUntil:
+          let ws = "\L" & repeat(' ',em.indentLevel+2)
+          em.col = em.content.len - a
+          em.content.insert(ws, a)
+          break
+
+proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
+
+  template endsInWhite(em): bool =
+    em.content.len > 0 and em.content[em.content.high] in {' ', '\L'}
+  template endsInAlpha(em): bool =
+    em.content.len > 0 and em.content[em.content.high] in SymChars+{'_'}
+
+  proc emitComment(em: var Emitter; tok: TToken) =
+    let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    if not endsInWhite(em):
+      wr(" ")
+      if em.lineSpan == 0 and max(em.col, LineCommentColumn) + lit.len <= MaxLineLen:
+        for i in 1 .. LineCommentColumn - em.col: wr(" ")
+    wr lit
+
+  var preventComment = case tok.tokType
+                       of tokKeywordLow..tokKeywordHigh:
+                          if endsInAlpha(em): wr(" ")
+                          wr(TokTypeToStr[tok.tokType])
+
+                          case tok.tokType
+                          of tkAnd: rememberSplit(splitAnd)
+                          of tkOr: rememberSplit(splitOr)
+                          of tkIn: rememberSplit(splitIn)
+                          else: 90
+                       else:
+                         "case returns value"
+
+
+  if tok.tokType == tkComment and tok.line == em.lastLineNumber and tok.indent >= 0:
+    # we have an inline comment so handle it before the indentation token:
+    emitComment(em, tok)
+    preventComment = true
+    em.fixedUntil = em.content.high
+
+  elif tok.indent >= 0:
+        em.indentLevel = tok.indent
+        # remove trailing whitespace:
+        while em.content.len > 0 and em.content[em.content.high] == ' ':
+          setLen(em.content, em.content.len-1)
+        wr("\L")
+        for i in 2..tok.line - em.lastLineNumber: wr("\L")
+        em.col = 0
+        for i in 1..tok.indent:
+          wr(" ")
+        em.fixedUntil = em.content.high
+
+  case tok.tokType
+  of tokKeywordLow..tokKeywordHigh:
+    if endsInAlpha(em): wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+
+    case tok.tokType
+    of tkAnd: rememberSplit(splitAnd)
+    of tkOr: rememberSplit(splitOr)
+    of tkIn: rememberSplit(splitIn)
+    else: discard
+
+  of tkColon:
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+  of tkSemicolon,
+     tkComma:
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+    rememberSplit(splitComma)
+  of tkParLe, tkParRi, tkBracketLe,
+     tkBracketRi, tkCurlyLe, tkCurlyRi,
+     tkBracketDotLe, tkBracketDotRi,
+     tkCurlyDotLe, tkCurlyDotRi,
+     tkParDotLe, tkParDotRi,
+     tkColonColon, tkDot, tkBracketLeColon:
+    wr(TokTypeToStr[tok.tokType])
+    if tok.tokType in splitters:
+      rememberSplit(splitParLe)
+  of tkEquals:
+    if not em.endsInWhite: wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+  of tkOpr, tkDotDot:
+    if not em.endsInWhite: wr(" ")
+    wr(tok.ident.s)
+    template isUnary(tok): bool =
+      tok.strongSpaceB == 0 and tok.strongSpaceA > 0
+
+    if not isUnary(tok) or em.lastTok in {tkOpr, tkDotDot}:
+      wr(" ")
+      rememberSplit(splitBinary)
+  of tkAccent:
+    wr(TokTypeToStr[tok.tokType])
+    em.inquote = not em.inquote
+  of tkComment:
+    if not preventComment:
+      emitComment(em, tok)
+  of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
+    let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
+    softLinebreak(em, lit)
+    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wr(" ")
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    wr lit
+  of tkEof: discard
+  else:
+    let lit = if tok.ident != nil: tok.ident.s else: tok.literal
+    softLinebreak(em, lit)
+    if endsInAlpha(em): wr(" ")
+    wr lit
+
+  em.lastTok = tok.tokType
+  em.lastLineNumber = tok.line + em.lineSpan
+  em.lineSpan = 0
+
+proc starWasExportMarker*(em: var Emitter) =
+  if em.content.endsWith(" * "):
+    setLen(em.content, em.content.len-3)
+    em.content.add("*")
+    dec em.col, 2
+
+type
+  Thing = ref object
+    grade: string
+    # this name is great
+    name: string
+
+proc f() =
+  var c: char
+  var str: string
+  if c == '\\':
+    # escape char
+    str &= c

nimpretty/tests/expected/exhaustive.nim

@@ -0,0 +1,325 @@
+discard """
+  outputsub: '''ObjectAssignmentError'''
+  exitcode: "1"
+"""
+
+import verylongnamehere, verylongnamehere,
+  verylongnamehereverylongnamehereverylong, namehere, verylongnamehere
+
+proc `[]=`() = discard "index setter"
+proc `putter=`() = discard cast[pointer](cast[int](buffer) + size)
+
+(not false)
+
+let expr = if true: "true" else: "false"
+
+var body = newNimNode(nnkIfExpr).add(
+  newNimNode(nnkElifBranch).add(
+    infix(newDotExpr(ident("a"), ident("kind")), "==", newDotExpr(ident("b"),
+        ident("kind"))),
+    condition
+  ),
+  newNimNode(nnkElse).add(newStmtList(newNimNode(nnkReturnStmt).add(ident(
+      "false"))))
+)
+
+# comment
+
+var x = 1
+
+type
+  GeneralTokenizer* = object of RootObj ## comment here
+    kind*: TokenClass         ## and here
+    start*, length*: int      ## you know how it goes...
+    buf: cstring
+    pos: int                  # other comment here.
+    state: TokenClass
+
+var x*: string
+var y: seq[string] #[ yay inline comments. So nice I have to care bout these. ]#
+
+echo "#", x, "##", y, "#" & "string" & $test
+
+echo (tup, here)
+echo(argA, argB)
+
+import macros
+
+## A documentation comment here.
+## That spans multiple lines.
+## And is not to be touched.
+
+const numbers = [4u8, 5'u16, 89898_00]
+
+macro m(n): untyped =
+  result = foo"string literal"
+
+{.push m.}
+proc p() = echo "p", 1+4 * 5, if true: 5 else: 6
+proc q(param: var ref ptr string) =
+  p()
+  if true:
+    echo a and b or not c and not -d
+{.pop.}
+
+q()
+
+when false:
+  # bug #4766
+  type
+    Plain = ref object
+      discard
+
+    Wrapped[T] = object
+      value: T
+
+  converter toWrapped[T](value: T): Wrapped[T] =
+    Wrapped[T](value: value)
+
+  let result = Plain()
+  discard $result
+
+when false:
+  # bug #3670
+  template someTempl(someConst: bool) =
+    when someConst:
+      var a: int
+    if true:
+      when not someConst:
+        var a: int
+      a = 5
+
+  someTempl(true)
+
+
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2018 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Layouter for nimpretty. Still primitive but useful.
+
+import idents, lexer, lineinfos, llstream, options, msgs, strutils
+from os import changeFileExt
+
+const
+  MaxLineLen = 80
+  LineCommentColumn = 30
+
+type
+  SplitKind = enum
+    splitComma, splitParLe, splitAnd, splitOr, splitIn, splitBinary
+
+  Emitter* = object
+    f: PLLStream
+    config: ConfigRef
+    fid: FileIndex
+    lastTok: TTokType
+    inquote {.pragmaHereWrongCurlyEnd.}: bool
+    col, lastLineNumber, lineSpan, indentLevel: int
+    content: string
+    fixedUntil: int           # marks where we must not go in the content
+    altSplitPos: array[SplitKind, int] # alternative split positions
+
+proc openEmitter*[T, S](em: var Emitter; config: ConfigRef;
+    fileIdx: FileIndex) {.pragmaHereWrongCurlyEnd.} =
+  let outfile = changeFileExt(config.toFullPath(fileIdx), ".pretty.nim")
+  em.f = llStreamOpen(outfile, fmWrite)
+  em.config = config
+  em.fid = fileIdx
+  em.lastTok = tkInvalid
+  em.inquote = false
+  em.col = 0
+  em.content = newStringOfCap(16_000)
+  if em.f == nil:
+    rawMessage(config, errGenerated, "cannot open file: " & outfile)
+
+proc closeEmitter*(em: var Emitter) {.inline.} =
+  em.f.llStreamWrite em.content
+  llStreamClose(em.f)
+
+proc countNewlines(s: string): int =
+  result = 0
+  for i in 0..<s.len:
+    if s[i+1] == '\L': inc result
+
+proc calcCol(em: var Emitter; s: string) =
+  var i = s.len-1
+  em.col = 0
+  while i >= 0 and s[i] != '\L':
+    dec i
+    inc em.col
+
+template wr(x) =
+  em.content.add x
+  inc em.col, x.len
+
+template goodCol(col): bool = col in 40..MaxLineLen
+
+const splitters = {tkComma, tkSemicolon, tkParLe, tkParDotLe,
+                   tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
+                   tkCurlyLe}
+
+template rememberSplit(kind) =
+  if goodCol(em.col):
+    em.altSplitPos[kind] = em.content.len
+
+proc softLinebreak(em: var Emitter; lit: string) =
+  # XXX Use an algorithm that is outlined here:
+  # https://llvm.org/devmtg/2013-04/jasper-slides.pdf
+  # +2 because we blindly assume a comma or ' &' might follow
+  if not em.inquote and em.col+lit.len+2 >= MaxLineLen:
+    if em.lastTok in splitters:
+      wr("\L")
+      em.col = 0
+      for i in 1..em.indentLevel+2: wr(" ")
+    else:
+      # search backwards for a good split position:
+      for a in em.altSplitPos:
+        if a > em.fixedUntil:
+          let ws = "\L" & repeat(' ', em.indentLevel+2)
+          em.col = em.content.len - a
+          em.content.insert(ws, a)
+          break
+
+proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
+
+  template endsInWhite(em): bool =
+    em.content.len > 0 and em.content[em.content.high] in {' ', '\L'}
+  template endsInAlpha(em): bool =
+    em.content.len > 0 and em.content[em.content.high] in SymChars+{'_'}
+
+  proc emitComment(em: var Emitter; tok: TToken) =
+    let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA,
+        tok.commentOffsetB)
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    if not endsInWhite(em):
+      wr(" ")
+      if em.lineSpan == 0 and max(em.col,
+          LineCommentColumn) + lit.len <= MaxLineLen:
+        for i in 1 .. LineCommentColumn - em.col: wr(" ")
+    wr lit
+
+  var preventComment = case tok.tokType
+    of tokKeywordLow..tokKeywordHigh:
+      if endsInAlpha(em): wr(" ")
+      wr(TokTypeToStr[tok.tokType])
+
+      case tok.tokType
+      of tkAnd: rememberSplit(splitAnd)
+      of tkOr: rememberSplit(splitOr)
+      of tkIn: rememberSplit(splitIn)
+      else: 90
+    else:
+      "case returns value"
+
+
+  if tok.tokType == tkComment and tok.line == em.lastLineNumber and
+      tok.indent >= 0:
+    # we have an inline comment so handle it before the indentation token:
+    emitComment(em, tok)
+    preventComment = true
+    em.fixedUntil = em.content.high
+
+  elif tok.indent >= 0:
+    em.indentLevel = tok.indent
+    # remove trailing whitespace:
+    while em.content.len > 0 and em.content[em.content.high] == ' ':
+      setLen(em.content, em.content.len-1)
+    wr("\L")
+    for i in 2..tok.line - em.lastLineNumber: wr("\L")
+    em.col = 0
+    for i in 1..tok.indent:
+      wr(" ")
+    em.fixedUntil = em.content.high
+
+  case tok.tokType
+  of tokKeywordLow..tokKeywordHigh:
+    if endsInAlpha(em): wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+
+    case tok.tokType
+    of tkAnd: rememberSplit(splitAnd)
+    of tkOr: rememberSplit(splitOr)
+    of tkIn: rememberSplit(splitIn)
+    else: discard
+
+  of tkColon:
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+  of tkSemicolon,
+     tkComma:
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+    rememberSplit(splitComma)
+  of tkParLe, tkParRi, tkBracketLe,
+     tkBracketRi, tkCurlyLe, tkCurlyRi,
+     tkBracketDotLe, tkBracketDotRi,
+     tkCurlyDotLe, tkCurlyDotRi,
+     tkParDotLe, tkParDotRi,
+     tkColonColon, tkDot, tkBracketLeColon:
+    wr(TokTypeToStr[tok.tokType])
+    if tok.tokType in splitters:
+      rememberSplit(splitParLe)
+  of tkEquals:
+    if not em.endsInWhite: wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+  of tkOpr, tkDotDot:
+    if not em.endsInWhite: wr(" ")
+    wr(tok.ident.s)
+    template isUnary(tok): bool =
+      tok.strongSpaceB == 0 and tok.strongSpaceA > 0
+
+    if not isUnary(tok) or em.lastTok in {tkOpr, tkDotDot}:
+      wr(" ")
+      rememberSplit(splitBinary)
+  of tkAccent:
+    wr(TokTypeToStr[tok.tokType])
+    em.inquote = not em.inquote
+  of tkComment:
+    if not preventComment:
+      emitComment(em, tok)
+  of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit,
+      tkGTripleStrLit, tkCharLit:
+    let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
+    softLinebreak(em, lit)
+    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wr(
+        " ")
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    wr lit
+  of tkEof: discard
+  else:
+    let lit = if tok.ident != nil: tok.ident.s else: tok.literal
+    softLinebreak(em, lit)
+    if endsInAlpha(em): wr(" ")
+    wr lit
+
+  em.lastTok = tok.tokType
+  em.lastLineNumber = tok.line + em.lineSpan
+  em.lineSpan = 0
+
+proc starWasExportMarker*(em: var Emitter) =
+  if em.content.endsWith(" * "):
+    setLen(em.content, em.content.len-3)
+    em.content.add("*")
+    dec em.col, 2
+
+type
+  Thing = ref object
+    grade: string
+    # this name is great
+    name: string
+
+proc f() =
+  var c: char
+  var str: string
+  if c == '\\':
+    # escape char
+    str &= c

tests/arithm/tnot.nim

@@ -0,0 +1,58 @@
+discard """
+  output: '''
+-5
+-5
+-5
+-5
+4
+4
+4
+4
+251
+65531
+4294967291
+18446744073709551611
+4
+4
+4
+4
+'''
+"""
+
+# Signed types
+block:
+  const t0: int8  = not 4
+  const t1: int16 = not 4
+  const t2: int32 = not 4
+  const t3: int64 = not 4
+  const t4: int8  = not -5
+  const t5: int16 = not -5
+  const t6: int32 = not -5
+  const t7: int64 = not -5
+  echo t0
+  echo t1
+  echo t2
+  echo t3
+  echo t4
+  echo t5
+  echo t6
+  echo t7
+
+# Unsigned types
+block:
+  const t0: uint8  = not 4'u8
+  const t1: uint16 = not 4'u16
+  const t2: uint32 = not 4'u32
+  const t3: uint64 = not 4'u64
+  const t4: uint8  = not 251'u8
+  const t5: uint16 = not 65531'u16
+  const t6: uint32 = not 4294967291'u32
+  const t7: uint64 = not 18446744073709551611'u64
+  echo t0
+  echo t1
+  echo t2
+  echo t3
+  echo t4
+  echo t5
+  echo t6
+  echo t7

tests/arithm/tshl.nim

@@ -0,0 +1,34 @@
+discard """
+  output: '''
+0
+0
+1
+1
+0
+0
+0
+1
+'''
+"""
+
+# Signed types
+block:
+  const t0: int8  = 1'i8 shl 8
+  const t1: int16 = 1'i16 shl 16
+  const t2: int32 = 1'i32 shl 32
+  const t3: int64 = 1'i64 shl 64
+  echo t0
+  echo t1
+  echo t2
+  echo t3
+
+# Unsigned types
+block:
+  const t0: uint8  = 1'u8 shl 8
+  const t1: uint16 = 1'u16 shl 16
+  const t2: uint32 = 1'u32 shl 32
+  const t3: uint64 = 1'u64 shl 64
+  echo t0
+  echo t1
+  echo t2
+  echo t3

tests/concepts/treversable.nim

@@ -0,0 +1,31 @@
+# issue 7705, 7703, 7702
+discard """
+  output: '''
+z
+e
+  '''
+"""
+
+type
+  Reversable*[T] = concept a
+    a[int] is T
+    a.high is int
+    a.len is int
+    a.low is int
+
+proc get[T](s: Reversable[T], n: int): T =
+  s[n]
+
+proc hi[T](s: Reversable[T]): int =
+  s.high
+
+proc lo[T](s: Reversable[T]): int =
+  s.low
+
+iterator reverse*[T](s: Reversable[T]): T =
+  assert hi(s) - lo(s) == len(s) - 1
+  for z in hi(s).countdown(lo(s)):
+    yield s.get(z)
+
+for s in @["e", "z"].reverse:
+  echo s

tests/niminaction/Chapter1/various1.nim

@@ -0,0 +1,45 @@
+discard """
+  exitCode: 0
+  outputsub: "Woof!"
+"""
+
+import strutils
+echo("hello".to_upper())
+echo("world".toUpper())
+
+type
+  Dog = object #<1>
+    age: int #<2>
+
+let dog = Dog(age: 3) #<3>
+
+proc showNumber(num: int | float) =
+  echo(num)
+
+showNumber(3.14)
+showNumber(42)
+
+for i in 0 .. <10:
+  echo(i)
+
+block: # Block added due to clash.
+  type
+    Dog = object
+
+  proc bark(self: Dog) = #<1>
+    echo("Woof!")
+
+  let dog = Dog()
+  dog.bark() #<2>
+
+import sequtils, future, strutils
+let list = @["Dominik Picheta", "Andreas Rumpf", "Desmond Hume"]
+list.map(
+  (x: string) -> (string, string) => (x.split[0], x.split[1])
+).echo
+
+import strutils
+let list1 = @["Dominik Picheta", "Andreas Rumpf", "Desmond Hume"]
+for name in list1:
+  echo((name.split[0], name.split[1]))
+

tests/niminaction/Chapter2/explicit_discard.nim

@@ -0,0 +1,7 @@
+discard """
+  line: 7
+  errormsg: "has to be discarded"
+"""
+
+proc myProc(name: string): string = "Hello " & name
+myProc("Dominik")

tests/niminaction/Chapter2/no_def_eq.nim

@@ -0,0 +1,16 @@
+discard """
+  line: 16
+  errormsg: "type mismatch"
+"""
+
+type
+    Dog = object
+      name: string
+
+    Cat = object
+      name: string
+
+let dog: Dog = Dog(name: "Fluffy")
+let cat: Cat = Cat(name: "Fluffy")
+
+echo(dog == cat)

tests/niminaction/Chapter2/no_iterator.nim

@@ -0,0 +1,7 @@
+discard """
+  line: 6
+  errormsg: "type mismatch"
+"""
+
+for i in 5:
+  echo i

tests/niminaction/Chapter2/no_seq_type.nim

@@ -0,0 +1,6 @@
+discard """
+  line: 6
+  errormsg: "cannot infer the type of the sequence"
+"""
+
+var list = @[]

tests/niminaction/Chapter2/resultaccept.nim

@@ -0,0 +1,28 @@
+discard """
+  output: ""
+"""
+
+# Page 35.
+
+proc implicit: string =
+  "I will be returned"
+  
+proc discarded: string =
+  discard "I will not be returned"
+  
+proc explicit: string =
+  return "I will be returned"
+
+proc resultVar: string =
+  result = "I will be returned"
+  
+proc resultVar2: string =
+  result = ""
+  result.add("I will be ")
+  result.add("returned")
+
+doAssert implicit() == "I will be returned"
+doAssert discarded() == nil
+doAssert explicit() == "I will be returned"
+doAssert resultVar() == "I will be returned"
+doAssert resultVar2() == "I will be returned"

tests/niminaction/Chapter2/resultreject.nim

@@ -0,0 +1,33 @@
+discard """
+  line: 27
+  errormsg: "has to be discarded"
+"""
+
+# Page 35.
+
+proc implicit: string =
+  "I will be returned"
+  
+proc discarded: string =
+  discard "I will not be returned"
+
+proc explicit: string =
+  return "I will be returned"
+
+proc resultVar: string =
+  result = "I will be returned"
+  
+proc resultVar2: string =
+  result = ""
+  result.add("I will be ")
+  result.add("returned")
+
+proc resultVar3: string =
+  result = "I am the result"
+  "I will cause an error"
+
+doAssert implicit() == "I will be returned"
+doAssert discarded() == nil
+doAssert explicit() == "I will be returned"
+doAssert resultVar() == "I will be returned"
+doAssert resultVar2() == "I will be returned"

tests/niminaction/Chapter2/various2.nim

@@ -0,0 +1,369 @@
+discard """
+  exitCode: 0
+  outputsub: '''42 is greater than 0'''
+"""
+
+if 42 >= 0:
+  echo "42 is greater than 0"
+
+
+echo("Output: ",
+  5)
+echo(5 +
+  5)
+# --- Removed code that is supposed to fail here. Not going to test those. ---
+
+# Single-line comment
+#[
+Multiline comment
+]#
+when false:
+  echo("Commented-out code")
+
+let decimal = 42
+let hex = 0x42
+let octal = 0o42
+let binary = 0b101010
+
+let a: int16 = 42
+let b = 42'i8
+
+let c = 1'f32 # --- Changed names here to avoid clashes ---
+let d = 1.0e19
+
+let e = false
+let f = true
+
+let g = 'A'
+let h = '\109'
+let i = '\x79'
+
+let text = "The book title is \"Nim in Action\""
+
+let filepath = "C:\\Program Files\\Nim"
+
+# --- Changed name here to avoid clashes ---
+let filepath1 = r"C:\Program Files\Nim"
+
+let multiLine = """foo
+  bar
+  baz
+"""
+echo multiLine
+
+import strutils
+# --- Changed name here to avoid clashes ---
+let multiLine1 = """foo
+  bar
+  baz
+"""
+echo multiLine1.unindent
+doAssert multiLine1.unindent == "foo\nbar\nbaz\n"
+
+proc fillString(): string =
+  result = ""
+  echo("Generating string")
+  for i in 0 .. 4:
+    result.add($i) #<1>
+
+const count = fillString()
+
+var
+  text1 = "hello"
+  number: int = 10
+  isTrue = false
+
+var 火 = "Fire"
+let ogień = true
+
+var `var` = "Hello"
+echo(`var`)
+
+proc myProc(name: string): string = "Hello " & name
+discard myProc("Dominik")
+
+proc bar(): int #<1>
+
+proc foo(): float = bar().float
+proc bar(): int = foo().int
+
+proc noReturn() = echo("Hello")
+proc noReturn2(): void = echo("Hello")
+
+proc noReturn3 = echo("Hello")
+
+proc message(recipient: string): auto =
+  "Hello " & recipient
+
+doAssert message("Dom") == "Hello Dom"
+
+proc max(a: int, b: int): int =
+  if a > b: a else: b
+
+doAssert max(5, 10) == 10
+
+proc max2(a, b: int): int =
+  if a > b: a else: b
+
+proc genHello(name: string, surname = "Doe"): string =
+  "Hello " & name & " " & surname
+
+# -- Leaving these as asserts as that is in the original code, just in case
+# -- somehow in the future `assert` is removed :)
+assert genHello("Peter") == "Hello Peter Doe"
+assert genHello("Peter", "Smith") == "Hello Peter Smith"
+
+proc genHello2(names: varargs[string]): string =
+  result = ""
+  for name in names:
+    result.add("Hello " & name & "\n")
+
+doAssert genHello2("John", "Bob") == "Hello John\nHello Bob\n"
+
+proc getUserCity(firstName, lastName: string): string =
+  case firstName
+  of "Damien": return "Tokyo"
+  of "Alex": return "New York"
+  else: return "Unknown"
+
+proc getUserCity(userID: int): string =
+  case userID
+  of 1: return "Tokyo"
+  of 2: return "New York"
+  else: return "Unknown"
+
+doAssert getUserCity("Damien", "Lundi") == "Tokyo"
+doAssert getUserCity(2) == "New York" # -- Errata here: missing closing "
+
+import sequtils
+let numbers = @[1, 2, 3, 4, 5, 6]
+let odd = filter(numbers, proc (x: int): bool = x mod 2 != 0)
+doAssert odd == @[1, 3, 5]
+
+import sequtils, future
+let numbers1 = @[1, 2, 3, 4, 5, 6]
+let odd1 = filter(numbers1, (x: int) -> bool => x mod 2 != 0)
+assert odd1 == @[1, 3, 5]
+
+proc isValid(x: int, validator: proc (x: int): bool) =
+  if validator(x): echo(x, " is valid")
+  else: echo(x, " is NOT valid")
+
+import future
+proc isValid2(x: int, validator: (x: int) -> bool) =
+  if validator(x): echo(x, " is valid")
+  else: echo(x, " is NOT valid")
+
+var list: array[3, int]
+list[0] = 1
+list[1] = 42
+assert list[0] == 1
+assert list[1] == 42
+assert list[2] == 0 #<1>
+
+echo list.repr #<2>
+
+# echo list[500]
+
+var list2: array[-10 .. -9, int]
+list2[-10] = 1
+list2[-9] = 2
+
+var list3 = ["Hi", "There"]
+
+var list4 = ["My", "name", "is", "Dominik"]
+for item in list4:
+  echo(item)
+
+for i in list4.low .. list4.high:
+  echo(list4[i])
+
+var list5: seq[int] = @[]
+doAssertRaises(IndexError):
+  list5[0] = 1
+
+list5.add(1)
+
+assert list5[0] == 1
+doAssertRaises(IndexError):
+  echo list5[42]
+
+# -- Errata: var list: seq[int]; echo(list[0]). This now creates an exception,
+# --         not a SIGSEGV.
+
+block:
+  var list = newSeq[string](3)
+  assert list[0] == nil
+  list[0] = "Foo"
+  list[1] = "Bar"
+  list[2] = "Baz"
+
+  list.add("Lorem")
+
+block:
+  let list = @[4, 8, 15, 16, 23, 42]
+  for i in 0 .. <list.len:
+    stdout.write($list[i] & " ")
+
+var collection: set[int16]
+doAssert collection == {}
+
+block:
+  let collection = {'a', 'x', 'r'}
+  doAssert 'a' in collection
+
+block:
+  let collection = {'a', 'T', 'z'}
+  let isAllLowerCase = {'A' .. 'Z'} * collection == {}
+  doAssert(not isAllLowerCase)
+
+let age = 10
+if age > 0 and age <= 10:
+  echo("You're still a child")
+elif age > 10 and age < 18:
+  echo("You're a teenager")
+else:
+  echo("You're an adult")
+
+let variable = "Arthur"
+case variable
+of "Arthur", "Zaphod", "Ford":
+  echo("Male")
+of "Marvin":
+  echo("Robot")
+of "Trillian":
+  echo("Female")
+else:
+  echo("Unknown")
+
+let ageDesc = if age < 18: "Non-Adult" else: "Adult"
+
+block:
+  var i = 0
+  while i < 3:
+    echo(i)
+    i.inc
+
+block label:
+  var i = 0
+  while true:
+    while i < 5:
+      if i > 3: break label
+      i.inc
+
+iterator values(): int =
+  var i = 0
+  while i < 5:
+    yield i
+    i.inc
+
+for value in values():
+  echo(value)
+
+import os
+for filename in walkFiles("*.nim"):
+  echo(filename)
+
+for item in @[1, 2, 3]:
+  echo(item)
+
+for i, value in @[1, 2, 3]: echo("Value at ", i, ": ", value)
+
+doAssertRaises(IOError):
+  proc second() =
+    raise newException(IOError, "Somebody set us up the bomb")
+
+  proc first() =
+    second()
+
+  first()
+
+block:
+  proc second() =
+    raise newException(IOError, "Somebody set us up the bomb")
+
+  proc first() =
+    try:
+      second()
+    except:
+      echo("Cannot perform second action because: " &
+        getCurrentExceptionMsg())
+
+  first()
+
+block:
+  type
+    Person = object
+      name: string
+      age: int
+
+  var person: Person
+  var person1 = Person(name: "Neo", age: 28)
+
+block:
+  type
+    PersonObj = object
+      name: string
+      age: int
+    PersonRef = ref PersonObj
+
+  # proc setName(person: PersonObj) =
+  #   person.name = "George"
+
+  proc setName(person: PersonRef) =
+    person.name = "George"
+
+block:
+  type
+    Dog = object
+      name: string
+
+    Cat = object
+      name: string
+
+  let dog: Dog = Dog(name: "Fluffy")
+  let cat: Cat = Cat(name: "Fluffy")
+
+block:
+  type
+    Dog = tuple
+      name: string
+
+    Cat = tuple
+      name: string
+
+  let dog: Dog = (name: "Fluffy")
+  let cat: Cat = (name: "Fluffy")
+
+  echo(dog == cat)
+
+block:
+  type
+    Point = tuple[x, y: int]
+    Point2 = (int, int)
+
+  let pos: Point = (x: 100, y: 50)
+  doAssert pos == (100, 50)
+
+  let pos1: Point = (x: 100, y: 50)
+  let (x, y) = pos1 #<1>
+  let (left, _) = pos1
+  doAssert x == pos1[0]
+  doAssert y == pos1[1]
+  doAssert left == x
+
+block:
+  type
+    Color = enum
+      colRed,
+      colGreen,
+      colBlue
+
+  let color: Color = colRed
+
+block:
+  type
+    Color {.pure.} = enum
+      red, green, blue
+
+  let color = Color.red

tests/niminaction/Chapter3/various3.nim

@@ -0,0 +1,93 @@
+import threadpool
+proc foo: string = "Dog"
+var x: FlowVar[string] = spawn foo()
+assert(^x == "Dog")
+
+block:
+  type
+    Box = object
+      case empty: bool
+      of false:
+        contents: string
+      else:
+        discard
+
+  var obj = Box(empty: false, contents: "Hello")
+  assert obj.contents == "Hello"
+
+  var obj2 = Box(empty: true)
+  doAssertRaises(FieldError):
+    echo(obj2.contents)
+
+import json
+assert parseJson("null").kind == JNull
+assert parseJson("true").kind == JBool
+assert parseJson("42").kind == JInt
+assert parseJson("3.14").kind == JFloat
+assert parseJson("\"Hi\"").kind == JString
+assert parseJson("""{ "key": "value" }""").kind == JObject
+assert parseJson("[1, 2, 3, 4]").kind == JArray
+
+import json
+let data = """
+  {"username": "Dominik"}
+"""
+
+let obj = parseJson(data) 
+assert obj.kind == JObject 
+assert obj["username"].kind == JString 
+assert obj["username"].str == "Dominik"
+
+block:
+  proc count10(): int =
+    for i in 0 .. <10:
+      result.inc
+  assert count10() == 10
+
+type
+  Point = tuple[x, y: int]
+
+var point = (5, 10)
+var point2 = (x: 5, y: 10)
+
+type
+  Human = object
+    name: string
+    age: int
+
+var jeff = Human(name: "Jeff", age: 23)
+var amy = Human(name: "Amy", age: 20)
+
+import asyncdispatch
+
+var future = newFuture[int]() 
+doAssert(not future.finished) 
+
+future.callback = 
+  proc (future: Future[int]) = 
+    echo("Future is no longer empty, ", future.read) 
+
+future.complete(42)
+
+import asyncdispatch, asyncfile
+
+when false:
+  var file = openAsync("")
+  let dataFut = file.readAll()
+  dataFut.callback =
+    proc (future: Future[string]) =
+      echo(future.read())
+
+  asyncdispatch.runForever()
+
+import asyncdispatch, asyncfile, os
+
+proc readFiles() {.async.} =
+  # --- Changed to getTempDir here.
+  var file = openAsync(getTempDir() / "test.txt", fmReadWrite)
+  let data = await file.readAll() 
+  echo(data) 
+  await file.write("Hello!\n") 
+
+waitFor readFiles()
+

tests/niminaction/Chapter3/various3.nim.cfg

@@ -0,0 +1 @@
+threads:on

tests/stdlib/tpegs.nim

@@ -0,0 +1,78 @@
+discard """
+  output: '''
+pkNonTerminal: Sum @(2, 3)
+  pkSequence: (Product (('+' / '-') Product)*)
+    pkNonTerminal: Product @(3, 7)
+      pkSequence: (Value (('*' / '/') Value)*)
+        pkNonTerminal: Value @(4, 5)
+          pkOrderedChoice: (([0-9] [0-9]*) / ('(' Expr ')'))
+            pkSequence: ([0-9] [0-9]*)
+              pkCharChoice: [0-9]
+              pkGreedyRepSet: [0-9]*
+            pkSequence: ('(' Expr ')')
+              pkChar: '('
+              pkNonTerminal: Expr @(1, 4)
+                pkNonTerminal: Sum @(2, 3)
+              pkChar: ')'
+        pkGreedyRep: (('*' / '/') Value)*
+          pkSequence: (('*' / '/') Value)
+            pkOrderedChoice: ('*' / '/')
+              pkChar: '*'
+              pkChar: '/'
+            pkNonTerminal: Value @(4, 5)
+    pkGreedyRep: (('+' / '-') Product)*
+      pkSequence: (('+' / '-') Product)
+        pkOrderedChoice: ('+' / '-')
+          pkChar: '+'
+          pkChar: '-'
+        pkNonTerminal: Product @(3, 7)
+'''
+"""
+
+import strutils, streams
+import pegs
+
+const
+  indent = "  "
+
+let
+  pegSrc = """
+Expr <- Sum
+Sum <- Product (('+' / '-') Product)*
+Product <- Value (('*' / '/') Value)*
+Value <- [0-9]+ / '(' Expr ')'
+  """
+  pegAst: Peg = pegSrc.peg
+
+var
+  outp = newStringStream()
+  processed: seq[string] = @[]
+
+proc prt(outp: Stream, kind: PegKind, s: string; level: int = 0) =
+  outp.writeLine indent.repeat(level) & "$1: $2" % [$kind, s]
+
+proc recLoop(p: Peg, level: int = 0) =
+  case p.kind
+  of pkEmpty..pkWhitespace:
+    discard
+  of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle:
+    outp.prt(p.kind, $p, level)
+  of pkChar, pkGreedyRepChar:
+    outp.prt(p.kind, $p, level)
+  of pkCharChoice, pkGreedyRepSet:
+    outp.prt(p.kind, $p, level)
+  of pkNonTerminal:
+    outp.prt(p.kind,
+      "$1 @($3, $4)" % [p.nt.name, $p.nt.rule.kind, $p.nt.line, $p.nt.col], level)
+    if not(p.nt.name in processed):
+      processed.add p.nt.name
+      p.nt.rule.recLoop level+1
+  of pkBackRef..pkBackRefIgnoreStyle:
+    outp.prt(p.kind, $p, level)
+  else:
+    outp.prt(p.kind, $p, level)
+    for s in items(p):
+      s.recLoop level+1
+
+pegAst.recLoop
+echo outp.data

tests/testament/categories.nim

@@ -266,8 +266,17 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
     testSpec r, makeTest(filename, options, cat, actionCompile), targetCPP
 
   let tests = [
+    "niminaction/Chapter1/various1",
+    "niminaction/Chapter2/various2",
+    "niminaction/Chapter2/resultaccept",
+    "niminaction/Chapter2/resultreject",
+    "niminaction/Chapter2/explicit_discard",
+    "niminaction/Chapter2/no_def_eq",
+    "niminaction/Chapter2/no_iterator",
+    "niminaction/Chapter2/no_seq_type",
     "niminaction/Chapter3/ChatApp/src/server",
     "niminaction/Chapter3/ChatApp/src/client",
+    "niminaction/Chapter3/various3",
     "niminaction/Chapter6/WikipediaStats/concurrency_regex",
     "niminaction/Chapter6/WikipediaStats/concurrency",
     "niminaction/Chapter6/WikipediaStats/naive",
@@ -278,8 +287,34 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
     "niminaction/Chapter7/Tweeter/src/tweeter",
     "niminaction/Chapter7/Tweeter/src/createDatabase",
     "niminaction/Chapter7/Tweeter/tests/database_test",
-    "niminaction/Chapter8/sdl/sdl_test",
+    "niminaction/Chapter8/sdl/sdl_test"
     ]
+
+  # Verify that the files have not been modified. Death shall fall upon
+  # whoever edits these hashes without dom96's permission, j/k. But please only
+  # edit when making a conscious breaking change, also please try to make your
+  # commit message clear and notify me so I can easily compile an errata later.
+  var testHashes: seq[string] = @[]
+
+  for test in tests:
+    testHashes.add(getMD5(readFile("tests" / test.addFileExt("nim")).string))
+
+  const refHashes = @[
+    "51afdfa84b3ca3d810809d6c4e5037ba", "30f07e4cd5eaec981f67868d4e91cfcf",
+    "d14e7c032de36d219c9548066a97e846", "2e40bfd5daadb268268727da91bb4e81",
+    "c5d3853ed0aba04bf6d35ba28a98dca0", "058603145ff92d46c009006b06e5b228",
+    "7b94a029b94ddb7efafddd546c965ff6", "586d74514394e49f2370dfc01dd9e830",
+    "e1901837b757c9357dc8d259fd0ef0f6", "097670c7ae12e825debaf8ec3995227b",
+    "a8cb7b78cc78d28535ab467361db5d6e", "bfaec2816a1848991b530c1ad17a0184",
+    "47cb71bb4c1198d6d29cdbee05aa10b9", "87e4436809f9d73324cfc4f57f116770",
+    "7b7db5cddc8cf8fa9b6776eef1d0a31d", "e6e40219f0f2b877869b738737b7685e",
+    "6532ee87d819f2605a443d5e94f9422a", "9a8fe78c588d08018843b64b57409a02",
+    "03a801275b8b76b4170c870cd0da079d", "20bb7d3e2d38d43b0cb5fcff4909a4a8",
+    "af6844598f534fab6942abfa4dfe9ab2", "2a7a17f84f6503d9bc89a5ab8feea127"
+  ]
+  doAssert testHashes == refHashes, "Nim in Action tests were changed."
+
+  # Run the tests.
   for testfile in tests:
     test "tests/" & testfile & ".nim", actionCompile
 
@@ -291,6 +326,7 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
 
 
 
+
 # ------------------------- manyloc -------------------------------------------
 #proc runSpecialTests(r: var TResults, options: string) =
 #  for t in ["lib/packages/docutils/highlite"]:

tests/testament/tester.nim

@@ -129,7 +129,7 @@ proc callCCompiler(cmdTemplate, filename, options: string,
   let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
                        "options", options, "file", filename.quoteShell,
                        "filedir", filename.getFileDir()])
-  var p = startProcess(command="gcc", args=c[5.. ^1],
+  var p = startProcess(command="gcc", args=c[5 .. ^1],
                        options={poStdErrToStdOut, poUsePath})
   let outp = p.outputStream
   var x = newStringOfCap(120)

tests/types/t7905.nim

@@ -0,0 +1,33 @@
+discard """
+  output: '''
+(member: "hello world")
+(member: 123.456)
+(member: "hello world", x: ...)
+(member: 123.456, x: ...)
+'''
+"""
+
+template foobar(arg: typed): untyped =
+  type
+    MyType = object
+      member: type(arg)
+
+  var myVar: MyType
+  myVar.member = arg
+  echo myVar
+
+foobar("hello world")
+foobar(123.456'f64)
+
+template foobarRec(arg: typed): untyped =
+  type
+    MyType = object
+      member: type(arg)
+      x: ref MyType
+
+  var myVar: MyType
+  myVar.member = arg
+  echo myVar
+
+foobarRec("hello world")
+foobarRec(123.456'f64)