Merge branch 'devel' into sorted_deduplicate

.gitignore

@@ -66,3 +66,6 @@ testresults/
 test.txt
 /test.ini
 
+tweeter.db
+tweeter_test.db
+megatest.nim

.gitlab-ci.yml

@@ -53,9 +53,8 @@ test-windows:
   <<: *win_set_path_def
   script:
     - call ci\deps.bat
-    - nim c --taintMode:on testament\tester
+    - nim c testament\tester
     - testament\tester.exe --pedantic all
   tags:
     - windows
     - fast
-

.travis.yml

@@ -51,7 +51,7 @@ script:
   #- nimble install sdl1
   #- nimble install jester@#head -y
   #- nimble install niminst
-  - nim c --taintMode:on -d:nimCoroutines testament/tester
+  - nim c -d:nimCoroutines testament/tester
   - testament/tester --pedantic all -d:nimCoroutines
   - nim c -o:bin/nimpretty nimpretty/nimpretty.nim
   - nim c -r nimpretty/tester.nim

appveyor.yml

@@ -53,8 +53,8 @@ build_script:
 #  - nimble install opengl -y
 #  - nimble install sdl1 -y
 #  - nimble install jester@#head -y
-  - nim c --taintMode:on -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester
+  - nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester
-  - nim c --taintMode:on -d:nimCoroutines testament/tester
+  - nim c -d:nimCoroutines testament/tester
 
 test_script:
   - testament\tester --pedantic all -d:nimCoroutines

changelog.md

@@ -29,6 +29,27 @@
 
 - `osproc.execProcess` now also takes a `workingDir` parameter.
 
+- `options.UnpackError` is no longer a ref type and inherits from `System.Defect` instead of `System.ValueError`.
+
+- nre's `RegexMatch.{captureBounds,captures}[]`  no longer return `Option` or
+  `nil`/`""`, respectivly. Use the newly added `n in p.captures` method to
+  check if a group is captured, otherwise you'll recieve an exception.
+
+- nre's `RegexMatch.{captureBounds,captures}.toTable` no longer accept a
+  default parameter. Instead uncaptured entries are left empty. Use
+  `Table.getOrDefault()` if you need defaults.
+
+- nre's `RegexMatch.captures.{items,toSeq}` now returns an `Option[string]`
+  instead of a `string`. With the removal of `nil` strings, this is the only
+  way to indicate a missing match. Inside your loops, instead of `capture ==
+  ""` or `capture == nil`, use `capture.isSome` to check if a capture is
+  present, and `capture.get` to get its value.
+
+- nre's `replace()` no longer throws `ValueError` when the replacement string
+  has missing captures. It instead throws `KeyError` for named captures, and
+  `IndexError` for un-named captures. This is consistant with
+  `RegexMatch.{captureBounds,captures}[]`.
+
 #### Breaking changes in the compiler
 
 - The compiler now implements the "generic symbol prepass" for `when` statements
@@ -66,6 +87,8 @@ proc enumToString*(enums: openArray[enum]): string =
   is instantiation of generic proc symbol.
 
 - Added the parameter ``isSorted`` for the ``sequtils.deduplicate`` proc.
+- There is a new stdlib mdoule `std/diff` to compute the famous "diff"
+  of two texts by line.
 
 ### Library changes
 
@@ -93,6 +116,9 @@ proc enumToString*(enums: openArray[enum]): string =
 - There is a new pragma block `noSideEffect` that works like
   the `gcsafe` pragma block.
 - added os.getCurrentProcessId()
+- User defined pragmas are now allowed in the pragma blocks
+- Pragma blocks are now longer eliminated from the typed AST tree to preserve
+  pragmas for further analysis by macros
 
 ### Language changes
 

compiler/ast.nim

@@ -723,10 +723,9 @@ type
       sons*: TNodeSeq
     comment*: string
 
-  TSymSeq* = seq[PSym]
   TStrTable* = object         # a table[PIdent] of PSym
     counter*: int
-    data*: TSymSeq
+    data*: seq[PSym]
 
   # -------------- backend information -------------------------------
   TLocKind* = enum
@@ -1087,9 +1086,6 @@ proc newSym*(symKind: TSymKind, name: PIdent, owner: PSym,
   result.id = getID()
   when debugIds:
     registerId(result)
-  #if result.id == 77131:
-  #  writeStacktrace()
-  #  echo name.s
 
 proc isMetaType*(t: PType): bool =
   return t.kind in tyMetaTypes or
@@ -1261,6 +1257,9 @@ proc `$`*(x: TLockLevel): string =
   elif x.ord == UnknownLockLevel.ord: result = "<unknown>"
   else: result = $int16(x)
 
+proc `$`*(s: PSym): string =
+  result = s.name.s & "@" & $s.id
+
 proc newType*(kind: TTypeKind, owner: PSym): PType =
   new(result)
   result.kind = kind

compiler/astalgo.nim

@@ -507,7 +507,7 @@ proc strTableContains*(t: TStrTable, n: PSym): bool =
     h = nextTry(h, high(t.data))
   result = false
 
-proc strTableRawInsert(data: var TSymSeq, n: PSym) =
+proc strTableRawInsert(data: var seq[PSym], n: PSym) =
   var h: Hash = n.name.h and high(data)
   if sfImmediate notin n.flags:
     # fast path:
@@ -535,7 +535,7 @@ proc strTableRawInsert(data: var TSymSeq, n: PSym) =
     data[h] = n
     if favPos >= 0: swap data[h], data[favPos]
 
-proc symTabReplaceRaw(data: var TSymSeq, prevSym: PSym, newSym: PSym) =
+proc symTabReplaceRaw(data: var seq[PSym], prevSym: PSym, newSym: PSym) =
   assert prevSym.name.h == newSym.name.h
   var h: Hash = prevSym.name.h and high(data)
   while data[h] != nil:
@@ -549,7 +549,7 @@ proc symTabReplace*(t: var TStrTable, prevSym: PSym, newSym: PSym) =
   symTabReplaceRaw(t.data, prevSym, newSym)
 
 proc strTableEnlarge(t: var TStrTable) =
-  var n: TSymSeq
+  var n: seq[PSym]
   newSeq(n, len(t.data) * GrowthFactor)
   for i in countup(0, high(t.data)):
     if t.data[i] != nil: strTableRawInsert(n, t.data[i])

compiler/ccgcalls.nim

@@ -63,26 +63,6 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
     add(pl, ~");$n")
     line(p, cpsStmts, pl)
 
-proc isInCurrentFrame(p: BProc, n: PNode): bool =
-  # checks if `n` is an expression that refers to the current frame;
-  # this does not work reliably because of forwarding + inlining can break it
-  case n.kind
-  of nkSym:
-    if n.sym.kind in {skVar, skResult, skTemp, skLet} and p.prc != nil:
-      result = p.prc.id == n.sym.owner.id
-  of nkDotExpr, nkBracketExpr:
-    if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyLent,tyPtr,tyRef}:
-      result = isInCurrentFrame(p, n.sons[0])
-  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-    result = isInCurrentFrame(p, n.sons[1])
-  of nkHiddenDeref, nkDerefExpr:
-    # what about: var x = addr(y); callAsOpenArray(x[])?
-    # *shrug* ``addr`` is unsafe anyway.
-    result = false
-  of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
-    result = isInCurrentFrame(p, n.sons[0])
-  else: discard
-
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
 proc openArrayLoc(p: BProc, n: PNode): Rope =

compiler/ccgexprs.nim

@@ -850,7 +850,6 @@ proc genUncheckedArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   var a, b: TLoc
   initLocExpr(p, x, a)
   initLocExpr(p, y, b)
-  var ty = skipTypes(a.t, abstractVarRange + abstractPtrs + tyUserTypeClasses)
   d.inheritLocation(a)
   putIntoDest(p, d, n, ropecg(p.module, "$1[$2]", rdLoc(a), rdCharLoc(b)),
               a.storage)
@@ -867,15 +866,15 @@ proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) =
       # semantic pass has already checked for const index expressions
       if firstOrd(p.config, ty) == 0:
         if (firstOrd(p.config, b.t) < firstOrd(p.config, ty)) or (lastOrd(p.config, b.t) > lastOrd(p.config, ty)):
-          linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)) #raiseIndexError();$n",
+          linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)) #raiseIndexError2($1, $2);$n",
                   rdCharLoc(b), intLiteral(lastOrd(p.config, ty)))
       else:
-        linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3) #raiseIndexError();$n",
+        linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3) #raiseIndexError3($1, $2, $3);$n",
                 rdCharLoc(b), first, intLiteral(lastOrd(p.config, ty)))
     else:
       let idx = getOrdValue(y)
       if idx < firstOrd(p.config, ty) or idx > lastOrd(p.config, ty):
-        localError(p.config, x.info, "index out of bounds")
+        localError(p.config, x.info, formatErrorIndexBound(idx, firstOrd(p.config, ty), lastOrd(p.config, ty)))
   d.inheritLocation(a)
   putIntoDest(p, d, n,
               ropecg(p.module, "$1[($2)- $3]", rdLoc(a), rdCharLoc(b), first), a.storage)
@@ -884,7 +883,6 @@ proc genCStringElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   var a, b: TLoc
   initLocExpr(p, x, a)
   initLocExpr(p, y, b)
-  var ty = skipTypes(a.t, abstractVarRange)
   inheritLocation(d, a)
   putIntoDest(p, d, n,
               ropecg(p.module, "$1[$2]", rdLoc(a), rdCharLoc(b)), a.storage)
@@ -915,7 +913,7 @@ proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   initLocExpr(p, x, a)
   initLocExpr(p, y, b) # emit range check:
   if optBoundsCheck in p.options:
-    linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)) #raiseIndexError();$n",
+    linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)) #raiseIndexError2($1,$2Len_0-1);$n",
             rdLoc(b), rdLoc(a)) # BUGFIX: ``>=`` and not ``>``!
   inheritLocation(d, a)
   putIntoDest(p, d, n,
@@ -931,11 +929,11 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   if optBoundsCheck in p.options:
     if ty.kind == tyString and (not defined(nimNoZeroTerminator) or optLaxStrings in p.options):
       linefmt(p, cpsStmts,
-              "if ((NU)($1) > (NU)$2) #raiseIndexError();$n",
+              "if ((NU)($1) > (NU)$2) #raiseIndexError2($1,$2);$n",
               rdLoc(b), lenExpr(p, a))
     else:
       linefmt(p, cpsStmts,
-              "if ((NU)($1) >= (NU)$2) #raiseIndexError();$n",
+              "if ((NU)($1) >= (NU)$2) #raiseIndexError2($1,$2-1);$n",
               rdLoc(b), lenExpr(p, a))
   if d.k == locNone: d.storage = OnHeap
   if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}:
@@ -1402,7 +1400,6 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) =
   else:
     var i: TLoc
     getTemp(p, getSysType(p.module.g.graph, unknownLineInfo(), tyInt), i)
-    let oldCode = p.s(cpsStmts)
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",  i.r, L.rope)
     initLoc(elem, locExpr, lodeTyp elemType(skipTypes(n.typ, abstractInst)), OnHeap)
     elem.r = ropecg(p.module, "$1$3[$2]", rdLoc(d), rdLoc(i), dataField(p))
@@ -2576,7 +2573,6 @@ proc genConstObjConstr(p: BProc; n: PNode): Rope =
 proc genConstSimpleList(p: BProc, n: PNode): Rope =
   var length = sonsLen(n)
   result = rope("{")
-  let t = n.typ.skipTypes(abstractInst)
   for i in countup(0, length - 2):
     addf(result, "$1,$n", [genNamedConstExpr(p, n.sons[i])])
   if length > 0:

compiler/ccgstmts.nim

@@ -559,7 +559,8 @@ proc genParForStmt(p: BProc, t: PNode) =
     initLocExpr(p, call.sons[1], rangeA)
     initLocExpr(p, call.sons[2], rangeB)
 
-    lineF(p, cpsStmts, "#pragma omp $4$n" &
+    # $n at the beginning because of #9710
+    lineF(p, cpsStmts, "$n#pragma omp $4$n" &
                         "for ($1 = $2; $1 <= $3; ++$1)",
                         [forLoopVar.loc.rdLoc,
                         rangeA.rdLoc, rangeB.rdLoc,

compiler/cgen.nim

@@ -16,6 +16,8 @@ import
   condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
   lowerings, tables, sets, ndi, lineinfos, pathutils, transf
 
+import system/helpers2
+
 when not defined(leanCompiler):
   import semparallel
 

compiler/cgmeth.nim

@@ -188,7 +188,7 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
   elif sfBase notin s.flags:
     message(g.config, s.info, warnUseBase)
 
-proc relevantCol(methods: TSymSeq, col: int): bool =
+proc relevantCol(methods: seq[PSym], col: int): bool =
   # returns true iff the position is relevant
   var t = methods[0].typ.sons[col].skipTypes(skipPtrs)
   if t.kind == tyObject:
@@ -206,7 +206,7 @@ proc cmpSignatures(a, b: PSym, relevantCols: IntSet): int =
       if (d != high(int)) and d != 0:
         return d
 
-proc sortBucket(a: var TSymSeq, relevantCols: IntSet) =
+proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
   # we use shellsort here; fast and simple
   var n = len(a)
   var h = 1
@@ -225,7 +225,7 @@ proc sortBucket(a: var TSymSeq, relevantCols: IntSet) =
       a[j] = v
     if h == 1: break
 
-proc genDispatcher(g: ModuleGraph; methods: TSymSeq, relevantCols: IntSet): PSym =
+proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PSym =
   var base = lastSon(methods[0].ast).sym
   result = base
   var paramLen = sonsLen(base.typ)

compiler/commands.nim

@@ -480,7 +480,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "native", "gdb":
       incl(conf.globalOptions, optCDebug)
       conf.options = conf.options + {optLineDir} - {optEndb}
-      defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
+      #defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
       undefSymbol(conf.symbols, "endb")
     else:
       localError(conf, info, "expected endb|gdb but found " & arg)
@@ -617,6 +617,14 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "run", "r":
     expectNoArg(conf, switch, arg, pass, info)
     incl(conf.globalOptions, optRun)
+  of "errormax":
+    expectArg(conf, switch, arg, pass, info)
+    # Note: `nim check` (etc) can overwrite this.
+    # `0` is meaningless, give it a useful meaning as in clang's -ferror-limit
+    # If user doesn't set this flag and the code doesn't either, it'd
+    # have the same effect as errorMax = 1
+    let ret = parseInt(arg)
+    conf.errorMax = if ret == 0: high(int) else: ret
   of "verbosity":
     expectArg(conf, switch, arg, pass, info)
     let verbosity = parseInt(arg)

compiler/destroyer.nim

@@ -116,7 +116,7 @@ Remarks: Rule 1.2 is not yet implemented because ``sink`` is currently
 
 import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
-  strutils, options, dfa, lowerings, tables, modulegraphs,
+  strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
   lineinfos, parampatterns
 
 const
@@ -127,20 +127,11 @@ type
     owner: PSym
     g: ControlFlowGraph
     jumpTargets: IntSet
-    tmpObj: PType
-    tmp: PSym
     destroys, topLevelVars: PNode
-    toDropBit: Table[int, PSym]
     graph: ModuleGraph
     emptyNode: PNode
     otherRead: PNode
 
-proc getTemp(c: var Con; typ: PType; info: TLineInfo): PNode =
-  # XXX why are temps fields in an object here?
-  let f = newSym(skField, getIdent(c.graph.cache, ":d" & $c.tmpObj.n.len), c.owner, info)
-  f.typ = typ
-  rawAddField c.tmpObj, f
-  result = rawDirectAccess(c.tmp, f)
 
 proc isHarmlessVar*(s: PSym; c: Con): bool =
   # 's' is harmless if it used only once and its
@@ -329,22 +320,11 @@ proc genDestroy(c: Con; t: PType; dest: PNode): PNode =
 proc addTopVar(c: var Con; v: PNode) =
   c.topLevelVars.add newTree(nkIdentDefs, v, c.emptyNode, c.emptyNode)
 
-proc dropBit(c: var Con; s: PSym): PSym =
+proc getTemp(c: var Con; typ: PType; info: TLineInfo): PNode =
-  result = c.toDropBit.getOrDefault(s.id)
+  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), c.owner, info)
-  assert result != nil
+  sym.typ = typ
-
+  result = newSymNode(sym)
-proc registerDropBit(c: var Con; s: PSym) =
+  c.addTopVar(result)
-  let result = newSym(skTemp, getIdent(c.graph.cache, s.name.s & "_AliveBit"), c.owner, s.info)
-  result.typ = getSysType(c.graph, s.info, tyBool)
-  let trueVal = newIntTypeNode(nkIntLit, 1, result.typ)
-  c.topLevelVars.add newTree(nkIdentDefs, newSymNode result, c.emptyNode, trueVal)
-  c.toDropBit[s.id] = result
-  # generate:
-  #  if not sinkParam_AliveBit: `=destroy`(sinkParam)
-  let t = s.typ.skipTypes({tyGenericInst, tyAlias, tySink})
-  if t.destructor != nil:
-    c.destroys.add newTree(nkIfStmt,
-      newTree(nkElifBranch, newSymNode result, genDestroy(c, t, newSymNode s)))
 
 proc p(n: PNode; c: var Con): PNode
 
@@ -355,16 +335,6 @@ template recurse(n, dest) =
 proc isSinkParam(s: PSym): bool {.inline.} =
   result = s.kind == skParam and s.typ.kind == tySink
 
-proc destructiveMoveSink(n: PNode; c: var Con): PNode =
-  # generate:  (chckMove(sinkParam_AliveBit); sinkParam_AliveBit = false; sinkParam)
-  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-  let bit = newSymNode dropBit(c, n.sym)
-  if optMoveCheck in c.owner.options:
-    result.add callCodegenProc(c.graph, "chckMove", bit.info, bit)
-  result.add newTree(nkAsgn, bit,
-    newIntTypeNode(nkIntLit, 0, getSysType(c.graph, n.info, tyBool)))
-  result.add n
-
 proc genMagicCall(n: PNode; c: var Con; magicname: string; m: TMagic): PNode =
   result = newNodeI(nkCall, n.info)
   result.add(newSymNode(createMagic(c.graph, magicname, m)))
@@ -395,6 +365,12 @@ proc destructiveMoveVar(n: PNode; c: var Con): PNode =
   result.add genWasMoved(n, c)
   result.add tempAsNode
 
+proc sinkParamIsLastReadCheck(c: var Con, s: PNode) = 
+  assert s.kind == nkSym and s.sym.kind == skParam
+  if not isLastRead(s, c):
+     localError(c.graph.config, c.otherRead.info, "sink parameter `" & $s.sym.name.s &
+         "` is already consumed at " & toFileLineCol(c. graph.config, s.info))
+
 proc passCopyToSink(n: PNode; c: var Con): PNode =
   result = newNodeIT(nkStmtListExpr, n.info, n.typ)
   let tmp = getTemp(c, n.typ, n.info)
@@ -411,6 +387,11 @@ proc passCopyToSink(n: PNode; c: var Con): PNode =
   result.add tmp
 
 proc pArg(arg: PNode; c: var Con; isSink: bool): PNode =
+  template pArgIfTyped(arg_part: PNode): PNode =
+    # typ is nil if we are in if/case expr branch with noreturn
+    if arg_part.typ == nil: p(arg_part, c)
+    else: pArg(arg_part, c, isSink)
+
   if isSink:
     if arg.kind in nkCallKinds:
       # recurse but skip the call expression in order to prevent
@@ -421,17 +402,53 @@ proc pArg(arg: PNode; c: var Con; isSink: bool): PNode =
       result.add arg[0]
       for i in 1..<arg.len:
         result.add pArg(arg[i], c, i < L and parameters[i].kind == tySink)
-    elif arg.kind in {nkObjConstr, nkCharLit..nkFloat128Lit}:
+    elif arg.kind in {nkBracket, nkObjConstr, nkTupleConstr, nkBracket, nkCharLit..nkFloat128Lit}:
       discard "object construction to sink parameter: nothing to do"
       result = arg
-    elif arg.kind == nkSym and arg.sym.kind in InterestingSyms and isLastRead(arg, c):
-      # if x is a variable and it its last read we eliminate its
-      # destructor invokation, but don't. We need to reset its memory
-      # to disable its destructor which we have not elided:
-      result = destructiveMoveVar(arg, c)
     elif arg.kind == nkSym and isSinkParam(arg.sym):
-      # mark the sink parameter as used:
+      # Sinked params can be consumed only once. We need to reset the memory
-      result = destructiveMoveSink(arg, c)
+      # to disable the destructor which we have not elided
+      sinkParamIsLastReadCheck(c, arg)
+      result = destructiveMoveVar(arg, c)
+    elif arg.kind == nkSym and arg.sym.kind in InterestingSyms and isLastRead(arg, c):
+      # it is the last read, can be sinked. We need to reset the memory
+      # to disable the destructor which we have not elided
+      result = destructiveMoveVar(arg, c)
+    elif arg.kind in {nkBlockExpr, nkBlockStmt}:
+      result = copyNode(arg)
+      result.add arg[0]
+      result.add pArg(arg[1], c, isSink)
+    elif arg.kind == nkStmtListExpr:
+      result = copyNode(arg)
+      for i in 0..arg.len-2:
+        result.add p(arg[i], c)
+      result.add pArg(arg[^1], c, isSink)
+    elif arg.kind in {nkIfExpr, nkIfStmt}:
+      result = copyNode(arg)
+      for i in 0..<arg.len:
+        var branch = copyNode(arg[i])
+        if arg[i].kind in {nkElifBranch, nkElifExpr}:   
+          branch.add p(arg[i][0], c)
+          branch.add pArgIfTyped(arg[i][1])
+        else:
+          branch.add pArgIfTyped(arg[i][0])
+        result.add branch
+    elif arg.kind == nkCaseStmt:
+      result = copyNode(arg)
+      result.add p(arg[0], c)
+      for i in 1..<arg.len:
+        var branch: PNode
+        if arg[i].kind == nkOfbranch:
+          branch = arg[i] # of branch conditions are constants
+          branch[^1] = pArgIfTyped(arg[i][^1])
+        elif arg[i].kind in {nkElifBranch, nkElifExpr}:
+          branch = copyNode(arg[i])   
+          branch.add p(arg[i][0], c)
+          branch.add pArgIfTyped(arg[i][1])
+        else:
+          branch = copyNode(arg[i]) 
+          branch.add pArgIfTyped(arg[i][0])
+        result.add branch     
     else:
       # an object that is not temporary but passed to a 'sink' parameter
       # results in a copy.
@@ -440,6 +457,11 @@ proc pArg(arg: PNode; c: var Con; isSink: bool): PNode =
     result = p(arg, c)
 
 proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
+  template moveOrCopyIfTyped(ri_part: PNode): PNode =
+    # typ is nil if we are in if/case expr branch with noreturn
+    if ri_part.typ == nil: p(ri_part, c)
+    else: moveOrCopy(dest, ri_part, c)
+
   case ri.kind
   of nkCallKinds:
     result = genSink(c, dest.typ, dest, ri)
@@ -464,6 +486,45 @@ proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
     for i in 0..ri.len-2:
       result.add p(ri[i], c)
     result.add moveOrCopy(dest, ri[^1], c)
+  of nkBlockExpr, nkBlockStmt:
+    result = newNodeI(nkBlockStmt, ri.info)
+    result.add ri[0] ## add label
+    result.add moveOrCopy(dest, ri[1], c)
+  of nkIfExpr, nkIfStmt:
+    result = newNodeI(nkIfStmt, ri.info)
+    for i in 0..<ri.len:
+      var branch = copyNode(ri[i])
+      if ri[i].kind in {nkElifBranch, nkElifExpr}:
+        branch.add p(ri[i][0], c)
+        branch.add moveOrCopyIfTyped(ri[i][1])
+      else:
+        branch.add moveOrCopyIfTyped(ri[i][0])
+      result.add branch
+  of nkCaseStmt:
+    result = newNodeI(nkCaseStmt, ri.info)
+    result.add p(ri[0], c)
+    for i in 1..<ri.len:
+      var branch: PNode
+      if ri[i].kind == nkOfbranch:
+        branch = ri[i] # of branch conditions are constants
+        branch[^1] = moveOrCopyIfTyped(ri[i][^1])
+      elif ri[i].kind in {nkElifBranch, nkElifExpr}:
+        branch = copyNode(ri[i])   
+        branch.add p(ri[i][0], c)
+        branch.add moveOrCopyIfTyped(ri[i][1])
+      else:
+        branch = copyNode(ri[i]) 
+        branch.add moveOrCopyIfTyped(ri[i][0])
+      result.add branch
+  of nkBracket:
+    # array constructor
+    result = genSink(c, dest.typ, dest, ri)
+    let ri2 = copyTree(ri)
+    for i in 0..<ri.len:
+      # everything that is passed to an array constructor is consumed,
+      # so these all act like 'sink' parameters:
+      ri2[i] = pArg(ri[i], c, isSink = true)
+    result.add ri2
   of nkObjConstr:
     result = genSink(c, dest.typ, dest, ri)
     let ri2 = copyTree(ri)
@@ -484,14 +545,17 @@ proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
         ri2[i] = pArg(ri[i], c, isSink = true)
     result.add ri2
   of nkSym:
-    if ri.sym.kind != skParam and isLastRead(ri, c):
+    if isSinkParam(ri.sym):
+      # Rule 3: `=sink`(x, z); wasMoved(z)
+      sinkParamIsLastReadCheck(c, ri)
+      var snk = genSink(c, dest.typ, dest, ri)
+      snk.add ri
+      result = newTree(nkStmtList, snk, genMagicCall(ri, c, "wasMoved", mWasMoved))   
+    elif ri.sym.kind != skParam and isLastRead(ri, c):
       # Rule 3: `=sink`(x, z); wasMoved(z)
       var snk = genSink(c, dest.typ, dest, ri)
-      snk.add p(ri, c)
+      snk.add ri
       result = newTree(nkStmtList, snk, genMagicCall(ri, c, "wasMoved", mWasMoved))
-    elif isSinkParam(ri.sym):
-      result = genSink(c, dest.typ, dest, ri)
-      result.add destructiveMoveSink(ri, c)
     else:
       result = genCopy(c, dest.typ, dest, ri)
       result.add p(ri, c)
@@ -512,13 +576,14 @@ proc p(n: PNode; c: var Con): PNode =
       if it.kind == nkVarTuple and hasDestructor(ri.typ):
         let x = lowerTupleUnpacking(c.graph, it, c.owner)
         result.add p(x, c)
-      elif it.kind == nkIdentDefs and hasDestructor(it[0].typ) and not isUnpackedTuple(it[0].sym):
+      elif it.kind == nkIdentDefs and hasDestructor(it[0].typ):
         for j in 0..L-2:
           let v = it[j]
           doAssert v.kind == nkSym
           # move the variable declaration to the top of the frame:
           c.addTopVar v
           # make sure it's destroyed at the end of the proc:
+          if not isUnpackedTuple(it[0].sym):
             c.destroys.add genDestroy(c, v.typ, v)
           if ri.kind != nkEmpty:
             let r = moveOrCopy(v, ri, c)
@@ -566,12 +631,8 @@ proc injectDestructorCalls*(g: ModuleGraph; owner: PSym; n: PNode): PNode =
     echo "injecting into ", n
   var c: Con
   c.owner = owner
-  c.tmp = newSym(skTemp, getIdent(g.cache, ":d"), owner, n.info)
-  c.tmpObj = createObj(g, owner, n.info)
-  c.tmp.typ = c.tmpObj
   c.destroys = newNodeI(nkStmtList, n.info)
   c.topLevelVars = newNodeI(nkVarSection, n.info)
-  c.toDropBit = initTable[int, PSym]()
   c.graph = g
   c.emptyNode = newNodeI(nkEmpty, n.info)
   let cfg = constructCfg(owner, n)
@@ -586,10 +647,10 @@ proc injectDestructorCalls*(g: ModuleGraph; owner: PSym; n: PNode): PNode =
     let params = owner.typ.n
     for i in 1 ..< params.len:
       let param = params[i].sym
-      if param.typ.kind == tySink: registerDropBit(c, param)
+      if param.typ.kind == tySink and hasDestructor(param.typ): 
+        c.destroys.add genDestroy(c, param.typ.skipTypes({tyGenericInst, tyAlias, tySink}), params[i])
+
   let body = p(n, c)
-  if c.tmp.typ.n.len > 0:
-    c.addTopVar(newSymNode c.tmp)
   result = newNodeI(nkStmtList, n.info)
   if c.topLevelVars.len > 0:
     result.add c.topLevelVars

compiler/dfa.nim

@@ -272,7 +272,7 @@ proc genReturn(c: var Con; n: PNode) =
   c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 const
-  InterestingSyms = {skVar, skResult, skLet}
+  InterestingSyms = {skVar, skResult, skLet, skParam}
 
 proc genUse(c: var Con; n: PNode) =
   var n = n

compiler/evaltempl.nim

@@ -37,18 +37,21 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
   case templ.kind
   of nkSym:
     var s = templ.sym
-    if s.owner.id == c.owner.id:
+    if s.owner == nil or s.owner.id == c.owner.id:
       if s.kind == skParam and sfGenSym notin s.flags:
         handleParam actual.sons[s.position]
-      elif s.kind == skGenericParam or
+      elif (s.owner != nil) and (s.kind == skGenericParam or
-           s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam:
+           s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam):
         handleParam actual.sons[s.owner.typ.len + s.position - 1]
       else:
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
           x = copySym(s)
-          x.owner = c.genSymOwner
+          # sem'check needs to set the owner properly later, see bug #9476
+          x.owner = nil # c.genSymOwner
+          #if x.kind == skParam and x.owner.kind == skModule:
+          #  internalAssert c.config, false
           idTablePut(c.mapping, s, x)
         result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
     else:
@@ -173,6 +176,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   initIdTable(ctx.mapping)
 
   let body = tmpl.getBody
+  #echo "instantion of ", renderTree(body, {renderIds})
   if isAtom(body):
     result = newNodeI(nkPar, body.info)
     evalTemplateAux(body, args, ctx, result)
@@ -189,5 +193,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
       evalTemplateAux(body.sons[i], args, ctx, result)
   result.flags.incl nfFromTemplate
   result = wrapInComesFrom(n.info, tmpl, result)
+  #if ctx.debugActive:
+  #  echo "instantion of ", renderTree(result, {renderIds})
   dec(conf.evalTemplateCounter)
 

compiler/extccomp.nim

@@ -763,6 +763,32 @@ proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx:
       rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" %
         cmds.join())
 
+proc linkViaResponseFile(conf: ConfigRef; cmd: string) =
+  # Extracting the linker.exe here is a bit hacky but the best solution
+  # given ``buildLib``'s design.
+  var i = 0
+  var last = 0
+  if cmd.len > 0 and cmd[0] == '"':
+    inc i
+    while i < cmd.len and cmd[i] != '"': inc i
+    last = i
+    inc i
+  else:
+    while i < cmd.len and cmd[i] != ' ': inc i
+    last = i
+  while i < cmd.len and cmd[i] == ' ': inc i
+  let linkerArgs = conf.projectName & "_" & "linkerArgs.txt"
+  let args = cmd.substr(i)
+  # GCC's response files don't support backslashes. Junk.
+  if conf.cCompiler == ccGcc:
+    writeFile(linkerArgs, args.replace('\\', '/'))
+  else:
+    writeFile(linkerArgs, args)
+  try:
+    execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs)
+  finally:
+    removeFile(linkerArgs)
+
 proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) =
   var
     linkCmd: string
@@ -795,6 +821,12 @@ proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) =
 
     linkCmd = getLinkCmd(conf, projectfile, objfiles)
     if optCompileOnly notin conf.globalOptions:
+      if defined(windows) and linkCmd.len > 8_000:
+        # Windows's command line limit is about 8K (don't laugh...) so C compilers on
+        # Windows support a feature where the command line can be passed via ``@linkcmd``
+        # to them.
+        linkViaResponseFile(conf, linkCmd)
+      else:
         execLinkCmd(conf, linkCmd)
   else:
     linkCmd = ""

compiler/jsgen.nim

@@ -945,7 +945,7 @@ proc needsNoCopy(p: PProc; y: PNode): bool =
   else:
     return (mapType(y.typ) != etyBaseIndex and
             (skipTypes(y.typ, abstractInst).kind in
-             {tyRef, tyPtr, tyLent, tyVar, tyCString} + IntegralTypes))
+             {tyRef, tyPtr, tyLent, tyVar, tyCString, tyProc} + IntegralTypes))
   return true
 
 proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =

compiler/lambdalifting.nim

@@ -407,11 +407,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
               obj.n[0].sym.id = -s.id
             else:
               addField(obj, s, c.graph.cache)
-      # but always return because the rest of the proc is only relevant when
-      # ow != owner:
-      return
     # direct or indirect dependency:
-    if (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
+    elif (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
       discard """
         proc outer() =
           var x: int

compiler/lookups.nim

@@ -169,7 +169,7 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) =
             getSymRepr(c.config, s))
       inc missingImpls
     elif {sfUsed, sfExported} * s.flags == {}:
-      if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam}:
+      if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam, skEnumField}:
         # XXX: implicit type params are currently skTypes
         # maybe they can be made skGenericParam as well.
         if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and

compiler/main.nim

@@ -266,11 +266,23 @@ proc mainCommand*(graph: ModuleGraph) =
       var libpaths = newJArray()
       for dir in conf.searchPaths: libpaths.elems.add(%dir.string)
 
+      var hints = newJObject() # consider factoring with `listHints`
+      for a in hintMin..hintMax:
+        let key = lineinfos.HintsToStr[ord(a) - ord(hintMin)]
+        hints[key] = %(a in conf.notes)
+      var warnings = newJObject()
+      for a in warnMin..warnMax:
+        let key = lineinfos.WarningsToStr[ord(a) - ord(warnMin)]
+        warnings[key] = %(a in conf.notes)
+
       var dumpdata = %[
         (key: "version", val: %VersionAsString),
         (key: "project_path", val: %conf.projectFull.string),
         (key: "defined_symbols", val: definedSymbols),
-        (key: "lib_paths", val: libpaths)
+        (key: "lib_paths", val: %libpaths),
+        (key: "out", val: %conf.outFile.string),
+        (key: "hints", val: hints),
+        (key: "warnings", val: warnings),
       ]
 
       msgWriteln(conf, $dumpdata, {msgStdout, msgSkipHook})

compiler/modulegraphs.nim

@@ -47,7 +47,7 @@ type
     doStopCompile*: proc(): bool {.closure.}
     usageSym*: PSym # for nimsuggest
     owners*: seq[PSym]
-    methods*: seq[tuple[methods: TSymSeq, dispatcher: PSym]] # needs serialization!
+    methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization!
     systemModule*: PSym
     sysTypes*: array[TTypeKind, PType]
     compilerprocs*: TStrTable

compiler/nim.nim

@@ -9,14 +9,14 @@
 
 when defined(gcc) and defined(windows):
   when defined(x86):
-    {.link: "icons/nim.res".}
+    {.link: "../icons/nim.res".}
   else:
-    {.link: "icons/nim_icon.o".}
+    {.link: "../icons/nim_icon.o".}
 
 when defined(amd64) and defined(windows) and defined(vcc):
-  {.link: "icons/nim-amd64-windows-vcc.res".}
+  {.link: "../icons/nim-amd64-windows-vcc.res".}
 when defined(i386) and defined(windows) and defined(vcc):
-  {.link: "icons/nim-i386-windows-vcc.res".}
+  {.link: "../icons/nim-i386-windows-vcc.res".}
 
 import
   commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,

compiler/pragmas.nim

@@ -1111,8 +1111,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         else: sym.flags.incl sfUsed
       of wLiftLocals: discard
       else: invalidPragma(c, it)
-    elif sym != nil and sym.kind in {skVar, skLet, skParam, skField, skProc,
+    elif sym == nil or (sym != nil and sym.kind in {skVar, skLet, skParam, 
-                                     skFunc, skConverter, skMethod, skType}:
+                      skField, skProc, skFunc, skConverter, skMethod, skType}):
       n.sons[i] = semCustomPragma(c, it)
     elif sym != nil:
       illegalCustomPragma(c, it, sym)
@@ -1138,7 +1138,7 @@ proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
             internalError(c.config, n.info, "implicitPragmas")
           inc i
         popInfoContext(c.config)
-        if sym.kind in routineKinds: mergePragmas(sym.ast, o)
+        if sym.kind in routineKinds and sym.ast != nil: mergePragmas(sym.ast, o)
 
     if lfExportLib in sym.loc.flags and sfExportc notin sym.flags:
       localError(c.config, n.info, ".dynlib requires .exportc")

compiler/sem.nim

@@ -207,6 +207,7 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
     if result.kind notin {kind, skTemp}:
       localError(c.config, n.info, "cannot use symbol of kind '" &
                  $result.kind & "' as a '" & $kind & "'")
+    when false:
       if sfGenSym in result.flags and result.kind notin {skTemplate, skMacro, skParam}:
         # declarative context, so produce a fresh gensym:
         result = copySym(result)

compiler/semasgn.nim

@@ -309,6 +309,10 @@ proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
   liftBodyAux(a, typ, body, newSymNode(dest).newDeref, newSymNode(src))
   # recursion is handled explicitly, do not register the type based operation
   # before 'liftBodyAux':
+  if c.config.selectedGC == gcDestructors and
+      typ.kind in {tySequence, tyString} and body.len == 0:
+    discard "do not cache it yet"
+  else:
     case kind
     of attachedAsgn: typ.assignment = result
     of attachedSink: typ.sink = result

compiler/semdata.nim

@@ -100,8 +100,8 @@ type
     compilesContextId*: int    # > 0 if we are in a ``compiles`` magic
     compilesContextIdGenerator*: int
     inGenericInst*: int        # > 0 if we are instantiating a generic
-    converters*: TSymSeq       # sequence of converters
+    converters*: seq[PSym]
-    patterns*: TSymSeq         # sequence of pattern matchers
+    patterns*: seq[PSym]       # sequence of pattern matchers
     optionStack*: seq[POptionEntry]
     symMapping*: TIdTable      # every gensym'ed symbol needs to be mapped
                                # to some new symbol in a generic instantiation
@@ -239,7 +239,7 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext =
   result.typesWithOps = @[]
   result.features = graph.config.features
 
-proc inclSym(sq: var TSymSeq, s: PSym) =
+proc inclSym(sq: var seq[PSym], s: PSym) =
   var L = len(sq)
   for i in countup(0, L - 1):
     if sq[i].id == s.id: return

compiler/semexprs.nim

@@ -1083,6 +1083,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
       # XXX see the hack in sigmatch.nim ...
       return s.typ.n
     elif sfGenSym in s.flags:
+      # the owner should have been set by now by addParamOrResult
+      internalAssert c.config, s.owner != nil
       if c.p.wasForwarded:
         # gensym'ed parameters that nevertheless have been forward declared
         # need a special fixup:
@@ -2289,6 +2291,8 @@ proc semBlock(c: PContext, n: PNode; flags: TExprFlags): PNode =
     var labl = newSymG(skLabel, n.sons[0], c)
     if sfGenSym notin labl.flags:
       addDecl(c, labl)
+    elif labl.owner == nil:
+      labl.owner = c.p.owner
     n.sons[0] = newSymNode(labl, n.sons[0].info)
     suggestSym(c.config, n.sons[0].info, labl, c.graph.usageSym)
     styleCheckDef(c.config, labl)

compiler/semfold.nim

@@ -15,6 +15,8 @@ import
   nversion, platform, math, msgs, os, condsyms, idents, renderer, types,
   commands, magicsys, modulegraphs, strtabs, lineinfos
 
+import system/helpers2
+
 proc newIntNodeT*(intVal: BiggestInt, n: PNode; g: ModuleGraph): PNode =
   case skipTypes(n.typ, abstractVarRange).kind
   of tyInt:
@@ -489,11 +491,11 @@ proc foldArrayAccess(m: PSym, n: PNode; g: ModuleGraph): PNode =
       result = x.sons[int(idx)]
       if result.kind == nkExprColonExpr: result = result.sons[1]
     else:
-      localError(g.config, n.info, "index out of bounds: " & $n)
+      localError(g.config, n.info, formatErrorIndexBound(idx, sonsLen(x)+1) & $n)
   of nkBracket:
     idx = idx - firstOrd(g.config, x.typ)
     if idx >= 0 and idx < x.len: result = x.sons[int(idx)]
-    else: localError(g.config, n.info, "index out of bounds: " & $n)
+    else: localError(g.config, n.info, formatErrorIndexBound(idx, x.len+1) & $n)
   of nkStrLit..nkTripleStrLit:
     result = newNodeIT(nkCharLit, x.info, n.typ)
     if idx >= 0 and idx < len(x.strVal):
@@ -501,7 +503,7 @@ proc foldArrayAccess(m: PSym, n: PNode; g: ModuleGraph): PNode =
     elif idx == len(x.strVal) and optLaxStrings in g.config.options:
       discard
     else:
-      localError(g.config, n.info, "index out of bounds: " & $n)
+      localError(g.config, n.info, formatErrorIndexBound(idx, len(x.strVal)-1) & $n)
   else: discard
 
 proc foldFieldAccess(m: PSym, n: PNode; g: ModuleGraph): PNode =
@@ -626,6 +628,14 @@ proc getConstExpr(m: PSym, n: PNode; g: ModuleGraph): PNode =
         # It doesn't matter if the argument is const or not for mLengthArray.
         # This fixes bug #544.
         result = newIntNodeT(lengthOrd(g.config, n.sons[1].typ), n, g)
+      of mSizeOf:
+        let size = getSize(g.config, n[1].typ)
+        if size >= 0:
+          result = newIntNode(nkIntLit, size)
+          result.info = n.info
+          result.typ = getSysType(g, n.info, tyInt)
+        else:
+          result = nil
       of mAstToStr:
         result = newStrNodeT(renderTree(n[1], {renderNoComments}), n, g)
       of mConStrStr:

compiler/seminst.nim

@@ -116,7 +116,7 @@ proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) =
     var x = PSym(idTableGet(symMap, s))
     if x != nil:
       n.sym = x
-    elif s.owner.kind == skPackage:
+    elif s.owner == nil or s.owner.kind == skPackage:
       #echo "copied this ", s.name.s
       x = copySym(s)
       x.owner = owner

compiler/semmagic.nim

@@ -321,8 +321,6 @@ proc semOf(c: PContext, n: PNode): PNode =
 proc magicsAfterOverloadResolution(c: PContext, n: PNode,
                                    flags: TExprFlags): PNode =
   ## This is the preferred code point to implement magics.
-  ## This function basically works like a macro, with the difference
-  ## that it is implemented in the compiler and not on the nimvm.
   ## ``c`` the current module, a symbol table to a very good approximation
   ## ``n`` the ast like it would be passed to a real macro
   ## ``flags`` Some flags for more contextual information on how the

compiler/semstmts.nim

@@ -497,8 +497,10 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
       var v = semIdentDef(c, a.sons[j], symkind)
       styleCheckDef(c.config, v)
       onDef(a[j].info, v)
-      if sfGenSym notin v.flags and not isDiscardUnderscore(v):
+      if sfGenSym notin v.flags:
-        addInterfaceDecl(c, v)
+        if not isDiscardUnderscore(v): addInterfaceDecl(c, v)
+      else:
+        if v.owner == nil: v.owner = c.p.owner
       when oKeepVariableNames:
         if c.inUnrolledContext > 0: v.flags.incl(sfShadowed)
         else:
@@ -573,6 +575,7 @@ proc semConst(c: PContext, n: PNode): PNode =
     setVarType(c, v, typ)
     v.ast = def               # no need to copy
     if sfGenSym notin v.flags: addInterfaceDecl(c, v)
+    elif v.owner == nil: v.owner = getCurrOwner(c)
     var b = newNodeI(nkConstDef, a.info)
     if importantComments(c.config): b.comment = a.comment
     addSon(b, newSymNode(v))
@@ -616,6 +619,7 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode =
       v.typ = iterBase
       n.sons[0] = newSymNode(v)
       if sfGenSym notin v.flags: addForVarDecl(c, v)
+      elif v.owner == nil: v.owner = getCurrOwner(c)
     else:
       localError(c.config, n.info, errWrongNumberOfVariables)
   elif length-2 != sonsLen(iter):
@@ -626,8 +630,9 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode =
       if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal)
       v.typ = iter.sons[i]
       n.sons[i] = newSymNode(v)
-      if sfGenSym notin v.flags and not isDiscardUnderscore(v):
+      if sfGenSym notin v.flags:
-        addForVarDecl(c, v)
+        if not isDiscardUnderscore(v): addForVarDecl(c, v)
+      elif v.owner == nil: v.owner = getCurrOwner(c)
   inc(c.p.nestedLoopCounter)
   openScope(c)
   n.sons[length-1] = semExprBranch(c, n.sons[length-1], flags)
@@ -922,6 +927,7 @@ proc typeSectionLeftSidePass(c: PContext, n: PNode) =
             s = typsym
       # add it here, so that recursive types are possible:
       if sfGenSym notin s.flags: addInterfaceDecl(c, s)
+      elif s.owner == nil: s.owner = getCurrOwner(c)
 
     if name.kind == nkPragmaExpr:
       a.sons[0].sons[0] = newSymNode(s)
@@ -1620,7 +1626,8 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     else:
       s.typ.callConv = lastOptionEntry(c).defaultCC
     # add it here, so that recursive procs are possible:
-    if sfGenSym in s.flags: discard
+    if sfGenSym in s.flags:
+      if s.owner == nil: s.owner = getCurrOwner(c)
     elif kind in OverloadableSyms:
       if not typeIsDetermined:
         addInterfaceOverloadableSymAt(c, oldScope, s)
@@ -1836,35 +1843,49 @@ proc semMacroDef(c: PContext, n: PNode): PNode =
   if n.sons[bodyPos].kind == nkEmpty:
     localError(c.config, n.info, errImplOfXexpected % s.name.s)
 
-proc evalInclude(c: PContext, n: PNode): PNode =
+proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult: PNode) =
-  result = newNodeI(nkStmtList, n.info)
+  var f = checkModuleName(c.config, it)
-  addSon(result, n)
-  for i in countup(0, sonsLen(n) - 1):
-    var f = checkModuleName(c.config, n.sons[i])
   if f != InvalidFileIDX:
     if containsOrIncl(c.includedFiles, f.int):
       localError(c.config, n.info, errRecursiveDependencyX % toFilename(c.config, f))
     else:
-        addSon(result, semStmt(c, c.graph.includeFileCallback(c.graph, c.module, f), {}))
+      addSon(includeStmtResult, semStmt(c, c.graph.includeFileCallback(c.graph, c.module, f), {}))
       excl(c.includedFiles, f.int)
 
+proc evalInclude(c: PContext, n: PNode): PNode =
+  result = newNodeI(nkStmtList, n.info)
+  addSon(result, n)
+  for i in countup(0, sonsLen(n) - 1):
+    var imp: PNode
+    let it = n.sons[i]
+    if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
+      let sep = it[0]
+      let dir = it[1]
+      imp = newNodeI(nkInfix, it.info)
+      imp.add sep
+      imp.add dir
+      imp.add sep # dummy entry, replaced in the loop
+      for x in it[2]:
+        imp.sons[2] = x
+        incMod(c, n, imp, result)
+    else:
+      incMod(c, n, it, result)
+
 proc setLine(n: PNode, info: TLineInfo) =
   for i in 0 ..< safeLen(n): setLine(n.sons[i], info)
   n.info = info
 
 proc semPragmaBlock(c: PContext, n: PNode): PNode =
+  checkSonsLen(n, 2, c.config)
   let pragmaList = n.sons[0]
   pragma(c, nil, pragmaList, exprPragmas)
-  result = semExpr(c, n.sons[1])
+  n[1] = semExpr(c, n[1])
-  n.sons[1] = result
+  result = n
+  result.typ = n[1].typ
   for i in 0 ..< pragmaList.len:
     case whichPragma(pragmaList.sons[i])
     of wLine: setLine(result, pragmaList.sons[i].info)
-    of wLocks, wGcSafe, wNosideeffect:
+    of wNoRewrite: incl(result.flags, nfNoRewrite)
-      result = n
-      result.typ = n.sons[1].typ
-    of wNoRewrite:
-      incl(result.flags, nfNoRewrite)
     else: discard
 
 proc semStaticStmt(c: PContext, n: PNode): PNode =

compiler/semtypes.nim

@@ -90,6 +90,8 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
         if sonsLen(v) == 2:
           strVal = v.sons[1] # second tuple part is the string value
           if skipTypes(strVal.typ, abstractInst).kind in {tyString, tyCString}:
+            if not isOrdinalType(v.sons[0].typ):
+              localError(c.config, v.sons[0].info, errOrdinalTypeExpected)
             x = getOrdValue(v.sons[0]) # first tuple part is the ordinal
           else:
             localError(c.config, strVal.info, errStringLiteralExpected)
@@ -99,6 +101,8 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
         strVal = v
         x = counter
       else:
+        if not isOrdinalType(v.typ):
+          localError(c.config, v.info, errOrdinalTypeExpected)
         x = getOrdValue(v)
       if i != 1:
         if x != counter: incl(result.flags, tfEnumHasHoles)
@@ -821,7 +825,11 @@ proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind) =
       a.typ = nn.typ
       addDecl(c, a)
   else:
-    if sfGenSym notin param.flags: addDecl(c, param)
+    if sfGenSym in param.flags:
+      # bug #XXX, fix the gensym'ed parameters owner:
+      if param.owner == nil:
+        param.owner = getCurrOwner(c)
+    else: addDecl(c, param)
 
 template shouldHaveMeta(t) =
   internalAssert c.config, tfHasMeta in t.flags

compiler/vm.nim

@@ -413,26 +413,12 @@ proc recSetFlagIsRef(arg: PNode) =
     arg.sons[i].recSetFlagIsRef
 
 proc setLenSeq(c: PCtx; node: PNode; newLen: int; info: TLineInfo) =
-  # FIXME: this doesn't attempt to solve incomplete
-  # support of tyPtr, tyRef in VM.
   let typ = node.typ.skipTypes(abstractInst+{tyRange}-{tyTypeDesc})
-  let typeEntry = typ.sons[0].skipTypes(abstractInst+{tyRange}-{tyTypeDesc})
-  let typeKind = case typeEntry.kind
-                 of tyUInt..tyUInt64: nkUIntLit
-                 of tyRange, tyEnum, tyBool, tyChar, tyInt..tyInt64: nkIntLit
-                 of tyFloat..tyFloat128: nkFloatLit
-                 of tyString: nkStrLit
-                 of tyObject: nkObjConstr
-                 of tySequence: nkNilLit
-                 of tyProc, tyTuple: nkTupleConstr
-                 else: nkEmpty
-
   let oldLen = node.len
   setLen(node.sons, newLen)
   if oldLen < newLen:
-    # TODO: This is still not correct for tyPtr, tyRef default value
     for i in oldLen ..< newLen:
-      node.sons[i] = newNodeI(typeKind, info)
+      node.sons[i] = getNullValue(typ.sons[0], info, c.config)
 
 const
   errIndexOutOfBounds = "index out of bounds"

doc/advopt.txt

@@ -13,6 +13,7 @@ Advanced commands:
   //genDepend               generate a DOT file containing the
                           module dependency graph
   //dump                    dump all defined conditionals and search paths
+                            see also: --dump.format:json (useful with: ` | jq`)
   //check                   checks the project for syntax and semantic
 
 Advanced options:
@@ -103,6 +104,7 @@ Advanced options:
                             value = number of processors (0 for auto-detect)
   --incremental:on|off      only recompile the changed modules (experimental!)
   --verbosity:0|1|2|3       set Nim's verbosity level (1 is default)
+  --errorMax:N              stop compilation after N errors; 0 means unlimited
   --experimental:$1
                             enable experimental language feature
   -v, --version             show detailed version information

doc/contributing.rst

@@ -263,17 +263,8 @@ Take advantage of no implicit bool conversion
   doAssert isValid() == true
   doAssert isValid() # preferred
 
-.. _immediately_invoked_lambdas:
-Immediately invoked lambdas (https://en.wikipedia.org/wiki/Immediately-invoked_function_expression)
-
-.. code-block:: nim
-
-  let a = (proc (): auto = getFoo())()
-  let a = block:  # preferred
-    getFoo()
-
 .. _design_for_mcs:
-Design with method call syntax (UFCS in other languages) chaining in mind
+Design with method call syntax chaining in mind
 
 .. code-block:: nim
 

doc/lib.rst

@@ -154,6 +154,10 @@ String handling
 * `std/wordwrap <wordwrap.html>`_
   This module contains an algorithm to wordwrap a Unicode string.
 
+* `std/diff <diff.html>`_
+  This module contains an algorithm to compute the famous "diff"
+  of two texts by line.
+
 
 Generic Operating System Services
 ---------------------------------

koch.nim

@@ -79,6 +79,7 @@ template withDir(dir, body) =
   finally:
     setCurrentdir(old)
 
+let origDir = getCurrentDir()
 setCurrentDir(getAppDir())
 
 proc tryExec(cmd: string): bool =
@@ -378,9 +379,7 @@ proc winRelease*() =
 template `|`(a, b): string = (if a.len > 0: a else: b)
 
 proc tests(args: string) =
-  # we compile the tester with taintMode:on to have a basic
+  nimexec "cc --opt:speed testament/tester"
-  # taint mode test :-)
-  nimexec "cc --taintMode:on --opt:speed testament/tester"
   # Since tests take a long time (on my machine), and we want to defy Murhpys
   # law - lets make sure the compiler really is freshly compiled!
   nimexec "c --lib:lib -d:release --opt:speed compiler/nim.nim"
@@ -415,6 +414,7 @@ proc temp(args: string) =
   let nimexec = findNim()
   exec(nimexec & " c -d:debug --debugger:native " & bootArgs & " " & (d / "compiler" / "nim"), 125)
   copyExe(output, finalDest)
+  setCurrentDir(origDir)
   if programArgs.len > 0: exec(finalDest & " " & programArgs)
 
 proc xtemp(cmd: string) =

lib/impure/nre.nim

@@ -65,7 +65,7 @@ runnableExamples:
   let hasVowel = firstVowel.isSome()
   if hasVowel:
     let matchBounds = firstVowel.get().captureBounds[-1]
-    doAssert matchBounds.get().a == 1
+    doAssert matchBounds.a == 1
 
 
 # Type definitions {{{
@@ -167,14 +167,15 @@ type
     ##     -  ``"abc".match(re"(?<letter>\w)").get.captures["letter"] == "a"``
     ##     -  ``"abc".match(re"(\w)\w").get.captures[-1] == "ab"``
     ##
-    ## ``captureBounds[]: Option[HSlice[int, int]]``
+    ## ``captureBounds[]: HSlice[int, int]``
     ##     gets the bounds of the given capture according to the same rules as
     ##     the above. If the capture is not filled, then ``None`` is returned.
     ##     The bounds are both inclusive.
     ##
-    ##     -  ``"abc".match(re"(\w)").get.captureBounds[0].get == 0 .. 0``
+    ##     -  ``"abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0``
-    ##     -  ``"abc".match(re"").get.captureBounds[-1].get == 0 .. -1``
+    ##     -  ``0 in "abc".match(re"(\w)").get.captureBounds == true``
-    ##     -  ``"abc".match(re"abc").get.captureBounds[-1].get == 0 .. 2``
+    ##     -  ``"abc".match(re"").get.captureBounds[-1] == 0 .. -1``
+    ##     -  ``"abc".match(re"abc").get.captureBounds[-1] == 0 .. 2``
     ##
     ## ``match: string``
     ##     the full text of the match.
@@ -227,9 +228,10 @@ runnableExamples:
     doAssert "abc".match(re"(?<letter>\w)").get.captures["letter"] == "a"
     doAssert "abc".match(re"(\w)\w").get.captures[-1] == "ab"
 
-    doAssert "abc".match(re"(\w)").get.captureBounds[0].get == 0 .. 0
+    doAssert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0
-    doAssert "abc".match(re"").get.captureBounds[-1].get == 0 .. -1
+    doAssert 0 in "abc".match(re"(\w)").get.captureBounds == true
-    doAssert "abc".match(re"abc").get.captureBounds[-1].get == 0 .. 2
+    doAssert "abc".match(re"").get.captureBounds[-1] == 0 .. -1
+    doAssert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2
 # }}}
 
 proc getinfo[T](pattern: Regex, opt: cint): T =
@@ -269,78 +271,99 @@ proc matchesCrLf(pattern: Regex): bool =
 # }}}
 
 # Capture accessors {{{
-proc captureBounds*(pattern: RegexMatch): CaptureBounds = return CaptureBounds(pattern)
+func captureBounds*(pattern: RegexMatch): CaptureBounds = return CaptureBounds(pattern)
 
-proc captures*(pattern: RegexMatch): Captures = return Captures(pattern)
+func captures*(pattern: RegexMatch): Captures = return Captures(pattern)
 
-proc `[]`*(pattern: CaptureBounds, i: int): Option[HSlice[int, int]] =
+func contains*(pattern: CaptureBounds, i: int): bool =
   let pattern = RegexMatch(pattern)
-  if pattern.pcreMatchBounds[i + 1].a != -1:
+  pattern.pcreMatchBounds[i + 1].a != -1
-    let bounds = pattern.pcreMatchBounds[i + 1]
-    return some(int(bounds.a) .. int(bounds.b-1))
-  else:
-    return none(HSlice[int, int])
 
-proc `[]`*(pattern: Captures, i: int): string =
+func contains*(pattern: Captures, i: int): bool =
+  i in CaptureBounds(pattern)
+
+func `[]`*(pattern: CaptureBounds, i: int): HSlice[int, int] =
+  let pattern = RegexMatch(pattern)
+  if not (i in pattern.captureBounds):
+    raise newException(IndexError, "Group '" & $i & "' was not captured")
+
+  let bounds = pattern.pcreMatchBounds[i + 1]
+  int(bounds.a)..int(bounds.b-1)
+
+func `[]`*(pattern: Captures, i: int): string =
   let pattern = RegexMatch(pattern)
   let bounds = pattern.captureBounds[i]
 
-  if bounds.isSome:
+  pattern.str.substr(bounds.a, bounds.b)
-    let bounds = bounds.get
-    return pattern.str.substr(bounds.a, bounds.b)
-  else:
-    return ""
 
-proc match*(pattern: RegexMatch): string =
+func match*(pattern: RegexMatch): string =
   return pattern.captures[-1]
 
-proc matchBounds*(pattern: RegexMatch): HSlice[int, int] =
+func matchBounds*(pattern: RegexMatch): HSlice[int, int] =
-  return pattern.captureBounds[-1].get
+  return pattern.captureBounds[-1]
 
-proc `[]`*(pattern: CaptureBounds, name: string): Option[HSlice[int, int]] =
+func contains*(pattern: CaptureBounds, name: string): bool =
   let pattern = RegexMatch(pattern)
-  return pattern.captureBounds[pattern.pattern.captureNameToId.fget(name)]
+  let nameToId = pattern.pattern.captureNameToId
+  if not (name in nameToId):
+      return false
+  nameToId[name] in pattern.captureBounds
 
-proc `[]`*(pattern: Captures, name: string): string =
+func contains*(pattern: Captures, name: string): bool =
+  name in CaptureBounds(pattern)
+
+func checkNamedCaptured(pattern: RegexMatch, name: string): void =
+  if not (name in pattern.captureBounds):
+    raise newException(KeyError, "Group '" & name & "' was not captured")
+
+func `[]`*(pattern: CaptureBounds, name: string): HSlice[int, int] =
   let pattern = RegexMatch(pattern)
-  return pattern.captures[pattern.pattern.captureNameToId.fget(name)]
+  checkNamedCaptured(pattern, name)
+  pattern.captureBounds[pattern.pattern.captureNameToId[name]]
 
-template toTableImpl(cond: untyped) {.dirty.} =
+func `[]`*(pattern: Captures, name: string): string =
+  let pattern = RegexMatch(pattern)
+  checkNamedCaptured(pattern, name)
+  return pattern.captures[pattern.pattern.captureNameToId[name]]
+
+template toTableImpl() {.dirty.} =
   for key in RegexMatch(pattern).pattern.captureNameId.keys:
-    let nextVal = pattern[key]
+    if key in pattern:
-    if cond:
+        result[key] = pattern[key]
-      result[key] = default
-    else:
-      result[key] = nextVal
 
-proc toTable*(pattern: Captures, default: string = ""): Table[string, string] =
+func toTable*(pattern: Captures): Table[string, string] =
   result = initTable[string, string]()
-  toTableImpl(nextVal.len == 0)
+  toTableImpl()
 
-proc toTable*(pattern: CaptureBounds, default = none(HSlice[int, int])):
+func toTable*(pattern: CaptureBounds): Table[string, HSlice[int, int]] =
-    Table[string, Option[HSlice[int, int]]] =
+  result = initTable[string, HSlice[int, int]]()
-  result = initTable[string, Option[HSlice[int, int]]]()
+  toTableImpl()
-  toTableImpl(nextVal.isNone)
 
-template itemsImpl(cond: untyped) {.dirty.} =
+template itemsImpl() {.dirty.} =
   for i in 0 ..< RegexMatch(pattern).pattern.captureCount:
-    let nextVal = pattern[i]
     # done in this roundabout way to avoid multiple yields (potential code
     # bloat)
-    let nextYieldVal = if cond: default else: nextVal
+    let nextYieldVal = if i in pattern:
+      some(pattern[i])
+    else:
+      default
+
     yield nextYieldVal
 
+iterator items*(pattern: CaptureBounds,
+                default = none(HSlice[int, int])): Option[HSlice[int, int]] =
+  itemsImpl()
 
-iterator items*(pattern: CaptureBounds, default = none(HSlice[int, int])): Option[HSlice[int, int]] =
+iterator items*(pattern: Captures,
-  itemsImpl(nextVal.isNone)
+                default: Option[string] = none(string)): Option[string] =
+  itemsImpl()
 
-iterator items*(pattern: Captures, default: string = ""): string =
+proc toSeq*(pattern: CaptureBounds,
-  itemsImpl(nextVal.len == 0)
+            default = none(HSlice[int, int])): seq[Option[HSlice[int, int]]] =
-
-proc toSeq*(pattern: CaptureBounds, default = none(HSlice[int, int])): seq[Option[HSlice[int, int]]] =
   accumulateResult(pattern.items(default))
 
-proc toSeq*(pattern: Captures, default: string = ""): seq[string] =
+proc toSeq*(pattern: Captures,
+            default: Option[string] = none(string)): seq[Option[string]] =
   accumulateResult(pattern.items(default))
 
 proc `$`*(pattern: RegexMatch): string =
@@ -652,7 +675,8 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string]
 
     for cap in match.captures:
       # if there are captures, include them in the result
-      result.add(cap)
+      if cap.isSome:
+        result.add(cap.get)
 
     if splits == maxSplit - 1:
       break
@@ -706,7 +730,8 @@ proc replace*(str: string, pattern: Regex,
   ## -  ``$#`` - first capture
   ## -  ``$0`` - full match
   ##
-  ## If a given capture is missing, a ``ValueError`` exception is thrown.
+  ## If a given capture is missing, ``IndexError`` thrown for un-named captures
+  ## and ``KeyError`` for named captures.
   replaceImpl(str, pattern, subproc(match))
 
 proc replace*(str: string, pattern: Regex,

lib/impure/nre/private/util.nim

@@ -1,17 +1,9 @@
 ## INTERNAL FILE FOR USE ONLY BY nre.nim.
 import tables
 
-proc fget*[K, V](self: Table[K, V], key: K): V =
-  if self.hasKey(key):
-    return self[key]
-  else:
-    raise newException(KeyError, "Key does not exist in table: " & $key)
-
 const Ident = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'}
 const StartIdent = Ident - {'0'..'9'}
 
-template checkNil(arg: string): string = arg
-
 template formatStr*(howExpr, namegetter, idgetter): untyped =
   let how = howExpr
   var val = newStringOfCap(how.len)
@@ -28,7 +20,7 @@ template formatStr*(howExpr, namegetter, idgetter): untyped =
         i += 2
       elif how[i + 1] == '#':
         var id {.inject.} = lastNum
-        val.add(checkNil(idgetter))
+        val.add(idgetter)
         lastNum += 1
         i += 2
       elif how[i + 1] in {'0'..'9'}:
@@ -37,7 +29,7 @@ template formatStr*(howExpr, namegetter, idgetter): untyped =
         while i < how.len and how[i] in {'0'..'9'}:
           id += (id * 10) + (ord(how[i]) - ord('0'))
           i += 1
-        val.add(checkNil(idgetter))
+        val.add(idgetter)
         lastNum = id + 1
       elif how[i + 1] in StartIdent:
         i += 1
@@ -45,7 +37,7 @@ template formatStr*(howExpr, namegetter, idgetter): untyped =
         while i < how.len and how[i] in Ident:
           name.add(how[i])
           i += 1
-        val.add(checkNil(namegetter))
+        val.add(namegetter)
       elif how[i + 1] == '{':
         i += 2
         var name {.inject.} = ""
@@ -53,7 +45,7 @@ template formatStr*(howExpr, namegetter, idgetter): untyped =
           name.add(how[i])
           i += 1
         i += 1
-        val.add(checkNil(namegetter))
+        val.add(namegetter)
       else:
         raise newException(Exception, "Syntax error in format string at " & $i)
   val

lib/pure/asyncdispatch.nim

@@ -1513,8 +1513,217 @@ proc poll*(timeout = 500) =
   ## `epoll`:idx: or `kqueue`:idx: primitive only once.
   discard runOnce(timeout)
 
-# Common procedures between current and upcoming asyncdispatch
+template createAsyncNativeSocketImpl(domain, sockType, protocol) =
-include includes/asynccommon
+  let handle = newNativeSocket(domain, sockType, protocol)
+  if handle == osInvalidSocket:
+    return osInvalidSocket.AsyncFD
+  handle.setBlocking(false)
+  when defined(macosx) and not defined(nimdoc):
+    handle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1)
+  result = handle.AsyncFD
+  register(result)
+
+proc createAsyncNativeSocket*(domain: cint, sockType: cint,
+                           protocol: cint): AsyncFD =
+  createAsyncNativeSocketImpl(domain, sockType, protocol)
+
+proc createAsyncNativeSocket*(domain: Domain = Domain.AF_INET,
+                           sockType: SockType = SOCK_STREAM,
+                           protocol: Protocol = IPPROTO_TCP): AsyncFD =
+  createAsyncNativeSocketImpl(domain, sockType, protocol)
+
+proc newAsyncNativeSocket*(domain: cint, sockType: cint,
+                           protocol: cint): AsyncFD {.deprecated: "use createAsyncNativeSocket instead".} =
+  createAsyncNativeSocketImpl(domain, sockType, protocol)
+
+proc newAsyncNativeSocket*(domain: Domain = Domain.AF_INET,
+                           sockType: SockType = SOCK_STREAM,
+                           protocol: Protocol = IPPROTO_TCP): AsyncFD
+                           {.deprecated: "use createAsyncNativeSocket instead".} =
+  createAsyncNativeSocketImpl(domain, sockType, protocol)
+
+when defined(windows) or defined(nimdoc):
+  proc bindToDomain(handle: SocketHandle, domain: Domain) =
+    # Extracted into a separate proc, because connect() on Windows requires
+    # the socket to be initially bound.
+    template doBind(saddr) =
+      if bindAddr(handle, cast[ptr SockAddr](addr(saddr)),
+                  sizeof(saddr).SockLen) < 0'i32:
+        raiseOSError(osLastError())
+
+    if domain == Domain.AF_INET6:
+      var saddr: Sockaddr_in6
+      saddr.sin6_family = uint16(toInt(domain))
+      doBind(saddr)
+    else:
+      var saddr: Sockaddr_in
+      saddr.sin_family = uint16(toInt(domain))
+      doBind(saddr)
+
+  proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): Future[void] =
+    let retFuture = newFuture[void]("doConnect")
+    result = retFuture
+
+    var ol = PCustomOverlapped()
+    GC_ref(ol)
+    ol.data = CompletionData(fd: socket, cb:
+      proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) =
+        if not retFuture.finished:
+          if errcode == OSErrorCode(-1):
+            retFuture.complete()
+          else:
+            retFuture.fail(newException(OSError, osErrorMsg(errcode)))
+    )
+
+    let ret = connectEx(socket.SocketHandle, addrInfo.ai_addr,
+                        cint(addrInfo.ai_addrlen), nil, 0, nil,
+                        cast[POVERLAPPED](ol))
+    if ret:
+      # Request to connect completed immediately.
+      retFuture.complete()
+      # We don't deallocate ``ol`` here because even though this completed
+      # immediately poll will still be notified about its completion and it
+      # will free ``ol``.
+    else:
+      let lastError = osLastError()
+      if lastError.int32 != ERROR_IO_PENDING:
+        # With ERROR_IO_PENDING ``ol`` will be deallocated in ``poll``,
+        # and the future will be completed/failed there, too.
+        GC_unref(ol)
+        retFuture.fail(newException(OSError, osErrorMsg(lastError)))
+else:
+  proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): Future[void] =
+    let retFuture = newFuture[void]("doConnect")
+    result = retFuture
+
+    proc cb(fd: AsyncFD): bool =
+      let ret = SocketHandle(fd).getSockOptInt(
+        cint(SOL_SOCKET), cint(SO_ERROR))
+      if ret == 0:
+        # We have connected.
+        retFuture.complete()
+        return true
+      elif ret == EINTR:
+        # interrupted, keep waiting
+        return false
+      else:
+        retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret))))
+        return true
+
+    let ret = connect(socket.SocketHandle,
+                      addrInfo.ai_addr,
+                      addrInfo.ai_addrlen.Socklen)
+    if ret == 0:
+      # Request to connect completed immediately.
+      retFuture.complete()
+    else:
+      let lastError = osLastError()
+      if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS:
+        addWrite(socket, cb)
+      else:
+        retFuture.fail(newException(OSError, osErrorMsg(lastError)))
+
+template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped,
+                           protocol: Protocol = IPPROTO_RAW) =
+  ## Iterates through the AddrInfo linked list asynchronously
+  ## until the connection can be established.
+  const shouldCreateFd = not declared(fd)
+
+  when shouldCreateFd:
+    let sockType = protocol.toSockType()
+
+    var fdPerDomain: array[low(Domain).ord..high(Domain).ord, AsyncFD]
+    for i in low(fdPerDomain)..high(fdPerDomain):
+      fdPerDomain[i] = osInvalidSocket.AsyncFD
+    template closeUnusedFds(domainToKeep = -1) {.dirty.} =
+      for i, fd in fdPerDomain:
+        if fd != osInvalidSocket.AsyncFD and i != domainToKeep:
+          fd.closeSocket()
+
+  var lastException: ref Exception
+  var curAddrInfo = addrInfo
+  var domain: Domain
+  when shouldCreateFd:
+    var curFd: AsyncFD
+  else:
+    var curFd = fd
+  proc tryNextAddrInfo(fut: Future[void]) {.gcsafe.} =
+    if fut == nil or fut.failed:
+      if fut != nil:
+        lastException = fut.readError()
+
+      while curAddrInfo != nil:
+        let domainOpt = curAddrInfo.ai_family.toKnownDomain()
+        if domainOpt.isSome:
+          domain = domainOpt.unsafeGet()
+          break
+        curAddrInfo = curAddrInfo.ai_next
+
+      if curAddrInfo == nil:
+        freeAddrInfo(addrInfo)
+        when shouldCreateFd:
+          closeUnusedFds()
+        if lastException != nil:
+          retFuture.fail(lastException)
+        else:
+          retFuture.fail(newException(
+            IOError, "Couldn't resolve address: " & address))
+        return
+
+      when shouldCreateFd:
+        curFd = fdPerDomain[ord(domain)]
+        if curFd == osInvalidSocket.AsyncFD:
+          try:
+            curFd = newAsyncNativeSocket(domain, sockType, protocol)
+          except:
+            freeAddrInfo(addrInfo)
+            closeUnusedFds()
+            raise getCurrentException()
+          when defined(windows):
+            curFd.SocketHandle.bindToDomain(domain)
+          fdPerDomain[ord(domain)] = curFd
+
+      doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo
+      curAddrInfo = curAddrInfo.ai_next
+    else:
+      freeAddrInfo(addrInfo)
+      when shouldCreateFd:
+        closeUnusedFds(ord(domain))
+        retFuture.complete(curFd)
+      else:
+        retFuture.complete()
+
+  tryNextAddrInfo(nil)
+
+proc dial*(address: string, port: Port,
+           protocol: Protocol = IPPROTO_TCP): Future[AsyncFD] =
+  ## Establishes connection to the specified ``address``:``port`` pair via the
+  ## specified protocol. The procedure iterates through possible
+  ## resolutions of the ``address`` until it succeeds, meaning that it
+  ## seamlessly works with both IPv4 and IPv6.
+  ## Returns the async file descriptor, registered in the dispatcher of
+  ## the current thread, ready to send or receive data.
+  let retFuture = newFuture[AsyncFD]("dial")
+  result = retFuture
+  let sockType = protocol.toSockType()
+
+  let aiList = getAddrInfo(address, port, Domain.AF_UNSPEC, sockType, protocol)
+  asyncAddrInfoLoop(aiList, noFD, protocol)
+
+proc connect*(socket: AsyncFD, address: string, port: Port,
+              domain = Domain.AF_INET): Future[void] =
+  let retFuture = newFuture[void]("connect")
+  result = retFuture
+
+  when defined(windows):
+    verifyPresence(socket)
+  else:
+    assert getSockDomain(socket.SocketHandle) == domain
+
+  let aiList = getAddrInfo(address, port, domain)
+  when defined(windows):
+    socket.SocketHandle.bindToDomain(domain)
+  asyncAddrInfoLoop(aiList, socket)
 
 proc sleepAsync*(ms: int | float): Future[void] =
   ## Suspends the execution of the current async procedure for the next

lib/pure/collections/lists.nim

@@ -140,11 +140,26 @@ proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} =
   ## exist, true otherwise.
   result = find(L, value) != nil
 
+proc append*[T](L: var SinglyLinkedList[T],
+                n: SinglyLinkedNode[T]) {.inline.} =
+  ## appends a node `n` to `L`. Efficiency: O(1).
+  n.next = nil
+  if L.tail != nil:
+    assert(L.tail.next == nil)
+    L.tail.next = n
+  L.tail = n
+  if L.head == nil: L.head = n
+
+proc append*[T](L: var SinglyLinkedList[T], value: T) {.inline.} =
+  ## appends a value to `L`. Efficiency: O(1).
+  append(L, newSinglyLinkedNode(value))
+
 proc prepend*[T](L: var SinglyLinkedList[T],
                  n: SinglyLinkedNode[T]) {.inline.} =
   ## prepends a node to `L`. Efficiency: O(1).
   n.next = L.head
   L.head = n
+  if L.tail == nil: L.tail = n
 
 proc prepend*[T](L: var SinglyLinkedList[T], value: T) {.inline.} =
   ## prepends a node to `L`. Efficiency: O(1).

lib/pure/collections/sharedstrings.nim

@@ -12,6 +12,8 @@
 type
   UncheckedCharArray = UncheckedArray[char]
 
+import system/helpers2
+
 type
   Buffer = ptr object
     refcount: int
@@ -49,11 +51,11 @@ proc len*(s: SharedString): int = s.len
 
 proc `[]`*(s: SharedString; i: Natural): char =
   if i < s.len: result = s.buffer.data[i+s.first]
-  else: raise newException(IndexError, "index out of bounds")
+  else: raise newException(IndexError, formatErrorIndexBound(i, s.len-1))
 
 proc `[]=`*(s: var SharedString; i: Natural; value: char) =
   if i < s.len: s.buffer.data[i+s.first] = value
-  else: raise newException(IndexError, "index out of bounds")
+  else: raise newException(IndexError, formatErrorIndexBound(i, s.len-1))
 
 proc `[]`*(s: SharedString; ab: HSlice[int, int]): SharedString =
   #incRef(src.buffer)

lib/pure/includes/asynccommon.nim

@@ -1,211 +0,0 @@
-template createAsyncNativeSocketImpl(domain, sockType, protocol) =
-  let handle = newNativeSocket(domain, sockType, protocol)
-  if handle == osInvalidSocket:
-    return osInvalidSocket.AsyncFD
-  handle.setBlocking(false)
-  when defined(macosx) and not defined(nimdoc):
-    handle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1)
-  result = handle.AsyncFD
-  register(result)
-
-proc createAsyncNativeSocket*(domain: cint, sockType: cint,
-                           protocol: cint): AsyncFD =
-  createAsyncNativeSocketImpl(domain, sockType, protocol)
-
-proc createAsyncNativeSocket*(domain: Domain = Domain.AF_INET,
-                           sockType: SockType = SOCK_STREAM,
-                           protocol: Protocol = IPPROTO_TCP): AsyncFD =
-  createAsyncNativeSocketImpl(domain, sockType, protocol)
-
-proc newAsyncNativeSocket*(domain: cint, sockType: cint,
-                           protocol: cint): AsyncFD {.deprecated: "use createAsyncNativeSocket instead".} =
-  createAsyncNativeSocketImpl(domain, sockType, protocol)
-
-proc newAsyncNativeSocket*(domain: Domain = Domain.AF_INET,
-                           sockType: SockType = SOCK_STREAM,
-                           protocol: Protocol = IPPROTO_TCP): AsyncFD
-                           {.deprecated: "use createAsyncNativeSocket instead".} =
-  createAsyncNativeSocketImpl(domain, sockType, protocol)
-
-when defined(windows) or defined(nimdoc):
-  proc bindToDomain(handle: SocketHandle, domain: Domain) =
-    # Extracted into a separate proc, because connect() on Windows requires
-    # the socket to be initially bound.
-    template doBind(saddr) =
-      if bindAddr(handle, cast[ptr SockAddr](addr(saddr)),
-                  sizeof(saddr).SockLen) < 0'i32:
-        raiseOSError(osLastError())
-
-    if domain == Domain.AF_INET6:
-      var saddr: Sockaddr_in6
-      saddr.sin6_family = uint16(toInt(domain))
-      doBind(saddr)
-    else:
-      var saddr: Sockaddr_in
-      saddr.sin_family = uint16(toInt(domain))
-      doBind(saddr)
-
-  proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): Future[void] =
-    let retFuture = newFuture[void]("doConnect")
-    result = retFuture
-
-    var ol = PCustomOverlapped()
-    GC_ref(ol)
-    ol.data = CompletionData(fd: socket, cb:
-      proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) =
-        if not retFuture.finished:
-          if errcode == OSErrorCode(-1):
-            retFuture.complete()
-          else:
-            retFuture.fail(newException(OSError, osErrorMsg(errcode)))
-    )
-
-    let ret = connectEx(socket.SocketHandle, addrInfo.ai_addr,
-                        cint(addrInfo.ai_addrlen), nil, 0, nil,
-                        cast[POVERLAPPED](ol))
-    if ret:
-      # Request to connect completed immediately.
-      retFuture.complete()
-      # We don't deallocate ``ol`` here because even though this completed
-      # immediately poll will still be notified about its completion and it
-      # will free ``ol``.
-    else:
-      let lastError = osLastError()
-      if lastError.int32 != ERROR_IO_PENDING:
-        # With ERROR_IO_PENDING ``ol`` will be deallocated in ``poll``,
-        # and the future will be completed/failed there, too.
-        GC_unref(ol)
-        retFuture.fail(newException(OSError, osErrorMsg(lastError)))
-else:
-  proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): Future[void] =
-    let retFuture = newFuture[void]("doConnect")
-    result = retFuture
-
-    proc cb(fd: AsyncFD): bool =
-      let ret = SocketHandle(fd).getSockOptInt(
-        cint(SOL_SOCKET), cint(SO_ERROR))
-      if ret == 0:
-        # We have connected.
-        retFuture.complete()
-        return true
-      elif ret == EINTR:
-        # interrupted, keep waiting
-        return false
-      else:
-        retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret))))
-        return true
-
-    let ret = connect(socket.SocketHandle,
-                      addrInfo.ai_addr,
-                      addrInfo.ai_addrlen.Socklen)
-    if ret == 0:
-      # Request to connect completed immediately.
-      retFuture.complete()
-    else:
-      let lastError = osLastError()
-      if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS:
-        addWrite(socket, cb)
-      else:
-        retFuture.fail(newException(OSError, osErrorMsg(lastError)))
-
-template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped,
-                           protocol: Protocol = IPPROTO_RAW) =
-  ## Iterates through the AddrInfo linked list asynchronously
-  ## until the connection can be established.
-  const shouldCreateFd = not declared(fd)
-
-  when shouldCreateFd:
-    let sockType = protocol.toSockType()
-
-    var fdPerDomain: array[low(Domain).ord..high(Domain).ord, AsyncFD]
-    for i in low(fdPerDomain)..high(fdPerDomain):
-      fdPerDomain[i] = osInvalidSocket.AsyncFD
-    template closeUnusedFds(domainToKeep = -1) {.dirty.} =
-      for i, fd in fdPerDomain:
-        if fd != osInvalidSocket.AsyncFD and i != domainToKeep:
-          fd.closeSocket()
-
-  var lastException: ref Exception
-  var curAddrInfo = addrInfo
-  var domain: Domain
-  when shouldCreateFd:
-    var curFd: AsyncFD
-  else:
-    var curFd = fd
-  proc tryNextAddrInfo(fut: Future[void]) {.gcsafe.} =
-    if fut == nil or fut.failed:
-      if fut != nil:
-        lastException = fut.readError()
-
-      while curAddrInfo != nil:
-        let domainOpt = curAddrInfo.ai_family.toKnownDomain()
-        if domainOpt.isSome:
-          domain = domainOpt.unsafeGet()
-          break
-        curAddrInfo = curAddrInfo.ai_next
-
-      if curAddrInfo == nil:
-        freeAddrInfo(addrInfo)
-        when shouldCreateFd:
-          closeUnusedFds()
-        if lastException != nil:
-          retFuture.fail(lastException)
-        else:
-          retFuture.fail(newException(
-            IOError, "Couldn't resolve address: " & address))
-        return
-
-      when shouldCreateFd:
-        curFd = fdPerDomain[ord(domain)]
-        if curFd == osInvalidSocket.AsyncFD:
-          try:
-            curFd = newAsyncNativeSocket(domain, sockType, protocol)
-          except:
-            freeAddrInfo(addrInfo)
-            closeUnusedFds()
-            raise getCurrentException()
-          when defined(windows):
-            curFd.SocketHandle.bindToDomain(domain)
-          fdPerDomain[ord(domain)] = curFd
-
-      doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo
-      curAddrInfo = curAddrInfo.ai_next
-    else:
-      freeAddrInfo(addrInfo)
-      when shouldCreateFd:
-        closeUnusedFds(ord(domain))
-        retFuture.complete(curFd)
-      else:
-        retFuture.complete()
-
-  tryNextAddrInfo(nil)
-
-proc dial*(address: string, port: Port,
-           protocol: Protocol = IPPROTO_TCP): Future[AsyncFD] =
-  ## Establishes connection to the specified ``address``:``port`` pair via the
-  ## specified protocol. The procedure iterates through possible
-  ## resolutions of the ``address`` until it succeeds, meaning that it
-  ## seamlessly works with both IPv4 and IPv6.
-  ## Returns the async file descriptor, registered in the dispatcher of
-  ## the current thread, ready to send or receive data.
-  let retFuture = newFuture[AsyncFD]("dial")
-  result = retFuture
-  let sockType = protocol.toSockType()
-
-  let aiList = getAddrInfo(address, port, Domain.AF_UNSPEC, sockType, protocol)
-  asyncAddrInfoLoop(aiList, noFD, protocol)
-
-proc connect*(socket: AsyncFD, address: string, port: Port,
-              domain = Domain.AF_INET): Future[void] =
-  let retFuture = newFuture[void]("connect")
-  result = retFuture
-
-  when defined(windows):
-    verifyPresence(socket)
-  else:
-    assert getSockDomain(socket.SocketHandle) == domain
-
-  let aiList = getAddrInfo(address, port, domain)
-  when defined(windows):
-    socket.SocketHandle.bindToDomain(domain)
-  asyncAddrInfoLoop(aiList, socket)

lib/pure/includes/oserr.nim

@@ -59,7 +59,8 @@ proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} =
   e.errorCode = errorCode.int32
   e.msg = osErrorMsg(errorCode)
   if additionalInfo.len > 0:
-    e.msg.add  "; Additional info: "
+    if e.msg[^1] != '\n': e.msg.add '\n'
+    e.msg.add  "Additional info: "
     e.msg.addQuoted additionalInfo
   if e.msg == "":
     e.msg = "unknown OS error"

lib/pure/memfiles.nim

@@ -36,11 +36,12 @@ type
     size*: int       ## size of the memory mapped file
 
     when defined(windows):
-      fHandle: Handle
+      fHandle*: Handle     ## **Caution**: Windows specific public field to allow
-      mapHandle: Handle
+                           ## even more low level trickery.
-      wasOpened: bool   ## only close if wasOpened
+      mapHandle*: Handle   ## **Caution**: Windows specific public field.
+      wasOpened*: bool     ## **Caution**: Windows specific public field.
     else:
-      handle: cint
+      handle*: cint        ## **Caution**: Posix specific public field.
 
 proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
              mappedSize = -1, offset = 0): pointer =
@@ -281,6 +282,35 @@ proc flush*(f: var MemFile; attempts: Natural = 3) =
       if lastErr != EBUSY.OSErrorCode:
         raiseOSError(lastErr, "error flushing mapping")
 
+when defined(posix) or defined(nimdoc):
+  proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} =
+    ## resize and re-map the file underlying an ``allowRemap MemFile``.
+    ## **Note**: this assumes the entire file is mapped read-write at offset zero.
+    ## Also, the value of ``.mem`` will probably change.
+    ## **Note**: This is not (yet) avaiable on Windows.
+    when defined(posix):
+      if f.handle == -1:
+        raise newException(IOError,
+                            "Cannot resize MemFile opened with allowRemap=false")
+      if ftruncate(f.handle, newFileSize) == -1:
+        raiseOSError(osLastError())
+      when defined(linux):                          #Maybe NetBSD, too?
+        #On Linux this can be over 100 times faster than a munmap,mmap cycle.
+        proc mremap(old: pointer; oldSize,newSize: csize; flags: cint): pointer {.
+          importc: "mremap", header: "<sys/mman.h>" .}
+        let newAddr = mremap(f.mem, csize(f.size), csize(newFileSize), cint(1))
+        if newAddr == cast[pointer](MAP_FAILED):
+          raiseOSError(osLastError())
+      else:
+        if munmap(f.mem, f.size) != 0:
+          raiseOSError(osLastError())
+        let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE,
+                            MAP_SHARED or MAP_POPULATE, f.handle, 0)
+        if newAddr == cast[pointer](MAP_FAILED):
+          raiseOSError(osLastError())
+      f.mem = newAddr
+      f.size = newFileSize
+
 proc close*(f: var MemFile) =
   ## closes the memory mapped file `f`. All changes are written back to the
   ## file system, if `f` was opened with write access.

lib/pure/options.nim

@@ -39,17 +39,18 @@
 ##
 ## .. code-block:: nim
 ##
-##   try:
+##    let found = "abc".find('c')
-##     assert("abc".find('c').get() == 2)  # Immediately extract the value
+##    assert found.isSome and found.get() == 2
-##   except UnpackError:  # If there is no value
-##     assert false  # This will not be reached, because the value is present
 ##   
 ## The ``get`` operation demonstrated above returns the underlying value, or
-## raises ``UnpackError`` if there is no value. There is another option for
+## raises ``UnpackError`` if there is no value. Note that ``UnpackError`` inherits
-## obtaining the value: ``unsafeGet``, but you must only use it when you are
+## from ``system.Defect``, and should therefore never be catched. Instead, rely on
-## absolutely sure the value is present (e.g. after checking ``isSome``). If
+## checking if the option contains a value with ``isSome`` and ``isNone``.
-## you do not care about the tiny overhead that ``get`` causes, you should
+##
-## simply never use ``unsafeGet``.
+## There is another option for obtaining the value: ``unsafeGet``, but you must
+## only use it when you are absolutely sure the value is present (e.g. after
+## checking ``isSome``). If you do not care about the tiny overhead that ``get``
+## causes, you should simply never use ``unsafeGet``.
 ##
 ## How to deal with an absence of a value:
 ##
@@ -61,12 +62,7 @@
 ##   assert(result == none(int))
 ##   # It has no value:
 ##   assert(result.isNone)
-##
+
-##   try:
-##     echo result.get()
-##     assert(false)  # This will not be reached
-##   except UnpackError:  # Because an exception is raised
-##     discard
 import typetraits
 
 type
@@ -81,7 +77,7 @@ type
       val: T
       has: bool
 
-  UnpackError* = ref object of ValueError
+  UnpackError* = object of Defect
 
 proc some*[T](val: T): Option[T] =
   ## Returns a ``Option`` that has this value.
@@ -129,7 +125,7 @@ proc get*[T](self: Option[T]): T =
   ## Returns contents of the Option. If it is none, then an exception is
   ## thrown.
   if self.isNone:
-    raise UnpackError(msg: "Can't obtain a value from a `none`")
+    raise newException(UnpackError, "Can't obtain a value from a `none`")
   self.val
 
 proc get*[T](self: Option[T], otherwise: T): T =
@@ -143,7 +139,7 @@ proc get*[T](self: var Option[T]): var T =
   ## Returns contents of the Option. If it is none, then an exception is
   ## thrown.
   if self.isNone:
-    raise UnpackError(msg: "Can't obtain a value from a `none`")
+    raise newException(UnpackError, "Can't obtain a value from a `none`")
   return self.val
 
 proc map*[T](self: Option[T], callback: proc (input: T)) =

lib/pure/subexes.nim

@@ -17,7 +17,7 @@
 
 from strutils import parseInt, cmpIgnoreStyle, Digits
 include "system/inclrtl"
-
+import system/helpers2
 
 proc findNormalized(x: string, inArray: openarray[string]): int =
   var i = 0
@@ -85,7 +85,7 @@ proc getFormatArg(p: var FormatParser, a: openArray[string]): int =
     result = parseInt(a[result])-1
   else:
     raiseInvalidFormat("'#', '$', number or identifier expected")
-  if result >=% a.len: raiseInvalidFormat("index out of bounds: " & $result)
+  if result >=% a.len: raiseInvalidFormat(formatErrorIndexBound(result, a.len))
   p.i = i
 
 proc scanDollar(p: var FormatParser, a: openarray[string], s: var string) {.

lib/std/diff.nim

@@ -0,0 +1,387 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2018 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements an algorithm to compute the
+## `diff`:idx: between two sequences of lines.
+
+# code owner: Arne Döring
+#
+# This is based on C# code written by Matthias Hertel, http://www.mathertel.de
+#
+# This Class implements the Difference Algorithm published in
+# "An O(ND) Difference Algorithm and its Variations" by Eugene Myers
+# Algorithmica Vol. 1 No. 2, 1986, p 251.
+
+import tables, strutils
+
+type
+  Item* = object    ## An Item in the list of differences.
+    startA*: int    ## Start Line number in Data A.
+    startB*: int    ## Start Line number in Data B.
+    deletedA*: int  ## Number of changes in Data A.
+    insertedB*: int ## Number of changes in Data B.
+
+  DiffData = object ## Data on one input file being compared.
+    data: seq[int] ## Buffer of numbers that will be compared.
+    modified: seq[bool] ## Array of booleans that flag for modified
+                        ## data. This is the result of the diff.
+                        ## This means deletedA in the first Data or
+                        ## inserted in the second Data.
+
+  Smsrd = object
+    x, y: int
+
+# template to avoid a seq copy. Required until ``sink`` parameters are ready.
+template newDiffData(initData: seq[int]; L: int): DiffData =
+  DiffData(
+    data: initData,
+    modified: newSeq[bool](L + 2)
+  )
+
+proc len(d: DiffData): int {.inline.} = d.data.len
+
+proc diffCodes(aText: string; h: var Table[string, int]): DiffData =
+  ## This function converts all textlines of the text into unique numbers for every unique textline
+  ## so further work can work only with simple numbers.
+  ## ``aText`` the input text
+  ## ``h`` This extern initialized hashtable is used for storing all ever used textlines.
+  ## ``trimSpace`` ignore leading and trailing space characters
+  ## Returns a array of integers.
+  var lastUsedCode = h.len
+  result.data = newSeq[int]()
+  for s in aText.splitLines:
+    if h.contains s:
+      result.data.add h[s]
+    else:
+      inc lastUsedCode
+      h[s] = lastUsedCode
+      result.data.add lastUsedCode
+  result.modified = newSeq[bool](result.data.len + 2)
+
+proc optimize(data: var DiffData) =
+  ## If a sequence of modified lines starts with a line that contains the same content
+  ## as the line that appends the changes, the difference sequence is modified so that the
+  ## appended line and not the starting line is marked as modified.
+  ## This leads to more readable diff sequences when comparing text files.
+  var startPos = 0
+  while startPos < data.len:
+    while startPos < data.len and not data.modified[startPos]:
+      inc startPos
+    var endPos = startPos
+    while endPos < data.len and data.modified[endPos]:
+      inc endPos
+
+    if endPos < data.len and data.data[startPos] == data.data[endPos]:
+      data.modified[startPos] = false
+      data.modified[endPos] = true
+    else:
+      startPos = endPos
+
+proc sms(dataA: var DiffData; lowerA, upperA: int; dataB: DiffData; lowerB, upperB: int;
+         downVector, upVector: var openArray[int]): Smsrd =
+  ## This is the algorithm to find the Shortest Middle Snake (sms).
+  ## ``dataA`` sequence A
+  ## ``lowerA`` lower bound of the actual range in dataA
+  ## ``upperA`` upper bound of the actual range in dataA (exclusive)
+  ## ``dataB`` sequence B
+  ## ``lowerB`` lower bound of the actual range in dataB
+  ## ``upperB`` upper bound of the actual range in dataB (exclusive)
+  ## ``downVector`` a vector for the (0,0) to (x,y) search. Passed as a parameter for speed reasons.
+  ## ``upVector`` a vector for the (u,v) to (N,M) search. Passed as a parameter for speed reasons.
+  ## Returns a MiddleSnakeData record containing x,y and u,v.
+
+  let max = dataA.len + dataB.len + 1
+
+  let downK = lowerA - lowerB # the k-line to start the forward search
+  let upK = upperA - upperB # the k-line to start the reverse search
+
+  let delta = (upperA - lowerA) - (upperB - lowerB)
+  let oddDelta = (delta and 1) != 0
+
+  # The vectors in the publication accepts negative indexes. the vectors implemented here are 0-based
+  # and are access using a specific offset: upOffset upVector and downOffset for downVector
+  let downOffset = max - downK
+  let upOffset = max - upK
+
+  let maxD = ((upperA - lowerA + upperB - lowerB) div 2) + 1
+
+  downVector[downOffset + downK + 1] = lowerA
+  upVector[upOffset + upK - 1] = upperA
+
+  for D in 0 .. maxD:
+    # Extend the forward path.
+    for k in countUp(downK - D, downK + D, 2):
+      # find the only or better starting point
+      var x: int
+      if k == downK - D:
+        x = downVector[downOffset + k + 1] # down
+      else:
+        x = downVector[downOffset + k - 1] + 1 # a step to the right
+        if k < downK + D and downVector[downOffset + k + 1] >= x:
+          x = downVector[downOffset + k + 1] # down
+
+      var y = x - k
+
+      # find the end of the furthest reaching forward D-path in diagonal k.
+      while x < upperA and y < upperB and dataA.data[x] == dataB.data[y]:
+        inc x
+        inc y
+
+      downVector[downOffset + k] = x
+
+      # overlap ?
+      if oddDelta and upK - D < k and k < upK + D:
+        if upVector[upOffset + k] <= downVector[downOffset + k]:
+          return Smsrd(x: downVector[downOffset + k],
+                       y: downVector[downOffset + k] - k)
+
+    # Extend the reverse path.
+    for k in countUp(upK - D, upK + D, 2):
+      # find the only or better starting point
+      var x: int
+      if k == upK + D:
+        x = upVector[upOffset + k - 1] # up
+      else:
+        x = upVector[upOffset + k + 1] - 1 # left
+        if k > upK - D and upVector[upOffset + k - 1] < x:
+          x = upVector[upOffset + k - 1] # up
+
+      var y = x - k
+      while x > lowerA and y > lowerB and dataA.data[x - 1] == dataB.data[y - 1]:
+        dec x
+        dec y
+
+      upVector[upOffset + k] = x
+
+      # overlap ?
+      if not oddDelta and downK-D <= k and k <= downK+D:
+        if upVector[upOffset + k] <= downVector[downOffset + k]:
+          return Smsrd(x: downVector[downOffset + k],
+                       y: downVector[downOffset + k] - k)
+
+  assert false, "the algorithm should never come here."
+
+proc lcs(dataA: var DiffData; lowerA, upperA: int; dataB: var DiffData; lowerB, upperB: int;
+         downVector, upVector: var openArray[int]) =
+  ## This is the divide-and-conquer implementation of the longes common-subsequence (lcs)
+  ## algorithm.
+  ## The published algorithm passes recursively parts of the A and B sequences.
+  ## To avoid copying these arrays the lower and upper bounds are passed while the sequences stay constant.
+  ## ``dataA`` sequence A
+  ## ``lowerA`` lower bound of the actual range in dataA
+  ## ``upperA`` upper bound of the actual range in dataA (exclusive)
+  ## ``dataB`` sequence B
+  ## ``lowerB`` lower bound of the actual range in dataB
+  ## ``upperB`` upper bound of the actual range in dataB (exclusive)
+  ## ``downVector`` a vector for the (0,0) to (x,y) search. Passed as a parameter for speed reasons.
+  ## ``upVector`` a vector for the (u,v) to (N,M) search. Passed as a parameter for speed reasons.
+
+  # make mutable copy:
+  var lowerA = lowerA
+  var lowerB = lowerB
+  var upperA = upperA
+  var upperB = upperB
+
+  # Fast walkthrough equal lines at the start
+  while lowerA < upperA and lowerB < upperB and dataA.data[lowerA] == dataB.data[lowerB]:
+    inc lowerA
+    inc lowerB
+
+  # Fast walkthrough equal lines at the end
+  while lowerA < upperA and lowerB < upperB and dataA.data[upperA - 1] == dataB.data[upperB - 1]:
+    dec upperA
+    dec upperB
+
+  if lowerA == upperA:
+    # mark as inserted lines.
+    while lowerB < upperB:
+      dataB.modified[lowerB] = true
+      inc lowerB
+
+  elif lowerB == upperB:
+    # mark as deleted lines.
+    while lowerA < upperA:
+      dataA.modified[lowerA] = true
+      inc lowerA
+
+  else:
+    # Find the middle snakea and length of an optimal path for A and B
+    let smsrd = sms(dataA, lowerA, upperA, dataB, lowerB, upperB, downVector, upVector)
+    # Debug.Write(2, "MiddleSnakeData", String.Format("{0},{1}", smsrd.x, smsrd.y))
+
+    # The path is from LowerX to (x,y) and (x,y) to UpperX
+    lcs(dataA, lowerA, smsrd.x, dataB, lowerB, smsrd.y, downVector, upVector)
+    lcs(dataA, smsrd.x, upperA, dataB, smsrd.y, upperB, downVector, upVector)  # 2002.09.20: no need for 2 points
+
+proc createDiffs(dataA, dataB: DiffData): seq[Item] =
+  ## Scan the tables of which lines are inserted and deleted,
+  ## producing an edit script in forward order.
+  var startA = 0
+  var startB = 0
+  var lineA = 0
+  var lineB = 0
+  while lineA < dataA.len or lineB < dataB.len:
+    if lineA < dataA.len and not dataA.modified[lineA] and
+       lineB < dataB.len and not dataB.modified[lineB]:
+      # equal lines
+      inc lineA
+      inc lineB
+    else:
+      # maybe deleted and/or inserted lines
+      startA = lineA
+      startB = lineB
+
+      while lineA < dataA.len and (lineB >= dataB.len or dataA.modified[lineA]):
+        inc lineA
+
+      while lineB < dataB.len and (lineA >= dataA.len or dataB.modified[lineB]):
+        inc lineB
+
+      if (startA < lineA) or (startB < lineB):
+        result.add Item(startA: startA,
+                        startB: startB,
+                        deletedA: lineA - startA,
+                        insertedB: lineB - startB)
+
+
+proc diffInt*(arrayA, arrayB: openArray[int]): seq[Item] =
+  ## Find the difference in 2 arrays of integers.
+  ## ``arrayA`` A-version of the numbers (usualy the old one)
+  ## ``arrayB`` B-version of the numbers (usualy the new one)
+  ## Returns a array of Items that describe the differences.
+
+  # The A-Version of the data (original data) to be compared.
+  var dataA = newDiffData(@arrayA, arrayA.len)
+
+  # The B-Version of the data (modified data) to be compared.
+  var dataB = newDiffData(@arrayB, arrayB.len)
+
+  let max = dataA.len + dataB.len + 1
+  ## vector for the (0,0) to (x,y) search
+  var downVector = newSeq[int](2 * max + 2)
+  ## vector for the (u,v) to (N,M) search
+  var upVector = newSeq[int](2 * max + 2)
+
+  lcs(dataA, 0, dataA.len, dataB, 0, dataB.len, downVector, upVector)
+  result = createDiffs(dataA, dataB)
+
+proc diffText*(textA, textB: string): seq[Item] =
+  ## Find the difference in 2 text documents, comparing by textlines.
+  ## The algorithm itself is comparing 2 arrays of numbers so when comparing 2 text documents
+  ## each line is converted into a (hash) number. This hash-value is computed by storing all
+  ## textlines into a common hashtable so i can find dublicates in there, and generating a
+  ## new number each time a new textline is inserted.
+  ## ``TextA`` A-version of the text (usualy the old one)
+  ## ``TextB`` B-version of the text (usualy the new one)
+  ## ``trimSpace`` When set to true, all leading and trailing whitespace characters are stripped out before the comparation is done.
+  ## ``ignoreSpace`` When set to true, all whitespace characters are converted to a single space character before the comparation is done.
+  ## ``ignoreCase`` When set to true, all characters are converted to their lowercase equivivalence before the comparation is done.
+  ## Returns a seq of Items that describe the differences.
+
+  # prepare the input-text and convert to comparable numbers.
+  var h = initTable[string, int]()  # TextA.len + TextB.len  <- probably wrong initial size
+  # The A-Version of the data (original data) to be compared.
+  var dataA = diffCodes(textA, h)
+
+  # The B-Version of the data (modified data) to be compared.
+  var dataB = diffCodes(textB, h)
+
+  h.clear # free up hashtable memory (maybe)
+
+  let max = dataA.len + dataB.len + 1
+  ## vector for the (0,0) to (x,y) search
+  var downVector = newSeq[int](2 * max + 2)
+  ## vector for the (u,v) to (N,M) search
+  var upVector = newSeq[int](2 * max + 2)
+
+  lcs(dataA, 0, dataA.len, dataB, 0, dataB.len, downVector, upVector)
+
+  optimize(dataA)
+  optimize(dataB)
+  result = createDiffs(dataA, dataB)
+
+when isMainModule:
+
+  proc testHelper(f: seq[Item]): string =
+    for it in f:
+      result.add(
+        $it.deletedA & "." & $it.insertedB & "." & $it.startA & "." & $it.startB & "*"
+      )
+
+  proc main() =
+    var a, b: string
+
+    stdout.writeLine("Diff Self Test...")
+
+    # test all changes
+    a = "a,b,c,d,e,f,g,h,i,j,k,l".replace(',', '\n')
+    b = "0,1,2,3,4,5,6,7,8,9".replace(',', '\n')
+    assert(testHelper(diffText(a, b)) ==
+      "12.10.0.0*",
+      "all-changes test failed.")
+    stdout.writeLine("all-changes test passed.")
+    # test all same
+    a = "a,b,c,d,e,f,g,h,i,j,k,l".replace(',', '\n')
+    b = a
+    assert(testHelper(diffText(a, b)) ==
+      "",
+      "all-same test failed.")
+    stdout.writeLine("all-same test passed.")
+
+    # test snake
+    a = "a,b,c,d,e,f".replace(',', '\n')
+    b = "b,c,d,e,f,x".replace(',', '\n')
+    assert(testHelper(diffText(a, b)) ==
+      "1.0.0.0*0.1.6.5*",
+      "snake test failed.")
+    stdout.writeLine("snake test passed.")
+
+    # 2002.09.20 - repro
+    a = "c1,a,c2,b,c,d,e,g,h,i,j,c3,k,l".replace(',', '\n')
+    b = "C1,a,C2,b,c,d,e,I1,e,g,h,i,j,C3,k,I2,l".replace(',', '\n')
+    assert(testHelper(diffText(a, b)) ==
+      "1.1.0.0*1.1.2.2*0.2.7.7*1.1.11.13*0.1.13.15*",
+      "repro20020920 test failed.")
+    stdout.writeLine("repro20020920 test passed.")
+
+    # 2003.02.07 - repro
+    a = "F".replace(',', '\n')
+    b = "0,F,1,2,3,4,5,6,7".replace(',', '\n')
+    assert(testHelper(diffText(a, b)) ==
+      "0.1.0.0*0.7.1.2*",
+      "repro20030207 test failed.")
+    stdout.writeLine("repro20030207 test passed.")
+
+    # Muegel - repro
+    a = "HELLO\nWORLD"
+    b = "\n\nhello\n\n\n\nworld\n"
+    assert(testHelper(diffText(a, b)) ==
+      "2.8.0.0*",
+      "repro20030409 test failed.")
+    stdout.writeLine("repro20030409 test passed.")
+
+    # test some differences
+    a = "a,b,-,c,d,e,f,f".replace(',', '\n')
+    b = "a,b,x,c,e,f".replace(',', '\n')
+    assert(testHelper(diffText(a, b)) ==
+      "1.1.2.2*1.0.4.4*1.0.7.6*",
+      "some-changes test failed.")
+    stdout.writeLine("some-changes test passed.")
+
+    # test one change within long chain of repeats
+    a = "a,a,a,a,a,a,a,a,a,a".replace(',', '\n')
+    b = "a,a,a,a,-,a,a,a,a,a".replace(',', '\n')
+    assert(testHelper(diffText(a, b)) ==
+      "0.1.4.4*1.0.9.10*",
+      "long chain of repeats test failed.")
+
+    stdout.writeLine("End.")
+    stdout.flushFile
+
+  main()

lib/system/chcks.nim

@@ -8,6 +8,7 @@
 #
 
 # Implementation of some runtime checks.
+import system/helpers2
 
 proc raiseRangeError(val: BiggestInt) {.compilerproc, noinline.} =
   when hostOS == "standalone":
@@ -15,6 +16,12 @@ proc raiseRangeError(val: BiggestInt) {.compilerproc, noinline.} =
   else:
     sysFatal(RangeError, "value out of range: ", $val)
 
+proc raiseIndexError3(i, a, b: int) {.compilerproc, noinline.} =
+  sysFatal(IndexError, formatErrorIndexBound(i, a, b))
+
+proc raiseIndexError2(i, n: int) {.compilerproc, noinline.} =
+  sysFatal(IndexError, formatErrorIndexBound(i, n))
+
 proc raiseIndexError() {.compilerproc, noinline.} =
   sysFatal(IndexError, "index out of bounds")
 
@@ -25,7 +32,7 @@ proc chckIndx(i, a, b: int): int =
   if i >= a and i <= b:
     return i
   else:
-    raiseIndexError()
+    raiseIndexError3(i, a, b)
 
 proc chckRange(i, a, b: int): int =
   if i >= a and i <= b:

lib/system/excpt.nim

@@ -454,16 +454,21 @@ when not defined(gcDestructors):
     shallowCopy(result, e.trace)
 
 when defined(nimRequiresNimFrame):
-  proc stackOverflow() {.noinline.} =
+  const nimCallDepthLimit {.intdefine.} = 2000
+
+  proc callDepthLimitReached() {.noinline.} =
     writeStackTrace()
-    showErrorMessage("Stack overflow\n")
+    showErrorMessage("Error: call depth limit reached in a debug build (" &
+        $nimCallDepthLimit & " function calls). You can change it with " &
+        "-d:nimCallDepthLimit=<int> but really try to avoid deep " &
+        "recursions instead.\n")
     quitOrDebug()
 
   proc nimFrame(s: PFrame) {.compilerRtl, inl, exportc: "nimFrame".} =
     s.calldepth = if framePtr == nil: 0 else: framePtr.calldepth+1
     s.prev = framePtr
     framePtr = s
-    if s.calldepth == 2000: stackOverflow()
+    if s.calldepth == nimCallDepthLimit: callDepthLimitReached()
 else:
   proc pushFrame(s: PFrame) {.compilerRtl, inl, exportc: "nimFrame".} =
     # XXX only for backwards compatibility

lib/system/helpers2.nim

@@ -0,0 +1,5 @@
+template formatErrorIndexBound*[T](i, a, b: T): string =
+  "index out of bounds: (a:" & $a & ") <= (i:" & $i & ") <= (b:" & $b & ") "
+
+template formatErrorIndexBound*[T](i, n: T): string =
+  "index out of bounds: (i:" & $i & ") <= (n:" & $n & ") "

testament/backend.nim

@@ -20,14 +20,11 @@ var
   thisCommit: CommitId
   thisBranch: string
 
-{.experimental.}
-proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip
-
 proc getMachine*(): MachineId =
-  var name = "hostname"()
+  var name = execProcess("hostname").string.strip
   if name.len == 0:
-    name = when defined(posix): getenv"HOSTNAME".string
+    name = when defined(posix): getenv("HOSTNAME").string
-           else: getenv"COMPUTERNAME".string
+           else: getenv("COMPUTERNAME").string
   if name.len == 0:
     quit "cannot determine the machine name"
 
@@ -35,8 +32,8 @@ proc getMachine*(): MachineId =
 
 proc getCommit(): CommitId =
   const commLen = "commit ".len
-  let hash = "git log -n 1"()[commLen..commLen+10]
+  let hash = execProcess("git log -n 1").string.strip[commLen..commLen+10]
-  thisBranch = "git symbolic-ref --short HEAD"()
+  thisBranch = execProcess("git symbolic-ref --short HEAD").string.strip
   if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD"
   result = CommitId(hash)
 
@@ -45,8 +42,7 @@ var
   currentCategory: string
   entries: int
 
-proc writeTestResult*(name, category, target,
+proc writeTestResult*(name, category, target, action, result, expected, given: string) =
-                      action, result, expected, given: string) =
   createDir("testresults")
   if currentCategory != category:
     if currentCategory.len > 0:

testament/categories.nim

@@ -10,6 +10,36 @@
 ## Include for the tester that contains test suites that test special features
 ## of the compiler.
 
+const
+  specialCategories = [
+    "assert",
+    "async",
+    "debugger",
+    "dll",
+    "examples",
+    "flags",
+    "gc",
+    "io",
+    "js",
+    "lib",
+    "longgc",
+    "manyloc",
+    "nimble-all",
+    "nimble-core",
+    "nimble-extra",
+    "niminaction",
+    "rodfiles",
+    "threads",
+    "untestable",
+    "stdlib",
+    "testdata",
+    "nimcache",
+    "coroutines",
+    "osproc",
+    "shouldfail",
+    "dir with space"
+  ]
+
 # included from tester.nim
 # ---------------- ROD file tests ---------------------------------------------
 
@@ -27,7 +57,7 @@ proc delNimCache(filename, options: string) =
 proc runRodFiles(r: var TResults, cat: Category, options: string) =
   template test(filename: string, clearCacheFirst=false) =
     if clearCacheFirst: delNimCache(filename, options)
-    testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun)
+    testSpec r, makeTest(rodfilesDir / filename, options, cat)
 
 
   # test basic recompilation scheme:
@@ -97,10 +127,12 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
     else:
       ""
 
-  testNoSpec c, makeTest("lib/nimrtl.nim",
+  var test1 = makeTest("lib/nimrtl.nim", options & " --app:lib -d:createNimRtl --threads:on", cat)
-    options & " --app:lib -d:createNimRtl --threads:on", cat, actionCompile)
+  test1.spec.action = actionCompile
-  testNoSpec c, makeTest("tests/dll/server.nim",
+  testSpec c, test1
-    options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat, actionCompile)
+  var test2 = makeTest("tests/dll/server.nim", options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat)
+  test2.spec.action = actionCompile
+  testSpec c, test2
 
   when defined(Windows):
     # windows looks in the dir of the exe (yay!):
@@ -120,7 +152,7 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
     safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
 
   testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl --threads:on" & rpath,
-                       cat, actionRun)
+                       cat)
 
 proc dllTests(r: var TResults, cat: Category, options: string) =
   # dummy compile result:
@@ -138,32 +170,32 @@ proc dllTests(r: var TResults, cat: Category, options: string) =
 proc gcTests(r: var TResults, cat: Category, options: string) =
   template testWithNone(filename: untyped) =
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " --gc:none", cat, actionRun)
+                  " --gc:none", cat)
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release --gc:none", cat, actionRun)
+                  " -d:release --gc:none", cat)
 
   template testWithoutMs(filename: untyped) =
-    testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options, cat)
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release", cat, actionRun)
+                  " -d:release", cat)
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release -d:useRealtimeGC", cat, actionRun)
+                  " -d:release -d:useRealtimeGC", cat)
 
   template testWithoutBoehm(filename: untyped) =
     testWithoutMs filename
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " --gc:markAndSweep", cat, actionRun)
+                  " --gc:markAndSweep", cat)
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release --gc:markAndSweep", cat, actionRun)
+                  " -d:release --gc:markAndSweep", cat)
   template test(filename: untyped) =
     testWithoutBoehm filename
     when not defined(windows) and not defined(android):
       # AR: cannot find any boehm.dll on the net, right now, so disabled
       # for windows:
       testSpec r, makeTest("tests/gc" / filename, options &
-                    " --gc:boehm", cat, actionRun)
+                    " --gc:boehm", cat)
       testSpec r, makeTest("tests/gc" / filename, options &
-                    " -d:release --gc:boehm", cat, actionRun)
+                    " -d:release --gc:boehm", cat)
 
   testWithoutBoehm "foreign_thr"
   test "gcemscripten"
@@ -196,17 +228,18 @@ proc longGCTests(r: var TResults, cat: Category, options: string) =
 
   var c = initResults()
   # According to ioTests, this should compile the file
-  testNoSpec c, makeTest("tests/realtimeGC/shared", options, cat, actionCompile)
+  testSpec c, makeTest("tests/realtimeGC/shared", options, cat)
-  testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat, actionRun)
+  #        ^- why is this not appended to r? Should this be discarded?
-  testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat, actionRun)
+  testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun
+  testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat)
 
 # ------------------------- threading tests -----------------------------------
 
 proc threadTests(r: var TResults, cat: Category, options: string) =
   template test(filename: untyped) =
-    testSpec r, makeTest(filename, options, cat, actionRun)
+    testSpec r, makeTest(filename, options, cat)
-    testSpec r, makeTest(filename, options & " -d:release", cat, actionRun)
+    testSpec r, makeTest(filename, options & " -d:release", cat)
-    testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat, actionRun)
+    testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat)
   for t in os.walkFiles("tests/threads/t*.nim"):
     test(t)
 
@@ -229,16 +262,16 @@ proc asyncTests(r: var TResults, cat: Category, options: string) =
 # ------------------------- debugger tests ------------------------------------
 
 proc debuggerTests(r: var TResults, cat: Category, options: string) =
-  testNoSpec r, makeTest("tools/nimgrep", options & " --debugger:on", cat)
+  var t = makeTest("tools/nimgrep", options & " --debugger:on", cat)
+  t.spec.action = actionCompile
+  testSpec r, t
 
 # ------------------------- JS tests ------------------------------------------
 
 proc jsTests(r: var TResults, cat: Category, options: string) =
   template test(filename: untyped) =
-    testSpec r, makeTest(filename, options & " -d:nodejs", cat,
+    testSpec r, makeTest(filename, options & " -d:nodejs", cat), {targetJS}
-                         actionRun), targetJS
+    testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat), {targetJS}
-    testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat,
-                         actionRun), targetJS
 
   for t in os.walkFiles("tests/js/t*.nim"):
     test(t)
@@ -259,14 +292,14 @@ proc jsTests(r: var TResults, cat: Category, options: string) =
 proc testNimInAction(r: var TResults, cat: Category, options: string) =
   let options = options & " --nilseqs:on"
 
-  template test(filename: untyped, action: untyped) =
+  template test(filename: untyped) =
-    testSpec r, makeTest(filename, options, cat, action)
+    testSpec r, makeTest(filename, options, cat)
 
   template testJS(filename: untyped) =
-    testSpec r, makeTest(filename, options, cat, actionCompile), targetJS
+    testSpec r, makeTest(filename, options, cat), {targetJS}
 
   template testCPP(filename: untyped) =
-    testSpec r, makeTest(filename, options, cat, actionCompile), targetCPP
+    testSpec r, makeTest(filename, options, cat), {targetCPP}
 
   let tests = [
     "niminaction/Chapter1/various1",
@@ -298,39 +331,43 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
   # edit when making a conscious breaking change, also please try to make your
   # commit message clear and notify me so I can easily compile an errata later.
   const refHashes = @[
-    "51afdfa84b3ca3d810809d6c4e5037ba", "30f07e4cd5eaec981f67868d4e91cfcf",
+    "51afdfa84b3ca3d810809d6c4e5037ba",
-    "d14e7c032de36d219c9548066a97e846", "2e40bfd5daadb268268727da91bb4e81",
+    "30f07e4cd5eaec981f67868d4e91cfcf",
-    "c5d3853ed0aba04bf6d35ba28a98dca0", "058603145ff92d46c009006b06e5b228",
+    "d14e7c032de36d219c9548066a97e846",
-    "7b94a029b94ddb7efafddd546c965ff6", "586d74514394e49f2370dfc01dd9e830",
+    "b335635562ff26ec0301bdd86356ac0c",
-    "13febc363ed82585f2a60de40ddfefda", "c11a013db35e798f44077bc0763cc86d",
+    "6c4add749fbf50860e2f523f548e6b0e",
-    "3e32e2c5e9a24bd13375e1cd0467079c", "0b9fe7ba159623d49ae60db18a15037c",
+    "76de5833a7cc46f96b006ce51179aeb1",
-    "b2dd5293d7f784824bbf9792c6fb51ad", "4c19d8d9026bfe151b31d7007fa3c237",
+    "705eff79844e219b47366bd431658961",
-    "9415c6a568cfceed08da8378e95b5cd5", "da520038c153f4054cb8cc5faa617714",
+    "a1e87b881c5eb161553d119be8b52f64",
-    "e6c6e061b6f77b2475db6fec7abfb7f4", "9a8fe78c588d08018843b64b57409a02",
+    "2d706a6ec68d2973ec7e733e6d5dce50",
-    "8b5d28e985c0542163927d253a3e4fc9", "783299b98179cc725f9c46b5e3b5381f",
+    "c11a013db35e798f44077bc0763cc86d",
-    "bc523f9a9921299090bac1af6c958e73", "80f9c3e594a798225046e8a42e990daf"
+    "3e32e2c5e9a24bd13375e1cd0467079c",
+    "a5452722b2841f0c1db030cf17708955",
+    "dc6c45eb59f8814aaaf7aabdb8962294",
+    "69d208d281a2e7bffd3eaf4bab2309b1",
+    "ec05666cfb60211bedc5e81d4c1caf3d",
+    "da520038c153f4054cb8cc5faa617714",
+    "59906c8cd819cae67476baa90a36b8c1",
+    "9a8fe78c588d08018843b64b57409a02",
+    "8b5d28e985c0542163927d253a3e4fc9",
+    "783299b98179cc725f9c46b5e3b5381f",
+    "1a2b3fba1187c68d6a9bfa66854f3318",
+    "80f9c3e594a798225046e8a42e990daf"
   ]
 
   for i, test in tests:
     let filename = "tests" / test.addFileExt("nim")
     let testHash = getMD5(readFile(filename).string)
     doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed."
-
   # Run the tests.
   for testfile in tests:
-    test "tests/" & testfile & ".nim", actionCompile
+    test "tests/" & testfile & ".nim"
-
   let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim"
   testJS jsFile
-
   let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim"
   testCPP cppFile
 
-
 # ------------------------- manyloc -------------------------------------------
-#proc runSpecialTests(r: var TResults, options: string) =
-#  for t in ["lib/packages/docutils/highlite"]:
-#    testSpec(r, t, options)
 
 proc findMainFile(dir: string): string =
   # finds the file belonging to ".nim.cfg"; if there is no such file
@@ -354,24 +391,29 @@ proc manyLoc(r: var TResults, cat: Category, options: string) =
       if dir.endsWith"named_argument_bug": continue
       let mainfile = findMainFile(dir)
       if mainfile != "":
-        testNoSpec r, makeTest(mainfile, options, cat)
+        var test = makeTest(mainfile, options, cat)
+        test.spec.action = actionCompile
+        testSpec r, test
 
 proc compileExample(r: var TResults, pattern, options: string, cat: Category) =
   for test in os.walkFiles(pattern):
-    testNoSpec r, makeTest(test, options, cat)
+    var test = makeTest(test, options, cat)
+    test.spec.action = actionCompile
+    testSpec r, test
 
 proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
-  for test in os.walkFiles(pattern):
+  for testFile in os.walkFiles(pattern):
-    let name = extractFilename(test)
+    let name = extractFilename(testFile)
     if name notin disabledFiles:
-      let contents = readFile(test).string
+      let contents = readFile(testFile).string
-      if contents.contains("when isMainModule"):
+      var testObj = makeTest(testFile, options, cat)
-        testSpec r, makeTest(test, options, cat, actionRunNoSpec)
+      if "when isMainModule" notin contents:
-      else:
+        testObj.spec.action = actionCompile
-        testNoSpec r, makeTest(test, options, cat, actionCompile)
+      testSpec r, testObj
 
 # ----------------------------- nimble ----------------------------------------
-type PackageFilter = enum
+type
+  PackageFilter = enum
     pfCoreOnly
     pfExtraOnly
     pfAll
@@ -404,7 +446,6 @@ proc getPackageDir(package: string): string =
 
 iterator listPackages(filter: PackageFilter): tuple[name, url: string] =
   let packageList = parseFile(packageIndex)
-
   for package in packageList.items():
     let
       name = package["name"].str
@@ -458,24 +499,99 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
 
 # ----------------------------------------------------------------------------
 
-const AdditionalCategories = ["debugger", "examples", "lib"]
+const AdditionalCategories = ["debugger", "examples", "lib", "megatest"]
 
 proc `&.?`(a, b: string): string =
   # candidate for the stdlib?
   result = if b.startswith(a): b else: a & b
 
-#proc `&?.`(a, b: string): string = # not used
-  # candidate for the stdlib?
-  #result = if a.endswith(b): a else: a & b
-
 proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
   let test = "tests" & DirSep &.? cat.string / test
   let target = if cat.string.normalize == "js": targetJS else: targetC
+  if existsFile(test):
+    testSpec r, makeTest(test, options, cat), {target}
+  else:
+    echo "[Warning] - ", test, " test does not exist"
 
-  if existsFile(test): testSpec r, makeTest(test, options, cat), target
+proc isJoinableSpec(spec: TSpec): bool =
-  else: echo "[Warning] - ", test, " test does not exist"
+  result = not spec.sortoutput and
+    spec.action == actionRun and
+    not fileExists(spec.file.changeFileExt("cfg")) and
+    not fileExists(parentDir(spec.file) / "nim.cfg") and
+    spec.cmd.len == 0 and
+    spec.err != reDisabled and
+    not spec.unjoinable and
+    spec.exitCode == 0 and
+    spec.input.len == 0 and
+    spec.nimout.len == 0 and
+    spec.outputCheck != ocSubstr and
+    spec.ccodeCheck.len == 0 and
+    (spec.targets == {} or spec.targets == {targetC})
 
-proc processCategory(r: var TResults, cat: Category, options: string) =
+proc norm(s: var string) =
+  while true:
+    let tmp = s.replace("\n\n", "\n")
+    if tmp == s: break
+    s = tmp
+  s = s.strip
+
+proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
+  ## returs a list of tests that have problems
+  var specs: seq[TSpec] = @[]
+
+  for kind, dir in walkDir(testsDir):
+    assert testsDir.startsWith(testsDir)
+    let cat = dir[testsDir.len .. ^1]
+    if kind == pcDir and cat notin specialCategories:
+      for file in os.walkFiles(testsDir / cat / "t*.nim"):
+        let spec = parseSpec(file)
+        if isJoinableSpec(spec):
+          specs.add spec
+
+  echo "joinable specs: ", specs.len
+
+  var megatest: string
+  for runSpec in specs:
+    megatest.add "import r\""
+    megatest.add runSpec.file
+    megatest.add "\"\n"
+
+  writeFile("megatest.nim", megatest)
+
+  const args = ["c", "-d:testing", "--listCmd", "megatest.nim"]
+  var (buf, exitCode) = execCmdEx2(command = "nim", args = args, options = {poStdErrToStdOut, poUsePath}, input = "")
+  if exitCode != 0:
+    echo buf
+    quit("megatest compilation failed")
+
+  (buf, exitCode) = execCmdEx("./megatest")
+  if exitCode != 0:
+    echo buf
+    quit("megatest execution failed")
+
+  norm buf
+  writeFile("outputGotten.txt", buf)
+  var outputExpected = ""
+  for i, runSpec in specs:
+    outputExpected.add runSpec.output.strip
+    outputExpected.add '\n'
+  norm outputExpected
+
+  if buf != outputExpected:
+    writeFile("outputExpected.txt", outputExpected)
+    discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt")
+    echo "output different!"
+    quit 1
+  else:
+    echo "output OK"
+    removeFile("outputGotten.txt")
+    removeFile("megatest.nim")
+  #testSpec r, makeTest("megatest", options, cat)
+
+# ---------------------------------------------------------------------------
+
+proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
+                     runJoinableTests: bool) =
   case cat.string.normalize
   of "rodfiles":
     when false:
@@ -524,10 +640,17 @@ proc processCategory(r: var TResults, cat: Category, options: string) =
   of "untestable":
     # We can't test it because it depends on a third party.
     discard # TODO: Move untestable tests to someplace else, i.e. nimble repo.
+  of "megatest":
+    runJoinedTest(r, cat, testsDir)
   else:
     var testsRun = 0
     for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"):
-      testSpec r, makeTest(name, options, cat)
+      var test = makeTest(name, options, cat)
+      if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
+        discard "run the test"
+      else:
+        test.spec.err = reJoined
+      testSpec r, test
       inc testsRun
     if testsRun == 0:
       echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run"

testament/specs.nim

@@ -9,21 +9,21 @@
 
 import parseutils, strutils, os, osproc, streams, parsecfg
 
-
 var compilerPrefix* = "compiler" / "nim"
 
 let isTravis* = existsEnv("TRAVIS")
 let isAppVeyor* = existsEnv("APPVEYOR")
 
-proc cmdTemplate*(): string =
-  compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file"
-
 type
   TTestAction* = enum
     actionRun = "run"
     actionCompile = "compile"
     actionReject = "reject"
-    actionRunNoSpec = "runNoSpec"
+
+  TOutputCheck* = enum
+    ocIgnore = "ignore"
+    ocEqual  = "equal"
+    ocSubstr = "substr"
 
   TResultEnum* = enum
     reNimcCrash,     # nim compiler seems to have crashed
@@ -38,8 +38,10 @@ type
     reExeNotFound,
     reInstallFailed     # package installation failed
     reBuildFailed       # package building failed
-    reIgnored,          # test is ignored
+    reDisabled,         # test is disabled
+    reJoined,           # test is disabled because it was joined into the megatest
     reSuccess           # test was successful
+    reInvalidSpec       # test had problems to parse the spec
 
   TTarget* = enum
     targetC = "C"
@@ -51,7 +53,9 @@ type
     action*: TTestAction
     file*, cmd*: string
     input*: string
-    outp*: string
+    outputCheck*: TOutputCheck
+    sortoutput*: bool
+    output*: string
     line*, column*: int
     tfile*: string
     tline*, tcolumn*: int
@@ -60,9 +64,16 @@ type
     ccodeCheck*: string
     maxCodeSize*: int
     err*: TResultEnum
-    substr*, sortoutput*: bool
     targets*: set[TTarget]
     nimout*: string
+    parseErrors*: string # when the spec definition is invalid, this is not empty.
+    unjoinable*: bool
+
+proc getCmd*(s: TSpec): string =
+  if s.cmd.len == 0:
+    result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file"
+  else:
+    result = s.cmd
 
 const
   targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"]
@@ -91,33 +102,6 @@ proc extractSpec(filename: string): string =
 when not defined(nimhygiene):
   {.pragma: inject.}
 
-template parseSpecAux(fillResult: untyped) =
-  var ss = newStringStream(extractSpec(filename))
-  var p {.inject.}: CfgParser
-  open(p, ss, filename, 1)
-  while true:
-    var e {.inject.} = next(p)
-    case e.kind
-    of cfgEof: break
-    of cfgSectionStart, cfgOption, cfgError:
-      echo ignoreMsg(p, e)
-    of cfgKeyValuePair:
-      fillResult
-  close(p)
-
-proc specDefaults*(result: var TSpec) =
-  result.msg = ""
-  result.outp = ""
-  result.nimout = ""
-  result.ccodeCheck = ""
-  result.cmd = cmdTemplate()
-  result.line = 0
-  result.column = 0
-  result.tfile = ""
-  result.tline = 0
-  result.tcolumn = 0
-  result.maxCodeSize = 0
-
 proc parseTargets*(value: string): set[TTarget] =
   for v in value.normalize.splitWhitespace:
     case v
@@ -127,34 +111,67 @@ proc parseTargets*(value: string): set[TTarget] =
     of "js": result.incl(targetJS)
     else: echo "target ignored: " & v
 
+proc addLine*(self: var string; a: string) =
+  self.add a
+  self.add "\n"
+
+proc addLine*(self: var string; a,b: string) =
+  self.add a
+  self.add b
+  self.add "\n"
+
 proc parseSpec*(filename: string): TSpec =
-  specDefaults(result)
   result.file = filename
-  parseSpecAux:
+  let specStr = extractSpec(filename)
+  var ss = newStringStream(specStr)
+  var p: CfgParser
+  open(p, ss, filename, 1)
+  while true:
+    var e = next(p)
+    case e.kind
+    of cfgKeyValuePair:
       case normalize(e.key)
       of "action":
         case e.value.normalize
-      of "compile": result.action = actionCompile
+        of "compile":
-      of "run": result.action = actionRun
+          result.action = actionCompile
-      of "reject": result.action = actionReject
+        of "run":
-      else: echo ignoreMsg(p, e)
-    of "file": result.file = e.value
-    of "line": discard parseInt(e.value, result.line)
-    of "column": discard parseInt(e.value, result.column)
-    of "tfile": result.tfile = e.value
-    of "tline": discard parseInt(e.value, result.tline)
-    of "tcolumn": discard parseInt(e.value, result.tcolumn)
-    of "output":
           result.action = actionRun
-      result.outp = e.value
+        of "reject":
+          result.action = actionReject
+        else:
+          result.parseErrors.addLine "cannot interpret as action: ", e.value
+      of "file":
+        if result.msg.len == 0 and result.nimout.len == 0:
+          result.parseErrors.addLine "errormsg or msg needs to be specified before file"
+        result.file = e.value
+      of "line":
+        if result.msg.len == 0 and result.nimout.len == 0:
+          result.parseErrors.addLine "errormsg, msg or nimout needs to be specified before line"
+        discard parseInt(e.value, result.line)
+      of "column":
+        if result.msg.len == 0 and result.nimout.len == 0:
+          result.parseErrors.addLine "errormsg or msg needs to be specified before column"
+        discard parseInt(e.value, result.column)
+      of "tfile":
+        result.tfile = e.value
+      of "tline":
+        discard parseInt(e.value, result.tline)
+      of "tcolumn":
+        discard parseInt(e.value, result.tcolumn)
+      of "output":
+        result.outputCheck = ocEqual
+        result.output = strip(e.value)
       of "input":
         result.input = e.value
       of "outputsub":
-      result.action = actionRun
+        result.outputCheck = ocSubstr
-      result.outp = e.value
+        result.output = strip(e.value)
-      result.substr = true
       of "sortoutput":
+        try:
           result.sortoutput  = parseCfgBool(e.value)
+        except:
+          result.parseErrors.addLine getCurrentExceptionMsg()
       of "exitcode":
         discard parseInt(e.value, result.exitCode)
         result.action = actionRun
@@ -167,41 +184,61 @@ proc parseSpec*(filename: string): TSpec =
         result.action = actionReject
       of "nimout":
         result.nimout = e.value
+      of "joinable":
+        result.unjoinable = not parseCfgBool(e.value)
       of "disabled":
         case e.value.normalize
-      of "y", "yes", "true", "1", "on": result.err = reIgnored
+        of "y", "yes", "true", "1", "on": result.err = reDisabled
         of "n", "no", "false", "0", "off": discard
         of "win", "windows":
-        when defined(windows): result.err = reIgnored
+          when defined(windows): result.err = reDisabled
         of "linux":
-        when defined(linux): result.err = reIgnored
+          when defined(linux): result.err = reDisabled
         of "bsd":
-        when defined(bsd): result.err = reIgnored
+          when defined(bsd): result.err = reDisabled
         of "macosx":
-        when defined(macosx): result.err = reIgnored
+          when defined(macosx): result.err = reDisabled
         of "unix":
-        when defined(unix): result.err = reIgnored
+          when defined(unix): result.err = reDisabled
         of "posix":
-        when defined(posix): result.err = reIgnored
+          when defined(posix): result.err = reDisabled
         of "travis":
-        if isTravis: result.err = reIgnored
+          if isTravis: result.err = reDisabled
         of "appveyor":
-        if isAppVeyor: result.err = reIgnored
+          if isAppVeyor: result.err = reDisabled
         else:
-        raise newException(ValueError, "cannot interpret as a bool: " & e.value)
+          result.parseErrors.addLine "cannot interpret as a bool: ", e.value
       of "cmd":
         if e.value.startsWith("nim "):
           result.cmd = compilerPrefix & e.value[3..^1]
         else:
           result.cmd = e.value
-    of "ccodecheck": result.ccodeCheck = e.value
+      of "ccodecheck":
-    of "maxcodesize": discard parseInt(e.value, result.maxCodeSize)
+        result.ccodeCheck = e.value
+      of "maxcodesize":
+        discard parseInt(e.value, result.maxCodeSize)
       of "target", "targets":
         for v in e.value.normalize.splitWhitespace:
           case v
-        of "c": result.targets.incl(targetC)
+          of "c":
-        of "cpp", "c++": result.targets.incl(targetCpp)
+            result.targets.incl(targetC)
-        of "objc": result.targets.incl(targetObjC)
+          of "cpp", "c++":
-        of "js": result.targets.incl(targetJS)
+            result.targets.incl(targetCpp)
-        else: echo ignoreMsg(p, e)
+          of "objc":
-    else: echo ignoreMsg(p, e)
+            result.targets.incl(targetObjC)
+          of "js":
+            result.targets.incl(targetJS)
+          else:
+            result.parseErrors.addLine "cannot interpret as a target: ", e.value
+      else:
+        result.parseErrors.addLine "invalid key for test spec: ", e.key
+
+    of cfgSectionStart:
+      result.parseErrors.addLine "section ignored: ", e.section
+    of cfgOption:
+      result.parseErrors.addLine "command ignored: ", e.key & ": " & e.value
+    of cfgError:
+      result.parseErrors.addLine e.msg
+    of cfgEof:
+      break
+  close(p)

testament/tester.nim

@@ -15,6 +15,7 @@ import
   algorithm, compiler/nodejs, times, sets, md5
 
 var useColors = true
+var backendLogging = true
 
 const
   resultsFile = "testresults.html"
@@ -27,6 +28,7 @@ Command:
   c|cat|category <category>   run all the tests of a certain category
   r|run <test>                run single test file
   html                        generate $1 from the database
+  stats                       generate statistics about test cases
 Arguments:
   arguments are passed to the compiler
 Options:
@@ -35,7 +37,8 @@ Options:
   --targets:"c c++ js objc" run tests for specified targets (default: all)
   --nim:path                use a particular nim executable (default: compiler/nim)
   --directory:dir           Change to directory dir before reading the tests or doing anything else.
-  --colors:on|off           turn messagescoloring on|off
+  --colors:on|off           Turn messagescoloring on|off.
+  --backendLogging:on|off   Disable or enable backend logging. By default turned on.
 """ % resultsFile
 
 type
@@ -48,7 +51,7 @@ type
     name: string
     cat: Category
     options: string
-    action: TTestAction
+    spec: TSpec
     startTime: float
 
 # ----------------------------------------------------------------------------
@@ -113,7 +116,7 @@ proc nimcacheDir(filename, options: string, target: TTarget): string =
 proc callCompiler(cmdTemplate, filename, options: string,
                   target: TTarget, extraOptions=""): TSpec =
   let nimcache = nimcacheDir(filename, options, target)
-  let options = options & " " & ("--nimCache:" & nimcache).quoteShell & extraOptions
+  let options = options & " " & quoteShell("--nimCache:" & nimcache) & extraOptions
   let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
                        "options", options, "file", filename.quoteShell,
                        "filedir", filename.getFileDir()])
@@ -138,7 +141,7 @@ proc callCompiler(cmdTemplate, filename, options: string,
   close(p)
   result.msg = ""
   result.file = ""
-  result.outp = ""
+  result.output = ""
   result.line = 0
   result.column = 0
   result.tfile = ""
@@ -170,7 +173,7 @@ proc callCCompiler(cmdTemplate, filename, options: string,
   result.nimout = ""
   result.msg = ""
   result.file = ""
-  result.outp = ""
+  result.output = ""
   result.line = -1
   while outp.readLine(x.TaintedString) or running(p):
     result.nimout.add(x & "\n")
@@ -219,18 +222,21 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
   let name = test.name.extractFilename & " " & $target & test.options
   let duration = epochTime() - test.startTime
   let durationStr = duration.formatFloat(ffDecimal, precision = 8).align(11)
+  if backendLogging:
     backend.writeTestResult(name = name,
                             category = test.cat.string,
                             target = $target,
-                          action = $test.action,
+                            action = $test.spec.action,
                             result = $success,
                             expected = expected,
                             given = given)
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
   if success == reSuccess:
     maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)"
-  elif success == reIgnored:
+  elif success == reDisabled:
     maybeStyledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
+  elif success == reJoined:
+    maybeStyledEcho styleDim, fgYellow, "JOINED: ", styleBright, fgCyan, name
   else:
     maybeStyledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
     maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
@@ -240,11 +246,11 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
     maybeStyledEcho fgYellow, "Gotten:"
     maybeStyledEcho styleBright, given, "\n"
 
-  if existsEnv("APPVEYOR"):
+  if backendLogging and existsEnv("APPVEYOR"):
     let (outcome, msg) =
       if success == reSuccess:
         ("Passed", "")
-      elif success == reIgnored:
+      elif success in {reDisabled, reJoined}:
         ("Skipped", "")
       else:
         ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
@@ -328,11 +334,6 @@ proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) =
       given.err = reMsgsDiffer
       return
 
-proc makeDeterministic(s: string): string =
-  var x = splitLines(s)
-  sort(x, system.cmp)
-  result = join(x, "\n")
-
 proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
                          expected: TSpec; r: var TResults) =
   var expectedmsg: string = ""
@@ -350,75 +351,58 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
   if given.err == reSuccess: inc(r.passed)
   r.addResult(test, target, expectedmsg, givenmsg, given.err)
 
-proc testSpec(r: var TResults, test: TTest, target = targetC) =
+proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
-  let tname = test.name.addFileExt(".nim")
+  var expected = test.spec
-  var expected: TSpec
+  if expected.parseErrors.len > 0:
-  if test.action != actionRunNoSpec:
+    # targetC is a lie, but parameter is required
-    expected = parseSpec(tname)
+    r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec)
-    if test.action == actionRun and expected.action == actionCompile:
+    inc(r.total)
-      expected.action = actionRun
+    return
-  else:
+  if expected.err in {reDisabled, reJoined}:
-    specDefaults expected
+    # targetC is a lie, but parameter is required
-    expected.action = actionRunNoSpec
+    r.addResult(test, targetC, "", "", expected.err)
-
-  if expected.err == reIgnored:
-    r.addResult(test, target, "", "", reIgnored)
     inc(r.skipped)
     inc(r.total)
     return
-
+  expected.targets.incl targets
-  if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true" and target == targetC and expected.targets == {}:
+  # still no target specified at all
-    expected.targets.incl(targetCpp)
+  if expected.targets == {}:
-  elif expected.targets == {}:
+    if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true":
-    expected.targets.incl(target)
+      expected.targets = {targetCpp}
-
+    else:
+      expected.targets = {targetC}
   for target in expected.targets:
     inc(r.total)
-    if target notin targets:
-      r.addResult(test, target, "", "", reIgnored)
-      inc(r.skipped)
-      continue
-
     case expected.action
     of actionCompile:
-      var given = callCompiler(expected.cmd, test.name, test.options, target,
+      var given = callCompiler(expected.getCmd, test.name, test.options, target,
         extraOptions=" --stdout --hint[Path]:off --hint[Processing]:off")
       compilerOutputTests(test, target, given, expected, r)
-    of actionRun, actionRunNoSpec:
+    of actionRun:
       # In this branch of code "early return" pattern is clearer than deep
       # nested conditionals - the empty rows in between to clarify the "danger"
-      var given = callCompiler(expected.cmd, test.name, test.options,
+      var given = callCompiler(expected.getCmd, test.name, test.options,
                                target)
-
-      # echo "expected.cmd: ", expected.cmd
-      # echo "nimout: ", given.nimout
-      # echo "outp:   ", given.outp
-      # echo "msg:    ", given.msg
-      # echo "err:    ", given.err
-
       if given.err != reSuccess:
         r.addResult(test, target, "", given.msg, given.err)
         continue
-
       let isJsTarget = target == targetJS
       var exeFile: string
       if isJsTarget:
-        let (_, file, _) = splitFile(tname)
+        let file = test.name.lastPathPart.changeFileExt("js")
-        exeFile = nimcacheDir(test.name, test.options, target) / file & ".js"
+        exeFile = nimcacheDir(test.name, test.options, target) / file
       else:
-        exeFile = changeFileExt(tname, ExeExt)
+        exeFile = changeFileExt(test.name, ExeExt)
 
       if not existsFile(exeFile):
-        r.addResult(test, target, expected.outp, "executable not found", reExeNotFound)
+        r.addResult(test, target, expected.output, "executable not found", reExeNotFound)
         continue
 
       let nodejs = if isJsTarget: findNodeJs() else: ""
       if isJsTarget and nodejs == "":
-        r.addResult(test, target, expected.outp, "nodejs binary not in PATH",
+        r.addResult(test, target, expected.output, "nodejs binary not in PATH",
                     reExeNotFound)
         continue
-
-
       var exeCmd: string
       var args: seq[string]
       if isJsTarget:
@@ -426,55 +410,44 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) =
         args.add exeFile
       else:
         exeCmd = exeFile
-
       var (buf, exitCode) = execCmdEx2(exeCmd, args, options = {poStdErrToStdOut}, input = expected.input)
-
       # Treat all failure codes from nodejs as 1. Older versions of nodejs used
       # to return other codes, but for us it is sufficient to know that it's not 0.
       if exitCode != 0: exitCode = 1
-
+      let bufB =
-      let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string))
+        if expected.sortoutput:
-                 else: strip(buf.string)
+          var x = splitLines(strip(buf.string))
-      let expectedOut = strip(expected.outp)
+          sort(x, system.cmp)
-
+          join(x, "\n")
+        else:
+          strip(buf.string)
       if exitCode != expected.exitCode:
         r.addResult(test, target, "exitcode: " & $expected.exitCode,
                           "exitcode: " & $exitCode & "\n\nOutput:\n" &
                           bufB, reExitCodesDiffer)
         continue
-
+      if (expected.outputCheck == ocEqual and expected.output != bufB) or
-      if bufB != expectedOut and expected.action != actionRunNoSpec:
+         (expected.outputCheck == ocSubstr and expected.output notin bufB):
-        if not (expected.substr and expectedOut in bufB):
         given.err = reOutputsDiffer
-          r.addResult(test, target, expected.outp, bufB, reOutputsDiffer)
+        r.addResult(test, target, expected.output, bufB, reOutputsDiffer)
         continue
-
       compilerOutputTests(test, target, given, expected, r)
       continue
-
     of actionReject:
-      var given = callCompiler(expected.cmd, test.name, test.options,
+      var given = callCompiler(expected.getCmd, test.name, test.options,
                                target)
       cmpMsgs(r, expected, given, test, target)
       continue
 
-proc testNoSpec(r: var TResults, test: TTest, target = targetC) =
+proc testC(r: var TResults, test: TTest, action: TTestAction) =
-  # does not extract the spec because the file is not supposed to have any
-  #let tname = test.name.addFileExt(".nim")
-  inc(r.total)
-  let given = callCompiler(cmdTemplate(), test.name, test.options, target)
-  r.addResult(test, target, "", given.msg, given.err)
-  if given.err == reSuccess: inc(r.passed)
-
-proc testC(r: var TResults, test: TTest) =
   # runs C code. Doesn't support any specs, just goes by exit code.
   let tname = test.name.addFileExt(".c")
   inc(r.total)
   maybeStyledEcho "Processing ", fgCyan, extractFilename(tname)
-  var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC)
+  var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC)
   if given.err != reSuccess:
     r.addResult(test, targetC, "", given.msg, given.err)
-  elif test.action == actionRun:
+  elif action == actionRun:
     let exeFile = changeFileExt(test.name, ExeExt)
     var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath})
     if exitCode != 0: given.err = reExitCodesDiffer
@@ -485,7 +458,6 @@ proc testExec(r: var TResults, test: TTest) =
   inc(r.total)
   let (outp, errC) = execCmdEx(test.options.strip())
   var given: TSpec
-  specDefaults(given)
   if errC == 0:
     given.err = reSuccess
   else:
@@ -495,11 +467,12 @@ proc testExec(r: var TResults, test: TTest) =
   if given.err == reSuccess: inc(r.passed)
   r.addResult(test, targetC, "", given.msg, given.err)
 
-proc makeTest(test, options: string, cat: Category, action = actionCompile,
+proc makeTest(test, options: string, cat: Category): TTest =
-              env: string = ""): TTest =
+  result.cat = cat
-  # start with 'actionCompile', will be overwritten in the spec:
+  result.name = test
-  result = TTest(cat: cat, name: test, options: options,
+  result.options = options
-                 action: action, startTime: epochTime())
+  result.spec = parseSpec(addFileExt(test, ".nim"))
+  result.startTime = epochTime()
 
 when defined(windows):
   const
@@ -512,10 +485,7 @@ else:
 
 include categories
 
-# proc runCaasTests(r: var TResults) =
+const testsDir = "tests" & DirSep
-#   for test, output, status, mode in caasTestsRunner():
-#     r.addResult(test, "", output & "-> " & $mode,
-#                 if status: reSuccess else: reOutputsDiffer)
 
 proc main() =
   os.putenv "NIMTEST_COLOR", "never"
@@ -524,10 +494,8 @@ proc main() =
   backend.open()
   var optPrintResults = false
   var optFailing = false
-
   var targetsStr = ""
 
-
   var p = initOptParser()
   p.next()
   while p.kind == cmdLongoption:
@@ -538,26 +506,38 @@ proc main() =
     of "targets":
       targetsStr = p.val.string
       targets = parseTargets(targetsStr)
-    of "nim": compilerPrefix = p.val.string
+    of "nim":
+      compilerPrefix = addFileExt(p.val.string, ExeExt)
     of "directory":
       setCurrentDir(p.val.string)
     of "colors":
-      if p.val.string == "on":
+      case p.val.string:
+      of "on":
         useColors = true
-      elif p.val.string == "off":
+      of "off":
         useColors = false
       else:
         quit Usage
+    of "backendlogging":
+      case p.val.string:
+      of "on":
+        backendLogging = true
+      of "off":
+        backendLogging = false
+      else:
+        quit Usage
     else:
       quit Usage
     p.next()
-  if p.kind != cmdArgument: quit Usage
+  if p.kind != cmdArgument:
+    quit Usage
   var action = p.key.string.normalize
   p.next()
   var r = initResults()
   case action
   of "all":
-    let testsDir = "tests" & DirSep
+    #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false)
+
     var myself = quoteShell(findExe("testament" / "tester"))
     if targetsStr.len > 0:
       myself &= " " & quoteShell("--targets:" & targetsStr)
@@ -570,14 +550,21 @@ proc main() =
       assert testsDir.startsWith(testsDir)
       let cat = dir[testsDir.len .. ^1]
       if kind == pcDir and cat notin ["testdata", "nimcache"]:
-        cmds.add(myself & " cat " & quoteShell(cat) & rest)
+        cmds.add(myself & " pcat " & quoteShell(cat) & rest)
     for cat in AdditionalCategories:
-      cmds.add(myself & " cat " & quoteShell(cat) & rest)
+      cmds.add(myself & " pcat " & quoteShell(cat) & rest)
     quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams})
   of "c", "cat", "category":
     var cat = Category(p.key)
     p.next
-    processCategory(r, cat, p.cmdLineRest.string)
+    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true)
+  of "pcat":
+    # 'pcat' is used for running a category in parallel. Currently the only
+    # difference is that we don't want to run joinable tests here as they
+    # are covered by the 'megatest' category.
+    var cat = Category(p.key)
+    p.next
+    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false)
   of "r", "run":
     let (dir, file) = splitPath(p.key.string)
     let (_, subdir) = splitPath(dir)

testament/tests/shouldfail/tcolumn.nim

@@ -1,7 +1,7 @@
 discard """
+errormsg: "undeclared identifier: 'undeclared'"
 line: 8
 column: 7
-errormsg: "undeclared identifier: 'undeclared'"
 """
 
 # test should fail because the line directive is wrong

testament/tests/shouldfail/terrormsg.nim

@@ -1,7 +1,7 @@
 discard """
+errormsg: "wrong error message"
 line: 8
 column: 6
-errormsg: "wrong error message"
 """
 
 # test should fail because the line directive is wrong

testament/tests/shouldfail/tfile.nim

@@ -1,6 +1,6 @@
 discard """
-file: "notthisfile.nim"
 errmsg: "undeclared identifier: 'undefined'"
+file: "notthisfile.nim"
 """
 
 echo undefined

testament/tests/shouldfail/tline.nim

@@ -1,7 +1,7 @@
 discard """
+errormsg: "undeclared identifier: 'undeclared'"
 line: 9
 column: 6
-errormsg: "undeclared identifier: 'undeclared'"
 """
 
 # test should fail because the line directive is wrong

tests/ambsym/tambsym.nim

@@ -1,7 +1,7 @@
 discard """
+  errormsg: "ambiguous identifier"
   file: "tambsym.nim"
   line: 11
-  errormsg: "ambiguous identifier"
 """
 # Test ambiguous symbols
 
@@ -11,5 +11,3 @@ var
   v: TExport #ERROR_MSG ambiguous identifier
 
 v = y
-
-

tests/ambsym/tambsym2.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tambsym2.nim"
   output: "7"
 """
 # Test overloading of procs with locals
@@ -20,5 +19,3 @@ m.len = 7
 m.data = "1234"
 
 x(m, 5) #OUT 7
-
-

tests/ambsym/tambsym3.nim

@@ -1,7 +1,7 @@
 discard """
+  errormsg: "ambiguous identifier"
   file: "tambsym3.nim"
   line: 11
-  errormsg: "ambiguous identifier"
 """
 # Test ambiguous symbols
 
@@ -11,5 +11,3 @@ var
   v = mDec #ERROR_MSG ambiguous identifier
 
 writeLine(stdout, ord(v))
-
-

tests/ambsym/tambsys.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tambsys.nim"
   output: ""
 """
 # Test ambiguous symbols
@@ -9,5 +8,3 @@ import mambsys1, mambsys2
 var
   v: mambsys1.TExport
 mambsys2.foo(3) #OUT
-
-

tests/array/tarray.nim

@@ -1,7 +1,5 @@
 discard """
-  file: "tarray.nim"
+output: '''
-  output:
-'''
 [4, 5, 6]
 
 [16, 25, 36]
@@ -30,6 +28,7 @@ dflfdjkl__abcdefgasfsgdfgsgdfggsdfasdfsafewfkljdsfajsdf
 kgdchlfniambejop
 fjpmholcibdgeakn
 '''
+joinable: false
 """
 
 block tarray:
@@ -358,7 +357,7 @@ block troofregression:
   echo testStr[testStr.len - 8 .. testStr.len - 1] & "__" & testStr[0 .. testStr.len - pred(rot)]
 
   var
-    instructions = readFile(getAppDir() / "troofregression2.txt").split(',')
+    instructions = readFile(parentDir(currentSourcePath) / "troofregression2.txt").split(',')
     programs = "abcdefghijklmnop"
 
   proc dance(dancers: string): string =

tests/array/tarraycons.nim

@@ -1,7 +1,7 @@
 discard """
+  errormsg: "invalid order in array constructor"
   file: "tarraycons.nim"
   line: 14
-  errormsg: "invalid order in array constructor"
 """
 
 type
@@ -19,6 +19,3 @@ const
   ]
 
 echo myMapping[eC][1]
-
-
-

tests/array/tarraycons_ptr_generic2.nim

@@ -1,7 +1,7 @@
 discard """
+  errormsg: "type mismatch: got <ptr Hard[system.string]> but expected 'Book[system.string]'"
   file: "tarraycons_ptr_generic2.nim"
   line: 17
-  errormsg: "type mismatch: got <ptr Hard[system.string]> but expected 'Book[system.string]'"
 """
 
 type

tests/assert/tassert.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tassert.nim"
   outputsub: "assertion failure!this shall be always written"
   exitcode: "1"
 """
@@ -19,5 +18,3 @@ finally:
   system.write(stdout, "this shall be always written")
 
 assert(false) #OUT assertion failure!this shall be always written
-
-

tests/assign/tassign.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tassign.nim"
   output:
 '''
 TEMP=C:\Programs\xyz\bin

tests/assign/tvariantasgn.nim

@@ -1,7 +1,7 @@
 discard """
-  file: "tvariantasgn.nim"
   output: "came here"
 """
+
 #BUG
 type
   TAnyKind = enum
@@ -26,5 +26,3 @@ nr.intVal = 78
 # s = nr # works
 nr = s # fails!
 echo "came here"
-
-

tests/async/t7192.nim

@@ -9,6 +9,6 @@ import asyncdispatch
 proc testCallback() =
   echo "testCallback()"
 
-when isMainModule:
+when true:
   callSoon(testCallback)
   poll()

tests/async/t7758.nim

@@ -1,7 +1,3 @@
-discard """
-  file: "t7758.nim"
-  exitcode: 0
-"""
 import asyncdispatch
 
 proc task() {.async.} =

tests/async/tasyncRecvLine.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tasyncRecvLine.nim"
 output: '''
 Hello World
 Hello World

tests/async/tasync_gcunsafe.nim

@@ -1,7 +1,7 @@
 discard """
+  errormsg: "'anotherGCSafeAsyncProcIter' is not GC-safe as it calls 'asyncGCUnsafeProc'"
   cmd: "nim c --threads:on $file"
   file: "asyncmacro.nim"
-  errormsg: "'anotherGCSafeAsyncProcIter' is not GC-safe as it calls 'asyncGCUnsafeProc'"
 """
 
 assert compileOption("threads"), "this test will not do anything useful without --threads:on"

tests/async/tasyncall.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tasyncall.nim"
   exitcode: 0
 """
 import times, sequtils

tests/async/tasyncawait.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tasyncawait.nim"
   output: "5000"
 """
 import asyncdispatch, nativesockets, net, strutils, os

tests/async/tasyncconnect.nim

@@ -1,7 +1,6 @@
 discard """
-  file: "tasyncconnect.nim"
-  exitcode: 1
   outputsub: "Error: unhandled exception: Connection refused"
+  exitcode: 1
 """
 
 import

tests/async/tasyncdial.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tasyncdial.nim"
   output: '''
 OK AF_INET
 OK AF_INET6

tests/async/tasyncexceptions.nim

@@ -1,7 +1,6 @@
 discard """
-  file: "tasyncexceptions.nim"
-  exitcode: 1
   outputsub: "Error: unhandled exception: foobar"
+  exitcode: 1
 """
 import asyncdispatch
 
@@ -28,7 +27,7 @@ proc serve() {.async.} =
     var fut = await accept()
     await processClient(fut)
 
-when isMainModule:
+when true:
   proc main =
     var fut = serve()
     fut.callback =

tests/async/tasyncfile.nim

@@ -1,10 +1,9 @@
 discard """
-  output: '''13
+output: '''
+13
 hello humans!
 13
 '''
-  file: "tasyncfile.nim"
-  exitcode: 0
 """
 import asyncfile, asyncdispatch, os
 
@@ -54,8 +53,7 @@ proc main() {.async.} =
 
   # Issue #7347
   block:
-    let appDir = getAppDir()
+    var file = openAsync( parentDir(currentSourcePath) / "hello.txt")
-    var file = openAsync(appDir & DirSep & "hello.txt")
     echo file.getFileSize()
     echo await file.readAll()
     echo file.getFilePos()

tests/async/tasyncrecursion.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tasyncrecursion.nim"
 output: "50005000"
 """
 import asyncdispatch
@@ -16,7 +15,7 @@ proc asyncRecursionTest*(): Future[int] {.async.} =
     inc(result, await asyncRecursionCycle(i))
     inc(i)
 
-when isMainModule:
+when true:
   setGlobalDispatcher(newDispatcher())
   var i = waitFor asyncRecursionTest()
   echo i

tests/async/tasyncsend4757.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tasyncsend4754.nim"
 output: "Finished"
 """
 

tests/async/tasyncssl.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tasyncssl.nim"
   cmd: "nim $target --hints:on --define:ssl $options $file"
   output: "500"
   disabled: "windows"

tests/async/tasynctry.nim

@@ -1,6 +1,4 @@
 discard """
-  file: "tasynctry.nim"
-  exitcode: 0
 output: '''
 Generic except: Test
 Specific except

tests/async/tawaitsemantics.nim

@@ -1,6 +1,4 @@
 discard """
-  file: "tawaitsemantics.nim"
-  exitcode: 0
 output: '''
 Error can be caught using yield
 Infix `or` raises

tests/async/tfuturestream.nim

@@ -1,6 +1,4 @@
 discard """
-  file: "tfuturestream.nim"
-  exitcode: 0
 output: '''
 0
 1

tests/async/tioselectors.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tioselectors.nim"
   output: "All tests passed!"
 """
 import selectors

tests/async/tnewasyncudp.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "tnewasyncudp.nim"
   output: "5000"
 """
 import asyncdispatch, nativesockets, net, strutils, os

tests/async/tpendingcheck.nim

@@ -1,6 +1,4 @@
 discard """
-  file: "tpendingcheck.nim"
-  exitcode: 0
   output: ""
 """
 
@@ -18,4 +16,3 @@ while not f.finished:
 f.read
 
 doAssert(not hasPendingOperations())
-

tests/async/twinasyncrw.nim

@@ -1,5 +1,4 @@
 discard """
-  file: "twinasyncrw.nim"
   output: "5000"
 """
 when defined(windows):

tests/benchmark.nim

@@ -1,47 +0,0 @@
-#
-#
-#            Nim Benchmark tool
-#        (c) Copyright 2012 Dominik Picheta
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This program runs benchmarks
-import osproc, os, times, json
-
-type
-  TBenchResult = tuple[file: string, success: bool, time: float]
-
-proc compileBench(file: string) =
-  ## Compiles ``file``.
-  doAssert(execCmdEx("nim c -d:release " & file).exitCode == QuitSuccess)
-
-proc runBench(file: string): TBenchResult =
-  ## Runs ``file`` and returns info on how long it took to run.
-  var start = epochTime()
-  if execCmdEx(file.addFileExt(ExeExt)).exitCode == QuitSuccess:
-    var t = epochTime() - start
-    result = (file, true, t)
-  else: result = (file, false, -1.0)
-
-proc genOutput(benches: seq[TBenchResult]): PJsonNode =
-  result = newJObject()
-  for i in benches:
-    if i.success:
-      result[i.file.extractFilename] = newJFloat(i.time)
-    else:
-      result[i.file.extractFilename] = newJNull()
-
-proc doBench(): seq[TBenchResult] =
-  result = @[]
-  for i in walkFiles("tests/benchmarks/*.nim"):
-    echo(i.extractFilename)
-    compileBench(i)
-    result.add(runBench(i))
-
-when isMainModule:
-  var b = doBench()
-  var output = genOutput(b)
-  writeFile("benchmarkResults.json", pretty(output))
-

tests/benchmarks/fannkuch.nim

@@ -1,69 +0,0 @@
-import os
-import strutils
-
-proc fannkuch(n: int): int =
-    var
-        count: seq[int]
-        maxFlips = 0
-        m = n-1
-        r = n
-        check = 0
-        perm1: seq[int]
-        perm: seq[int]
-
-    newSeq(count, n+1)
-    newSeq(perm1, n)
-    newSeq(perm, n)
-    for i in 0 .. n-1:
-        count[i] = i+1
-        perm1[i] = i
-        perm[i]  = i
-    count[n] = n+1
-
-    while true:
-        if check < 30:
-            for i in items(perm1):
-                write(stdout, $(i+1))
-            echo("")
-            inc(check)
-
-        while r != 1:
-            count[r-1] = r
-            dec (r)
-
-        if perm1[0] != 0 and perm1[m] != m:
-            # perm = perm1
-            # The above line is between 3 and 4 times slower than the loop below!
-            for i in 0 .. n-1:
-                perm[i] = perm1[i]
-            var flipsCount = 0
-            var k = perm[0]
-            while k != 0:
-                for i in 0 .. (k div 2):
-                    swap(perm[i], perm[k-i])
-                inc(flipsCount)
-                k = perm[0]
-
-            if flipsCount > maxFlips:
-                maxFlips = flipsCount
-
-        block makePerm:
-            while r != n:
-                var tmp = perm1[0]
-                # # perm1.delete (0)
-                # # perm1.insert (tmp, r)
-                # # The above is about twice as slow as the following:
-                # moveMem (addr (perm1[0]), addr (perm1[1]), r * sizeof (int))
-                # The call to moveMem is about 50% slower than the loop below!
-                for i in 0 .. r-1:
-                    perm1[i] = perm1[i+1]
-                perm1[r] = tmp
-
-                dec(count[r])
-                if count[r] > 0:
-                    break makePerm
-                inc(r)
-            return maxFlips
-
-var n = 10
-echo("Pfannkuchen(" & $n & ") = " & $fannkuch(n))

tests/benchmarks/quicksort.nim

@@ -1,54 +0,0 @@
-import os
-import strutils
-
-# Generate some pseudo-random data
-var seed: tuple[s1, s2, s3: int32] = (2'i32, 8'i32, 16'i32)
-
-proc random(): int32 =
-    seed = (((((((seed[0] and 0x0007_FFFF'i32) shl 13'i32) xor seed[0]) shr
-               19'i32) and 0x0000_1FFF'i32) xor
-             ((seed[0] and 0x000F_FFFE'i32) shl 12'i32)),
-
-            ((((((seed[1] and 0x3FFF_FFFF'i32) shl  2'i32) xor seed[1]) shr
-               25'i32) and 0x0000_007F'i32) xor
-             ((seed[1] and 0x0FFF_FFF8'i32) shl  4'i32)),
-
-            ((((((seed[2] and 0x1FFF_FFFF'i32) shl  3'i32) xor seed[2]) shr
-               11'i32) and 0x001F_FFFF'i32) xor
-             ((seed[2] and 0x0000_7FF0'i32) shl 17'i32)))
-    return seed[0] xor seed[1] xor seed[2]
-
-var n = 9999999
-
-var data: seq[int32]
-newSeq(data, n)
-for i in 0 .. data.high():
-    data[i] = random()
-
-
-proc `$`(d: seq[int32]): string =
-    result = "[ "
-    for i in items(d):
-        result.addSep(", ", 2)
-        result.add($(i and 0xFFFF_FFFF'i64))
-    result.add(" ]")
-
-# Sort the data
-proc sort(start, stop: int) =
-    if stop <= start+1:
-        return
-
-    var j = start
-    for i in start..stop-2:
-        if data[i] <% data[stop-1]:
-            swap(data[i], data[j])
-            inc(j)
-    swap(data[j], data[stop-1])
-
-    sort(start, j)
-    sort(j+1, stop)
-
-sort(0, data.len)
-echo(data[n div 2 - 1] and 0xFFFF_FFFF'i64, ", ",
-     data[n div 2] and 0xFFFF_FFFF'i64, ", ",
-     data[n div 2 + 1] and 0xFFFF_FFFF'i64)

tests/bind/tbind.nim

@@ -1,7 +1,5 @@
 discard """
-  file: "tbind.nim"
+output: '''
-  output:
-'''
 3
 1
 1

tests/bind/tbind2.nim

@@ -1,7 +1,7 @@
 discard """
+  errormsg: "ambiguous call"
   file: "tbind2.nim"
   line: 12
-  errormsg: "ambiguous call"
 """
 # Test the new ``bind`` keyword for templates
 
@@ -12,6 +12,3 @@ template tempBind(x, y): untyped =
   (bind p1(x, y))  #ERROR_MSG ambiguous call
 
 echo tempBind(1'i8, 2'i8)
-
-
-

tests/bind/tdatabind.nim

@@ -74,7 +74,7 @@ proc propertyBind*[T](p1: var TProperty[T], p2: var TProperty[T]) =
 proc `->`[T](p1: var TProperty[T], p2: var TProperty[T]) =
   propertyBind(p2,p1)
 
-when isMainModule:
+when true:
   # Initial value testing
   var myProp = newProperty(5)
 

tests/bind/tinvalidbindtypedesc.nim

@@ -1,6 +1,6 @@
 discard """
-  line: 10
   errormsg: "type mismatch: got <type float, string>"
+  line: 10
 """
 
 proc foo(T: typedesc; some: T) =
@@ -8,4 +8,3 @@ proc foo(T: typedesc; some: T) =
 
 foo int, 4
 foo float, "bad"
-

tests/bind/tnicerrorforsymchoice.nim

@@ -1,6 +1,6 @@
 discard """
-  line: 18
   errormsg: "type mismatch: got <proc (s: TScgi: ScgiState or AsyncScgiState) | proc (client: AsyncSocket, headers: StringTableRef, input: string){.noSideEffect, gcsafe, locks: 0.}>"
+  line: 18
 """
 
 #bug #442

tests/borrow/tinvalidborrow.nim

@@ -1,6 +1,6 @@
 discard """
-  line: 11
   errormsg: "no symbol to borrow from found"
+  line: 11
 """
 
 # bug #516
@@ -14,4 +14,3 @@ var
   d, e: TAtom
 
 echo( $(d == e) )
-

tests/caas/absurd_nesting.nim

@@ -1,29 +0,0 @@
-# Tries to test the full ownership path generated by idetools.
-
-proc lev1(t1: string) =
-  var temp = t1
-  for i in 0..len(temp)-1:
-    temp[i] = chr(int(temp[i]) + 1)
-
-  proc lev2(t2: string) =
-    var temp = t2
-    for i in 0..len(temp)-1:
-      temp[i] = chr(int(temp[i]) + 1)
-
-    proc lev3(t3: string) =
-      var temp = t3
-      for i in 0..len(temp)-1:
-        temp[i] = chr(int(temp[i]) + 1)
-
-      proc lev4(t4: string) =
-        var temp = t4
-        for i in 0..len(temp)-1:
-          temp[i] = chr(int(temp[i]) + 1)
-
-        echo temp & "(lev4)"
-      lev4(temp & "(lev3)")
-    lev3(temp & "(lev2)")
-  lev2(temp & "(lev1)")
-
-when isMainModule:
-  lev1("abcd")