Merge branch 'devel' into expand-amb-identifier-output

.github/ISSUE_TEMPLATE/bug_report.md

@@ -38,6 +38,7 @@ Hello World!
 * Your Nim version (output of `nim -v`).
 * Was it working in the previous Nim releases?
 * A link to a related issue or discussion.
+* A project reference where (and how) the issue causes problems.
 -->
 ```
 $ nim -v

.gitignore

@@ -52,6 +52,8 @@ xcuserdata/
 /compiler/nim.dot
 /reject.json
 /run.json
+*.json
+/pkgstemp/**/*
 # for `nim doc foo.nim`
 /*.html
 lib/**/*.html
@@ -59,6 +61,8 @@ lib/**/*.html
 
 /testresults.json
 testament.db
+/tests/**/*.json
+/tests/**/*.js
 /csources
 dist/
 

changelog.md

@@ -47,11 +47,17 @@
 
 - `system.ValueError` now inherits from `system.CatchableError` instead of `system.Defect`.
 
-- The procs `parseutils.parseBiggsetInt`, `parseutils.parseInt`,
+- The procs `parseutils.parseBiggestInt`, `parseutils.parseInt`,
   `parseutils.parseBiggestUInt` and `parseutils.parseUInt` now raise a
   `ValueError` when the parsed integer is outside of the valid range.
   Previously they sometimes raised a `OverflowError` and sometimes returned `0`.
 
+- The procs `parseutils.parseBin`, `parseutils.parseOct` and `parseutils.parseHex`
+  were not clearing their `var` parameter `number` and used to push its value to
+  the left when storing the parsed string into it. Now they always set the value
+  of the parameter to `0` before storing the result of the parsing, unless the
+  string to parse is not valid and then the value of `number` is not changed.
+
 - `streams.StreamObject` now restricts its fields to only raise `system.Defect`,
   `system.IOError` and `system.OSError`.
   This change only affects custom stream implementations.
@@ -91,6 +97,7 @@
 - Custom types that should be supported by `strformat` (&) now need an
   explicit overload of `formatValue`.
 
+
 #### Breaking changes in the compiler
 
 - The compiler now implements the "generic symbol prepass" for `when` statements
@@ -107,6 +114,7 @@ proc enumToString*(enums: openArray[enum]): string =
 ```
 
 - ``discard x`` is now illegal when `x` is a function symbol.
+
 - Implicit imports via ``--import: module`` in a config file are now restricted
   to the main package.
 
@@ -152,6 +160,9 @@ proc enumToString*(enums: openArray[enum]): string =
 
 - Added `system.default`.
 
+- Added `sequtils.items` for closure iterators, allows closure iterators 
+  to be used by the the mapIt, filterIt, allIt, anyIt, etc.
+
 
 ### Library changes
 
@@ -179,6 +190,7 @@ proc enumToString*(enums: openArray[enum]): string =
   differently.
 
 - `securehash` is moved to `lib/deprecated`
+
 - The switch ``-d:useWinAnsi`` is not supported anymore.
 
 
@@ -202,14 +214,19 @@ proc enumToString*(enums: openArray[enum]): string =
   it's more recognizable and allows tools like github to recognize it as Nim,
   see [#9647](https://github.com/nim-lang/Nim/issues/9647).
   The previous extension will continue to work.
+
 - Pragma syntax is now consistent. Previous syntax where type pragmas did not
   follow the type name is now deprecated. Also pragma before generic parameter
   list is deprecated to be consistent with how pragmas are used with a proc. See
   [#8514](https://github.com/nim-lang/Nim/issues/8514) and
   [#1872](https://github.com/nim-lang/Nim/issues/1872) for further details.
 
+- Hash sets and tables are initialized by default. The explicit `initHashSet`,
+  `initTable`, etc. are not needed anymore.
+
 
 ### Tool changes
+
 - `jsondoc` now include a `moduleDescription` field with the module
   description. `jsondoc0` shows comments as it's own objects as shown in the
   documentation.
@@ -218,6 +235,7 @@ proc enumToString*(enums: openArray[enum]): string =
 
 
 ### Compiler changes
+
 - The deprecated `fmod` proc is now unavailable on the VM'.
 - A new `--outdir` option was added.
 - The compiled JavaScript file for the project produced by executing `nim js`
@@ -226,4 +244,5 @@ proc enumToString*(enums: openArray[enum]): string =
   The compiler also provides a new more flexible API for handling the
   hot code reloading events in the code.
 
+
 ### Bugfixes

compiler/ast.nim

@@ -644,7 +644,7 @@ type
     mSwap, mIsNil, mArrToSeq, mCopyStr, mCopyStrLast,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
     mMove, mWasMoved, mDestroy,
-    mDefault, mAccessEnv, mReset,
+    mDefault, mUnown, mAccessEnv, mReset,
     mArray, mOpenArray, mRange, mSet, mSeq, mOpt, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
     mOrdinal,
@@ -877,6 +877,13 @@ type
 
   TTypeSeq* = seq[PType]
   TLockLevel* = distinct int16
+
+  TTypeAttachedOp* = enum ## as usual, order is important here
+    attachedDestructor,
+    attachedAsgn,
+    attachedSink,
+    attachedDeepCopy
+
   TType* {.acyclic.} = object of TIdObj # \
                               # types are identical iff they have the
                               # same id; there may be multiple copies of a type
@@ -897,12 +904,7 @@ type
     owner*: PSym              # the 'owner' of the type
     sym*: PSym                # types have the sym associated with them
                               # it is used for converting types to strings
-    destructor*: PSym         # destructor. warning: nil here may not necessary
+    attachedOps*: array[TTypeAttachedOp, PSym] # destructors, etc.
-                              # mean that there is no destructor.
-                              # see instantiateDestructor in semdestruct.nim
-    deepCopy*: PSym           # overriden 'deepCopy' operation
-    assignment*: PSym         # overriden '=' operation
-    sink*: PSym               # overriden '=sink' operation
     methods*: seq[(int,PSym)] # attached methods
     size*: BiggestInt         # the size of the type in bytes
                               # -1 means that the size is unkwown
@@ -1275,6 +1277,7 @@ const
   UnspecifiedLockLevel* = TLockLevel(-1'i16)
   MaxLockLevel* = 1000'i16
   UnknownLockLevel* = TLockLevel(1001'i16)
+  AttachedOpToStr*: array[TTypeAttachedOp, string] = ["=destroy", "=", "=sink", "=deepcopy"]
 
 proc `$`*(x: TLockLevel): string =
   if x.ord == UnspecifiedLockLevel.ord: result = "<unspecified>"
@@ -1341,10 +1344,7 @@ proc assignType*(dest, src: PType) =
   dest.n = src.n
   dest.size = src.size
   dest.align = src.align
-  dest.destructor = src.destructor
+  dest.attachedOps = src.attachedOps
-  dest.deepCopy = src.deepCopy
-  dest.sink = src.sink
-  dest.assignment = src.assignment
   dest.lockLevel = src.lockLevel
   # this fixes 'type TLock = TSysLock':
   if src.sym != nil:
@@ -1831,3 +1831,7 @@ proc addParam*(procType: PType; param: PSym) =
   param.position = procType.len-1
   addSon(procType.n, newSymNode(param))
   rawAddSon(procType, param.typ)
+
+template destructor*(t: PType): PSym = t.attachedOps[attachedDestructor]
+template assignment*(t: PType): PSym = t.attachedOps[attachedAsgn]
+template asink*(t: PType): PSym = t.attachedOps[attachedSink]

compiler/ccgcalls.nim

@@ -46,13 +46,23 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
       add(pl, ~")")
-      if p.module.compileToCpp and lfSingleUse in d.flags:
+      if p.module.compileToCpp:
-        # do not generate spurious temporaries for C++! For C we're better off
+        if lfSingleUse in d.flags:
-        # with them to prevent undefined behaviour and because the codegen
+          # do not generate spurious temporaries for C++! For C we're better off
-        # is free to emit expressions multiple times!
+          # with them to prevent undefined behaviour and because the codegen
-        d.k = locCall
+          # is free to emit expressions multiple times!
-        d.r = pl
+          d.k = locCall
-        excl d.flags, lfSingleUse
+          d.r = pl
+          excl d.flags, lfSingleUse
+        else:
+          if d.k == locNone and p.splitDecls == 0:
+            getTempCpp(p, typ.sons[0], d, pl)
+          else:
+            if d.k == locNone: getTemp(p, typ.sons[0], d)
+            var list: TLoc
+            initLoc(list, locCall, d.lode, OnUnknown)
+            list.r = pl
+            genAssignment(p, d, list, {}) # no need for deep copying
       else:
         if d.k == locNone: getTemp(p, typ.sons[0], d)
         assert(d.t != nil)        # generate an assignment to d:
@@ -218,10 +228,12 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     genParamLoop(pl)
 
   template genCallPattern {.dirty.} =
-    lineF(p, cpsStmts, callPattern & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
+    if tfIterator in typ.flags:
+      lineF(p, cpsStmts, PatIter & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
+    else:
+      lineF(p, cpsStmts, PatProc & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
 
   let rawProc = getRawProcType(p, typ)
-  let callPattern = if tfIterator in typ.flags: PatIter else: PatProc
   if typ.sons[0] != nil:
     if isInvalidReturnType(p.config, typ.sons[0]):
       if sonsLen(ri) > 1: add(pl, ~", ")
@@ -246,7 +258,11 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, d.lode, OnUnknown)
-      list.r = callPattern % [rdLoc(op), pl, pl.addComma, rawProc]
+      if tfIterator in typ.flags:
+        list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
+      else:
+        list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
+
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
     genCallPattern()

compiler/ccgexprs.nim

@@ -99,17 +99,17 @@ proc bitSetToWord(s: TBitSet, size: int): BiggestInt =
     if j < len(s): result = result or (ze64(s[j]) shl (j * 8))
 
 proc genRawSetData(cs: TBitSet, size: int): Rope =
-  var frmt: FormatStr
   if size > 8:
     result = "{$n" % []
     for i in countup(0, size - 1):
       if i < size - 1:
         # not last iteration?
-        if (i + 1) mod 8 == 0: frmt = "0x$1,$n"
+        if (i + 1) mod 8 == 0:
-        else: frmt = "0x$1, "
+          addf(result, "0x$1,$n", [rope(toHex(ze64(cs[i]), 2))])
+        else:
+          addf(result, "0x$1, ", [rope(toHex(ze64(cs[i]), 2))])
       else:
-        frmt = "0x$1}$n"
+        addf(result, "0x$1}$n", [rope(toHex(ze64(cs[i]), 2))])
-      addf(result, frmt, [rope(toHex(ze64(cs[i]), 2))])
   else:
     result = intLiteral(bitSetToWord(cs, size))
     #  result := rope('0x' + ToHex(bitSetToWord(cs, size), size * 2))
@@ -510,10 +510,7 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
       "mulInt64", "divInt64", "modInt64",
       "addInt64", "subInt64"
     ]
-    opr: array[mAddI..mPred, string] = [
+    opr: array[mAddI..mPred, string] = ["+", "-", "*", "/", "%", "+", "-"]
-      "($#)($# + $#)", "($#)($# - $#)", "($#)($# * $#)",
-      "($#)($# / $#)", "($#)($# % $#)",
-      "($#)($# + $#)", "($#)($# - $#)"]
   var a, b: TLoc
   assert(e.sons[1].typ != nil)
   assert(e.sons[2].typ != nil)
@@ -523,7 +520,7 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
   # later via 'chckRange'
   let t = e.typ.skipTypes(abstractRange)
   if optOverflowCheck notin p.options:
-    let res = opr[m] % [getTypeDesc(p.module, e.typ), rdLoc(a), rdLoc(b)]
+    let res = "($1)($2 $3 $4)" % [getTypeDesc(p.module, e.typ), rdLoc(a), rope(opr[m]), rdLoc(b)]
     putIntoDest(p, d, e, res)
   else:
     let res = binaryArithOverflowRaw(p, t, a, b,
@@ -531,11 +528,6 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
     putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res])
 
 proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
-  const
-    opr: array[mUnaryMinusI..mAbsI, string] = [
-      mUnaryMinusI: "((NI$2)-($1))",
-      mUnaryMinusI64: "-($1)",
-      mAbsI: "($1 > 0? ($1) : -($1))"]
   var
     a: TLoc
     t: PType
@@ -545,54 +537,17 @@ proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
   if optOverflowCheck in p.options:
     linefmt(p, cpsStmts, "if ($1 == $2) #raiseOverflow();$n",
             [rdLoc(a), intLiteral(firstOrd(p.config, t))])
-  putIntoDest(p, d, e, opr[m] % [rdLoc(a), rope(getSize(p.config, t) * 8)])
+  case m
+  of mUnaryMinusI:
+    putIntoDest(p, d, e, "((NI$2)-($1))" % [rdLoc(a), rope(getSize(p.config, t) * 8)])
+  of mUnaryMinusI64:
+    putIntoDest(p, d, e, "-($1)" % [rdLoc(a)])
+  of mAbsI:
+    putIntoDest(p, d, e, "($1 > 0? ($1) : -($1))" % [rdLoc(a)])
+  else:
+    assert(false, $m)
 
 proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
-  const
-    binArithTab: array[mAddF64..mXor, string] = [
-      "(($4)($1) + ($4)($2))", # AddF64
-      "(($4)($1) - ($4)($2))", # SubF64
-      "(($4)($1) * ($4)($2))", # MulF64
-      "(($4)($1) / ($4)($2))", # DivF64
-      "($4)((NU$5)($1) >> (NU$3)($2))", # ShrI
-      "($4)((NU$3)($1) << (NU$3)($2))", # ShlI
-      "($4)((NI$3)($1) >> (NU$3)($2))", # AshrI
-      "($4)($1 & $2)",      # BitandI
-      "($4)($1 | $2)",      # BitorI
-      "($4)($1 ^ $2)",      # BitxorI
-      "(($1 <= $2) ? $1 : $2)", # MinI
-      "(($1 >= $2) ? $1 : $2)", # MaxI
-      "(($1 <= $2) ? $1 : $2)", # MinF64
-      "(($1 >= $2) ? $1 : $2)", # MaxF64
-      "($4)((NU$3)($1) + (NU$3)($2))", # AddU
-      "($4)((NU$3)($1) - (NU$3)($2))", # SubU
-      "($4)((NU$3)($1) * (NU$3)($2))", # MulU
-      "($4)((NU$3)($1) / (NU$3)($2))", # DivU
-      "($4)((NU$3)($1) % (NU$3)($2))", # ModU
-      "($1 == $2)",           # EqI
-      "($1 <= $2)",           # LeI
-      "($1 < $2)",            # LtI
-      "($1 == $2)",           # EqF64
-      "($1 <= $2)",           # LeF64
-      "($1 < $2)",            # LtF64
-      "((NU$3)($1) <= (NU$3)($2))", # LeU
-      "((NU$3)($1) < (NU$3)($2))", # LtU
-      "((NU64)($1) <= (NU64)($2))", # LeU64
-      "((NU64)($1) < (NU64)($2))", # LtU64
-      "($1 == $2)",           # EqEnum
-      "($1 <= $2)",           # LeEnum
-      "($1 < $2)",            # LtEnum
-      "((NU8)($1) == (NU8)($2))", # EqCh
-      "((NU8)($1) <= (NU8)($2))", # LeCh
-      "((NU8)($1) < (NU8)($2))", # LtCh
-      "($1 == $2)",           # EqB
-      "($1 <= $2)",           # LeB
-      "($1 < $2)",            # LtB
-      "($1 == $2)",           # EqRef
-      "($1 == $2)",           # EqPtr
-      "($1 <= $2)",           # LePtr
-      "($1 < $2)",            # LtPtr
-      "($1 != $2)"]           # Xor
   var
     a, b: TLoc
     s, k: BiggestInt
@@ -603,9 +558,59 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   # BUGFIX: cannot use result-type here, as it may be a boolean
   s = max(getSize(p.config, a.t), getSize(p.config, b.t)) * 8
   k = getSize(p.config, a.t) * 8
-  putIntoDest(p, d, e,
+
-              binArithTab[op] % [rdLoc(a), rdLoc(b), rope(s),
+  template applyFormat(frmt: untyped) =
-                                      getSimpleTypeDesc(p.module, e.typ), rope(k)])
+    putIntoDest(p, d, e, frmt % [
+      rdLoc(a), rdLoc(b), rope(s),
+      getSimpleTypeDesc(p.module, e.typ), rope(k)]
+    )
+
+  case op
+  of mAddF64: applyFormat("(($4)($1) + ($4)($2))")
+  of mSubF64: applyFormat("(($4)($1) - ($4)($2))")
+  of mMulF64: applyFormat("(($4)($1) * ($4)($2))")
+  of mDivF64: applyFormat("(($4)($1) / ($4)($2))")
+  of mShrI: applyFormat("($4)((NU$5)($1) >> (NU$3)($2))")
+  of mShlI: applyFormat("($4)((NU$3)($1) << (NU$3)($2))")
+  of mAshrI: applyFormat("($4)((NI$3)($1) >> (NU$3)($2))")
+  of mBitandI: applyFormat("($4)($1 & $2)")
+  of mBitorI: applyFormat("($4)($1 | $2)")
+  of mBitxorI: applyFormat("($4)($1 ^ $2)")
+  of mMinI: applyFormat("(($1 <= $2) ? $1 : $2)")
+  of mMaxI: applyFormat("(($1 >= $2) ? $1 : $2)")
+  of mMinF64: applyFormat("(($1 <= $2) ? $1 : $2)")
+  of mMaxF64: applyFormat("(($1 >= $2) ? $1 : $2)")
+  of mAddU: applyFormat("($4)((NU$3)($1) + (NU$3)($2))")
+  of mSubU: applyFormat("($4)((NU$3)($1) - (NU$3)($2))")
+  of mMulU: applyFormat("($4)((NU$3)($1) * (NU$3)($2))")
+  of mDivU: applyFormat("($4)((NU$3)($1) / (NU$3)($2))")
+  of mModU: applyFormat("($4)((NU$3)($1) % (NU$3)($2))")
+  of mEqI: applyFormat("($1 == $2)")
+  of mLeI: applyFormat("($1 <= $2)")
+  of mLtI: applyFormat("($1 < $2)")
+  of mEqF64: applyFormat("($1 == $2)")
+  of mLeF64: applyFormat("($1 <= $2)")
+  of mLtF64: applyFormat("($1 < $2)")
+  of mLeU: applyFormat("((NU$3)($1) <= (NU$3)($2))")
+  of mLtU: applyFormat("((NU$3)($1) < (NU$3)($2))")
+  of mLeU64: applyFormat("((NU64)($1) <= (NU64)($2))")
+  of mLtU64: applyFormat("((NU64)($1) < (NU64)($2))")
+  of mEqEnum: applyFormat("($1 == $2)")
+  of mLeEnum: applyFormat("($1 <= $2)")
+  of mLtEnum: applyFormat("($1 < $2)")
+  of mEqCh: applyFormat("((NU8)($1) == (NU8)($2))")
+  of mLeCh: applyFormat("((NU8)($1) <= (NU8)($2))")
+  of mLtCh: applyFormat("((NU8)($1) < (NU8)($2))")
+  of mEqB: applyFormat("($1 == $2)")
+  of mLeB: applyFormat("($1 <= $2)")
+  of mLtB: applyFormat("($1 < $2)")
+  of mEqRef: applyFormat("($1 == $2)")
+  of mEqUntracedRef: applyFormat("($1 == $2)")
+  of mLePtr: applyFormat("($1 <= $2)")
+  of mLtPtr: applyFormat("($1 < $2)")
+  of mXor: applyFormat("($1 != $2)")
+  else:
+    assert(false, $op)
 
 proc genEqProc(p: BProc, e: PNode, d: var TLoc) =
   var a, b: TLoc
@@ -628,7 +633,8 @@ proc genIsNil(p: BProc, e: PNode, d: var TLoc) =
 
 proc unaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   const
-    unArithTab: array[mNot..mToBiggestInt, string] = ["!($1)", # Not
+    unArithTab: array[mNot..mToBiggestInt, string] = [
+      "!($1)", # Not
       "$1",                   # UnaryPlusI
       "($3)((NU$2) ~($1))",   # BitnotI
       "$1",                   # UnaryPlusF64
@@ -654,10 +660,58 @@ proc unaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   assert(e.sons[1].typ != nil)
   initLocExpr(p, e.sons[1], a)
   t = skipTypes(e.typ, abstractRange)
-  putIntoDest(p, d, e,
+
-              unArithTab[op] % [rdLoc(a), rope(getSize(p.config, t) * 8),
+
+  template applyFormat(frmt: untyped) =
+    putIntoDest(p, d, e, frmt % [rdLoc(a), rope(getSize(p.config, t) * 8),
                 getSimpleTypeDesc(p.module, e.typ)])
 
+
+  case op
+  of mNot:
+    applyFormat("!($1)")
+  of mUnaryPlusI:
+    applyFormat("$1")
+  of mBitnotI:
+    applyFormat("($3)((NU$2) ~($1))")
+  of mUnaryPlusF64:
+    applyFormat("$1")
+  of mUnaryMinusF64:
+    applyFormat("-($1)")
+  of mAbsF64:
+    applyFormat("($1 < 0? -($1) : ($1))")
+    # BUGFIX: fabs() makes problems for Tiny C
+  of mZe8ToI:
+    applyFormat("(($3)(NU)(NU8)($1))")
+  of mZe8ToI64:
+    applyFormat("(($3)(NU64)(NU8)($1))")
+  of mZe16ToI:
+    applyFormat("(($3)(NU)(NU16)($1))")
+  of mZe16ToI64:
+    applyFormat("(($3)(NU64)(NU16)($1))")
+  of mZe32ToI64:
+    applyFormat("(($3)(NU64)(NU32)($1))")
+  of mZeIToI64:
+    applyFormat("(($3)(NU64)(NU)($1))")
+  of mToU8:
+    applyFormat("(($3)(NU8)(NU)($1))")
+  of mToU16:
+    applyFormat("(($3)(NU16)(NU)($1))")
+  of mToU32:
+    applyFormat("(($3)(NU32)(NU64)($1))")
+  of mToFloat:
+    applyFormat("((double) ($1))")
+  of mToBiggestFloat:
+    applyFormat("((double) ($1))")
+  of mToInt:
+    applyFormat("float64ToInt32($1)")
+  of mToBiggestInt:
+    applyFormat("float64ToInt64($1)")
+  else:
+    assert false, $op
+
+
+
 proc isCppRef(p: BProc; typ: PType): bool {.inline.} =
   result = p.module.compileToCpp and
       skipTypes(typ, abstractInstOwned).kind == tyVar and
@@ -942,6 +996,23 @@ proc genBracketExpr(p: BProc; n: PNode; d: var TLoc) =
   of tyTuple: genTupleElem(p, n, d)
   else: internalError(p.config, n.info, "expr(nkBracketExpr, " & $ty.kind & ')')
 
+proc isSimpleExpr(n: PNode): bool =
+  # calls all the way down --> can stay expression based
+  case n.kind
+  of nkCallKinds, nkDotExpr, nkPar, nkTupleConstr,
+      nkObjConstr, nkBracket, nkCurly, nkHiddenDeref, nkDerefExpr, nkHiddenAddr,
+      nkHiddenStdConv, nkHiddenSubConv, nkConv, nkAddr:
+    for c in n:
+      if not isSimpleExpr(c): return false
+    result = true
+  of nkStmtListExpr:
+    for i in 0..n.len-2:
+      if n[i].kind notin {nkCommentStmt, nkEmpty}: return false
+    result = isSimpleExpr(n.lastSon)
+  else:
+    if n.isAtom:
+      result = true
+
 proc genAndOr(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
   # how to generate code?
   #  'expr1 and expr2' becomes:
@@ -963,24 +1034,41 @@ proc genAndOr(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
   # tmp = a
   # end:
   # a = tmp
-  var
+  when false:
-    L: TLabel
+    #if isSimpleExpr(e) and p.module.compileToCpp:
-    tmp: TLoc
+    var tmpA, tmpB: TLoc
-  getTemp(p, e.typ, tmp)      # force it into a temp!
+    #getTemp(p, e.typ, tmpA)
-  inc p.splitDecls
+    #getTemp(p, e.typ, tmpB)
-  expr(p, e.sons[1], tmp)
+    initLocExprSingleUse(p, e.sons[1], tmpA)
-  L = getLabel(p)
+    initLocExprSingleUse(p, e.sons[2], tmpB)
-  if m == mOr:
+    tmpB.k = locExpr
-    lineF(p, cpsStmts, "if ($1) goto $2;$n", [rdLoc(tmp), L])
+    if m == mOr:
+      tmpB.r = "((" & rdLoc(tmpA) & ")||(" & rdLoc(tmpB) & "))"
+    else:
+      tmpB.r = "((" & rdLoc(tmpA) & ")&&(" & rdLoc(tmpB) & "))"
+    if d.k == locNone:
+      d = tmpB
+    else:
+      genAssignment(p, d, tmpB, {})
   else:
-    lineF(p, cpsStmts, "if (!($1)) goto $2;$n", [rdLoc(tmp), L])
+    var
-  expr(p, e.sons[2], tmp)
+      L: TLabel
-  fixLabel(p, L)
+      tmp: TLoc
-  if d.k == locNone:
+    getTemp(p, e.typ, tmp)      # force it into a temp!
-    d = tmp
+    inc p.splitDecls
-  else:
+    expr(p, e.sons[1], tmp)
-    genAssignment(p, d, tmp, {}) # no need for deep copying
+    L = getLabel(p)
-  dec p.splitDecls
+    if m == mOr:
+      lineF(p, cpsStmts, "if ($1) goto $2;$n", [rdLoc(tmp), L])
+    else:
+      lineF(p, cpsStmts, "if (!($1)) goto $2;$n", [rdLoc(tmp), L])
+    expr(p, e.sons[2], tmp)
+    fixLabel(p, L)
+    if d.k == locNone:
+      d = tmp
+    else:
+      genAssignment(p, d, tmp, {}) # no need for deep copying
+    dec p.splitDecls
 
 proc genEcho(p: BProc, n: PNode) =
   # this unusal way of implementing it ensures that e.g. ``echo("hallo", 45)``
@@ -1675,7 +1763,7 @@ proc fewCmps(conf: ConfigRef; s: PNode): bool =
   else:
     result = sonsLen(s) <= 8  # 8 seems to be a good value
 
-proc binaryExprIn(p: BProc, e: PNode, a, b, d: var TLoc, frmt: string) =
+template binaryExprIn(p: BProc, e: PNode, a, b, d: var TLoc, frmt: string) =
   putIntoDest(p, d, e, frmt % [rdLoc(a), rdSetElemLoc(p.config, b, a.t)])
 
 proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) =
@@ -1686,7 +1774,7 @@ proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) =
   of 8: binaryExprIn(p, e, a, b, d, "(($1 &((NU64)1<<((NU)($2)&63U)))!=0)")
   else: binaryExprIn(p, e, a, b, d, "(($1[(NU)($2)>>3] &(1U<<((NU)($2)&7U)))!=0)")
 
-proc binaryStmtInExcl(p: BProc, e: PNode, d: var TLoc, frmt: string) =
+template binaryStmtInExcl(p: BProc, e: PNode, d: var TLoc, frmt: string) =
   var a, b: TLoc
   assert(d.k == locNone)
   initLocExpr(p, e.sons[1], a)
@@ -1753,13 +1841,19 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of 1, 2, 4, 8:
     case op
     of mIncl:
-      var ts = "NU" & $(size * 8)
+      case size
-      binaryStmtInExcl(p, e, d,
+      of 1: binaryStmtInExcl(p, e, d, "$1 |= ((NU8)1)<<(($2) & 7);$n")
-          "$1 |= ((" & ts & ")1)<<(($2)%(sizeof(" & ts & ")*8));$n")
+      of 2: binaryStmtInExcl(p, e, d, "$1 |= ((NU16)1)<<(($2) & 15);$n")
+      of 4: binaryStmtInExcl(p, e, d, "$1 |= ((NU32)1)<<(($2) & 31);$n")
+      of 8: binaryStmtInExcl(p, e, d, "$1 |= ((NU64)1)<<(($2) & 63);$n")
+      else: assert(false, $size)
     of mExcl:
-      var ts = "NU" & $(size * 8)
+      case size
-      binaryStmtInExcl(p, e, d, "$1 &= ~(((" & ts & ")1) << (($2) % (sizeof(" &
+      of 1: binaryStmtInExcl(p, e, d, "$1 &= ~(((NU8)1) << (($2) & 7));$n")
-          ts & ")*8)));$n")
+      of 2: binaryStmtInExcl(p, e, d, "$1 &= ~(((NU16)1) << (($2) & 15));$n")
+      of 4: binaryStmtInExcl(p, e, d, "$1 &= ~(((NU32)1) << (($2) & 31));$n")
+      of 8: binaryStmtInExcl(p, e, d, "$1 &= ~(((NU64)1) << (($2) & 63));$n")
+      else: assert(false, $size)
     of mCard:
       if size <= 4: unaryExprChar(p, e, d, "#countBits32($1)")
       else: unaryExprChar(p, e, d, "#countBits64($1)")
@@ -1824,14 +1918,14 @@ proc genSomeCast(p: BProc, e: PNode, d: var TLoc) =
   var a: TLoc
   initLocExpr(p, e.sons[1], a)
   let etyp = skipTypes(e.typ, abstractRange+{tyOwned})
+  let srcTyp = skipTypes(e.sons[1].typ, abstractRange)
   if etyp.kind in ValueTypes and lfIndirect notin a.flags:
     putIntoDest(p, d, e, "(*($1*) ($2))" %
         [getTypeDesc(p.module, e.typ), addrLoc(p.config, a)], a.storage)
-  elif etyp.kind == tyProc and etyp.callConv == ccClosure:
+  elif etyp.kind == tyProc and etyp.callConv == ccClosure and srcTyp.callConv != ccClosure:
     putIntoDest(p, d, e, "(($1) ($2))" %
         [getClosureType(p.module, etyp, clHalfWithEnv), rdCharLoc(a)], a.storage)
   else:
-    let srcTyp = skipTypes(e.sons[1].typ, abstractRange)
     # C++ does not like direct casts from pointer to shorter integral types
     if srcTyp.kind in {tyPtr, tyPointer} and etyp.kind in IntegralTypes:
       putIntoDest(p, d, e, "(($1) (ptrdiff_t) ($2))" %
@@ -2085,7 +2179,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mCharToStr: genDollar(p, e, d, "#nimCharToStr($1)")
   of mFloatToStr: genDollar(p, e, d, "#nimFloatToStr($1)")
   of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)")
-  of mStrToStr: expr(p, e.sons[1], d)
+  of mStrToStr, mUnown: expr(p, e.sons[1], d)
   of mEnumToStr:
     if optNimV2 in p.config.globalOptions:
       genEnumToStr(p, e, d)
@@ -2228,14 +2322,14 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) =
           initLocExpr(p, it.sons[0], a)
           initLocExpr(p, it.sons[1], b)
           lineF(p, cpsStmts, "for ($1 = $3; $1 <= $4; $1++) $n" &
-              "$2 |=((" & ts & ")(1)<<(($1)%(sizeof(" & ts & ")*8)));$n", [
+              "$2 |=(($5)(1)<<(($1)%(sizeof($5)*8)));$n", [
               rdLoc(idx), rdLoc(d), rdSetElemLoc(p.config, a, e.typ),
-              rdSetElemLoc(p.config, b, e.typ)])
+              rdSetElemLoc(p.config, b, e.typ), rope(ts)])
         else:
           initLocExpr(p, it, a)
           lineF(p, cpsStmts,
-               "$1 |=((" & ts & ")(1)<<(($2)%(sizeof(" & ts & ")*8)));$n",
+               "$1 |=(($3)(1)<<(($2)%(sizeof($3)*8)));$n",
-               [rdLoc(d), rdSetElemLoc(p.config, a, e.typ)])
+               [rdLoc(d), rdSetElemLoc(p.config, a, e.typ), rope(ts)])
 
 proc genTupleConstr(p: BProc, n: PNode, d: var TLoc) =
   var rec: TLoc
@@ -2609,7 +2703,10 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
             genProc(p.module, prc)
   of nkParForStmt: genParForStmt(p, n)
   of nkState: genState(p, n)
-  of nkGotoState: genGotoState(p, n)
+  of nkGotoState:
+    # simply never set it back to 0 here from here on...
+    inc p.splitDecls
+    genGotoState(p, n)
   of nkBreakState: genBreakState(p, n, d)
   else: internalError(p.config, n.info, "expr(" & $n.kind & "); unknown node kind")
 

compiler/ccgstmts.nim

@@ -207,7 +207,7 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
     let tryStmt = p.nestedTryStmts.pop
     if not p.module.compileToCpp or optNoCppExceptions in p.config.globalOptions:
       # Pop safe points generated by try
-      if not tryStmt.inExcept and not isDefined(p.config, "nimQuirky"):
+      if not tryStmt.inExcept:
         linefmt(p, cpsStmts, "#popSafePoint();$n", [])
 
     # Pop this try-stmt of the list of nested trys
@@ -227,8 +227,9 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
   if not p.module.compileToCpp or optNoCppExceptions in p.config.globalOptions:
     # Pop exceptions that was handled by the
     # except-blocks we are in
-    for i in countdown(howManyExcepts-1, 0):
+    if not p.noSafePoints:
-      linefmt(p, cpsStmts, "#popCurrentException();$n", [])
+      for i in countdown(howManyExcepts-1, 0):
+        linefmt(p, cpsStmts, "#popCurrentException();$n", [])
 
 proc genGotoState(p: BProc, n: PNode) =
   # we resist the temptation to translate it into duff's device as it later
@@ -449,7 +450,7 @@ proc genReturnStmt(p: BProc, t: PNode) =
   blockLeaveActions(p,
     howManyTrys    = p.nestedTryStmts.len,
     howManyExcepts = p.inExceptBlockLen)
-  if (p.finallySafePoints.len > 0) and not isDefined(p.config, "nimQuirky"):
+  if (p.finallySafePoints.len > 0) and not p.noSafePoints:
     # If we're in a finally block, and we came here by exception
     # consume it before we return.
     var safePoint = p.finallySafePoints[p.finallySafePoints.len-1]
@@ -1004,6 +1005,8 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
       (t.kind == nkHiddenTryStmt and sfSystemModule in p.module.module.flags)
   if not quirkyExceptions:
     p.module.includeHeader("<setjmp.h>")
+  else:
+    p.noSafePoints = true
   genLineDir(p, t)
   discard cgsym(p.module, "Exception")
   var safePoint: Rope
@@ -1021,7 +1024,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
     startBlock(p, "if ($1.status == 0) {$n", [safePoint])
   var length = sonsLen(t)
-  add(p.nestedTryStmts, (t, false))
+  add(p.nestedTryStmts, (t, quirkyExceptions))
   expr(p, t.sons[0], d)
   if not quirkyExceptions:
     linefmt(p, cpsStmts, "#popSafePoint();$n", [])
@@ -1135,9 +1138,9 @@ proc genAsmStmt(p: BProc, t: PNode) =
   # work:
   if p.prc == nil:
     # top level asm statement?
-    addf(p.module.s[cfsProcHeaders], CC[p.config.cCompiler].asmStmtFrmt, [s])
+    add(p.module.s[cfsProcHeaders], runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s]))
   else:
-    lineF(p, cpsStmts, CC[p.config.cCompiler].asmStmtFrmt, [s])
+    add(p.s(cpsStmts), indentLine(p, runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s])))
 
 proc determineSection(n: PNode): TCFileSection =
   result = cfsProcHeaders

compiler/ccgtypes.nim

@@ -338,12 +338,17 @@ proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope =
     if result == nil: result = cacheGetType(m.typeCache, sig)
 
 proc structOrUnion(t: PType): Rope =
+  let cachedUnion {.global.} = rope("union")
+  let cachedStruct {.global.} = rope("struct")
   let t = t.skipTypes({tyAlias, tySink})
-  (if tfUnion in t.flags: rope("union") else: rope("struct"))
+  if tfUnion in t.flags: cachedUnion
+  else: cachedStruct
 
-proc getForwardStructFormat(m: BModule): string =
+proc addForwardStructFormat(m: BModule, structOrUnion: Rope, typename: Rope) =
-  if m.compileToCpp: result = "$1 $2;$n"
+  if m.compileToCpp:
-  else: result = "typedef $1 $2 $2;$n"
+    m.s[cfsForwardTypes].addf "$1 $2;$n", [structOrUnion, typename]
+  else:
+    m.s[cfsForwardTypes].addf "typedef $1 $2 $2;$n", [structOrUnion, typename]
 
 proc seqStar(m: BModule): string =
   if m.config.selectedGC == gcDestructors: result = ""
@@ -360,8 +365,7 @@ proc getTypeForward(m: BModule, typ: PType; sig: SigHash): Rope =
     result = getTypeName(m, typ, sig)
     m.forwTypeCache[sig] = result
     if not isImportedType(concrete):
-      addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
+      addForwardStructFormat(m, structOrUnion(typ), result)
-          [structOrUnion(typ), result])
     else:
       pushType(m, concrete)
     doAssert m.forwTypeCache[sig] == result
@@ -733,8 +737,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
     if result == nil:
       result = getTypeName(m, origTyp, sig)
       if not isImportedType(t):
-        addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
+        addForwardStructFormat(m, structOrUnion(t), result)
-            [structOrUnion(t), result])
       m.forwTypeCache[sig] = result
     assert(cacheGetType(m.typeCache, sig) == nil)
     m.typeCache[sig] = result & seqStar(m)
@@ -845,8 +848,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
         result = getTypeName(m, origTyp, sig)
         m.forwTypeCache[sig] = result
         if not isImportedType(t):
-          addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
+          addForwardStructFormat(m, structOrUnion(t), result)
-             [structOrUnion(t), result])
         assert m.forwTypeCache[sig] == result
       m.typeCache[sig] = result # always call for sideeffects:
       if not incompleteType(t):
@@ -905,7 +907,8 @@ proc finishTypeDescriptions(m: BModule) =
     discard getTypeDesc(m, m.typeStack[i])
     inc(i)
 
-template cgDeclFrmt*(s: PSym): string = s.constraint.strVal
+template cgDeclFrmt*(s: PSym): string =
+  s.constraint.strVal
 
 proc isReloadable(m: BModule, prc: PSym): bool =
   return m.hcrOn and sfNonReloadable notin prc.flags
@@ -943,7 +946,7 @@ proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope =
          params])
   else:
     let asPtrStr = if asPtr: (rope("(*") & name & ")") else: name
-    result = prc.cgDeclFrmt % [rettype, asPtrStr, params]
+    result = runtimeFormat(prc.cgDeclFrmt, [rettype, asPtrStr, params])
 
 # ------------------ type info generation -------------------------------------
 
@@ -1346,10 +1349,10 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
     # results are not deterministic!
     genTupleInfo(m, t, origType, result, info)
   else: internalError(m.config, "genTypeInfo(" & $t.kind & ')')
-  if t.deepCopy != nil:
+  if t.attachedOps[attachedDeepCopy] != nil:
-    genDeepCopyProc(m, t.deepCopy, result)
+    genDeepCopyProc(m, t.attachedOps[attachedDeepCopy], result)
-  elif origType.deepCopy != nil:
+  elif origType.attachedOps[attachedDeepCopy] != nil:
-    genDeepCopyProc(m, origType.deepCopy, result)
+    genDeepCopyProc(m, origType.attachedOps[attachedDeepCopy], result)
   result = prefixTI.rope & result & ")".rope
 
 proc genTypeSection(m: BModule, n: PNode) =

compiler/cgen.nim

@@ -206,15 +206,15 @@ proc indentLine(p: BProc, r: Rope): Rope =
     prepend(result, "\t".rope)
 
 template appcg(m: BModule, c: var Rope, frmt: FormatStr,
-           args: varargs[untyped]) =
+           args: untyped) =
   add(c, ropecg(m, frmt, args))
 
 template appcg(m: BModule, sec: TCFileSection, frmt: FormatStr,
-           args: varargs[untyped]) =
+           args: untyped) =
   add(m.s[sec], ropecg(m, frmt, args))
 
 template appcg(p: BProc, sec: TCProcSection, frmt: FormatStr,
-           args: varargs[untyped]) =
+           args: untyped) =
   add(p.s(sec), ropecg(p.module, frmt, args))
 
 template line(p: BProc, sec: TCProcSection, r: Rope) =
@@ -224,7 +224,7 @@ template line(p: BProc, sec: TCProcSection, r: string) =
   add(p.s(sec), indentLine(p, r.rope))
 
 template lineF(p: BProc, sec: TCProcSection, frmt: FormatStr,
-              args: openarray[Rope]) =
+              args: untyped) =
   add(p.s(sec), indentLine(p, frmt % args))
 
 template lineCg(p: BProc, sec: TCProcSection, frmt: FormatStr,
@@ -440,6 +440,15 @@ proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
   result.flags = {}
   constructLoc(p, result, not needsInit)
 
+proc getTempCpp(p: BProc, t: PType, result: var TLoc; value: Rope) =
+  inc(p.labels)
+  result.r = "T" & rope(p.labels) & "_"
+  linefmt(p, cpsStmts, "$1 $2 = $3;$n", [getTypeDesc(p.module, t), result.r, value])
+  result.k = locTemp
+  result.lode = lodeTyp t
+  result.storage = OnStack
+  result.flags = {}
+
 proc getIntTemp(p: BProc, result: var TLoc) =
   inc(p.labels)
   result.r = "T" & rope(p.labels) & "_"
@@ -484,7 +493,7 @@ proc localVarDecl(p: BProc; n: PNode): Rope =
     add(result, " ")
     add(result, s.loc.r)
   else:
-    result = s.cgDeclFrmt % [result, s.loc.r]
+    result = runtimeFormat(s.cgDeclFrmt, [result, s.loc.r])
 
 proc assignLocalVar(p: BProc, n: PNode) =
   #assert(s.loc.k == locNone) # not yet assigned
@@ -535,7 +544,7 @@ proc assignGlobalVar(p: BProc, n: PNode) =
         if sfVolatile in s.flags: add(decl, " volatile")
         addf(decl, " $1;$n", [s.loc.r])
       else:
-        decl = (s.cgDeclFrmt & ";$n") % [td, s.loc.r]
+        decl = runtimeFormat(s.cgDeclFrmt & ";$n", [td, s.loc.r])
       add(p.module.s[cfsVars], decl)
   if p.withinLoop > 0:
     # fixes tests/run/tzeroarray:

compiler/cgendata.nim

@@ -70,6 +70,7 @@ type
     threadVarAccessed*: bool  # true if the proc already accessed some threadvar
     hasCurFramePointer*: bool # true if _nimCurFrame var needed to recover after
                               # exception is generated
+    noSafePoints*: bool       # the proc doesn't use safe points in exception handling
     lastLineInfo*: TLineInfo  # to avoid generating excessive 'nimln' statements
     currLineInfo*: TLineInfo  # AST codegen will make this superfluous
     nestedTryStmts*: seq[tuple[n: PNode, inExcept: bool]]

compiler/dfa.nim

@@ -567,19 +567,36 @@ proc genReturn(c: var Con; n: PNode) =
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
-  PathKinds* = {nkDotExpr, nkCheckedFieldExpr,
+  PathKinds0 = {nkDotExpr, nkCheckedFieldExpr,
                 nkBracketExpr, nkDerefExpr, nkHiddenDeref,
                 nkAddr, nkHiddenAddr,
-                nkHiddenStdConv, nkHiddenSubConv, nkObjDownConv, nkObjUpConv}
+                nkObjDownConv, nkObjUpConv}
+  PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv}
+
+proc getRoot(n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of PathKinds0:
+      result = result[0]
+    of PathKinds1:
+      result = result[1]
+    else: break
+
+proc skipConvDfa*(n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of nkObjDownConv, nkObjUpConv:
+      result = result[0]
+    of PathKinds1:
+      result = result[1]
+    else: break
 
 proc genUse(c: var Con; orig: PNode) =
-  var n = orig
+  let n = dfa.getRoot(orig)
-  var iters = 0
-  while n.kind in PathKinds:
-    n = n[0]
-    inc iters
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: orig, kind: use, sym: if iters > 0: nil else: n.sym)
+    c.code.add Instr(n: orig, kind: use, sym: if orig != n: nil else: n.sym)
 
 proc aliases(obj, field: PNode): bool =
   var n = field
@@ -590,7 +607,7 @@ proc aliases(obj, field: PNode): bool =
     if sameTrees(obj, n): return true
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv, nkHiddenDeref:
+       nkObjDownConv, nkObjUpConv, nkHiddenDeref, nkDerefExpr:
       n = n[0]
     of nkBracketExpr:
       let x = n[0]
@@ -616,13 +633,19 @@ proc instrTargets*(ins: Instr; loc: PNode): bool =
     # use x; question does it affect 'x.f'? Yes.
     result = aliases(ins.n, loc) or aliases(loc, ins.n)
 
-proc isAnalysableFieldAccess*(n: PNode; owner: PSym): bool =
+proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
-  var n = n
+  var n = orig
   while true:
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv, nkHiddenDeref:
+       nkObjDownConv, nkObjUpConv:
       n = n[0]
+    of nkHiddenDeref, nkDerefExpr:
+      # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
+      # pointer indirection.
+      n = n[0]
+      return n.kind == nkSym and n.sym.owner == owner and (isSinkParam(n.sym) or
+          n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned, tyVar})
     of nkBracketExpr:
       let x = n[0]
       if x.typ != nil and x.typ.skipTypes(abstractInst).kind == tyTuple:
@@ -633,11 +656,25 @@ proc isAnalysableFieldAccess*(n: PNode; owner: PSym): bool =
       break
   # XXX Allow closure deref operations here if we know
   # the owner controlled the closure allocation?
-  result = n.kind == nkSym and n.sym.owner == owner and owner.kind != skModule
+  result = n.kind == nkSym and n.sym.owner == owner and
+    owner.kind != skModule and
+    (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
+  # Note: There is a different move analyzer possible that checks for
+  # consume(param.key); param.key = newValue  for all paths. Then code like
+  #
+  #   let splited = split(move self.root, x)
+  #   self.root = merge(splited.lower, splited.greater)
+  #
+  # could be written without the ``move self.root``. However, this would be
+  # wrong! Then the write barrier for the ``self.root`` assignment would
+  # free the old data and all is lost! Lesson: Don't be too smart, trust the
+  # lower level C++ optimizer to specialize this code.
 
 proc genDef(c: var Con; n: PNode) =
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
     c.code.add Instr(n: n, kind: def, sym: n.sym)
+  elif isAnalysableFieldAccess(n, c.owner):
+    c.code.add Instr(n: n, kind: def, sym: nil)
 
 proc canRaise(fn: PNode): bool =
   const magicsThatCanRaise = {
@@ -715,7 +752,7 @@ proc gen(c: var Con; n: PNode) =
     # "uses" 'i'. But we are only talking about builtin array indexing so
     # it doesn't matter and 'x = 34' is NOT a usage of 'x'.
     genDef(c, n[0])
-  of PathKinds:
+  of PathKinds0 - {nkHiddenStdConv, nkHiddenSubConv, nkObjDownConv, nkObjUpConv}:
     genUse(c, n)
   of nkIfStmt, nkIfExpr: genIf(c, n)
   of nkWhenStmt:
@@ -732,8 +769,8 @@ proc gen(c: var Con; n: PNode) =
      nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr:
     for x in n: gen(c, x)
   of nkPragmaBlock: gen(c, n.lastSon)
-  of nkDiscardStmt: gen(c, n.sons[0])
+  of nkDiscardStmt, nkObjDownConv, nkObjUpConv: gen(c, n.sons[0])
-  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast:
+  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, nkHiddenSubConv, nkHiddenStdConv:
     gen(c, n.sons[1])
   of nkStringToCString, nkCStringToString: gen(c, n.sons[0])
   of nkVarSection, nkLetSection: genVarSection(c, n)

compiler/docgen.nim

@@ -172,7 +172,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
                                 outExt, RelativeDir"htmldocs", false)
   result.thisDir = result.destFile.splitFile.dir
 
-proc dispA(conf: ConfigRef; dest: var Rope, xml, tex: string, args: openArray[Rope]) =
+template dispA(conf: ConfigRef; dest: var Rope, xml, tex: string, args: openArray[Rope]) =
   if conf.cmd != cmdRst2tex: addf(dest, xml, args)
   else: addf(dest, tex, args)
 

compiler/docgen2.nim

@@ -20,11 +20,12 @@ type
   TGen = object of PPassContext
     doc: PDoc
     module: PSym
+    config: ConfigRef
   PGen = ref TGen
 
 template shouldProcess(g): bool =
   (g.module.owner.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or
-      sfMainModule in g.module.flags
+      sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =
   var g = PGen(p)
@@ -60,6 +61,7 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   var g: PGen
   new(g)
   g.module = module
+  g.config = graph.config
   var d = newDocumentor(AbsoluteFile toFullPath(graph.config, FileIndex module.position),
       graph.cache, graph.config, ext)
   d.hasToc = true

compiler/extccomp.nim

@@ -376,7 +376,7 @@ proc nameToCC*(name: string): TSystemCC =
   result = ccNone
 
 proc isVSCompatible*(conf: ConfigRef): bool =
-  return conf.cCompiler == ccVcc or 
+  return conf.cCompiler == ccVcc or
           conf.cCompiler == ccClangCl or
           (conf.cCompiler == ccIcl and conf.target.hostOS in osDos..osWindows)
 
@@ -738,7 +738,7 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
     # way of being able to debug and rebuild the program at the same time. This
     # is accomplished using the /PDB:<filename> flag (there also exists the
     # /PDBALTPATH:<filename> flag). The only downside is that the .pdb files are
-    # atleast 300kb big (when linking statically to the runtime - or else 5mb+) 
+    # atleast 300kb big (when linking statically to the runtime - or else 5mb+)
     # and will quickly accumulate. There is a hacky solution: we could try to
     # delete all .pdb files with a pattern and swallow exceptions.
     #
@@ -910,7 +910,8 @@ proc callCCompiler*(conf: ConfigRef) =
                        else: AbsoluteFile(conf.projectName)
       linkCmd = getLinkCmd(conf, mainOutput, objfiles)
       if optCompileOnly notin conf.globalOptions:
-        if defined(windows) and linkCmd.len > 8_000:
+        const MaxCmdLen = when defined(windows): 8_000 else: 32_000
+        if linkCmd.len > MaxCmdLen:
           # Windows's command line limit is about 8K (don't laugh...) so C compilers on
           # Windows support a feature where the command line can be passed via ``@linkcmd``
           # to them.

compiler/injectdestructors.nim

@@ -136,7 +136,7 @@ to do it.
 import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
   strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
-  lineinfos, parampatterns
+  lineinfos, parampatterns, sighashes
 
 const
   InterestingSyms = {skVar, skResult, skLet, skForVar, skTemp}
@@ -195,15 +195,17 @@ proc isLastRead(n: PNode; c: var Con): bool =
   # first we need to search for the instruction that belongs to 'n':
   c.otherRead = nil
   var instr = -1
+  let m = dfa.skipConvDfa(n)
+
   for i in 0..<c.g.len:
     # This comparison is correct and MUST not be ``instrTargets``:
-    if c.g[i].kind == use and c.g[i].n == n:
+    if c.g[i].kind == use and c.g[i].n == m:
       if instr < 0:
         instr = i
         break
 
   dbg:
-    echo "starting point for ", n, " is ", instr
+    echo "starting point for ", n, " is ", instr, " ", n.kind
 
   if instr < 0: return false
   # we go through all paths beginning from 'instr+1' and need to
@@ -314,24 +316,37 @@ proc makePtrType(c: Con, baseType: PType): PType =
   result = newType(tyPtr, c.owner)
   addSonSkipIntLit(result, baseType)
 
-template genOp(opr, opname, ri) =
+proc addDestroy(c: var Con; n: PNode) =
-  let op = opr
+  # append to front:
+  c.destroys = newTree(nkStmtList, n, c.destroys)
+
+proc genOp(c: Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
+  var op = t.attachedOps[kind]
+
   if op == nil:
-    globalError(c.graph.config, dest.info, "internal error: '" & opname &
+    # give up and find the canonical type instead:
+    let h = sighashes.hashType(t, {CoType, CoConsiderOwned})
+    let canon = c.graph.canonTypes.getOrDefault(h)
+    if canon != nil:
+      op = canon.attachedOps[kind]
+
+  if op == nil:
+    globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
       "' operator not found for type " & typeToString(t))
   elif op.ast[genericParamsPos].kind != nkEmpty:
-    globalError(c.graph.config, dest.info, "internal error: '" & opname &
+    globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
       "' operator is generic")
-  if sfError in op.flags: checkForErrorPragma(c, t, ri, opname)
+  if sfError in op.flags: checkForErrorPragma(c, t, ri, AttachedOpToStr[kind])
   let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
   addrExp.add(dest)
   result = newTree(nkCall, newSymNode(op), addrExp)
 
 proc genSink(c: Con; t: PType; dest, ri: PNode): PNode =
   let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  let op = if t.sink != nil: t.sink else: t.assignment
+  let k = if t.attachedOps[attachedSink] != nil: attachedSink
-  if op != nil:
+           else: attachedAsgn
-    genOp(op, "=sink", ri)
+  if t.attachedOps[k] != nil:
+    result = genOp(c, t, k, dest, ri)
   else:
     # in rare cases only =destroy exists but no sink or assignment
     # (see Pony object in tmove_objconstr.nim)
@@ -342,15 +357,15 @@ proc genCopy(c: Con; t: PType; dest, ri: PNode): PNode =
   if tfHasOwned in t.flags:
     checkForErrorPragma(c, t, ri, "=")
   let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  genOp(t.assignment, "=", ri)
+  result = genOp(c, t, attachedAsgn, dest, ri)
 
 proc genCopyNoCheck(c: Con; t: PType; dest, ri: PNode): PNode =
   let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  genOp(t.assignment, "=", ri)
+  result = genOp(c, t, attachedAsgn, dest, ri)
 
 proc genDestroy(c: Con; t: PType; dest: PNode): PNode =
   let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  genOp(t.destructor, "=destroy", nil)
+  result = genOp(c, t, attachedDestructor, dest, nil)
 
 proc addTopVar(c: var Con; v: PNode) =
   c.topLevelVars.add newTree(nkIdentDefs, v, c.emptyNode, c.emptyNode)
@@ -408,6 +423,17 @@ proc sinkParamIsLastReadCheck(c: var Con, s: PNode) =
      localError(c.graph.config, c.otherRead.info, "sink parameter `" & $s.sym.name.s &
          "` is already consumed at " & toFileLineCol(c. graph.config, s.info))
 
+proc isSinkTypeForParam(t: PType): bool =
+  # a parameter like 'seq[owned T]' must not be used only once, but its
+  # elements must, so we detect this case here:
+  result = t.skipTypes({tyGenericInst, tyAlias}).kind in {tySink, tyOwned}
+  when false:
+    if isSinkType(t):
+      if t.skipTypes({tyGenericInst, tyAlias}).kind in {tyArray, tyVarargs, tyOpenArray, tySequence}:
+        result = false
+      else:
+        result = true
+
 proc passCopyToSink(n: PNode; c: var Con): PNode =
   result = newNodeIT(nkStmtListExpr, n.info, n.typ)
   let tmp = getTemp(c, n.typ, n.info)
@@ -441,7 +467,7 @@ proc pArg(arg: PNode; c: var Con; isSink: bool): PNode =
       let L = if parameters != nil: parameters.len else: 0
       result.add arg[0]
       for i in 1..<arg.len:
-        result.add pArg(arg[i], c, i < L and isSinkType(parameters[i]))
+        result.add pArg(arg[i], c, i < L and isSinkTypeForParam(parameters[i]))
     elif arg.kind in {nkBracket, nkObjConstr, nkTupleConstr, nkBracket, nkCharLit..nkTripleStrLit}:
       discard "object construction to sink parameter: nothing to do"
       result = arg
@@ -518,16 +544,22 @@ proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
     let L = if parameters != nil: parameters.len else: 0
     ri2.add ri[0]
     for i in 1..<ri.len:
-      ri2.add pArg(ri[i], c, i < L and isSinkType(parameters[i]))
+      ri2.add pArg(ri[i], c, i < L and isSinkTypeForParam(parameters[i]))
     #recurse(ri, ri2)
     result.add ri2
   of nkBracketExpr:
     if ri[0].kind == nkSym and isUnpackedTuple(ri[0].sym):
       # unpacking of tuple: move out the elements
       result = genSink(c, dest.typ, dest, ri)
+      result.add p(ri, c)
+    elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c):
+      # Rule 3: `=sink`(x, z); wasMoved(z)
+      var snk = genSink(c, dest.typ, dest, ri)
+      snk.add ri
+      result = newTree(nkStmtList, snk, genWasMoved(ri, c))
     else:
       result = genCopy(c, dest.typ, dest, ri)
-    result.add p(ri, c)
+      result.add p(ri, c)
   of nkStmtListExpr:
     result = newNodeI(nkStmtList, ri.info)
     for i in 0..ri.len-2:
@@ -610,7 +642,9 @@ proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
       result = genCopy(c, dest.typ, dest, ri)
       result.add p(ri, c)
   of nkHiddenSubConv, nkHiddenStdConv:
-    if ri[1].kind in movableNodeKinds:
+    if sameType(ri.typ, ri[1].typ):
+      result = moveOrCopy(dest, ri[1], c)
+    elif ri[1].kind in movableNodeKinds:
       result = moveOrCopy(dest, ri[1], c)
       var b = newNodeIT(ri.kind, ri.info, ri.typ)
       b.add ri[0] # add empty node
@@ -673,6 +707,15 @@ proc injectDefaultCalls(n: PNode, c: var Con) =
 proc isCursor(n: PNode): bool {.inline.} =
   result = n.kind == nkSym and sfCursor in n.sym.flags
 
+proc keepVar(n, it: PNode, c: var Con): PNode =
+  # keep the var but transform 'ri':
+  result = copyNode(n)
+  var itCopy = copyNode(it)
+  for j in 0..it.len-2:
+    itCopy.add it[j]
+  itCopy.add p(it[it.len-1], c)
+  result.add itCopy
+
 proc p(n: PNode; c: var Con): PNode =
   case n.kind
   of nkVarSection, nkLetSection:
@@ -694,24 +737,17 @@ proc p(n: PNode; c: var Con): PNode =
             c.addTopVar v
             # make sure it's destroyed at the end of the proc:
             if not isUnpackedTuple(it[0].sym):
-              c.destroys.add genDestroy(c, v.typ, v)
+              c.addDestroy genDestroy(c, v.typ, v)
-            if ri.kind != nkEmpty:
+          if ri.kind != nkEmpty:
-              let r = moveOrCopy(v, ri, c)
+            let r = moveOrCopy(v, ri, c)
-              result.add r
+            result.add r
       else:
-        # keep it, but transform 'ri':
+        result.add keepVar(n, it, c)
-        var varSection = copyNode(n)
-        var itCopy = copyNode(it)
-        for j in 0..L-2:
-          itCopy.add it[j]
-        itCopy.add p(ri, c)
-        varSection.add itCopy
-        result.add varSection
   of nkCallKinds:
     let parameters = n[0].typ
     let L = if parameters != nil: parameters.len else: 0
     for i in 1 ..< n.len:
-      n.sons[i] = pArg(n[i], c, i < L and isSinkType(parameters[i]))
+      n.sons[i] = pArg(n[i], c, i < L and isSinkTypeForParam(parameters[i]))
     if n.typ != nil and hasDestructor(n.typ):
       discard "produce temp creation"
       result = newNodeIT(nkStmtListExpr, n.info, n.typ)
@@ -720,11 +756,11 @@ proc p(n: PNode; c: var Con): PNode =
       sinkExpr.add n
       result.add sinkExpr
       result.add tmp
-      c.destroys.add genDestroy(c, n.typ, tmp)
+      c.addDestroy genDestroy(c, n.typ, tmp)
     else:
       result = n
   of nkAsgn, nkFastAsgn:
-    if hasDestructor(n[0].typ):
+    if hasDestructor(n[0].typ) and n[1].kind notin {nkProcDef, nkDo, nkLambda, nkClosure}:
       result = moveOrCopy(n[0], n[1], c)
     else:
       result = copyNode(n)
@@ -783,8 +819,8 @@ proc injectDestructorCalls*(g: ModuleGraph; owner: PSym; n: PNode): PNode =
     let params = owner.typ.n
     for i in 1 ..< params.len:
       let param = params[i].sym
-      if isSinkParam(param) and hasDestructor(param.typ.skipTypes({tySink})):
+      if isSinkTypeForParam(param.typ) and hasDestructor(param.typ.skipTypes({tySink})):
-        c.destroys.add genDestroy(c, param.typ.skipTypes({tyGenericInst, tyAlias, tySink}), params[i])
+        c.addDestroy genDestroy(c, param.typ.skipTypes({tyGenericInst, tyAlias, tySink}), params[i])
 
   #if optNimV2 in c.graph.config.globalOptions:
   #  injectDefaultCalls(n, c)

compiler/jsgen.nim

@@ -32,9 +32,9 @@ import
   ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options,
   nversion, nimsets, msgs, std / sha1, bitsets, idents, types, os, tables,
   times, ropes, math, passes, ccgutils, wordrecg, renderer,
-  intsets, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, 
+  intsets, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils,
   pathutils, transf
-  
+
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -365,92 +365,92 @@ proc genOr(p: PProc, a, b: PNode, r: var TCompRes) =
     line(p, "}")
 
 type
-  TMagicFrmt = array[0..3, string]
+  TMagicFrmt = array[0..1, string]
   TMagicOps = array[mAddI..mStrToStr, TMagicFrmt]
 
-const # magic checked op; magic unchecked op; checked op; unchecked op
+const # magic checked op; magic unchecked op;
-  jsOps: TMagicOps = [
+  jsMagics: TMagicOps = [
-    ["addInt", "", "addInt($1, $2)", "($1 + $2)"], # AddI
+    ["addInt", ""], # AddI
-    ["subInt", "", "subInt($1, $2)", "($1 - $2)"], # SubI
+    ["subInt", ""], # SubI
-    ["mulInt", "", "mulInt($1, $2)", "($1 * $2)"], # MulI
+    ["mulInt", ""], # MulI
-    ["divInt", "", "divInt($1, $2)", "Math.trunc($1 / $2)"], # DivI
+    ["divInt", ""], # DivI
-    ["modInt", "", "modInt($1, $2)", "Math.trunc($1 % $2)"], # ModI
+    ["modInt", ""], # ModI
-    ["addInt", "", "addInt($1, $2)", "($1 + $2)"], # Succ
+    ["addInt", ""], # Succ
-    ["subInt", "", "subInt($1, $2)", "($1 - $2)"], # Pred
+    ["subInt", ""], # Pred
-    ["", "", "($1 + $2)", "($1 + $2)"], # AddF64
+    ["", ""], # AddF64
-    ["", "", "($1 - $2)", "($1 - $2)"], # SubF64
+    ["", ""], # SubF64
-    ["", "", "($1 * $2)", "($1 * $2)"], # MulF64
+    ["", ""], # MulF64
-    ["", "", "($1 / $2)", "($1 / $2)"], # DivF64
+    ["", ""], # DivF64
-    ["", "", "", ""], # ShrI
+    ["", ""], # ShrI
-    ["", "", "($1 << $2)", "($1 << $2)"], # ShlI
+    ["", ""], # ShlI
-    ["", "", "($1 >> $2)", "($1 >> $2)"], # AshrI
+    ["", ""], # AshrI
-    ["", "", "($1 & $2)", "($1 & $2)"], # BitandI
+    ["", ""], # BitandI
-    ["", "", "($1 | $2)", "($1 | $2)"], # BitorI
+    ["", ""], # BitorI
-    ["", "", "($1 ^ $2)", "($1 ^ $2)"], # BitxorI
+    ["", ""], # BitxorI
-    ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinI
+    ["nimMin", "nimMin"], # MinI
-    ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxI
+    ["nimMax", "nimMax"], # MaxI
-    ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinF64
+    ["nimMin", "nimMin"], # MinF64
-    ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxF64
+    ["nimMax", "nimMax"], # MaxF64
-    ["", "", "", ""], # addU
+    ["", ""], # addU
-    ["", "", "", ""], # subU
+    ["", ""], # subU
-    ["", "", "", ""], # mulU
+    ["", ""], # mulU
-    ["", "", "", ""], # divU
+    ["", ""], # divU
-    ["", "", "($1 % $2)", "($1 % $2)"], # modU
+    ["", ""], # modU
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqI
+    ["", ""], # EqI
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # LeI
+    ["", ""], # LeI
-    ["", "", "($1 < $2)", "($1 < $2)"], # LtI
+    ["", ""], # LtI
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqF64
+    ["", ""], # EqF64
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # LeF64
+    ["", ""], # LeF64
-    ["", "", "($1 < $2)", "($1 < $2)"], # LtF64
+    ["", ""], # LtF64
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # leU
+    ["", ""], # leU
-    ["", "", "($1 < $2)", "($1 < $2)"], # ltU
+    ["", ""], # ltU
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # leU64
+    ["", ""], # leU64
-    ["", "", "($1 < $2)", "($1 < $2)"], # ltU64
+    ["", ""], # ltU64
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqEnum
+    ["", ""], # EqEnum
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # LeEnum
+    ["", ""], # LeEnum
-    ["", "", "($1 < $2)", "($1 < $2)"], # LtEnum
+    ["", ""], # LtEnum
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqCh
+    ["", ""], # EqCh
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # LeCh
+    ["", ""], # LeCh
-    ["", "", "($1 < $2)", "($1 < $2)"], # LtCh
+    ["", ""], # LtCh
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqB
+    ["", ""], # EqB
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # LeB
+    ["", ""], # LeB
-    ["", "", "($1 < $2)", "($1 < $2)"], # LtB
+    ["", ""], # LtB
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqRef
+    ["", ""], # EqRef
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqUntracedRef
+    ["", ""], # EqUntracedRef
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # LePtr
+    ["", ""], # LePtr
-    ["", "", "($1 < $2)", "($1 < $2)"], # LtPtr
+    ["", ""], # LtPtr
-    ["", "", "($1 != $2)", "($1 != $2)"], # Xor
+    ["", ""], # Xor
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqCString
+    ["", ""], # EqCString
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqProc
+    ["", ""], # EqProc
-    ["negInt", "", "negInt($1)", "-($1)"], # UnaryMinusI
+    ["negInt", ""], # UnaryMinusI
-    ["negInt64", "", "negInt64($1)", "-($1)"], # UnaryMinusI64
+    ["negInt64", ""], # UnaryMinusI64
-    ["absInt", "", "absInt($1)", "Math.abs($1)"], # AbsI
+    ["absInt", ""], # AbsI
-    ["", "", "!($1)", "!($1)"], # Not
+    ["", ""], # Not
-    ["", "", "+($1)", "+($1)"], # UnaryPlusI
+    ["", ""], # UnaryPlusI
-    ["", "", "~($1)", "~($1)"], # BitnotI
+    ["", ""], # BitnotI
-    ["", "", "+($1)", "+($1)"], # UnaryPlusF64
+    ["", ""], # UnaryPlusF64
-    ["", "", "-($1)", "-($1)"], # UnaryMinusF64
+    ["", ""], # UnaryMinusF64
-    ["", "", "Math.abs($1)", "Math.abs($1)"], # AbsF64
+    ["", ""], # AbsF64
-    ["Ze8ToI", "Ze8ToI", "Ze8ToI($1)", "Ze8ToI($1)"], # mZe8ToI
+    ["Ze8ToI", "Ze8ToI"], # mZe8ToI
-    ["Ze8ToI64", "Ze8ToI64", "Ze8ToI64($1)", "Ze8ToI64($1)"], # mZe8ToI64
+    ["Ze8ToI64", "Ze8ToI64"], # mZe8ToI64
-    ["Ze16ToI", "Ze16ToI", "Ze16ToI($1)", "Ze16ToI($1)"], # mZe16ToI
+    ["Ze16ToI", "Ze16ToI"], # mZe16ToI
-    ["Ze16ToI64", "Ze16ToI64", "Ze16ToI64($1)", "Ze16ToI64($1)"], # mZe16ToI64
+    ["Ze16ToI64", "Ze16ToI64"], # mZe16ToI64
-    ["Ze32ToI64", "Ze32ToI64", "Ze32ToI64($1)", "Ze32ToI64($1)"], # mZe32ToI64
+    ["Ze32ToI64", "Ze32ToI64"], # mZe32ToI64
-    ["ZeIToI64", "ZeIToI64", "ZeIToI64($1)", "ZeIToI64($1)"], # mZeIToI64
+    ["ZeIToI64", "ZeIToI64"], # mZeIToI64
-    ["toU8", "toU8", "toU8($1)", "toU8($1)"], # toU8
+    ["toU8", "toU8"], # toU8
-    ["toU16", "toU16", "toU16($1)", "toU16($1)"], # toU16
+    ["toU16", "toU16"], # toU16
-    ["toU32", "toU32", "toU32($1)", "toU32($1)"], # toU32
+    ["toU32", "toU32"], # toU32
-    ["", "", "$1", "$1"],     # ToFloat
+    ["", ""],     # ToFloat
-    ["", "", "$1", "$1"],     # ToBiggestFloat
+    ["", ""],     # ToBiggestFloat
-    ["", "", "Math.trunc($1)", "Math.trunc($1)"], # ToInt
+    ["", ""], # ToInt
-    ["", "", "Math.trunc($1)", "Math.trunc($1)"], # ToBiggestInt
+    ["", ""], # ToBiggestInt
-    ["nimCharToStr", "nimCharToStr", "nimCharToStr($1)", "nimCharToStr($1)"],
+    ["nimCharToStr", "nimCharToStr"],
-    ["nimBoolToStr", "nimBoolToStr", "nimBoolToStr($1)", "nimBoolToStr($1)"],
+    ["nimBoolToStr", "nimBoolToStr"],
-    ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")"],
+    ["cstrToNimstr", "cstrToNimstr"],
-    ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")"],
+    ["cstrToNimstr", "cstrToNimstr"],
-    ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")"],
+    ["cstrToNimstr", "cstrToNimstr"],
-    ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr($1)", "cstrToNimstr($1)"],
+    ["cstrToNimstr", "cstrToNimstr"],
-    ["", "", "$1", "$1"]]
+    ["", ""]]
 
 proc needsTemp(p: PProc; n: PNode): bool =
   # check if n contains a call to determine
@@ -478,7 +478,7 @@ proc maybeMakeTemp(p: PProc, n: PNode; x: TCompRes): tuple[a, tmp: Rope] =
   else:
     (a: a, tmp: b)
 
-proc binaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
+template binaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
   # $1 and $2 in the `frmt` string bind to lhs and rhs of the expr,
   # if $3 or $4 are present they will be substituted with temps for
   # lhs and rhs respectively
@@ -490,8 +490,8 @@ proc binaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
   var
     a, tmp = x.rdLoc
     b, tmp2 = y.rdLoc
-  if "$3" in frmt: (a, tmp) = maybeMakeTemp(p, n[1], x)
+  when "$3" in frmt: (a, tmp) = maybeMakeTemp(p, n[1], x)
-  if "$4" in frmt: (a, tmp) = maybeMakeTemp(p, n[1], x)
+  when "$4" in frmt: (a, tmp) = maybeMakeTemp(p, n[1], x)
 
   r.res = frmt % [a, b, tmp, tmp2]
   r.kind = resExpr
@@ -520,7 +520,7 @@ proc binaryUintExpr(p: PProc, n: PNode, r: var TCompRes, op: string,
     r.res = "(($1 $2 $3) $4)" % [x.rdLoc, rope op, y.rdLoc, trimmer]
   r.kind = resExpr
 
-proc ternaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
+template ternaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
   var x, y, z: TCompRes
   useMagic(p, magic)
   gen(p, n.sons[1], x)
@@ -529,7 +529,7 @@ proc ternaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
   r.res = frmt % [x.rdLoc, y.rdLoc, z.rdLoc]
   r.kind = resExpr
 
-proc unaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
+template unaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
   # $1 binds to n[1], if $2 is present it will be substituted to a tmp of $1
   useMagic(p, magic)
   gen(p, n.sons[1], r)
@@ -541,15 +541,108 @@ proc unaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
 proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   var
     x, y: TCompRes
+    xLoc,yLoc: Rope
   let i = ord(optOverflowCheck notin p.options)
-  useMagic(p, jsOps[op][i])
+  useMagic(p, jsMagics[op][i])
   if sonsLen(n) > 2:
     gen(p, n.sons[1], x)
     gen(p, n.sons[2], y)
-    r.res = jsOps[op][i + 2] % [x.rdLoc, y.rdLoc]
+    xLoc = x.rdLoc
+    yLoc = y.rdLoc
   else:
     gen(p, n.sons[1], r)
-    r.res = jsOps[op][i + 2] % [r.rdLoc]
+    xLoc = r.rdLoc
+
+  template applyFormat(frmtA, frmtB: string) =
+    if i == 0:
+      r.res = frmtA % [xLoc, yLoc]
+    else:
+      r.res = frmtB % [xLoc, yLoc]
+
+  case op:
+  of mAddI: applyFormat("addInt($1, $2)", "($1 + $2)")
+  of mSubI: applyFormat("subInt($1, $2)", "($1 - $2)")
+  of mMulI: applyFormat("mulInt($1, $2)", "($1 * $2)")
+  of mDivI: applyFormat("divInt($1, $2)", "Math.trunc($1 / $2)")
+  of mModI: applyFormat("modInt($1, $2)", "Math.trunc($1 % $2)")
+  of mSucc: applyFormat("addInt($1, $2)", "($1 + $2)")
+  of mPred: applyFormat("subInt($1, $2)", "($1 - $2)")
+  of mAddF64: applyFormat("($1 + $2)", "($1 + $2)")
+  of mSubF64: applyFormat("($1 - $2)", "($1 - $2)")
+  of mMulF64: applyFormat("($1 * $2)", "($1 * $2)")
+  of mDivF64: applyFormat("($1 / $2)", "($1 / $2)")
+  of mShrI: applyFormat("", "")
+  of mShlI: applyFormat("($1 << $2)", "($1 << $2)")
+  of mAshrI: applyFormat("($1 >> $2)", "($1 >> $2)")
+  of mBitandI: applyFormat("($1 & $2)", "($1 & $2)")
+  of mBitorI: applyFormat("($1 | $2)", "($1 | $2)")
+  of mBitxorI: applyFormat("($1 ^ $2)", "($1 ^ $2)")
+  of mMinI: applyFormat("nimMin($1, $2)", "nimMin($1, $2)")
+  of mMaxI: applyFormat("nimMax($1, $2)", "nimMax($1, $2)")
+  of mMinF64: applyFormat("nimMin($1, $2)", "nimMin($1, $2)")
+  of mMaxF64: applyFormat("nimMax($1, $2)", "nimMax($1, $2)")
+  of mAddU: applyFormat("", "")
+  of msubU: applyFormat("", "")
+  of mmulU: applyFormat("", "")
+  of mdivU: applyFormat("", "")
+  of mmodU: applyFormat("($1 % $2)", "($1 % $2)")
+  of mEqI: applyFormat("($1 == $2)", "($1 == $2)")
+  of mLeI: applyFormat("($1 <= $2)", "($1 <= $2)")
+  of mLtI: applyFormat("($1 < $2)", "($1 < $2)")
+  of mEqF64: applyFormat("($1 == $2)", "($1 == $2)")
+  of mLeF64: applyFormat("($1 <= $2)", "($1 <= $2)")
+  of mLtF64: applyFormat("($1 < $2)", "($1 < $2)")
+  of mleU: applyFormat("($1 <= $2)", "($1 <= $2)")
+  of mltU: applyFormat("($1 < $2)", "($1 < $2)")
+  of mleU64: applyFormat("($1 <= $2)", "($1 <= $2)")
+  of mltU64: applyFormat("($1 < $2)", "($1 < $2)")
+  of mEqEnum: applyFormat("($1 == $2)", "($1 == $2)")
+  of mLeEnum: applyFormat("($1 <= $2)", "($1 <= $2)")
+  of mLtEnum: applyFormat("($1 < $2)", "($1 < $2)")
+  of mEqCh: applyFormat("($1 == $2)", "($1 == $2)")
+  of mLeCh: applyFormat("($1 <= $2)", "($1 <= $2)")
+  of mLtCh: applyFormat("($1 < $2)", "($1 < $2)")
+  of mEqB: applyFormat("($1 == $2)", "($1 == $2)")
+  of mLeB: applyFormat("($1 <= $2)", "($1 <= $2)")
+  of mLtB: applyFormat("($1 < $2)", "($1 < $2)")
+  of mEqRef: applyFormat("($1 == $2)", "($1 == $2)")
+  of mEqUntracedRef: applyFormat("($1 == $2)", "($1 == $2)")
+  of mLePtr: applyFormat("($1 <= $2)", "($1 <= $2)")
+  of mLtPtr: applyFormat("($1 < $2)", "($1 < $2)")
+  of mXor: applyFormat("($1 != $2)", "($1 != $2)")
+  of mEqCString: applyFormat("($1 == $2)", "($1 == $2)")
+  of mEqProc: applyFormat("($1 == $2)", "($1 == $2)")
+  of mUnaryMinusI: applyFormat("negInt($1)", "-($1)")
+  of mUnaryMinusI64: applyFormat("negInt64($1)", "-($1)")
+  of mAbsI: applyFormat("absInt($1)", "Math.abs($1)")
+  of mNot: applyFormat("!($1)", "!($1)")
+  of mUnaryPlusI: applyFormat("+($1)", "+($1)")
+  of mBitnotI: applyFormat("~($1)", "~($1)")
+  of mUnaryPlusF64: applyFormat("+($1)", "+($1)")
+  of mUnaryMinusF64: applyFormat("-($1)", "-($1)")
+  of mAbsF64: applyFormat("Math.abs($1)", "Math.abs($1)")
+  of mZe8ToI: applyFormat("Ze8ToI($1)", "Ze8ToI($1)")
+  of mZe8ToI64: applyFormat("Ze8ToI64($1)", "Ze8ToI64($1)")
+  of mZe16ToI: applyFormat("Ze16ToI($1)", "Ze16ToI($1)")
+  of mZe16ToI64: applyFormat("Ze16ToI64($1)", "Ze16ToI64($1)")
+  of mZe32ToI64: applyFormat("Ze32ToI64($1)", "Ze32ToI64($1)")
+  of mZeIToI64: applyFormat("ZeIToI64($1)", "ZeIToI64($1)")
+  of mtoU8: applyFormat("toU8($1)", "toU8($1)")
+  of mtoU16: applyFormat("toU16($1)", "toU16($1)")
+  of mtoU32: applyFormat("toU32($1)", "toU32($1)")
+  of mToFloat: applyFormat("$1", "$1")
+  of mToBiggestFloat: applyFormat("$1", "$1")
+  of mToInt: applyFormat("Math.trunc($1)", "Math.trunc($1)")
+  of mToBiggestInt: applyFormat("Math.trunc($1)", "Math.trunc($1)")
+  of mCharToStr: applyFormat("nimCharToStr($1)", "nimCharToStr($1)")
+  of mBoolToStr: applyFormat("nimBoolToStr($1)", "nimBoolToStr($1)")
+  of mIntToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
+  of mInt64ToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
+  of mFloatToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
+  of mCStrToStr: applyFormat("cstrToNimstr($1)", "cstrToNimstr($1)")
+  of mStrToStr, mUnown: applyFormat("$1", "$1")
+  else:
+    assert false, $op
 
 proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   case op
@@ -1268,6 +1361,9 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
       internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
     r.res = s.loc.r
   of skProc, skFunc, skConverter, skMethod:
+    if sfCompileTime in s.flags:
+      localError(p.config, n.info, "request to generate code for .compileTime proc: " &
+          s.name.s)
     discard mangleName(p.module, s)
     r.res = s.loc.r
     if lfNoDecl in s.loc.flags or s.magic != mNone or
@@ -1603,6 +1699,9 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     else:
       varCode = "var $2"
   else:
+    # Is this really a thought through feature?  this basically unused
+    # feature makes it impossible for almost all format strings in
+    # this function to be checked at compile time.
     varCode = v.constraint.strVal
 
   if n.kind == nkEmpty:
@@ -1611,8 +1710,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       lineF(p, "var $1 = null;$n", [varName])
       lineF(p, "var $1_Idx = 0;$n", [varName])
     else:
-      lineF(p, varCode & " = $3;$n",
+      line(p, runtimeFormat(varCode & " = $3;$n", [returnType, varName, createVar(p, v.typ, isIndirect(v))]))
-                [returnType, varName, createVar(p, v.typ, isIndirect(v))])
   else:
     gen(p, n, a)
     case mapType(p, v.typ)
@@ -1626,29 +1724,29 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       let targetBaseIndex = {sfAddrTaken, sfGlobal} * v.flags == {}
       if a.typ == etyBaseIndex:
         if targetBaseIndex:
-          lineF(p, varCode & " = $3, $2_Idx = $4;$n",
+          line(p, runtimeFormat(varCode & " = $3, $2_Idx = $4;$n",
-                   [returnType, v.loc.r, a.address, a.res])
+                   [returnType, v.loc.r, a.address, a.res]))
         else:
           if isIndirect(v):
-            lineF(p, varCode & " = [[$3, $4]];$n",
+            line(p, runtimeFormat(varCode & " = [[$3, $4]];$n",
-                     [returnType, v.loc.r, a.address, a.res])
+                     [returnType, v.loc.r, a.address, a.res]))
           else:
-            lineF(p, varCode & " = [$3, $4];$n",
+            line(p, runtimeFormat(varCode & " = [$3, $4];$n",
-                     [returnType, v.loc.r, a.address, a.res])
+                     [returnType, v.loc.r, a.address, a.res]))
       else:
         if targetBaseIndex:
           let tmp = p.getTemp
           lineF(p, "var $1 = $2, $3 = $1[0], $3_Idx = $1[1];$n",
                    [tmp, a.res, v.loc.r])
         else:
-          lineF(p, varCode & " = $3;$n", [returnType, v.loc.r, a.res])
+          line(p, runtimeFormat(varCode & " = $3;$n", [returnType, v.loc.r, a.res]))
       return
     else:
       s = a.res
     if isIndirect(v):
-      lineF(p, varCode & " = [$3];$n", [returnType, v.loc.r, s])
+      line(p, runtimeFormat(varCode & " = [$3];$n", [returnType, v.loc.r, s]))
     else:
-      lineF(p, varCode & " = $3;$n", [returnType, v.loc.r, s])
+      line(p, runtimeFormat(varCode & " = $3;$n", [returnType, v.loc.r, s]))
 
   if useReloadingGuard:
     dec p.extraIndent
@@ -2117,7 +2215,7 @@ proc genReturnStmt(p: PProc, n: PNode) =
   lineF(p, "break BeforeRet;$n", [])
 
 proc frameCreate(p: PProc; procname, filename: Rope): Rope =
-  let frameFmt =
+  const frameFmt =
     "var F={procname:$1,prev:framePtr,filename:$2,line:0};$n"
 
   result = p.indentLine(frameFmt % [procname, filename])
@@ -2185,7 +2283,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
 
   var def: Rope
   if not prc.constraint.isNil:
-    def = (prc.constraint.strVal & " {$n$#$#$#$#$#") %
+    def = runtimeFormat(prc.constraint.strVal & " {$n$#$#$#$#$#",
             [ returnType,
               name,
               header,
@@ -2193,7 +2291,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
               optionalLine(p.locals),
               optionalLine(resultAsgn),
               optionalLine(genProcBody(p, prc)),
-              optionalLine(p.indentLine(returnStmt))]
+              optionalLine(p.indentLine(returnStmt))])
   else:
     result = ~"\L"
 

compiler/lambdalifting.nim

@@ -261,7 +261,7 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
 
 proc freshVarForClosureIter*(g: ModuleGraph; s, owner: PSym): PNode =
   let envParam = getHiddenParam(g, owner)
-  let obj = envParam.typ.lastSon
+  let obj = envParam.typ.skipTypes({tyOwned, tyRef})
   addField(obj, s, g.cache)
 
   var access = newSymNode(envParam)
@@ -320,15 +320,23 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
     rawAddSon(result, obj)
     c.ownerToType[owner.id] = result
 
+proc asOwnedRef(c: DetectionPass; t: PType): PType =
+  if optNimV2 in c.graph.config.globalOptions:
+    assert t.kind == tyRef
+    result = newType(tyOwned, t.owner)
+    result.rawAddSon t
+  else:
+    result = t
+
 proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym;
                           info: TLineInfo): PType =
   var r = c.getEnvTypeForOwner(owner, info)
   result = newType(tyPtr, owner)
-  rawAddSon(result, r.base)
+  rawAddSon(result, r.skipTypes({tyOwned, tyRef}))
 
 proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   let refObj = c.getEnvTypeForOwner(dest, info) # getHiddenParam(dest).typ
-  let obj = refObj.lastSon
+  let obj = refObj.skipTypes({tyOwned, tyRef})
   # The assumption here is that gcDestructors means we cannot deal
   # with cycles properly, so it's better to produce a weak ref (=ptr) here.
   # This seems to be generally correct but since it's a bit risky it's only
@@ -343,7 +351,7 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   let upIdent = getIdent(c.graph.cache, upName)
   let upField = lookupInRecord(obj.n, upIdent)
   if upField != nil:
-    if upField.typ.base != fieldType.base:
+    if upField.typ.skipTypes({tyOwned, tyRef}) != fieldType.skipTypes({tyOwned, tyRef}):
       localError(c.graph.config, dep.info, "internal error: up references do not agree")
   else:
     let result = newSym(skField, upIdent, obj.owner, obj.owner.info)
@@ -414,8 +422,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
         addClosureParam(c, owner, n.info)
         if interestingIterVar(s):
           if not c.capturedVars.containsOrIncl(s.id):
-            let obj = getHiddenParam(c.graph, owner).typ.lastSon
+            let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef})
-            #let obj = c.getEnvTypeForOwner(s.owner).lastSon
+            #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef})
 
             if s.name.id == getIdent(c.graph.cache, ":state").id:
               obj.n[0].sym.id = -s.id
@@ -440,8 +448,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       #echo "capturing ", n.info
       # variable 's' is actually captured:
       if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
-        let obj = c.getEnvTypeForOwner(ow, n.info).lastSon
+        let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef})
-        #getHiddenParam(owner).typ.lastSon
+        #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef})
         addField(obj, s, c.graph.cache)
       # create required upFields:
       var w = owner.skipGenericOwner
@@ -530,14 +538,14 @@ proc setupEnvVar(owner: PSym; d: DetectionPass;
     let envVarType = d.ownerToType.getOrDefault(owner.id)
     if envVarType.isNil:
       localError d.graph.config, owner.info, "internal error: could not determine closure type"
-    result = newEnvVar(d.graph.cache, owner, envVarType)
+    result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType))
     c.envVars[owner.id] = result
 
 proc getUpViaParam(g: ModuleGraph; owner: PSym): PNode =
   let p = getHiddenParam(g, owner)
   result = p.newSymNode
   if owner.isIterator:
-    let upField = lookupInRecord(p.typ.lastSon.n, getIdent(g.cache, upName))
+    let upField = lookupInRecord(p.typ.skipTypes({tyOwned, tyRef}).n, getIdent(g.cache, upName))
     if upField == nil:
       localError(g.config, owner.info, "could not find up reference for closure iter")
     else:
@@ -566,10 +574,10 @@ proc rawClosureCreation(owner: PSym;
         # add ``env.param = param``
         result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info))
 
-  let upField = lookupInRecord(env.typ.lastSon.n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(env.typ.skipTypes({tyOwned, tyRef}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
     let up = getUpViaParam(d.graph, owner)
-    if up != nil and upField.typ.base == up.typ.base:
+    if up != nil and upField.typ.skipTypes({tyOwned, tyRef}) == up.typ.skipTypes({tyOwned, tyRef}):
       result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
                  up, env.info))
     #elif oldenv != nil and oldenv.typ == upField.typ:
@@ -584,11 +592,11 @@ proc closureCreationForIter(iter: PNode;
   let owner = iter.sym.skipGenericOwner
   var v = newSym(skVar, getIdent(d.graph.cache, envName), owner, iter.info)
   incl(v.flags, sfShadowed)
-  v.typ = getHiddenParam(d.graph, iter.sym).typ
+  v.typ = asOwnedRef(d, getHiddenParam(d.graph, iter.sym).typ)
   var vnode: PNode
   if owner.isIterator:
     let it = getHiddenParam(d.graph, owner)
-    addUniqueField(it.typ.sons[0], v, d.graph.cache)
+    addUniqueField(it.typ.skipTypes({tyOwned, tyRef}), v, d.graph.cache)
     vnode = indirectAccess(newSymNode(it), v, v.info)
   else:
     vnode = v.newSymNode
@@ -597,10 +605,10 @@ proc closureCreationForIter(iter: PNode;
     result.add(vs)
   result.add(newCall(getSysSym(d.graph, iter.info, "internalNew"), vnode))
 
-  let upField = lookupInRecord(v.typ.lastSon.n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
     let u = setupEnvVar(owner, d, c)
-    if u.typ.base == upField.typ.base:
+    if u.typ.skipTypes({tyOwned, tyRef}) == upField.typ.skipTypes({tyOwned, tyRef}):
       result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info),
                  u, iter.info))
     else:
@@ -610,7 +618,7 @@ proc closureCreationForIter(iter: PNode;
 proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
                      c: var LiftingPass): PNode =
   let access = setupEnvVar(owner, d, c)
-  let obj = access.typ.sons[0]
+  let obj = access.typ.skipTypes({tyOwned, tyRef})
   let field = getFieldFromObj(obj, n.sym)
   if field != nil:
     result = rawIndirectAccess(access, field, n.info)
@@ -619,7 +627,7 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
     result = n
 
 proc getStateField*(g: ModuleGraph; owner: PSym): PSym =
-  getHiddenParam(g, owner).typ.sons[0].n.sons[0].sym
+  getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef}).n.sons[0].sym
 
 proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
                       c: var LiftingPass): PNode
@@ -644,7 +652,7 @@ proc symToClosure(n: PNode; owner: PSym; d: DetectionPass;
     while true:
       if access.typ == wanted:
         return makeClosure(d.graph, s, access, n.info)
-      let obj = access.typ.sons[0]
+      let obj = access.typ.skipTypes({tyOwned, tyRef})
       let upField = lookupInRecord(obj.n, getIdent(d.graph.cache, upName))
       if upField == nil:
         localError(d.graph.config, n.info, "internal error: no environment found")

compiler/liftdestructors.nim

@@ -126,7 +126,7 @@ proc newDeepCopyCall(op: PSym; x, y: PNode): PNode =
 
 proc useNoGc(c: TLiftCtx; t: PType): bool {.inline.} =
   result = optNimV2 in c.graph.config.globalOptions and
-    (tfHasGCedMem in t.flags or t.isGCedMem)
+    ({tfHasGCedMem, tfHasOwned} * t.flags != {} or t.isGCedMem)
 
 proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
                         field: var PSym): bool =
@@ -171,7 +171,7 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
     body.add newAsgnCall(c.graph, op, x, y)
     result = true
 
-proc addDestructorCall(c: var TLiftCtx; t: PType; body, x: PNode): bool =
+proc addDestructorCall(c: var TLiftCtx; t: PType; body, x: PNode) =
   var op = t.destructor
   if op == nil and useNoGc(c, t):
     op = produceSym(c.c, t, attachedDestructor, c.info)
@@ -182,7 +182,6 @@ proc addDestructorCall(c: var TLiftCtx; t: PType; body, x: PNode): bool =
     markUsed(c.graph.config, c.info, op, c.graph.usageSym)
     onUse(c.info, op)
     body.add destructorCall(c.graph, op, x)
-    result = true
   elif useNoGc(c, t):
     internalError(c.graph.config, c.info,
       "type-bound operator could not be resolved")
@@ -197,7 +196,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
         assert t.typeInst != nil
         # patch generic destructor:
         op = c.c.instTypeBoundOp(c.c, op, t.typeInst, c.info, attachedAsgn, 1)
-        t.destructor = op
+        t.attachedOps[attachedDestructor] = op
 
       markUsed(c.graph.config, c.info, op, c.graph.usageSym)
       onUse(c.info, op)
@@ -207,9 +206,9 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
   of attachedAsgn:
     result = considerAsgnOrSink(c, t, body, x, y, t.assignment)
   of attachedSink:
-    result = considerAsgnOrSink(c, t, body, x, y, t.sink)
+    result = considerAsgnOrSink(c, t, body, x, y, t.asink)
   of attachedDeepCopy:
-    let op = t.deepCopy
+    let op = t.attachedOps[attachedDeepCopy]
     if op != nil:
       markUsed(c.graph.config, c.info, op, c.graph.usageSym)
       onUse(c.info, op)
@@ -323,8 +322,8 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add moveCall
     # alternatively we could do this:
     when false:
-      doAssert t.sink != nil
+      doAssert t.asink != nil
-      body.add newAsgnCall(c.graph, t.sink, x, y)
+      body.add newAsgnCall(c.graph, t.asink, x, y)
   of attachedDestructor:
     doAssert t.destructor != nil
     body.add destructorCall(c.graph, t.destructor, x)
@@ -365,10 +364,10 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   #var disposeCall = genBuiltin(c.graph, mDispose, "dispose", x)
 
   if isFinal(elemType):
-    discard addDestructorCall(c, elemType, actions, genDeref(x))
+    addDestructorCall(c, elemType, actions, genDeref(x))
     actions.add callCodegenProc(c.graph, "nimRawDispose", c.info, x)
   else:
-    discard addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(x))
+    addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(x))
     actions.add callCodegenProc(c.graph, "nimDestroyAndDispose", c.info, x)
 
   case c.kind
@@ -502,27 +501,10 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 proc produceSymDistinctType(c: PContext; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym =
   assert typ.kind == tyDistinct
   let baseType = typ[0]
-  case kind
+  if baseType.attachedOps[kind] == nil:
-  of attachedAsgn:
+    discard produceSym(c, baseType, kind, info)
-    if baseType.assignment == nil:
+  typ.attachedOps[kind] = baseType.attachedOps[kind]
-      discard produceSym(c, baseType, kind, info)
+  result = typ.attachedOps[kind]
-    typ.assignment = baseType.assignment
-    result = typ.assignment
-  of attachedSink:
-    if baseType.sink == nil:
-      discard produceSym(c, baseType, kind, info)
-    typ.sink = baseType.sink
-    result = typ.sink
-  of attachedDeepCopy:
-    if baseType.deepCopy == nil:
-      discard produceSym(c, baseType, kind, info)
-    typ.deepCopy = baseType.deepCopy
-    result = typ.deepCopy
-  of attachedDestructor:
-    if baseType.destructor == nil:
-      discard produceSym(c, baseType, kind, info)
-    typ.destructor = baseType.destructor
-    result = typ.destructor
 
 proc produceSym(c: PContext; typ: PType; kind: TTypeAttachedOp;
               info: TLineInfo): PSym =
@@ -536,11 +518,7 @@ proc produceSym(c: PContext; typ: PType; kind: TTypeAttachedOp;
   a.c = c
   let g = c.graph
   let body = newNodeI(nkStmtList, info)
-  let procname = case kind
+  let procname = getIdent(g.cache, AttachedOpToStr[kind])
-                 of attachedAsgn: getIdent(g.cache, "=")
-                 of attachedSink: getIdent(g.cache, "=sink")
-                 of attachedDeepCopy: getIdent(g.cache, "=deepcopy")
-                 of attachedDestructor: getIdent(g.cache, "=destroy")
 
   result = newSym(skProc, procname, typ.owner, info)
   a.fn = result
@@ -557,11 +535,7 @@ proc produceSym(c: PContext; typ: PType; kind: TTypeAttachedOp;
     result.typ.addParam src
 
   # register this operation already:
-  case kind
+  typ.attachedOps[kind] = result
-  of attachedAsgn: typ.assignment = result
-  of attachedSink: typ.sink = result
-  of attachedDeepCopy: typ.deepCopy = result
-  of attachedDestructor: typ.destructor = result
 
   var tk: TTypeKind
   if optNimV2 in c.graph.config.globalOptions:
@@ -636,23 +610,15 @@ proc createTypeBoundOps(c: PContext; orig: PType; info: TLineInfo) =
   # 5. We have a (custom) generic destructor.
   let typ = canon.skipTypes({tyGenericInst, tyAlias})
   # we generate the destructor first so that other operators can depend on it:
-  if typ.destructor == nil:
+  for k in attachedDestructor..attachedSink:
-    discard produceSym(c, typ, attachedDestructor, info)
+    if typ.attachedOps[k] == nil:
-  else:
+      discard produceSym(c, typ, k, info)
-    inst(typ.destructor, typ)
+    else:
-  if typ.assignment == nil:
+      inst(typ.attachedOps[k], typ)
-    discard produceSym(c, typ, attachedAsgn, info)
-  else:
-    inst(typ.assignment, typ)
-  if typ.sink == nil:
-    discard produceSym(c, typ, attachedSink, info)
-  else:
-    inst(typ.sink, typ)
 
   if overwrite:
-    orig.destructor = typ.destructor
+    for k in attachedDestructor..attachedSink:
-    orig.assignment = typ.assignment
+      orig.attachedOps[k] = typ.attachedOps[k]
-    orig.sink = typ.sink
 
   if not isTrival(orig.destructor):
     #or not isTrival(orig.assignment) or

compiler/modules.nim

@@ -17,9 +17,9 @@ import
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
 
-proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: string) =
+proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
   let
-    pck = getPackageName(graph.config, filename)
+    pck = getPackageName(graph.config, filename.string)
     pck2 = if pck.len > 0: pck else: "unknown"
     pack = getIdent(graph.cache, pck2)
   var packSym = graph.packageSyms.strTableGet(pack)
@@ -27,6 +27,22 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
     packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, result.info)
     initStrTable(packSym.tab)
     graph.packageSyms.strTableAdd(packSym)
+  else:
+    let existing = strTableGet(packSym.tab, result.name)
+    if existing != nil and existing.info.fileIndex != result.info.fileIndex:
+      when false:
+        # we used to produce an error:
+        localError(graph.config, result.info,
+          "module names need to be unique per Nimble package; module clashes with " &
+            toFullPath(graph.config, existing.info.fileIndex))
+      else:
+        # but starting with version 0.20 we now produce a fake Nimble package instead
+        # to resolve the conflicts:
+        let pck3 = fakePackageName(graph.config, filename)
+        packSym = newSym(skPackage, getIdent(graph.cache, pck3), nil, result.info)
+        initStrTable(packSym.tab)
+        graph.packageSyms.strTableAdd(packSym)
+
   result.owner = packSym
   result.position = int fileIdx
 
@@ -37,13 +53,7 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
   incl(result.flags, sfUsed)
   initStrTable(result.tab)
   strTableAdd(result.tab, result) # a module knows itself
-  let existing = strTableGet(packSym.tab, result.name)
+  strTableAdd(packSym.tab, result)
-  if existing != nil and existing.info.fileIndex != result.info.fileIndex:
-    localError(graph.config, result.info,
-      "module names need to be unique per Nimble package; module clashes with " &
-        toFullPath(graph.config, existing.info.fileIndex))
-  # strTableIncl() for error corrections:
-  discard strTableIncl(packSym.tab, result)
 
 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   # We cannot call ``newSym`` here, because we have to circumvent the ID
@@ -51,7 +61,7 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   new(result)
   result.id = -1             # for better error checking
   result.kind = skModule
-  let filename = toFullPath(graph.config, fileIdx)
+  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
   result.name = getIdent(graph.cache, splitFile(filename).name)
   if not isNimIdentifier(result.name.s):
     rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
@@ -61,8 +71,8 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
 proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
   result = graph.getModule(fileIdx)
   if result == nil:
-    let filename = toFullPath(graph.config, fileIdx)
+    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-    let (r, id) = loadModuleSym(graph, fileIdx, AbsoluteFile filename)
+    let (r, id) = loadModuleSym(graph, fileIdx, filename)
     result = r
     if result == nil:
       result = newModule(graph, fileIdx)

compiler/packagehandling.nim

@@ -29,9 +29,6 @@ proc getPackageName*(conf: ConfigRef; path: string): string =
       for file in walkFiles(d / "*.nimble"):
         result = file.splitFile.name
         break packageSearch
-      for file in walkFiles(d / "*.babel"):
-        result = file.splitFile.name
-        break packageSearch
   # we also store if we didn't find anything:
   when not defined(nimNoNilSeqs):
     if result.isNil: result = ""
@@ -41,10 +38,19 @@ proc getPackageName*(conf: ConfigRef; path: string): string =
     dec parents
     if parents <= 0: break
 
+proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string =
+  # foo/../bar becomes foo7_7bar
+  result = relativeTo(path, conf.projectPath, '/').string.multiReplace(
+    {"/": "7", "..": "_", "7": "77", "_": "__"})
+
 proc withPackageName*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile =
   let x = getPackageName(conf, path.string)
   if x.len == 0:
     result = path
   else:
     let (p, file, ext) = path.splitFile
-    result = p / RelativeFile((x & '_' & file) & ext)
+    if x == "stdlib":
+      # Hot code reloading now relies on 'stdlib_system' names etc.
+      result = p / RelativeFile((x & '_' & file) & ext)
+    else:
+      result = p / RelativeFile(fakePackageName(conf, path))

compiler/ropes.nim

@@ -211,7 +211,7 @@ proc ropeConcat*(a: varargs[Rope]): Rope =
 proc prepend*(a: var Rope, b: Rope) = a = b & a
 proc prepend*(a: var Rope, b: string) = a = b & a
 
-proc `%`*(frmt: FormatStr, args: openArray[Rope]): Rope =
+proc runtimeFormat*(frmt: FormatStr, args: openArray[Rope]): Rope =
   var i = 0
   var length = len(frmt)
   result = nil
@@ -269,7 +269,10 @@ proc `%`*(frmt: FormatStr, args: openArray[Rope]): Rope =
       add(result, substr(frmt, start, i - 1))
   assert(ropeInvariant(result))
 
-proc addf*(c: var Rope, frmt: FormatStr, args: openArray[Rope]) =
+proc `%`*(frmt: static[FormatStr], args: openArray[Rope]): Rope =
+  runtimeFormat(frmt, args)
+
+template addf*(c: var Rope, frmt: FormatStr, args: openArray[Rope]) =
   ## shortcut for ``add(c, frmt % args)``.
   add(c, frmt % args)
 

compiler/semdata.nim

@@ -72,12 +72,6 @@ type
 
   TExprFlags* = set[TExprFlag]
 
-  TTypeAttachedOp* = enum
-    attachedAsgn,
-    attachedSink,
-    attachedDeepCopy,
-    attachedDestructor
-
   PContext* = ref TContext
   TContext* = object of TPassContext # a context represents a module
     enforceVoidContext*: PType

compiler/semexprs.nim

@@ -937,7 +937,7 @@ proc semExprNoType(c: PContext, n: PNode): PNode =
   let isPush = hintExtendedContext in c.config.notes
   if isPush: pushInfoContext(c.config, n.info)
   result = semExpr(c, n, {efWantStmt})
-  result = discardCheck(c, result, {})
+  discardCheck(c, result, {})
   if isPush: popInfoContext(c.config)
 
 proc isTypeExpr(n: PNode): bool =
@@ -1526,17 +1526,52 @@ proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
       x.typ.flags.incl tfVarIsPtr
       #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info
 
-proc asgnToResult(c: PContext, n, le, ri: PNode) =
+proc borrowCheck(c: PContext, n, le, ri: PNode) =
+  const
+    PathKinds0 = {nkDotExpr, nkCheckedFieldExpr,
+                  nkBracketExpr, nkAddr, nkHiddenAddr,
+                  nkObjDownConv, nkObjUpConv}
+    PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv}
+
+  proc getRoot(n: PNode; followDeref: bool): PNode =
+    result = n
+    while true:
+      case result.kind
+      of nkDerefExpr, nkHiddenDeref:
+        if followDeref: result = result[0]
+        else: break
+      of PathKinds0:
+        result = result[0]
+      of PathKinds1:
+        result = result[1]
+      else: break
+
+  proc scopedLifetime(c: PContext; ri: PNode): bool {.inline.} =
+    let n = getRoot(ri, followDeref = false)
+    result = (ri.kind in nkCallKinds+{nkObjConstr}) or
+      (n.kind == nkSym and n.sym.owner == c.p.owner)
+
+  proc escapes(c: PContext; le: PNode): bool {.inline.} =
+    # param[].foo[] = self  definitely escapes, we don't need to
+    # care about pointer derefs:
+    let n = getRoot(le, followDeref = true)
+    result = n.kind == nkSym and n.sym.kind == skParam
+
   # Special typing rule: do not allow to pass 'owned T' to 'T' in 'result = x':
-  if ri.typ != nil and ri.typ.skipTypes(abstractInst).kind == tyOwned and
+  const absInst = abstractInst - {tyOwned}
-      le.typ != nil and le.typ.skipTypes(abstractInst).kind != tyOwned and ri.kind in nkCallKinds:
+  if ri.typ != nil and ri.typ.skipTypes(absInst).kind == tyOwned and
-    localError(c.config, n.info, "cannot return an owned pointer as an unowned pointer; " &
+      le.typ != nil and le.typ.skipTypes(absInst).kind != tyOwned and
-      "use 'owned(" & typeToString(le.typ) & ")' as the return type")
+      scopedLifetime(c, ri):
+    if le.kind == nkSym and le.sym.kind == skResult:
+      localError(c.config, n.info, "cannot return an owned pointer as an unowned pointer; " &
+        "use 'owned(" & typeToString(le.typ) & ")' as the return type")
+    elif escapes(c, le):
+      localError(c.config, n.info,
+        "assignment produces a dangling ref: the unowned ref lives longer than the owned ref")
 
 template resultTypeIsInferrable(typ: PType): untyped =
   typ.isMetaType and typ.kind != tyTypeDesc
 
-
 proc goodLineInfo(arg: PNode): TLineinfo =
   if arg.kind == nkStmtListExpr and arg.len > 0:
     goodLineInfo(arg[^1])
@@ -1623,7 +1658,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
           c.p.owner.typ.sons[0] = rhsTyp
         else:
           typeMismatch(c.config, n.info, lhs.typ, rhsTyp)
-      asgnToResult(c, n, n.sons[0], rhs)
+    borrowCheck(c, n, lhs, rhs)
 
     n.sons[1] = fitNode(c, le, rhs, goodLineInfo(n[1]))
     liftTypeBoundOps(c, lhs.typ, lhs.info)
@@ -1673,7 +1708,7 @@ proc semProcBody(c: PContext, n: PNode): PNode =
       a.sons[1] = result
       result = semAsgn(c, a)
   else:
-    result = discardCheck(c, result, {})
+    discardCheck(c, result, {})
 
   if c.p.owner.kind notin {skMacro, skTemplate} and
      c.p.resultSym != nil and c.p.resultSym.typ.isMetaType:

compiler/semmagic.nim

@@ -322,6 +322,39 @@ proc semOf(c: PContext, n: PNode): PNode =
   n.typ = getSysType(c.graph, n.info, tyBool)
   result = n
 
+proc semUnown(c: PContext; n: PNode): PNode =
+  proc unownedType(c: PContext; t: PType): PType =
+    case t.kind
+    of tyTuple:
+      var elems = newSeq[PType](t.len)
+      var someChange = false
+      for i in 0..<t.len:
+        elems[i] = unownedType(c, t[i])
+        if elems[i] != t[i]: someChange = true
+      if someChange:
+        result = newType(tyTuple, t.owner)
+        # we have to use 'rawAddSon' here so that type flags are
+        # properly computed:
+        for e in elems: result.rawAddSon(e)
+      else:
+        result = t
+    of tyOwned: result = t.sons[0]
+    of tySequence, tyOpenArray, tyArray, tyVarargs, tyVar, tyLent,
+       tyGenericInst, tyAlias:
+      let L = t.len-1
+      let b = unownedType(c, t[L])
+      if b != t[L]:
+        result = copyType(t, t.owner, keepId = false)
+        result[L] = b
+        result.flags.excl tfHasOwned
+      else:
+        result = t
+    else:
+      result = t
+
+  result = copyTree(n[1])
+  result.typ = unownedType(c, result.typ)
+
 proc magicsAfterOverloadResolution(c: PContext, n: PNode,
                                    flags: TExprFlags): PNode =
   ## This is the preferred code point to implement magics.
@@ -433,4 +466,6 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     let t = n[1].typ.skipTypes(abstractVar)
     if t.destructor != nil:
       result.sons[0] = newSymNode(t.destructor)
+  of mUnown:
+    result = semUnown(c, n)
   else: result = n

compiler/sempass2.nim

@@ -527,7 +527,7 @@ proc isNoEffectList(n: PNode): bool {.inline.} =
   n.len == 0 or (n[tagEffects] == nil and n[exceptionEffects] == nil)
 
 proc isTrival(caller: PNode): bool {.inline.} =
-  result = caller.kind == nkSym and caller.sym.magic in {mEqProc, mIsNil}
+  result = caller.kind == nkSym and caller.sym.magic in {mEqProc, mIsNil, mMove, mWasMoved}
 
 proc trackOperand(tracked: PEffects, n: PNode, paramType: PType; caller: PNode) =
   let a = skipConvAndClosure(n)
@@ -738,7 +738,7 @@ proc track(tracked: PEffects, n: PNode) =
         mergeTags(tracked, effectList.sons[tagEffects], n)
         gcsafeAndSideeffectCheck()
     if a.kind != nkSym or a.sym.magic != mNBindSym:
-      for i in 1 ..< len(n): trackOperand(tracked, n.sons[i], paramType(op, i), a)
+      for i in 1 ..< n.len: trackOperand(tracked, n.sons[i], paramType(op, i), a)
     if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}:
       # may not look like an assignment, but it is:
       let arg = n.sons[1]
@@ -754,7 +754,7 @@ proc track(tracked: PEffects, n: PNode) =
       track(tracked, n.sons[i])
   of nkDotExpr:
     guardDotAccess(tracked, n)
-    for i in 0 ..< len(n): track(tracked, n.sons[i])
+    for i in 0 ..< n.len: track(tracked, n.sons[i])
   of nkCheckedFieldExpr:
     track(tracked, n.sons[0])
     if warnProveField in tracked.config.notes:
@@ -821,7 +821,7 @@ proc track(tracked: PEffects, n: PNode) =
   of nkForStmt, nkParForStmt:
     # we are very conservative here and assume the loop is never executed:
     let oldState = tracked.init.len
-    for i in 0 .. len(n)-3:
+    for i in 0 .. n.len-3:
       let it = n[i]
       track(tracked, it)
       if tracked.owner.kind != skMacro:
@@ -830,13 +830,19 @@ proc track(tracked: PEffects, n: PNode) =
             createTypeBoundOps(tracked.c, x.typ, x.info)
         else:
           createTypeBoundOps(tracked.c, it.typ, it.info)
-    for i in len(n)-2..len(n)-1:
+    let iterCall = n[n.len-2]
-      track(tracked, n.sons[i])
+    let loopBody = n[n.len-1]
+    if tracked.owner.kind != skMacro and iterCall.safelen > 1:
+      # XXX this is a bit hacky:
+      if iterCall[1].typ != nil and iterCall[1].typ.skipTypes(abstractVar).kind notin {tyVarargs, tyOpenArray}:
+        createTypeBoundOps(tracked.c, iterCall[1].typ, iterCall[1].info)
+    track(tracked, iterCall)
+    track(tracked, loopBody)
     setLen(tracked.init, oldState)
   of nkObjConstr:
     when false: track(tracked, n.sons[0])
     let oldFacts = tracked.guards.s.len
-    for i in 1 ..< len(n):
+    for i in 1 ..< n.len:
       let x = n.sons[i]
       track(tracked, x)
       if x.sons[0].kind == nkSym and sfDiscriminant in x.sons[0].sym.flags:

compiler/semstmts.nim

@@ -43,7 +43,7 @@ proc semDiscard(c: PContext, n: PNode): PNode =
     n.sons[0] = semExprWithType(c, n.sons[0])
     let sonType = n.sons[0].typ
     let sonKind = n.sons[0].kind
-    if isEmptyType(sonType) or sonType.kind == tyNone or n.sons[0].kind == nkTypeOfExpr:
+    if isEmptyType(sonType) or sonType.kind in {tyNone, tyTypeDesc} or sonKind == nkTypeOfExpr:
       localError(c.config, n.info, errInvalidDiscard)
     if sonType.kind == tyProc and sonKind notin nkCallKinds:
       # tyProc is disallowed to prevent ``discard foo`` to be valid, when ``discard foo()`` is meant.
@@ -130,13 +130,13 @@ proc fixNilType(c: PContext; n: PNode) =
     for it in n: fixNilType(c, it)
   n.typ = nil
 
-proc discardCheck(c: PContext, expr: PNode, flags: TExprFlags): PNode =
+proc discardCheck(c: PContext, result: PNode, flags: TExprFlags) =
-  result = expr
   if c.matchedConcept != nil or efInTypeof in flags: return
 
   if result.typ != nil and result.typ.kind notin {tyStmt, tyVoid}:
     if implicitlyDiscardable(result):
-      result = newNode(nkDiscardStmt, result.info, @[result])
+      var n = newNodeI(nkDiscardStmt, result.info, 1)
+      n[0] = result
     elif result.typ.kind != tyError and c.config.cmd != cmdInteractive:
       var n = result
       while n.kind in skipForDiscardable: n = n.lastSon
@@ -168,8 +168,7 @@ proc semIf(c: PContext, n: PNode; flags: TExprFlags): PNode =
     else: illFormedAst(it, c.config)
   if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or
       (not hasElse and efInTypeof notin flags):
-    for it in n:
+    for it in n: discardCheck(c, it.lastSon, flags)
-      it.sons[^1] = discardCheck(c, it.sons[^1], flags)
     result.kind = nkIfStmt
     # propagate any enforced VoidContext:
     if typ == c.enforceVoidContext: result.typ = c.enforceVoidContext
@@ -267,14 +266,12 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags): PNode =
 
   dec c.p.inTryStmt
   if isEmptyType(typ) or typ.kind in {tyNil, tyExpr}:
-    n.sons[0] = discardCheck(c, n.sons[0], flags)
+    discardCheck(c, n.sons[0], flags)
-    for i in 1..n.len-1:
+    for i in 1..n.len-1: discardCheck(c, n.sons[i].lastSon, flags)
-      n.sons[i].sons[^1] = discardCheck(c, n.sons[i].sons[^1], flags)
     if typ == c.enforceVoidContext:
       result.typ = c.enforceVoidContext
   else:
-    if n.lastSon.kind == nkFinally:
+    if n.lastSon.kind == nkFinally: discardCheck(c, n.lastSon.lastSon, flags)
-      n.sons[^1].sons[^1] = discardCheck(c, n.sons[^1].sons[^1], flags)
     n.sons[0] = fitNode(c, typ, n.sons[0], n.sons[0].info)
     for i in 1..last:
       var it = n.sons[i]
@@ -474,10 +471,14 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
                      ($typ.kind).substr(2).toLowerAscii)
         elif typ.kind == tyProc and tfUnresolved in typ.flags:
           localError(c.config, def.info, errProcHasNoConcreteType % def.renderTree)
-        elif symkind == skVar and typ.kind == tyOwned and def.kind notin nkCallKinds:
+        when false:
-          # special type inference rule: 'var it = ownedPointer' is turned
+          # XXX This typing rule is neither documented nor complete enough to
-          # into an unowned pointer.
+          # justify it. Instead use the newer 'unowned x' until we figured out
-          typ = typ.lastSon
+          # a more general solution.
+          if symkind == skVar and typ.kind == tyOwned and def.kind notin nkCallKinds:
+            # special type inference rule: 'var it = ownedPointer' is turned
+            # into an unowned pointer.
+            typ = typ.lastSon
     else:
       if symkind == skLet: localError(c.config, a.info, errLetNeedsInit)
 
@@ -729,7 +730,7 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode =
   openScope(c)
   n.sons[length-1] = semExprBranch(c, n.sons[length-1], flags)
   if efInTypeof notin flags:
-    n.sons[^1] = discardCheck(c, n.sons[^1], flags)
+    discardCheck(c, n.sons[length-1], flags)
   closeScope(c)
   dec(c.p.nestedLoopCounter)
 
@@ -919,8 +920,7 @@ proc semCase(c: PContext, n: PNode; flags: TExprFlags): PNode =
   closeScope(c)
   if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or
       (not hasElse and efInTypeof notin flags):
-    for i in 1..n.len-1:
+    for i in 1..n.len-1: discardCheck(c, n.sons[i].lastSon, flags)
-      n.sons[i].sons[^1] = discardCheck(c, n.sons[i].sons[^1], flags)
     # propagate any enforced VoidContext:
     if typ == c.enforceVoidContext:
       result.typ = c.enforceVoidContext
@@ -1568,7 +1568,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
       if obj.kind in {tyObject, tyDistinct, tySequence, tyString}:
         obj = canonType(c, obj)
         if obj.destructor.isNil:
-          obj.destructor = s
+          obj.attachedOps[attachedDestructor] = s
         else:
           prevDestructor(c, obj.destructor, obj, n.info)
         noError = true
@@ -1592,7 +1592,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
         elif t.kind == tyGenericInvocation: t = t.sons[0]
         else: break
       if t.kind in {tyObject, tyDistinct, tyEnum, tySequence, tyString}:
-        if t.deepCopy.isNil: t.deepCopy = s
+        if t.attachedOps[attachedDeepCopy].isNil: t.attachedOps[attachedDeepCopy] = s
         else:
           localError(c.config, n.info, errGenerated,
                      "cannot bind another 'deepCopy' to: " & typeToString(t))
@@ -1631,11 +1631,11 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
         # attach these ops to the canonical tySequence
         obj = canonType(c, obj)
         #echo "ATTACHING TO ", obj.id, " ", s.name.s, " ", cast[int](obj)
-        let opr = if s.name.s == "=": addr(obj.assignment) else: addr(obj.sink)
+        let k = if name == "=": attachedAsgn else: attachedSink
-        if opr[].isNil:
+        if obj.attachedOps[k].isNil:
-          opr[] = s
+          obj.attachedOps[k] = s
         else:
-          prevDestructor(c, opr[], obj, n.info)
+          prevDestructor(c, obj.attachedOps[k], obj, n.info)
         if obj.owner.getModule != s.getModule:
           localError(c.config, n.info, errGenerated,
             "type bound operation `" & name & "` can be defined only in the same module with its type (" & obj.typeToString() & ")")
@@ -1828,7 +1828,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
 
   if sfOverriden in s.flags or s.name.s[0] == '=': semOverride(c, s, n)
   if s.name.s[0] in {'.', '('}:
-    if s.name.s in [".", ".()", ".="] and {destructor, dotOperators} * c.features == {}:
+    if s.name.s in [".", ".()", ".="] and {Feature.destructor, dotOperators} * c.features == {}:
       localError(c.config, n.info, "the overloaded " & s.name.s &
         " operator has to be enabled with {.experimental: \"dotOperators\".}")
     elif s.name.s == "()" and callOperator notin c.features:
@@ -2124,7 +2124,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
       n.typ = n.sons[i].typ
       if not isEmptyType(n.typ): n.kind = nkStmtListExpr
     elif i != last or voidContext:
-      n.sons[i] = discardCheck(c, n.sons[i], flags)
+      discardCheck(c, n.sons[i], flags)
     else:
       n.typ = n.sons[i].typ
       if not isEmptyType(n.typ): n.kind = nkStmtListExpr

compiler/semtempl.nim

@@ -559,9 +559,11 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
   else:
     s = semIdentVis(c, skTemplate, n.sons[0], {})
 
-  if s.owner != nil and sfSystemModule in s.owner.flags and
+  if s.owner != nil:
-      s.name.s in ["!=", ">=", ">", "incl", "excl", "in", "notin", "isnot"]:
+    const names = ["!=", ">=", ">", "incl", "excl", "in", "notin", "isnot"]
-    incl(s.flags, sfCallsite)
+    if sfSystemModule in s.owner.flags and s.name.s in names or
+       s.owner.name.s == "vm" and s.name.s == "stackTrace":
+      incl(s.flags, sfCallsite)
 
   styleCheckDef(c.config, s)
   onDef(n[0].info, s)

compiler/semtypes.nim

@@ -1348,8 +1348,8 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
   # special check for generic object with
   # generic/partial specialized parent
   let tx = result.skipTypes(abstractPtrs, 50)
-  if tx.isNil:
+  if tx.isNil or isTupleRecursive(tx):
-    localError(c.config, n.info, "invalid recursion in type '$1'" % typeToString(result[0]))
+    localError(c.config, n.info, "illegal recursion in type '$1'" % typeToString(result[0]))
     return errorType(c)
   if tx != result and tx.kind == tyObject and tx.sons[0] != nil:
     semObjectTypeForInheritedGenericInst(c, n, tx)

compiler/semtypinst.nim

@@ -380,13 +380,13 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   if newbody.isGenericAlias: newbody = newbody.skipGenericAlias
   rawAddSon(result, newbody)
   checkPartialConstructedType(cl.c.config, cl.info, newbody)
-  let dc = newbody.deepCopy
+  let dc = newbody.attachedOps[attachedDeepCopy]
   if not cl.allowMetaTypes:
-    if dc != nil and sfFromGeneric notin newbody.deepCopy.flags:
+    if dc != nil and sfFromGeneric notin newbody.attachedOps[attachedDeepCopy].flags:
       # 'deepCopy' needs to be instantiated for
       # generics *when the type is constructed*:
-      newbody.deepCopy = cl.c.instTypeBoundOp(cl.c, dc, result, cl.info,
+      newbody.attachedOps[attachedDeepCopy] = cl.c.instTypeBoundOp(cl.c, dc, result, cl.info,
-                                              attachedDeepCopy, 1)
+                                                                   attachedDeepCopy, 1)
     if bodyIsNew and newbody.typeInst == nil:
       #doassert newbody.typeInst == nil
       newbody.typeInst = result
@@ -592,7 +592,8 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
         skipIntLiteralParams(result)
 
       of tySequence:
-        if cl.isReturnType and cl.c.config.selectedGc == gcDestructors and result.destructor.isNil and
+        if cl.isReturnType and cl.c.config.selectedGc == gcDestructors and
+            result.attachedOps[attachedDestructor].isNil and
             result[0].kind != tyEmpty and optNimV2 notin cl.c.config.globalOptions:
           let s = cl.c.graph.sysTypes[tySequence]
           var old = copyType(s, s.owner, keepId=false)
@@ -601,9 +602,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
           old.n = nil
           old.flags = {tfHasAsgn}
           old.addSonSkipIntLit result[0]
-          result.destructor = old.destructor
+          result.attachedOps = old.attachedOps
-          result.assignment = old.assignment
-          result.sink = old.sink
           cl.c.typesWithOps.add((result, old))
 
       else: discard
@@ -619,19 +618,19 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
         result.n = replaceObjBranches(cl, result.n)
 
 template typeBound(c, newty, oldty, field, info) =
-  let opr = newty.field
+  let opr = newty.attachedOps[field]
   if opr != nil and sfFromGeneric notin opr.flags:
     # '=' needs to be instantiated for generics when the type is constructed:
     #echo "DESTROY: instantiating ", astToStr(field), " for ", typeToString(oldty)
-    newty.field = c.instTypeBoundOp(c, opr, oldty, info, attachedAsgn, 1)
+    newty.attachedOps[field] = c.instTypeBoundOp(c, opr, oldty, info, attachedAsgn, 1)
 
 proc instAllTypeBoundOp*(c: PContext, info: TLineInfo) =
   var i = 0
   while i < c.typesWithOps.len:
     let (newty, oldty) = c.typesWithOps[i]
-    typeBound(c, newty, oldty, destructor, info)
+    typeBound(c, newty, oldty, attachedDestructor, info)
-    typeBound(c, newty, oldty, sink, info)
+    typeBound(c, newty, oldty, attachedSink, info)
-    typeBound(c, newty, oldty, assignment, info)
+    typeBound(c, newty, oldty, attachedAsgn, info)
     inc i
   setLen(c.typesWithOps, 0)
 

compiler/sighashes.nim

@@ -349,7 +349,7 @@ proc hashBodyTree(graph: ModuleGraph, c: var MD5Context, n: PNode) =
     c &= n.strVal
   else:
     for i in 0..<n.len:
-      hashTree(c, n.sons[i])
+      hashBodyTree(graph, c, n.sons[i])
 
 proc symBodyDigest*(graph: ModuleGraph, sym: PSym): SigHash =
   ## compute unique digest of the proc/func/method symbols
@@ -369,7 +369,7 @@ proc symBodyDigest*(graph: ModuleGraph, sym: PSym): SigHash =
   if sym.ast != nil:
     md5Init(c)
     c.md5Update(cast[cstring](result.addr), sizeof(result))
-    c.hashTree(sym.ast[bodyPos])
+    hashBodyTree(graph, c, sym.ast[bodyPos])
     c.md5Final(result.Md5Digest)
     graph.symBodyHashes[sym.id] = result
 

compiler/sigmatch.nim

@@ -2092,14 +2092,21 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
         result = localConvMatch(c, m, f, a, arg)
       else:
         r = typeRel(m, base(f), a)
-        if r >= isGeneric:
+        case r
+        of isGeneric:
           inc(m.convMatches)
           result = copyTree(arg)
-          if r == isGeneric:
+          result.typ = getInstantiatedType(c, arg, m, base(f))
-            result.typ = getInstantiatedType(c, arg, m, base(f))
           m.baseTypeMatch = true
-        # bug #4799, varargs accepting subtype relation object
+        of isFromIntLit:
-        elif r == isSubtype:
+          inc(m.intConvMatches, 256)
+          result = implicitConv(nkHiddenStdConv, f[0], arg, m, c)
+          m.baseTypeMatch = true
+        of isEqual:
+          inc(m.convMatches)
+          result = copyTree(arg)
+          m.baseTypeMatch = true
+        of isSubtype: # bug #4799, varargs accepting subtype relation object
           inc(m.subtypeMatches)
           if base(f).kind == tyTypeDesc:
             result = arg

compiler/transf.nim

@@ -651,6 +651,8 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
       var t = formal.typ
       if formal.ast != nil and formal.ast.typ.destructor != nil and t.destructor == nil:
         t = formal.ast.typ # better use the type that actually has a destructor.
+      elif t.destructor == nil and arg.typ.destructor != nil:
+        t = arg.typ
       # generate a temporary and produce an assignment statement:
       var temp = newTemp(c, t, formal.info)
       #temp.sym.flags.incl sfCursor

compiler/trees.nim

@@ -101,7 +101,7 @@ proc isDeepConstExpr*(n: PNode): bool =
       if not isDeepConstExpr(n.sons[i]): return false
     if n.typ.isNil: result = true
     else:
-      let t = n.typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink})
+      let t = n.typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink, tyOwned})
       if t.kind in {tyRef, tyPtr}: return false
       if t.kind != tyObject or not isCaseObj(t.n):
         result = true

compiler/vmgen.nim

@@ -931,6 +931,21 @@ proc ldNullOpcode(t: PType): TOpcode =
   assert t != nil
   if fitsRegister(t): opcLdNullReg else: opcLdNull
 
+proc whichAsgnOpc(n: PNode): TOpcode =
+  case n.typ.skipTypes(abstractRange+{tyOwned}-{tyTypeDesc}).kind
+  of tyBool, tyChar, tyEnum, tyOrdinal, tyInt..tyInt64, tyUInt..tyUInt64:
+    opcAsgnInt
+  of tyString, tyCString:
+    opcAsgnStr
+  of tyFloat..tyFloat128:
+    opcAsgnFloat
+  of tyRef, tyNil, tyVar, tyLent, tyPtr:
+    opcAsgnRef
+  else:
+    opcAsgnComplex
+
+proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode = opc
+
 proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   case m
   of mAnd: c.genAndOr(n, opcFJmp, dest)
@@ -957,7 +972,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.genNarrow(n.sons[1], d)
     c.genAsgnPatch(n.sons[1], d)
     c.freeTemp(d)
-  of mOrd, mChr, mArrToSeq: c.gen(n.sons[1], dest)
+  of mOrd, mChr, mArrToSeq, mUnown: c.gen(n.sons[1], dest)
   of mNew, mNewFinalize:
     unused(c, n, dest)
     c.genNew(n)
@@ -1330,6 +1345,17 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mRunnableExamples:
     discard "just ignore any call to runnableExamples"
   of mDestroy: discard "ignore calls to the default destructor"
+  of mMove:
+    let arg = n[1]
+    let a = c.genx(arg)
+    assert dest >= 0
+    if dest < 0: dest = c.getTemp(arg.typ)
+    gABC(c, arg, whichAsgnOpc(arg), dest, a, 1)
+    # XXX use ldNullOpcode() here?
+    c.gABx(n, opcLdNull, a, c.genType(arg.typ))
+    c.gABx(n, opcNodeToReg, a, a)
+    c.genAsgnPatch(arg, a)
+    c.freeTemp(a)
   else:
     # mGCref, mGCunref,
     globalError(c.config, n.info, "cannot generate code for: " & $m)
@@ -1423,21 +1449,6 @@ proc genDeref(c: PCtx, n: PNode, dest: var TDest, flags: TGenFlags) =
     if {gfNodeAddr, gfNode} * flags == {} and fitsRegister(n.typ):
       c.gABC(n, opcNodeToReg, dest, dest)
 
-proc whichAsgnOpc(n: PNode): TOpcode =
-  case n.typ.skipTypes(abstractRange+{tyOwned}-{tyTypeDesc}).kind
-  of tyBool, tyChar, tyEnum, tyOrdinal, tyInt..tyInt64, tyUInt..tyUInt64:
-    opcAsgnInt
-  of tyString, tyCString:
-    opcAsgnStr
-  of tyFloat..tyFloat128:
-    opcAsgnFloat
-  of tyRef, tyNil, tyVar, tyLent, tyPtr:
-    opcAsgnRef
-  else:
-    opcAsgnComplex
-
-proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode = opc
-
 proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) =
   let tmp = c.genx(ri)
   assert dest >= 0
@@ -1585,8 +1596,8 @@ proc genGlobalInit(c: PCtx; n: PNode; s: PSym) =
   #   var decls{.compileTime.}: seq[NimNode] = @[]
   let dest = c.getTemp(s.typ)
   c.gABx(n, opcLdGlobal, dest, s.position)
-  if s.ast != nil:
+  if s.astdef != nil:
-    let tmp = c.genx(s.ast)
+    let tmp = c.genx(s.astdef)
     c.genAdditionalCopy(n, opcWrDeref, dest, 0, tmp)
     c.freeTemp(dest)
     c.freeTemp(tmp)

doc/docgen.rst

@@ -18,6 +18,21 @@ from input .nim files and projects, as well as HTML and LaTeX from input RST
 dependencies (``import``), any top-level documentation comments (##), and
 exported symbols (*), including procedures, types, and variables.
 
+Quick start
+-----------
+
+Generate HTML documentation for a file:
+
+::
+  nim doc <filename>.nim
+
+Generate HTML documentation for a whole project:
+
+::
+  # delete any htmldocs/*.idx file before starting
+  nim doc --project --index:on --git.url:<url> --git.commit:<tag> <main_filename>.nim
+  nim buildIndex -o:htmldocs/theindex.html htmldocs
+
 
 Documentation Comments
 ----------------------
@@ -186,21 +201,25 @@ file.
 See source switch
 -----------------
 
+The ``docSeeSrcUrl`` switch is deprecated. Use:
+
 ::
-  nim doc2 --docSeeSrcUrl:txt filename.nim
+  nim doc2 --git.url:<url> filename.nim
 
-When you pass the ``docSeeSrcUrl`` switch to docgen, after each documented item
+With the ``git.url`` switch the *See source* hyperlink will appear below each
-in your source code the hyper link *See source* will appear pointing to the
+documented item in your source code pointing to the implementation of that
-implementation of that item on a GitHub repository. You can click the link to
+item on a GitHub repository.
-see the implementation of the item.
+You can click the link to see the implementation of the item.
 
-If you want to reuse this feature in your own documentation you will have to
+The ``git.commit`` switch overrides the hardcoded `devel` branch in config/nimdoc.cfg.
-modify ``config/nimdoc.cfg`` to contain a ``doc.item.seesrc`` value with a
+This is useful to link to a different branch e.g. `--git.commit:master`,
-hyper link to your own code repository. As you will see by the comments in that
+or to a tag e.g. `--git.commit:1.2.3` or a commit.
-file, the value ``txt`` passed on the command line will be used in the HTML
-template along others like ``$path`` and ``$line``.
 
-In the case of Nim's own documentation, the ``txt`` value is just a commit
+Source URLs are generated as `href="${url}/tree/${commit}/${path}#L${line}"` by default and this compatible with GitHub but not with GitLab.
+
+You can edit ``config/nimdoc.cfg`` and modify the ``doc.item.seesrc`` value with a hyperlink to your own code repository.
+
+In the case of Nim's own documentation, the ``commit`` value is just a commit
 hash to append to a formatted URL to https://github.com/Araq/Nim. The
 ``tools/nimweb.nim`` helper queries the current git commit hash during doc
 generation, but since you might be working on an unpublished repository, it
@@ -231,7 +250,7 @@ HTML anchor generation
 ======================
 
 When you run the ``rst2html`` command, all sections in the RST document will
-get an anchor you can hyper link to. Usually you can guess the anchor lower
+get an anchor you can hyperlink to. Usually you can guess the anchor lower
 casing the section title and replacing spaces with dashes, and in any case you
 can get it from the table of contents. But when you run the ``doc`` or ``doc2``
 commands to generate API documentation, some symbol get one or two anchors at
@@ -249,7 +268,7 @@ will be their unquoted value after removing parameters, return types and
 pragmas. The plain name allows short and nice linking of symbols which works
 unless you have a module with collisions due to overloading.
 
-If you hyper link a plain name symbol and there are other matches on the same
+If you hyperlink a plain name symbol and there are other matches on the same
 HTML file, most browsers will go to the first one. To differentiate the rest,
 you will need to use the complex name. A complex name for a callable type is
 made up from several parts:
@@ -313,14 +332,14 @@ columns is:
 
 1. Mandatory term being indexed. Terms can include quoting according to
    Nim's rules (eg. \`^\`).
-2. Base filename plus anchor hyper link (eg.
+2. Base filename plus anchor hyperlink (eg.
    ``algorithm.html#*,int,SortOrder``).
-3. Optional human readable string to display as hyper link. If the value is not
+3. Optional human readable string to display as hyperlink. If the value is not
-   present or is the empty string, the hyper link will be rendered
+   present or is the empty string, the hyperlink will be rendered
    using the term. Prefix whitespace indicates that this entry is
    not for an API symbol but for a TOC entry.
-4. Optional title or description of the hyper link. Browsers usually display
+4. Optional title or description of the hyperlink. Browsers usually display
-   this as a tooltip after hovering a moment over the hyper link.
+   this as a tooltip after hovering a moment over the hyperlink.
 
 The index generation tools try to differentiate between documentation
 generated from ``.nim`` files and documentation generated from ``.txt`` or

doc/manual.rst

@@ -22,7 +22,10 @@ precise wording. This manual is constantly evolving into a proper specification.
 **Note**: The experimental features of Nim are
 covered `here <manual_experimental.html>`_.
 
-This document describes the lexis, the syntax, and the semantics of Nim.
+This document describes the lexis, the syntax, and the semantics of the Nim language.
+
+To learn how to compile Nim programs and generate documentation see
+`Compiler User Guide <nimc.html>`_ and `DocGen Tools Guide <docgen.html>`_.
 
 The language constructs are explained using an extended BNF, in which ``(a)*``
 means 0 or more ``a``'s, ``a+`` means 1 or more ``a``'s, and ``(a)?`` means an
@@ -2357,7 +2360,7 @@ argument's resolution:
   rem unresolvedExpression(undeclaredIdentifier)
 
 ``untyped`` and ``varargs[untyped]`` are the only metatype that are lazy in this sense, the other
-metatypes ``typed`` and ``type`` are not lazy.
+metatypes ``typed`` and ``typedesc`` are not lazy.
 
 
 Varargs matching
@@ -4434,7 +4437,7 @@ templates:
 | ``notin`` and ``isnot`` have the obvious meanings.
 
 The "types" of templates can be the symbols ``untyped``,
-``typed`` or ``type``. These are "meta types", they can only be used in certain
+``typed`` or ``typedesc``. These are "meta types", they can only be used in certain
 contexts. Regular types can be used too; this implies that ``typed`` expressions
 are expected.
 
@@ -4598,7 +4601,7 @@ In templates identifiers can be constructed with the backticks notation:
 .. code-block:: nim
     :test: "nim c $1"
 
-  template typedef(name: untyped, typ: type) =
+  template typedef(name: untyped, typ: typedesc) =
     type
       `T name`* {.inject.} = typ
       `P name`* {.inject.} = ref `T name`
@@ -4660,7 +4663,7 @@ template cannot be accessed in the instantiation context:
 .. code-block:: nim
     :test: "nim c $1"
 
-  template newException*(exceptn: type, message: string): untyped =
+  template newException*(exceptn: typedesc, message: string): untyped =
     var
       e: ref exceptn  # e is implicitly gensym'ed here
     new(e)
@@ -5095,27 +5098,26 @@ expression by coercing it to a corresponding ``static`` type:
 The complier will report any failure to evaluate the expression or a
 possible type mismatch error.
 
-type[T]
+typedesc[T]
--------
+-----------
 
 In many contexts, Nim allows you to treat the names of types as regular
 values. These values exists only during the compilation phase, but since
-all values must have a type, ``type`` is considered their special type.
+all values must have a type, ``typedesc`` is considered their special type.
 
-``type`` acts like a generic type. For instance, the type of the symbol
+``typedesc`` acts like a generic type. For instance, the type of the symbol
-``int`` is ``type[int]``. Just like with regular generic types, when the
+``int`` is ``typedesc[int]``. Just like with regular generic types, when the
-generic param is ommited, ``type`` denotes the type class of all types.
+generic param is ommited, ``typedesc`` denotes the type class of all types.
-As a syntactic convenience, you can also use ``type`` as a modifier.
+As a syntactic convenience, you can also use ``typedesc`` as a modifier.
-``type int`` is considered the same as ``type[int]``.
 
-Procs featuring ``type`` params are considered implicitly generic.
+Procs featuring ``typedesc`` params are considered implicitly generic.
 They will be instantiated for each unique combination of supplied types
 and within the body of the proc, the name of each param will refer to
 the bound concrete type:
 
 .. code-block:: nim
 
-  proc new(T: type): ref T =
+  proc new(T: typedesc): ref T =
     echo "allocating ", T.name
     new(result)
 
@@ -5126,14 +5128,14 @@ When multiple type params are present, they will bind freely to different
 types. To force a bind-once behavior one can use an explicit generic param:
 
 .. code-block:: nim
-  proc acceptOnlyTypePairs[T, U](A, B: type[T]; C, D: type[U])
+  proc acceptOnlyTypePairs[T, U](A, B: typedesc[T]; C, D: typedesc[U])
 
 Once bound, type params can appear in the rest of the proc signature:
 
 .. code-block:: nim
     :test: "nim c $1"
 
-  template declareVariableWithType(T: type, value: T) =
+  template declareVariableWithType(T: typedesc, value: T) =
     var x: T = value
 
   declareVariableWithType int, 42
@@ -5145,8 +5147,8 @@ types that will match the type param:
 .. code-block:: nim
     :test: "nim c $1"
 
-  template maxval(T: type int): int = high(int)
+  template maxval(T: typedesc[int]): int = high(int)
-  template maxval(T: type float): float = Inf
+  template maxval(T: typedesc[float]): float = Inf
 
   var i = int.maxval
   var f = float.maxval
@@ -5159,7 +5161,8 @@ The constraint can be a concrete type or a type class.
 typeof operator
 ---------------
 
-**Note**: ``typeof(x)`` can also be written as ``type(x)``.
+**Note**: ``typeof(x)`` can also be written as ``type(x)`` but ``type(x)``
+is discouraged.
 
 You can obtain the type of a given expression by constructing a ``typeof``
 value from it (in many other languages this is known as the `typeof`:idx:
@@ -6581,7 +6584,7 @@ Custom pragmas are defined using templates annotated with pragma ``pragma``:
 .. code-block:: nim
   template dbTable(name: string, table_space: string = "") {.pragma.}
   template dbKey(name: string = "", primary_key: bool = false) {.pragma.}
-  template dbForeignKey(t: type) {.pragma.}
+  template dbForeignKey(t: typedesc) {.pragma.}
   template dbIgnore {.pragma.}
 
 

doc/manual_experimental.rst

@@ -486,9 +486,9 @@ The concept types can be parametric just like the regular generic types:
     M.data[m * M.N + n] = v
 
   # Adapt the Matrix type to the concept's requirements
-  template Rows*(M: type Matrix): int = M.M
+  template Rows*(M: typedesc[Matrix]): int = M.M
-  template Cols*(M: type Matrix): int = M.N
+  template Cols*(M: typedesc[Matrix]): int = M.N
-  template ValueType*(M: type Matrix): type = M.T
+  template ValueType*(M: typedesc[Matrix]): typedesc = M.T
 
   -------------
   ### usage.nim

lib/core/macros.nim

@@ -1076,20 +1076,24 @@ proc expectKind*(n: NimNode; k: set[NimNodeKind]) {.compileTime.} =
   ## macros that check the AST that is passed to them.
   if n.kind notin k: error("Expected one of " & $k & ", got " & $n.kind, n)
 
-proc newProc*(name = newEmptyNode(); params: openArray[NimNode] = [newEmptyNode()];
+proc newProc*(name = newEmptyNode();
-    body: NimNode = newStmtList(), procType = nnkProcDef): NimNode {.compileTime.} =
+              params: openArray[NimNode] = [newEmptyNode()];
+              body: NimNode = newStmtList();
+              procType = nnkProcDef;
+              pragmas: NimNode = newEmptyNode()): NimNode {.compileTime.} =
   ## shortcut for creating a new proc
   ##
   ## The ``params`` array must start with the return type of the proc,
   ## followed by a list of IdentDefs which specify the params.
   if procType notin RoutineNodes:
     error("Expected one of " & $RoutineNodes & ", got " & $procType)
+  pragmas.expectKind({nnkEmpty, nnkPragma})
   result = newNimNode(procType).add(
     name,
     newEmptyNode(),
     newEmptyNode(),
     newNimNode(nnkFormalParams).add(params),
-    newEmptyNode(),  # pragmas
+    pragmas,
     newEmptyNode(),
     body)
 

lib/core/runtime_v2.nim

@@ -47,8 +47,9 @@ proc nimNewObj(size: int): pointer {.compilerRtl.} =
   when defined(nimscript):
     discard
   elif defined(useMalloc):
-    result = c_malloc(s) +! sizeof(RefHeader)
+    var orig = c_malloc(s)
-    nimZeroMem(result, s)
+    nimZeroMem(orig, s)
+    result = orig +! sizeof(RefHeader)
   else:
     result = alloc0(s) +! sizeof(RefHeader)
   inc allocs

lib/core/seqs.nim

@@ -104,6 +104,10 @@ proc newSeqPayload(cap, elemSize: int): pointer {.compilerRtl, raises: [].} =
 proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize: int): pointer {.
     compilerRtl, noSideEffect, raises: [].} =
   {.noSideEffect.}:
+    template `+!`(p: pointer, s: int): pointer =
+      cast[pointer](cast[int](p) +% s)
+
+    const headerSize = sizeof(int) + sizeof(Allocator)
     if addlen <= 0:
       result = p
     elif p == nil:
@@ -112,14 +116,23 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize: int): pointer {.
       # Note: this means we cannot support things that have internal pointers as
       # they get reallocated here. This needs to be documented clearly.
       var p = cast[ptr PayloadBase](p)
-      let allocator = if p.allocator == nil: getLocalAllocator() else: p.allocator
       let cap = max(resize(p.cap), len+addlen)
-      var q = cast[ptr PayloadBase](allocator.realloc(allocator, p,
+      if p.allocator == nil:
-        sizeof(int) + sizeof(Allocator) + elemSize * p.cap,
+        let allocator = getLocalAllocator()
-        sizeof(int) + sizeof(Allocator) + elemSize * cap))
+        var q = cast[ptr PayloadBase](allocator.alloc(allocator,
-      q.allocator = allocator
+          headerSize + elemSize * cap))
-      q.cap = cap
+        copyMem(q +! headerSize, p +! headerSize, len * elemSize)
-      result = q
+        q.allocator = allocator
+        q.cap = cap
+        result = q
+      else:
+        let allocator = p.allocator
+        var q = cast[ptr PayloadBase](allocator.realloc(allocator, p,
+          headerSize + elemSize * p.cap,
+          headerSize + elemSize * cap))
+        q.allocator = allocator
+        q.cap = cap
+        result = q
 
 proc shrink*[T](x: var seq[T]; newLen: Natural) =
   mixin `=destroy`
@@ -140,6 +153,20 @@ proc grow*[T](x: var seq[T]; newLen: Natural; value: T) =
   for i in oldLen .. newLen-1:
     xu.p.data[i] = value
 
+proc add*[T](x: var seq[T]; value: sink T) {.magic: "AppendSeqElem", noSideEffect.} =
+  ## Generic proc for adding a data item `y` to a container `x`.
+  ##
+  ## For containers that have an order, `add` means *append*. New generic
+  ## containers should also call their adding proc `add` for consistency.
+  ## Generic code becomes much easier to write if the Nim naming scheme is
+  ## respected.
+  let oldLen = x.len
+  var xu = cast[ptr NimSeqV2[T]](addr x)
+  if xu.p == nil or xu.p.cap < oldLen+1:
+    xu.p = cast[typeof(xu.p)](prepareSeqAdd(oldLen, xu.p, 1, sizeof(T)))
+  xu.len = oldLen+1
+  xu.p.data[oldLen] = value
+
 proc setLen[T](s: var seq[T], newlen: Natural) =
   {.noSideEffect.}:
     if newlen < s.len:

lib/impure/db_mysql.nim

@@ -14,7 +14,7 @@
 ## `db_postgres <db_postgres.html>`_.
 ##
 ## Parameter substitution
-## ----------------------
+## ======================
 ##
 ## All ``db_*`` modules support the same form of parameter substitution.
 ## That is, using the ``?`` (question mark) to signify the place where a
@@ -25,10 +25,10 @@
 ##
 ##
 ## Examples
-## --------
+## ========
 ##
 ## Opening a connection to a database
-## ==================================
+## ----------------------------------
 ##
 ## .. code-block:: Nim
 ##     import db_mysql
@@ -36,7 +36,7 @@
 ##     db.close()
 ##
 ## Creating a table
-## ================
+## ----------------
 ##
 ## .. code-block:: Nim
 ##      db.exec(sql"DROP TABLE IF EXISTS myTable")
@@ -45,14 +45,14 @@
 ##                       name varchar(50) not null)"""))
 ##
 ## Inserting data
-## ==============
+## --------------
 ##
 ## .. code-block:: Nim
 ##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",
 ##             "Dominik")
 ##
 ## Larger example
-## ==============
+## --------------
 ##
 ## .. code-block:: Nim
 ##

lib/impure/db_odbc.nim

@@ -20,7 +20,7 @@
 ## `db_mysql <db_mysql.html>`_.
 ##
 ## Parameter substitution
-## ----------------------
+## ======================
 ##
 ## All ``db_*`` modules support the same form of parameter substitution.
 ## That is, using the ``?`` (question mark) to signify the place where a
@@ -31,10 +31,10 @@
 ##
 ##
 ## Examples
-## --------
+## ========
 ##
 ## Opening a connection to a database
-## ==================================
+## ----------------------------------
 ##
 ## .. code-block:: Nim
 ##     import db_odbc
@@ -42,7 +42,7 @@
 ##     db.close()
 ##
 ## Creating a table
-## ================
+## ----------------
 ##
 ## .. code-block:: Nim
 ##      db.exec(sql"DROP TABLE IF EXISTS myTable")
@@ -51,14 +51,14 @@
 ##                       name varchar(50) not null)"""))
 ##
 ## Inserting data
-## ==============
+## --------------
 ##
 ## .. code-block:: Nim
 ##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",
 ##             "Andreas")
 ##
 ## Large example
-## =============
+## -------------
 ##
 ## .. code-block:: Nim
 ##

lib/impure/db_postgres.nim

@@ -14,7 +14,7 @@
 ## `db_mysql <db_mysql.html>`_.
 ##
 ## Parameter substitution
-## ----------------------
+## ======================
 ##
 ## All ``db_*`` modules support the same form of parameter substitution.
 ## That is, using the ``?`` (question mark) to signify the place where a
@@ -38,10 +38,10 @@
 ##           3)
 ##
 ## Examples
-## --------
+## ========
 ##
 ## Opening a connection to a database
-## ==================================
+## ----------------------------------
 ##
 ## .. code-block:: Nim
 ##     import db_postgres
@@ -49,7 +49,7 @@
 ##     db.close()
 ##
 ## Creating a table
-## ================
+## ----------------
 ##
 ## .. code-block:: Nim
 ##      db.exec(sql"DROP TABLE IF EXISTS myTable")
@@ -58,7 +58,7 @@
 ##                       name varchar(50) not null)"""))
 ##
 ## Inserting data
-## ==============
+## --------------
 ##
 ## .. code-block:: Nim
 ##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",

lib/impure/db_sqlite.nim

@@ -14,7 +14,7 @@
 ## `db_mysql <db_mysql.html>`_.
 ##
 ## Parameter substitution
-## ----------------------
+## ======================
 ##
 ## All ``db_*`` modules support the same form of parameter substitution.
 ## That is, using the ``?`` (question mark) to signify the place where a
@@ -24,10 +24,10 @@
 ##     sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)"
 ##
 ## Examples
-## --------
+## ========
 ##
 ## Opening a connection to a database
-## ==================================
+## ----------------------------------
 ##
 ## .. code-block:: Nim
 ##     import db_sqlite
@@ -35,7 +35,7 @@
 ##     db.close()
 ##
 ## Creating a table
-## ================
+## ----------------
 ##
 ## .. code-block:: Nim
 ##      db.exec(sql"DROP TABLE IF EXISTS myTable")
@@ -44,14 +44,14 @@
 ##                       name varchar(50) not null)"""))
 ##
 ## Inserting data
-## ==============
+## --------------
 ##
 ## .. code-block:: Nim
 ##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",
 ##             "Jack")
 ##
 ## Larger example
-## ==============
+## --------------
 ##
 ## .. code-block:: nim
 ##

lib/impure/nre.nim

@@ -6,18 +6,6 @@
 #    distribution, for details about the copyright.
 #
 
-
-from pcre import nil
-import nre/private/util
-import tables
-from strutils import `%`
-from math import ceil
-import options
-from unicode import runeLenAt
-
-export options
-
-
 ## What is NRE?
 ## ============
 ##
@@ -67,6 +55,15 @@ runnableExamples:
     let matchBounds = firstVowel.get().captureBounds[-1]
     doAssert matchBounds.a == 1
 
+from pcre import nil
+import nre/private/util
+import tables
+from strutils import `%`
+from math import ceil
+import options
+from unicode import runeLenAt
+
+export options
 
 # Type definitions {{{
 type

lib/js/asyncjs.nim

@@ -50,7 +50,7 @@
 ##   proc loadGame(name: string): Future[Game] {.async.}
 ##
 ## JavaScript compatibility
-## ~~~~~~~~~~~~~~~~~~~~~~~~~
+## ========================
 ##
 ## Nim currently generates `async/await` JavaScript code which is supported in modern
 ## EcmaScript and most modern versions of browsers, Node.js and Electron.

lib/pure/asyncdispatch.nim

@@ -7,23 +7,6 @@
 #    distribution, for details about the copyright.
 #
 
-include "system/inclrtl"
-
-import os, tables, strutils, times, heapqueue, lists, options, asyncstreams
-import options, math
-import asyncfutures except callSoon
-
-import nativesockets, net, deques
-
-export Port, SocketFlag
-export asyncfutures except callSoon
-export asyncstreams
-
-#{.injectStmt: newGcInvariant().}
-
-## AsyncDispatch
-## *************
-##
 ## This module implements asynchronous IO. This includes a dispatcher,
 ## a ``Future`` type implementation, and an ``async`` macro which allows
 ## asynchronous code to be written in a synchronous style with the ``await``
@@ -82,7 +65,7 @@ export asyncstreams
 ## error), if there is no error however it returns the value of the future.
 ##
 ## Asynchronous procedures
-## -----------------------
+## =======================
 ##
 ## Asynchronous procedures remove the pain of working with callbacks. They do
 ## this by allowing you to write asynchronous code the same way as you would
@@ -116,7 +99,7 @@ export asyncstreams
 ## exceptions in async procs.
 ##
 ## Handling Exceptions
-## ~~~~~~~~~~~~~~~~~~~
+## -------------------
 ##
 ## The most reliable way to handle exceptions is to use ``yield`` on a future
 ## then check the future's ``failed`` property. For example:
@@ -142,7 +125,7 @@ export asyncstreams
 ##
 ##
 ## Discarding futures
-## ------------------
+## ==================
 ##
 ## Futures should **never** be discarded. This is because they may contain
 ## errors. If you do not care for the result of a Future then you should
@@ -151,17 +134,31 @@ export asyncstreams
 ## ``waitFor`` for that purpose.
 ##
 ## Examples
-## --------
+## ========
 ##
 ## For examples take a look at the documentation for the modules implementing
 ## asynchronous IO. A good place to start is the
 ## `asyncnet module <asyncnet.html>`_.
 ##
 ## Limitations/Bugs
-## ----------------
+## ================
 ##
 ## * The effect system (``raises: []``) does not work with async procedures.
 
+include "system/inclrtl"
+
+import os, tables, strutils, times, heapqueue, lists, options, asyncstreams
+import options, math
+import asyncfutures except callSoon
+
+import nativesockets, net, deques
+
+export Port, SocketFlag
+export asyncfutures except callSoon
+export asyncstreams
+
+#{.injectStmt: newGcInvariant().}
+
 # TODO: Check if yielded future is nil and throw a more meaningful exception
 
 type
@@ -1827,4 +1824,4 @@ proc setEvent*(ev: AsyncEvent) {.deprecated.} =
   ## Set event ``ev`` to signaled state.
   ##
   ## **Deprecated since v0.18.0:** Use ``trigger`` instead.
-  ev.trigger()
+  ev.trigger()

lib/pure/asyncftpclient.nim

@@ -16,7 +16,7 @@
 ## * Navigation through the FTP server's directories.
 ##
 ## Connecting to an FTP server
-## ------------------------
+## ===========================
 ##
 ## In order to begin any sort of transfer of files you must first
 ## connect to an FTP server. You can do so with the ``connect`` procedure.
@@ -34,7 +34,7 @@
 ## client will be connected after the ``await ftp.connect()`` call.
 ##
 ## Uploading a new file
-## --------------------
+## ====================
 ##
 ## After a connection is made you can use the ``store`` procedure to upload
 ## a new file to the FTP server. Make sure to check you are in the correct
@@ -53,7 +53,7 @@
 ##      waitFor(main())
 ##
 ## Checking the progress of a file transfer
-## ----------------------------------------
+## ========================================
 ##
 ## The progress of either a file upload or a file download can be checked
 ## by specifying a ``onProgressChanged`` procedure to the ``store`` or

lib/pure/asynchttpserver.nim

@@ -14,9 +14,8 @@
 ## application you should use a reverse proxy (for example nginx) instead of
 ## allowing users to connect directly to this server.
 ##
-##
 ## Basic usage
-## -----------
+## ===========
 ##
 ## This example will create an HTTP server on port 8080. The server will
 ## respond to all requests with a ``200 OK`` response code and "Hello World"

lib/pure/asyncmacro.nim

@@ -7,8 +7,6 @@
 #    distribution, for details about the copyright.
 #
 
-## AsyncMacro
-## *************
 ## `asyncdispatch` module depends on the `asyncmacro` module to work properly.
 
 import macros, strutils, asyncfutures

lib/pure/asyncnet.nim

@@ -11,7 +11,7 @@
 ## asynchronous dispatcher defined in the ``asyncdispatch`` module.
 ##
 ## Asynchronous IO in Nim
-## ----------------------
+## ======================
 ##
 ## Async IO in Nim consists of multiple layers (from highest to lowest):
 ##
@@ -49,7 +49,7 @@
 ## over all the layers, providing some extra features such as buffering.
 ##
 ## SSL
-## ----
+## ===
 ##
 ## SSL can be enabled by compiling with the ``-d:ssl`` flag.
 ##
@@ -58,10 +58,10 @@
 ## the newly created SSL context to get an SSL socket.
 ##
 ## Examples
-## --------
+## ========
 ##
 ## Chat server
-## ^^^^^^^^^^^
+## -----------
 ##
 ## The following example demonstrates a simple chat server.
 ##

lib/pure/base64.nim

@@ -15,7 +15,6 @@
 ## bytes (i.e., a total of 24 bits) can therefore be represented by
 ## four 6-bit Base64 digits.
 ##
-##
 ## Basic usage
 ## ===========
 ##

lib/pure/collections/hashcommon.nim

@@ -0,0 +1,68 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2019 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# An ``include`` file which contains common code for
+# hash sets and tables.
+
+const
+  growthFactor = 2
+
+when not defined(nimHasDefault):
+  template default[T](t: typedesc[T]): T =
+    var v: T
+    v
+
+# hcode for real keys cannot be zero.  hcode==0 signifies an empty slot.  These
+# two procs retain clarity of that encoding without the space cost of an enum.
+proc isEmpty(hcode: Hash): bool {.inline.} =
+  result = hcode == 0
+
+proc isFilled(hcode: Hash): bool {.inline.} =
+  result = hcode != 0
+
+proc nextTry(h, maxHash: Hash): Hash {.inline.} =
+  result = (h + 1) and maxHash
+
+proc mustRehash(length, counter: int): bool {.inline.} =
+  assert(length > counter)
+  result = (length * 2 < counter * 3) or (length - counter < 4)
+
+template rawGetKnownHCImpl() {.dirty.} =
+  if t.dataLen == 0:
+    return -1
+  var h: Hash = hc and maxHash(t)   # start with real hash value
+  while isFilled(t.data[h].hcode):
+    # Compare hc THEN key with boolean short circuit. This makes the common case
+    # zero ==key's for missing (e.g.inserts) and exactly one ==key for present.
+    # It does slow down succeeding lookups by one extra Hash cmp&and..usually
+    # just a few clock cycles, generally worth it for any non-integer-like A.
+    if t.data[h].hcode == hc and t.data[h].key == key:
+      return h
+    h = nextTry(h, maxHash(t))
+  result = -1 - h                   # < 0 => MISSING; insert idx = -1 - result
+
+proc rawGetKnownHC[X, A](t: X, key: A, hc: Hash): int {.inline.} =
+  rawGetKnownHCImpl()
+
+template genHashImpl(key, hc: typed) =
+  hc = hash(key)
+  if hc == 0:       # This almost never taken branch should be very predictable.
+    hc = 314159265  # Value doesn't matter; Any non-zero favorite is fine.
+
+template genHash(key: typed): Hash =
+  var res: Hash
+  genHashImpl(key, res)
+  res
+
+template rawGetImpl() {.dirty.} =
+  genHashImpl(key, hc)
+  rawGetKnownHCImpl()
+
+proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline.} =
+  rawGetImpl()

lib/pure/collections/lists.nim

@@ -80,14 +80,15 @@ type
   DoublyLinkedNodeObj*[T] = object ## A node a doubly linked list consists of.
     ##
     ## It consists of a `value` field, and pointers to `next` and `prev`.
-    next*, prev*: ref DoublyLinkedNodeObj[T]
+    next*: <//>(ref DoublyLinkedNodeObj[T])
+    prev*: ref DoublyLinkedNodeObj[T]
     value*: T
   DoublyLinkedNode*[T] = ref DoublyLinkedNodeObj[T]
 
   SinglyLinkedNodeObj*[T] = object ## A node a singly linked list consists of.
     ##
     ## It consists of a `value` field, and a pointer to `next`.
-    next*: ref SinglyLinkedNodeObj[T]
+    next*: <//>(ref SinglyLinkedNodeObj[T])
     value*: T
   SinglyLinkedNode*[T] = ref SinglyLinkedNodeObj[T]
 
@@ -95,19 +96,22 @@ type
     ##
     ## Use `initSinglyLinkedList proc <#initSinglyLinkedList>`_ to create
     ## a new empty list.
-    head*, tail*: SinglyLinkedNode[T]
+    head*: <//>(SinglyLinkedNode[T])
+    tail*: SinglyLinkedNode[T]
 
   DoublyLinkedList*[T] = object ## A doubly linked list.
     ##
     ## Use `initDoublyLinkedList proc <#initDoublyLinkedList>`_ to create
     ## a new empty list.
-    head*, tail*: DoublyLinkedNode[T]
+    head*: <//>(DoublyLinkedNode[T])
+    tail*: DoublyLinkedNode[T]
 
   SinglyLinkedRing*[T] = object ## A singly linked ring.
     ##
     ## Use `initSinglyLinkedRing proc <#initSinglyLinkedRing>`_ to create
     ## a new empty ring.
-    head*, tail*: SinglyLinkedNode[T]
+    head*: <//>(SinglyLinkedNode[T])
+    tail*: SinglyLinkedNode[T]
 
   DoublyLinkedRing*[T] = object ## A doubly linked ring.
     ##
@@ -147,7 +151,7 @@ proc initDoublyLinkedRing*[T](): DoublyLinkedRing[T] =
     var a = initDoublyLinkedRing[int]()
   discard
 
-proc newDoublyLinkedNode*[T](value: T): DoublyLinkedNode[T] =
+proc newDoublyLinkedNode*[T](value: T): <//>(DoublyLinkedNode[T]) =
   ## Creates a new doubly linked node with the given `value`.
   runnableExamples:
     var n = newDoublyLinkedNode[int](5)
@@ -156,7 +160,7 @@ proc newDoublyLinkedNode*[T](value: T): DoublyLinkedNode[T] =
   new(result)
   result.value = value
 
-proc newSinglyLinkedNode*[T](value: T): SinglyLinkedNode[T] =
+proc newSinglyLinkedNode*[T](value: T): <//>(SinglyLinkedNode[T]) =
   ## Creates a new singly linked node with the given `value`.
   runnableExamples:
     var n = newSinglyLinkedNode[int](5)

lib/pure/collections/sequtils.nim

@@ -949,6 +949,14 @@ macro mapLiterals*(constructor, op: untyped;
   ## works for nested tuples of arrays of sets etc.
   result = mapLitsImpl(constructor, op, nested.boolVal)
 
+iterator items*[T](xs: iterator: T): T =
+  ## iterates over each element yielded by a closure iterator. This may
+  ## not seem particularly useful on its own, but this allows closure
+  ## iterators to be used by the the mapIt, filterIt, allIt, anyIt, etc.
+  ## templates.
+  for x in xs():
+    yield x
+
 when isMainModule:
   import strutils
   from algorithm import sorted
@@ -1382,5 +1390,13 @@ when isMainModule:
     doAssert outp == @[@["a", "b"], @["c", "d"]]
 
 
+  block:
+    proc iter(len: int): auto =
+      result = iterator(): int =
+        for i in 0..<len:
+          yield i
+
+    doAssert: iter(3).mapIt(2*it).foldl(a + b) == 6
+
   when not defined(testing):
     echo "Finished doc tests"

lib/pure/collections/setimpl.nim

@@ -0,0 +1,153 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2019 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# An ``include`` file for the different hash set implementations.
+
+
+template maxHash(t): untyped = high(t.data)
+template dataLen(t): untyped = len(t.data)
+
+include hashcommon
+
+template initImpl(s: typed, size: int) =
+  assert isPowerOfTwo(size)
+  when s is OrderedSet:
+    s.first = -1
+    s.last = -1
+  s.counter = 0
+  newSeq(s.data, size)
+
+template rawInsertImpl() {.dirty.} =
+  if data.len == 0:
+    initImpl(s, defaultInitialSize)
+  data[h].key = key
+  data[h].hcode = hc
+
+proc rawInsert[A](s: var HashSet[A], data: var KeyValuePairSeq[A], key: A,
+                  hc: Hash, h: Hash) =
+  rawInsertImpl()
+
+proc enlarge[A](s: var HashSet[A]) =
+  var n: KeyValuePairSeq[A]
+  newSeq(n, len(s.data) * growthFactor)
+  swap(s.data, n)                   # n is now old seq
+  for i in countup(0, high(n)):
+    if isFilled(n[i].hcode):
+      var j = -1 - rawGetKnownHC(s, n[i].key, n[i].hcode)
+      rawInsert(s, s.data, n[i].key, n[i].hcode, j)
+
+template inclImpl() {.dirty.} =
+  if s.data.len == 0:
+    initImpl(s, defaultInitialSize)
+  var hc: Hash
+  var index = rawGet(s, key, hc)
+  if index < 0:
+    if mustRehash(len(s.data), s.counter):
+      enlarge(s)
+      index = rawGetKnownHC(s, key, hc)
+    rawInsert(s, s.data, key, hc, -1 - index)
+    inc(s.counter)
+
+template containsOrInclImpl() {.dirty.} =
+  if s.data.len == 0:
+    initImpl(s, defaultInitialSize)
+  var hc: Hash
+  var index = rawGet(s, key, hc)
+  if index >= 0:
+    result = true
+  else:
+    if mustRehash(len(s.data), s.counter):
+      enlarge(s)
+      index = rawGetKnownHC(s, key, hc)
+    rawInsert(s, s.data, key, hc, -1 - index)
+    inc(s.counter)
+
+template doWhile(a, b) =
+  while true:
+    b
+    if not a: break
+
+proc exclImpl[A](s: var HashSet[A], key: A) : bool {. inline .} =
+  var hc: Hash
+  var i = rawGet(s, key, hc)
+  var msk = high(s.data)
+  result = true
+
+  if i >= 0:
+    result = false
+    dec(s.counter)
+    while true:         # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1
+      var j = i         # The correctness of this depends on (h+1) in nextTry,
+      var r = j         # though may be adaptable to other simple sequences.
+      s.data[i].hcode = 0              # mark current EMPTY
+      s.data[i].key = default(type(s.data[i].key))
+      doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)):
+        i = (i + 1) and msk            # increment mod table size
+        if isEmpty(s.data[i].hcode):   # end of collision cluster; So all done
+          return
+        r = s.data[i].hcode and msk    # "home" location of key@i
+      shallowCopy(s.data[j], s.data[i]) # data[i] will be marked EMPTY next loop
+
+template dollarImpl() {.dirty.} =
+  result = "{"
+  for key in items(s):
+    if result.len > 1: result.add(", ")
+    result.addQuoted(key)
+  result.add("}")
+
+
+
+# --------------------------- OrderedSet ------------------------------
+
+proc rawGet[A](t: OrderedSet[A], key: A, hc: var Hash): int {.inline.} =
+  rawGetImpl()
+
+proc rawInsert[A](s: var OrderedSet[A], data: var OrderedKeyValuePairSeq[A],
+                  key: A, hc: Hash, h: Hash) =
+  rawInsertImpl()
+  data[h].next = -1
+  if s.first < 0: s.first = h
+  if s.last >= 0: data[s.last].next = h
+  s.last = h
+
+proc enlarge[A](s: var OrderedSet[A]) =
+  var n: OrderedKeyValuePairSeq[A]
+  newSeq(n, len(s.data) * growthFactor)
+  var h = s.first
+  s.first = -1
+  s.last = -1
+  swap(s.data, n)
+  while h >= 0:
+    var nxt = n[h].next
+    if isFilled(n[h].hcode):
+      var j = -1 - rawGetKnownHC(s, n[h].key, n[h].hcode)
+      rawInsert(s, s.data, n[h].key, n[h].hcode, j)
+    h = nxt
+
+proc exclImpl[A](s: var OrderedSet[A], key: A) : bool {.inline.} =
+  if len(s.data) == 0:
+    return true
+  var n: OrderedKeyValuePairSeq[A]
+  newSeq(n, len(s.data))
+  var h = s.first
+  s.first = -1
+  s.last = -1
+  swap(s.data, n)
+  let hc = genHash(key)
+  result = true
+  while h >= 0:
+    var nxt = n[h].next
+    if isFilled(n[h].hcode):
+      if n[h].hcode == hc and n[h].key == key:
+        dec s.counter
+        result = false
+      else:
+        var j = -1 - rawGetKnownHC(s, n[h].key, n[h].hcode)
+        rawInsert(s, s.data, n[h].key, n[h].hcode, j)
+    h = nxt

lib/pure/collections/sets.nim

@@ -26,7 +26,7 @@
 ##   `symmetric difference <#symmetricDifference,HashSet[A],HashSet[A]>`_
 ##
 ## .. code-block::
-##   echo toHashSet([9, 5, 1])         # {9, 1, 5}
+##   echo toHashSet([9, 5, 1])     # {9, 1, 5}
 ##   echo toOrderedSet([9, 5, 1])  # {9, 5, 1}
 ##
 ##   let
@@ -69,148 +69,32 @@ type
     data: KeyValuePairSeq[A]
     counter: int
 
+type
+  OrderedKeyValuePair[A] = tuple[
+    hcode: Hash, next: int, key: A]
+  OrderedKeyValuePairSeq[A] = seq[OrderedKeyValuePair[A]]
+  OrderedSet* {.myShallow.} [A] = object ## \
+    ## A generic hash set that remembers insertion order.
+    ##
+    ## Use `init proc <#init,OrderedSet[A],int>`_ or `initOrderedSet proc
+    ## <#initOrderedSet,int>`_ before calling other procs on it.
+    data: OrderedKeyValuePairSeq[A]
+    counter, first, last: int
 
-# ---------------------- helpers -----------------------------------
+const
+  defaultInitialSize* = 64
 
-const growthFactor = 2
+include setimpl
-
-when not defined(nimHasDefault):
-  template default[T](t: typedesc[T]): T =
-    ## Used by clear methods to get a default value.
-    var v: T
-    v
-
-# hcode for real keys cannot be zero.  hcode==0 signifies an empty slot.  These
-# two procs retain clarity of that encoding without the space cost of an enum.
-proc isEmpty(hcode: Hash): bool {.inline.} =
-  result = hcode == 0
-
-proc isFilled(hcode: Hash): bool {.inline.} =
-  result = hcode != 0
-
-proc nextTry(h, maxHash: Hash): Hash {.inline.} =
-  result = (h + 1) and maxHash
-
-template rawGetKnownHCImpl() {.dirty.} =
-  var h: Hash = hc and high(s.data)  # start with real hash value
-  while isFilled(s.data[h].hcode):
-    # Compare hc THEN key with boolean short circuit. This makes the common case
-    # zero ==key's for missing (e.g.inserts) and exactly one ==key for present.
-    # It does slow down succeeding lookups by one extra Hash cmp&and..usually
-    # just a few clock cycles, generally worth it for any non-integer-like A.
-    if s.data[h].hcode == hc and s.data[h].key == key:  # compare hc THEN key
-      return h
-    h = nextTry(h, high(s.data))
-  result = -1 - h                   # < 0 => MISSING; insert idx = -1 - result
-
-template genHash(key: typed): Hash =
-  var hc = hash(key)
-  if hc == 0:       # This almost never taken branch should be very predictable.
-    hc = 314159265  # Value doesn't matter; Any non-zero favorite is fine.
-  hc
-
-template rawGetImpl() {.dirty.} =
-  hc = genHash(key)
-  rawGetKnownHCImpl()
-
-template rawInsertImpl() {.dirty.} =
-  data[h].key = key
-  data[h].hcode = hc
-
-proc rawGetKnownHC[A](s: HashSet[A], key: A, hc: Hash): int {.inline.} =
-  rawGetKnownHCImpl()
-
-proc rawGet[A](s: HashSet[A], key: A, hc: var Hash): int {.inline.} =
-  rawGetImpl()
-
-proc rawInsert[A](s: var HashSet[A], data: var KeyValuePairSeq[A], key: A,
-                  hc: Hash, h: Hash) =
-  rawInsertImpl()
-
-proc enlarge[A](s: var HashSet[A]) =
-  var n: KeyValuePairSeq[A]
-  newSeq(n, len(s.data) * growthFactor)
-  swap(s.data, n)                   # n is now old seq
-  for i in countup(0, high(n)):
-    if isFilled(n[i].hcode):
-      var j = -1 - rawGetKnownHC(s, n[i].key, n[i].hcode)
-      rawInsert(s, s.data, n[i].key, n[i].hcode, j)
-
-template inclImpl() {.dirty.} =
-  var hc: Hash
-  var index = rawGet(s, key, hc)
-  if index < 0:
-    if mustRehash(len(s.data), s.counter):
-      enlarge(s)
-      index = rawGetKnownHC(s, key, hc)
-    rawInsert(s, s.data, key, hc, -1 - index)
-    inc(s.counter)
-
-template containsOrInclImpl() {.dirty.} =
-  var hc: Hash
-  var index = rawGet(s, key, hc)
-  if index >= 0:
-    result = true
-  else:
-    if mustRehash(len(s.data), s.counter):
-      enlarge(s)
-      index = rawGetKnownHC(s, key, hc)
-    rawInsert(s, s.data, key, hc, -1 - index)
-    inc(s.counter)
-
-template doWhile(a, b) =
-  while true:
-    b
-    if not a: break
-
-proc exclImpl[A](s: var HashSet[A], key: A) : bool {. inline .} =
-  assert s.isValid, "The set needs to be initialized."
-  var hc: Hash
-  var i = rawGet(s, key, hc)
-  var msk = high(s.data)
-  result = true
-
-  if i >= 0:
-    result = false
-    dec(s.counter)
-    while true:         # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1
-      var j = i         # The correctness of this depends on (h+1) in nextTry,
-      var r = j         # though may be adaptable to other simple sequences.
-      s.data[i].hcode = 0              # mark current EMPTY
-      s.data[i].key = default(type(s.data[i].key))
-      doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)):
-        i = (i + 1) and msk            # increment mod table size
-        if isEmpty(s.data[i].hcode):   # end of collision cluster; So all done
-          return
-        r = s.data[i].hcode and msk    # "home" location of key@i
-      shallowCopy(s.data[j], s.data[i]) # data[i] will be marked EMPTY next loop
-
-proc mustRehash(length, counter: int): bool {.inline.} =
-  assert(length > counter)
-  result = (length * 2 < counter * 3) or (length - counter < 4)
-
-template dollarImpl() {.dirty.} =
-  result = "{"
-  for key in items(s):
-    if result.len > 1: result.add(", ")
-    result.addQuoted(key)
-  result.add("}")
 
 proc rightSize*(count: Natural): int {.inline.}
 
 
-
-
-
-
-
-
 # ---------------------------------------------------------------------
 # ------------------------------ HashSet ------------------------------
 # ---------------------------------------------------------------------
 
 
-proc init*[A](s: var HashSet[A], initialSize=64) =
+proc init*[A](s: var HashSet[A], initialSize = defaultInitialSize) =
   ## Initializes a hash set.
   ##
   ## The `initialSize` parameter needs to be a power of two (default: 64).
@@ -218,9 +102,8 @@ proc init*[A](s: var HashSet[A], initialSize=64) =
   ## `math.nextPowerOfTwo proc <math.html#nextPowerOfTwo>`_ or `rightSize proc
   ## <#rightSize,Natural>`_ from this module.
   ##
-  ## All set variables must be initialized before
+  ## Starting from Nim v0.20, sets are initialized by default and it is
-  ## use with other procs from this module, with the exception of `isValid proc
+  ## not necessary to call this function explicitly.
-  ## <#isValid,HashSet[A]>`_ and `len() <#len,HashSet[A]>`_.
   ##
   ## You can call this proc on a previously initialized hash set, which will
   ## discard all its values. This might be more convenient than iterating over
@@ -235,42 +118,26 @@ proc init*[A](s: var HashSet[A], initialSize=64) =
     init(a)
     assert a.isValid
 
-  assert isPowerOfTwo(initialSize)
+  initImpl(s, initialSize)
-  s.counter = 0
-  newSeq(s.data, initialSize)
 
-proc initHashSet*[A](initialSize=64): HashSet[A] =
+proc initHashSet*[A](initialSize = defaultInitialSize): HashSet[A] =
   ## Wrapper around `init proc <#init,HashSet[A],int>`_ for initialization of
   ## hash sets.
   ##
   ## Returns an empty hash set you can assign directly in ``var`` blocks in a
   ## single line.
   ##
+  ## Starting from Nim v0.20, sets are initialized by default and it is
+  ## not necessary to call this function explicitly.
+  ##
   ## See also:
   ## * `toHashSet proc <#toHashSet,openArray[A]>`_
   runnableExamples:
     var a = initHashSet[int]()
-    assert a.isValid
     a.incl(3)
     assert len(a) == 1
-  result.init(initialSize)
 
-proc isValid*[A](s: HashSet[A]): bool =
+  result.init(initialSize)
-  ## Returns `true` if the set has been initialized (with `initHashSet proc
-  ## <#initHashSet,int>`_ or `init proc <#init,HashSet[A],int>`_).
-  ##
-  ## Most operations over an uninitialized set will crash at runtime and
-  ## `assert <system.html#assert>`_ in debug builds. You can use this proc in
-  ## your own procs to verify that sets passed to your procs are correctly
-  ## initialized.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block ::
-  ##   proc savePreferences(options: HashSet[string]) =
-  ##     assert options.isValid, "Pass an initialized set!"
-  ##     # Do stuff here, may crash in release builds!
-  result = s.data.len > 0
 
 proc `[]`*[A](s: var HashSet[A], key: A): var A =
   ## Returns the element that is actually stored in `s` which has the same
@@ -278,7 +145,6 @@ proc `[]`*[A](s: var HashSet[A], key: A): var A =
   ##
   ## This is useful when one overloaded `hash` and `==` but still needs
   ## reference semantics for sharing.
-  assert s.isValid, "The set needs to be initialized."
   var hc: Hash
   var index = rawGet(s, key, hc)
   if index >= 0: result = s.data[index].key
@@ -305,7 +171,6 @@ proc contains*[A](s: HashSet[A], key: A): bool =
     assert values.contains(2)
     assert 2 in values
 
-  assert s.isValid, "The set needs to be initialized."
   var hc: Hash
   var index = rawGet(s, key, hc)
   result = index >= 0
@@ -325,7 +190,6 @@ proc incl*[A](s: var HashSet[A], key: A) =
     values.incl(2)
     assert values.len == 1
 
-  assert s.isValid, "The set needs to be initialized."
   inclImpl()
 
 proc incl*[A](s: var HashSet[A], other: HashSet[A]) =
@@ -344,8 +208,6 @@ proc incl*[A](s: var HashSet[A], other: HashSet[A]) =
     values.incl(others)
     assert values.len == 5
 
-  assert s.isValid, "The set `s` needs to be initialized."
-  assert other.isValid, "The set `other` needs to be initialized."
   for item in other: incl(s, item)
 
 proc toHashSet*[A](keys: openArray[A]): HashSet[A] =
@@ -368,14 +230,6 @@ proc toHashSet*[A](keys: openArray[A]): HashSet[A] =
   result = initHashSet[A](rightSize(keys.len))
   for key in items(keys): result.incl(key)
 
-proc initSet*[A](initialSize=64): HashSet[A] {.deprecated:
-     "Deprecated since v0.20, use `initHashSet`"} = initHashSet[A](initialSize)
-  ## Deprecated since v0.20, use `initHashSet`.
-
-proc toSet*[A](keys: openArray[A]): HashSet[A] {.deprecated:
-     "Deprecated since v0.20, use `toHashSet`"} = toHashSet[A](keys)
-  ## Deprecated since v0.20, use `toHashSet`.
-
 iterator items*[A](s: HashSet[A]): A =
   ## Iterates over elements of the set `s`.
   ##
@@ -395,8 +249,7 @@ iterator items*[A](s: HashSet[A]): A =
   ##   assert a.len == 2
   ##   echo b
   ##   # --> {(a: 1, b: 3), (a: 0, b: 4)}
-  assert s.isValid, "The set needs to be initialized."
+  for h in 0 .. high(s.data):
-  for h in 0..high(s.data):
     if isFilled(s.data[h].hcode): yield s.data[h].key
 
 proc containsOrIncl*[A](s: var HashSet[A], key: A): bool =
@@ -417,7 +270,6 @@ proc containsOrIncl*[A](s: var HashSet[A], key: A): bool =
     assert values.containsOrIncl(2) == true
     assert values.containsOrIncl(3) == false
 
-  assert s.isValid, "The set needs to be initialized."
   containsOrInclImpl()
 
 proc excl*[A](s: var HashSet[A], key: A) =
@@ -434,6 +286,7 @@ proc excl*[A](s: var HashSet[A], key: A) =
     s.excl(2)
     s.excl(2)
     assert s.len == 3
+
   discard exclImpl(s, key)
 
 proc excl*[A](s: var HashSet[A], other: HashSet[A]) =
@@ -453,8 +306,6 @@ proc excl*[A](s: var HashSet[A], other: HashSet[A]) =
     assert len(numbers) == 3
     ## numbers == {1, 3, 5}
 
-  assert s.isValid, "The set `s` needs to be initialized."
-  assert other.isValid, "The set `other` needs to be initialized."
   for item in other: discard exclImpl(s, item)
 
 proc missingOrExcl*[A](s: var HashSet[A], key: A): bool =
@@ -474,6 +325,7 @@ proc missingOrExcl*[A](s: var HashSet[A], key: A): bool =
     assert s.missingOrExcl(4) == true
     assert s.missingOrExcl(6) == false
     assert s.missingOrExcl(6) == true
+
   exclImpl(s, key)
 
 proc pop*[A](s: var HashSet[A]): A =
@@ -489,7 +341,7 @@ proc pop*[A](s: var HashSet[A]): A =
     assert s.pop == 2
     doAssertRaises(KeyError, echo s.pop)
 
-  for h in 0..high(s.data):
+  for h in 0 .. high(s.data):
     if isFilled(s.data[h].hcode):
       result = s.data[h].key
       excl(s, result)
@@ -510,7 +362,7 @@ proc clear*[A](s: var HashSet[A]) =
     assert len(s) == 0
 
   s.counter = 0
-  for i in 0..<s.data.len:
+  for i in 0 ..< s.data.len:
     s.data[i].hcode = 0
     s.data[i].key = default(type(s.data[i].key))
 
@@ -525,6 +377,7 @@ proc len*[A](s: HashSet[A]): int =
     assert len(a) == 0
     let s = toHashSet([3, 5, 7])
     assert len(s) == 3
+
   result = s.counter
 
 proc card*[A](s: HashSet[A]): int =
@@ -554,8 +407,6 @@ proc union*[A](s1, s2: HashSet[A]): HashSet[A] =
       c = union(a, b)
     assert c == toHashSet(["a", "b", "c"])
 
-  assert s1.isValid, "The set `s1` needs to be initialized."
-  assert s2.isValid, "The set `s2` needs to be initialized."
   result = s1
   incl(result, s2)
 
@@ -579,9 +430,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] =
       c = intersection(a, b)
     assert c == toHashSet(["b"])
 
-  assert s1.isValid, "The set `s1` needs to be initialized."
+  result = initHashSet[A](max(min(s1.data.len, s2.data.len), 2))
-  assert s2.isValid, "The set `s2` needs to be initialized."
-  result = initHashSet[A](min(s1.data.len, s2.data.len))
   for item in s1:
     if item in s2: incl(result, item)
 
@@ -604,8 +453,6 @@ proc difference*[A](s1, s2: HashSet[A]): HashSet[A] =
       c = difference(a, b)
     assert c == toHashSet(["a"])
 
-  assert s1.isValid, "The set `s1` needs to be initialized."
-  assert s2.isValid, "The set `s2` needs to be initialized."
   result = initHashSet[A]()
   for item in s1:
     if not contains(s2, item):
@@ -631,8 +478,6 @@ proc symmetricDifference*[A](s1, s2: HashSet[A]): HashSet[A] =
       c = symmetricDifference(a, b)
     assert c == toHashSet(["a", "c"])
 
-  assert s1.isValid, "The set `s1` needs to be initialized."
-  assert s2.isValid, "The set `s2` needs to be initialized."
   result = s1
   for item in s2:
     if containsOrIncl(result, item): excl(result, item)
@@ -663,8 +508,6 @@ proc disjoint*[A](s1, s2: HashSet[A]): bool =
     assert disjoint(a, b) == false
     assert disjoint(a, b - a) == true
 
-  assert s1.isValid, "The set `s1` needs to be initialized."
-  assert s2.isValid, "The set `s2` needs to be initialized."
   for item in s1:
     if item in s2: return false
   return true
@@ -681,6 +524,7 @@ proc `<`*[A](s, t: HashSet[A]): bool =
       c = intersection(a, b)
     assert c < a and c < b
     assert(not (a < a))
+
   s.counter != t.counter and s <= t
 
 proc `<=`*[A](s, t: HashSet[A]): bool =
@@ -711,6 +555,7 @@ proc `==`*[A](s, t: HashSet[A]): bool =
       a = toHashSet([1, 2])
       b = toHashSet([2, 1])
     assert a == b
+
   s.counter == t.counter and s <= t
 
 proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] =
@@ -729,8 +574,7 @@ proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] =
 
 proc hash*[A](s: HashSet[A]): Hash =
   ## Hashing of HashSet.
-  assert s.isValid, "The set needs to be initialized."
+  for h in 0 .. high(s.data):
-  for h in 0..high(s.data):
     result = result xor s.data[h].hcode
   result = !$result
 
@@ -747,7 +591,6 @@ proc `$`*[A](s: HashSet[A]): string =
   ##   # --> {2, 4, 5}
   ##   echo toHashSet(["no", "esc'aping", "is \" provided"])
   ##   # --> {no, esc'aping, is " provided}
-  assert s.isValid, "The set needs to be initialized."
   dollarImpl()
 
 proc rightSize*(count: Natural): int {.inline.} =
@@ -760,8 +603,28 @@ proc rightSize*(count: Natural): int {.inline.} =
   result = nextPowerOfTwo(count * 3 div 2  +  4)
 
 
+proc initSet*[A](initialSize = defaultInitialSize): HashSet[A] {.deprecated:
+     "Deprecated since v0.20, use `initHashSet`"} = initHashSet[A](initialSize)
+  ## Deprecated since v0.20, use `initHashSet <#initHashSet,int>`_.
 
+proc toSet*[A](keys: openArray[A]): HashSet[A] {.deprecated:
+     "Deprecated since v0.20, use `toHashSet`"} = toHashSet[A](keys)
+  ## Deprecated since v0.20, use `toHashSet <#toHashSet,openArray[A]>`_.
 
+proc isValid*[A](s: HashSet[A]): bool {.deprecated:
+     "Deprecated since v0.20; sets are initialized by default"} =
+  ## **Deprecated since v0.20; sets are initialized by default**
+  ##
+  ## Returns `true` if the set has been initialized (with `initHashSet proc
+  ## <#initHashSet,int>`_ or `init proc <#init,HashSet[A],int>`_).
+  ##
+  ## **Examples:**
+  ##
+  ## .. code-block ::
+  ##   proc savePreferences(options: HashSet[string]) =
+  ##     assert options.isValid, "Pass an initialized set!"
+  ##     # Do stuff here, may crash in release builds!
+  result = s.data.len > 0
 
 
 
@@ -769,89 +632,19 @@ proc rightSize*(count: Natural): int {.inline.} =
 # --------------------------- OrderedSet ------------------------------
 # ---------------------------------------------------------------------
 
-type
-  OrderedKeyValuePair[A] = tuple[
-    hcode: Hash, next: int, key: A]
-  OrderedKeyValuePairSeq[A] = seq[OrderedKeyValuePair[A]]
-  OrderedSet* {.myShallow.} [A] = object ## \
-    ## A generic hash set that remembers insertion order.
-    ##
-    ## Use `init proc <#init,OrderedSet[A],int>`_ or `initOrderedSet proc
-    ## <#initOrderedSet,int>`_ before calling other procs on it.
-    data: OrderedKeyValuePairSeq[A]
-    counter, first, last: int
-
-
-# ---------------------- helpers -----------------------------------
-
 template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} =
-  var h = s.first
+  if s.data.len > 0:
-  var idx = 0
+    var h = s.first
-  while h >= 0:
+    var idx = 0
-    var nxt = s.data[h].next
+    while h >= 0:
-    if isFilled(s.data[h].hcode):
+      var nxt = s.data[h].next
-      yieldStmt
+      if isFilled(s.data[h].hcode):
-      inc(idx)
+        yieldStmt
-    h = nxt
+        inc(idx)
-
+      h = nxt
-proc rawGetKnownHC[A](s: OrderedSet[A], key: A, hc: Hash): int {.inline.} =
-  rawGetKnownHCImpl()
-
-proc rawGet[A](s: OrderedSet[A], key: A, hc: var Hash): int {.inline.} =
-  rawGetImpl()
-
-proc rawInsert[A](s: var OrderedSet[A], data: var OrderedKeyValuePairSeq[A],
-                  key: A, hc: Hash, h: Hash) =
-  rawInsertImpl()
-  data[h].next = -1
-  if s.first < 0: s.first = h
-  if s.last >= 0: data[s.last].next = h
-  s.last = h
-
-proc enlarge[A](s: var OrderedSet[A]) =
-  var n: OrderedKeyValuePairSeq[A]
-  newSeq(n, len(s.data) * growthFactor)
-  var h = s.first
-  s.first = -1
-  s.last = -1
-  swap(s.data, n)
-  while h >= 0:
-    var nxt = n[h].next
-    if isFilled(n[h].hcode):
-      var j = -1 - rawGetKnownHC(s, n[h].key, n[h].hcode)
-      rawInsert(s, s.data, n[h].key, n[h].hcode, j)
-    h = nxt
-
-proc isValid*[A](s: OrderedSet[A]): bool
-
-proc exclImpl[A](s: var OrderedSet[A], key: A) : bool {. inline .} =
-  assert s.isValid, "The set needs to be initialized."
-  var n: OrderedKeyValuePairSeq[A]
-  newSeq(n, len(s.data))
-  var h = s.first
-  s.first = -1
-  s.last = -1
-  swap(s.data, n)
-  let hc = genHash(key)
-  result = true
-  while h >= 0:
-    var nxt = n[h].next
-    if isFilled(n[h].hcode):
-      if n[h].hcode == hc and n[h].key == key:
-        dec s.counter
-        result = false
-      else:
-        var j = -1 - rawGetKnownHC(s, n[h].key, n[h].hcode)
-        rawInsert(s, s.data, n[h].key, n[h].hcode, j)
-    h = nxt
 
 
-
+proc init*[A](s: var OrderedSet[A], initialSize = defaultInitialSize) =
-# -----------------------------------------------------------------------
-
-
-
-proc init*[A](s: var OrderedSet[A], initialSize=64) =
   ## Initializes an ordered hash set.
   ##
   ## The `initialSize` parameter needs to be a power of two (default: 64).
@@ -859,9 +652,8 @@ proc init*[A](s: var OrderedSet[A], initialSize=64) =
   ## `math.nextPowerOfTwo proc <math.html#nextPowerOfTwo>`_ or `rightSize proc
   ## <#rightSize,Natural>`_ from this module.
   ##
-  ## All set variables must be initialized before
+  ## Starting from Nim v0.20, sets are initialized by default and it is
-  ## use with other procs from this module, with the exception of `isValid proc
+  ## not necessary to call this function explicitly.
-  ## <#isValid,HashSet[A]>`_ and `len() <#len,HashSet[A]>`_.
   ##
   ## You can call this proc on a previously initialized hash set, which will
   ## discard all its values. This might be more convenient than iterating over
@@ -876,26 +668,25 @@ proc init*[A](s: var OrderedSet[A], initialSize=64) =
     init(a)
     assert a.isValid
 
-  assert isPowerOfTwo(initialSize)
+  initImpl(s, initialSize)
-  s.counter = 0
-  s.first = -1
-  s.last = -1
-  newSeq(s.data, initialSize)
 
-proc initOrderedSet*[A](initialSize=64): OrderedSet[A] =
+proc initOrderedSet*[A](initialSize = defaultInitialSize): OrderedSet[A] =
   ## Wrapper around `init proc <#init,OrderedSet[A],int>`_ for initialization of
   ## ordered hash sets.
   ##
   ## Returns an empty ordered hash set you can assign directly in ``var`` blocks
   ## in a single line.
   ##
+  ## Starting from Nim v0.20, sets are initialized by default and it is
+  ## not necessary to call this function explicitly.
+  ##
   ## See also:
   ## * `toOrderedSet proc <#toOrderedSet,openArray[A]>`_
   runnableExamples:
     var a = initOrderedSet[int]()
-    assert a.isValid
     a.incl(3)
     assert len(a) == 1
+
   result.init(initialSize)
 
 proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] =
@@ -918,23 +709,6 @@ proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] =
   result = initOrderedSet[A](rightSize(keys.len))
   for key in items(keys): result.incl(key)
 
-proc isValid*[A](s: OrderedSet[A]): bool =
-  ## Returns `true` if the set has been initialized (with `initHashSet proc
-  ## <#initOrderedSet,int>`_ or `init proc <#init,OrderedSet[A],int>`_).
-  ##
-  ## Most operations over an uninitialized set will crash at runtime and
-  ## `assert <system.html#assert>`_ in debug builds. You can use this proc in
-  ## your own procs to verify that sets passed to your procs are correctly
-  ## initialized.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block ::
-  ##   proc savePreferences(options: OrderedSet[string]) =
-  ##     assert options.isValid, "Pass an initialized set!"
-  ##     # Do stuff here, may crash in release builds!
-  result = s.data.len > 0
-
 proc contains*[A](s: OrderedSet[A], key: A): bool =
   ## Returns true if `key` is in `s`.
   ##
@@ -952,7 +726,6 @@ proc contains*[A](s: OrderedSet[A], key: A): bool =
     assert values.contains(2)
     assert 2 in values
 
-  assert s.isValid, "The set needs to be initialized."
   var hc: Hash
   var index = rawGet(s, key, hc)
   result = index >= 0
@@ -972,7 +745,6 @@ proc incl*[A](s: var OrderedSet[A], key: A) =
     values.incl(2)
     assert values.len == 1
 
-  assert s.isValid, "The set needs to be initialized."
   inclImpl()
 
 proc incl*[A](s: var HashSet[A], other: OrderedSet[A]) =
@@ -988,8 +760,7 @@ proc incl*[A](s: var HashSet[A], other: OrderedSet[A]) =
       others = toOrderedSet([3, 4, 5])
     values.incl(others)
     assert values.len == 5
-  assert s.isValid, "The set `s` needs to be initialized."
+
-  assert other.isValid, "The set `other` needs to be initialized."
   for item in items(other): incl(s, item)
 
 proc containsOrIncl*[A](s: var OrderedSet[A], key: A): bool =
@@ -1009,7 +780,6 @@ proc containsOrIncl*[A](s: var OrderedSet[A], key: A): bool =
     assert values.containsOrIncl(2) == true
     assert values.containsOrIncl(3) == false
 
-  assert s.isValid, "The set needs to be initialized."
   containsOrInclImpl()
 
 proc excl*[A](s: var OrderedSet[A], key: A) =
@@ -1025,6 +795,7 @@ proc excl*[A](s: var OrderedSet[A], key: A) =
     s.excl(2)
     s.excl(2)
     assert s.len == 3
+
   discard exclImpl(s, key)
 
 proc missingOrExcl*[A](s: var OrderedSet[A], key: A): bool =
@@ -1044,6 +815,7 @@ proc missingOrExcl*[A](s: var OrderedSet[A], key: A): bool =
     assert s.missingOrExcl(4) == true
     assert s.missingOrExcl(6) == false
     assert s.missingOrExcl(6) == true
+
   exclImpl(s, key)
 
 proc clear*[A](s: var OrderedSet[A]) =
@@ -1059,7 +831,7 @@ proc clear*[A](s: var OrderedSet[A]) =
   s.counter = 0
   s.first = -1
   s.last = -1
-  for i in 0..<s.data.len:
+  for i in 0 ..< s.data.len:
     s.data[i].hcode = 0
     s.data[i].next = 0
     s.data[i].key = default(type(s.data[i].key))
@@ -1075,6 +847,7 @@ proc len*[A](s: OrderedSet[A]): int {.inline.} =
     assert len(a) == 0
     let s = toHashSet([3, 5, 7])
     assert len(s) == 3
+
   result = s.counter
 
 proc card*[A](s: OrderedSet[A]): int {.inline.} =
@@ -1110,7 +883,6 @@ proc `==`*[A](s, t: OrderedSet[A]): bool =
 
 proc hash*[A](s: OrderedSet[A]): Hash =
   ## Hashing of OrderedSet.
-  assert s.isValid, "The set needs to be initialized."
   forAllOrderedPairs:
     result = result !& s.data[h].hcode
   result = !$result
@@ -1129,7 +901,6 @@ proc `$`*[A](s: OrderedSet[A]): string =
   ##   # --> {2, 4, 5}
   ##   echo toOrderedSet(["no", "esc'aping", "is \" provided"])
   ##   # --> {no, esc'aping, is " provided}
-  assert s.isValid, "The set needs to be initialized."
   dollarImpl()
 
 
@@ -1152,11 +923,9 @@ iterator items*[A](s: OrderedSet[A]): A =
   ##   # --> Got 5
   ##   # --> Got 8
   ##   # --> Got 4
-  assert s.isValid, "The set needs to be initialized."
   forAllOrderedPairs:
     yield s.data[h].key
 
-
 iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] =
   ## Iterates through (position, value) tuples of OrderedSet `s`.
   runnableExamples:
@@ -1166,12 +935,27 @@ iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] =
       p.add(x)
     assert p == @[(0, 'a'), (1, 'b'), (2, 'r'), (3, 'c'), (4, 'd')]
 
-  assert s.isValid, "The set needs to be initialized"
   forAllOrderedPairs:
     yield (idx, s.data[h].key)
 
 
 
+proc isValid*[A](s: OrderedSet[A]): bool {.deprecated:
+     "Deprecated since v0.20; sets are initialized by default"} =
+  ## **Deprecated since v0.20; sets are initialized by default**
+  ##
+  ## Returns `true` if the set has been initialized (with `initHashSet proc
+  ## <#initOrderedSet,int>`_ or `init proc <#init,OrderedSet[A],int>`_).
+  ##
+  ## **Examples:**
+  ##
+  ## .. code-block ::
+  ##   proc savePreferences(options: OrderedSet[string]) =
+  ##     assert options.isValid, "Pass an initialized set!"
+  ##     # Do stuff here, may crash in release builds!
+  result = s.data.len > 0
+
+
 # -----------------------------------------------------------------------
 
 
@@ -1179,7 +963,7 @@ iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] =
 when isMainModule and not defined(release):
   proc testModule() =
     ## Internal micro test to validate docstrings and such.
-    block isValidTest:
+    block isValidTest: # isValid is deprecated
       var options: HashSet[string]
       proc savePreferences(options: HashSet[string]) =
         assert options.isValid, "Pass an initialized set!"
@@ -1280,7 +1064,7 @@ when isMainModule and not defined(release):
       var b = a.map(proc (x: int): string = $x)
       assert b == toHashSet(["1", "2", "3"])
 
-    block isValidTest:
+    block isValidTest: # isValid is deprecated
       var cards: OrderedSet[string]
       proc saveTarotCards(cards: OrderedSet[string]) =
         assert cards.isValid, "Pass an initialized set!"
@@ -1393,6 +1177,69 @@ when isMainModule and not defined(release):
       bb.incl(y)
       assert aa == bb
 
+    block setsWithoutInit:
+      var
+        a: HashSet[int]
+        b: HashSet[int]
+        c: HashSet[int]
+        d: HashSet[int]
+        e: HashSet[int]
+
+      doAssert a.containsOrIncl(3) == false
+      doAssert a.contains(3)
+      doAssert a.len == 1
+      doAssert a.containsOrIncl(3)
+      a.incl(3)
+      doAssert a.len == 1
+      a.incl(6)
+      doAssert a.len == 2
+
+      b.incl(5)
+      doAssert b.len == 1
+      b.excl(5)
+      b.excl(c)
+      doAssert b.missingOrExcl(5)
+      doAssert b.disjoint(c)
+
+      d = b + c
+      doAssert d.len == 0
+      d = b * c
+      doAssert d.len == 0
+      d = b - c
+      doAssert d.len == 0
+      d = b -+- c
+      doAssert d.len == 0
+
+      doAssert (d < e) == false
+      doAssert d <= e
+      doAssert d == e
+
+    block setsWithoutInit:
+      var
+        a: OrderedSet[int]
+        b: OrderedSet[int]
+        c: OrderedSet[int]
+        d: HashSet[int]
+
+
+      doAssert a.containsOrIncl(3) == false
+      doAssert a.contains(3)
+      doAssert a.len == 1
+      doAssert a.containsOrIncl(3)
+      a.incl(3)
+      doAssert a.len == 1
+      a.incl(6)
+      doAssert a.len == 2
+
+      b.incl(5)
+      doAssert b.len == 1
+      doAssert b.missingOrExcl(5) == false
+      doAssert b.missingOrExcl(5)
+
+      doAssert c.missingOrExcl(9)
+      d.incl(c)
+      doAssert d.len == 0
+
     when not defined(testing):
       echo "Micro tests run successfully."
 

lib/pure/collections/sharedtables.nim

@@ -29,6 +29,14 @@ template maxHash(t): untyped = t.dataLen-1
 
 include tableimpl
 
+template st_maybeRehashPutImpl(enlarge) {.dirty.} =
+  if mustRehash(t.dataLen, t.counter):
+    enlarge(t)
+    index = rawGetKnownHC(t, key, hc)
+  index = -1 - index                  # important to transform for mgetOrPutImpl
+  rawInsert(t, t.data, key, val, hc, index)
+  inc(t.counter)
+
 proc enlarge[A, B](t: var SharedTable[A, B]) =
   let oldSize = t.dataLen
   let size = oldSize * growthFactor
@@ -176,7 +184,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A,
       var val: B
       mapper(key, val, pairExists)
       if pairExists:
-        maybeRehashPutImpl(enlarge)
+        st_maybeRehashPutImpl(enlarge)
 
 proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) =
   ## puts a (key, value)-pair into `t`.

lib/pure/collections/tableimpl.nim

@@ -9,49 +9,7 @@
 
 # An ``include`` file for the different table implementations.
 
-# hcode for real keys cannot be zero.  hcode==0 signifies an empty slot.  These
+include hashcommon
-# two procs retain clarity of that encoding without the space cost of an enum.
-proc isEmpty(hcode: Hash): bool {.inline.} =
-  result = hcode == 0
-
-proc isFilled(hcode: Hash): bool {.inline.} =
-  result = hcode != 0
-
-const
-  growthFactor = 2
-
-proc mustRehash(length, counter: int): bool {.inline.} =
-  assert(length > counter)
-  result = (length * 2 < counter * 3) or (length - counter < 4)
-
-proc nextTry(h, maxHash: Hash): Hash {.inline.} =
-  result = (h + 1) and maxHash
-
-template rawGetKnownHCImpl() {.dirty.} =
-  var h: Hash = hc and maxHash(t)   # start with real hash value
-  while isFilled(t.data[h].hcode):
-    # Compare hc THEN key with boolean short circuit. This makes the common case
-    # zero ==key's for missing (e.g.inserts) and exactly one ==key for present.
-    # It does slow down succeeding lookups by one extra Hash cmp&and..usually
-    # just a few clock cycles, generally worth it for any non-integer-like A.
-    if t.data[h].hcode == hc and t.data[h].key == key:
-      return h
-    h = nextTry(h, maxHash(t))
-  result = -1 - h                   # < 0 => MISSING; insert idx = -1 - result
-
-template genHashImpl(key, hc: typed) =
-  hc = hash(key)
-  if hc == 0:       # This almost never taken branch should be very predictable.
-    hc = 314159265  # Value doesn't matter; Any non-zero favorite is fine.
-
-template genHash(key: typed): Hash =
-  var res: Hash
-  genHashImpl(key, res)
-  res
-
-template rawGetImpl() {.dirty.} =
-  genHashImpl(key, hc)
-  rawGetKnownHCImpl()
 
 template rawGetDeepImpl() {.dirty.} =   # Search algo for unconditional add
   genHashImpl(key, hc)
@@ -65,20 +23,16 @@ template rawInsertImpl() {.dirty.} =
   data[h].val = val
   data[h].hcode = hc
 
-proc rawGetKnownHC[X, A](t: X, key: A, hc: Hash): int {.inline.} =
-  rawGetKnownHCImpl()
-
 proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline.} =
   rawGetDeepImpl()
 
-proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline.} =
-  rawGetImpl()
-
 proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B],
                      key: A, val: B, hc: Hash, h: Hash) =
   rawInsertImpl()
 
 template addImpl(enlarge) {.dirty.} =
+  if t.dataLen == 0:
+    initImpl(t, defaultInitialSize)
   if mustRehash(t.dataLen, t.counter): enlarge(t)
   var hc: Hash
   var j = rawGetDeep(t, key, hc)
@@ -86,6 +40,8 @@ template addImpl(enlarge) {.dirty.} =
   inc(t.counter)
 
 template maybeRehashPutImpl(enlarge) {.dirty.} =
+  if t.dataLen == 0:
+    initImpl(t, defaultInitialSize)
   if mustRehash(t.dataLen, t.counter):
     enlarge(t)
     index = rawGetKnownHC(t, key, hc)
@@ -94,12 +50,16 @@ template maybeRehashPutImpl(enlarge) {.dirty.} =
   inc(t.counter)
 
 template putImpl(enlarge) {.dirty.} =
+  if t.dataLen == 0:
+    initImpl(t, defaultInitialSize)
   var hc: Hash
   var index = rawGet(t, key, hc)
   if index >= 0: t.data[index].val = val
   else: maybeRehashPutImpl(enlarge)
 
 template mgetOrPutImpl(enlarge) {.dirty.} =
+  if t.dataLen == 0:
+    initImpl(t, defaultInitialSize)
   var hc: Hash
   var index = rawGet(t, key, hc)
   if index < 0:
@@ -109,6 +69,8 @@ template mgetOrPutImpl(enlarge) {.dirty.} =
   result = t.data[index].val
 
 template hasKeyOrPutImpl(enlarge) {.dirty.} =
+  if t.dataLen == 0:
+    initImpl(t, defaultInitialSize)
   var hc: Hash
   var index = rawGet(t, key, hc)
   if index < 0:
@@ -116,11 +78,6 @@ template hasKeyOrPutImpl(enlarge) {.dirty.} =
     maybeRehashPutImpl(enlarge)
   else: result = true
 
-when not defined(nimHasDefault):
-  template default[T](t: typedesc[T]): T =
-    var v: T
-    v
-
 template delImplIdx(t, i) =
   let msk = maxHash(t)
   if i >= 0:
@@ -150,9 +107,53 @@ template delImpl() {.dirty.} =
   delImplIdx(t, i)
 
 template clearImpl() {.dirty.} =
-  for i in 0 ..< t.data.len:
+  for i in 0 ..< t.dataLen:
     when compiles(t.data[i].hcode): # CountTable records don't contain a hcode
       t.data[i].hcode = 0
     t.data[i].key = default(type(t.data[i].key))
     t.data[i].val = default(type(t.data[i].val))
   t.counter = 0
+
+template initImpl(result: typed, size: int) =
+  assert isPowerOfTwo(size)
+  result.counter = 0
+  newSeq(result.data, size)
+
+template insertImpl() = # for CountTable
+  if t.dataLen == 0: initImpl(t, defaultInitialSize)
+  if mustRehash(len(t.data), t.counter): enlarge(t)
+  ctRawInsert(t, t.data, key, val)
+  inc(t.counter)
+
+template getOrDefaultImpl(t, key): untyped =
+  mixin rawGet
+  var hc: Hash
+  var index = rawGet(t, key, hc)
+  if index >= 0: result = t.data[index].val
+
+template getOrDefaultImpl(t, key, default: untyped): untyped =
+  mixin rawGet
+  var hc: Hash
+  var index = rawGet(t, key, hc)
+  result = if index >= 0: t.data[index].val else: default
+
+template dollarImpl(): untyped {.dirty.} =
+  if t.len == 0:
+    result = "{:}"
+  else:
+    result = "{"
+    for key, val in pairs(t):
+      if result.len > 1: result.add(", ")
+      result.addQuoted(key)
+      result.add(": ")
+      result.addQuoted(val)
+    result.add("}")
+
+template equalsImpl(s, t: typed): typed =
+  if s.counter == t.counter:
+    # different insertion orders mean different 'data' seqs, so we have
+    # to use the slow route here:
+    for key, val in s:
+      if not t.hasKey(key): return false
+      if t.getOrDefault(key) != val: return false
+    return true

lib/pure/collections/tables.nim

@@ -19,7 +19,7 @@
 ## For consistency with every other data type in Nim these have **value**
 ## semantics, this means that ``=`` performs a copy of the hash table.
 ##
-## For `ref semantics<manual.html#types-ref-and-pointer-types>`_
+## For `ref semantics<manual.html#types-reference-and-pointer-types>`_
 ## use their ``Ref`` variants: `TableRef<#TableRef>`_,
 ## `OrderedTableRef<#OrderedTableRef>`_, and `CountTableRef<#CountTableRef>`_.
 ##
@@ -237,9 +237,13 @@ type
     ## For creating a new empty TableRef, use `newTable proc
     ## <#newTable,int>`_.
 
+const
+  defaultInitialSize* = 64
 
 # ------------------------------ helpers ---------------------------------
 
+# Do NOT move these to tableimpl.nim, because sharedtables uses that
+# file and has its own implementation.
 template maxHash(t): untyped = high(t.data)
 template dataLen(t): untyped = len(t.data)
 
@@ -260,30 +264,6 @@ template get(t, key): untyped =
     else:
       raise newException(KeyError, "key not found")
 
-template getOrDefaultImpl(t, key): untyped =
-  mixin rawGet
-  var hc: Hash
-  var index = rawGet(t, key, hc)
-  if index >= 0: result = t.data[index].val
-
-template getOrDefaultImpl(t, key, default: untyped): untyped =
-  mixin rawGet
-  var hc: Hash
-  var index = rawGet(t, key, hc)
-  result = if index >= 0: t.data[index].val else: default
-
-template dollarImpl(): untyped {.dirty.} =
-  if t.len == 0:
-    result = "{:}"
-  else:
-    result = "{"
-    for key, val in pairs(t):
-      if result.len > 1: result.add(", ")
-      result.addQuoted(key)
-      result.add(": ")
-      result.addQuoted(val)
-    result.add("}")
-
 proc enlarge[A, B](t: var Table[A, B]) =
   var n: KeyValuePairSeq[A, B]
   newSeq(n, len(t.data) * growthFactor)
@@ -294,16 +274,8 @@ proc enlarge[A, B](t: var Table[A, B]) =
       var j: Hash = eh and maxHash(t)
       while isFilled(t.data[j].hcode):
         j = nextTry(j, maxHash(t))
-      rawInsert(t, t.data, n[i].key, n[i].val, eh, j)
+      rawInsert(t, t.data, move n[i].key, move n[i].val, eh, j)
 
-template equalsImpl(s, t: typed): typed =
-  if s.counter == t.counter:
-    # different insertion orders mean different 'data' seqs, so we have
-    # to use the slow route here:
-    for key, val in s:
-      if not t.hasKey(key): return false
-      if t.getOrDefault(key) != val: return false
-    return true
 
 
 
@@ -311,7 +283,7 @@ template equalsImpl(s, t: typed): typed =
 # ------------------------------ Table ------------------------------
 # -------------------------------------------------------------------
 
-proc initTable*[A, B](initialSize=64): Table[A, B] =
+proc initTable*[A, B](initialsize = defaultInitialSize): Table[A, B] =
   ## Creates a new hash table that is empty.
   ##
   ## ``initialSize`` must be a power of two (default: 64).
@@ -320,6 +292,9 @@ proc initTable*[A, B](initialSize=64): Table[A, B] =
   ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_
   ## from this module.
   ##
+  ## Starting from Nim v0.20, tables are initialized by default and it is
+  ## not necessary to call this function explicitly.
+  ##
   ## See also:
   ## * `toTable proc<#toTable,openArray[]>`_
   ## * `newTable proc<#newTable,int>`_ for creating a `TableRef`
@@ -327,9 +302,7 @@ proc initTable*[A, B](initialSize=64): Table[A, B] =
     let
       a = initTable[int, string]()
       b = initTable[char, seq[int]]()
-  assert isPowerOfTwo(initialSize)
+  initImpl(result, initialSize)
-  result.counter = 0
-  newSeq(result.data, initialSize)
 
 proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] =
   ## Creates a new hash table that contains the given ``pairs``.
@@ -343,6 +316,7 @@ proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] =
     let a = [('a', 5), ('b', 9)]
     let b = toTable(a)
     assert b == {'a': 5, 'b': 9}.toTable
+
   result = initTable[A, B](rightSize(pairs.len))
   for key, val in items(pairs): result[key] = val
 
@@ -398,6 +372,7 @@ proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) =
     a['x'] = 7
     a['y'] = 33
     doAssert a == {'x': 7, 'y': 33}.toTable
+
   putImpl(enlarge)
 
 proc hasKey*[A, B](t: Table[A, B], key: A): bool =
@@ -414,6 +389,7 @@ proc hasKey*[A, B](t: Table[A, B], key: A): bool =
     let a = {'a': 5, 'b': 9}.toTable
     doAssert a.hasKey('a') == true
     doAssert a.hasKey('z') == false
+
   var hc: Hash
   result = rawGet(t, key, hc) >= 0
 
@@ -424,6 +400,7 @@ proc contains*[A, B](t: Table[A, B], key: A): bool =
     let a = {'a': 5, 'b': 9}.toTable
     doAssert 'b' in a == true
     doAssert a.contains('z') == false
+
   return hasKey[A, B](t, key)
 
 proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool =
@@ -443,6 +420,7 @@ proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool =
     if a.hasKeyOrPut('z', 50):
       a['z'] = 99
     doAssert a == {'a': 99, 'b': 9, 'z': 50}.toTable
+
   hasKeyOrPutImpl(enlarge)
 
 proc getOrDefault*[A, B](t: Table[A, B], key: A): B =
@@ -461,6 +439,7 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A): B =
     let a = {'a': 5, 'b': 9}.toTable
     doAssert a.getOrDefault('a') == 5
     doAssert a.getOrDefault('z') == 0
+
   getOrDefaultImpl(t, key)
 
 proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B =
@@ -478,6 +457,7 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B =
     let a = {'a': 5, 'b': 9}.toTable
     doAssert a.getOrDefault('a', 99) == 5
     doAssert a.getOrDefault('z', 99) == 99
+
   getOrDefaultImpl(t, key, default)
 
 proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B =
@@ -497,6 +477,7 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B =
     doAssert a.mgetOrPut('a', 99) == 5
     doAssert a.mgetOrPut('z', 99) == 99
     doAssert a == {'a': 5, 'b': 9, 'z': 99}.toTable
+
   mgetOrPutImpl(enlarge)
 
 proc len*[A, B](t: Table[A, B]): int =
@@ -504,6 +485,7 @@ proc len*[A, B](t: Table[A, B]): int =
   runnableExamples:
     let a = {'a': 5, 'b': 9}.toTable
     doAssert len(a) == 2
+
   result = t.counter
 
 proc add*[A, B](t: var Table[A, B], key: A, val: B) =
@@ -527,6 +509,7 @@ proc del*[A, B](t: var Table[A, B], key: A) =
     doAssert a == {'b': 9, 'c': 13}.toTable
     a.del('z')
     doAssert a == {'b': 9, 'c': 13}.toTable
+
   delImpl()
 
 proc take*[A, B](t: var Table[A, B], key: A, val: var B): bool =
@@ -568,6 +551,7 @@ proc clear*[A, B](t: var Table[A, B]) =
     doAssert len(a) == 3
     clear(a)
     doAssert len(a) == 0
+
   clearImpl()
 
 proc `$`*[A, B](t: Table[A, B]): string =
@@ -583,6 +567,7 @@ proc `==`*[A, B](s, t: Table[A, B]): bool =
       a = {'a': 5, 'b': 9, 'c': 13}.toTable
       b = {'b': 9, 'c': 13, 'a': 5}.toTable
     doAssert a == b
+
   equalsImpl(s, t)
 
 proc rightSize*(count: Natural): int {.inline.} =
@@ -692,6 +677,7 @@ iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) =
     for k, v in a.mpairs:
       v.add(v[0] + 10)
     doAssert a == {'e': @[2, 4, 6, 8, 12], 'o': @[1, 5, 7, 9, 11]}.toTable
+
   for h in 0..high(t.data):
     if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val)
 
@@ -709,6 +695,7 @@ iterator keys*[A, B](t: Table[A, B]): A =
     for k in a.keys:
       a[k].add(99)
     doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.toTable
+
   for h in 0..high(t.data):
     if isFilled(t.data[h].hcode): yield t.data[h].key
 
@@ -726,6 +713,7 @@ iterator values*[A, B](t: Table[A, B]): B =
       }.toTable
     for v in a.values:
       doAssert v.len == 4
+
   for h in 0..high(t.data):
     if isFilled(t.data[h].hcode): yield t.data[h].val
 
@@ -744,6 +732,7 @@ iterator mvalues*[A, B](t: var Table[A, B]): var B =
     for v in a.mvalues:
       v.add(99)
     doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.toTable
+
   for h in 0..high(t.data):
     if isFilled(t.data[h].hcode): yield t.data[h].val
 
@@ -779,7 +768,7 @@ iterator allValues*[A, B](t: Table[A, B]; key: A): B =
 # -------------------------------------------------------------------
 
 
-proc newTable*[A, B](initialSize=64): TableRef[A, B] =
+proc newTable*[A, B](initialsize = defaultInitialSize): <//>TableRef[A, B] =
   ## Creates a new ref hash table that is empty.
   ##
   ## ``initialSize`` must be a power of two (default: 64).
@@ -796,10 +785,11 @@ proc newTable*[A, B](initialSize=64): TableRef[A, B] =
     let
       a = newTable[int, string]()
       b = newTable[char, seq[int]]()
+
   new(result)
   result[] = initTable[A, B](initialSize)
 
-proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] =
+proc newTable*[A, B](pairs: openArray[(A, B)]): <//>TableRef[A, B] =
   ## Creates a new ref hash table that contains the given ``pairs``.
   ##
   ## ``pairs`` is a container consisting of ``(key, value)`` tuples.
@@ -811,10 +801,11 @@ proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] =
     let a = [('a', 5), ('b', 9)]
     let b = newTable(a)
     assert b == {'a': 5, 'b': 9}.newTable
+
   new(result)
   result[] = toTable[A, B](pairs)
 
-proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] =
+proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): <//>TableRef[C, B] =
   ## Index the collection with the proc provided.
   # TODO: As soon as supported, change collection: A to collection: A[B]
   result = newTable[C, B]()
@@ -842,6 +833,7 @@ proc `[]`*[A, B](t: TableRef[A, B], key: A): var B =
     doAssert a['a'] == 5
     doAssertRaises(KeyError):
       echo a['z']
+
   result = t[][key]
 
 proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) =
@@ -857,6 +849,7 @@ proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) =
     a['x'] = 7
     a['y'] = 33
     doAssert a == {'x': 7, 'y': 33}.newTable
+
   t[][key] = val
 
 proc hasKey*[A, B](t: TableRef[A, B], key: A): bool =
@@ -874,6 +867,7 @@ proc hasKey*[A, B](t: TableRef[A, B], key: A): bool =
     let a = {'a': 5, 'b': 9}.newTable
     doAssert a.hasKey('a') == true
     doAssert a.hasKey('z') == false
+
   result = t[].hasKey(key)
 
 proc contains*[A, B](t: TableRef[A, B], key: A): bool =
@@ -883,6 +877,7 @@ proc contains*[A, B](t: TableRef[A, B], key: A): bool =
     let a = {'a': 5, 'b': 9}.newTable
     doAssert 'b' in a == true
     doAssert a.contains('z') == false
+
   return hasKey[A, B](t, key)
 
 proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool =
@@ -902,6 +897,7 @@ proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool =
     if a.hasKeyOrPut('z', 50):
       a['z'] = 99
     doAssert a == {'a': 99, 'b': 9, 'z': 50}.newTable
+
   t[].hasKeyOrPut(key, val)
 
 proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B =
@@ -920,6 +916,7 @@ proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B =
     let a = {'a': 5, 'b': 9}.newTable
     doAssert a.getOrDefault('a') == 5
     doAssert a.getOrDefault('z') == 0
+
   getOrDefault(t[], key)
 
 proc getOrDefault*[A, B](t: TableRef[A, B], key: A, default: B): B =
@@ -937,6 +934,7 @@ proc getOrDefault*[A, B](t: TableRef[A, B], key: A, default: B): B =
     let a = {'a': 5, 'b': 9}.newTable
     doAssert a.getOrDefault('a', 99) == 5
     doAssert a.getOrDefault('z', 99) == 99
+
   getOrDefault(t[], key, default)
 
 proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B =
@@ -956,6 +954,7 @@ proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B =
     doAssert a.mgetOrPut('a', 99) == 5
     doAssert a.mgetOrPut('z', 99) == 99
     doAssert a == {'a': 5, 'b': 9, 'z': 99}.newTable
+
   t[].mgetOrPut(key, val)
 
 proc len*[A, B](t: TableRef[A, B]): int =
@@ -963,6 +962,7 @@ proc len*[A, B](t: TableRef[A, B]): int =
   runnableExamples:
     let a = {'a': 5, 'b': 9}.newTable
     doAssert len(a) == 2
+
   result = t.counter
 
 proc add*[A, B](t: TableRef[A, B], key: A, val: B) =
@@ -988,6 +988,7 @@ proc del*[A, B](t: TableRef[A, B], key: A) =
     doAssert a == {'b': 9, 'c': 13}.newTable
     a.del('z')
     doAssert a == {'b': 9, 'c': 13}.newTable
+
   t[].del(key)
 
 proc take*[A, B](t: TableRef[A, B], key: A, val: var B): bool =
@@ -1012,6 +1013,7 @@ proc take*[A, B](t: TableRef[A, B], key: A, val: var B): bool =
     doAssert a.take('z', i) == false
     doAssert a == {'a': 5, 'c': 13}.newTable
     doAssert i == 0
+
   result = t[].take(key, val)
 
 proc clear*[A, B](t: TableRef[A, B]) =
@@ -1025,6 +1027,7 @@ proc clear*[A, B](t: TableRef[A, B]) =
     doAssert len(a) == 3
     clear(a)
     doAssert len(a) == 0
+
   clearImpl()
 
 proc `$`*[A, B](t: TableRef[A, B]): string =
@@ -1041,6 +1044,7 @@ proc `==`*[A, B](s, t: TableRef[A, B]): bool =
       a = {'a': 5, 'b': 9, 'c': 13}.newTable
       b = {'b': 9, 'c': 13, 'a': 5}.newTable
     doAssert a == b
+
   if isNil(s): result = isNil(t)
   elif isNil(t): result = false
   else: equalsImpl(s[], t[])
@@ -1089,6 +1093,7 @@ iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) =
     for k, v in a.mpairs:
       v.add(v[0] + 10)
     doAssert a == {'e': @[2, 4, 6, 8, 12], 'o': @[1, 5, 7, 9, 11]}.newTable
+
   for h in 0..high(t.data):
     if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val)
 
@@ -1106,6 +1111,7 @@ iterator keys*[A, B](t: TableRef[A, B]): A =
     for k in a.keys:
       a[k].add(99)
     doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.newTable
+
   for h in 0..high(t.data):
     if isFilled(t.data[h].hcode): yield t.data[h].key
 
@@ -1123,6 +1129,7 @@ iterator values*[A, B](t: TableRef[A, B]): B =
       }.newTable
     for v in a.values:
       doAssert v.len == 4
+
   for h in 0..high(t.data):
     if isFilled(t.data[h].hcode): yield t.data[h].val
 
@@ -1140,6 +1147,7 @@ iterator mvalues*[A, B](t: TableRef[A, B]): var B =
     for v in a.mvalues:
       v.add(99)
     doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.newTable
+
   for h in 0..high(t.data):
     if isFilled(t.data[h].hcode): yield t.data[h].val
 
@@ -1218,7 +1226,7 @@ template forAllOrderedPairs(yieldStmt: untyped): typed {.dirty.} =
 
 # ----------------------------------------------------------------------
 
-proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] =
+proc initOrderedTable*[A, B](initialsize = defaultInitialSize): OrderedTable[A, B] =
   ## Creates a new ordered hash table that is empty.
   ##
   ## ``initialSize`` must be a power of two (default: 64).
@@ -1227,6 +1235,9 @@ proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] =
   ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_
   ## from this module.
   ##
+  ## Starting from Nim v0.20, tables are initialized by default and it is
+  ## not necessary to call this function explicitly.
+  ##
   ## See also:
   ## * `toOrderedTable proc<#toOrderedTable,openArray[]>`_
   ## * `newOrderedTable proc<#newOrderedTable,int>`_ for creating an
@@ -1235,11 +1246,9 @@ proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] =
     let
       a = initOrderedTable[int, string]()
       b = initOrderedTable[char, seq[int]]()
-  assert isPowerOfTwo(initialSize)
+  initImpl(result, initialSize)
-  result.counter = 0
   result.first = -1
   result.last = -1
-  newSeq(result.data, initialSize)
 
 proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] =
   ## Creates a new ordered hash table that contains the given ``pairs``.
@@ -1254,6 +1263,7 @@ proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] =
     let a = [('a', 5), ('b', 9)]
     let b = toOrderedTable(a)
     assert b == {'a': 5, 'b': 9}.toOrderedTable
+
   result = initOrderedTable[A, B](rightSize(pairs.len))
   for key, val in items(pairs): result[key] = val
 
@@ -1278,6 +1288,7 @@ proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B =
     doAssert a['a'] == 5
     doAssertRaises(KeyError):
       echo a['z']
+
   get(t, key)
 
 proc `[]`*[A, B](t: var OrderedTable[A, B], key: A): var B=
@@ -1309,6 +1320,7 @@ proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) =
     a['x'] = 7
     a['y'] = 33
     doAssert a == {'x': 7, 'y': 33}.toOrderedTable
+
   putImpl(enlarge)
 
 proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool =
@@ -1326,6 +1338,7 @@ proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool =
     let a = {'a': 5, 'b': 9}.toOrderedTable
     doAssert a.hasKey('a') == true
     doAssert a.hasKey('z') == false
+
   var hc: Hash
   result = rawGet(t, key, hc) >= 0
 
@@ -1336,6 +1349,7 @@ proc contains*[A, B](t: OrderedTable[A, B], key: A): bool =
     let a = {'a': 5, 'b': 9}.toOrderedTable
     doAssert 'b' in a == true
     doAssert a.contains('z') == false
+
   return hasKey[A, B](t, key)
 
 proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool =
@@ -1355,6 +1369,7 @@ proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool =
     if a.hasKeyOrPut('z', 50):
       a['z'] = 99
     doAssert a == {'a': 99, 'b': 9, 'z': 50}.toOrderedTable
+
   hasKeyOrPutImpl(enlarge)
 
 proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B =
@@ -1373,6 +1388,7 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B =
     let a = {'a': 5, 'b': 9}.toOrderedTable
     doAssert a.getOrDefault('a') == 5
     doAssert a.getOrDefault('z') == 0
+
   getOrDefaultImpl(t, key)
 
 proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B =
@@ -1390,6 +1406,7 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B =
     let a = {'a': 5, 'b': 9}.toOrderedTable
     doAssert a.getOrDefault('a', 99) == 5
     doAssert a.getOrDefault('z', 99) == 99
+
   getOrDefaultImpl(t, key, default)
 
 proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B =
@@ -1409,6 +1426,7 @@ proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B =
     doAssert a.mgetOrPut('a', 99) == 5
     doAssert a.mgetOrPut('z', 99) == 99
     doAssert a == {'a': 5, 'b': 9, 'z': 99}.toOrderedTable
+
   mgetOrPutImpl(enlarge)
 
 proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} =
@@ -1416,6 +1434,7 @@ proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} =
   runnableExamples:
     let a = {'a': 5, 'b': 9}.toOrderedTable
     doAssert len(a) == 2
+
   result = t.counter
 
 proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) =
@@ -1440,6 +1459,7 @@ proc del*[A, B](t: var OrderedTable[A, B], key: A) =
     doAssert a == {'b': 9, 'c': 13}.toOrderedTable
     a.del('z')
     doAssert a == {'b': 9, 'c': 13}.toOrderedTable
+
   var n: OrderedKeyValuePairSeq[A, B]
   newSeq(n, len(t.data))
   var h = t.first
@@ -1467,6 +1487,7 @@ proc clear*[A, B](t: var OrderedTable[A, B]) =
     doAssert len(a) == 3
     clear(a)
     doAssert len(a) == 0
+
   clearImpl()
   t.first = -1
   t.last = -1
@@ -1602,6 +1623,7 @@ iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) =
     for k, v in a.mpairs:
       v.add(v[0] + 10)
     doAssert a == {'o': @[1, 5, 7, 9, 11], 'e': @[2, 4, 6, 8, 12]}.toOrderedTable
+
   forAllOrderedPairs:
     yield (t.data[h].key, t.data[h].val)
 
@@ -1619,6 +1641,7 @@ iterator keys*[A, B](t: OrderedTable[A, B]): A =
     for k in a.keys:
       a[k].add(99)
     doAssert a == {'o': @[1, 5, 7, 9, 99], 'e': @[2, 4, 6, 8, 99]}.toOrderedTable
+
   forAllOrderedPairs:
     yield t.data[h].key
 
@@ -1636,6 +1659,7 @@ iterator values*[A, B](t: OrderedTable[A, B]): B =
       }.toOrderedTable
     for v in a.values:
       doAssert v.len == 4
+
   forAllOrderedPairs:
     yield t.data[h].val
 
@@ -1666,7 +1690,7 @@ iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B =
 # --------------------------- OrderedTableRef -------------------------------
 # ---------------------------------------------------------------------------
 
-proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] =
+proc newOrderedTable*[A, B](initialsize = defaultInitialSize): <//>OrderedTableRef[A, B] =
   ## Creates a new ordered ref hash table that is empty.
   ##
   ## ``initialSize`` must be a power of two (default: 64).
@@ -1687,7 +1711,7 @@ proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] =
   new(result)
   result[] = initOrderedTable[A, B](initialSize)
 
-proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] =
+proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): <//>OrderedTableRef[A, B] =
   ## Creates a new ordered ref hash table that contains the given ``pairs``.
   ##
   ## ``pairs`` is a container consisting of ``(key, value)`` tuples.
@@ -1700,6 +1724,7 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] =
     let a = [('a', 5), ('b', 9)]
     let b = newOrderedTable(a)
     assert b == {'a': 5, 'b': 9}.newOrderedTable
+
   result = newOrderedTable[A, B](rightSize(pairs.len))
   for key, val in items(pairs): result.add(key, val)
 
@@ -1740,6 +1765,7 @@ proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) =
     a['x'] = 7
     a['y'] = 33
     doAssert a == {'x': 7, 'y': 33}.newOrderedTable
+
   t[][key] = val
 
 proc hasKey*[A, B](t: OrderedTableRef[A, B], key: A): bool =
@@ -1757,6 +1783,7 @@ proc hasKey*[A, B](t: OrderedTableRef[A, B], key: A): bool =
     let a = {'a': 5, 'b': 9}.newOrderedTable
     doAssert a.hasKey('a') == true
     doAssert a.hasKey('z') == false
+
   result = t[].hasKey(key)
 
 proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool =
@@ -1766,6 +1793,7 @@ proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool =
     let a = {'a': 5, 'b': 9}.newOrderedTable
     doAssert 'b' in a == true
     doAssert a.contains('z') == false
+
   return hasKey[A, B](t, key)
 
 proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool =
@@ -1785,6 +1813,7 @@ proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool =
     if a.hasKeyOrPut('z', 50):
       a['z'] = 99
     doAssert a == {'a': 99, 'b': 9, 'z': 50}.newOrderedTable
+
   result = t[].hasKeyOrPut(key, val)
 
 proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A): B =
@@ -1803,6 +1832,7 @@ proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A): B =
     let a = {'a': 5, 'b': 9}.newOrderedTable
     doAssert a.getOrDefault('a') == 5
     doAssert a.getOrDefault('z') == 0
+
   getOrDefault(t[], key)
 
 proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A, default: B): B =
@@ -1820,6 +1850,7 @@ proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A, default: B): B =
     let a = {'a': 5, 'b': 9}.newOrderedTable
     doAssert a.getOrDefault('a', 99) == 5
     doAssert a.getOrDefault('z', 99) == 99
+
   getOrDefault(t[], key, default)
 
 proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B =
@@ -1839,6 +1870,7 @@ proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B =
     doAssert a.mgetOrPut('a', 99) == 5
     doAssert a.mgetOrPut('z', 99) == 99
     doAssert a == {'a': 5, 'b': 9, 'z': 99}.newOrderedTable
+
   result = t[].mgetOrPut(key, val)
 
 proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} =
@@ -1846,6 +1878,7 @@ proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} =
   runnableExamples:
     let a = {'a': 5, 'b': 9}.newOrderedTable
     doAssert len(a) == 2
+
   result = t.counter
 
 proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: B) =
@@ -1868,6 +1901,7 @@ proc del*[A, B](t: var OrderedTableRef[A, B], key: A) =
     doAssert a == {'b': 9, 'c': 13}.newOrderedTable
     a.del('z')
     doAssert a == {'b': 9, 'c': 13}.newOrderedTable
+
   t[].del(key)
 
 proc clear*[A, B](t: var OrderedTableRef[A, B]) =
@@ -1880,6 +1914,7 @@ proc clear*[A, B](t: var OrderedTableRef[A, B]) =
     doAssert len(a) == 3
     clear(a)
     doAssert len(a) == 0
+
   clear(t[])
 
 proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x,y: (A, B)): int, order = SortOrder.Ascending) =
@@ -1899,6 +1934,7 @@ proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x,y: (A, B)): int, order =
     doAssert a == {'a': 10, 'b': 20, 'c': 0}.newOrderedTable
     a.sort(system.cmp, order=SortOrder.Descending)
     doAssert a == {'c': 0, 'b': 20, 'a': 10}.newOrderedTable
+
   t[].sort(cmp, order=order)
 
 proc `$`*[A, B](t: OrderedTableRef[A, B]): string =
@@ -1915,6 +1951,7 @@ proc `==`*[A, B](s, t: OrderedTableRef[A, B]): bool =
       a = {'a': 5, 'b': 9, 'c': 13}.newOrderedTable
       b = {'b': 9, 'c': 13, 'a': 5}.newOrderedTable
     doAssert a != b
+
   if isNil(s): result = isNil(t)
   elif isNil(t): result = false
   else: result = s[] == t[]
@@ -1964,6 +2001,7 @@ iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) =
     for k, v in a.mpairs:
       v.add(v[0] + 10)
     doAssert a == {'o': @[1, 5, 7, 9, 11], 'e': @[2, 4, 6, 8, 12]}.newOrderedTable
+
   forAllOrderedPairs:
     yield (t.data[h].key, t.data[h].val)
 
@@ -1981,6 +2019,7 @@ iterator keys*[A, B](t: OrderedTableRef[A, B]): A =
     for k in a.keys:
       a[k].add(99)
     doAssert a == {'o': @[1, 5, 7, 9, 99], 'e': @[2, 4, 6, 8, 99]}.newOrderedTable
+
   forAllOrderedPairs:
     yield t.data[h].key
 
@@ -1998,6 +2037,7 @@ iterator values*[A, B](t: OrderedTableRef[A, B]): B =
       }.newOrderedTable
     for v in a.values:
       doAssert v.len == 4
+
   forAllOrderedPairs:
     yield t.data[h].val
 
@@ -2016,6 +2056,7 @@ iterator mvalues*[A, B](t: OrderedTableRef[A, B]): var B =
     for v in a.mvalues:
       v.add(99)
     doAssert a == {'o': @[1, 5, 7, 9, 99], 'e': @[2, 4, 6, 8, 99]}.newOrderedTable
+
   forAllOrderedPairs:
     yield t.data[h].val
 
@@ -2046,7 +2087,7 @@ type
 
 # ------------------------------ helpers ---------------------------------
 
-proc rawInsert[A](t: CountTable[A], data: var seq[tuple[key: A, val: int]],
+proc ctRawInsert[A](t: CountTable[A], data: var seq[tuple[key: A, val: int]],
                   key: A, val: int) =
   var h: Hash = hash(key) and high(data)
   while data[h].val != 0: h = nextTry(h, high(data))
@@ -2057,10 +2098,12 @@ proc enlarge[A](t: var CountTable[A]) =
   var n: seq[tuple[key: A, val: int]]
   newSeq(n, len(t.data) * growthFactor)
   for i in countup(0, high(t.data)):
-    if t.data[i].val != 0: rawInsert(t, n, t.data[i].key, t.data[i].val)
+    if t.data[i].val != 0: ctRawInsert(t, n, t.data[i].key, t.data[i].val)
   swap(t.data, n)
 
 proc rawGet[A](t: CountTable[A], key: A): int =
+  if t.data.len == 0:
+    return -1
   var h: Hash = hash(key) and high(t.data) # start with real hash value
   while t.data[h].val != 0:
     if t.data[h].key == key: return h
@@ -2075,7 +2118,7 @@ proc inc*[A](t: var CountTable[A], key: A, val = 1)
 
 # ----------------------------------------------------------------------
 
-proc initCountTable*[A](initialSize=64): CountTable[A] =
+proc initCountTable*[A](initialsize = defaultInitialSize): CountTable[A] =
   ## Creates a new count table that is empty.
   ##
   ## ``initialSize`` must be a power of two (default: 64).
@@ -2084,13 +2127,14 @@ proc initCountTable*[A](initialSize=64): CountTable[A] =
   ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_
   ## from this module.
   ##
+  ## Starting from Nim v0.20, tables are initialized by default and it is
+  ## not necessary to call this function explicitly.
+  ##
   ## See also:
   ## * `toCountTable proc<#toCountTable,openArray[A]>`_
   ## * `newCountTable proc<#newCountTable,int>`_ for creating a
   ##   `CountTableRef`
-  assert isPowerOfTwo(initialSize)
+  initImpl(result, initialSize)
-  result.counter = 0
-  newSeq(result.data, initialSize)
 
 proc toCountTable*[A](keys: openArray[A]): CountTable[A] =
   ## Creates a new count table with every member of a container ``keys``
@@ -2130,9 +2174,7 @@ proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
   if h >= 0:
     t.data[h].val = val
   else:
-    if mustRehash(len(t.data), t.counter): enlarge(t)
+    insertImpl()
-    rawInsert(t, t.data, key, val)
-    inc(t.counter)
 
 proc inc*[A](t: var CountTable[A], key: A, val = 1) =
   ## Increments ``t[key]`` by ``val`` (default: 1).
@@ -2141,14 +2183,13 @@ proc inc*[A](t: var CountTable[A], key: A, val = 1) =
     a.inc('a')
     a.inc('b', 10)
     doAssert a == toCountTable("aaabbbbbbbbbbb")
+
   var index = rawGet(t, key)
   if index >= 0:
     inc(t.data[index].val, val)
     if t.data[index].val == 0: dec(t.counter)
   else:
-    if mustRehash(len(t.data), t.counter): enlarge(t)
+    insertImpl()
-    rawInsert(t, t.data, key, val)
-    inc(t.counter)
 
 proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] =
   ## Returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n)
@@ -2229,6 +2270,7 @@ proc sort*[A](t: var CountTable[A], order = SortOrder.Descending) =
     doAssert toSeq(a.values) == @[5, 2, 2, 1, 1]
     a.sort(SortOrder.Ascending)
     doAssert toSeq(a.values) == @[1, 1, 2, 2, 5]
+
   t.data.sort(cmp=ctCmp, order=order)
 
 proc merge*[A](s: var CountTable[A], t: CountTable[A]) =
@@ -2238,6 +2280,7 @@ proc merge*[A](s: var CountTable[A], t: CountTable[A]) =
     let b = toCountTable("bcc")
     a.merge(b)
     doAssert a == toCountTable("aaabbbccc")
+
   for key, value in t:
     s.inc(key, value)
 
@@ -2248,6 +2291,7 @@ proc merge*[A](s, t: CountTable[A]): CountTable[A] =
       a = toCountTable("aaabbc")
       b = toCountTable("bcc")
     doAssert merge(a, b) == toCountTable("aaabbbccc")
+
   result = initCountTable[A](nextPowerOfTwo(max(s.len, t.len)))
   for table in @[s, t]:
     for key, value in table:
@@ -2306,6 +2350,7 @@ iterator mpairs*[A](t: var CountTable[A]): (A, var int) =
     for k, v in mpairs(a):
       v = 2
     doAssert a == toCountTable("aabbccddrr")
+
   for h in 0..high(t.data):
     if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val)
 
@@ -2320,6 +2365,7 @@ iterator keys*[A](t: CountTable[A]): A =
     for k in keys(a):
       a[k] = 2
     doAssert a == toCountTable("aabbccddrr")
+
   for h in 0..high(t.data):
     if t.data[h].val != 0: yield t.data[h].key
 
@@ -2334,6 +2380,7 @@ iterator values*[A](t: CountTable[A]): int =
     let a = toCountTable("abracadabra")
     for v in values(a):
       assert v < 10
+
   for h in 0..high(t.data):
     if t.data[h].val != 0: yield t.data[h].val
 
@@ -2349,6 +2396,7 @@ iterator mvalues*[A](t: var CountTable[A]): var int =
     for v in mvalues(a):
       v = 2
     doAssert a == toCountTable("aabbccddrr")
+
   for h in 0..high(t.data):
     if t.data[h].val != 0: yield t.data[h].val
 
@@ -2364,7 +2412,7 @@ iterator mvalues*[A](t: var CountTable[A]): var int =
 
 proc inc*[A](t: CountTableRef[A], key: A, val = 1)
 
-proc newCountTable*[A](initialSize=64): CountTableRef[A] =
+proc newCountTable*[A](initialsize = defaultInitialSize): <//>CountTableRef[A] =
   ## Creates a new ref count table that is empty.
   ##
   ## ``initialSize`` must be a power of two (default: 64).
@@ -2381,7 +2429,7 @@ proc newCountTable*[A](initialSize=64): CountTableRef[A] =
   new(result)
   result[] = initCountTable[A](initialSize)
 
-proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] =
+proc newCountTable*[A](keys: openArray[A]): <//>CountTableRef[A] =
   ## Creates a new ref count table with every member of a container ``keys``
   ## having a count of how many times it occurs in that container.
   result = newCountTable[A](rightSize(keys.len))
@@ -2493,6 +2541,7 @@ proc merge*[A](s, t: CountTableRef[A]) =
       b = newCountTable("bcc")
     a.merge(b)
     doAssert a == newCountTable("aaabbbccc")
+
   s[].merge(t[])
 
 proc `$`*[A](t: CountTableRef[A]): string =
@@ -2551,6 +2600,7 @@ iterator mpairs*[A](t: CountTableRef[A]): (A, var int) =
     for k, v in mpairs(a):
       v = 2
     doAssert a == newCountTable("aabbccddrr")
+
   for h in 0..high(t.data):
     if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val)
 
@@ -2565,6 +2615,7 @@ iterator keys*[A](t: CountTableRef[A]): A =
     for k in keys(a):
       a[k] = 2
     doAssert a == newCountTable("aabbccddrr")
+
   for h in 0..high(t.data):
     if t.data[h].val != 0: yield t.data[h].key
 
@@ -2579,6 +2630,7 @@ iterator values*[A](t: CountTableRef[A]): int =
     let a = newCountTable("abracadabra")
     for v in values(a):
       assert v < 10
+
   for h in 0..high(t.data):
     if t.data[h].val != 0: yield t.data[h].val
 
@@ -2593,6 +2645,7 @@ iterator mvalues*[A](t: CountTableRef[A]): var int =
     for v in mvalues(a):
       v = 2
     doAssert a == newCountTable("aabbccddrr")
+
   for h in 0..high(t.data):
     if t.data[h].val != 0: yield t.data[h].val
 
@@ -2813,3 +2866,99 @@ when isMainModule:
     doAssert "test1" == orf.getOrDefault("test1", "test1")
     orf["test2"] = "test2"
     doAssert "test2" == orf.getOrDefault("test2", "test1")
+
+  block tableWithoutInit:
+    var
+      a: Table[string, int]
+      b: Table[string, int]
+      c: Table[string, int]
+      d: Table[string, int]
+      e: Table[string, int]
+
+    a["a"] = 7
+    doAssert a.hasKey("a")
+    doAssert a.len == 1
+    doAssert a["a"] == 7
+    a["a"] = 9
+    doAssert a.len == 1
+    doAssert a["a"] == 9
+
+    doAssert b.hasKeyOrPut("b", 5) == false
+    doAssert b.hasKey("b")
+    doAssert b.hasKeyOrPut("b", 8)
+    doAssert b["b"] == 5
+
+    doAssert c.getOrDefault("a") == 0
+    doAssert c.getOrDefault("a", 3) == 3
+    c["a"] = 6
+    doAssert c.getOrDefault("a", 3) == 6
+
+    doAssert d.mgetOrPut("a", 3) == 3
+    doAssert d.mgetOrPut("a", 6) == 3
+
+    var x = 99
+    doAssert e.take("a", x) == false
+    doAssert x == 99
+    e["a"] = 77
+    doAssert e.take("a", x)
+    doAssert x == 77
+
+  block orderedTableWithoutInit:
+    var
+      a: OrderedTable[string, int]
+      b: OrderedTable[string, int]
+      c: OrderedTable[string, int]
+      d: OrderedTable[string, int]
+
+    a["a"] = 7
+    doAssert a.hasKey("a")
+    doAssert a.len == 1
+    doAssert a["a"] == 7
+    a["a"] = 9
+    doAssert a.len == 1
+    doAssert a["a"] == 9
+
+    doAssert b.hasKeyOrPut("b", 5) == false
+    doAssert b.hasKey("b")
+    doAssert b.hasKeyOrPut("b", 8)
+    doAssert b["b"] == 5
+
+    doAssert c.getOrDefault("a") == 0
+    doAssert c.getOrDefault("a", 3) == 3
+    c["a"] = 6
+    doAssert c.getOrDefault("a", 3) == 6
+
+    doAssert d.mgetOrPut("a", 3) == 3
+    doAssert d.mgetOrPut("a", 6) == 3
+
+  block countTableWithoutInit:
+    var
+      a: CountTable[string]
+      b: CountTable[string]
+      c: CountTable[string]
+      d: CountTable[string]
+      e: CountTable[string]
+
+    a["a"] = 7
+    doAssert a.hasKey("a")
+    doAssert a.len == 1
+    doAssert a["a"] == 7
+    a["a"] = 9
+    doAssert a.len == 1
+    doAssert a["a"] == 9
+
+    doAssert b["b"] == 0
+    b.inc("b")
+    doAssert b["b"] == 1
+
+    doAssert c.getOrDefault("a") == 0
+    doAssert c.getOrDefault("a", 3) == 3
+    c["a"] = 6
+    doAssert c.getOrDefault("a", 3) == 6
+
+    e["f"] = 3
+    merge(d, e)
+    doAssert d.hasKey("f")
+    d.inc("f")
+    merge(d, e)
+    doAssert d["f"] == 7

lib/pure/dynlib.nim

@@ -12,10 +12,10 @@
 ## Windows ``LoadLibrary``.
 ##
 ## Examples
-## --------
+## ========
 ##
 ## Loading a simple C function
-## ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+## ---------------------------
 ##
 ## The following example demonstrates loading a function called 'greet'
 ## from a library that is determined at runtime based upon a language choice.

lib/pure/htmlgen.nim

@@ -16,7 +16,8 @@
 ## generator. Each commonly used HTML tag has a corresponding macro
 ## that generates a string with its HTML representation.
 ##
-## Example:
+## Examples
+## ========
 ##
 ## .. code-block:: Nim
 ##   var nim = "Nim"

lib/pure/includes/osenv.nim

@@ -1,6 +1,6 @@
 # Include file that implements 'getEnv' and friends. Do not import it!
 
-when not declared(os):
+when not declared(os) and not declared(ospaths):
   {.error: "This is an include file for os.nim!".}
 
 from parseutils import skipIgnoreCase

lib/pure/includes/oserr.nim

@@ -1,6 +1,6 @@
 # Include file that implements 'osErrorMsg' and friends. Do not import it!
 
-when not declared(os):
+when not declared(os) and not declared(ospaths):
   {.error: "This is an include file for os.nim!".}
 
 when not defined(nimscript):

lib/pure/parsecfg.nim

@@ -12,48 +12,46 @@
 ## format, but much more powerful, as it is not a line based parser. String
 ## literals, raw string literals and triple quoted string literals are supported
 ## as in the Nim programming language.
-
+##
-## This is an example of how a configuration file may look like:
+## Example of how a configuration file may look like:
 ##
 ## .. include:: ../../doc/mytest.cfg
 ##     :literal:
 ##
-
+## Here is an example of how to use the configuration file parser:
-##[ Here is an example of how to use the configuration file parser:
+##
-
+## .. code-block:: nim
-.. code-block:: nim
+##
-
+##    import os, parsecfg, strutils, streams
-    import
+##
-      os, parsecfg, strutils, streams
+##    var f = newFileStream(paramStr(1), fmRead)
-
+##    if f != nil:
-    var f = newFileStream(paramStr(1), fmRead)
+##      var p: CfgParser
-    if f != nil:
+##      open(p, f, paramStr(1))
-      var p: CfgParser
+##      while true:
-      open(p, f, paramStr(1))
+##        var e = next(p)
-      while true:
+##        case e.kind
-        var e = next(p)
+##        of cfgEof: break
-        case e.kind
+##        of cfgSectionStart:   ## a ``[section]`` has been parsed
-        of cfgEof: break
+##          echo("new section: " & e.section)
-        of cfgSectionStart:   ## a ``[section]`` has been parsed
+##        of cfgKeyValuePair:
-          echo("new section: " & e.section)
+##          echo("key-value-pair: " & e.key & ": " & e.value)
-        of cfgKeyValuePair:
+##        of cfgOption:
-          echo("key-value-pair: " & e.key & ": " & e.value)
+##          echo("command: " & e.key & ": " & e.value)
-        of cfgOption:
+##        of cfgError:
-          echo("command: " & e.key & ": " & e.value)
+##          echo(e.msg)
-        of cfgError:
+##      close(p)
-          echo(e.msg)
+##    else:
-      close(p)
+##      echo("cannot open: " & paramStr(1))
-    else:
+##
-      echo("cannot open: " & paramStr(1))
+##
-
-]##
-
 ## Examples
-## --------
+## ========
 ##
-## This is an example of a configuration file.
+## Configuration file example
+## --------------------------
 ##
-## ::
+## .. code-block:: nim
 ##
 ##     charset = "utf-8"
 ##     [Package]
@@ -64,8 +62,8 @@
 ##     qq = "10214028"
 ##     email = "lihaifeng@wxm.com"
 ##
-## Creating a configuration file.
+## Creating a configuration file
-## ==============================
+## -----------------------------
 ## .. code-block:: nim
 ##
 ##     import parsecfg
@@ -78,8 +76,8 @@
 ##     dict.setSectionKey("Author","email","lihaifeng@wxm.com")
 ##     dict.writeConfig("config.ini")
 ##
-## Reading a configuration file.
+## Reading a configuration file
-## =============================
+## ----------------------------
 ## .. code-block:: nim
 ##
 ##     import parsecfg
@@ -92,8 +90,8 @@
 ##     var email = dict.getSectionValue("Author","email")
 ##     echo pname & "\n" & name & "\n" & qq & "\n" & email
 ##
-## Modifying a configuration file.
+## Modifying a configuration file
-## ===============================
+## ------------------------------
 ## .. code-block:: nim
 ##
 ##     import parsecfg
@@ -101,8 +99,8 @@
 ##     dict.setSectionKey("Author","name","lhf")
 ##     dict.writeConfig("config.ini")
 ##
-## Deleting a section key in a configuration file.
+## Deleting a section key in a configuration file
-## ===============================================
+## ----------------------------------------------
 ## .. code-block:: nim
 ##
 ##     import parsecfg
@@ -443,17 +441,17 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} =
 
 # ---------------- Configuration file related operations ----------------
 type
-  Config* = OrderedTableRef[string, OrderedTableRef[string, string]]
+  Config* = OrderedTableRef[string, <//>OrderedTableRef[string, string]]
 
 proc newConfig*(): Config =
   ## Create a new configuration table.
   ## Useful when wanting to create a configuration file.
-  result = newOrderedTable[string, OrderedTableRef[string, string]]()
+  result = newOrderedTable[string, <//>OrderedTableRef[string, string]]()
 
-proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config =
+proc loadConfig*(stream: Stream, filename: string = "[stream]"): <//>Config =
   ## Load the specified configuration from stream into a new Config instance.
   ## `filename` parameter is only used for nicer error messages.
-  var dict = newOrderedTable[string, OrderedTableRef[string, string]]()
+  var dict = newOrderedTable[string, <//>OrderedTableRef[string, string]]()
   var curSection = "" ## Current section,
                       ## the default value of the current section is "",
                       ## which means that the current section is a common
@@ -483,7 +481,7 @@ proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config =
   close(p)
   result = dict
 
-proc loadConfig*(filename: string): Config =
+proc loadConfig*(filename: string): <//>Config =
   ## Load the specified configuration file into a new Config instance.
   let file = open(filename, fmRead)
   let fileStream = newFileStream(file)

lib/pure/parseutils.nim

@@ -64,104 +64,154 @@ const
 proc toLower(c: char): char {.inline.} =
   result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c
 
-proc parseHex*(s: string, number: var int, start = 0; maxLen = 0): int {.
+proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, maxLen = 0): int
-  rtl, extern: "npuParseHex", noSideEffect.}  =
+  {.noSideEffect.} =
+  ## Parses a binary number and stores its value in ``number``.
+  ##
+  ## Returns the number of the parsed characters or 0 in case of an error.
+  ## If error, the value of ``number`` is not changed.
+  ##
+  ## If ``maxLen == 0``, the parsing continues until the first non-bin character
+  ## or to the end of the string. Otherwise, no more than ``maxLen`` characters
+  ## are parsed starting from the ``start`` position.
+  ## 
+  ## It does not check for overflow. If the value represented by the string is
+  ## too big to fit into ``number``, only the value of last fitting characters
+  ## will be stored in ``number`` without producing an error.
+  runnableExamples:
+    var num: int
+    doAssert parseBin("0100_1110_0110_1001_1110_1101", num) == 29
+    doAssert num == 5138925
+    doAssert parseBin("3", num) == 0
+    var num8: int8
+    doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8) == 32
+    doAssert num8 == 0b1110_1101'i8
+    doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8, 3, 9) == 9
+    doAssert num8 == 0b0100_1110'i8
+    var num8u: uint8
+    doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8u) == 32
+    doAssert num8u == 237
+    var num64: int64
+    doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40
+    doAssert num64 == 336784608873
+  var i = start
+  var output = T(0)
+  var foundDigit = false
+  let last = min(s.len, if maxLen == 0: s.len else: i + maxLen)
+  if i + 1 < last and s[i] == '0' and (s[i+1] in {'b', 'B'}): inc(i, 2)
+  while i < last:
+    case s[i]
+    of '_': discard
+    of '0'..'1':
+      output = output shl 1 or T(ord(s[i]) - ord('0'))
+      foundDigit = true
+    else: break
+    inc(i)
+  if foundDigit:
+    number = output
+    result = i - start
+
+proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, maxLen = 0): int
+  {.noSideEffect.} =
+  ## Parses an octal number and stores its value in ``number``.
+  ##
+  ## Returns the number of the parsed characters or 0 in case of an error.
+  ## If error, the value of ``number`` is not changed.
+  ##
+  ## If ``maxLen == 0``, the parsing continues until the first non-oct character
+  ## or to the end of the string. Otherwise, no more than ``maxLen`` characters
+  ## are parsed starting from the ``start`` position.
+  ## 
+  ## It does not check for overflow. If the value represented by the string is
+  ## too big to fit into ``number``, only the value of last fitting characters
+  ## will be stored in ``number`` without producing an error.
+  runnableExamples:
+    var num: int
+    doAssert parseOct("0o23464755", num) == 10
+    doAssert num == 5138925
+    doAssert parseOct("8", num) == 0
+    var num8: int8
+    doAssert parseOct("0o_1464_755", num8) == 11
+    doAssert num8 == -19
+    doAssert parseOct("0o_1464_755", num8, 3, 3) == 3
+    doAssert num8 == 102
+    var num8u: uint8
+    doAssert parseOct("1464755", num8u) == 7
+    doAssert num8u == 237
+    var num64: int64
+    doAssert parseOct("2346475523464755", num64) == 16
+    doAssert num64 == 86216859871725
+  var i = start
+  var output = T(0)
+  var foundDigit = false
+  let last = min(s.len, if maxLen == 0: s.len else: i + maxLen)
+  if i + 1 < last and s[i] == '0' and (s[i+1] in {'o', 'O'}): inc(i, 2)
+  while i < last:
+    case s[i]
+    of '_': discard
+    of '0'..'7':
+      output = output shl 3 or T(ord(s[i]) - ord('0'))
+      foundDigit = true
+    else: break
+    inc(i)
+  if foundDigit:
+    number = output
+    result = i - start
+
+proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, maxLen = 0): int
+  {.noSideEffect.} =
   ## Parses a hexadecimal number and stores its value in ``number``.
   ##
-  ## Returns the number of the parsed characters or 0 in case of an error. This
+  ## Returns the number of the parsed characters or 0 in case of an error.
-  ## proc is sensitive to the already existing value of ``number`` and will
+  ## If error, the value of ``number`` is not changed.
-  ## likely not do what you want unless you make sure ``number`` is zero. You
-  ## can use this feature to *chain* calls, though the result int will quickly
-  ## overflow.
   ##
-  ## If ``maxLen == 0`` the length of the hexadecimal number has no upper bound.
+  ## If ``maxLen == 0``, the parsing continues until the first non-hex character
-  ## Else no more than ``start + maxLen`` characters are parsed, up to the
+  ## or to the end of the string. Otherwise, no more than ``maxLen`` characters
-  ## length of the string.
+  ## are parsed starting from the ``start`` position.
+  ## 
+  ## It does not check for overflow. If the value represented by the string is
+  ## too big to fit into ``number``, only the value of last fitting characters
+  ## will be stored in ``number`` without producing an error.
   runnableExamples:
-    var value = 0
+    var num: int
-    discard parseHex("0x38", value)
+    doAssert parseHex("4E_69_ED", num) == 8
-    assert value == 56
+    doAssert num == 5138925
-    discard parseHex("0x34", value)
+    doAssert parseHex("X", num) == 0
-    assert value == 56 * 256 + 52
+    doAssert parseHex("#ABC", num) == 4
-    value = -1
+    var num8: int8
-    discard parseHex("0x38", value)
+    doAssert parseHex("0x_4E_69_ED", num8) == 11
-    assert value == -200
+    doAssert num8 == 0xED'i8
+    doAssert parseHex("0x_4E_69_ED", num8, 3, 2) == 2
+    doAssert num8 == 0x4E'i8
+    var num8u: uint8
+    doAssert parseHex("0x_4E_69_ED", num8u) == 11
+    doAssert num8u == 237
+    var num64: int64
+    doAssert parseHex("4E69ED4E69ED", num64) == 12
+    doAssert num64 == 86216859871725
   var i = start
+  var output = T(0)
   var foundDigit = false
-  # get last index based on minimum `start + maxLen` or `s.len`
+  let last = min(s.len, if maxLen == 0: s.len else: i + maxLen)
-  let last = min(s.len, if maxLen == 0: s.len else: i+maxLen)
+  if i + 1 < last and s[i] == '0' and (s[i+1] in {'x', 'X'}): inc(i, 2)
-  if i+1 < last and s[i] == '0' and (s[i+1] in {'x', 'X'}): inc(i, 2)
   elif i < last and s[i] == '#': inc(i)
   while i < last:
     case s[i]
     of '_': discard
     of '0'..'9':
-      number = number shl 4 or (ord(s[i]) - ord('0'))
+      output = output shl 4 or T(ord(s[i]) - ord('0'))
       foundDigit = true
     of 'a'..'f':
-      number = number shl 4 or (ord(s[i]) - ord('a') + 10)
+      output = output shl 4 or T(ord(s[i]) - ord('a') + 10)
       foundDigit = true
     of 'A'..'F':
-      number = number shl 4 or (ord(s[i]) - ord('A') + 10)
+      output = output shl 4 or T(ord(s[i]) - ord('A') + 10)
       foundDigit = true
     else: break
     inc(i)
-  if foundDigit: result = i-start
+  if foundDigit:
-
+    number = output
-proc parseOct*(s: string, number: var int, start = 0, maxLen = 0): int  {.
+    result = i - start
-  rtl, extern: "npuParseOct", noSideEffect.} =
-  ## Parses an octal number and stores its value in ``number``. Returns
-  ## the number of the parsed characters or 0 in case of an error.
-  ##
-  ## If ``maxLen == 0`` the length of the octal number has no upper bound.
-  ## Else no more than ``start + maxLen`` characters are parsed, up to the
-  ## length of the string.
-  runnableExamples:
-    var res: int
-    doAssert parseOct("12", res) == 2
-    doAssert res == 10
-    doAssert parseOct("9", res) == 0
-  var i = start
-  var foundDigit = false
-  # get last index based on minimum `start + maxLen` or `s.len`
-  let last = min(s.len, if maxLen == 0: s.len else: i+maxLen)
-  if i+1 < last and s[i] == '0' and (s[i+1] in {'o', 'O'}): inc(i, 2)
-  while i < last:
-    case s[i]
-    of '_': discard
-    of '0'..'7':
-      number = number shl 3 or (ord(s[i]) - ord('0'))
-      foundDigit = true
-    else: break
-    inc(i)
-  if foundDigit: result = i-start
-
-proc parseBin*(s: string, number: var int, start = 0, maxLen = 0): int {.
-  rtl, extern: "npuParseBin", noSideEffect.} =
-  ## Parses an binary number and stores its value in ``number``. Returns
-  ## the number of the parsed characters or 0 in case of an error.
-  ##
-  ## If ``maxLen == 0`` the length of the binary number has no upper bound.
-  ## Else no more than ``start + maxLen`` characters are parsed, up to the
-  ## length of the string.
-  runnableExamples:
-    var res: int
-    doAssert parseBin("010011100110100101101101", res) == 24
-    doAssert parseBin("3", res) == 0
-  var i = start
-  var foundDigit = false
-  # get last index based on minimum `start + maxLen` or `s.len`
-  let last = min(s.len, if maxLen == 0: s.len else: i+maxLen)
-  if i+1 < last and s[i] == '0' and (s[i+1] in {'b', 'B'}): inc(i, 2)
-  while i < last:
-    case s[i]
-    of '_': discard
-    of '0'..'1':
-      number = number shl 1 or (ord(s[i]) - ord('0'))
-      foundDigit = true
-    else: break
-    inc(i)
-  if foundDigit: result = i-start
 
 proc parseIdent*(s: string, ident: var string, start = 0): int =
   ## Parses an identifier and stores it in ``ident``. Returns
@@ -605,11 +655,6 @@ when isMainModule:
   var value = 0
   discard parseHex("0x38", value)
   doAssert value == 56
-  discard parseHex("0x34", value)
-  doAssert value == 56 * 256 + 52
-  value = -1
-  discard parseHex("0x38", value)
-  doAssert value == -200
 
   value = -1
   doAssert(parseSaturatedNatural("848", value) == 3)

lib/pure/random.nim

@@ -129,7 +129,7 @@ proc next*(r: var Rand): uint64 =
   ##   a given upper bound
   ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
   ## * `rand proc<#rand,Rand,HSlice[T,T]>`_ that accepts a slice
-  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer type
+  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type
   ## * `skipRandomNumbers proc<#skipRandomNumbers,Rand>`_
   runnableExamples:
     var r = initRand(2019)
@@ -242,7 +242,7 @@ proc rand*(r: var Rand; max: Natural): int {.benign.} =
   ##   random number generator
   ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
   ## * `rand proc<#rand,Rand,HSlice[T,T]>`_ that accepts a slice
-  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer type
+  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type
   runnableExamples:
     var r = initRand(123)
     doAssert r.rand(100) == 0
@@ -268,7 +268,7 @@ proc rand*(max: int): int {.benign.} =
   ##   provided state
   ## * `rand proc<#rand,float>`_ that returns a float
   ## * `rand proc<#rand,HSlice[T,T]>`_ that accepts a slice
-  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer type
+  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type
   runnableExamples:
     randomize(123)
     doAssert rand(100) == 0
@@ -285,7 +285,7 @@ proc rand*(r: var Rand; max: range[0.0 .. high(float)]): float {.benign.} =
   ##   random number generator
   ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
   ## * `rand proc<#rand,Rand,HSlice[T,T]>`_ that accepts a slice
-  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer type
+  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type
   runnableExamples:
     var r = initRand(234)
     let f = r.rand(1.0)
@@ -311,7 +311,7 @@ proc rand*(max: float): float {.benign.} =
   ##   provided state
   ## * `rand proc<#rand,int>`_ that returns an integer
   ## * `rand proc<#rand,HSlice[T,T]>`_ that accepts a slice
-  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer type
+  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type
   runnableExamples:
     randomize(234)
     let f = rand(1.0)
@@ -327,7 +327,7 @@ proc rand*[T: Ordinal](r: var Rand; x: HSlice[T, T]): T =
   ##   default random number generator
   ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
   ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer type
+  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type
   runnableExamples:
     var r = initRand(345)
     doAssert r.rand(1..6) == 4
@@ -349,7 +349,7 @@ proc rand*[T: Ordinal](x: HSlice[T, T]): T =
   ##   a provided state
   ## * `rand proc<#rand,int>`_ that returns an integer
   ## * `rand proc<#rand,float>`_ that returns a floating point number
-  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer type
+  ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type
   runnableExamples:
     randomize(345)
     doAssert rand(1..6) == 4
@@ -383,7 +383,13 @@ proc rand*[T: SomeInteger](t: typedesc[T]): T =
     doAssert rand(uint32) == 578980729'u32
     doAssert rand(uint32) == 4052940463'u32
     doAssert rand(uint32) == 2163872389'u32
-  result = cast[T](state.next)
+    doAssert rand(range[1..16]) == 11
+    doAssert rand(range[1..16]) == 4
+    doAssert rand(range[1..16]) == 16
+  when T is range:
+    result = rand(state, low(T)..high(T))
+  else:
+    result = cast[T](state.next)
 
 proc rand*[T](a: openArray[T]): T {.deprecated.} =
   ## **Deprecated since version 0.20.0:**

lib/pure/strformat.nim

@@ -419,6 +419,9 @@ proc formatValue*(result: var string; value: SomeInteger; specifier: string) =
   ## Standard format implementation for ``SomeInteger``. It makes little
   ## sense to call this directly, but it is required to exist
   ## by the ``&`` macro.
+  if specifier.len == 0:
+    result.add $value
+    return
   let spec = parseStandardFormatSpecifier(specifier)
   var radix = 10
   case spec.typ
@@ -432,10 +435,13 @@ proc formatValue*(result: var string; value: SomeInteger; specifier: string) =
       " of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ)
   result.add formatInt(value, radix, spec)
 
-proc formatValue*(result: var string; value: SomeFloat; specifier: string): void =
+proc formatValue*(result: var string; value: SomeFloat; specifier: string) =
   ## Standard format implementation for ``SomeFloat``. It makes little
   ## sense to call this directly, but it is required to exist
   ## by the ``&`` macro.
+  if specifier.len == 0:
+    result.add $value
+    return
   let spec = parseStandardFormatSpecifier(specifier)
 
   var fmode = ffDefault
@@ -457,7 +463,7 @@ proc formatValue*(result: var string; value: SomeFloat; specifier: string): void
   if value >= 0.0:
     if spec.sign != '-':
       sign = true
-      if  value == 0.0:
+      if value == 0.0:
         if 1.0 / value == Inf:
           # only insert the sign if value != negZero
           f.insert($spec.sign, 0)
@@ -467,16 +473,16 @@ proc formatValue*(result: var string; value: SomeFloat; specifier: string): void
     sign = true
 
   if spec.padWithZero:
-    var sign_str = ""
+    var signStr = ""
     if sign:
-      sign_str = $f[0]
+      signStr = $f[0]
       f = f[1..^1]
 
     let toFill = spec.minimumWidth - f.len - ord(sign)
     if toFill > 0:
       f = repeat('0', toFill) & f
     if sign:
-      f = sign_str & f
+      f = signStr & f
 
   # the default for numbers is right-alignment:
   let align = if spec.align == '\0': '>' else: spec.align
@@ -712,6 +718,13 @@ when isMainModule:
   for s in invalidUtf8:
     check &"{s:>5}", repeat(" ", 5-s.len) & s
 
+  # bug #11089
+  let flfoo: float = 1.0
+  check &"{flfoo}", "1.0"
+
+  # bug #11092
+  check &"{high(int64)}", "9223372036854775807"
+  check &"{low(int64)}", "-9223372036854775808"
 
   import json
 

lib/pure/times.nim

@@ -17,7 +17,8 @@
   resolution used by ``getTime()`` depends on the platform and backend
   (JS is limited to millisecond precision).
 
-  Examples:
+  Examples
+  ========
 
   .. code-block:: nim
     import times, os
@@ -38,7 +39,7 @@
     echo "One month from now     : ", now() + 1.months
 
   Parsing and Formatting Dates
-  ----------------------------
+  ============================
 
   The ``DateTime`` type can be parsed and formatted using the different
   ``parse`` and ``format`` procedures.
@@ -128,7 +129,7 @@
   only for years in the range 1..9999).
 
   Duration vs TimeInterval
-  ----------------------------
+  ============================
   The ``times`` module exports two similiar types that are both used to
   represent some amount of time: `Duration <#Duration>`_ and
   `TimeInterval <#TimeInterval>`_.
@@ -137,7 +138,7 @@
   needed).
 
   Duration
-  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  ----------------------------
   A ``Duration`` represents a duration of time stored as seconds and
   nanoseconds. A ``Duration`` is always fully normalized, so
 ``initDuration(hours = 1)`` and ``initDuration(minutes = 60)`` are equivilant.
@@ -147,7 +148,7 @@
   is more performant and easier to understand it should generally prefered.
 
   TimeInterval
-  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  ----------------------------
   A ``TimeInterval`` represents some amount of time expressed in calendar
   units, for example "1 year and 2 days". Since some units cannot be
   normalized (the length of a year is different for leap years for example),
@@ -164,7 +165,7 @@
   ``Duration`` doesn't have.
 
   How long is a day?
-  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  ----------------------------
   It should be especially noted that the handling of days differs between
   ``TimeInterval`` and ``Duration``. The ``Duration`` type always treats a day
   as exactly 86400 seconds. For ``TimeInterval``, it's more complex.
@@ -2651,7 +2652,7 @@ proc microseconds*(dur: Duration): int {.inline, deprecated.} =
     doAssert dur.microseconds == 7008
   result = convert(Nanoseconds, Microseconds, dur.nanosecond)
 
-proc nanoseconds*(dur: Duration): NanosecondRange {.inline.} =
+proc nanoseconds*(dur: Duration): NanosecondRange {.inline, deprecated.} =
   ## Number of whole microseconds represented by the **fractional**
   ## part of the duration.
   ##

lib/pure/unittest.nim

@@ -57,8 +57,8 @@
 ##   # Run suites starting with 'bug #' and standalone tests starting with '#'
 ##   nim c -r test 'bug #*::' '::#*'
 ##
-## Example
+## Examples
-## -------
+## ========
 ##
 ## .. code:: nim
 ##
@@ -177,13 +177,13 @@ proc addOutputFormatter*(formatter: OutputFormatter) =
   formatters.add(formatter)
 
 proc newConsoleOutputFormatter*(outputLevel: OutputLevel = PRINT_ALL,
-                                colorOutput = true): ConsoleOutputFormatter =
+                                colorOutput = true): <//>ConsoleOutputFormatter =
   ConsoleOutputFormatter(
     outputLevel: outputLevel,
     colorOutput: colorOutput
   )
 
-proc defaultConsoleFormatter*(): ConsoleOutputFormatter =
+proc defaultConsoleFormatter*(): <//>ConsoleOutputFormatter =
   when declared(stdout):
     # Reading settings
     # On a terminal this branch is executed
@@ -263,7 +263,7 @@ proc xmlEscape(s: string): string =
       else:
         result.add(c)
 
-proc newJUnitOutputFormatter*(stream: Stream): JUnitOutputFormatter =
+proc newJUnitOutputFormatter*(stream: Stream): <//>JUnitOutputFormatter =
   ## Creates a formatter that writes report to the specified stream in
   ## JUnit format.
   ## The ``stream`` is NOT closed automatically when the test are finished,

lib/system.nim

@@ -1619,7 +1619,7 @@ template `isnot`*(x, y: untyped): untyped = not (x is y)
   ##   assert @[1, 2] isnot enum
 
 when defined(nimV2) and not defined(nimscript):
-  type owned*{.magic: "BuiltinType".}[T]
+  type owned*{.magic: "BuiltinType".}[T] ## type constructor to mark a ref/ptr or a closure as `owned`.
 
   proc new*[T](a: var owned(ref T)) {.magic: "New", noSideEffect.}
     ## Creates a new object of type ``T`` and returns a safe (traced)
@@ -1637,8 +1637,16 @@ when defined(nimV2) and not defined(nimscript):
       var r: owned(ref t)
     new(r)
     return r
+
+  proc unown*[T](x: T): T {.magic: "Unown", noSideEffect.}
+    ## Use the expression ``x`` ignoring its ownership attribute.
+
+  # This is only required to make 0.20 compile with the 0.19 line.
+  template `<//>`*(t: untyped): untyped = owned(t)
+
 else:
   template owned*(t: typeDesc): typedesc = t
+  template unown*(x: typed): typed = x
 
   proc new*[T](a: var ref T) {.magic: "New", noSideEffect.}
     ## Creates a new object of type ``T`` and returns a safe (traced)
@@ -1657,6 +1665,9 @@ else:
     new(r)
     return r
 
+  # This is only required to make 0.20 compile with the 0.19 line.
+  template `<//>`*(t: untyped): untyped = t
+
 template disarm*(x: typed) =
   ## Useful for ``disarming`` dangling pointers explicitly for the
   ## --newruntime. Regardless of whether --newruntime is used or not
@@ -1926,13 +1937,11 @@ const
     ## failure.
 
 when defined(nodejs) and not defined(nimscript):
-  var programResult* {.importc: "process.exitCode".}: int
+  var programResult* {.importc: "process.exitCode", deprecated.}: int
   programResult = 0
-else:
+elif hostOS != "standalone":
-  var programResult* {.compilerproc, exportc: "nim_program_result".}: int
+  var programResult* {.compilerproc, exportc: "nim_program_result", deprecated.}: int
-    ## Modify this variable to specify the exit code of the program
+    ## deprecated, prefer ``quit``
-    ## under normal circumstances. When the program is terminated
-    ## prematurely using `quit proc <#quit,int>`_, this value is ignored.
 
 when defined(nimdoc):
   proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.}
@@ -1994,18 +2003,7 @@ when not defined(JS) and not defined(nimscript) and hostOS != "standalone":
 when not defined(JS) and not defined(nimscript) and hasAlloc and not defined(gcDestructors):
   proc addChar(s: NimString, c: char): NimString {.compilerProc, benign.}
 
-when defined(gcDestructors):
+when not defined(gcDestructors):
-  proc add*[T](x: var seq[T], y: sink T) {.magic: "AppendSeqElem", noSideEffect.} =
-    ## Generic proc for adding a data item `y` to a container `x`.
-    ##
-    ## For containers that have an order, `add` means *append*. New generic
-    ## containers should also call their adding proc `add` for consistency.
-    ## Generic code becomes much easier to write if the Nim naming scheme is
-    ## respected.
-    let xl = x.len
-    setLen(x, xl + 1)
-    x[xl] = y
-else:
   proc add*[T](x: var seq[T], y: T) {.magic: "AppendSeqElem", noSideEffect.}
     ## Generic proc for adding a data item `y` to a container `x`.
     ##
@@ -2239,14 +2237,15 @@ proc addQuitProc*(quitProc: proc() {.noconv.}) {.
 # not be called explicitly! The user may decide to do this manually though.
 
 when not defined(nimscript) and not defined(JS):
-  proc zeroMem*(p: pointer, size: Natural) {.inline, benign.}
+  proc zeroMem*(p: pointer, size: Natural) {.inline, noSideEffect,
+    tags: [], locks: 0, raises: [].}
     ## Overwrites the contents of the memory at ``p`` with the value 0.
     ##
     ## Exactly ``size`` bytes will be overwritten. Like any procedure
     ## dealing with raw memory this is **unsafe**.
 
   proc copyMem*(dest, source: pointer, size: Natural) {.inline, benign,
-    tags: [], locks: 0.}
+    tags: [], locks: 0, raises: [].}
     ## Copies the contents from the memory at ``source`` to the memory
     ## at ``dest``.
     ## Exactly ``size`` bytes will be copied. The memory
@@ -2254,7 +2253,7 @@ when not defined(nimscript) and not defined(JS):
     ## memory this is **unsafe**.
 
   proc moveMem*(dest, source: pointer, size: Natural) {.inline, benign,
-    tags: [], locks: 0.}
+    tags: [], locks: 0, raises: [].}
     ## Copies the contents from the memory at ``source`` to the memory
     ## at ``dest``.
     ##
@@ -2263,7 +2262,8 @@ when not defined(nimscript) and not defined(JS):
     ## and is thus somewhat more safe than ``copyMem``. Like any procedure
     ## dealing with raw memory this is still **unsafe**, though.
 
-  proc equalMem*(a, b: pointer, size: Natural): bool {.inline, noSideEffect, tags: [], locks: 0.}
+  proc equalMem*(a, b: pointer, size: Natural): bool {.inline, noSideEffect,
+    tags: [], locks: 0, raises: [].}
     ## Compares the memory blocks ``a`` and ``b``. ``size`` bytes will
     ## be compared.
     ##

lib/system/ansi_c.nim

@@ -148,4 +148,4 @@ proc rawWrite*(f: CFilePtr, s: cstring) {.compilerproc, nonreloadable, inline.}
   # we cannot throw an exception here!
   discard c_fwrite(s, 1, s.len, f)
 
-{.pop}
+{.pop.}

lib/system/fatal.nim

@@ -10,6 +10,8 @@ when hostOS == "standalone":
     panic(arg)
 
 elif defined(nimQuirky) and not defined(nimscript):
+  import ansi_c
+
   proc name(t: typedesc): string {.magic: "TypeTrait".}
 
   proc sysFatal(exceptn: typedesc, message, arg: string) {.inline, noReturn.} =

lib/system/strmantle.nim

@@ -147,58 +147,58 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
     has_sign = false
 
   # Sign?
-  if s[i] == '+' or s[i] == '-':
+  if i < s.len and (s[i] == '+' or s[i] == '-'):
     has_sign = true
     if s[i] == '-':
       sign = -1.0
     inc(i)
 
   # NaN?
-  if s[i] == 'N' or s[i] == 'n':
+  if i+2 < s.len and (s[i] == 'N' or s[i] == 'n'):
     if s[i+1] == 'A' or s[i+1] == 'a':
       if s[i+2] == 'N' or s[i+2] == 'n':
-        if s[i+3] notin IdentChars:
+        if i+3 >= s.len or s[i+3] notin IdentChars:
           number = NaN
           return i+3 - start
     return 0
 
   # Inf?
-  if s[i] == 'I' or s[i] == 'i':
+  if i+2 < s.len and (s[i] == 'I' or s[i] == 'i'):
     if s[i+1] == 'N' or s[i+1] == 'n':
       if s[i+2] == 'F' or s[i+2] == 'f':
-        if s[i+3] notin IdentChars:
+        if i+3 >= s.len or s[i+3] notin IdentChars:
           number = Inf*sign
           return i+3 - start
     return 0
 
-  if s[i] in {'0'..'9'}:
+  if i < s.len and s[i] in {'0'..'9'}:
     first_digit = (s[i].ord - '0'.ord)
   # Integer part?
-  while s[i] in {'0'..'9'}:
+  while i < s.len and s[i] in {'0'..'9'}:
     inc(kdigits)
     integer = integer * 10'u64 + (s[i].ord - '0'.ord).uint64
     inc(i)
-    while s[i] == '_': inc(i)
+    while i < s.len and s[i] == '_': inc(i)
 
   # Fractional part?
-  if s[i] == '.':
+  if i < s.len and s[i] == '.':
     inc(i)
     # if no integer part, Skip leading zeros
     if kdigits <= 0:
-      while s[i] == '0':
+      while i < s.len and s[i] == '0':
         inc(frac_exponent)
         inc(i)
-        while s[i] == '_': inc(i)
+        while i < s.len and s[i] == '_': inc(i)
 
-    if first_digit == -1 and s[i] in {'0'..'9'}:
+    if first_digit == -1 and i < s.len and s[i] in {'0'..'9'}:
       first_digit = (s[i].ord - '0'.ord)
     # get fractional part
-    while s[i] in {'0'..'9'}:
+    while i < s.len and s[i] in {'0'..'9'}:
       inc(fdigits)
       inc(frac_exponent)
       integer = integer * 10'u64 + (s[i].ord - '0'.ord).uint64
       inc(i)
-      while s[i] == '_': inc(i)
+      while i < s.len and s[i] == '_': inc(i)
 
   # if has no digits: return error
   if kdigits + fdigits <= 0 and
@@ -206,7 +206,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
      (i == start + 1 and has_sign)): # or only '+' or '-
     return 0
 
-  if s[i] in {'e', 'E'}:
+  if i+1 < s.len and s[i] in {'e', 'E'}:
     inc(i)
     if s[i] == '+' or s[i] == '-':
       if s[i] == '-':
@@ -215,10 +215,10 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
       inc(i)
     if s[i] notin {'0'..'9'}:
       return 0
-    while s[i] in {'0'..'9'}:
+    while i < s.len and s[i] in {'0'..'9'}:
       exponent = exponent * 10 + (ord(s[i]) - ord('0'))
       inc(i)
-      while s[i] == '_': inc(i) # underscores are allowed and ignored
+      while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored
 
   var real_exponent = exp_sign*exponent - frac_exponent
   let exp_negative = real_exponent < 0
@@ -259,12 +259,12 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
   result = i - start
   i = start
   # re-parse without error checking, any error should be handled by the code above.
-  if s[i] == '.': i.inc
+  if i < s.len and s[i] == '.': i.inc
-  while s[i] in {'0'..'9','+','-'}:
+  while i < s.len and s[i] in {'0'..'9','+','-'}:
     if ti < maxlen:
       t[ti] = s[i]; inc(ti)
     inc(i)
-    while s[i] in {'.', '_'}: # skip underscore and decimal point
+    while i < s.len and s[i] in {'.', '_'}: # skip underscore and decimal point
       inc(i)
 
   # insert exponent

lib/system/threads.nim

@@ -19,7 +19,8 @@
 ## to prevent race conditions and improves efficiency. See `the manual for
 ## details of this memory model <manual.html#threads>`_.
 ##
-## Example:
+## Examples
+## ========
 ##
 ## .. code-block:: Nim
 ##
@@ -328,7 +329,7 @@ else:
 proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) =
   createThread[void](t, tp)
 
-## we need to cache current threadId to not perform syscall all the time
+# we need to cache current threadId to not perform syscall all the time
 var threadId {.threadvar.}: int
 
 when defined(windows):

nimdoc/testproject/expected/testproject.html

@@ -878,7 +878,9 @@ function main() {
   <span class="Identifier">subdir</span> <span class="Operator">/</span> <span class="Identifier">subdir_b</span> <span class="Operator">/</span> <span class="Identifier">utils</span>
 
 <span class="Identifier">doAssert</span> <span class="Identifier">bar</span><span class="Other">(</span><span class="DecNumber">3</span><span class="Other">,</span> <span class="DecNumber">4</span><span class="Other">)</span> <span class="Operator">==</span> <span class="DecNumber">7</span>
-<span class="Identifier">foo</span><span class="Other">(</span><span class="Identifier">enumValueA</span><span class="Other">,</span> <span class="Identifier">enumValueB</span><span class="Other">)</span></pre></p>
+<span class="Identifier">foo</span><span class="Other">(</span><span class="Identifier">enumValueA</span><span class="Other">,</span> <span class="Identifier">enumValueB</span><span class="Other">)</span>
+<span class="Keyword">for</span> <span class="Identifier">x</span> <span class="Keyword">in</span> <span class="StringLit">&quot;xx&quot;</span><span class="Other">:</span>
+  <span class="Keyword">discard</span></pre></p>
   <div class="section" id="6">
 <h1><a class="toc-backref" href="#6">Imports</a></h1>
 <dl class="item">

nimdoc/testproject/testproject.nim

@@ -6,6 +6,8 @@ runnableExamples:
   import subdir / subdir_b / utils
   doAssert bar(3, 4) == 7
   foo(enumValueA, enumValueB)
+  # bug #11078
+  for x in "xx": discard
 
 
 template foo*(a, b: SomeType) =

testament/categories.nim

@@ -11,6 +11,7 @@
 ## of the compiler.
 
 import important_packages
+import sequtils
 
 const
   specialCategories = [
@@ -514,30 +515,21 @@ proc testNimblePackages(r: var TResults, cat: Category) =
       let buildPath = packagesDir / name
       if not existsDir(buildPath):
         if hasDep:
-          let nimbleProcess = startProcess("nimble", "", ["install", "-y", name],
+          let (nimbleOutput, nimbleStatus) = execCmdEx2("nimble", ["install", "-y", name])
-                                           options = {poUsePath, poStdErrToStdOut})
-          let nimbleStatus = waitForExitEx(nimbleProcess)
-          nimbleProcess.close
           if nimbleStatus != QuitSuccess:
-            r.addResult(test, targetC, "", "'nimble install' failed", reInstallFailed)
+            r.addResult(test, targetC, "", "'nimble install' failed\n" & nimbleOutput, reInstallFailed)
             continue
 
-        let installProcess = startProcess("git", "", ["clone", url, buildPath],
+        let (installOutput, installStatus) = execCmdEx2("git", ["clone", url, buildPath])
-                                          options = {poUsePath, poStdErrToStdOut})
-        let installStatus = waitForExitEx(installProcess)
-        installProcess.close
         if installStatus != QuitSuccess:
-          r.addResult(test, targetC, "", "'git clone' failed", reInstallFailed)
+          r.addResult(test, targetC, "", "'git clone' failed\n" & installOutput, reInstallFailed)
           continue
 
       let cmdArgs = parseCmdLine(cmd)
-      let buildProcess = startProcess(cmdArgs[0], buildPath, cmdArgs[1..^1],
-                                      options = {poUsePath, poStdErrToStdOut})
-      let buildStatus = waitForExitEx(buildProcess)
-      buildProcess.close
 
+      let (buildOutput, buildStatus) = execCmdEx2(cmdArgs[0], cmdArgs[1..^1], workingDir=buildPath)
       if buildStatus != QuitSuccess:
-        r.addResult(test, targetC, "", "package test failed", reBuildFailed)
+        r.addResult(test, targetC, "", "package test failed\n" & buildOutput, reBuildFailed)
       else:
         inc r.passed
         r.addResult(test, targetC, "", "", reSuccess)
@@ -653,8 +645,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
 
   let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd", "megatest.nim"]
   proc onStdout(line: string) = echo line
-  var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "",
+  var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "")
-    onStdout = if verboseMegatest: onStdout else: nil)
   if exitCode != 0:
     echo buf.string
     quit("megatest compilation failed")
@@ -688,8 +679,22 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
 
 # ---------------------------------------------------------------------------
 
-proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
+proc loadSkipFrom(name: string): seq[string] =
+  # One skip per line, comments start with #
+  # used by `nlvm` (at least)
+  try:
+    for line in lines(name):
+      let sline = line.strip()
+      if sline.len > 0 and not sline.startsWith("#"):
+        result.add sline
+  except:
+    echo "Could not load " & name & ", ignoring"
+
+proc processCategory(r: var TResults, cat: Category,
+                     options, testsDir, skipFrom: string,
                      runJoinableTests: bool) =
+  let skips = loadSkipFrom(skipFrom)
+
   case cat.string.normalize
   of "rodfiles":
     when false:
@@ -745,6 +750,9 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
 
     for i, name in files:
       var test = makeTest(name, options, cat)
+      if skips.anyIt(it in name):
+        test.spec.err = reDisabled
+
       if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
         discard "run the test"
       else:

testament/tester.nim

@@ -19,8 +19,6 @@ include compiler/nodejs
 var useColors = true
 var backendLogging = true
 var simulate = false
-var verboseMegatest = false # very verbose but can be useful
-var verboseCommands = false
 
 const
   testsDir = "tests" & DirSep
@@ -39,8 +37,6 @@ Arguments:
   arguments are passed to the compiler
 Options:
   --print                   also print results to the console
-  --verboseMegatest         log to stdout megatetest compilation
-  --verboseCommands         log to stdout info about commands being run
   --simulate                see what tests would be run but don't run them (for debugging)
   --failing                 only show failing/ignored tests
   --targets:"c c++ js objc" run tests for specified targets (default: all)
@@ -48,6 +44,7 @@ Options:
   --directory:dir           Change to directory dir before reading the tests or doing anything else.
   --colors:on|off           Turn messagescoloring on|off.
   --backendLogging:on|off   Disable or enable backend logging. By default turned on.
+  --skipFrom:file           Read tests to skip from `file` - one test per line, # comments ignored
 """ % resultsFile
 
 type
@@ -91,11 +88,11 @@ proc getFileDir(filename: string): string =
   if not result.isAbsolute():
     result = getCurrentDir() / result
 
-proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOption], input: string, onStdout: proc(line: string) = nil): tuple[
+proc execCmdEx2(command: string, args: openarray[string]; workingDir, input: string = ""): tuple[
                 output: TaintedString,
                 exitCode: int] {.tags:
                 [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
-  var p = startProcess(command, args=args, options=options)
+  var p = startProcess(command, workingDir=workingDir, args=args, options={poStdErrToStdOut, poUsePath})
   var outp = outputStream(p)
 
   # There is no way to provide input for the child process
@@ -111,20 +108,11 @@ proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOp
     if outp.readLine(line):
       result.output.string.add(line.string)
       result.output.string.add("\n")
-      if onStdout != nil: onStdout(line.string)
     else:
       result.exitCode = peekExitCode(p)
       if result.exitCode != -1: break
   close(p)
 
-  if verboseCommands:
-    var command2 = command
-    if args.len > 0: command2.add " " & args.quoteShellCommand
-    echo (msg: "execCmdEx2",
-      command: command2,
-      options: options,
-      exitCode: result.exitCode)
-
 proc nimcacheDir(filename, options: string, target: TTarget): string =
   ## Give each test a private nimcache dir so they don't clobber each other's.
   let hashInput = options & $target
@@ -266,17 +254,25 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
     maybeStyledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
     maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
     maybeStyledEcho styleBright, fgRed, "Failure: ", $success
-    maybeStyledEcho fgYellow, "Expected:"
+    if success in {reBuildFailed, reNimcCrash, reInstallFailed}:
-    maybeStyledEcho styleBright, expected, "\n"
+      # expected is empty, no reason to print it.
-    maybeStyledEcho fgYellow, "Gotten:"
+      echo given
-    maybeStyledEcho styleBright, given, "\n"
+    else:
+      maybeStyledEcho fgYellow, "Expected:"
+      maybeStyledEcho styleBright, expected, "\n"
+      maybeStyledEcho fgYellow, "Gotten:"
+      maybeStyledEcho styleBright, given, "\n"
+
 
   if backendLogging and existsEnv("APPVEYOR"):
     let (outcome, msg) =
-      if success == reSuccess:
+      case success
+      of reSuccess:
         ("Passed", "")
-      elif success in {reDisabled, reJoined}:
+      of reDisabled, reJoined:
         ("Skipped", "")
+      of reBuildFailed, reNimcCrash, reInstallFailed:
+        ("Failed", "Failure: " & $success & "\n" & given)
       else:
         ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
     var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options,
@@ -320,7 +316,7 @@ proc generatedFile(test: TTest, target: TTarget): string =
     let (_, name, _) = test.name.splitFile
     let ext = targetToExt[target]
     result = nimcacheDir(test.name, test.options, target) /
-             ("compiler_" & name.changeFileExt(ext))
+              name.replace("_", "__").changeFileExt(ext)
 
 proc needsCodegenCheck(spec: TSpec): bool =
   result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0
@@ -424,7 +420,7 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
       # nested conditionals - the empty rows in between to clarify the "danger"
       var given = callCompiler(expected.getCmd, test.name, test.options, target)
       if given.err != reSuccess:
-        r.addResult(test, target, "", given.msg, given.err)
+        r.addResult(test, target, "", given.nimout, given.err)
         continue
       let isJsTarget = target == targetJS
       var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt)
@@ -445,7 +441,7 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
         args = concat(@[exeFile], args)
       else:
         exeCmd = exeFile
-      var (buf, exitCode) = execCmdEx2(exeCmd, args, options = {poStdErrToStdOut}, input = expected.input)
+      var (buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input)
       # Treat all failure codes from nodejs as 1. Older versions of nodejs used
       # to return other codes, but for us it is sufficient to know that it's not 0.
       if exitCode != 0: exitCode = 1
@@ -514,6 +510,8 @@ const disabledFilesDefault = @[
   "LockFreeHash.nim",
   "sharedstrings.nim",
   "tableimpl.nim",
+  "setimpl.nim",
+  "hashcommon.nim",
 
   # Error: undeclared identifier: 'hasThreadSupport'
   "ioselectors_epoll.nim",
@@ -545,14 +543,13 @@ proc main() =
   var optFailing = false
   var targetsStr = ""
   var isMainProcess = true
+  var skipFrom = ""
 
   var p = initOptParser()
   p.next()
   while p.kind == cmdLongoption:
     case p.key.string.normalize
     of "print", "verbose": optPrintResults = true
-    of "verbosemegatest": verboseMegatest = true
-    of "verbosecommands": verboseCommands = true
     of "failing": optFailing = true
     of "pedantic": discard "now always enabled"
     of "targets":
@@ -580,6 +577,8 @@ proc main() =
         backendLogging = false
       else:
         quit Usage
+    of "skipfrom":
+      skipFrom = p.val.string
     else:
       quit Usage
     p.next()
@@ -598,6 +597,9 @@ proc main() =
 
     myself &= " " & quoteShell("--nim:" & compilerPrefix)
 
+    if skipFrom.len > 0:
+      myself &= " " & quoteShell("--skipFrom:" & skipFrom)
+
     var cats: seq[string]
     let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: ""
     for kind, dir in walkDir(testsDir):
@@ -617,13 +619,13 @@ proc main() =
     if simulate:
       for i, cati in cats:
         progressStatus(i)
-        processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false)
+        processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = false)
     else:
       quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
   of "c", "cat", "category":
     var cat = Category(p.key)
     p.next
-    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true)
+    processCategory(r, cat, p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = true)
   of "pcat":
     # 'pcat' is used for running a category in parallel. Currently the only
     # difference is that we don't want to run joinable tests here as they
@@ -631,7 +633,7 @@ proc main() =
     isMainProcess = false
     var cat = Category(p.key)
     p.next
-    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false)
+    processCategory(r, cat, p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = false)
   of "r", "run":
     # at least one directory is required in the path, to use as a category name
     let pathParts = split(p.key.string, {DirSep, AltSep})

tests/ccgbugs/t5701.nim

@@ -2,6 +2,7 @@ discard """
   output: '''(1, 1)
 (2, 2)
 (3, 3)
+@[1, 2, 3, 4]
 '''
 """
 
@@ -15,3 +16,12 @@ proc foo(args: varargs[int]) =
     discard
 
 foo(1,2,3)
+
+# 10999
+
+proc varargsToSeq(vals: varargs[int32]): seq[int32] =
+  result = newSeqOfCap[int32](vals.len)
+  for v in vals:
+    result.add v
+
+echo varargsToSeq(1, 2, 3, 4)

tests/destructor/tdestructor.nim

@@ -7,8 +7,8 @@ mygeneric1 constructed
 mygeneric1 destroyed
 ----
 mygeneric2 constructed
-myobj destroyed
 mygeneric2 destroyed
+myobj destroyed
 ----
 mygeneric3 constructed
 mygeneric1 destroyed
@@ -20,10 +20,10 @@ mygeneric2 destroyed
 ----
 ----
 myobj destroyed
+myobj destroyed
+myobj destroyed
+myobj destroyed
 mygeneric1 destroyed
-myobj destroyed
-myobj destroyed
-myobj destroyed
 ---
 myobj destroyed
 myobj destroyed

tests/destructor/tdestructor3.nim

@@ -4,8 +4,8 @@ destroy
 destroy
 5
 123
-destroy Foo: 5
+destroy Foo: 123
-destroy Foo: 123'''
+destroy Foo: 5'''
 joinable: false
 """
 

tests/destructor/tdont_return_unowned_from_owned.nim

@@ -0,0 +1,43 @@
+discard """
+  cmd: "nim check --newruntime --hints:off $file"
+  nimout: '''tdont_return_unowned_from_owned.nim(24, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type
+tdont_return_unowned_from_owned.nim(27, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type
+tdont_return_unowned_from_owned.nim(30, 6) Error: type mismatch: got <Obj>
+but expected one of:
+proc new[T](a: var ref T; finalizer: proc (x: ref T) {.nimcall.})
+2 other mismatching symbols have been suppressed; compile with --showAllMismatches:on to see them
+
+expression: new(result)
+tdont_return_unowned_from_owned.nim(30, 6) Error: illformed AST:
+tdont_return_unowned_from_owned.nim(38, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref
+tdont_return_unowned_from_owned.nim(39, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref
+tdont_return_unowned_from_owned.nim(43, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type
+'''
+  errormsg: "cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type"
+  line: 43
+"""
+# bug #11073
+type
+  Obj = ref object
+
+proc newObjA(): Obj =
+  result = new Obj
+
+proc newObjB(): Obj =
+  result = Obj()
+
+proc newObjC(): Obj =
+  new(result)
+
+let a = newObjA()
+let b = newObjB()
+let c = newObjC()
+
+proc testA(result: var (RootRef, RootRef)) =
+  let r: owned RootRef = RootRef()
+  result[0] = r
+  result[1] = RootRef()
+
+proc testB(): RootRef =
+  let r: owned RootRef = RootRef()
+  result = r

tests/destructor/tgcdestructors.nim

@@ -4,7 +4,9 @@ discard """
 ho
 ha
 @["arg", "asdfklasdfkl", "asdkfj", "dfasj", "klfjl"]
-22 22'''
+@[1, 2, 3]
+@["red", "yellow", "orange", "rtrt1", "pink"]
+30 30'''
 """
 
 import allocators
@@ -132,6 +134,41 @@ proc other =
 
 other()
 
+# bug #11050
+
+type
+  Obj* = object
+    f*: seq[int]
+
+method main(o: Obj) {.base.} =
+  for newb in o.f:
+    discard
+
+# test that o.f was not moved!
+proc testforNoMove =
+  var o = Obj(f: @[1, 2, 3])
+  main(o)
+  echo o.f
+
+testforNoMove()
+
+# bug #11065
+type
+  Warm = seq[string]
+
+proc testWarm =
+  var w: Warm
+  w = @["red", "yellow", "orange"]
+
+  var x = "rt"
+  var y = "rt1"
+  w.add(x & y)
+
+  w.add("pink")
+  echo w
+
+testWarm()
+
 #echo s
 let (a, d) = allocCounters()
 discard cprintf("%ld %ld\n", a, d)

tests/destructor/tobjfield_analysis.nim

@@ -2,30 +2,46 @@ discard """
   output: '''works'''
 """
 
-type
+#  bug #11095
-  MyVal = object
-    f: ptr float
 
-proc `=destroy`(x: var MyVal) =
+type
+  MyVal[T] = object
+    f: ptr T
+
+proc `=destroy`[T](x: var MyVal[T]) =
   if x.f != nil:
     dealloc(x.f)
 
-proc `=sink`(x1: var MyVal, x2: Myval) =
+proc `=sink`[T](x1: var MyVal[T], x2: MyVal[T]) =
   if x1.f != x2.f:
     `=destroy`(x1)
     x1.f = x2.f
 
-proc `=`(x1: var MyVal, x2: Myval) {.error.}
+proc `=`[T](x1: var MyVal[T], x2: MyVal[T]) {.error.}
 
-proc newVal(x: float): MyVal =
+proc newVal[T](x: sink T): MyVal[T] =
-  result.f = create(float)
+  result.f = create(T)
   result.f[] = x
 
-proc sinkMe(x: sink MyVal) =
+proc set[T](x: var MyVal[T], val: T) =
+  x.f[] = val
+
+proc sinkMe[T](x: sink MyVal[T]) =
   discard
 
+var flag = false
+
 proc main =
-  var y = (newVal(3.0), newVal(4.0))
+  var y = case flag
+    of true:
+      var x1 = newVal[float](1.0)
+      var x2 = newVal[float](2.0)
+      (newVal(x1), newVal(x2))
+
+    of false:
+      var x1 = newVal[float](1.0)
+      var x2 = newVal[float](2.0)
+      (newVal(x1), newVal(x2))
 
   sinkMe y[0]
   sinkMe y[1]

tests/destructor/towned_binary_tree.nim

@@ -0,0 +1,92 @@
+discard """
+  cmd: '''nim c --newruntime $file'''
+  output: '''331665
+allocs 0'''
+"""
+
+#  bug #11053
+
+import random
+
+type Node = ref object
+  x, y: int32
+  left, right: owned Node
+
+proc newNode(x: int32): owned Node =
+  result = Node(x: x, y: rand(high int32).int32)
+
+proc merge(lower, greater: owned Node): owned Node =
+  if lower.isNil:
+    result = greater
+  elif greater.isNil:
+    result = lower
+  elif lower.y < greater.y:
+    lower.right = merge(lower.right, greater)
+    result = lower
+  else:
+    greater.left = merge(lower, greater.left)
+    result = greater
+
+proc splitBinary(orig: owned Node, value: int32): (owned Node, owned Node) =
+  if orig.isNil:
+    result = (nil, nil)
+  elif orig.x < value:
+    let splitPair = splitBinary(orig.right, value)
+    orig.right = splitPair[0]
+    result = (orig, splitPair[1])
+  else:
+    let splitPair = splitBinary(orig.left, value)
+    orig.left = splitPair[1]
+    result = (splitPair[0], orig)
+
+proc merge3(lower, equal, greater: owned Node): owned Node =
+  merge(merge(lower, equal), greater)
+
+proc split(orig: owned Node, value: int32): tuple[lower, equal, greater: owned Node] =
+  let
+    (lower, equalGreater) = splitBinary(orig, value)
+    (equal, greater) = splitBinary(equalGreater, value + 1)
+  result = (lower, equal, greater)
+
+type Tree = object
+  root: owned Node
+
+proc hasValue(self: var Tree, x: int32): bool =
+  let splited = split(move self.root, x)
+  result = not splited.equal.isNil
+  self.root = merge3(splited.lower, splited.equal, splited.greater)
+
+proc insert(self: var Tree, x: int32) =
+  var splited = split(move self.root, x)
+  if splited.equal.isNil:
+    splited.equal = newNode(x)
+  self.root = merge3(splited.lower, splited.equal, splited.greater)
+
+proc erase(self: var Tree, x: int32) =
+  let splited = split(move self.root, x)
+  self.root = merge(splited.lower, splited.greater)
+
+proc main() =
+  var
+    tree = Tree()
+    cur = 5'i32
+    res = 0
+
+  for i in 1 ..< 1000000:
+    let a = i mod 3
+    cur = (cur * 57 + 43) mod 10007
+    case a:
+    of 0:
+      tree.insert(cur)
+    of 1:
+      tree.erase(cur)
+    of 2:
+      if tree.hasValue(cur):
+        res += 1
+    else:
+      discard
+  echo res
+
+when isMainModule:
+  main()
+  echo "allocs ", allocs

tests/destructor/twidgets.nim

@@ -0,0 +1,78 @@
+discard """
+  cmd: '''nim c --newruntime $file'''
+  output: '''button
+clicked!
+1 1  alloc/dealloc pairs: 0'''
+"""
+
+import core / allocators
+import system / ansi_c
+
+type
+  Widget* = ref object of RootObj
+    drawImpl: owned(proc (self: Widget))
+
+  Button* = ref object of Widget
+    caption: string
+    onclick: owned(proc())
+
+  Window* = ref object of Widget
+    elements: seq[owned Widget]
+
+
+proc newButton(caption: string; onclick: owned(proc())): owned Button =
+  proc draw(self: Widget) =
+    let b = Button(self)
+    echo b.caption
+
+  #result = Button(drawImpl: draw, caption: caption, onclick: onclick)
+  new(result)
+  result.drawImpl = draw
+  result.caption = caption
+  result.onclick = onclick
+
+iterator unitems*[T](a: seq[owned T]): T {.inline.} =
+  ## Iterates over each item of `a`.
+  var i = 0
+  let L = len(a)
+  while i < L:
+    yield a[i]
+    inc(i)
+    assert(len(a) == L, "seq modified while iterating over it")
+
+proc newWindow(): owned Window =
+  proc windraw(self: Widget) =
+    let w = Window(self)
+    for i in 0..<len(w.elements):
+      let e = Widget(w.elements[i])
+      let d = (proc(self: Widget))e.drawImpl
+      if not d.isNil: d(e)
+
+  result = Window(drawImpl: windraw, elements: @[])
+
+proc draw(w: Widget) =
+  let d = (proc(self: Widget))w.drawImpl
+  if not d.isNil: d(w)
+
+proc add*(w: Window; elem: owned Widget) =
+  w.elements.add elem
+
+proc main =
+  var w = newWindow()
+
+  var b = newButton("button", nil)
+  let u: Button = b
+  b.onclick = proc () =
+    b.caption = "clicked!"
+  w.add b
+
+  w.draw()
+  # simulate button click:
+  u.onclick()
+
+  w.draw()
+
+main()
+
+let (a, d) = allocCounters()
+discard cprintf("%ld %ld  alloc/dealloc pairs: %ld\n", a, d, allocs)

tests/destructor/twidgets_unown.nim

@@ -0,0 +1,68 @@
+discard """
+  cmd: '''nim c --newruntime $file'''
+  output: '''button
+clicked!
+3 3  alloc/dealloc pairs: 0'''
+"""
+
+import core / allocators
+import system / ansi_c
+
+type
+  Widget* = ref object of RootObj
+    drawImpl: owned(proc (self: Widget))
+
+  Button* = ref object of Widget
+    caption: string
+    onclick: owned(proc())
+
+  Window* = ref object of Widget
+    elements: seq[owned Widget]
+
+
+proc newButton(caption: string; onclick: owned(proc())): owned Button =
+  proc draw(self: Widget) =
+    let b = Button(self)
+    echo b.caption
+
+  #result = Button(drawImpl: draw, caption: caption, onclick: onclick)
+  new(result)
+  result.drawImpl = draw
+  result.caption = caption
+  result.onclick = onclick
+
+proc newWindow(): owned Window =
+  proc windraw(self: Widget) =
+    let w = Window(self)
+    for e in unown(w.elements):
+      let d = unown e.drawImpl
+      if not d.isNil: d(e)
+
+  result = Window(drawImpl: windraw, elements: @[])
+
+proc draw(w: Widget) =
+  let d = unown w.drawImpl
+  if not d.isNil: d(w)
+
+proc add*(w: Window; elem: owned Widget) =
+  w.elements.add elem
+
+proc main =
+  var w = newWindow()
+
+  var b = newButton("button", nil)
+  let u = unown b
+  b.onclick = proc () =
+    b.caption = "clicked!"
+  w.add b
+
+  w.draw()
+  # simulate button click:
+  u.onclick()
+
+  w.draw()
+
+main()
+
+let (a, d) = allocCounters()
+discard cprintf("%ld %ld  alloc/dealloc pairs: %ld\n", a, d, allocs)

tests/discard/tdiscardable.nim

@@ -3,6 +3,8 @@ output: '''
 tdiscardable
 1
 1
+something defered
+something defered
 '''
 """
 
@@ -45,3 +47,21 @@ proc foo: (proc: int) =
   return bar
 
 discard foo()
+
+# bug #10842
+
+proc myDiscardable(): int {.discardable.} =
+  discard
+
+proc main1() =
+  defer:
+    echo "something defered"
+  discard myDiscardable()
+
+proc main2() =
+  defer:
+    echo "something defered"
+  myDiscardable()
+
+main1()
+main2()

tests/macros/tnewproc.nim

@@ -0,0 +1,51 @@
+import macros
+
+macro test(a: untyped): untyped =
+  # proc hello*(x: int = 3, y: float32): int {.inline.} = discard
+  let
+    nameNode = nnkPostfix.newTree(
+      newIdentNode("*"),
+      newIdentNode("hello")
+    )
+    params = @[
+      newIdentNode("int"),
+      nnkIdentDefs.newTree(
+        newIdentNode("x"),
+        newIdentNode("int"),
+        newLit(3)
+      ),
+      nnkIdentDefs.newTree(
+        newIdentNode("y"),
+        newIdentNode("float32"),
+        newEmptyNode()
+      )
+    ]
+    paramsNode = nnkFormalParams.newTree(params)
+    pragmasNode = nnkPragma.newTree(
+      newIdentNode("inline")
+    )
+    bodyNode = nnkStmtList.newTree(
+      nnkDiscardStmt.newTree(
+        newEmptyNode()
+      )
+    )
+
+  var
+    expected = nnkProcDef.newTree(
+      nameNode,
+      newEmptyNode(),
+      newEmptyNode(),
+      paramsNode,
+      pragmasNode,
+      newEmptyNode(),
+      bodyNode
+    )
+
+  doAssert expected == newProc(name=nameNode, params=params,
+                                    body = bodyNode, pragmas=pragmasNode)
+  expected.pragma = newEmptyNode()
+  doAssert expected == newProc(name=nameNode, params=params,
+                                    body = bodyNode)
+
+test:
+  42

tests/modules/mnotuniquename.nim

@@ -0,0 +1 @@
+proc flat*() = echo "flat"

tests/modules/tnotuniquename.nim

@@ -1,7 +1,10 @@
 discard """
-  errormsg: "module names need to be unique per Nimble package"
+  output: '''nested
-  file: "tnotuniquename/mnotuniquename.nim"
+flat'''
 """
 
 import mnotuniquename
-import tnotuniquename/mnotuniquename as nun
+import tnotuniquename_dir/mnotuniquename as nun
+
+nested()
+flat()

tests/modules/tnotuniquename2.nim

@@ -1,6 +1,7 @@
 discard """
   errormsg: "module names need to be unique per Nimble package"
   file: "tnotuniquename/mnotuniquename.nim"
+  disabled: "true"
 """
 
 import mnotuniquename

tests/modules/tnotuniquename_dir/mnotuniquename.nim

@@ -0,0 +1,2 @@
+
+proc nested*() = echo "nested"

tests/types/tillegaltuplerecursiongeneric.nim

@@ -1,5 +1,5 @@
 discard """
-  errormsg: "illegal recursion in type 'RefTreeInt'"
+  errormsg: "illegal recursion in type 'RefTree'"
 """
 
 type

tests/types/tillegaltuplerecursiongeneric2.nim

@@ -0,0 +1,9 @@
+discard """
+  errormsg: "illegal recursion in type 'TPearl'"
+"""
+
+type
+  TPearl[T] = tuple
+    next: TPearl[T]
+
+var x: TPearl[int]

tests/types/tillegaltyperecursion2.nim

@@ -1,5 +1,5 @@
 discard """
-  errormsg: "invalid recursion in type 'Executor'"
+  errormsg: "illegal recursion in type 'Executor'"
   line: 8
 """
 # bug reported by PR #5637