Merge branch 'devel' into araq

.travis.yml

@@ -16,10 +16,12 @@ addons:
     - libcurl4-openssl-dev
     - libsdl1.2-dev
     - libgc-dev
+    - libsfml-dev
 
 before_install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install boehmgc; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install sfml; fi
 
 before_script:
   - set -e

changelog.md

@@ -5,3 +5,39 @@
 - Removed basic2d/basic3d out of the stdlib and into Nimble packages.
   These packages deprecated however, use the ``glm``, ``arraymancer``, ``neo``
   or another package.
+- Arrays of char cannot be converted to ``cstring`` anymore, pointers to
+  arrays of char can! This means ``$`` for arrays can finally exist
+  in ``system.nim`` and do the right thing.
+- ``echo`` now works with strings that contain ``\0`` (the binary zero is not
+  shown) and ``nil`` strings are equal to empty strings.
+- JSON: Deprecated `getBVal`, `getFNum`, and `getNum` in favour to
+  `getBool`, `getFloat`, `getBiggestInt`. Also `getInt` procedure was added.
+- `reExtended` is no longer default for the `re` constructor in the `re`
+  module.
+- The overloading rules changed slightly so that constrained generics are
+  preferred over unconstrained generics. (Bug #6526)
+- It is now possible to forward declare object types so that mutually
+  recursive types can be created across module boundaries. See
+  [package level objects](https://nim-lang.org/docs/manual.html#package-level-objects)
+  for more information.
+- The **unary** ``<`` is now deprecated, for ``.. <`` use ``..<`` for other usages
+  use the ``pred`` proc.
+- We changed how array accesses "from backwards" like ``a[^1]`` or ``a[0..^1]`` are
+  implemented. These are now implemented purely in ``system.nim`` without compiler
+  support. There is a new "heterogenous" slice type ``system.HSlice`` that takes 2
+  generic parameters which can be ``BackwardsIndex`` indices. ``BackwardsIndex`` is
+  produced by ``system.^``.
+  This means if you overload ``[]`` or ``[]=`` you need to ensure they also work
+  with ``system.BackwardsIndex`` (if applicable for the accessors).
+- ``mod`` and bitwise ``and`` do not produce ``range`` subtypes anymore. This
+  turned out to be more harmful than helpful and the language is simpler
+  without this special typing rule.
+- Added ``algorithm.rotateLeft``.
+- ``rationals.toRational`` now uses an algorithm based on continued fractions.
+  This means its results are more precise and it can't run into an infinite loop
+  anymore.
+- Added ``typetraits.$`` as an alias for ``typetraits.name``.
+- ``os.getEnv`` now takes an optional ``default`` parameter that tells ``getEnv``
+  what to return if the environment variable does not exist.
+- Removed PDCurses wrapper from the stdlib and published it as a separate 
+  Nimble package.

compiler/ast.nim

@@ -874,7 +874,8 @@ type
                               # mean that there is no destructor.
                               # see instantiateDestructor in semdestruct.nim
     deepCopy*: PSym           # overriden 'deepCopy' operation
-    assignment*: PSym         # overriden '=' operator
+    assignment*: PSym         # overriden '=' operation
+    sink*: PSym               # overriden '=sink' operation
     methods*: seq[(int,PSym)] # attached methods
     size*: BiggestInt         # the size of the type in bytes
                               # -1 means that the size is unkwown
@@ -1019,16 +1020,11 @@ proc add*(father, son: PNode) =
 
 type Indexable = PNode | PType
 
-template `[]`*(n: Indexable, i: int): Indexable =
+template `[]`*(n: Indexable, i: int): Indexable = n.sons[i]
-  n.sons[i]
+template `[]=`*(n: Indexable, i: int; x: Indexable) = n.sons[i] = x
 
-template `-|`*(b, s: untyped): untyped =
+template `[]`*(n: Indexable, i: BackwardsIndex): Indexable = n[n.len - i.int]
-  (if b >= 0: b else: s.len + b)
+template `[]=`*(n: Indexable, i: BackwardsIndex; x: Indexable) = n[n.len - i.int] = x
-
-# son access operators with support for negative indices
-template `{}`*(n: Indexable, i: int): untyped = n[i -| n]
-template `{}=`*(n: Indexable, i: int, s: Indexable) =
-  n.sons[i -| n] = s
 
 when defined(useNodeIds):
   const nodeIdToDebug* = -1 # 299750 # 300761 #300863 # 300879
@@ -1050,6 +1046,8 @@ proc newNode*(kind: TNodeKind): PNode =
 
 proc newTree*(kind: TNodeKind; children: varargs[PNode]): PNode =
   result = newNode(kind)
+  if children.len > 0:
+    result.info = children[0].info
   result.sons = @children
 
 proc newIntNode*(kind: TNodeKind, intVal: BiggestInt): PNode =
@@ -1293,6 +1291,7 @@ proc assignType*(dest, src: PType) =
   dest.align = src.align
   dest.destructor = src.destructor
   dest.deepCopy = src.deepCopy
+  dest.sink = src.sink
   dest.assignment = src.assignment
   dest.lockLevel = src.lockLevel
   # this fixes 'type TLock = TSysLock':
@@ -1391,6 +1390,14 @@ proc skipTypes*(t: PType, kinds: TTypeKinds): PType =
   result = t
   while result.kind in kinds: result = lastSon(result)
 
+proc skipTypes*(t: PType, kinds: TTypeKinds; maxIters: int): PType =
+  result = t
+  var i = maxIters
+  while result.kind in kinds:
+    result = lastSon(result)
+    dec i
+    if i == 0: return nil
+
 proc skipTypesOrNil*(t: PType, kinds: TTypeKinds): PType =
   ## same as skipTypes but handles 'nil'
   result = t
@@ -1403,7 +1410,7 @@ proc isGCedMem*(t: PType): bool {.inline.} =
            t.kind == tyProc and t.callConv == ccClosure
 
 proc propagateToOwner*(owner, elem: PType) =
-  const HaveTheirOwnEmpty = {tySequence, tySet, tyPtr, tyRef, tyProc}
+  const HaveTheirOwnEmpty = {tySequence, tyOpt, tySet, tyPtr, tyRef, tyProc}
   owner.flags = owner.flags + (elem.flags * {tfHasMeta})
   if tfNotNil in elem.flags:
     if owner.kind in {tyGenericInst, tyGenericBody, tyGenericInvocation}:
@@ -1419,9 +1426,9 @@ proc propagateToOwner*(owner, elem: PType) =
     owner.flags.incl tfHasMeta
 
   if tfHasAsgn in elem.flags:
-    let o2 = elem.skipTypes({tyGenericInst, tyAlias})
+    let o2 = owner.skipTypes({tyGenericInst, tyAlias})
     if o2.kind in {tyTuple, tyObject, tyArray,
-                   tySequence, tySet, tyDistinct}:
+                   tySequence, tyOpt, tySet, tyDistinct}:
       o2.flags.incl tfHasAsgn
       owner.flags.incl tfHasAsgn
 
@@ -1662,3 +1669,10 @@ when false:
     if n.isNil: return true
     for i in 0 ..< n.safeLen:
       if n[i].containsNil: return true
+
+template hasDestructor*(t: PType): bool = tfHasAsgn in t.flags
+template incompleteType*(t: PType): bool =
+  t.sym != nil and {sfForward, sfNoForward} * t.sym.flags == {sfForward}
+
+template typeCompleted*(s: PSym) =
+  incl s.flags, sfNoForward

compiler/ccgexprs.nim

@@ -270,10 +270,10 @@ proc genGenericAsgn(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
            addrLoc(dest), addrLoc(src), rdLoc(dest))
     else:
       linefmt(p, cpsStmts, "#genericShallowAssign((void*)$1, (void*)$2, $3);$n",
-              addrLoc(dest), addrLoc(src), genTypeInfo(p.module, dest.t))
+              addrLoc(dest), addrLoc(src), genTypeInfo(p.module, dest.t, dest.lode.info))
   else:
     linefmt(p, cpsStmts, "#genericAssign((void*)$1, (void*)$2, $3);$n",
-            addrLoc(dest), addrLoc(src), genTypeInfo(p.module, dest.t))
+            addrLoc(dest), addrLoc(src), genTypeInfo(p.module, dest.t, dest.lode.info))
 
 proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
   # This function replaces all other methods for generating
@@ -291,7 +291,8 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
       genRefAssign(p, dest, src, flags)
     else:
       linefmt(p, cpsStmts, "#genericSeqAssign($1, $2, $3);$n",
-              addrLoc(dest), rdLoc(src), genTypeInfo(p.module, dest.t))
+              addrLoc(dest), rdLoc(src),
+              genTypeInfo(p.module, dest.t, dest.lode.info))
   of tyString:
     if (needToCopy notin flags and src.storage != OnStatic) or canMove(src.lode):
       genRefAssign(p, dest, src, flags)
@@ -352,7 +353,8 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     if needsComplexAssignment(dest.t):
       linefmt(p, cpsStmts,     # XXX: is this correct for arrays?
            "#genericAssignOpenArray((void*)$1, (void*)$2, $1Len_0, $3);$n",
-           addrLoc(dest), addrLoc(src), genTypeInfo(p.module, dest.t))
+           addrLoc(dest), addrLoc(src),
+           genTypeInfo(p.module, dest.t, dest.lode.info))
     else:
       useStringh(p.module)
       linefmt(p, cpsStmts,
@@ -393,14 +395,17 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) =
   of tyPtr, tyRef, tyProc, tyTuple, tyObject, tyArray:
     # XXX optimize this
     linefmt(p, cpsStmts, "#genericDeepCopy((void*)$1, (void*)$2, $3);$n",
-            addrLoc(dest), addrLocOrTemp(src), genTypeInfo(p.module, dest.t))
+            addrLoc(dest), addrLocOrTemp(src),
+            genTypeInfo(p.module, dest.t, dest.lode.info))
   of tySequence, tyString:
     linefmt(p, cpsStmts, "#genericSeqDeepCopy($1, $2, $3);$n",
-            addrLoc(dest), rdLoc(src), genTypeInfo(p.module, dest.t))
+            addrLoc(dest), rdLoc(src),
+            genTypeInfo(p.module, dest.t, dest.lode.info))
   of tyOpenArray, tyVarargs:
     linefmt(p, cpsStmts,
          "#genericDeepCopyOpenArray((void*)$1, (void*)$2, $1Len_0, $3);$n",
-         addrLoc(dest), addrLocOrTemp(src), genTypeInfo(p.module, dest.t))
+         addrLoc(dest), addrLocOrTemp(src),
+         genTypeInfo(p.module, dest.t, dest.lode.info))
   of tySet:
     if mapType(ty) == ctArray:
       useStringh(p.module)
@@ -965,6 +970,8 @@ proc genEcho(p: BProc, n: PNode) =
   # this unusal way of implementing it ensures that e.g. ``echo("hallo", 45)``
   # is threadsafe.
   internalAssert n.kind == nkBracket
+  if platform.targetOS == osGenode:
+    # bypass libc and print directly to the Genode LOG session
     var args: Rope = nil
     var a: TLoc
     for i in countup(0, n.len-1):
@@ -973,11 +980,16 @@ proc genEcho(p: BProc, n: PNode) =
       else:
         initLocExpr(p, n.sons[i], a)
         addf(args, ", $1? ($1)->data:\"nil\"", [rdLoc(a)])
-  if platform.targetOS == osGenode:
-    # bypass libc and print directly to the Genode LOG session
     p.module.includeHeader("<base/log.h>")
     linefmt(p, cpsStmts, """Genode::log(""$1);$n""", args)
   else:
+    if n.len == 0:
+      linefmt(p, cpsStmts, "#echoBinSafe(NIM_NIL, $1);$n", n.len.rope)
+    else:
+      var a: TLoc
+      initLocExpr(p, n, a)
+      linefmt(p, cpsStmts, "#echoBinSafe($1, $2);$n", a.rdLoc, n.len.rope)
+    when false:
       p.module.includeHeader("<stdio.h>")
       linefmt(p, cpsStmts, "printf($1$2);$n",
               makeCString(repeat("%s", n.len) & tnl), args)
@@ -1094,7 +1106,8 @@ proc genReset(p: BProc, n: PNode) =
   var a: TLoc
   initLocExpr(p, n.sons[1], a)
   linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
-          addrLoc(a), genTypeInfo(p.module, skipTypes(a.t, {tyVar})))
+          addrLoc(a),
+          genTypeInfo(p.module, skipTypes(a.t, {tyVar}), n.info))
 
 proc rawGenNew(p: BProc, a: TLoc, sizeExpr: Rope) =
   var sizeExpr = sizeExpr
@@ -1108,7 +1121,7 @@ proc rawGenNew(p: BProc, a: TLoc, sizeExpr: Rope) =
     sizeExpr = "sizeof($1)" %
         [getTypeDesc(p.module, bt)]
   let args = [getTypeDesc(p.module, typ),
-              genTypeInfo(p.module, typ),
+              genTypeInfo(p.module, typ, a.lode.info),
               sizeExpr]
   if a.storage == OnHeap and usesNativeGC():
     # use newObjRC1 as an optimization
@@ -1138,7 +1151,7 @@ proc genNew(p: BProc, e: PNode) =
 proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope) =
   let seqtype = skipTypes(dest.t, abstractVarRange)
   let args = [getTypeDesc(p.module, seqtype),
-              genTypeInfo(p.module, seqtype), length]
+              genTypeInfo(p.module, seqtype, dest.lode.info), length]
   var call: TLoc
   initLoc(call, locExpr, dest.lode, OnHeap)
   if dest.storage == OnHeap and usesNativeGC():
@@ -1166,7 +1179,7 @@ proc genNewSeqOfCap(p: BProc; e: PNode; d: var TLoc) =
   putIntoDest(p, d, e, ropecg(p.module,
               "($1)#nimNewSeqOfCap($2, $3)", [
               getTypeDesc(p.module, seqtype),
-              genTypeInfo(p.module, seqtype), a.rdLoc]))
+              genTypeInfo(p.module, seqtype, e.info), a.rdLoc]))
   gcUsage(e)
 
 proc genConstExpr(p: BProc, n: PNode): Rope
@@ -1248,10 +1261,11 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) =
   if d.k == locNone:
     getTemp(p, n.typ, d)
   # generate call to newSeq before adding the elements per hand:
-  var L = int(lengthOrd(n.sons[1].typ))
+  let L = int(lengthOrd(n.sons[1].typ))
-
   genNewSeqAux(p, d, intLiteral(L))
   initLocExpr(p, n.sons[1], a)
+  # bug #5007; do not produce excessive C source code:
+  if L < 10:
     for i in countup(0, L - 1):
       initLoc(elem, locExpr, lodeTyp elemType(skipTypes(n.typ, abstractInst)), OnHeap)
       elem.r = rfmt(nil, "$1->data[$2]", rdLoc(d), intLiteral(i))
@@ -1259,6 +1273,19 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) =
       initLoc(arr, locExpr, lodeTyp elemType(skipTypes(n.sons[1].typ, abstractInst)), a.storage)
       arr.r = rfmt(nil, "$1[$2]", rdLoc(a), intLiteral(i))
       genAssignment(p, elem, arr, {afDestIsNil, needToCopy})
+  else:
+    var i: TLoc
+    getTemp(p, getSysType(tyInt), i)
+    let oldCode = p.s(cpsStmts)
+    linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",  i.r, L.rope)
+    initLoc(elem, locExpr, lodeTyp elemType(skipTypes(n.typ, abstractInst)), OnHeap)
+    elem.r = rfmt(nil, "$1->data[$2]", rdLoc(d), rdLoc(i))
+    elem.storage = OnHeap # we know that sequences are on the heap
+    initLoc(arr, locExpr, lodeTyp elemType(skipTypes(n.sons[1].typ, abstractInst)), a.storage)
+    arr.r = rfmt(nil, "$1[$2]", rdLoc(a), rdLoc(i))
+    genAssignment(p, elem, arr, {afDestIsNil, needToCopy})
+    lineF(p, cpsStmts, "}$n", [])
+
 
 proc genNewFinalize(p: BProc, e: PNode) =
   var
@@ -1269,7 +1296,7 @@ proc genNewFinalize(p: BProc, e: PNode) =
   initLocExpr(p, e.sons[1], a)
   initLocExpr(p, e.sons[2], f)
   initLoc(b, locExpr, a.lode, OnHeap)
-  ti = genTypeInfo(p.module, refType)
+  ti = genTypeInfo(p.module, refType, e.info)
   addf(p.module.s[cfsTypeInit3], "$1->finalizer = (void*)$2;$n", [ti, rdLoc(f)])
   b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", [
       getTypeDesc(p.module, refType),
@@ -1279,10 +1306,10 @@ proc genNewFinalize(p: BProc, e: PNode) =
   genObjectInit(p, cpsStmts, bt, a, false)
   gcUsage(e)
 
-proc genOfHelper(p: BProc; dest: PType; a: Rope): Rope =
+proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo): Rope =
   # unfortunately 'genTypeInfo' sets tfObjHasKids as a side effect, so we
   # have to call it here first:
-  let ti = genTypeInfo(p.module, dest)
+  let ti = genTypeInfo(p.module, dest, info)
   if tfFinal in dest.flags or (objHasKidsValid in p.module.flags and
                                tfObjHasKids notin dest.flags):
     result = "$1.m_type == $2" % [a, ti]
@@ -1295,7 +1322,7 @@ proc genOfHelper(p: BProc; dest: PType; a: Rope): Rope =
   when false:
     # former version:
     result = rfmt(p.module, "#isObj($1.m_type, $2)",
-                  a, genTypeInfo(p.module, dest))
+                  a, genTypeInfo(p.module, dest, info))
 
 proc genOf(p: BProc, x: PNode, typ: PType, d: var TLoc) =
   var a: TLoc
@@ -1317,9 +1344,9 @@ proc genOf(p: BProc, x: PNode, typ: PType, d: var TLoc) =
     globalError(x.info, errGenerated,
       "no 'of' operator available for pure objects")
   if nilCheck != nil:
-    r = rfmt(p.module, "(($1) && ($2))", nilCheck, genOfHelper(p, dest, r))
+    r = rfmt(p.module, "(($1) && ($2))", nilCheck, genOfHelper(p, dest, r, x.info))
   else:
-    r = rfmt(p.module, "($1)", genOfHelper(p, dest, r))
+    r = rfmt(p.module, "($1)", genOfHelper(p, dest, r, x.info))
   putIntoDest(p, d, x, r, a.storage)
 
 proc genOf(p: BProc, n: PNode, d: var TLoc) =
@@ -1342,12 +1369,12 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) =
   of tyEnum, tyOrdinal:
     putIntoDest(p, d, e,
                 ropecg(p.module, "#reprEnum((NI)$1, $2)", [
-                rdLoc(a), genTypeInfo(p.module, t)]), a.storage)
+                rdLoc(a), genTypeInfo(p.module, t, e.info)]), a.storage)
   of tyString:
     putIntoDest(p, d, e, ropecg(p.module, "#reprStr($1)", [rdLoc(a)]), a.storage)
   of tySet:
     putIntoDest(p, d, e, ropecg(p.module, "#reprSet($1, $2)", [
-                addrLoc(a), genTypeInfo(p.module, t)]), a.storage)
+                addrLoc(a), genTypeInfo(p.module, t, e.info)]), a.storage)
   of tyOpenArray, tyVarargs:
     var b: TLoc
     case a.t.kind
@@ -1362,22 +1389,22 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) =
     else: internalError(e.sons[0].info, "genRepr()")
     putIntoDest(p, d, e,
         ropecg(p.module, "#reprOpenArray($1, $2)", [rdLoc(b),
-        genTypeInfo(p.module, elemType(t))]), a.storage)
+        genTypeInfo(p.module, elemType(t), e.info)]), a.storage)
   of tyCString, tyArray, tyRef, tyPtr, tyPointer, tyNil, tySequence:
     putIntoDest(p, d, e,
                 ropecg(p.module, "#reprAny($1, $2)", [
-                rdLoc(a), genTypeInfo(p.module, t)]), a.storage)
+                rdLoc(a), genTypeInfo(p.module, t, e.info)]), a.storage)
   of tyEmpty, tyVoid:
     localError(e.info, "'repr' doesn't support 'void' type")
   else:
     putIntoDest(p, d, e, ropecg(p.module, "#reprAny($1, $2)",
-                              [addrLoc(a), genTypeInfo(p.module, t)]),
+                              [addrLoc(a), genTypeInfo(p.module, t, e.info)]),
                                a.storage)
   gcUsage(e)
 
 proc genGetTypeInfo(p: BProc, e: PNode, d: var TLoc) =
   let t = e.sons[1].typ
-  putIntoDest(p, d, e, genTypeInfo(p.module, t))
+  putIntoDest(p, d, e, genTypeInfo(p.module, t, e.info))
 
 proc genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) =
   var a: TLoc
@@ -1959,10 +1986,10 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) =
         t = skipTypes(t.sons[0], skipPtrs)
     if nilCheck != nil:
       linefmt(p, cpsStmts, "if ($1) #chckObj($2.m_type, $3);$n",
-              nilCheck, r, genTypeInfo(p.module, dest))
+              nilCheck, r, genTypeInfo(p.module, dest, n.info))
     else:
       linefmt(p, cpsStmts, "#chckObj($1.m_type, $2);$n",
-              r, genTypeInfo(p.module, dest))
+              r, genTypeInfo(p.module, dest, n.info))
   if n.sons[0].typ.kind != tyObject:
     putIntoDest(p, d, n,
                 "(($1) ($2))" % [getTypeDesc(p.module, n.typ), rdLoc(a)], a.storage)
@@ -2245,7 +2272,7 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope =
       result = rope"{NIM_NIL, NIM_NIL}"
   of tyObject:
     if not isObjLackingTypeField(t) and not p.module.compileToCpp:
-      result = "{{$1}}" % [genTypeInfo(p.module, t)]
+      result = "{{$1}}" % [genTypeInfo(p.module, t, info)]
     else:
       result = rope"{}"
   of tyArray, tyTuple: result = rope"{}"
@@ -2290,7 +2317,7 @@ proc getNullValueAuxT(p: BProc; orig, t: PType; obj, cons: PNode, result: var Ro
     base = skipTypes(base, skipPtrs)
     getNullValueAuxT(p, orig, base, base.n, cons, result, count)
   elif not isObjLackingTypeField(t) and not p.module.compileToCpp:
-    addf(result, "$1", [genTypeInfo(p.module, orig)])
+    addf(result, "$1", [genTypeInfo(p.module, orig, obj.info)])
     inc count
   getNullValueAux(p, t, obj, cons, result, count)
   # do not emit '{}' as that is not valid C:

compiler/ccgstmts.nim

@@ -20,7 +20,7 @@ proc registerGcRoot(p: BProc, v: PSym) =
       containsGarbageCollectedRef(v.loc.t):
     # we register a specialized marked proc here; this has the advantage
     # that it works out of the box for thread local storage then :-)
-    let prc = genTraverseProcForGlobal(p.module, v)
+    let prc = genTraverseProcForGlobal(p.module, v, v.info)
     appcg(p.module, p.module.initProc.procSec(cpsInit),
       "#nimRegisterGlobalMarker($1);$n", [prc])
 
@@ -141,9 +141,13 @@ template preserveBreakIdx(body: untyped): untyped =
   p.breakIdx = oldBreakIdx
 
 proc genState(p: BProc, n: PNode) =
-  internalAssert n.len == 1 and n.sons[0].kind == nkIntLit
+  internalAssert n.len == 1
+  let n0 = n[0]
+  if n0.kind == nkIntLit:
     let idx = n.sons[0].intVal
     linefmt(p, cpsStmts, "STATE$1: ;$n", idx.rope)
+  elif n0.kind == nkStrLit:
+    linefmt(p, cpsStmts, "$1: ;$n", n0.strVal.rope)
 
 proc genGotoState(p: BProc, n: PNode) =
   # we resist the temptation to translate it into duff's device as it later
@@ -156,8 +160,13 @@ proc genGotoState(p: BProc, n: PNode) =
   lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)])
   p.beforeRetNeeded = true
   lineF(p, cpsStmts, "case -1: goto BeforeRet_;$n", [])
-  for i in 0 .. lastOrd(n.sons[0].typ):
+  var statesCounter = lastOrd(n.sons[0].typ)
-    lineF(p, cpsStmts, "case $1: goto STATE$1;$n", [rope(i)])
+  if n.len >= 2 and n[1].kind == nkIntLit:
+    statesCounter = n[1].intVal
+  let prefix = if n.len == 3 and n[2].kind == nkStrLit: n[2].strVal.rope
+               else: rope"STATE"
+  for i in 0 .. statesCounter:
+    lineF(p, cpsStmts, "case $2: goto $1$2;$n", [prefix, rope(i)])
   lineF(p, cpsStmts, "}$n", [])
 
 proc genBreakState(p: BProc, n: PNode) =
@@ -826,7 +835,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
         if orExpr != nil: add(orExpr, "||")
         appcg(p.module, orExpr,
               "#isObj($1.exp->m_type, $2)",
-              [exc, genTypeInfo(p.module, t.sons[i].sons[j].typ)])
+              [exc, genTypeInfo(p.module, t[i][j].typ, t[i][j].info)])
       lineF(p, cpsStmts, "if ($1) ", [orExpr])
       startBlock(p)
       expr(p, t.sons[i].sons[blen-1], d)
@@ -935,7 +944,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
           "#isObj(#getCurrentException()->Sup.m_type, $1)"
           else: "#isObj(#getCurrentException()->m_type, $1)"
         appcg(p.module, orExpr, isObjFormat,
-              [genTypeInfo(p.module, t.sons[i].sons[j].typ)])
+              [genTypeInfo(p.module, t[i][j].typ, t[i][j].info)])
       if i > 1: line(p, cpsStmts, "else ")
       startBlock(p, "if ($1) {$n", [orExpr])
       linefmt(p, cpsStmts, "$1.status = 0;$n", safePoint)
@@ -1053,7 +1062,7 @@ proc genWatchpoint(p: BProc, n: PNode) =
   let typ = skipTypes(n.sons[1].typ, abstractVarRange)
   lineCg(p, cpsStmts, "#dbgRegisterWatchpoint($1, (NCSTRING)$2, $3);$n",
         [a.addrLoc, makeCString(renderTree(n.sons[1])),
-        genTypeInfo(p.module, typ)])
+        genTypeInfo(p.module, typ, n.info)])
 
 proc genPragma(p: BProc, n: PNode) =
   for i in countup(0, sonsLen(n) - 1):
@@ -1083,7 +1092,7 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
                           field: PSym) =
   var t = skipTypes(objtype, abstractVar)
   assert t.kind == tyObject
-  discard genTypeInfo(p.module, t)
+  discard genTypeInfo(p.module, t, a.lode.info)
   var L = lengthOrd(field.typ)
   if not containsOrIncl(p.module.declaredThings, field.id):
     appcg(p.module, cfsVars, "extern $1",
@@ -1103,19 +1112,46 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   genDiscriminantCheck(p, a, tmp, dotExpr.sons[0].typ, dotExpr.sons[1].sym)
   genAssignment(p, a, tmp, {})
 
+proc patchAsgnStmtListExpr(father, orig, n: PNode) =
+  case n.kind
+  of nkDerefExpr, nkHiddenDeref:
+    let asgn = copyNode(orig)
+    asgn.add orig[0]
+    asgn.add n
+    father.add asgn
+  of nkStmtList, nkStmtListExpr:
+    for x in n:
+      patchAsgnStmtListExpr(father, orig, x)
+  else:
+    father.add n
+
 proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
   if e.sons[0].kind == nkSym and sfGoto in e.sons[0].sym.flags:
     genLineDir(p, e)
     genGotoVar(p, e.sons[1])
   elif not fieldDiscriminantCheckNeeded(p, e):
+    # this fixes bug #6422 but we really need to change the representation of
+    # arrays in the backend...
+    let le = e[0]
+    let ri = e[1]
+    var needsRepair = false
+    var it = ri
+    while it.kind in {nkStmtList, nkStmtListExpr}:
+      it = it.lastSon
+      needsRepair = true
+    if it.kind in {nkDerefExpr, nkHiddenDeref} and needsRepair:
+      var patchedTree = newNodeI(nkStmtList, e.info)
+      patchAsgnStmtListExpr(patchedTree, e, ri)
+      genStmts(p, patchedTree)
+      return
+
     var a: TLoc
-    if e[0].kind in {nkDerefExpr, nkHiddenDeref}:
+    if le.kind in {nkDerefExpr, nkHiddenDeref}:
-      genDeref(p, e[0], a, enforceDeref=true)
+      genDeref(p, le, a, enforceDeref=true)
     else:
-      initLocExpr(p, e.sons[0], a)
+      initLocExpr(p, le, a)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)
     assert(a.t != nil)
-    let ri = e.sons[1]
     genLineDir(p, ri)
     loadInto(p, e.sons[0], ri, a)
   else:

compiler/ccgtrav.nim

@@ -66,7 +66,7 @@ proc genTraverseProc(c: var TTraversalClosure, accessor: Rope, typ: PType) =
 
   var p = c.p
   case typ.kind
-  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct:
+  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred:
     genTraverseProc(c, accessor, lastSon(typ))
   of tyArray:
     let arraySize = lengthOrd(typ.sons[0])
@@ -151,8 +151,8 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash;
   m.s[cfsProcHeaders].addf("$1;$n", [header])
   m.s[cfsProcs].add(generatedProc)
 
-proc genTraverseProcForGlobal(m: BModule, s: PSym): Rope =
+proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope =
-  discard genTypeInfo(m, s.loc.t)
+  discard genTypeInfo(m, s.loc.t, info)
 
   var c: TTraversalClosure
   var p = newProc(nil, m)

compiler/ccgtypes.nim

@@ -92,7 +92,7 @@ proc mangleParamName(m: BModule; s: PSym): Rope =
 
 proc mangleLocalName(p: BProc; s: PSym): Rope =
   assert s.kind in skLocalVars+{skTemp}
-  assert sfGlobal notin s.flags
+  #assert sfGlobal notin s.flags
   result = s.loc.r
   if result == nil:
     var key = s.name.s.mangle
@@ -119,7 +119,7 @@ proc scopeMangledParam(p: BProc; param: PSym) =
 
 const
   irrelevantForBackend = {tyGenericBody, tyGenericInst, tyGenericInvocation,
-                          tyDistinct, tyRange, tyStatic, tyAlias}
+                          tyDistinct, tyRange, tyStatic, tyAlias, tyInferred}
 
 proc typeName(typ: PType): Rope =
   let typ = typ.skipTypes(irrelevantForBackend)
@@ -278,7 +278,10 @@ proc ccgIntroducedPtr(s: PSym): bool =
   elif tfByCopy in pt.flags: return false
   case pt.kind
   of tyObject:
-    if (optByRef in s.options) or (getSize(pt) > platform.floatSize * 2):
+    if s.typ.sym != nil and sfForward in s.typ.sym.flags:
+      # forwarded objects are *always* passed by pointers for consistency!
+      result = true
+    elif (optByRef in s.options) or (getSize(pt) > platform.floatSize * 3):
       result = true           # requested anyway
     elif (tfFinal in pt.flags) and (pt.sons[0] == nil):
       result = false          # no need, because no subtyping possible
@@ -286,7 +289,7 @@ proc ccgIntroducedPtr(s: PSym): bool =
       result = true           # ordinary objects are always passed by reference,
                               # otherwise casting doesn't work
   of tyTuple:
-    result = (getSize(pt) > platform.floatSize*2) or (optByRef in s.options)
+    result = (getSize(pt) > platform.floatSize*3) or (optByRef in s.options)
   else: result = false
 
 proc fillResult(param: PNode) =
@@ -806,7 +809,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       var chunkStart = 0
       while i < cppName.data.len:
         if cppName.data[i] == '\'':
-          var chunkEnd = <i
+          var chunkEnd = i-1
           var idx, stars: int
           if scanCppGenericSlot(cppName.data, i, idx, stars):
             result.add cppName.data.substr(chunkStart, chunkEnd)
@@ -854,6 +857,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
              [structOrUnion(t), result])
         assert m.forwTypeCache[sig] == result
       m.typeCache[sig] = result # always call for sideeffects:
+      if not incompleteType(t):
         let recdesc = if t.kind != tyTuple: getRecordDesc(m, t, result, check)
                       else: getTupleDesc(m, t, result, check)
         if not isImportedType(t):
@@ -935,12 +939,13 @@ proc genProcHeader(m: BModule, prc: PSym): Rope =
 
 # ------------------ type info generation -------------------------------------
 
-proc genTypeInfo(m: BModule, t: PType): Rope
+proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope
 proc getNimNode(m: BModule): Rope =
   result = "$1[$2]" % [m.typeNodesName, rope(m.typeNodes)]
   inc(m.typeNodes)
 
-proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) =
+proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
+                        name, base: Rope; info: TLineInfo) =
   var nimtypeKind: int
   #allocMemTI(m, typ, name)
   if isObjLackingTypeField(typ):
@@ -970,15 +975,19 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) =
          [name])
   addf(m.s[cfsVars], "TNimType $1;$n", [name])
 
-proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope) =
+proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope;
+                    info: TLineInfo) =
   var base: Rope
   if sonsLen(typ) > 0 and typ.lastSon != nil:
     var x = typ.lastSon
     if typ.kind == tyObject: x = x.skipTypes(skipPtrs)
-    base = genTypeInfo(m, x)
+    if typ.kind == tyPtr and x.kind == tyObject and incompleteType(x):
+      base = rope("0")
+    else:
+      base = genTypeInfo(m, x, info)
   else:
     base = rope("0")
-  genTypeInfoAuxBase(m, typ, origType, name, base)
+  genTypeInfoAuxBase(m, typ, origType, name, base, info)
 
 proc discriminatorTableName(m: BModule, objtype: PType, d: PSym): Rope =
   # bugfix: we need to search the type that contains the discriminator:
@@ -994,19 +1003,20 @@ proc discriminatorTableDecl(m: BModule, objtype: PType, d: PSym): Rope =
   var tmp = discriminatorTableName(m, objtype, d)
   result = "TNimNode* $1[$2];$n" % [tmp, rope(lengthOrd(d.typ)+1)]
 
-proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope) =
+proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
+                     info: TLineInfo) =
   case n.kind
   of nkRecList:
     var L = sonsLen(n)
     if L == 1:
-      genObjectFields(m, typ, origType, n.sons[0], expr)
+      genObjectFields(m, typ, origType, n.sons[0], expr, info)
     elif L > 0:
       var tmp = getTempName(m)
       addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(L)])
       for i in countup(0, L-1):
         var tmp2 = getNimNode(m)
         addf(m.s[cfsTypeInit3], "$1[$2] = &$3;$n", [tmp, rope(i), tmp2])
-        genObjectFields(m, typ, origType, n.sons[i], tmp2)
+        genObjectFields(m, typ, origType, n.sons[i], tmp2, info)
       addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2; $1.sons = &$3[0];$n",
            [expr, rope(L), tmp])
     else:
@@ -1024,14 +1034,14 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope) =
         "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" &
         "$1.name = $5;$n" & "$1.sons = &$6[0];$n" &
         "$1.len = $7;$n", [expr, getTypeDesc(m, origType), field.loc.r,
-                           genTypeInfo(m, field.typ),
+                           genTypeInfo(m, field.typ, info),
                            makeCString(field.name.s),
                            tmp, rope(L)])
     addf(m.s[cfsData], "TNimNode* $1[$2];$n", [tmp, rope(L+1)])
     for i in countup(1, sonsLen(n)-1):
       var b = n.sons[i]           # branch
       var tmp2 = getNimNode(m)
-      genObjectFields(m, typ, origType, lastSon(b), tmp2)
+      genObjectFields(m, typ, origType, lastSon(b), tmp2, info)
       case b.kind
       of nkOfBranch:
         if sonsLen(b) < 2:
@@ -1059,15 +1069,20 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope) =
       addf(m.s[cfsTypeInit3], "$1.kind = 1;$n" &
           "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" &
           "$1.name = $5;$n", [expr, getTypeDesc(m, origType),
-          field.loc.r, genTypeInfo(m, field.typ), makeCString(field.name.s)])
+          field.loc.r, genTypeInfo(m, field.typ, info), makeCString(field.name.s)])
   else: internalError(n.info, "genObjectFields")
 
-proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope) =
+proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) =
-  if typ.kind == tyObject: genTypeInfoAux(m, typ, origType, name)
+  if typ.kind == tyObject:
-  else: genTypeInfoAuxBase(m, typ, origType, name, rope("0"))
+    if incompleteType(typ):
+      localError(info, "request for RTTI generation for incomplete object: " &
+                        typeToString(typ))
+    genTypeInfoAux(m, typ, origType, name, info)
+  else:
+    genTypeInfoAuxBase(m, typ, origType, name, rope("0"), info)
   var tmp = getNimNode(m)
   if not isImportedType(typ):
-    genObjectFields(m, typ, origType, typ.n, tmp)
+    genObjectFields(m, typ, origType, typ.n, tmp, info)
   addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, tmp])
   var t = typ.sons[0]
   while t != nil:
@@ -1075,8 +1090,8 @@ proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope) =
     t.flags.incl tfObjHasKids
     t = t.sons[0]
 
-proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope) =
+proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) =
-  genTypeInfoAuxBase(m, typ, typ, name, rope("0"))
+  genTypeInfoAuxBase(m, typ, typ, name, rope("0"), info)
   var expr = getNimNode(m)
   var length = sonsLen(typ)
   if length > 0:
@@ -1090,7 +1105,7 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope) =
           "$1.offset = offsetof($2, Field$3);$n" &
           "$1.typ = $4;$n" &
           "$1.name = \"Field$3\";$n",
-           [tmp2, getTypeDesc(m, origType), rope(i), genTypeInfo(m, a)])
+           [tmp2, getTypeDesc(m, origType), rope(i), genTypeInfo(m, a, info)])
     addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2; $1.sons = &$3[0];$n",
          [expr, rope(length), tmp])
   else:
@@ -1098,12 +1113,12 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope) =
          [expr, rope(length)])
   addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, expr])
 
-proc genEnumInfo(m: BModule, typ: PType, name: Rope) =
+proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   # Type information for enumerations is quite heavy, so we do some
   # optimizations here: The ``typ`` field is never set, as it is redundant
   # anyway. We generate a cstring array and a loop over it. Exceptional
   # positions will be reset after the loop.
-  genTypeInfoAux(m, typ, typ, name)
+  genTypeInfoAux(m, typ, typ, name, info)
   var nodePtrs = getTempName(m)
   var length = sonsLen(typ.n)
   addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n",
@@ -1141,15 +1156,15 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope) =
     # 1 << 2 is {ntfEnumHole}
     addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [name])
 
-proc genSetInfo(m: BModule, typ: PType, name: Rope) =
+proc genSetInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   assert(typ.sons[0] != nil)
-  genTypeInfoAux(m, typ, typ, name)
+  genTypeInfoAux(m, typ, typ, name, info)
   var tmp = getNimNode(m)
   addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 0;$n" & "$3.node = &$1;$n",
        [tmp, rope(firstOrd(typ)), name])
 
-proc genArrayInfo(m: BModule, typ: PType, name: Rope) =
+proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
-  genTypeInfoAuxBase(m, typ, typ, name, genTypeInfo(m, typ.sons[1]))
+  genTypeInfoAuxBase(m, typ, typ, name, genTypeInfo(m, typ.sons[1], info), info)
 
 proc fakeClosureType(owner: PSym): PType =
   # we generate the same RTTI as for a tuple[pointer, ref tuple[]]
@@ -1171,11 +1186,11 @@ proc genDeepCopyProc(m: BModule; s: PSym; result: Rope) =
   addf(m.s[cfsTypeInit3], "$1.deepcopy =(void* (N_RAW_NIMCALL*)(void*))$2;$n",
      [result, s.loc.r])
 
-proc genTypeInfo(m: BModule, t: PType): Rope =
+proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
   let origType = t
   var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses)
   if t.kind == tyOpt:
-    return genTypeInfo(m, optLowering(t))
+    return genTypeInfo(m, optLowering(t), info)
 
   let sig = hashType(origType)
   result = m.typeInfoMarker.getOrDefault(sig)
@@ -1197,7 +1212,7 @@ proc genTypeInfo(m: BModule, t: PType): Rope =
   let owner = t.skipTypes(typedescPtrs).owner.getModule
   if owner != m.module:
     # make sure the type info is created in the owner module
-    discard genTypeInfo(m.g.modules[owner.position], origType)
+    discard genTypeInfo(m.g.modules[owner.position], origType, info)
     # reference the type info as extern here
     discard cgsym(m, "TNimType")
     discard cgsym(m, "TNimNode")
@@ -1208,35 +1223,35 @@ proc genTypeInfo(m: BModule, t: PType): Rope =
   case t.kind
   of tyEmpty, tyVoid: result = rope"0"
   of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar:
-    genTypeInfoAuxBase(m, t, t, result, rope"0")
+    genTypeInfoAuxBase(m, t, t, result, rope"0", info)
   of tyStatic:
-    if t.n != nil: result = genTypeInfo(m, lastSon t)
+    if t.n != nil: result = genTypeInfo(m, lastSon t, info)
     else: internalError("genTypeInfo(" & $t.kind & ')')
   of tyUserTypeClasses:
     internalAssert t.isResolvedUserTypeClass
-    return genTypeInfo(m, t.lastSon)
+    return genTypeInfo(m, t.lastSon, info)
   of tyProc:
     if t.callConv != ccClosure:
-      genTypeInfoAuxBase(m, t, t, result, rope"0")
+      genTypeInfoAuxBase(m, t, t, result, rope"0", info)
     else:
       let x = fakeClosureType(t.owner)
-      genTupleInfo(m, x, x, result)
+      genTupleInfo(m, x, x, result, info)
   of tySequence, tyRef, tyOptAsRef:
-    genTypeInfoAux(m, t, t, result)
+    genTypeInfoAux(m, t, t, result, info)
     if gSelectedGC >= gcMarkAndSweep:
       let markerProc = genTraverseProc(m, origType, sig, tiNew)
       addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc])
-  of tyPtr, tyRange: genTypeInfoAux(m, t, t, result)
+  of tyPtr, tyRange: genTypeInfoAux(m, t, t, result, info)
-  of tyArray: genArrayInfo(m, t, result)
+  of tyArray: genArrayInfo(m, t, result, info)
-  of tySet: genSetInfo(m, t, result)
+  of tySet: genSetInfo(m, t, result, info)
-  of tyEnum: genEnumInfo(m, t, result)
+  of tyEnum: genEnumInfo(m, t, result, info)
-  of tyObject: genObjectInfo(m, t, origType, result)
+  of tyObject: genObjectInfo(m, t, origType, result, info)
   of tyTuple:
     # if t.n != nil: genObjectInfo(m, t, result)
     # else:
     # BUGFIX: use consistently RTTI without proper field names; otherwise
     # results are not deterministic!
-    genTupleInfo(m, t, origType, result)
+    genTupleInfo(m, t, origType, result, info)
   else: internalError("genTypeInfo(" & $t.kind & ')')
   if t.deepCopy != nil:
     genDeepCopyProc(m, t.deepCopy, result)

compiler/cgen.nim

@@ -271,11 +271,11 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: TLoc,
       while (s.kind == tyObject) and (s.sons[0] != nil):
         add(r, ".Sup")
         s = skipTypes(s.sons[0], skipPtrs)
-    linefmt(p, section, "$1.m_type = $2;$n", r, genTypeInfo(p.module, t))
+    linefmt(p, section, "$1.m_type = $2;$n", r, genTypeInfo(p.module, t, a.lode.info))
   of frEmbedded:
     # worst case for performance:
     var r = if takeAddr: addrLoc(a) else: rdLoc(a)
-    linefmt(p, section, "#objectInit($1, $2);$n", r, genTypeInfo(p.module, t))
+    linefmt(p, section, "#objectInit($1, $2);$n", r, genTypeInfo(p.module, t, a.lode.info))
 
 type
   TAssignmentFlag = enum
@@ -306,7 +306,7 @@ proc resetLoc(p: BProc, loc: var TLoc) =
       linefmt(p, cpsStmts, "#chckNil((void*)$1);$n", addrLoc(loc))
     if loc.storage != OnStack:
       linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
-              addrLoc(loc), genTypeInfo(p.module, loc.t))
+              addrLoc(loc), genTypeInfo(p.module, loc.t, loc.lode.info))
       # XXX: generated reset procs should not touch the m_type
       # field, so disabling this should be safe:
       genObjectInit(p, cpsStmts, loc.t, loc, true)
@@ -381,7 +381,7 @@ proc localDebugInfo(p: BProc, s: PSym) =
   lineF(p, cpsInit,
        "FR_.s[$1].address = (void*)$3; FR_.s[$1].typ = $4; FR_.s[$1].name = $2;$n",
        [p.maxFrameLen.rope, makeCString(normalize(s.name.s)), a,
-        genTypeInfo(p.module, s.loc.t)])
+        genTypeInfo(p.module, s.loc.t, s.info)])
   inc(p.maxFrameLen)
   inc p.blocks[p.blocks.len-1].frameLen
 
@@ -451,7 +451,7 @@ proc assignGlobalVar(p: BProc, n: PNode) =
     appcg(p.module, p.module.s[cfsDebugInit],
           "#dbgRegisterGlobal($1, &$2, $3);$n",
          [makeCString(normalize(s.owner.name.s & '.' & s.name.s)),
-          s.loc.r, genTypeInfo(p.module, s.typ)])
+          s.loc.r, genTypeInfo(p.module, s.typ, n.info)])
 
 proc assignParam(p: BProc, s: PSym) =
   assert(s.loc.r != nil)
@@ -663,6 +663,13 @@ proc closureSetup(p: BProc, prc: PSym) =
   linefmt(p, cpsStmts, "$1 = ($2) ClE_0;$n",
           rdLoc(env.loc), getTypeDesc(p.module, env.typ))
 
+proc containsResult(n: PNode): bool =
+  if n.kind == nkSym and n.sym.kind == skResult:
+    result = true
+  else:
+    for i in 0..<n.safeLen:
+      if containsResult(n[i]): return true
+
 proc easyResultAsgn(n: PNode): PNode =
   const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt} +
                     declarativeDefs
@@ -672,7 +679,7 @@ proc easyResultAsgn(n: PNode): PNode =
     while i < n.len and n[i].kind in harmless: inc i
     if i < n.len: result = easyResultAsgn(n[i])
   of nkAsgn, nkFastAsgn:
-    if n[0].kind == nkSym and skResult == n[0].sym.kind:
+    if n[0].kind == nkSym and n[0].sym.kind == skResult and not containsResult(n[1]):
       incl n.flags, nfPreventCg
       return n[1]
   of nkReturnStmt:
@@ -1006,10 +1013,9 @@ proc genMainProc(m: BModule) =
     ComponentConstruct =
       "void Libc::Component::construct(Libc::Env &env) {$N" &
       "\tgenodeEnv = &env;$N" &
-      "\tLibc::with_libc([&] () {$n\t" &
+      "\tLibc::with_libc([&] () {$N\t" &
       MainProcs &
       "\t});$N" &
-      "\tenv.parent().exit(0);$N" &
       "}$N$N"
 
   var nimMain, otherMain: FormatStr

compiler/cgendata.nim

@@ -154,7 +154,7 @@ proc includeHeader*(this: BModule; header: string) =
 
 proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # section in the current block
-  result = p.blocks[^1].sections[s]
+  result = p.blocks[p.blocks.len-1].sections[s]
 
 proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # top level proc sections

compiler/commands.nim

@@ -81,15 +81,17 @@ proc writeVersionInfo(pass: TCmdLinePass) =
   if pass == passCmd1:
     msgWriteln(`%`(HelpMessage, [VersionAsString,
                                  platform.OS[platform.hostOS].name,
-                                 CPU[platform.hostCPU].name]))
+                                 CPU[platform.hostCPU].name]),
+               {msgStdout})
 
     const gitHash = gorge("git log -n 1 --format=%H").strip
     when gitHash.len == 40:
-      msgWriteln("git hash: " & gitHash)
+      msgWriteln("git hash: " & gitHash, {msgStdout})
 
     msgWriteln("active boot switches:" & usedRelease & usedAvoidTimeMachine &
       usedTinyC & usedGnuReadline & usedNativeStacktrace & usedNoCaas &
-      usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC)
+      usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC,
+               {msgStdout})
     msgQuit(0)
 
 var
@@ -249,6 +251,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
     result = gOptions * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck}
   of "infchecks": result = contains(gOptions, optInfCheck)
   of "nanchecks": result = contains(gOptions, optNaNCheck)
+  of "nilchecks": result = contains(gOptions, optNilCheck)
   of "objchecks": result = contains(gOptions, optObjCheck)
   of "fieldchecks": result = contains(gOptions, optFieldCheck)
   of "rangechecks": result = contains(gOptions, optRangeCheck)
@@ -340,7 +343,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     # keep the old name for compat
     if pass in {passCmd2, passPP} and not options.gNoNimblePath:
       expectArg(switch, arg, pass, info)
-      let path = processPath(arg, info, notRelativeToProj=true)
+      var path = processPath(arg, info, notRelativeToProj=true)
+      let nimbleDir = getEnv("NIMBLE_DIR")
+      if nimbleDir.len > 0 and pass == passPP: path = nimbleDir / "pkgs"
       nimblePath(path, info)
   of "nonimblepath", "nobabelpath":
     expectNoArg(switch, arg, pass, info)
@@ -471,6 +476,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitch({optNaNCheck, optInfCheck}, arg, pass, info)
   of "infchecks": processOnOffSwitch({optInfCheck}, arg, pass, info)
   of "nanchecks": processOnOffSwitch({optNaNCheck}, arg, pass, info)
+  of "nilchecks": processOnOffSwitch({optNilCheck}, arg, pass, info)
   of "objchecks": processOnOffSwitch({optObjCheck}, arg, pass, info)
   of "fieldchecks": processOnOffSwitch({optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch({optRangeCheck}, arg, pass, info)
@@ -662,6 +668,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectArg(switch, arg, pass, info)
     if config != nil:
       config.cppDefine(arg)
+  of "newruntime":
+    expectNoArg(switch, arg, pass, info)
+    newDestructors = true
+    defineSymbol("nimNewRuntime")
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(switch, arg)
     else: invalidCmdLineOption(pass, switch, info)

compiler/condsyms.nim

@@ -108,3 +108,4 @@ proc initDefines*() =
   defineSymbol("nimHasCppDefine")
   defineSymbol("nimGenericInOutFlags")
   when false: defineSymbol("nimHasOpt")
+  defineSymbol("nimNoArrayToCstringConversion")

compiler/depends.nim

@@ -10,7 +10,7 @@
 # This module implements a dependency file generator.
 
 import
-  os, options, ast, astalgo, msgs, ropes, idents, passes, importer
+  os, options, ast, astalgo, msgs, ropes, idents, passes, modulepaths
 
 from modulegraphs import ModuleGraph
 

compiler/destroyer.nim

@@ -0,0 +1,318 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Injects destructor calls into Nim code as well as
+## an optimizer that optimizes copies to moves. This is implemented as an
+## AST to AST transformation so that every backend benefits from it.
+
+## Rules for destructor injections:
+##
+## foo(bar(X(), Y()))
+## X and Y get destroyed after bar completes:
+##
+## foo( (tmpX = X(); tmpY = Y(); tmpBar = bar(tmpX, tmpY);
+##       destroy(tmpX); destroy(tmpY);
+##       tmpBar))
+## destroy(tmpBar)
+##
+## var x = f()
+## body
+##
+## is the same as:
+##
+##  var x;
+##  try:
+##    move(x, f())
+##  finally:
+##    destroy(x)
+##
+## But this really just an optimization that tries to avoid to
+## introduce too many temporaries, the 'destroy' is caused by
+## the 'f()' call. No! That is not true for 'result = f()'!
+##
+## x = y where y is read only once
+## is the same as:  move(x, y)
+##
+## Actually the more general rule is: The *last* read of ``y``
+## can become a move if ``y`` is the result of a construction.
+##
+## We also need to keep in mind here that the number of reads is
+## control flow dependent:
+## let x = foo()
+## while true:
+##   y = x  # only one read, but the 2nd iteration will fail!
+## This also affects recursions! Only usages that do not cross
+## a loop boundary (scope) and are not used in function calls
+## are safe.
+##
+##
+## x = f() is the same as:  move(x, f())
+##
+## x = y
+## is the same as:  copy(x, y)
+##
+## Reassignment works under this scheme:
+## var x = f()
+## x = y
+##
+## is the same as:
+##
+##  var x;
+##  try:
+##    move(x, f())
+##    copy(x, y)
+##  finally:
+##    destroy(x)
+##
+##  result = f()  must not destroy 'result'!
+##
+## The produced temporaries clutter up the code and might lead to
+## inefficiencies. A better strategy is to collect all the temporaries
+## in a single object that we put into a single try-finally that
+## surrounds the proc body. This means the code stays quite efficient
+## when compiled to C. In fact, we do the same for variables, so
+## destructors are called when the proc returns, not at scope exit!
+## This makes certains idioms easier to support. (Taking the slice
+## of a temporary object.)
+##
+## foo(bar(X(), Y()))
+## X and Y get destroyed after bar completes:
+##
+## var tmp: object
+## foo( (move tmp.x, X(); move tmp.y, Y(); tmp.bar = bar(tmpX, tmpY);
+##       tmp.bar))
+## destroy(tmp.bar)
+## destroy(tmp.x); destroy(tmp.y)
+
+
+import
+  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
+  strutils, options, dfa, lowerings, rodread
+
+const
+  InterestingSyms = {skVar, skResult, skLet}
+
+type
+  Con = object
+    owner: PSym
+    g: ControlFlowGraph
+    jumpTargets: IntSet
+    tmpObj: PType
+    tmp: PSym
+    destroys, topLevelVars: PNode
+
+proc isHarmlessVar*(s: PSym; c: Con): bool =
+  # 's' is harmless if it used only once and its
+  # definition/usage are not split by any labels:
+  #
+  # let s = foo()
+  # while true:
+  #   a[i] = s
+  #
+  # produces:
+  #
+  # def s
+  # L1:
+  #   use s
+  # goto L1
+  #
+  # let s = foo()
+  # if cond:
+  #   a[i] = s
+  # else:
+  #   a[j] = s
+  #
+  # produces:
+  #
+  # def s
+  # fork L2
+  # use s
+  # goto L3
+  # L2:
+  # use s
+  # L3
+  #
+  # So this analysis is for now overly conservative, but correct.
+  var defsite = -1
+  var usages = 0
+  for i in 0..<c.g.len:
+    case c.g[i].kind
+    of def:
+      if c.g[i].sym == s:
+        if defsite < 0: defsite = i
+        else: return false
+    of use:
+      if c.g[i].sym == s:
+        if defsite < 0: return false
+        for j in defsite .. i:
+          # not within the same basic block?
+          if j in c.jumpTargets: return false
+        # if we want to die after the first 'use':
+        if usages > 1: return false
+        inc usages
+    of useWithinCall:
+      if c.g[i].sym == s: return false
+    of goto, fork:
+      discard "we do not perform an abstract interpretation yet"
+
+template interestingSym(s: PSym): bool =
+  s.owner == c.owner and s.kind in InterestingSyms and hasDestructor(s.typ)
+
+proc patchHead(n: PNode) =
+  if n.kind in nkCallKinds and n[0].kind == nkSym and n.len > 1:
+    let s = n[0].sym
+    if sfFromGeneric in s.flags and s.name.s[0] == '=' and
+        s.name.s in ["=sink", "=", "=destroy"]:
+      excl(s.flags, sfFromGeneric)
+      patchHead(s.getBody)
+      let t = n[1].typ.skipTypes({tyVar, tyGenericInst, tyAlias, tyInferred})
+      template patch(op, field) =
+        if s.name.s == op and field != nil and field != s:
+          n.sons[0].sym = field
+      patch "=sink", t.sink
+      patch "=", t.assignment
+      patch "=destroy", t.destructor
+  for x in n:
+    patchHead(x)
+
+proc genSink(t: PType; dest: PNode): PNode =
+  let t = t.skipTypes({tyGenericInst, tyAlias})
+  let op = if t.sink != nil: t.sink else: t.assignment
+  assert op != nil
+  patchHead op.ast[bodyPos]
+  result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest))
+
+proc genCopy(t: PType; dest: PNode): PNode =
+  let t = t.skipTypes({tyGenericInst, tyAlias})
+  assert t.assignment != nil
+  patchHead t.assignment.ast[bodyPos]
+  result = newTree(nkCall, newSymNode(t.assignment), newTree(nkHiddenAddr, dest))
+
+proc genDestroy(t: PType; dest: PNode): PNode =
+  let t = t.skipTypes({tyGenericInst, tyAlias})
+  assert t.destructor != nil
+  patchHead t.destructor.ast[bodyPos]
+  result = newTree(nkCall, newSymNode(t.destructor), newTree(nkHiddenAddr, dest))
+
+proc addTopVar(c: var Con; v: PNode) =
+  c.topLevelVars.add newTree(nkIdentDefs, v, emptyNode, emptyNode)
+
+proc p(n: PNode; c: var Con): PNode
+
+template recurse(n, dest) =
+  for i in 0..<n.len:
+    dest.add p(n[i], c)
+
+proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
+  if ri.kind in nkCallKinds:
+    result = genSink(ri.typ, dest)
+    # watch out and no not transform 'ri' twice if it's a call:
+    let ri2 = copyNode(ri)
+    recurse(ri, ri2)
+    result.add ri2
+  elif ri.kind == nkSym and isHarmlessVar(ri.sym, c):
+    result = genSink(ri.typ, dest)
+    result.add p(ri, c)
+  else:
+    result = genCopy(ri.typ, dest)
+    result.add p(ri, c)
+
+proc p(n: PNode; c: var Con): PNode =
+  case n.kind
+  of nkVarSection, nkLetSection:
+    discard "transform; var x = y to  var x; x op y  where op is a move or copy"
+    result = newNodeI(nkStmtList, n.info)
+
+    for i in 0..<n.len:
+      let it = n[i]
+      let L = it.len-1
+      let ri = it[L]
+      if it.kind == nkVarTuple and hasDestructor(ri.typ):
+        let x = lowerTupleUnpacking(it, c.owner)
+        result.add p(x, c)
+      elif it.kind == nkIdentDefs and hasDestructor(it[0].typ):
+        for j in 0..L-2:
+          let v = it[j]
+          doAssert v.kind == nkSym
+          # move the variable declaration to the top of the frame:
+          c.addTopVar v
+          # make sure it's destroyed at the end of the proc:
+          c.destroys.add genDestroy(v.typ, v)
+          if ri.kind != nkEmpty:
+            let r = moveOrCopy(v, ri, c)
+            result.add r
+      else:
+        # keep it, but transform 'ri':
+        var varSection = copyNode(n)
+        var itCopy = copyNode(it)
+        for j in 0..L-1:
+          itCopy.add it[j]
+        itCopy.add p(ri, c)
+        varSection.add itCopy
+        result.add varSection
+  of nkCallKinds:
+    if n.typ != nil and hasDestructor(n.typ):
+      discard "produce temp creation"
+      result = newNodeIT(nkStmtListExpr, n.info, n.typ)
+      let f = newSym(skField, getIdent(":d" & $c.tmpObj.n.len), c.owner, n.info)
+      f.typ = n.typ
+      rawAddField c.tmpObj, f
+      var m = genSink(n.typ, rawDirectAccess(c.tmp, f))
+      var call = copyNode(n)
+      recurse(n, call)
+      m.add call
+      result.add m
+      result.add rawDirectAccess(c.tmp, f)
+      c.destroys.add genDestroy(n.typ, rawDirectAccess(c.tmp, f))
+    else:
+      result = copyNode(n)
+      recurse(n, result)
+  of nkAsgn, nkFastAsgn:
+    if hasDestructor(n[0].typ):
+      result = moveOrCopy(n[0], n[1], c)
+    else:
+      result = copyNode(n)
+      recurse(n, result)
+  of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef,
+      nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
+    result = n
+  else:
+    result = copyNode(n)
+    recurse(n, result)
+
+proc injectDestructorCalls*(owner: PSym; n: PNode): PNode =
+  var c: Con
+  c.owner = owner
+  c.tmp = newSym(skTemp, getIdent":d", owner, n.info)
+  c.tmpObj = createObj(owner, n.info)
+  c.tmp.typ = c.tmpObj
+  c.destroys = newNodeI(nkStmtList, n.info)
+  c.topLevelVars = newNodeI(nkVarSection, n.info)
+  let cfg = constructCfg(owner, n)
+  shallowCopy(c.g, cfg)
+  c.jumpTargets = initIntSet()
+  for i in 0..<c.g.len:
+    if c.g[i].kind in {goto, fork}:
+      c.jumpTargets.incl(i+c.g[i].dest)
+  let body = p(n, c)
+  if c.tmp.typ.n.len > 0:
+    c.addTopVar(newSymNode c.tmp)
+  result = newNodeI(nkStmtList, n.info)
+  if c.topLevelVars.len > 0:
+    result.add c.topLevelVars
+  if c.destroys.len > 0:
+    result.add newTryFinally(body, c.destroys)
+  else:
+    result.add body
+
+  when defined(nimDebugDestroys):
+    if owner.name.s == "createSeq":
+      echo "------------------------------------"
+      echo owner.name.s, " transformed to: "
+      echo result

compiler/dfa.nim

@@ -0,0 +1,439 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Data flow analysis for Nim. For now the task is to prove that every
+## usage of a local variable 'v' is covered by an initialization to 'v'
+## first.
+## We transform the AST into a linear list of instructions first to
+## make this easier to handle: There are only 2 different branching
+## instructions: 'goto X' is an unconditional goto, 'fork X'
+## is a conditional goto (either the next instruction or 'X' can be
+## taken). Exhaustive case statements are translated
+## so that the last branch is transformed into an 'else' branch.
+## ``return`` and ``break`` are all covered by 'goto'.
+## The case to detect is ``use v`` that is not dominated by
+## a ``def v``.
+## The data structures and algorithms used here are inspired by
+## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
+## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
+
+import ast, astalgo, types, intsets, tables, msgs
+
+type
+  InstrKind* = enum
+    goto, fork, def, use,
+    useWithinCall # this strange special case is used to get more
+                  # move optimizations out of regular code
+                  # XXX This is still overly pessimistic in
+                  # call(let x = foo; bar(x))
+  Instr* = object
+    n*: PNode
+    case kind*: InstrKind
+    of def, use, useWithinCall: sym*: PSym
+    of goto, fork: dest*: int
+
+  ControlFlowGraph* = seq[Instr]
+
+  TPosition = distinct int
+  TBlock = object
+    label: PSym
+    fixups: seq[TPosition]
+
+  ValueKind = enum
+    undef, value, valueOrUndef
+
+  Con = object
+    code: ControlFlowGraph
+    inCall: int
+    blocks: seq[TBlock]
+
+proc debugInfo(info: TLineInfo): string =
+  result = info.toFilename & ":" & $info.line
+
+proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
+  # for debugging purposes
+  # first iteration: compute all necessary labels:
+  var jumpTargets = initIntSet()
+  let last = if last < 0: c.len-1 else: min(last, c.len-1)
+  for i in start..last:
+    if c[i].kind in {goto, fork}:
+      jumpTargets.incl(i+c[i].dest)
+  var i = start
+  while i <= last:
+    if i in jumpTargets: result.add("L" & $i & ":\n")
+    result.add "\t"
+    result.add $c[i].kind
+    result.add "\t"
+    case c[i].kind
+    of def, use, useWithinCall:
+      result.add c[i].sym.name.s
+    of goto, fork:
+      result.add "L"
+      result.add c[i].dest+i
+    result.add("\t#")
+    result.add(debugInfo(c[i].n.info))
+    result.add("\n")
+    inc i
+  if i in jumpTargets: result.add("L" & $i & ": End\n")
+
+
+proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
+  ## echos the ControlFlowGraph for debugging purposes.
+  var buf = ""
+  codeListing(c, buf, start, last)
+  echo buf
+
+proc forkI(c: var Con; n: PNode): TPosition =
+  result = TPosition(c.code.len)
+  c.code.add Instr(n: n, kind: fork, dest: 0)
+
+proc gotoI(c: var Con; n: PNode): TPosition =
+  result = TPosition(c.code.len)
+  c.code.add Instr(n: n, kind: goto, dest: 0)
+
+proc genLabel(c: Con): TPosition =
+  result = TPosition(c.code.len)
+
+proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
+  let dist = p.int - c.code.len
+  internalAssert(-0x7fff < dist and dist < 0x7fff)
+  c.code.add Instr(n: n, kind: goto, dest: dist)
+
+proc patch(c: var Con, p: TPosition) =
+  # patch with current index
+  let p = p.int
+  let diff = c.code.len - p
+  internalAssert(-0x7fff < diff and diff < 0x7fff)
+  c.code[p].dest = diff
+
+proc popBlock(c: var Con; oldLen: int) =
+  for f in c.blocks[oldLen].fixups:
+    c.patch(f)
+  c.blocks.setLen(oldLen)
+
+template withBlock(labl: PSym; body: untyped) {.dirty.} =
+  var oldLen {.gensym.} = c.blocks.len
+  c.blocks.add TBlock(label: labl, fixups: @[])
+  body
+  popBlock(c, oldLen)
+
+proc isTrue(n: PNode): bool =
+  n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
+    n.kind == nkIntLit and n.intVal != 0
+
+proc gen(c: var Con; n: PNode) # {.noSideEffect.}
+
+proc genWhile(c: var Con; n: PNode) =
+  # L1:
+  #   cond, tmp
+  #   fjmp tmp, L2
+  #   body
+  #   jmp L1
+  # L2:
+  let L1 = c.genLabel
+  withBlock(nil):
+    if isTrue(n.sons[0]):
+      c.gen(n.sons[1])
+      c.jmpBack(n, L1)
+    else:
+      c.gen(n.sons[0])
+      let L2 = c.forkI(n)
+      c.gen(n.sons[1])
+      c.jmpBack(n, L1)
+      c.patch(L2)
+
+proc genBlock(c: var Con; n: PNode) =
+  withBlock(n.sons[0].sym):
+    c.gen(n.sons[1])
+
+proc genBreak(c: var Con; n: PNode) =
+  let L1 = c.gotoI(n)
+  if n.sons[0].kind == nkSym:
+    #echo cast[int](n.sons[0].sym)
+    for i in countdown(c.blocks.len-1, 0):
+      if c.blocks[i].label == n.sons[0].sym:
+        c.blocks[i].fixups.add L1
+        return
+    globalError(n.info, errGenerated, "VM problem: cannot find 'break' target")
+  else:
+    c.blocks[c.blocks.high].fixups.add L1
+
+proc genIf(c: var Con, n: PNode) =
+  var endings: seq[TPosition] = @[]
+  for i in countup(0, len(n) - 1):
+    var it = n.sons[i]
+    if it.len == 2:
+      c.gen(it.sons[0].sons[1])
+      var elsePos = c.forkI(it.sons[0].sons[1])
+      c.gen(it.sons[1])
+      if i < sonsLen(n)-1:
+        endings.add(c.gotoI(it.sons[1]))
+      c.patch(elsePos)
+    else:
+      c.gen(it.sons[0])
+  for endPos in endings: c.patch(endPos)
+
+proc genAndOr(c: var Con; n: PNode) =
+  #   asgn dest, a
+  #   fork L1
+  #   asgn dest, b
+  # L1:
+  c.gen(n.sons[1])
+  let L1 = c.forkI(n)
+  c.gen(n.sons[2])
+  c.patch(L1)
+
+proc genCase(c: var Con; n: PNode) =
+  #  if (!expr1) goto L1;
+  #    thenPart
+  #    goto LEnd
+  #  L1:
+  #  if (!expr2) goto L2;
+  #    thenPart2
+  #    goto LEnd
+  #  L2:
+  #    elsePart
+  #  Lend:
+  var endings: seq[TPosition] = @[]
+  c.gen(n.sons[0])
+  for i in 1 ..< n.len:
+    let it = n.sons[i]
+    if it.len == 1:
+      c.gen(it.sons[0])
+    else:
+      let elsePos = c.forkI(it.lastSon)
+      c.gen(it.lastSon)
+      if i < sonsLen(n)-1:
+        endings.add(c.gotoI(it.lastSon))
+      c.patch(elsePos)
+  for endPos in endings: c.patch(endPos)
+
+proc genTry(c: var Con; n: PNode) =
+  var endings: seq[TPosition] = @[]
+  let elsePos = c.forkI(n)
+  c.gen(n.sons[0])
+  c.patch(elsePos)
+  for i in 1 ..< n.len:
+    let it = n.sons[i]
+    if it.kind != nkFinally:
+      var blen = len(it)
+      let endExcept = c.forkI(it)
+      c.gen(it.lastSon)
+      if i < sonsLen(n)-1:
+        endings.add(c.gotoI(it))
+      c.patch(endExcept)
+  for endPos in endings: c.patch(endPos)
+  let fin = lastSon(n)
+  if fin.kind == nkFinally:
+    c.gen(fin.sons[0])
+
+proc genRaise(c: var Con; n: PNode) =
+  gen(c, n.sons[0])
+  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+
+proc genReturn(c: var Con; n: PNode) =
+  if n.sons[0].kind != nkEmpty: gen(c, n.sons[0])
+  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+
+const
+  InterestingSyms = {skVar, skResult, skLet}
+
+proc genUse(c: var Con; n: PNode) =
+  var n = n
+  while n.kind in {nkDotExpr, nkCheckedFieldExpr,
+                   nkBracketExpr, nkDerefExpr, nkHiddenDeref,
+                   nkAddr, nkHiddenAddr}:
+    n = n[0]
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    if c.inCall > 0:
+      c.code.add Instr(n: n, kind: useWithinCall, sym: n.sym)
+    else:
+      c.code.add Instr(n: n, kind: use, sym: n.sym)
+
+proc genDef(c: var Con; n: PNode) =
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: n, kind: def, sym: n.sym)
+
+proc genCall(c: var Con; n: PNode) =
+  gen(c, n[0])
+  var t = n[0].typ
+  if t != nil: t = t.skipTypes(abstractInst)
+  inc c.inCall
+  for i in 1..<n.len:
+    gen(c, n[i])
+    if t != nil and i < t.len and t.sons[i].kind == tyVar:
+      genDef(c, n[i])
+  dec c.inCall
+
+proc genMagic(c: var Con; n: PNode; m: TMagic) =
+  case m
+  of mAnd, mOr: c.genAndOr(n)
+  of mNew, mNewFinalize:
+    genDef(c, n[1])
+    for i in 2..<n.len: gen(c, n[i])
+  of mExit:
+    genCall(c, n)
+    c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+  else:
+    genCall(c, n)
+
+proc genVarSection(c: var Con; n: PNode) =
+  for a in n:
+    if a.kind == nkCommentStmt: continue
+    if a.kind == nkVarTuple:
+      gen(c, a.lastSon)
+      for i in 0 .. a.len-3: genDef(c, a[i])
+    else:
+      gen(c, a.lastSon)
+      if a.lastSon.kind != nkEmpty:
+        genDef(c, a.sons[0])
+
+proc gen(c: var Con; n: PNode) =
+  case n.kind
+  of nkSym: genUse(c, n)
+  of nkCallKinds:
+    if n.sons[0].kind == nkSym:
+      let s = n.sons[0].sym
+      if s.magic != mNone:
+        genMagic(c, n, s.magic)
+      else:
+        genCall(c, n)
+    else:
+      genCall(c, n)
+  of nkCharLit..nkNilLit: discard
+  of nkAsgn, nkFastAsgn:
+    gen(c, n[1])
+    genDef(c, n[0])
+  of nkDotExpr, nkCheckedFieldExpr, nkBracketExpr,
+     nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr:
+    gen(c, n[0])
+  of nkIfStmt, nkIfExpr: genIf(c, n)
+  of nkWhenStmt:
+    # This is "when nimvm" node. Chose the first branch.
+    gen(c, n.sons[0].sons[1])
+  of nkCaseStmt: genCase(c, n)
+  of nkWhileStmt: genWhile(c, n)
+  of nkBlockExpr, nkBlockStmt: genBlock(c, n)
+  of nkReturnStmt: genReturn(c, n)
+  of nkRaiseStmt: genRaise(c, n)
+  of nkBreakStmt: genBreak(c, n)
+  of nkTryStmt: genTry(c, n)
+  of nkStmtList, nkStmtListExpr, nkChckRangeF, nkChckRange64, nkChckRange,
+     nkBracket, nkCurly, nkPar, nkClosure, nkObjConstr:
+    for x in n: gen(c, x)
+  of nkPragmaBlock: gen(c, n.lastSon)
+  of nkDiscardStmt: gen(c, n.sons[0])
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv, nkExprColonExpr, nkExprEqExpr,
+     nkCast:
+    gen(c, n.sons[1])
+  of nkObjDownConv, nkStringToCString, nkCStringToString: gen(c, n.sons[0])
+  of nkVarSection, nkLetSection: genVarSection(c, n)
+  else: discard
+
+proc dfa(code: seq[Instr]) =
+  # We aggressively push 'undef' values for every 'use v' instruction
+  # until they are eliminated via a 'def v' instructions.
+  # If we manage to push one 'undef' to a 'use' instruction, we produce
+  # an error:
+  var undef = initIntSet()
+  for i in 0..<code.len:
+    if code[i].kind == use: undef.incl(code[i].sym.id)
+
+  var s = newSeq[IntSet](code.len)
+  for i in 0..<code.len:
+    assign(s[i], undef)
+
+  # In the original paper, W := {0,...,n} is done. This is wasteful, we
+  # have no intention to analyse a program like
+  #
+  # return 3
+  # echo a + b
+  #
+  # any further than necessary.
+  var w = @[0]
+  while w.len > 0:
+    var pc = w[^1]
+    # this simulates a single linear control flow execution:
+    while true:
+      # according to the paper, it is better to shrink the working set here
+      # in this inner loop:
+      let widx = w.find(pc)
+      if widx >= 0: w.del(widx)
+      # our interpretation ![I!]:
+      var sid = -1
+      case code[pc].kind
+      of goto, fork: discard
+      of use, useWithinCall:
+        let sym = code[pc].sym
+        if s[pc].contains(sym.id):
+          localError(code[pc].n.info, "variable read before initialized: " & sym.name.s)
+      of def:
+        sid = code[pc].sym.id
+
+      var pc2: int
+      if code[pc].kind == goto:
+        pc2 = pc + code[pc].dest
+      else:
+        pc2 = pc + 1
+        if code[pc].kind == fork:
+          let l = pc + code[pc].dest
+          if sid >= 0 and s[l].missingOrExcl(sid):
+            w.add l
+
+      if sid >= 0 and s[pc2].missingOrExcl(sid):
+        pc = pc2
+      else:
+        break
+      if pc >= code.len: break
+
+    when false:
+      case code[pc].kind
+      of use:
+        let s = code[pc].sym
+        if undefB.contains(s.id):
+          localError(code[pc].n.info, "variable read before initialized: " & s.name.s)
+          break
+        inc pc
+      of def:
+        let s = code[pc].sym
+        # exclude 'undef' for s for this path through the graph.
+        if not undefB.missingOrExcl(s.id):
+          inc pc
+        else:
+          break
+        #undefB.excl s.id
+        #inc pc
+        when false:
+          let prev = bindings.getOrDefault(s.id)
+          if prev != value:
+            # well now it has a value and we made progress, so
+            bindings[s.id] = value
+            inc pc
+          else:
+            break
+      of fork:
+        let diff = code[pc].dest
+        # we follow pc + 1 and remember the label for later:
+        w.add pc+diff
+        inc pc
+      of goto:
+        let diff = code[pc].dest
+        pc = pc + diff
+      if pc >= code.len: break
+
+proc dataflowAnalysis*(s: PSym; body: PNode) =
+  var c = Con(code: @[], blocks: @[])
+  gen(c, body)
+  #echoCfg(c.code)
+  dfa(c.code)
+
+proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
+  ## constructs a control flow graph for ``body``.
+  var c = Con(code: @[], blocks: @[])
+  shallowCopy(result, c.code)

compiler/docgen.nim

@@ -15,8 +15,8 @@ import
   ast, strutils, strtabs, options, msgs, os, ropes, idents,
   wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast,
   packages/docutils/rst, packages/docutils/rstgen, times,
-  packages/docutils/highlite, importer, sempass2, json, xmltree, cgi,
+  packages/docutils/highlite, sempass2, json, xmltree, cgi,
-  typesrenderer, astalgo
+  typesrenderer, astalgo, modulepaths
 
 type
   TSections = array[TSymKind, Rope]

compiler/evaltempl.nim

@@ -77,7 +77,7 @@ proc evalTemplateArgs(n: PNode, s: PSym; fromHlo: bool): PNode =
     # now that we have working untyped parameters.
     genericParams = if sfImmediate in s.flags or fromHlo: 0
                     else: s.ast[genericParamsPos].len
-    expectedRegularParams = <s.typ.len
+    expectedRegularParams = s.typ.len-1
     givenRegularParams = totalParams - genericParams
   if givenRegularParams < 0: givenRegularParams = 0
 

compiler/filter_tmpl.nim

@@ -13,14 +13,10 @@ import
   llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
   renderer, filters
 
-proc filterTmpl*(stdin: PLLStream, filename: string, call: PNode): PLLStream
-  # #! template(subsChar='$', metaChar='#') | standard(version="0.7.2")
-# implementation
-
 type
   TParseState = enum
     psDirective, psTempl
-  TTmplParser{.final.} = object
+  TTmplParser = object
     inp: PLLStream
     state: TParseState
     info: TLineInfo
@@ -61,6 +57,10 @@ proc scanPar(p: var TTmplParser, d: int) =
 proc withInExpr(p: TTmplParser): bool {.inline.} =
   result = p.par > 0 or p.bracket > 0 or p.curly > 0
 
+const
+  LineContinuationOprs = {'+', '-', '*', '/', '\\', '<', '>', '^',
+                          '|', '%', '&', '$', '@', '~', ','}
+
 proc parseLine(p: var TTmplParser) =
   var j = 0
   while p.x[j] == ' ': inc(j)
@@ -77,7 +77,7 @@ proc parseLine(p: var TTmplParser) =
       inc(j)
 
     scanPar(p, j)
-    p.pendingExprLine = withInExpr(p) or llstream.endsWithOpr(p.x)
+    p.pendingExprLine = withInExpr(p) or p.x.endsWith(LineContinuationOprs)
     case keyw
     of "end":
       if p.indent >= 2:
@@ -88,14 +88,14 @@ proc parseLine(p: var TTmplParser) =
       llStreamWrite(p.outp, spaces(p.indent))
       llStreamWrite(p.outp, "#end")
     of "if", "when", "try", "while", "for", "block", "case", "proc", "iterator",
-       "converter", "macro", "template", "method":
+       "converter", "macro", "template", "method", "func":
       llStreamWrite(p.outp, spaces(p.indent))
       llStreamWrite(p.outp, substr(p.x, d))
       inc(p.indent, 2)
     of "elif", "of", "else", "except", "finally":
       llStreamWrite(p.outp, spaces(p.indent - 2))
       llStreamWrite(p.outp, substr(p.x, d))
-    of "wLet", "wVar", "wConst", "wType":
+    of "let", "var", "const", "type":
       llStreamWrite(p.outp, spaces(p.indent))
       llStreamWrite(p.outp, substr(p.x, d))
       if not p.x.contains({':', '='}):
@@ -199,7 +199,7 @@ proc parseLine(p: var TTmplParser) =
           inc(j)
     llStreamWrite(p.outp, "\\n\"")
 
-proc filterTmpl(stdin: PLLStream, filename: string, call: PNode): PLLStream =
+proc filterTmpl*(stdin: PLLStream, filename: string, call: PNode): PLLStream =
   var p: TTmplParser
   p.info = newLineInfo(filename, 0, 0)
   p.outp = llStreamOpen("")

compiler/filters.nim

@@ -13,14 +13,6 @@ import
   llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
   renderer
 
-proc filterReplace*(stdin: PLLStream, filename: string, call: PNode): PLLStream
-proc filterStrip*(stdin: PLLStream, filename: string, call: PNode): PLLStream
-  # helpers to retrieve arguments:
-proc charArg*(n: PNode, name: string, pos: int, default: char): char
-proc strArg*(n: PNode, name: string, pos: int, default: string): string
-proc boolArg*(n: PNode, name: string, pos: int, default: bool): bool
-# implementation
-
 proc invalidPragma(n: PNode) =
   localError(n.info, errXNotAllowedHere, renderTree(n, {renderNoComments}))
 
@@ -35,26 +27,26 @@ proc getArg(n: PNode, name: string, pos: int): PNode =
     elif i == pos:
       return n.sons[i]
 
-proc charArg(n: PNode, name: string, pos: int, default: char): char =
+proc charArg*(n: PNode, name: string, pos: int, default: char): char =
   var x = getArg(n, name, pos)
   if x == nil: result = default
   elif x.kind == nkCharLit: result = chr(int(x.intVal))
   else: invalidPragma(n)
 
-proc strArg(n: PNode, name: string, pos: int, default: string): string =
+proc strArg*(n: PNode, name: string, pos: int, default: string): string =
   var x = getArg(n, name, pos)
   if x == nil: result = default
   elif x.kind in {nkStrLit..nkTripleStrLit}: result = x.strVal
   else: invalidPragma(n)
 
-proc boolArg(n: PNode, name: string, pos: int, default: bool): bool =
+proc boolArg*(n: PNode, name: string, pos: int, default: bool): bool =
   var x = getArg(n, name, pos)
   if x == nil: result = default
   elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "true") == 0: result = true
   elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "false") == 0: result = false
   else: invalidPragma(n)
 
-proc filterStrip(stdin: PLLStream, filename: string, call: PNode): PLLStream =
+proc filterStrip*(stdin: PLLStream, filename: string, call: PNode): PLLStream =
   var pattern = strArg(call, "startswith", 1, "")
   var leading = boolArg(call, "leading", 2, true)
   var trailing = boolArg(call, "trailing", 3, true)
@@ -68,7 +60,7 @@ proc filterStrip(stdin: PLLStream, filename: string, call: PNode): PLLStream =
       llStreamWriteln(result, line)
   llStreamClose(stdin)
 
-proc filterReplace(stdin: PLLStream, filename: string, call: PNode): PLLStream =
+proc filterReplace*(stdin: PLLStream, filename: string, call: PNode): PLLStream =
   var sub = strArg(call, "sub", 1, "")
   if len(sub) == 0: invalidPragma(call)
   var by = strArg(call, "by", 2, "")

compiler/gorgeimpl.nim

@@ -7,8 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## Module that implements ``gorge`` for the compiler as well as
+## Module that implements ``gorge`` for the compiler.
-## the scriptable import mechanism.
 
 import msgs, securehash, os, osproc, streams, strutils, options
 
@@ -56,28 +55,3 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo): (string, int) =
       result = p.readOutput
     except IOError, OSError:
       result = ("", -1)
-
-proc scriptableImport*(pkg, subdir: string; info: TLineInfo): string =
-  var cmd = getConfigVar("resolver.exe")
-  if cmd.len == 0: cmd = "nimresolve"
-  else: cmd = quoteShell(cmd)
-  cmd.add " --source:"
-  cmd.add quoteShell(info.toFullPath())
-  cmd.add " --stdlib:"
-  cmd.add quoteShell(options.libpath)
-  cmd.add "  --project:"
-  cmd.add quoteShell(gProjectFull)
-  if subdir.len != 0:
-    cmd.add " --subdir:"
-    cmd.add quoteShell(subdir)
-  if options.gNoNimblePath:
-    cmd.add " --nonimblepath"
-  cmd.add ' '
-  cmd.add quoteShell(pkg)
-  let (res, exitCode) = opGorge(cmd, "", cmd, info)
-  if exitCode == 0:
-    result = res.strip()
-  elif res.len > 0:
-    localError(info, res)
-  else:
-    localError(info, "cannot resolve: " & (pkg / subdir))

compiler/hlo.nim

@@ -36,7 +36,7 @@ proc applyPatterns(c: PContext, n: PNode): PNode =
   # we apply the last pattern first, so that pattern overriding is possible;
   # however the resulting AST would better not trigger the old rule then
   # anymore ;-)
-  for i in countdown(<c.patterns.len, 0):
+  for i in countdown(c.patterns.len-1, 0):
     let pattern = c.patterns[i]
     if not isNil(pattern):
       let x = applyRule(c, pattern, result)

compiler/importer.nim

@@ -11,83 +11,11 @@
 
 import
   intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups,
-  semdata, passes, renderer, gorgeimpl
+  semdata, passes, renderer, modulepaths
 
 proc evalImport*(c: PContext, n: PNode): PNode
 proc evalFrom*(c: PContext, n: PNode): PNode
 
-proc lookupPackage(pkg, subdir: PNode): string =
-  let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
-  case pkg.kind
-  of nkStrLit, nkRStrLit, nkTripleStrLit:
-    result = scriptableImport(pkg.strVal, sub, pkg.info)
-  of nkIdent:
-    result = scriptableImport(pkg.ident.s, sub, pkg.info)
-  else:
-    localError(pkg.info, "package name must be an identifier or string literal")
-    result = ""
-
-proc getModuleName*(n: PNode): string =
-  # This returns a short relative module name without the nim extension
-  # e.g. like "system", "importer" or "somepath/module"
-  # The proc won't perform any checks that the path is actually valid
-  case n.kind
-  of nkStrLit, nkRStrLit, nkTripleStrLit:
-    try:
-      result = pathSubs(n.strVal, n.info.toFullPath().splitFile().dir)
-    except ValueError:
-      localError(n.info, "invalid path: " & n.strVal)
-      result = n.strVal
-  of nkIdent:
-    result = n.ident.s
-  of nkSym:
-    result = n.sym.name.s
-  of nkInfix:
-    let n0 = n[0]
-    let n1 = n[1]
-    if n0.kind == nkIdent and n0.ident.id == getIdent("as").id:
-      # XXX hack ahead:
-      n.kind = nkImportAs
-      n.sons[0] = n.sons[1]
-      n.sons[1] = n.sons[2]
-      n.sons.setLen(2)
-      return getModuleName(n.sons[0])
-    if n1.kind == nkPrefix and n1[0].kind == nkIdent and n1[0].ident.s == "$":
-      if n0.kind == nkIdent and n0.ident.s == "/":
-        result = lookupPackage(n1[1], n[2])
-      else:
-        localError(n.info, "only '/' supported with $package notation")
-        result = ""
-    else:
-      # hacky way to implement 'x / y /../ z':
-      result = getModuleName(n1)
-      result.add renderTree(n0, {renderNoComments})
-      result.add getModuleName(n[2])
-  of nkPrefix:
-    if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$":
-      result = lookupPackage(n[1], nil)
-    else:
-      # hacky way to implement 'x / y /../ z':
-      result = renderTree(n, {renderNoComments}).replace(" ")
-  of nkDotExpr:
-    result = renderTree(n, {renderNoComments}).replace(".", "/")
-  of nkImportAs:
-    result = getModuleName(n.sons[0])
-  else:
-    localError(n.info, errGenerated, "invalid module name: '$1'" % n.renderTree)
-    result = ""
-
-proc checkModuleName*(n: PNode; doLocalError=true): int32 =
-  # This returns the full canonical path for a given module import
-  let modulename = n.getModuleName
-  let fullPath = findModule(modulename, n.info.toFullPath)
-  if fullPath.len == 0:
-    if doLocalError:
-      localError(n.info, errCannotOpenFile, modulename)
-    result = InvalidFileIDX
-  else:
-    result = fullPath.fileInfoIdx
-
 proc importPureEnumField*(c: PContext; s: PSym) =
   var check = strTableGet(c.importTable.symbols, s.name)
   if check == nil:

compiler/lexer.nim

@@ -126,6 +126,10 @@ type
     literal*: string          # the parsed (string) literal; and
                               # documentation comments are here too
     line*, col*: int
+    when defined(nimpretty):
+      offsetA*, offsetB*: int   # used for pretty printing so that literals
+                                # like 0b01 or  r"\L" are unaffected
+      commentOffsetA*, commentOffsetB*: int
 
   TErrorHandler* = proc (info: TLineInfo; msg: TMsgKind; arg: string)
   TLexer* = object of TBaseLexer
@@ -141,10 +145,19 @@ type
     when defined(nimsuggest):
       previousToken: TLineInfo
 
+when defined(nimpretty):
+  var
+    gIndentationWidth*: int
+
 var gLinesCompiled*: int  # all lines that have been compiled
 
 proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
-  newLineInfo(L.fileIdx, tok.line, tok.col)
+  result = newLineInfo(L.fileIdx, tok.line, tok.col)
+  when defined(nimpretty):
+    result.offsetA = tok.offsetA
+    result.offsetB = tok.offsetB
+    result.commentOffsetA = tok.commentOffsetA
+    result.commentOffsetB = tok.commentOffsetB
 
 proc isKeyword*(kind: TTokType): bool =
   result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
@@ -192,6 +205,9 @@ proc initToken*(L: var TToken) =
   L.fNumber = 0.0
   L.base = base10
   L.ident = nil
+  when defined(nimpretty):
+    L.commentOffsetA = 0
+    L.commentOffsetB = 0
 
 proc fillToken(L: var TToken) =
   L.tokType = tkInvalid
@@ -202,6 +218,9 @@ proc fillToken(L: var TToken) =
   L.fNumber = 0.0
   L.base = base10
   L.ident = nil
+  when defined(nimpretty):
+    L.commentOffsetA = 0
+    L.commentOffsetB = 0
 
 proc openLexer*(lex: var TLexer, fileIdx: int32, inputstream: PLLStream;
                  cache: IdentCache) =
@@ -245,11 +264,13 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
 proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
   result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
 
-template tokenBegin(pos) {.dirty.} =
+template tokenBegin(tok, pos) {.dirty.} =
   when defined(nimsuggest):
     var colA = getColNumber(L, pos)
+  when defined(nimpretty):
+    tok.offsetA = L.offsetBase + pos
 
-template tokenEnd(pos) {.dirty.} =
+template tokenEnd(tok, pos) {.dirty.} =
   when defined(nimsuggest):
     let colB = getColNumber(L, pos)+1
     if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and
@@ -257,8 +278,10 @@ template tokenEnd(pos) {.dirty.} =
       L.cursor = CursorPosition.InToken
       gTrackPos.col = colA.int16
     colA = 0
+  when defined(nimpretty):
+    tok.offsetB = L.offsetBase + pos
 
-template tokenEndIgnore(pos) =
+template tokenEndIgnore(tok, pos) =
   when defined(nimsuggest):
     let colB = getColNumber(L, pos)
     if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and
@@ -266,8 +289,10 @@ template tokenEndIgnore(pos) =
       gTrackPos.fileIndex = trackPosInvalidFileIdx
       gTrackPos.line = -1
     colA = 0
+  when defined(nimpretty):
+    tok.offsetB = L.offsetBase + pos
 
-template tokenEndPrevious(pos) =
+template tokenEndPrevious(tok, pos) =
   when defined(nimsuggest):
     # when we detect the cursor in whitespace, we attach the track position
     # to the token that came before that, but only if we haven't detected
@@ -279,6 +304,8 @@ template tokenEndPrevious(pos) =
       gTrackPos = L.previousToken
       gTrackPosAttached = true
     colA = 0
+  when defined(nimpretty):
+    tok.offsetB = L.offsetBase + pos
 
 {.push overflowChecks: off.}
 # We need to parse the largest uint literal without overflow checks
@@ -363,7 +390,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
   result.literal = ""
   result.base = base10
   startpos = L.bufpos
-  tokenBegin(startPos)
+  tokenBegin(result, startPos)
 
   # First stage: find out base, make verifications, build token literal string
   if L.buf[L.bufpos] == '0' and L.buf[L.bufpos + 1] in baseCodeChars + {'O'}:
@@ -573,7 +600,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
     lexMessageLitNum(L, errInvalidNumber, startpos)
   except OverflowError, RangeError:
     lexMessageLitNum(L, errNumberOutOfRange, startpos)
-  tokenEnd(postPos-1)
+  tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
 proc handleHexChar(L: var TLexer, xi: var int) =
@@ -691,10 +718,11 @@ proc handleCRLF(L: var TLexer, pos: int): int =
   else: result = pos
 
 proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
-  var pos = L.bufpos + 1          # skip "
+  var pos = L.bufpos
   var buf = L.buf                 # put `buf` in a register
   var line = L.lineNumber         # save linenumber for better error message
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
+  inc pos # skip "
   if buf[pos] == '\"' and buf[pos+1] == '\"':
     tok.tokType = tkTripleStrLit # long string literal:
     inc(pos, 2)               # skip ""
@@ -710,18 +738,18 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
       of '\"':
         if buf[pos+1] == '\"' and buf[pos+2] == '\"' and
             buf[pos+3] != '\"':
-          tokenEndIgnore(pos+2)
+          tokenEndIgnore(tok, pos+2)
           L.bufpos = pos + 3 # skip the three """
           break
         add(tok.literal, '\"')
         inc(pos)
       of CR, LF:
-        tokenEndIgnore(pos)
+        tokenEndIgnore(tok, pos)
         pos = handleCRLF(L, pos)
         buf = L.buf
         add(tok.literal, tnl)
       of nimlexbase.EndOfFile:
-        tokenEndIgnore(pos)
+        tokenEndIgnore(tok, pos)
         var line2 = L.lineNumber
         L.lineNumber = line
         lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart)
@@ -742,11 +770,11 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
           inc(pos, 2)
           add(tok.literal, '"')
         else:
-          tokenEndIgnore(pos)
+          tokenEndIgnore(tok, pos)
           inc(pos) # skip '"'
           break
       elif c in {CR, LF, nimlexbase.EndOfFile}:
-        tokenEndIgnore(pos)
+        tokenEndIgnore(tok, pos)
         lexMessage(L, errClosingQuoteExpected)
         break
       elif (c == '\\') and not rawMode:
@@ -759,7 +787,7 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
     L.bufpos = pos
 
 proc getCharacter(L: var TLexer, tok: var TToken) =
-  tokenBegin(L.bufpos)
+  tokenBegin(tok, L.bufpos)
   inc(L.bufpos)               # skip '
   var c = L.buf[L.bufpos]
   case c
@@ -769,14 +797,14 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
     tok.literal = $c
     inc(L.bufpos)
   if L.buf[L.bufpos] != '\'': lexMessage(L, errMissingFinalQuote)
-  tokenEndIgnore(L.bufpos)
+  tokenEndIgnore(tok, L.bufpos)
   inc(L.bufpos)               # skip '
 
 proc getSymbol(L: var TLexer, tok: var TToken) =
   var h: Hash = 0
   var pos = L.bufpos
   var buf = L.buf
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   while true:
     var c = buf[pos]
     case c
@@ -793,7 +821,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
         break
       inc(pos)
     else: break
-  tokenEnd(pos-1)
+  tokenEnd(tok, pos-1)
   h = !$h
   tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
   L.bufpos = pos
@@ -814,7 +842,7 @@ proc endOperator(L: var TLexer, tok: var TToken, pos: int,
 proc getOperator(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
   var buf = L.buf
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   var h: Hash = 0
   while true:
     var c = buf[pos]
@@ -822,7 +850,7 @@ proc getOperator(L: var TLexer, tok: var TToken) =
     h = h !& ord(c)
     inc(pos)
   endOperator(L, tok, pos, h)
-  tokenEnd(pos-1)
+  tokenEnd(tok, pos-1)
   # advance pos but don't store it in L.bufpos so the next token (which might
   # be an operator too) gets the preceding spaces:
   tok.strongSpaceB = 0
@@ -837,7 +865,7 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
   var pos = start
   var buf = L.buf
   var toStrip = 0
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   # detect the amount of indentation:
   if isDoc:
     toStrip = getColNumber(L, pos)
@@ -864,36 +892,37 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
       if isDoc:
         if buf[pos+1] == '#' and buf[pos+2] == '#':
           if nesting == 0:
-            tokenEndIgnore(pos+2)
+            tokenEndIgnore(tok, pos+2)
             inc(pos, 3)
             break
           dec nesting
         tok.literal.add ']'
       elif buf[pos+1] == '#':
         if nesting == 0:
-          tokenEndIgnore(pos+1)
+          tokenEndIgnore(tok, pos+1)
           inc(pos, 2)
           break
         dec nesting
       inc pos
     of CR, LF:
-      tokenEndIgnore(pos)
+      tokenEndIgnore(tok, pos)
       pos = handleCRLF(L, pos)
       buf = L.buf
       # strip leading whitespace:
+      when defined(nimpretty): tok.literal.add "\L"
       if isDoc:
-        tok.literal.add "\n"
+        when not defined(nimpretty): tok.literal.add "\n"
         inc tok.iNumber
         var c = toStrip
         while buf[pos] == ' ' and c > 0:
           inc pos
           dec c
     of nimlexbase.EndOfFile:
-      tokenEndIgnore(pos)
+      tokenEndIgnore(tok, pos)
       lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
       break
     else:
-      if isDoc: tok.literal.add buf[pos]
+      if isDoc or defined(nimpretty): tok.literal.add buf[pos]
       inc(pos)
   L.bufpos = pos
 
@@ -907,7 +936,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
   if buf[pos+2] == '[':
     skipMultiLineComment(L, tok, pos+3, true)
     return
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   inc(pos, 2)
 
   var toStrip = 0
@@ -921,7 +950,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
       if buf[pos] == '\\': lastBackslash = pos+1
       add(tok.literal, buf[pos])
       inc(pos)
-    tokenEndIgnore(pos)
+    tokenEndIgnore(tok, pos)
     pos = handleCRLF(L, pos)
     buf = L.buf
     var indent = 0
@@ -940,14 +969,14 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     else:
       if buf[pos] > ' ':
         L.indentAhead = indent
-      tokenEndIgnore(pos)
+      tokenEndIgnore(tok, pos)
       break
   L.bufpos = pos
 
 proc skip(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
   var buf = L.buf
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   tok.strongSpaceA = 0
   while true:
     case buf[pos]
@@ -958,7 +987,7 @@ proc skip(L: var TLexer, tok: var TToken) =
       if not L.allowTabs: lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
       inc(pos)
     of CR, LF:
-      tokenEndPrevious(pos)
+      tokenEndPrevious(tok, pos)
       pos = handleCRLF(L, pos)
       buf = L.buf
       var indent = 0
@@ -980,18 +1009,27 @@ proc skip(L: var TLexer, tok: var TToken) =
     of '#':
       # do not skip documentation comment:
       if buf[pos+1] == '#': break
+      when defined(nimpretty):
+        tok.commentOffsetA = L.offsetBase + pos
       if buf[pos+1] == '[':
         skipMultiLineComment(L, tok, pos+2, false)
         pos = L.bufpos
         buf = L.buf
+        when defined(nimpretty):
+          tok.commentOffsetB = L.offsetBase + pos
       else:
-        tokenBegin(pos)
+        tokenBegin(tok, pos)
         while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos)
-        tokenEndIgnore(pos+1)
+        tokenEndIgnore(tok, pos+1)
+        when defined(nimpretty):
+          tok.commentOffsetB = L.offsetBase + pos + 1
     else:
       break                   # EndOfFile also leaves the loop
-  tokenEndPrevious(pos-1)
+  tokenEndPrevious(tok, pos-1)
   L.bufpos = pos
+  when defined(nimpretty):
+    if gIndentationWidth <= 0:
+      gIndentationWidth = tok.indent
 
 proc rawGetTok*(L: var TLexer, tok: var TToken) =
   template atTokenEnd() {.dirty.} =
@@ -1014,7 +1052,7 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
   var c = L.buf[L.bufpos]
   tok.line = L.lineNumber
   tok.col = getColNumber(L, L.bufpos)
-  if c in SymStartChars - {'r', 'R', 'l'}:
+  if c in SymStartChars - {'r', 'R'}:
     getSymbol(L, tok)
   else:
     case c
@@ -1031,12 +1069,6 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
     of ',':
       tok.tokType = tkComma
       inc(L.bufpos)
-    of 'l':
-      # if we parsed exactly one character and its a small L (l), this
-      # is treated as a warning because it may be confused with the number 1
-      if L.buf[L.bufpos+1] notin (SymChars + {'_'}):
-        lexMessage(L, warnSmallLshouldNotBeUsed)
-      getSymbol(L, tok)
     of 'r', 'R':
       if L.buf[L.bufpos + 1] == '\"':
         inc(L.bufpos)

compiler/llstream.nim

@@ -84,7 +84,6 @@ const
   AdditionalLineContinuationOprs = {'#', ':', '='}
 
 proc endsWithOpr*(x: string): bool =
-  # also used by the standard template filter:
   result = x.endsWith(LineContinuationOprs)
 
 proc continueLine(line: string, inTripleString: bool): bool {.inline.} =

compiler/lookups.nim

@@ -46,7 +46,11 @@ proc considerQuotedIdent*(n: PNode, origin: PNode = nil): PIdent =
         of nkSym: id.add(x.sym.name.s)
         else: handleError(n, origin)
       result = getIdent(id)
-  of nkOpenSymChoice, nkClosedSymChoice: result = n.sons[0].sym.name
+  of nkOpenSymChoice, nkClosedSymChoice:
+    if n[0].kind == nkSym:
+      result = n.sons[0].sym.name
+    else:
+      handleError(n, origin)
   else:
     handleError(n, origin)
 
@@ -155,7 +159,7 @@ proc ensureNoMissingOrUnusedSymbols(scope: PScope) =
   var s = initTabIter(it, scope.symbols)
   var missingImpls = 0
   while s != nil:
-    if sfForward in s.flags:
+    if sfForward in s.flags and s.kind != skType:
       # too many 'implementation of X' errors are annoying
       # and slow 'suggest' down:
       if missingImpls == 0:
@@ -379,7 +383,11 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         result = errorSym(c, n.sons[1])
   of nkClosedSymChoice, nkOpenSymChoice:
     o.mode = oimSymChoice
+    if n[0].kind == nkSym:
       result = n.sons[0].sym
+    else:
+      o.mode = oimDone
+      return nil
     o.symChoiceIndex = 1
     o.inSymChoice = initIntSet()
     incl(o.inSymChoice, result.id)

compiler/lowerings.nim

@@ -70,6 +70,9 @@ proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
   lit.intVal = i
   addSon(result, lit)
 
+proc newTryFinally*(body, final: PNode): PNode =
+  result = newTree(nkTryStmt, body, newTree(nkFinally, final))
+
 proc lowerTupleUnpackingForAsgn*(n: PNode; owner: PSym): PNode =
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
@@ -139,6 +142,14 @@ proc rawIndirectAccess*(a: PNode; field: PSym; info: TLineInfo): PNode =
   addSon(result, newSymNode(field))
   result.typ = field.typ
 
+proc rawDirectAccess*(obj, field: PSym): PNode =
+  # returns a.field as a node
+  assert field.kind == skField
+  result = newNodeI(nkDotExpr, field.info)
+  addSon(result, newSymNode obj)
+  addSon(result, newSymNode field)
+  result.typ = field.typ
+
 proc lookupInRecord(n: PNode, id: int): PSym =
   result = nil
   case n.kind
@@ -171,8 +182,9 @@ proc addField*(obj: PType; s: PSym) =
   field.position = sonsLen(obj.n)
   addSon(obj.n, newSymNode(field))
 
-proc addUniqueField*(obj: PType; s: PSym) =
+proc addUniqueField*(obj: PType; s: PSym): PSym {.discardable.} =
-  if lookupInRecord(obj.n, s.id) == nil:
+  result = lookupInRecord(obj.n, s.id)
+  if result == nil:
     var field = newSym(skField, getIdent(s.name.s & $obj.n.len), s.owner, s.info)
     field.id = -s.id
     let t = skipIntLit(s.typ)
@@ -180,6 +192,7 @@ proc addUniqueField*(obj: PType; s: PSym) =
     assert t.kind != tyStmt
     field.position = sonsLen(obj.n)
     addSon(obj.n, newSymNode(field))
+    result = field
 
 proc newDotExpr(obj, b: PSym): PNode =
   result = newNodeI(nkDotExpr, obj.info)

compiler/modulepaths.nim

@@ -0,0 +1,172 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import ast, renderer, strutils, msgs, options, idents, os
+
+import nimblecmd
+
+const
+  considerParentDirs = not defined(noParentProjects)
+  considerNimbleDirs = not defined(noNimbleDirs)
+
+proc findInNimbleDir(pkg, subdir, dir: string): string =
+  var best = ""
+  var bestv = ""
+  for k, p in os.walkDir(dir, relative=true):
+    if k == pcDir and p.len > pkg.len+1 and
+        p[pkg.len] == '-' and p.startsWith(pkg):
+      let (_, a) = getPathVersion(p)
+      if bestv.len == 0 or bestv < a:
+        bestv = a
+        best = dir / p
+
+  if best.len > 0:
+    var f: File
+    if open(f, best / changeFileExt(pkg, ".nimble-link")):
+      # the second line contains what we're interested in, see:
+      # https://github.com/nim-lang/nimble#nimble-link
+      var override = ""
+      discard readLine(f, override)
+      discard readLine(f, override)
+      close(f)
+      if not override.isAbsolute():
+        best = best / override
+      else:
+        best = override
+  let f = if subdir.len == 0: pkg else: subdir
+  let res = addFileExt(best / f, "nim")
+  if best.len > 0 and fileExists(res):
+    result = res
+
+const stdlibDirs = [
+  "pure", "core", "arch",
+  "pure/collections",
+  "pure/concurrency", "impure",
+  "wrappers", "wrappers/linenoise",
+  "windows", "posix", "js"]
+
+proc resolveDollar(project, source, pkg, subdir: string; info: TLineInfo): string =
+  template attempt(a) =
+    let x = addFileExt(a, "nim")
+    if fileExists(x): return x
+
+  case pkg
+  of "stdlib":
+    if subdir.len == 0:
+      return options.libpath
+    else:
+      for candidate in stdlibDirs:
+        attempt(options.libpath / candidate / subdir)
+  of "root":
+    let root = project.splitFile.dir
+    if subdir.len == 0:
+      return root
+    else:
+      attempt(root / subdir)
+  else:
+    when considerParentDirs:
+      var p = parentDir(source.splitFile.dir)
+      # support 'import $karax':
+      let f = if subdir.len == 0: pkg else: subdir
+
+      while p.len > 0:
+        let dir = p / pkg
+        if dirExists(dir):
+          attempt(dir / f)
+          # 2nd attempt: try to use 'karax/karax'
+          attempt(dir / pkg / f)
+          # 3rd attempt: try to use 'karax/src/karax'
+          attempt(dir / "src" / f)
+          attempt(dir / "src" / pkg / f)
+        p = parentDir(p)
+
+    when considerNimbleDirs:
+      if not options.gNoNimblePath:
+        var nimbleDir = getEnv("NIMBLE_DIR")
+        if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
+        result = findInNimbleDir(pkg, subdir, nimbleDir / "pkgs")
+        if result.len > 0: return result
+        when not defined(windows):
+          result = findInNimbleDir(pkg, subdir, "/opt/nimble/pkgs")
+          if result.len > 0: return result
+
+proc scriptableImport(pkg, sub: string; info: TLineInfo): string =
+  result = resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
+
+proc lookupPackage(pkg, subdir: PNode): string =
+  let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
+  case pkg.kind
+  of nkStrLit, nkRStrLit, nkTripleStrLit:
+    result = scriptableImport(pkg.strVal, sub, pkg.info)
+  of nkIdent:
+    result = scriptableImport(pkg.ident.s, sub, pkg.info)
+  else:
+    localError(pkg.info, "package name must be an identifier or string literal")
+    result = ""
+
+proc getModuleName*(n: PNode): string =
+  # This returns a short relative module name without the nim extension
+  # e.g. like "system", "importer" or "somepath/module"
+  # The proc won't perform any checks that the path is actually valid
+  case n.kind
+  of nkStrLit, nkRStrLit, nkTripleStrLit:
+    try:
+      result = pathSubs(n.strVal, n.info.toFullPath().splitFile().dir)
+    except ValueError:
+      localError(n.info, "invalid path: " & n.strVal)
+      result = n.strVal
+  of nkIdent:
+    result = n.ident.s
+  of nkSym:
+    result = n.sym.name.s
+  of nkInfix:
+    let n0 = n[0]
+    let n1 = n[1]
+    if n0.kind == nkIdent and n0.ident.id == getIdent("as").id:
+      # XXX hack ahead:
+      n.kind = nkImportAs
+      n.sons[0] = n.sons[1]
+      n.sons[1] = n.sons[2]
+      n.sons.setLen(2)
+      return getModuleName(n.sons[0])
+    if n1.kind == nkPrefix and n1[0].kind == nkIdent and n1[0].ident.s == "$":
+      if n0.kind == nkIdent and n0.ident.s == "/":
+        result = lookupPackage(n1[1], n[2])
+      else:
+        localError(n.info, "only '/' supported with $package notation")
+        result = ""
+    else:
+      # hacky way to implement 'x / y /../ z':
+      result = getModuleName(n1)
+      result.add renderTree(n0, {renderNoComments})
+      result.add getModuleName(n[2])
+  of nkPrefix:
+    if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$":
+      result = lookupPackage(n[1], nil)
+    else:
+      # hacky way to implement 'x / y /../ z':
+      result = renderTree(n, {renderNoComments}).replace(" ")
+  of nkDotExpr:
+    result = renderTree(n, {renderNoComments}).replace(".", "/")
+  of nkImportAs:
+    result = getModuleName(n.sons[0])
+  else:
+    localError(n.info, errGenerated, "invalid module name: '$1'" % n.renderTree)
+    result = ""
+
+proc checkModuleName*(n: PNode; doLocalError=true): int32 =
+  # This returns the full canonical path for a given module import
+  let modulename = n.getModuleName
+  let fullPath = findModule(modulename, n.info.toFullPath)
+  if fullPath.len == 0:
+    if doLocalError:
+      localError(n.info, errCannotOpenFile, modulename)
+    result = InvalidFileIDX
+  else:
+    result = fullPath.fileInfoIdx

compiler/msgs.nim

@@ -498,6 +498,9 @@ type
                                # only 8 bytes.
     line*, col*: int16
     fileIndex*: int32
+    when defined(nimpretty):
+      offsetA*, offsetB*: int
+      commentOffsetA*, commentOffsetB*: int
 
   TErrorOutput* = enum
     eStdOut

compiler/nimlexbase.nim

@@ -46,6 +46,7 @@ type
                               # private data:
     sentinel*: int
     lineStart*: int           # index of last line start in buffer
+    offsetBase*: int          # use ``offsetBase + bufpos`` to get the offset
 
 
 proc openBaseLexer*(L: var TBaseLexer, inputstream: PLLStream,
@@ -122,7 +123,8 @@ proc fillBaseLexer(L: var TBaseLexer, pos: int): int =
     result = pos + 1          # nothing to do
   else:
     fillBuffer(L)
-    L.bufpos = 0              # XXX: is this really correct?
+    L.offsetBase += pos + 1
+    L.bufpos = 0
     result = 0
   L.lineStart = result
 
@@ -146,6 +148,7 @@ proc skipUTF8BOM(L: var TBaseLexer) =
 proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) =
   assert(bufLen > 0)
   L.bufpos = 0
+  L.offsetBase = 0
   L.bufLen = bufLen
   L.buf = cast[cstring](alloc(bufLen * chrSize))
   L.sentinel = bufLen - 1

compiler/nversion.nim

@@ -13,5 +13,5 @@
 const
   MaxSetElements* = 1 shl 16  # (2^16) to support unicode character sets?
   VersionAsString* = system.NimVersion
-  RodFileVersion* = "1222"       # modify this if the rod-format changes!
+  RodFileVersion* = "1223"       # modify this if the rod-format changes!
 

compiler/options.nim

@@ -144,6 +144,7 @@ var
   gPreciseStack*: bool = false
   gNoNimblePath* = false
   gExperimentalMode*: bool
+  newDestructors*: bool
 
 proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools}
 proc usesNativeGC*(): bool {.inline.} = gSelectedGC >= gcRefc

compiler/parser.nim

@@ -28,7 +28,7 @@ import
   llstream, lexer, idents, strutils, ast, astalgo, msgs
 
 type
-  TParser*{.final.} = object   # A TParser object represents a file that
+  TParser* = object            # A TParser object represents a file that
                                # is being parsed
     currInd: int               # current indentation level
     firstTok, strongSpaces: bool # Has the first token been read?
@@ -1905,7 +1905,7 @@ proc parseVariable(p: var TParser): PNode =
   #| variable = (varTuple / identColonEquals) colonBody? indAndComment
   if p.tok.tokType == tkParLe: result = parseVarTuple(p)
   else: result = parseIdentColonEquals(p, {withPragma, withDot})
-  result{-1} = postExprBlocks(p, result{-1})
+  result[^1] = postExprBlocks(p, result[^1])
   indAndComment(p, result)
 
 proc parseBind(p: var TParser, k: TNodeKind): PNode =

compiler/passes.nim

@@ -15,6 +15,7 @@ import
   condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
   nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder
 
+
 type
   TPassContext* = object of RootObj # the pass's context
     fromCache*: bool  # true if created by "openCached"
@@ -211,7 +212,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
             if n.kind == nkEmpty: break
             sl.add n
           if sfReorder in module.flags:
-            sl = reorder sl
+            sl = reorder(graph, sl, module, cache)
           discard processTopLevelStmt(sl, a)
           break
         elif not processTopLevelStmt(n, a): break

compiler/pragmas.nim

@@ -25,7 +25,7 @@ const
     wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC,
     wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
     wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe,
-    wOverride, wConstructor, wExportNims, wUsed}
+    wOverride, wConstructor, wExportNims, wUsed, wLiftLocals}
   converterPragmas* = procPragmas
   methodPragmas* = procPragmas+{wBase}-{wImportCpp}
   templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty,
@@ -55,7 +55,7 @@ const
     wPure, wHeader, wCompilerproc, wFinal, wSize, wExtern, wShallow,
     wImportCpp, wImportObjC, wError, wIncompleteStruct, wByCopy, wByRef,
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
-    wBorrow, wGcSafe, wExportNims, wPartial, wUsed, wExplain}
+    wBorrow, wGcSafe, wExportNims, wPartial, wUsed, wExplain, wPackage}
   fieldPragmas* = {wImportc, wExportc, wDeprecated, wExtern,
     wImportCpp, wImportObjC, wError, wGuard, wBitsize, wUsed}
   varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl,
@@ -70,6 +70,14 @@ const
                       wThread, wRaises, wLocks, wTags, wGcSafe}
   allRoutinePragmas* = methodPragmas + iteratorPragmas + lambdaPragmas
 
+proc getPragmaVal*(procAst: PNode; name: TSpecialWord): PNode =
+  let p = procAst[pragmasPos]
+  if p.kind == nkEmpty: return nil
+  for it in p:
+    if it.kind == nkExprColonExpr and it[0].kind == nkIdent and
+        it[0].ident.id == ord(name):
+      return it[1]
+
 proc pragma*(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords)
 # implementation
 
@@ -799,6 +807,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         noVal(it)
         if sym.typ == nil or tfFinal in sym.typ.flags: invalidPragma(it)
         else: incl(sym.typ.flags, tfInheritable)
+      of wPackage:
+        noVal(it)
+        if sym.typ == nil: invalidPragma(it)
+        else: incl(sym.flags, sfForward)
       of wAcyclic:
         noVal(it)
         if sym.typ == nil: invalidPragma(it)
@@ -974,6 +986,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         noVal(it)
         if sym == nil: invalidPragma(it)
         else: sym.flags.incl sfUsed
+      of wLiftLocals: discard
       else: invalidPragma(it)
     else: invalidPragma(it)
 

compiler/renderer.nim

@@ -17,12 +17,12 @@ type
     renderNone, renderNoBody, renderNoComments, renderDocComments,
     renderNoPragmas, renderIds, renderNoProcDefs
   TRenderFlags* = set[TRenderFlag]
-  TRenderTok*{.final.} = object
+  TRenderTok* = object
     kind*: TTokType
     length*: int16
 
   TRenderTokSeq* = seq[TRenderTok]
-  TSrcGen*{.final.} = object
+  TSrcGen* = object
     indent*: int
     lineLen*: int
     pos*: int              # current position for iteration over the buffer
@@ -37,15 +37,11 @@ type
     inGenericParams: bool
     checkAnon: bool        # we're in a context that can contain sfAnon
     inPragma: int
+    when defined(nimpretty):
+      pendingNewlineCount: int
+      origContent: string
 
 
-proc renderModule*(n: PNode, filename: string, renderFlags: TRenderFlags = {})
-proc renderTree*(n: PNode, renderFlags: TRenderFlags = {}): string
-proc initTokRender*(r: var TSrcGen, n: PNode, renderFlags: TRenderFlags = {})
-proc getNextTok*(r: var TSrcGen, kind: var TTokType, literal: var string)
-
-proc `$`*(n: PNode): string = n.renderTree
-# implementation
 # We render the source code in a two phases: The first
 # determines how long the subtree will likely be, the second
 # phase appends to a buffer that will be the output.
@@ -67,9 +63,31 @@ proc renderDefinitionName*(s: PSym, noQuotes = false): string =
   else:
     result = '`' & x & '`'
 
+when not defined(nimpretty):
   const
     IndentWidth = 2
-  longIndentWid = 4
+    longIndentWid = IndentWidth * 2
+else:
+  template IndentWidth: untyped = lexer.gIndentationWidth
+  template longIndentWid: untyped = IndentWidth() * 2
+
+  proc minmaxLine(n: PNode): (int, int) =
+    case n.kind
+    of nkTripleStrLit:
+      result = (n.info.line.int, n.info.line.int + countLines(n.strVal))
+    of nkCommentStmt:
+      result = (n.info.line.int, n.info.line.int + countLines(n.comment))
+    else:
+      result = (n.info.line.int, n.info.line.int)
+    for i in 0 ..< safeLen(n):
+      let (currMin, currMax) = minmaxLine(n[i])
+      if currMin < result[0]: result[0] = currMin
+      if currMax > result[1]: result[1] = currMax
+
+  proc lineDiff(a, b: PNode): int =
+    result = minmaxLine(b)[0] - minmaxLine(a)[1]
+
+const
   MaxLineLen = 80
   LineCommentColumn = 30
 
@@ -95,7 +113,11 @@ proc addTok(g: var TSrcGen, kind: TTokType, s: string) =
 
 proc addPendingNL(g: var TSrcGen) =
   if g.pendingNL >= 0:
-    addTok(g, tkSpaces, "\n" & spaces(g.pendingNL))
+    when defined(nimpretty):
+      let newlines = repeat("\n", clamp(g.pendingNewlineCount, 1, 3))
+    else:
+      const newlines = "\n"
+    addTok(g, tkSpaces, newlines & spaces(g.pendingNL))
     g.lineLen = g.pendingNL
     g.pendingNL = - 1
     g.pendingWhitespace = -1
@@ -119,11 +141,17 @@ proc putNL(g: var TSrcGen) =
 
 proc optNL(g: var TSrcGen, indent: int) =
   g.pendingNL = indent
-  g.lineLen = indent          # BUGFIX
+  g.lineLen = indent
+  when defined(nimpretty): g.pendingNewlineCount = 0
 
 proc optNL(g: var TSrcGen) =
   optNL(g, g.indent)
 
+proc optNL(g: var TSrcGen; a, b: PNode) =
+  g.pendingNL = g.indent
+  g.lineLen = g.indent
+  when defined(nimpretty): g.pendingNewlineCount = lineDiff(a, b)
+
 proc indentNL(g: var TSrcGen) =
   inc(g.indent, IndentWidth)
   g.pendingNL = g.indent
@@ -284,8 +312,8 @@ proc gcoms(g: var TSrcGen) =
   for i in countup(0, high(g.comStack)): gcom(g, g.comStack[i])
   popAllComs(g)
 
-proc lsub(n: PNode): int
+proc lsub(g: TSrcGen; n: PNode): int
-proc litAux(n: PNode, x: BiggestInt, size: int): string =
+proc litAux(g: TSrcGen; n: PNode, x: BiggestInt, size: int): string =
   proc skip(t: PType): PType =
     result = t
     while result.kind in {tyGenericInst, tyRange, tyVar, tyDistinct,
@@ -302,14 +330,23 @@ proc litAux(n: PNode, x: BiggestInt, size: int): string =
   elif nfBase16 in n.flags: result = "0x" & toHex(x, size * 2)
   else: result = $x
 
-proc ulitAux(n: PNode, x: BiggestInt, size: int): string =
+proc ulitAux(g: TSrcGen; n: PNode, x: BiggestInt, size: int): string =
   if nfBase2 in n.flags: result = "0b" & toBin(x, size * 8)
   elif nfBase8 in n.flags: result = "0o" & toOct(x, size * 3)
   elif nfBase16 in n.flags: result = "0x" & toHex(x, size * 2)
   else: result = $x
   # XXX proper unsigned output!
 
-proc atom(n: PNode): string =
+proc atom(g: TSrcGen; n: PNode): string =
+  when defined(nimpretty):
+    let comment = if n.info.commentOffsetA < n.info.commentOffsetB:
+                    " " & substr(g.origContent, n.info.commentOffsetA, n.info.commentOffsetB)
+                  else:
+                    ""
+    if n.info.offsetA <= n.info.offsetB:
+      # for some constructed tokens this can not be the case and we're better
+      # off to not mess with the offset then.
+      return substr(g.origContent, n.info.offsetA, n.info.offsetB) & comment
   var f: float32
   case n.kind
   of nkEmpty: result = ""
@@ -319,30 +356,30 @@ proc atom(n: PNode): string =
   of nkRStrLit: result = "r\"" & replace(n.strVal, "\"", "\"\"")  & '\"'
   of nkTripleStrLit: result = "\"\"\"" & n.strVal & "\"\"\""
   of nkCharLit: result = '\'' & toNimChar(chr(int(n.intVal))) & '\''
-  of nkIntLit: result = litAux(n, n.intVal, 4)
+  of nkIntLit: result = litAux(g, n, n.intVal, 4)
-  of nkInt8Lit: result = litAux(n, n.intVal, 1) & "\'i8"
+  of nkInt8Lit: result = litAux(g, n, n.intVal, 1) & "\'i8"
-  of nkInt16Lit: result = litAux(n, n.intVal, 2) & "\'i16"
+  of nkInt16Lit: result = litAux(g, n, n.intVal, 2) & "\'i16"
-  of nkInt32Lit: result = litAux(n, n.intVal, 4) & "\'i32"
+  of nkInt32Lit: result = litAux(g, n, n.intVal, 4) & "\'i32"
-  of nkInt64Lit: result = litAux(n, n.intVal, 8) & "\'i64"
+  of nkInt64Lit: result = litAux(g, n, n.intVal, 8) & "\'i64"
-  of nkUIntLit: result = ulitAux(n, n.intVal, 4) & "\'u"
+  of nkUIntLit: result = ulitAux(g, n, n.intVal, 4) & "\'u"
-  of nkUInt8Lit: result = ulitAux(n, n.intVal, 1) & "\'u8"
+  of nkUInt8Lit: result = ulitAux(g, n, n.intVal, 1) & "\'u8"
-  of nkUInt16Lit: result = ulitAux(n, n.intVal, 2) & "\'u16"
+  of nkUInt16Lit: result = ulitAux(g, n, n.intVal, 2) & "\'u16"
-  of nkUInt32Lit: result = ulitAux(n, n.intVal, 4) & "\'u32"
+  of nkUInt32Lit: result = ulitAux(g, n, n.intVal, 4) & "\'u32"
-  of nkUInt64Lit: result = ulitAux(n, n.intVal, 8) & "\'u64"
+  of nkUInt64Lit: result = ulitAux(g, n, n.intVal, 8) & "\'u64"
   of nkFloatLit:
     if n.flags * {nfBase2, nfBase8, nfBase16} == {}: result = $(n.floatVal)
-    else: result = litAux(n, (cast[PInt64](addr(n.floatVal)))[] , 8)
+    else: result = litAux(g, n, (cast[PInt64](addr(n.floatVal)))[] , 8)
   of nkFloat32Lit:
     if n.flags * {nfBase2, nfBase8, nfBase16} == {}:
       result = $n.floatVal & "\'f32"
     else:
       f = n.floatVal.float32
-      result = litAux(n, (cast[PInt32](addr(f)))[], 4) & "\'f32"
+      result = litAux(g, n, (cast[PInt32](addr(f)))[], 4) & "\'f32"
   of nkFloat64Lit:
     if n.flags * {nfBase2, nfBase8, nfBase16} == {}:
       result = $n.floatVal & "\'f64"
     else:
-      result = litAux(n, (cast[PInt64](addr(n.floatVal)))[], 8) & "\'f64"
+      result = litAux(g, n, (cast[PInt64](addr(n.floatVal)))[], 8) & "\'f64"
   of nkNilLit: result = "nil"
   of nkType:
     if (n.typ != nil) and (n.typ.sym != nil): result = n.typ.sym.name.s
@@ -351,21 +388,21 @@ proc atom(n: PNode): string =
     internalError("rnimsyn.atom " & $n.kind)
     result = ""
 
-proc lcomma(n: PNode, start: int = 0, theEnd: int = - 1): int =
+proc lcomma(g: TSrcGen; n: PNode, start: int = 0, theEnd: int = - 1): int =
   assert(theEnd < 0)
   result = 0
   for i in countup(start, sonsLen(n) + theEnd):
-    inc(result, lsub(n.sons[i]))
+    inc(result, lsub(g, n.sons[i]))
     inc(result, 2)            # for ``, ``
   if result > 0:
     dec(result, 2)            # last does not get a comma!
 
-proc lsons(n: PNode, start: int = 0, theEnd: int = - 1): int =
+proc lsons(g: TSrcGen; n: PNode, start: int = 0, theEnd: int = - 1): int =
   assert(theEnd < 0)
   result = 0
-  for i in countup(start, sonsLen(n) + theEnd): inc(result, lsub(n.sons[i]))
+  for i in countup(start, sonsLen(n) + theEnd): inc(result, lsub(g, n.sons[i]))
 
-proc lsub(n: PNode): int =
+proc lsub(g: TSrcGen; n: PNode): int =
   # computes the length of a tree
   if isNil(n): return 0
   if n.comment != nil: return MaxLineLen + 1
@@ -373,108 +410,108 @@ proc lsub(n: PNode): int =
   of nkEmpty: result = 0
   of nkTripleStrLit:
     if containsNL(n.strVal): result = MaxLineLen + 1
-    else: result = len(atom(n))
+    else: result = len(atom(g, n))
   of succ(nkEmpty)..pred(nkTripleStrLit), succ(nkTripleStrLit)..nkNilLit:
-    result = len(atom(n))
+    result = len(atom(g, n))
   of nkCall, nkBracketExpr, nkCurlyExpr, nkConv, nkPattern, nkObjConstr:
-    result = lsub(n.sons[0]) + lcomma(n, 1) + 2
+    result = lsub(g, n.sons[0]) + lcomma(g, n, 1) + 2
-  of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: result = lsub(n[1])
+  of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: result = lsub(g, n[1])
-  of nkCast: result = lsub(n.sons[0]) + lsub(n.sons[1]) + len("cast[]()")
+  of nkCast: result = lsub(g, n.sons[0]) + lsub(g, n.sons[1]) + len("cast[]()")
-  of nkAddr: result = (if n.len>0: lsub(n.sons[0]) + len("addr()") else: 4)
+  of nkAddr: result = (if n.len>0: lsub(g, n.sons[0]) + len("addr()") else: 4)
-  of nkStaticExpr: result = lsub(n.sons[0]) + len("static_")
+  of nkStaticExpr: result = lsub(g, n.sons[0]) + len("static_")
-  of nkHiddenAddr, nkHiddenDeref: result = lsub(n.sons[0])
+  of nkHiddenAddr, nkHiddenDeref: result = lsub(g, n.sons[0])
-  of nkCommand: result = lsub(n.sons[0]) + lcomma(n, 1) + 1
+  of nkCommand: result = lsub(g, n.sons[0]) + lcomma(g, n, 1) + 1
-  of nkExprEqExpr, nkAsgn, nkFastAsgn: result = lsons(n) + 3
+  of nkExprEqExpr, nkAsgn, nkFastAsgn: result = lsons(g, n) + 3
-  of nkPar, nkCurly, nkBracket, nkClosure: result = lcomma(n) + 2
+  of nkPar, nkCurly, nkBracket, nkClosure: result = lcomma(g, n) + 2
-  of nkArgList: result = lcomma(n)
+  of nkArgList: result = lcomma(g, n)
   of nkTableConstr:
-    result = if n.len > 0: lcomma(n) + 2 else: len("{:}")
+    result = if n.len > 0: lcomma(g, n) + 2 else: len("{:}")
   of nkClosedSymChoice, nkOpenSymChoice:
-    result = lsons(n) + len("()") + sonsLen(n) - 1
+    result = lsons(g, n) + len("()") + sonsLen(n) - 1
-  of nkTupleTy: result = lcomma(n) + len("tuple[]")
+  of nkTupleTy: result = lcomma(g, n) + len("tuple[]")
   of nkTupleClassTy: result = len("tuple")
-  of nkDotExpr: result = lsons(n) + 1
+  of nkDotExpr: result = lsons(g, n) + 1
-  of nkBind: result = lsons(n) + len("bind_")
+  of nkBind: result = lsons(g, n) + len("bind_")
-  of nkBindStmt: result = lcomma(n) + len("bind_")
+  of nkBindStmt: result = lcomma(g, n) + len("bind_")
-  of nkMixinStmt: result = lcomma(n) + len("mixin_")
+  of nkMixinStmt: result = lcomma(g, n) + len("mixin_")
-  of nkCheckedFieldExpr: result = lsub(n.sons[0])
+  of nkCheckedFieldExpr: result = lsub(g, n.sons[0])
-  of nkLambda: result = lsons(n) + len("proc__=_")
+  of nkLambda: result = lsons(g, n) + len("proc__=_")
-  of nkDo: result = lsons(n) + len("do__:_")
+  of nkDo: result = lsons(g, n) + len("do__:_")
   of nkConstDef, nkIdentDefs:
-    result = lcomma(n, 0, - 3)
+    result = lcomma(g, n, 0, - 3)
     var L = sonsLen(n)
-    if n.sons[L - 2].kind != nkEmpty: result = result + lsub(n.sons[L - 2]) + 2
+    if n.sons[L - 2].kind != nkEmpty: result = result + lsub(g, n.sons[L - 2]) + 2
-    if n.sons[L - 1].kind != nkEmpty: result = result + lsub(n.sons[L - 1]) + 3
+    if n.sons[L - 1].kind != nkEmpty: result = result + lsub(g, n.sons[L - 1]) + 3
-  of nkVarTuple: result = lcomma(n, 0, - 3) + len("() = ") + lsub(lastSon(n))
+  of nkVarTuple: result = lcomma(g, n, 0, - 3) + len("() = ") + lsub(g, lastSon(n))
-  of nkChckRangeF: result = len("chckRangeF") + 2 + lcomma(n)
+  of nkChckRangeF: result = len("chckRangeF") + 2 + lcomma(g, n)
-  of nkChckRange64: result = len("chckRange64") + 2 + lcomma(n)
+  of nkChckRange64: result = len("chckRange64") + 2 + lcomma(g, n)
-  of nkChckRange: result = len("chckRange") + 2 + lcomma(n)
+  of nkChckRange: result = len("chckRange") + 2 + lcomma(g, n)
   of nkObjDownConv, nkObjUpConv, nkStringToCString, nkCStringToString:
     result = 2
-    if sonsLen(n) >= 1: result = result + lsub(n.sons[0])
+    if sonsLen(n) >= 1: result = result + lsub(g, n.sons[0])
-    result = result + lcomma(n, 1)
+    result = result + lcomma(g, n, 1)
-  of nkExprColonExpr: result = lsons(n) + 2
+  of nkExprColonExpr: result = lsons(g, n) + 2
-  of nkInfix: result = lsons(n) + 2
+  of nkInfix: result = lsons(g, n) + 2
   of nkPrefix:
-    result = lsons(n)+1+(if n.len > 0 and n.sons[1].kind == nkInfix: 2 else: 0)
+    result = lsons(g, n)+1+(if n.len > 0 and n.sons[1].kind == nkInfix: 2 else: 0)
-  of nkPostfix: result = lsons(n)
+  of nkPostfix: result = lsons(g, n)
-  of nkCallStrLit: result = lsons(n)
+  of nkCallStrLit: result = lsons(g, n)
-  of nkPragmaExpr: result = lsub(n.sons[0]) + lcomma(n, 1)
+  of nkPragmaExpr: result = lsub(g, n.sons[0]) + lcomma(g, n, 1)
-  of nkRange: result = lsons(n) + 2
+  of nkRange: result = lsons(g, n) + 2
-  of nkDerefExpr: result = lsub(n.sons[0]) + 2
+  of nkDerefExpr: result = lsub(g, n.sons[0]) + 2
-  of nkAccQuoted: result = lsons(n) + 2
+  of nkAccQuoted: result = lsons(g, n) + 2
   of nkIfExpr:
-    result = lsub(n.sons[0].sons[0]) + lsub(n.sons[0].sons[1]) + lsons(n, 1) +
+    result = lsub(g, n.sons[0].sons[0]) + lsub(g, n.sons[0].sons[1]) + lsons(g, n, 1) +
         len("if_:_")
-  of nkElifExpr: result = lsons(n) + len("_elif_:_")
+  of nkElifExpr: result = lsons(g, n) + len("_elif_:_")
-  of nkElseExpr: result = lsub(n.sons[0]) + len("_else:_") # type descriptions
+  of nkElseExpr: result = lsub(g, n.sons[0]) + len("_else:_") # type descriptions
-  of nkTypeOfExpr: result = (if n.len > 0: lsub(n.sons[0]) else: 0)+len("type()")
+  of nkTypeOfExpr: result = (if n.len > 0: lsub(g, n.sons[0]) else: 0)+len("type()")
-  of nkRefTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("ref")
+  of nkRefTy: result = (if n.len > 0: lsub(g, n.sons[0])+1 else: 0) + len("ref")
-  of nkPtrTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("ptr")
+  of nkPtrTy: result = (if n.len > 0: lsub(g, n.sons[0])+1 else: 0) + len("ptr")
-  of nkVarTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("var")
+  of nkVarTy: result = (if n.len > 0: lsub(g, n.sons[0])+1 else: 0) + len("var")
   of nkDistinctTy:
-    result = len("distinct") + (if n.len > 0: lsub(n.sons[0])+1 else: 0)
+    result = len("distinct") + (if n.len > 0: lsub(g, n.sons[0])+1 else: 0)
     if n.len > 1:
       result += (if n[1].kind == nkWith: len("_with_") else: len("_without_"))
-      result += lcomma(n[1])
+      result += lcomma(g, n[1])
-  of nkStaticTy: result = (if n.len > 0: lsub(n.sons[0]) else: 0) +
+  of nkStaticTy: result = (if n.len > 0: lsub(g, n.sons[0]) else: 0) +
                                                          len("static[]")
-  of nkTypeDef: result = lsons(n) + 3
+  of nkTypeDef: result = lsons(g, n) + 3
-  of nkOfInherit: result = lsub(n.sons[0]) + len("of_")
+  of nkOfInherit: result = lsub(g, n.sons[0]) + len("of_")
-  of nkProcTy: result = lsons(n) + len("proc_")
+  of nkProcTy: result = lsons(g, n) + len("proc_")
-  of nkIteratorTy: result = lsons(n) + len("iterator_")
+  of nkIteratorTy: result = lsons(g, n) + len("iterator_")
-  of nkSharedTy: result = lsons(n) + len("shared_")
+  of nkSharedTy: result = lsons(g, n) + len("shared_")
   of nkEnumTy:
     if sonsLen(n) > 0:
-      result = lsub(n.sons[0]) + lcomma(n, 1) + len("enum_")
+      result = lsub(g, n.sons[0]) + lcomma(g, n, 1) + len("enum_")
     else:
       result = len("enum")
-  of nkEnumFieldDef: result = lsons(n) + 3
+  of nkEnumFieldDef: result = lsons(g, n) + 3
   of nkVarSection, nkLetSection:
     if sonsLen(n) > 1: result = MaxLineLen + 1
-    else: result = lsons(n) + len("var_")
+    else: result = lsons(g, n) + len("var_")
   of nkUsingStmt:
     if sonsLen(n) > 1: result = MaxLineLen + 1
-    else: result = lsons(n) + len("using_")
+    else: result = lsons(g, n) + len("using_")
-  of nkReturnStmt: result = lsub(n.sons[0]) + len("return_")
+  of nkReturnStmt: result = lsub(g, n.sons[0]) + len("return_")
-  of nkRaiseStmt: result = lsub(n.sons[0]) + len("raise_")
+  of nkRaiseStmt: result = lsub(g, n.sons[0]) + len("raise_")
-  of nkYieldStmt: result = lsub(n.sons[0]) + len("yield_")
+  of nkYieldStmt: result = lsub(g, n.sons[0]) + len("yield_")
-  of nkDiscardStmt: result = lsub(n.sons[0]) + len("discard_")
+  of nkDiscardStmt: result = lsub(g, n.sons[0]) + len("discard_")
-  of nkBreakStmt: result = lsub(n.sons[0]) + len("break_")
+  of nkBreakStmt: result = lsub(g, n.sons[0]) + len("break_")
-  of nkContinueStmt: result = lsub(n.sons[0]) + len("continue_")
+  of nkContinueStmt: result = lsub(g, n.sons[0]) + len("continue_")
-  of nkPragma: result = lcomma(n) + 4
+  of nkPragma: result = lcomma(g, n) + 4
   of nkCommentStmt: result = if n.comment.isNil: 0 else: len(n.comment)
-  of nkOfBranch: result = lcomma(n, 0, - 2) + lsub(lastSon(n)) + len("of_:_")
+  of nkOfBranch: result = lcomma(g, n, 0, - 2) + lsub(g, lastSon(n)) + len("of_:_")
-  of nkImportAs: result = lsub(n.sons[0]) + len("_as_") + lsub(n.sons[1])
+  of nkImportAs: result = lsub(g, n.sons[0]) + len("_as_") + lsub(g, n.sons[1])
-  of nkElifBranch: result = lsons(n) + len("elif_:_")
+  of nkElifBranch: result = lsons(g, n) + len("elif_:_")
-  of nkElse: result = lsub(n.sons[0]) + len("else:_")
+  of nkElse: result = lsub(g, n.sons[0]) + len("else:_")
-  of nkFinally: result = lsub(n.sons[0]) + len("finally:_")
+  of nkFinally: result = lsub(g, n.sons[0]) + len("finally:_")
-  of nkGenericParams: result = lcomma(n) + 2
+  of nkGenericParams: result = lcomma(g, n) + 2
   of nkFormalParams:
-    result = lcomma(n, 1) + 2
+    result = lcomma(g, n, 1) + 2
-    if n.sons[0].kind != nkEmpty: result = result + lsub(n.sons[0]) + 2
+    if n.sons[0].kind != nkEmpty: result = result + lsub(g, n.sons[0]) + 2
   of nkExceptBranch:
-    result = lcomma(n, 0, -2) + lsub(lastSon(n)) + len("except_:_")
+    result = lcomma(g, n, 0, -2) + lsub(g, lastSon(n)) + len("except_:_")
   else: result = MaxLineLen + 1
 
 proc fits(g: TSrcGen, x: int): bool =
@@ -517,7 +554,7 @@ proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
                theEnd: int = - 1, separator = tkComma) =
   for i in countup(start, sonsLen(n) + theEnd):
     var c = i < sonsLen(n) + theEnd
-    var sublen = lsub(n.sons[i]) + ord(c)
+    var sublen = lsub(g, n.sons[i]) + ord(c)
     if not fits(g, sublen) and (ind + sublen < MaxLineLen): optNL(g, ind)
     let oldLen = g.tokens.len
     gsub(g, n.sons[i])
@@ -564,12 +601,12 @@ proc gsection(g: var TSrcGen, n: PNode, c: TContext, kind: TTokType,
     gcoms(g)
   dedent(g)
 
-proc longMode(n: PNode, start: int = 0, theEnd: int = - 1): bool =
+proc longMode(g: TSrcGen; n: PNode, start: int = 0, theEnd: int = - 1): bool =
   result = n.comment != nil
   if not result:
     # check further
     for i in countup(start, sonsLen(n) + theEnd):
-      if (lsub(n.sons[i]) > MaxLineLen):
+      if (lsub(g, n.sons[i]) > MaxLineLen):
         result = true
         break
 
@@ -577,12 +614,16 @@ proc gstmts(g: var TSrcGen, n: PNode, c: TContext, doIndent=true) =
   if n.kind == nkEmpty: return
   if n.kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
     if doIndent: indentNL(g)
-    for i in countup(0, sonsLen(n) - 1):
+    let L = n.len
-      optNL(g)
+    for i in 0 .. L-1:
-      if n.sons[i].kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
+      if i > 0:
-        gstmts(g, n.sons[i], c, doIndent=false)
+        optNL(g, n[i-1], n[i])
       else:
-        gsub(g, n.sons[i])
+        optNL(g)
+      if n[i].kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
+        gstmts(g, n[i], c, doIndent=false)
+      else:
+        gsub(g, n[i])
       gcoms(g)
     if doIndent: dedent(g)
   else:
@@ -597,7 +638,7 @@ proc gif(g: var TSrcGen, n: PNode) =
   gsub(g, n.sons[0].sons[0])
   initContext(c)
   putWithSpace(g, tkColon, ":")
-  if longMode(n) or (lsub(n.sons[0].sons[1]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[0].sons[1]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[0].sons[1], c)
@@ -612,7 +653,7 @@ proc gwhile(g: var TSrcGen, n: PNode) =
   gsub(g, n.sons[0])
   putWithSpace(g, tkColon, ":")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[1]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[1]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[1], c)
@@ -621,7 +662,7 @@ proc gpattern(g: var TSrcGen, n: PNode) =
   var c: TContext
   put(g, tkCurlyLe, "{")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[0]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[0]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n, c)
@@ -632,7 +673,7 @@ proc gpragmaBlock(g: var TSrcGen, n: PNode) =
   gsub(g, n.sons[0])
   putWithSpace(g, tkColon, ":")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[1]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[1]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[1], c)
@@ -642,7 +683,7 @@ proc gtry(g: var TSrcGen, n: PNode) =
   put(g, tkTry, "try")
   putWithSpace(g, tkColon, ":")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[0]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[0]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[0], c)
@@ -653,8 +694,8 @@ proc gfor(g: var TSrcGen, n: PNode) =
   var length = sonsLen(n)
   putWithSpace(g, tkFor, "for")
   initContext(c)
-  if longMode(n) or
+  if longMode(g, n) or
-      (lsub(n.sons[length - 1]) + lsub(n.sons[length - 2]) + 6 + g.lineLen >
+      (lsub(g, n.sons[length - 1]) + lsub(g, n.sons[length - 2]) + 6 + g.lineLen >
       MaxLineLen):
     incl(c.flags, rfLongMode)
   gcomma(g, n, c, 0, - 3)
@@ -670,7 +711,7 @@ proc gcase(g: var TSrcGen, n: PNode) =
   initContext(c)
   var length = sonsLen(n)
   var last = if n.sons[length-1].kind == nkElse: -2 else: -1
-  if longMode(n, 0, last): incl(c.flags, rfLongMode)
+  if longMode(g, n, 0, last): incl(c.flags, rfLongMode)
   putWithSpace(g, tkCase, "case")
   gsub(g, n.sons[0])
   gcoms(g)
@@ -678,7 +719,7 @@ proc gcase(g: var TSrcGen, n: PNode) =
   gsons(g, n, c, 1, last)
   if last == - 2:
     initContext(c)
-    if longMode(n.sons[length - 1]): incl(c.flags, rfLongMode)
+    if longMode(g, n.sons[length - 1]): incl(c.flags, rfLongMode)
     gsub(g, n.sons[length - 1], c)
 
 proc gproc(g: var TSrcGen, n: PNode) =
@@ -740,7 +781,7 @@ proc gblock(g: var TSrcGen, n: PNode) =
   else:
     put(g, tkBlock, "block")
   putWithSpace(g, tkColon, ":")
-  if longMode(n) or (lsub(n.sons[1]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[1]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)
   # XXX I don't get why this is needed here! gstmts should already handle this!
@@ -753,7 +794,7 @@ proc gstaticStmt(g: var TSrcGen, n: PNode) =
   putWithSpace(g, tkStatic, "static")
   putWithSpace(g, tkColon, ":")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[0]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[0]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[0], c)
@@ -771,7 +812,7 @@ proc gident(g: var TSrcGen, n: PNode) =
       (n.typ != nil and tfImplicitTypeParam in n.typ.flags): return
 
   var t: TTokType
-  var s = atom(n)
+  var s = atom(g, n)
   if (s[0] in lexer.SymChars):
     if (n.kind == nkIdent):
       if (n.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
@@ -818,26 +859,26 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   case n.kind                 # atoms:
   of nkTripleStrLit: putRawStr(g, tkTripleStrLit, n.strVal)
   of nkEmpty: discard
-  of nkType: put(g, tkInvalid, atom(n))
+  of nkType: put(g, tkInvalid, atom(g, n))
   of nkSym, nkIdent: gident(g, n)
-  of nkIntLit: put(g, tkIntLit, atom(n))
+  of nkIntLit: put(g, tkIntLit, atom(g, n))
-  of nkInt8Lit: put(g, tkInt8Lit, atom(n))
+  of nkInt8Lit: put(g, tkInt8Lit, atom(g, n))
-  of nkInt16Lit: put(g, tkInt16Lit, atom(n))
+  of nkInt16Lit: put(g, tkInt16Lit, atom(g, n))
-  of nkInt32Lit: put(g, tkInt32Lit, atom(n))
+  of nkInt32Lit: put(g, tkInt32Lit, atom(g, n))
-  of nkInt64Lit: put(g, tkInt64Lit, atom(n))
+  of nkInt64Lit: put(g, tkInt64Lit, atom(g, n))
-  of nkUIntLit: put(g, tkUIntLit, atom(n))
+  of nkUIntLit: put(g, tkUIntLit, atom(g, n))
-  of nkUInt8Lit: put(g, tkUInt8Lit, atom(n))
+  of nkUInt8Lit: put(g, tkUInt8Lit, atom(g, n))
-  of nkUInt16Lit: put(g, tkUInt16Lit, atom(n))
+  of nkUInt16Lit: put(g, tkUInt16Lit, atom(g, n))
-  of nkUInt32Lit: put(g, tkUInt32Lit, atom(n))
+  of nkUInt32Lit: put(g, tkUInt32Lit, atom(g, n))
-  of nkUInt64Lit: put(g, tkUInt64Lit, atom(n))
+  of nkUInt64Lit: put(g, tkUInt64Lit, atom(g, n))
-  of nkFloatLit: put(g, tkFloatLit, atom(n))
+  of nkFloatLit: put(g, tkFloatLit, atom(g, n))
-  of nkFloat32Lit: put(g, tkFloat32Lit, atom(n))
+  of nkFloat32Lit: put(g, tkFloat32Lit, atom(g, n))
-  of nkFloat64Lit: put(g, tkFloat64Lit, atom(n))
+  of nkFloat64Lit: put(g, tkFloat64Lit, atom(g, n))
-  of nkFloat128Lit: put(g, tkFloat128Lit, atom(n))
+  of nkFloat128Lit: put(g, tkFloat128Lit, atom(g, n))
-  of nkStrLit: put(g, tkStrLit, atom(n))
+  of nkStrLit: put(g, tkStrLit, atom(g, n))
-  of nkRStrLit: put(g, tkRStrLit, atom(n))
+  of nkRStrLit: put(g, tkRStrLit, atom(g, n))
-  of nkCharLit: put(g, tkCharLit, atom(n))
+  of nkCharLit: put(g, tkCharLit, atom(g, n))
-  of nkNilLit: put(g, tkNil, atom(n))    # complex expressions
+  of nkNilLit: put(g, tkNil, atom(g, n))    # complex expressions
   of nkCall, nkConv, nkDotCall, nkPattern, nkObjConstr:
     if n.len > 0 and isBracket(n[0]):
       gsub(g, n, 1)
@@ -1003,7 +1044,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     gsub(g, n, 1)
     put(g, tkSpaces, Space)
     gsub(g, n, 0)        # binary operator
-    if not fits(g, lsub(n.sons[2]) + lsub(n.sons[0]) + 1):
+    if not fits(g, lsub(g, n.sons[2]) + lsub(g, n.sons[0]) + 1):
       optNL(g, g.indent + longIndentWid)
     else:
       put(g, tkSpaces, Space)
@@ -1011,6 +1052,10 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   of nkPrefix:
     gsub(g, n, 0)
     if n.len > 1:
+      let opr = if n[0].kind == nkIdent: n[0].ident
+                elif n[0].kind == nkSym: n[0].sym.name
+                else: nil
+      if n[1].kind == nkPrefix or (opr != nil and renderer.isKeyword(opr)):
         put(g, tkSpaces, Space)
       if n.sons[1].kind == nkInfix:
         put(g, tkParLe, "(")
@@ -1316,7 +1361,10 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     gstmts(g, n.sons[0], c)
   of nkExceptBranch:
     optNL(g)
+    if n.len != 1:
       putWithSpace(g, tkExcept, "except")
+    else:
+      put(g, tkExcept, "except")
     gcomma(g, n, 0, -2)
     putWithSpace(g, tkColon, ":")
     gcoms(g)
@@ -1363,7 +1411,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     #nkNone, nkExplicitTypeListCall:
     internalError(n.info, "rnimsyn.gsub(" & $n.kind & ')')
 
-proc renderTree(n: PNode, renderFlags: TRenderFlags = {}): string =
+proc renderTree*(n: PNode, renderFlags: TRenderFlags = {}): string =
   var g: TSrcGen
   initSrcGen(g, renderFlags)
   # do not indent the initial statement list so that
@@ -1375,12 +1423,20 @@ proc renderTree(n: PNode, renderFlags: TRenderFlags = {}): string =
     gsub(g, n)
   result = g.buf
 
-proc renderModule(n: PNode, filename: string,
+proc `$`*(n: PNode): string = n.renderTree
+
+proc renderModule*(n: PNode, infile, outfile: string,
                    renderFlags: TRenderFlags = {}) =
   var
     f: File
     g: TSrcGen
   initSrcGen(g, renderFlags)
+  when defined(nimpretty):
+    try:
+      g.origContent = readFile(infile)
+    except IOError:
+      rawMessage(errCannotOpenFile, infile)
+
   for i in countup(0, sonsLen(n) - 1):
     gsub(g, n.sons[i])
     optNL(g)
@@ -1391,17 +1447,17 @@ proc renderModule(n: PNode, filename: string,
   gcoms(g)
   if optStdout in gGlobalOptions:
     write(stdout, g.buf)
-  elif open(f, filename, fmWrite):
+  elif open(f, outfile, fmWrite):
     write(f, g.buf)
     close(f)
   else:
-    rawMessage(errCannotOpenFile, filename)
+    rawMessage(errCannotOpenFile, outfile)
 
-proc initTokRender(r: var TSrcGen, n: PNode, renderFlags: TRenderFlags = {}) =
+proc initTokRender*(r: var TSrcGen, n: PNode, renderFlags: TRenderFlags = {}) =
   initSrcGen(r, renderFlags)
   gsub(r, n)
 
-proc getNextTok(r: var TSrcGen, kind: var TTokType, literal: var string) =
+proc getNextTok*(r: var TSrcGen, kind: var TTokType, literal: var string) =
   if r.idx < len(r.tokens):
     kind = r.tokens[r.idx].kind
     var length = r.tokens[r.idx].length.int

compiler/reorder.nim

@@ -1,9 +1,36 @@
 
-import intsets, tables, ast, idents, renderer
+import 
+  intsets, ast, idents, algorithm, renderer, parser, ospaths, strutils, 
+  sequtils, msgs, modulegraphs, syntaxes, options, modulepaths, tables
 
-const
+type
-  nfTempMark = nfTransf
+  DepN = ref object
-  nfPermMark = nfNoRewrite
+    pnode: PNode
+    id, idx, lowLink: int
+    onStack: bool
+    kids: seq[DepN]
+    hAQ, hIS, hB, hCmd: int
+    when not defined(release):
+      expls: seq[string]
+  DepG = seq[DepN]
+
+when not defined(release):
+  var idNames = newTable[int, string]()
+
+proc newDepN(id: int, pnode: PNode): DepN =
+  new(result)
+  result.id = id
+  result.pnode = pnode
+  result.idx = -1
+  result.lowLink = -1
+  result.onStack = false
+  result.kids = @[]
+  result.hAQ = -1
+  result.hIS = -1
+  result.hB = -1
+  result.hCmd = -1
+  when not defined(release):
+    result.expls = @[]
 
 proc accQuoted(n: PNode): PIdent =
   var id = ""
@@ -21,10 +48,19 @@ proc addDecl(n: PNode; declares: var IntSet) =
   of nkPragmaExpr: addDecl(n[0], declares)
   of nkIdent:
     declares.incl n.ident.id
+    when not defined(release):
+      idNames[n.ident.id] = n.ident.s
   of nkSym:
     declares.incl n.sym.name.id
+    when not defined(release):
+      idNames[n.sym.name.id] = n.sym.name.s
   of nkAccQuoted:
-    declares.incl accQuoted(n).id
+    let a = accQuoted(n)
+    declares.incl a.id
+    when not defined(release):
+      idNames[a.id] = a.s
+  of nkEnumFieldDef:
+    addDecl(n[0], declares)
   else: discard
 
 proc computeDeps(n: PNode, declares, uses: var IntSet; topLevel: bool) =
@@ -32,7 +68,7 @@ proc computeDeps(n: PNode, declares, uses: var IntSet; topLevel: bool) =
   template decl(n) =
     if topLevel: addDecl(n, declares)
   case n.kind
-  of procDefs:
+  of procDefs, nkMacroDef, nkTemplateDef:
     decl(n[0])
     for i in 1..bodyPos: deps(n[i])
   of nkLetSection, nkVarSection, nkUsingStmt:
@@ -44,43 +80,358 @@ proc computeDeps(n: PNode, declares, uses: var IntSet; topLevel: bool) =
     for a in n:
       if a.len >= 3:
         decl(a[0])
-        for i in 1..<a.len: deps(a[i])
+        for i in 1..<a.len:
+          if a[i].kind == nkEnumTy:
+            # declare enum members
+            for b in a[i]:
+              decl(b)
+          else:
+            deps(a[i])
+  of nkIdentDefs:
+    for i in 1..<n.len: # avoid members identifiers in object definition
+      deps(n[i])
   of nkIdent: uses.incl n.ident.id
   of nkSym: uses.incl n.sym.name.id
   of nkAccQuoted: uses.incl accQuoted(n).id
   of nkOpenSymChoice, nkClosedSymChoice:
     uses.incl n.sons[0].sym.name.id
-  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse:
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
     for i in 0..<len(n): computeDeps(n[i], declares, uses, topLevel)
+  of nkPragma:
+    let a = n.sons[0]
+    if a.kind == nkExprColonExpr and a.sons[0].kind == nkIdent and
+       a.sons[0].ident.s == "pragma":
+        # user defined pragma
+        decl(a.sons[1])
+    else:
+      for i in 0..<safeLen(n): deps(n[i])
   else:
     for i in 0..<safeLen(n): deps(n[i])
 
-proc visit(i: int; all, res: PNode; deps: var seq[(IntSet, IntSet)]): bool =
+proc cleanPath(s: string): string =
-  let n = all[i]
+  # Here paths may have the form A / B or "A/B"
-  if nfTempMark in n.flags:
+  result = ""
-    # not a DAG!
+  for c in s:
+    if c != ' ' and c != '\"':
+      result.add c
+
+proc joinPath(parts: seq[string]): string =
+  let nb = parts.len
+  assert nb > 0
+  if nb == 1:
+    return parts[0]
+  result = parts[0] / parts[1]
+  for i in 2..<parts.len:
+    result = result / parts[i]
+
+proc getIncludePath(n: PNode, modulePath: string): string =
+  let istr = n.renderTree.cleanPath
+  let (pdir, _) = modulePath.splitPath
+  let p = istr.split('/').joinPath.addFileExt("nim")
+  result = pdir / p
+
+proc hasIncludes(n:PNode): bool =
+  for a in n:
+    if a.kind == nkIncludeStmt:
       return true
-  if nfPermMark notin n.flags:
+
-    incl n.flags, nfTempMark
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: int32;
-    var uses = deps[i][1]
+                    cache: IdentCache): PNode {.procvar.} =
-    for j in 0..<all.len:
+  result = syntaxes.parseFile(fileIdx, cache)
-      if j != i:
+  graph.addDep(s, fileIdx)
+  graph.addIncludeDep(s.position.int32, fileIdx)
+
+proc expandIncludes(graph: ModuleGraph, module: PSym, n: PNode, 
+                    modulePath: string, includedFiles: var IntSet,
+                    cache: IdentCache): PNode =
+  # Parses includes and injects them in the current tree
+  if not n.hasIncludes:
+    return n
+  result = newNodeI(nkStmtList, n.info)
+  for a in n:
+    if a.kind == nkIncludeStmt:
+      for i in 0..<a.len:
+        var f = checkModuleName(a.sons[i])
+        if f != InvalidFileIDX:
+          if containsOrIncl(includedFiles, f):
+            localError(a.info, errRecursiveDependencyX, f.toFilename)
+          else:
+            let nn = includeModule(graph, module, f, cache)
+            let nnn = expandIncludes(graph, module, nn, modulePath, 
+                                      includedFiles, cache)
+            excl(includedFiles, f)
+            for b in nnn:
+              result.add b
+    else:
+      result.add a
+
+proc splitSections(n: PNode): PNode =
+  # Split typeSections and ConstSections into
+  # sections that contain only one definition
+  assert n.kind == nkStmtList
+  result = newNodeI(nkStmtList, n.info)
+  for a in n:
+    if a.kind in {nkTypeSection, nkConstSection} and a.len > 1:
+      for b in a:
+        var s = newNode(a.kind)
+        s.info = b.info
+        s.add b
+        result.add s
+    else:
+      result.add a
+
+proc haveSameKind(dns: seq[DepN]): bool =
+  # Check if all the nodes in a strongly connected
+  # component have the same kind
+  result = true
+  let kind = dns[0].pnode.kind
+  for dn in dns:
+    if dn.pnode.kind != kind:
+      return false
+
+proc mergeSections(comps: seq[seq[DepN]], res: PNode) =
+  # Merges typeSections and ConstSections when they form
+  # a strong component (ex: circular type definition)
+  for c in comps:
+    assert c.len > 0
+    if c.len == 1:
+      res.add c[0].pnode
+    else:
+      let fstn = c[0].pnode
+      let kind = fstn.kind
+      # always return to the original order when we got circular dependencies
+      let cs = c.sortedByIt(it.id)
+      if kind in {nkTypeSection, nkConstSection} and haveSameKind(cs):
+        # Circular dependency between type or const sections, we just
+        # need to merge them
+        var sn = newNode(kind)
+        for dn in cs:
+          sn.add dn.pnode.sons[0]
+        res.add sn
+      else:
+          # Problematic circular dependency, we arrange the nodes into
+          # their original relative order and make sure to re-merge
+          # consecutive type and const sections
+          var wmsg = "Circular dependency detected. reorder pragma may not be able to" &
+            " reorder some nodes properely"
+          when not defined(release):
+            wmsg &= ":\n"
+            for i in 0..<cs.len-1:
+                for j in i..<cs.len:
+                  for ci in 0..<cs[i].kids.len:
+                    if cs[i].kids[ci].id == cs[j].id:
+                      wmsg &= "line " & $cs[i].pnode.info.line &
+                        " depends on line " & $cs[j].pnode.info.line &
+                        ": " & cs[i].expls[ci] & "\n"
+            for j in 0..<cs.len-1:
+                for ci in 0..<cs[^1].kids.len:
+                  if cs[^1].kids[ci].id == cs[j].id:
+                    wmsg &= "line " & $cs[^1].pnode.info.line &
+                      " depends on line " & $cs[j].pnode.info.line &
+                      ": " & cs[^1].expls[ci] & "\n"
+          message(cs[0].pnode.info, warnUser, wmsg)
+
+          var i = 0
+          while i < cs.len:
+            if cs[i].pnode.kind in {nkTypeSection, nkConstSection}:
+              let ckind = cs[i].pnode.kind
+              var sn = newNode(ckind)
+              sn.add cs[i].pnode[0]
+              inc i
+              while i < cs.len and cs[i].pnode.kind == ckind :
+                sn.add cs[i].pnode[0]
+                inc i
+              res.add sn
+            else:
+              res.add cs[i].pnode
+              inc i
+
+proc hasImportStmt(n: PNode): bool =
+  # Checks if the node is an import statement or
+  # i it contains one
+  case n.kind
+  of nkImportStmt, nkFromStmt, nkImportExceptStmt:
+    return true
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
+    for a in n:
+      if a.hasImportStmt:
+        return true
+  else:
+    result = false
+
+proc hasImportStmt(n: DepN): bool =
+  if n.hIS < 0:
+    n.hIS = ord(n.pnode.hasImportStmt)
+  result = bool(n.hIS)
+
+proc hasCommand(n: PNode): bool =
+  # Checks if the node is a command or a call
+  # or if it contains one
+  case n.kind
+  of nkCommand, nkCall:
+    result = true
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse,
+      nkStaticStmt, nkLetSection, nkConstSection, nkVarSection,
+      nkIdentDefs:
+        for a in n:
+          if a.hasCommand:
+            return true
+  else:
+    return false
+
+proc hasCommand(n: DepN): bool =
+  if n.hCmd < 0:
+    n.hCmd = ord(n.pnode.hasCommand)
+  result = bool(n.hCmd)
+
+proc hasAccQuoted(n: PNode): bool =
+  if n.kind == nkAccQuoted:
+    return true
+  for a in n:
+    if hasAccQuoted(a):
+      return true
+
+const extandedProcDefs = procDefs + {nkMacroDef,  nkTemplateDef}
+
+proc hasAccQuotedDef(n: PNode): bool =
+  # Checks if the node is a function, macro, template ...
+  # with a quoted name or if it contains one
+  case n.kind
+  of extandedProcDefs:
+    result = n[0].hasAccQuoted
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
+    for a in n:
+      if a.hasAccQuotedDef:
+        return true
+  else:
+    result = false
+
+proc hasAccQuotedDef(n: DepN): bool =
+  if n.hAQ < 0:
+    n.hAQ = ord(n.pnode.hasAccQuotedDef)
+  result = bool(n.hAQ)
+
+proc hasBody(n: PNode): bool =
+  # Checks if the node is a function, macro, template ...
+  # with a body or if it contains one
+  case n.kind
+  of nkCommand, nkCall:
+    result = true
+  of extandedProcDefs:
+    result = n[^1].kind == nkStmtList
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
+    for a in n:
+      if a.hasBody:
+        return true
+  else:
+    result = false
+
+proc hasBody(n: DepN): bool =
+  if n.hB < 0:
+    n.hB = ord(n.pnode.hasBody)
+  result = bool(n.hB)
+
+proc intersects(s1, s2: IntSet): bool =
+  for a in s1:
+    if s2.contains(a):
+      return true
+
+proc buildGraph(n: PNode, deps: seq[(IntSet, IntSet)]): DepG =
+  # Build a dependency graph
+  result = newSeqOfCap[DepN](deps.len)
+  for i in 0..<deps.len:
+    result.add newDepN(i, n.sons[i])
+  for i in 0..<deps.len:
+    var ni = result[i]
+    let uses = deps[i][1]
+    let niHasBody = ni.hasBody
+    let niHasCmd = ni.hasCommand
+    for j in 0..<deps.len:
+      if i == j: continue
+      var nj = result[j]
       let declares = deps[j][0]
+      if j < i and nj.hasCommand and niHasCmd:
+        # Preserve order for commands and calls
+        ni.kids.add nj
+        when not defined(release):
+          ni.expls.add "both have commands and one comes after the other"
+      elif j < i and nj.hasImportStmt:
+        # Every node that comes after an import statement must
+        # depend on that import
+        ni.kids.add nj
+        when not defined(release):
+          ni.expls.add "parent is, or contains, an import statement and child comes after it"
+      elif j < i and niHasBody and nj.hasAccQuotedDef:
+        # Every function, macro, template... with a body depends
+        # on precedent function declarations that have quoted names.
+        # That's because it is hard to detect the use of functions
+        # like "[]=", "[]", "or" ... in their bodies.
+        ni.kids.add nj
+        when not defined(release):
+          ni.expls.add "one declares a quoted identifier and the other has a body and comes after it"
+      elif j < i and niHasBody and not nj.hasBody and
+        intersects(deps[i][0], declares):
+          # Keep function declaration before function definition
+          ni.kids.add nj
+          when not defined(release):
+            for dep in deps[i][0]:
+              if dep in declares:
+                ni.expls.add "one declares \"" & idNames[dep] & "\" and the other defines it"
+      else:
         for d in declares:
           if uses.contains(d):
-            let oldLen = res.len
+            ni.kids.add nj
-            if visit(j, all, res, deps):
+            when not defined(release):
-              result = true
+              ni.expls.add "one declares \"" & idNames[d] & "\" and the other uses it"
-              # rollback what we did, it turned out to be a dependency that caused
-              # trouble:
-              for k in oldLen..<res.len:
-                res.sons[k].flags = res.sons[k].flags - {nfPermMark, nfTempMark}
-              if oldLen != res.len: res.sons.setLen oldLen
-            break
-    n.flags = n.flags + {nfPermMark} - {nfTempMark}
-    res.add n
 
-proc reorder*(n: PNode): PNode =
+proc strongConnect(v: var DepN, idx: var int, s: var seq[DepN],
+                   res: var seq[seq[DepN]]) =
+  # Recursive part of trajan's algorithm
+  v.idx = idx
+  v.lowLink = idx
+  inc idx
+  s.add v
+  v.onStack = true
+  for w in v.kids.mitems:
+    if w.idx < 0:
+      strongConnect(w, idx, s, res)
+      v.lowLink = min(v.lowLink, w.lowLink)
+    elif w.onStack:
+      v.lowLink = min(v.lowLink, w.idx)
+  if v.lowLink == v.idx:
+    var comp = newSeq[DepN]()
+    while true:
+      var w = s.pop
+      w.onStack = false
+      comp.add w
+      if w.id == v.id: break
+    res.add comp
+
+proc getStrongComponents(g: var DepG): seq[seq[DepN]] =
+  ## Tarjan's algorithm. Performs a topological sort
+  ## and detects strongly connected components.
+  result = newSeq[seq[DepN]]()
+  var s = newSeq[DepN]()
+  var idx = 0
+  for v in g.mitems:
+    if v.idx < 0:
+      strongConnect(v, idx, s, result)
+
+proc hasForbiddenPragma(n: PNode): bool =
+  # Checks if the tree node has some pragmas that do not
+  # play well with reordering, like the push/pop pragma
+  for a in n:
+    if a.kind == nkPragma and a[0].kind == nkIdent and
+        a[0].ident.s == "push":
+          return true
+
+proc reorder*(graph: ModuleGraph, n: PNode, module: PSym, cache: IdentCache): PNode =
+  if n.hasForbiddenPragma:
+    return n
+  var includedFiles = initIntSet()
+  let mpath = module.fileIdx.toFullPath
+  let n = expandIncludes(graph, module, n, mpath, 
+                          includedFiles, cache).splitSections
   result = newNodeI(nkStmtList, n.info)
   var deps = newSeq[(IntSet, IntSet)](n.len)
   for i in 0..<n.len:
@@ -88,15 +439,6 @@ proc reorder*(n: PNode): PNode =
     deps[i][1] = initIntSet()
     computeDeps(n[i], deps[i][0], deps[i][1], true)
 
-  for i in 0 .. n.len-1:
+  var g = buildGraph(n, deps)
-    discard visit(i, n, result, deps)
+  let comps = getStrongComponents(g)
-  for i in 0..<result.len:
+  mergeSections(comps, result)
-    result.sons[i].flags = result.sons[i].flags - {nfTempMark, nfPermMark}
-  when false:
-    # reverse the result:
-    let L = result.len-1
-    for i in 0 .. result.len div 2:
-      result.sons[i].flags = result.sons[i].flags - {nfTempMark, nfPermMark}
-      result.sons[L - i].flags = result.sons[L - i].flags - {nfTempMark, nfPermMark}
-      swap(result.sons[i], result.sons[L - i])
-  #echo result

compiler/rodread.nim

@@ -336,10 +336,13 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType =
   if r.s[r.pos] == '\17':
     inc(r.pos)
     result.assignment = rrGetSym(r, decodeVInt(r.s, r.pos), info)
-  while r.s[r.pos] == '\18':
+  if r.s[r.pos] == '\18':
+    inc(r.pos)
+    result.sink = rrGetSym(r, decodeVInt(r.s, r.pos), info)
+  while r.s[r.pos] == '\19':
     inc(r.pos)
     let x = decodeVInt(r.s, r.pos)
-    doAssert r.s[r.pos] == '\19'
+    doAssert r.s[r.pos] == '\20'
     inc(r.pos)
     let y = rrGetSym(r, decodeVInt(r.s, r.pos), info)
     result.methods.safeAdd((x, y))

compiler/rodutils.nim

@@ -10,7 +10,7 @@
 ## Serialization utilities for the compiler.
 import strutils
 
-proc c_sprintf(buf, frmt: cstring) {.importc: "sprintf", header: "<stdio.h>", nodecl, varargs.}
+proc c_snprintf(s: cstring; n:uint; frmt: cstring): cint {.importc: "snprintf", header: "<stdio.h>", nodecl, varargs.}
 
 proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
   if f != f:
@@ -20,10 +20,15 @@ proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
   elif f == 0.5 * f:
     if f > 0.0: result = "INF"
     else: result = "-INF"
+  else:
+    when defined(nimNoArrayToCstringConversion):
+      result = newString(81)
+      let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
+      setLen(result, n)
     else:
       var buf: array[0..80, char]
-    c_sprintf(buf, "%#.16e" & literalPostfix, f)
+      discard c_snprintf(buf.cstring, buf.len.uint, "%#.16e%s", f, literalPostfix.cstring)
-    result = $buf
+      result = $buf.cstring
 
 proc encodeStr*(s: string, result: var string) =
   for i in countup(0, len(s) - 1):
@@ -133,4 +138,3 @@ iterator decodeStrArray*(s: cstring): string =
   while s[i] != '\0':
     yield decodeStr(s, i)
     if s[i] == ' ': inc i
-

compiler/rodwrite.nim

@@ -13,8 +13,8 @@
 
 import
   intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform,
-  condsyms, ropes, idents, securehash, rodread, passes, importer, idgen,
+  condsyms, ropes, idents, securehash, rodread, passes, idgen,
-  rodutils
+  rodutils, modulepaths
 
 from modulegraphs import ModuleGraph
 
@@ -245,10 +245,14 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
     add(result, '\17')
     encodeVInt(t.assignment.id, result)
     pushSym(w, t.assignment)
-  for i, s in items(t.methods):
+  if t.sink != nil:
     add(result, '\18')
-    encodeVInt(i, result)
+    encodeVInt(t.sink.id, result)
+    pushSym(w, t.sink)
+  for i, s in items(t.methods):
     add(result, '\19')
+    encodeVInt(i, result)
+    add(result, '\20')
     encodeVInt(s.id, result)
     pushSym(w, s)
   encodeLoc(w, t.loc, result)

compiler/scriptconfig.nim

@@ -79,7 +79,7 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
     setResult(a, osproc.execCmd getString(a, 0))
 
   cbconf getEnv:
-    setResult(a, os.getEnv(a.getString 0))
+    setResult(a, os.getEnv(a.getString 0, a.getString 1))
   cbconf existsEnv:
     setResult(a, os.existsEnv(a.getString 0))
   cbconf dirExists:

compiler/sem.nim

@@ -12,7 +12,7 @@
 import
   ast, strutils, hashes, options, lexer, astalgo, trees, treetab,
   wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
-  magicsys, parser, nversion, nimsets, semfold, importer,
+  magicsys, parser, nversion, nimsets, semfold, modulepaths, importer,
   procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
   intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity,
@@ -522,6 +522,10 @@ proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
   else:
     result = n
   result = semStmt(c, result)
+  when false:
+    # Code generators are lazy now and can deal with undeclared procs, so these
+    # steps are not required anymore and actually harmful for the upcoming
+    # destructor support.
     # BUGFIX: process newly generated generics here, not at the end!
     if c.lastGenericIdx < c.generics.len:
       var a = newNodeI(nkStmtList, n.info)

compiler/semasgn.nim

@@ -22,7 +22,8 @@ type
     recurse: bool
 
 proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode)
-proc liftBody(c: PContext; typ: PType; info: TLineInfo): PSym
+proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
+              info: TLineInfo): PSym {.discardable.}
 
 proc at(a, i: PNode, elemType: PType): PNode =
   result = newNodeI(nkBracketExpr, a.info, 2)
@@ -92,24 +93,24 @@ proc newAsgnStmt(le, ri: PNode): PNode =
   result.sons[0] = le
   result.sons[1] = ri
 
-proc newDestructorCall(op: PSym; x: PNode): PNode =
+proc newOpCall(op: PSym; x: PNode): PNode =
   result = newNodeIT(nkCall, x.info, op.typ.sons[0])
   result.add(newSymNode(op))
   result.add x
 
-proc newDeepCopyCall(op: PSym; x, y: PNode): PNode =
+proc destructorCall(c: PContext; op: PSym; x: PNode): PNode =
-  result = newAsgnStmt(x, newDestructorCall(op, y))
+  result = newNodeIT(nkCall, x.info, op.typ.sons[0])
+  result.add(newSymNode(op))
+  if newDestructors:
+    result.add genAddr(c, x)
+  else:
+    result.add x
 
-proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
+proc newDeepCopyCall(op: PSym; x, y: PNode): PNode =
-  case c.kind
+  result = newAsgnStmt(x, newOpCall(op, y))
-  of attachedDestructor:
+
-    let op = t.destructor
+proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
-    if op != nil:
+                        field: PSym): bool =
-      markUsed(c.info, op, c.c.graph.usageSym)
-      styleCheckUse(c.info, op)
-      body.add newDestructorCall(op, x)
-      result = true
-  of attachedAsgn:
   if tfHasAsgn in t.flags:
     var op: PSym
     if sameType(t, c.asgnForType):
@@ -120,13 +121,27 @@ proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
         c.recurse = true
         return false
     else:
-        op = t.assignment
+      op = field
       if op == nil:
-          op = liftBody(c.c, t, c.info)
+        op = liftBody(c.c, t, c.kind, c.info)
     markUsed(c.info, op, c.c.graph.usageSym)
     styleCheckUse(c.info, op)
     body.add newAsgnCall(c.c, op, x, y)
     result = true
+
+proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
+  case c.kind
+  of attachedDestructor:
+    let op = t.destructor
+    if op != nil:
+      markUsed(c.info, op, c.c.graph.usageSym)
+      styleCheckUse(c.info, op)
+      body.add destructorCall(c.c, op, x)
+      result = true
+  of attachedAsgn:
+    result = considerAsgnOrSink(c, t, body, x, y, t.assignment)
+  of attachedSink:
+    result = considerAsgnOrSink(c, t, body, x, y, t.sink)
   of attachedDeepCopy:
     let op = t.deepCopy
     if op != nil:
@@ -188,7 +203,7 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       tyPtr, tyString, tyRef, tyOpt:
     defaultOp(c, t, body, x, y)
   of tyArray, tySequence:
-    if tfHasAsgn in t.flags:
+    if {tfHasAsgn, tfUncheckedArray} * t.flags == {tfHasAsgn}:
       if t.kind == tySequence:
         # XXX add 'nil' handling here
         body.add newSeqCall(c.c, x, y)
@@ -245,12 +260,20 @@ proc addParam(procType: PType; param: PSym) =
   addSon(procType.n, newSymNode(param))
   rawAddSon(procType, param.typ)
 
-proc liftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
+proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
+              info: TLineInfo): PSym {.discardable.} =
   var a: TLiftCtx
   a.info = info
   a.c = c
+  a.kind = kind
   let body = newNodeI(nkStmtList, info)
-  result = newSym(skProc, getIdent":lifted=", typ.owner, info)
+  let procname = case kind
+                 of attachedAsgn: getIdent"="
+                 of attachedSink: getIdent"=sink"
+                 of attachedDeepCopy: getIdent"=deepcopy"
+                 of attachedDestructor: getIdent"=destroy"
+
+  result = newSym(skProc, procname, typ.owner, info)
   a.fn = result
   a.asgnForType = typ
 
@@ -261,9 +284,17 @@ proc liftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
 
   result.typ = newProcType(info, typ.owner)
   result.typ.addParam dest
+  if kind != attachedDestructor:
     result.typ.addParam src
 
   liftBodyAux(a, typ, body, newSymNode(dest).newDeref, newSymNode(src))
+  # recursion is handled explicitly, do not register the type based operation
+  # before 'liftBodyAux':
+  case kind
+  of attachedAsgn: typ.assignment = result
+  of attachedSink: typ.sink = result
+  of attachedDeepCopy: typ.deepCopy = result
+  of attachedDestructor: typ.destructor = result
 
   var n = newNodeI(nkProcDef, info, bodyPos+1)
   for i in 0 ..< n.len: n.sons[i] = emptyNode
@@ -272,16 +303,29 @@ proc liftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
   n.sons[bodyPos] = body
   result.ast = n
 
-  # register late as recursion is handled differently
-  typ.assignment = result
-  #echo "Produced this ", n
 
 proc getAsgnOrLiftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
   let t = typ.skipTypes({tyGenericInst, tyVar, tyAlias})
   result = t.assignment
   if result.isNil:
-    result = liftBody(c, t, info)
+    result = liftBody(c, t, attachedAsgn, info)
 
 proc overloadedAsgn(c: PContext; dest, src: PNode): PNode =
   let a = getAsgnOrLiftBody(c, dest.typ, dest.info)
   result = newAsgnCall(c, a, dest, src)
+
+proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
+  ## In the semantic pass this is called in strategic places
+  ## to ensure we lift assignment, destructors and moves properly.
+  ## The later 'destroyer' pass depends on it.
+  if not newDestructors or not hasDestructor(typ): return
+  # do not produce wrong liftings while we're still instantiating generics:
+  if c.typesWithOps.len > 0: return
+  let typ = typ.skipTypes({tyGenericInst, tyAlias})
+  # we generate the destructor first so that other operators can depend on it:
+  if typ.destructor == nil:
+    liftBody(c, typ, attachedDestructor, info)
+  if typ.assignment == nil:
+    liftBody(c, typ, attachedAsgn, info)
+  if typ.sink == nil:
+    liftBody(c, typ, attachedSink, info)

compiler/semcall.nim

@@ -27,7 +27,7 @@ proc sameMethodDispatcher(a, b: PSym): bool =
       # method collide[T](a: TThing, b: TUnit[T]) is instantiated and not
       # method collide[T](a: TUnit[T], b: TThing)! This means we need to
       # *instantiate* every candidate! However, we don't keep more than 2-3
-      # candidated around so we cannot implement that for now. So in order
+      # candidates around so we cannot implement that for now. So in order
       # to avoid subtle problems, the call remains ambiguous and needs to
       # be disambiguated by the programmer; this way the right generic is
       # instantiated.
@@ -90,6 +90,10 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
     if c.currentScope.symbols.counter == counterInitial or syms != nil:
       matches(c, n, orig, z)
       if z.state == csMatch:
+        #if sym.name.s == "==" and (n.info ?? "temp3"):
+        #  echo typeToString(sym.typ)
+        #  writeMatches(z)
+
         # little hack so that iterators are preferred over everything else:
         if sym.kind == skIterator: inc(z.exactMatches, 200)
         case best.state

compiler/semdata.nim

@@ -37,7 +37,6 @@ type
                               # in standalone ``except`` and ``finally``
     next*: PProcCon           # used for stacking procedure contexts
     wasForwarded*: bool       # whether the current proc has a separate header
-    bracketExpr*: PNode       # current bracket expression (for ^ support)
     mapping*: TIdTable
 
   TMatchedConcept* = object
@@ -70,6 +69,7 @@ type
 
   TTypeAttachedOp* = enum
     attachedAsgn,
+    attachedSink,
     attachedDeepCopy,
     attachedDestructor
 
@@ -131,6 +131,11 @@ type
     recursiveDep*: string
     suggestionsMade*: bool
     inTypeContext*: int
+    typesWithOps*: seq[(PType, PType)] #\
+      # We need to instantiate the type bound ops lazily after
+      # the generic type has been constructed completely. See
+      # tests/destructor/topttree.nim for an example that
+      # would otherwise fail.
 
 proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair =
   result.genericSym = s
@@ -218,6 +223,7 @@ proc newContext*(graph: ModuleGraph; module: PSym; cache: IdentCache): PContext
   result.cache = cache
   result.graph = graph
   initStrTable(result.signatures)
+  result.typesWithOps = @[]
 
 
 proc inclSym(sq: var TSymSeq, s: PSym) =
@@ -333,7 +339,7 @@ proc makeNotType*(c: PContext, t1: PType): PType =
 
 proc nMinusOne*(n: PNode): PNode =
   result = newNode(nkCall, n.info, @[
-    newSymNode(getSysMagic("<", mUnaryLt)),
+    newSymNode(getSysMagic("pred", mPred)),
     n])
 
 # Remember to fix the procs below this one when you make changes!

compiler/semdestruct.nim

@@ -184,62 +184,3 @@ proc createDestructorCall(c: PContext, s: PSym): PNode =
       useSym(destructableT.destructor, c.graph.usageSym),
       useSym(s, c.graph.usageSym)]))
     result = newNode(nkDefer, s.info, @[call])
-
-proc insertDestructors(c: PContext,
-                       varSection: PNode): tuple[outer, inner: PNode] =
-  # Accepts a var or let section.
-  #
-  # When a var section has variables with destructors
-  # the var section is split up and finally blocks are inserted
-  # immediately after all "destructable" vars
-  #
-  # In case there were no destrucable variables, the proc returns
-  # (nil, nil) and the enclosing stmt-list requires no modifications.
-  #
-  # Otherwise, after the try blocks are created, the rest of the enclosing
-  # stmt-list should be inserted in the most `inner` such block (corresponding
-  # to the last variable).
-  #
-  # `outer` is a statement list that should replace the original var section.
-  # It will include the new truncated var section followed by the outermost
-  # try block.
-  let totalVars = varSection.sonsLen
-  for j in countup(0, totalVars - 1):
-    let
-      varId = varSection[j][0]
-      varTyp = varId.sym.typ
-      info = varId.info
-
-    if varTyp == nil or sfGlobal in varId.sym.flags: continue
-    let destructableT = instantiateDestructor(c, varTyp)
-
-    if destructableT != nil:
-      var tryStmt = newNodeI(nkTryStmt, info)
-
-      if j < totalVars - 1:
-        var remainingVars = newNodeI(varSection.kind, info)
-        remainingVars.sons = varSection.sons[(j+1)..varSection.len-1]
-        let (outer, inner) = insertDestructors(c, remainingVars)
-        if outer != nil:
-          tryStmt.addSon(outer)
-          result.inner = inner
-        else:
-          result.inner = newNodeI(nkStmtList, info)
-          result.inner.addSon(remainingVars)
-          tryStmt.addSon(result.inner)
-      else:
-        result.inner = newNodeI(nkStmtList, info)
-        tryStmt.addSon(result.inner)
-
-      tryStmt.addSon(
-        newNode(nkFinally, info, @[
-          semStmt(c, newNode(nkCall, info, @[
-            useSym(destructableT.destructor, c.graph.usageSym),
-            useSym(varId.sym, c.graph.usageSym)]))]))
-
-      result.outer = newNodeI(nkStmtList, info)
-      varSection.sons.setLen(j+1)
-      result.outer.addSon(varSection)
-      result.outer.addSon(tryStmt)
-
-      return

compiler/semexprs.nim

@@ -669,6 +669,7 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode =
     analyseIfAddressTakenInCall(c, result)
     if callee.magic != mNone:
       result = magicsAfterOverloadResolution(c, result, flags)
+    if result.typ != nil: liftTypeBoundOps(c, result.typ, n.info)
   if c.matchedConcept == nil:
     result = evalAtCompileTime(c, result)
 
@@ -1187,7 +1188,6 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
      tyCString:
     if n.len != 2: return nil
     n.sons[0] = makeDeref(n.sons[0])
-    c.p.bracketExpr = n.sons[0]
     for i in countup(1, sonsLen(n) - 1):
       n.sons[i] = semExprWithType(c, n.sons[i],
                                   flags*{efInTypeof, efDetermineType})
@@ -1208,7 +1208,6 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
   of tyTuple:
     if n.len != 2: return nil
     n.sons[0] = makeDeref(n.sons[0])
-    c.p.bracketExpr = n.sons[0]
     # [] operator for tuples requires constant expression:
     n.sons[1] = semConstExpr(c, n.sons[1])
     if skipTypes(n.sons[1].typ, {tyGenericInst, tyRange, tyOrdinal, tyAlias}).kind in
@@ -1246,17 +1245,13 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
       of skType:
         result = symNodeFromType(c, semTypeNode(c, n, nil), n.info)
       else:
-        c.p.bracketExpr = n.sons[0]
+        discard
-    else:
-      c.p.bracketExpr = n.sons[0]
 
 proc semArrayAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
-  let oldBracketExpr = c.p.bracketExpr
   result = semSubscript(c, n, flags)
   if result == nil:
     # overloaded [] operator:
     result = semExpr(c, buildOverloadedSubscripts(n, getIdent"[]"))
-  c.p.bracketExpr = oldBracketExpr
 
 proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode =
   var id = considerQuotedIdent(a[1], a)
@@ -1295,9 +1290,12 @@ proc takeImplicitAddr(c: PContext, n: PNode): PNode =
 proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
   if le.kind == nkHiddenDeref:
     var x = le.sons[0]
-    if x.typ.kind == tyVar and x.kind == nkSym and x.sym.kind == skResult:
+    if x.typ.kind == tyVar and x.kind == nkSym:
+      if x.sym.kind == skResult:
         n.sons[0] = x # 'result[]' --> 'result'
         n.sons[1] = takeImplicitAddr(c, ri)
+      if x.sym.kind != skParam:
+        # XXX This is hacky. See bug #4910.
         x.typ.flags.incl tfVarIsPtr
       #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info
 
@@ -1325,7 +1323,6 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
   of nkBracketExpr:
     # a[i] = x
     # --> `[]=`(a, i, x)
-    let oldBracketExpr = c.p.bracketExpr
     a = semSubscript(c, a, {efLValue})
     if a == nil:
       result = buildOverloadedSubscripts(n.sons[0], getIdent"[]=")
@@ -1335,9 +1332,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
         return n
       else:
         result = semExprNoType(c, result)
-        c.p.bracketExpr = oldBracketExpr
         return result
-    c.p.bracketExpr = oldBracketExpr
   of nkCurlyExpr:
     # a{i} = x -->  `{}=`(a, i, x)
     result = buildOverloadedSubscripts(n.sons[0], getIdent"{}=")
@@ -1382,9 +1377,12 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
           typeMismatch(n.info, lhs.typ, rhs.typ)
 
     n.sons[1] = fitNode(c, le, rhs, n.info)
+    if not newDestructors:
       if tfHasAsgn in lhs.typ.flags and not lhsIsResult and
           mode != noOverloadedAsgn:
         return overloadedAsgn(c, lhs, n.sons[1])
+    else:
+      liftTypeBoundOps(c, lhs.typ, lhs.info)
 
     fixAbstractType(c, n)
     asgnToResultVar(c, n, n.sons[0], n.sons[1])
@@ -1657,7 +1655,7 @@ proc semQuoteAst(c: PContext, n: PNode): PNode =
   # We transform the do block into a template with a param for
   # each interpolation. We'll pass this template to getAst.
   var
-    quotedBlock = n{-1}
+    quotedBlock = n[^1]
     op = if n.len == 3: expectString(c, n[1]) else: "``"
     quotes = newSeq[PNode](1)
       # the quotes will be added to a nkCall statement
@@ -2099,7 +2097,7 @@ proc shouldBeBracketExpr(n: PNode): bool =
     let b = a[0]
     if b.kind in nkSymChoices:
       for i in 0..<b.len:
-        if b[i].sym.magic == mArrGet:
+        if b[i].kind == nkSym and b[i].sym.magic == mArrGet:
           let be = newNodeI(nkBracketExpr, n.info)
           for i in 1..<a.len: be.add(a[i])
           n.sons[0] = be
@@ -2356,6 +2354,10 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     if not n.sons[0].typ.isEmptyType and not implicitlyDiscardable(n.sons[0]):
       localError(n.info, errGenerated, "'defer' takes a 'void' expression")
     #localError(n.info, errGenerated, "'defer' not allowed in this context")
+  of nkGotoState, nkState:
+    if n.len != 1 and n.len != 2: illFormedAst(n)
+    for i in 0 ..< n.len:
+      n.sons[i] = semExpr(c, n.sons[i])
   else:
     localError(n.info, errInvalidExpressionX,
                renderTree(n, {renderNoComments}))

compiler/semmagic.nim

@@ -38,9 +38,7 @@ proc skipAddr(n: PNode): PNode {.inline.} =
 proc semArrGet(c: PContext; n: PNode; flags: TExprFlags): PNode =
   result = newNodeI(nkBracketExpr, n.info)
   for i in 1..<n.len: result.add(n[i])
-  let oldBracketExpr = c.p.bracketExpr
   result = semSubscript(c, result, flags)
-  c.p.bracketExpr = oldBracketExpr
   if result.isNil:
     let x = copyTree(n)
     x.sons[0] = newIdentNode(getIdent"[]", n.info)
@@ -146,8 +144,14 @@ proc evalTypeTrait(traitCall: PNode, operand: PType, context: PSym): PNode =
     result = res.base.toNode(traitCall.info)
   of "stripGenericParams":
     result = uninstantiate(operand).toNode(traitCall.info)
+  of "supportsCopyMem":
+    let t = operand.skipTypes({tyVar, tyGenericInst, tyAlias, tyInferred})
+    let complexObj = containsGarbageCollectedRef(t) or
+                     hasDestructor(t)
+    result = newIntNodeT(ord(not complexObj), traitCall)
   else:
-    internalAssert false
+    localError(traitCall.info, "unknown trait")
+    result = emptyNode
 
 proc semTypeTraits(c: PContext, n: PNode): PNode =
   checkMinSonsLen(n, 2)
@@ -246,7 +250,11 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     result = semTypeOf(c, n.sons[1])
   of mArrGet: result = semArrGet(c, n, flags)
   of mArrPut: result = semArrPut(c, n, flags)
-  of mAsgn: result = semAsgnOpr(c, n)
+  of mAsgn:
+    if n[0].sym.name.s == "=":
+      result = semAsgnOpr(c, n)
+    else:
+      result = n
   of mIsPartOf: result = semIsPartOf(c, n, flags)
   of mTypeTrait: result = semTypeTraits(c, n)
   of mAstToStr:
@@ -264,35 +272,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
   of mDotDot:
     result = n
   of mRoof:
-    let bracketExpr = if n.len == 3: n.sons[2] else: c.p.bracketExpr
+    localError(n.info, "builtin roof operator is not supported anymore")
-    if bracketExpr.isNil:
-      localError(n.info, "no surrounding array access context for '^'")
-      result = n.sons[1]
-    elif bracketExpr.checkForSideEffects != seNoSideEffect:
-      localError(n.info, "invalid context for '^' as '$#' has side effects" %
-        renderTree(bracketExpr))
-      result = n.sons[1]
-    elif bracketExpr.typ.isStrangeArray:
-      localError(n.info, "invalid context for '^' as len!=high+1 for '$#'" %
-        renderTree(bracketExpr))
-      result = n.sons[1]
-    else:
-      # ^x  is rewritten to: len(a)-x
-      let lenExpr = newNodeI(nkCall, n.info)
-      lenExpr.add newIdentNode(getIdent"len", n.info)
-      lenExpr.add bracketExpr
-      let lenExprB = semExprWithType(c, lenExpr)
-      if lenExprB.typ.isNil or not isOrdinalType(lenExprB.typ):
-        localError(n.info, "'$#' has to be of an ordinal type for '^'" %
-          renderTree(lenExpr))
-        result = n.sons[1]
-      else:
-        result = newNodeIT(nkCall, n.info, getSysType(tyInt))
-        let subi = getSysMagic("-", mSubI)
-        #echo "got ", typeToString(subi.typ)
-        result.add newSymNode(subi, n.info)
-        result.add lenExprB
-        result.add n.sons[1]
   of mPlugin:
     let plugin = getPlugin(n[0].sym)
     if plugin.isNil:

compiler/semobjconstr.nim

@@ -78,13 +78,13 @@ proc caseBranchMatchesExpr(branch, matched: PNode): bool =
 
 proc pickCaseBranch(caseExpr, matched: PNode): PNode =
   # XXX: Perhaps this proc already exists somewhere
-  let endsWithElse = caseExpr{-1}.kind == nkElse
+  let endsWithElse = caseExpr[^1].kind == nkElse
   for i in 1 .. caseExpr.len - 1 - int(endsWithElse):
     if caseExpr[i].caseBranchMatchesExpr(matched):
       return caseExpr[i]
 
   if endsWithElse:
-    return caseExpr{-1}
+    return caseExpr[^1]
 
 iterator directFieldsInRecList(recList: PNode): PNode =
   # XXX: We can remove this case by making all nkOfBranch nodes
@@ -136,17 +136,20 @@ proc semConstructFields(c: PContext, recNode: PNode,
 
   of nkRecCase:
     template fieldsPresentInBranch(branchIdx: int): string =
-      fieldsPresentInInitExpr(recNode[branchIdx]{-1}, initExpr)
+      let branch = recNode[branchIdx]
+      let fields = branch[branch.len - 1]
+      fieldsPresentInInitExpr(fields, initExpr)
 
     template checkMissingFields(branchNode: PNode) =
-      checkForMissingFields(branchNode{-1}, initExpr)
+      let fields = branchNode[branchNode.len - 1]
+      checkForMissingFields(fields, initExpr)
 
     let discriminator = recNode.sons[0];
     internalAssert discriminator.kind == nkSym
     var selectedBranch = -1
 
     for i in 1 ..< recNode.len:
-      let innerRecords = recNode[i]{-1}
+      let innerRecords = recNode[i][^1]
       let status = semConstructFields(c, innerRecords, initExpr, flags)
       if status notin {initNone, initUnknown}:
         mergeInitStatus(result, status)

compiler/sempass2.nim

@@ -11,23 +11,14 @@ import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
   wordrecg, strutils, options, guards, writetracking
 
+when defined(useDfa):
+  import dfa
+
 # Second semantic checking pass over the AST. Necessary because the old
 # way had some inherent problems. Performs:
 #
 # * effect+exception tracking
 # * "usage before definition" checking
-# * checks for invalid usages of compiletime magics (not implemented)
-# * checks for invalid usages of NimNode (not implemented)
-# * later: will do an escape analysis for closures at least
-
-# Predefined effects:
-#   io, time (time dependent), gc (performs GC'ed allocation), exceptions,
-#   side effect (accesses global), store (stores into *type*),
-#   store_unknown (performs some store) --> store(any)|store(x)
-#   load (loads from *type*), recursive (recursive call), unsafe,
-#   endless (has endless loops), --> user effects are defined over *patterns*
-#   --> a TR macro can annotate the proc with user defined annotations
-#   --> the effect system can access these
 
 # ------------------------ exception and tag tracking -------------------------
 
@@ -248,6 +239,7 @@ proc useVar(a: PEffects, n: PNode) =
         (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem):
       #if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n)
       markGcUnsafe(a, s)
+      markSideEffect(a, s)
     else:
       markSideEffect(a, s)
 
@@ -590,6 +582,12 @@ proc trackOperand(tracked: PEffects, n: PNode, paramType: PType) =
   if paramType != nil and paramType.kind == tyVar:
     if n.kind == nkSym and isLocalVar(tracked, n.sym):
       makeVolatile(tracked, n.sym)
+  if paramType != nil and paramType.kind == tyProc and tfGcSafe in paramType.flags:
+    let argtype = skipTypes(a.typ, abstractInst)
+    # XXX figure out why this can be a non tyProc here. See httpclient.nim for an
+    # example that triggers it.
+    if argtype.kind == tyProc and notGcSafe(argtype) and not tracked.inEnforcedGcSafe:
+      localError(n.info, $n & " is not GC safe")
   notNilCheck(tracked, n, paramType)
 
 proc breaksBlock(n: PNode): bool =
@@ -979,7 +977,10 @@ proc trackProc*(s: PSym, body: PNode) =
     message(s.info, warnLockLevel,
       "declared lock level is $1, but real lock level is $2" %
         [$s.typ.lockLevel, $t.maxLockLevel])
-  if s.kind == skFunc: trackWrites(s, body)
+  when false:
+    if s.kind == skFunc:
+      when defined(dfa): dataflowAnalysis(s, body)
+      trackWrites(s, body)
 
 proc trackTopLevelStmt*(module: PSym; n: PNode) =
   if n.kind in {nkPragma, nkMacroDef, nkTemplateDef, nkProcDef, nkFuncDef,

compiler/semstmts.nim

@@ -100,6 +100,7 @@ proc semProc(c: PContext, n: PNode): PNode
 include semdestruct
 
 proc semDestructorCheck(c: PContext, n: PNode, flags: TExprFlags) {.inline.} =
+  if not newDestructors:
     if efAllowDestructor notin flags and
         n.kind in nkCallKinds+{nkObjConstr,nkBracket}:
       if instantiateDestructor(c, n.typ) != nil:
@@ -384,7 +385,7 @@ proc checkNilable(v: PSym) =
       {tfNotNil, tfNeedsInit} * v.typ.flags != {}:
     if v.ast.isNil:
       message(v.info, warnProveInit, v.name.s)
-    elif tfNeedsInit in v.typ.flags and tfNotNil notin v.ast.typ.flags:
+    elif tfNotNil in v.typ.flags and tfNotNil notin v.ast.typ.flags:
       message(v.info, warnProveInit, v.name.s)
 
 include semasgn
@@ -399,7 +400,7 @@ proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) =
   # in order for this transformation to be correct.
   let L = identDefs.len
   let value = identDefs[L-1]
-  if value.typ != nil and tfHasAsgn in value.typ.flags:
+  if value.typ != nil and tfHasAsgn in value.typ.flags and not newDestructors:
     # the spec says we need to rewrite 'var x = T()' to 'var x: T; x = T()':
     identDefs.sons[L-1] = emptyNode
     if result.kind != nkStmtList:
@@ -552,6 +553,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
     # this can only happen for errornous var statements:
     if typ == nil: continue
     typeAllowedCheck(a.info, typ, symkind)
+    liftTypeBoundOps(c, typ, a.info)
     var tup = skipTypes(typ, {tyGenericInst, tyAlias})
     if a.kind == nkVarTuple:
       if tup.kind != tyTuple:
@@ -607,7 +609,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         if def.kind == nkPar: v.ast = def[j]
         setVarType(v, tup.sons[j])
         b.sons[j] = newSymNode(v)
-      addDefer(c, result, v)
+      if not newDestructors: addDefer(c, result, v)
       checkNilable(v)
       if sfCompileTime in v.flags: hasCompileTime = true
   if hasCompileTime: vm.setupCompileTimeVar(c.module, c.cache, result)
@@ -773,24 +775,55 @@ proc typeSectionLeftSidePass(c: PContext, n: PNode) =
     checkSonsLen(a, 3)
     let name = a.sons[0]
     var s: PSym
-    if name.kind == nkDotExpr:
+    if name.kind == nkDotExpr and a[2].kind == nkObjectTy:
-      s = qualifiedLookUp(c, name, {checkUndeclared, checkModule})
+      let pkgName = considerQuotedIdent(name[0])
-      if s.kind != skType or
+      let typName = considerQuotedIdent(name[1])
-         s.typ.skipTypes(abstractPtrs).kind != tyObject or
+      let pkg = c.graph.packageSyms.strTableGet(pkgName)
-         tfPartial notin s.typ.skipTypes(abstractPtrs).flags:
+      if pkg.isNil or pkg.kind != skPackage:
-        localError(name.info, "only .partial objects can be extended")
+        localError(name.info, "unknown package name: " & pkgName.s)
+      else:
+        let typsym = pkg.tab.strTableGet(typName)
+        if typsym.isNil:
+          s = semIdentDef(c, name[1], skType)
+          s.typ = newTypeS(tyObject, c)
+          s.typ.sym = s
+          s.flags.incl sfForward
+          pkg.tab.strTableAdd s
+          addInterfaceDecl(c, s)
+        elif typsym.kind == skType and sfForward in typsym.flags:
+          s = typsym
+          addInterfaceDecl(c, s)
+        else:
+          localError(name.info, typsym.name.s & " is not a type that can be forwarded")
+          s = typsym
     else:
       s = semIdentDef(c, name, skType)
       s.typ = newTypeS(tyForward, c)
       s.typ.sym = s             # process pragmas:
       if name.kind == nkPragmaExpr:
         pragma(c, s, name.sons[1], typePragmas)
+      if sfForward in s.flags:
+        # check if the symbol already exists:
+        let pkg = c.module.owner
+        if not isTopLevel(c) or pkg.isNil:
+          localError(name.info, "only top level types in a package can be 'package'")
+        else:
+          let typsym = pkg.tab.strTableGet(s.name)
+          if typsym != nil:
+            if sfForward notin typsym.flags or sfNoForward notin typsym.flags:
+              typeCompleted(typsym)
+              typsym.info = s.info
+            else:
+              localError(name.info, "cannot complete type '" & s.name.s & "' twice; " &
+                      "previous type completion was here: " & $typsym.info)
+            s = typsym
       # add it here, so that recursive types are possible:
       if sfGenSym notin s.flags: addInterfaceDecl(c, s)
+
     a.sons[0] = newSymNode(s)
 
 proc checkCovariantParamsUsages(genericType: PType) =
-  var body = genericType{-1}
+  var body = genericType[^1]
 
   proc traverseSubTypes(t: PType): bool =
     template error(msg) = localError(genericType.sym.info, msg)
@@ -1112,7 +1145,7 @@ proc semProcAnnotation(c: PContext, prc: PNode;
                        validPragmas: TSpecialWords): PNode =
   var n = prc.sons[pragmasPos]
   if n == nil or n.kind == nkEmpty: return
-  for i in countup(0, <n.len):
+  for i in countup(0, n.len-1):
     var it = n.sons[i]
     var key = if it.kind == nkExprColonExpr: it.sons[0] else: it
     let m = lookupMacro(c, key)
@@ -1276,6 +1309,27 @@ proc maybeAddResult(c: PContext, s: PSym, n: PNode) =
 proc semOverride(c: PContext, s: PSym, n: PNode) =
   case s.name.s.normalize
   of "destroy", "=destroy":
+    if newDestructors:
+      let t = s.typ
+      var noError = false
+      if t.len == 2 and t.sons[0] == nil and t.sons[1].kind == tyVar:
+        var obj = t.sons[1].sons[0]
+        while true:
+          incl(obj.flags, tfHasAsgn)
+          if obj.kind == tyGenericBody: obj = obj.lastSon
+          elif obj.kind == tyGenericInvocation: obj = obj.sons[0]
+          else: break
+        if obj.kind in {tyObject, tyDistinct}:
+          if obj.destructor.isNil:
+            obj.destructor = s
+          else:
+            localError(n.info, errGenerated,
+              "cannot bind another '" & s.name.s & "' to: " & typeToString(obj))
+          noError = true
+      if not noError and sfSystemModule notin s.owner.flags:
+        localError(n.info, errGenerated,
+          "signature for '" & s.name.s & "' must be proc[T: object](x: var T)")
+    else:
       doDestructorStuff(c, s, n)
       if not experimentalMode(c):
         localError n.info, "use the {.experimental.} pragma to enable destructors"
@@ -1303,7 +1357,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
       localError(n.info, errGenerated,
                  "signature for 'deepCopy' must be proc[T: ptr|ref](x: T): T")
     incl(s.flags, sfUsed)
-  of "=":
+  of "=", "=sink":
     if s.magic == mAsgn: return
     incl(s.flags, sfUsed)
     let t = s.typ
@@ -1321,14 +1375,16 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
           objB = objB.sons[0]
         else: break
       if obj.kind in {tyObject, tyDistinct} and sameType(obj, objB):
-        if obj.assignment.isNil:
+        let opr = if s.name.s == "=": addr(obj.assignment) else: addr(obj.sink)
-          obj.assignment = s
+        if opr[].isNil:
+          opr[] = s
         else:
           localError(n.info, errGenerated,
-                     "cannot bind another '=' to: " & typeToString(obj))
+                     "cannot bind another '" & s.name.s & "' to: " & typeToString(obj))
         return
+    if sfSystemModule notin s.owner.flags:
       localError(n.info, errGenerated,
-               "signature for '=' must be proc[T: object](x: var T; y: T)")
+                "signature for '" & s.name.s & "' must be proc[T: object](x: var T; y: T)")
   else:
     if sfOverriden in s.flags:
       localError(n.info, errGenerated,
@@ -1808,7 +1864,8 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
       of LastBlockStmts:
         for j in countup(i + 1, length - 1):
           case n.sons[j].kind
-          of nkPragma, nkCommentStmt, nkNilLit, nkEmpty: discard
+          of nkPragma, nkCommentStmt, nkNilLit, nkEmpty, nkBlockExpr,
+             nkBlockStmt, nkState: discard
           else: localError(n.sons[j].info, errStmtInvalidAfterReturn)
       else: discard
 

compiler/semtempl.nim

@@ -113,13 +113,9 @@ type
     owner: PSym
     cursorInBody: bool # only for nimsuggest
     scopeN: int
-    bracketExpr: PNode
 
 template withBracketExpr(ctx, x, body: untyped) =
-  let old = ctx.bracketExpr
-  ctx.bracketExpr = x
   body
-  ctx.bracketExpr = old
 
 proc getIdentNode(c: var TemplCtx, n: PNode): PNode =
   case n.kind
@@ -304,18 +300,6 @@ proc semTemplBodySons(c: var TemplCtx, n: PNode): PNode =
   for i in 0 ..< n.len:
     result.sons[i] = semTemplBody(c, n.sons[i])
 
-proc oprIsRoof(n: PNode): bool =
-  const roof = "^"
-  case n.kind
-  of nkIdent: result = n.ident.s == roof
-  of nkSym: result = n.sym.name.s == roof
-  of nkAccQuoted:
-    if n.len == 1:
-      result = oprIsRoof(n.sons[0])
-  of nkOpenSymChoice, nkClosedSymChoice:
-    result = oprIsRoof(n.sons[0])
-  else: discard
-
 proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   result = n
   semIdeForTemplateOrGenericCheck(n, c.cursorInBody)
@@ -506,8 +490,6 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
       result = semTemplBodySons(c, n)
   of nkCallKinds-{nkPostfix}:
     result = semTemplBodySons(c, n)
-    if c.bracketExpr != nil and n.len == 2 and oprIsRoof(n.sons[0]):
-      result.add c.bracketExpr
   of nkDotExpr, nkAccQuoted:
     # dotExpr is ambiguous: note that we explicitly allow 'x.TemplateParam',
     # so we use the generic code for nkDotExpr too

compiler/semtypes.nim

@@ -1167,7 +1167,10 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
 
   # special check for generic object with
   # generic/partial specialized parent
-  let tx = result.skipTypes(abstractPtrs)
+  let tx = result.skipTypes(abstractPtrs, 50)
+  if tx.isNil:
+    localError(n.info, "invalid recursion in type '$1'" % typeToString(result[0]))
+    return errorType(c)
   if tx != result and tx.kind == tyObject and tx.sons[0] != nil:
     semObjectTypeForInheritedGenericInst(c, n, tx)
 
@@ -1295,7 +1298,6 @@ proc symFromExpectedTypeNode(c: PContext, n: PNode): PSym =
 
 proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
   result = nil
-  when defined(nimsuggest):
   inc c.inTypeContext
 
   if gCmd == cmdIdeTools: suggestExpr(c, n)
@@ -1511,8 +1513,13 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     localError(n.info, errTypeExpected)
     result = newOrPrevType(tyError, prev, c)
   n.typ = result
-  when defined(nimsuggest):
   dec c.inTypeContext
+  if c.inTypeContext == 0: instAllTypeBoundOp(c, n.info)
+
+when false:
+  proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
+    result = semTypeNodeInner(c, n, prev)
+    instAllTypeBoundOp(c, n.info)
 
 proc setMagicType(m: PSym, kind: TTypeKind, size: int) =
   # source : https://en.wikipedia.org/wiki/Data_structure_alignment#x86
@@ -1614,8 +1621,8 @@ proc semGenericParamList(c: PContext, n: PNode, father: PType = nil): PNode =
     var a = n.sons[i]
     if a.kind != nkIdentDefs: illFormedAst(n)
     let L = a.len
-    var def = a{-1}
+    var def = a[^1]
-    let constraint = a{-2}
+    let constraint = a[^2]
     var typ: PType
 
     if constraint.kind != nkEmpty:

compiler/semtypinst.nim

@@ -77,7 +77,7 @@ type
     topLayer*: TIdTable
     nextLayer*: ptr LayeredIdTable
 
-  TReplTypeVars* {.final.} = object
+  TReplTypeVars* = object
     c*: PContext
     typeMap*: ptr LayeredIdTable # map PType to PType
     symMap*: TIdTable         # map PSym to PSym
@@ -261,6 +261,17 @@ proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
   if not (t.kind in tyMetaTypes or
          (t.kind == tyStatic and t.n == nil)):
     result.flags.excl tfInstClearedFlags
+  when false:
+    if newDestructors:
+      result.assignment = nil
+      #result.destructor = nil
+      result.sink = nil
+
+template typeBound(c, newty, oldty, field, info) =
+  let opr = newty.field
+  if opr != nil and sfFromGeneric notin opr.flags:
+    # '=' needs to be instantiated for generics when the type is constructed:
+    newty.field = c.instTypeBoundOp(c, opr, oldty, info, attachedAsgn, 1)
 
 proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   # tyGenericInvocation[A, tyGenericInvocation[A, B]]
@@ -357,11 +368,10 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
         assert newbody.kind in {tyRef, tyPtr}
         assert newbody.lastSon.typeInst == nil
         newbody.lastSon.typeInst = result
-    let asgn = newbody.assignment
+    if newDestructors:
-    if asgn != nil and sfFromGeneric notin asgn.flags:
+      cl.c.typesWithOps.add((newbody, result))
-      # '=' needs to be instantiated for generics when the type is constructed:
+    else:
-      newbody.assignment = cl.c.instTypeBoundOp(cl.c, asgn, result, cl.info,
+      typeBound(cl.c, newbody, result, assignment, cl.info)
-                                                attachedAsgn, 1)
     let methods = skipTypes(bbody, abstractPtrs).methods
     for col, meth in items(methods):
       # we instantiate the known methods belonging to that type, this causes
@@ -529,6 +539,17 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
 
       else: discard
 
+proc instAllTypeBoundOp*(c: PContext, info: TLineInfo) =
+  if not newDestructors: return
+  var i = 0
+  while i < c.typesWithOps.len:
+    let (newty, oldty) = c.typesWithOps[i]
+    typeBound(c, newty, oldty, destructor, info)
+    typeBound(c, newty, oldty, sink, info)
+    typeBound(c, newty, oldty, assignment, info)
+    inc i
+  setLen(c.typesWithOps, 0)
+
 proc initTypeVars*(p: PContext, typeMap: ptr LayeredIdTable, info: TLineInfo;
                    owner: PSym): TReplTypeVars =
   initIdTable(result.symMap)

compiler/sigmatch.nim

@@ -55,6 +55,7 @@ type
                              # a distrinct type
     typedescMatched*: bool
     isNoCall*: bool          # misused for generic type instantiations C[T]
+    mutabilityProblem*: uint8 # tyVar mismatch
     inferredTypes: seq[PType] # inferred types during the current signature
                               # matching. they will be reset if the matching
                               # is not successful. may replace the bindings
@@ -66,7 +67,6 @@ type
                               # or when the explain pragma is used. may be
                               # triggered with an idetools command in the
                               # future.
-    mutabilityProblem*: uint8 # tyVar mismatch
     inheritancePenalty: int  # to prefer closest father object type
 
   TTypeRelFlag* = enum
@@ -224,7 +224,8 @@ proc complexDisambiguation(a, b: PType): int =
     let x = a.sons[i].sumGeneric
     let y = b.sons[i].sumGeneric
     #if ggDebug:
-    #  echo "came her ", typeToString(a.sons[i]), " ", typeToString(b.sons[i])
+    #echo "came herA ", typeToString(a.sons[i]), " ", x
+    #echo "came herB ", typeToString(b.sons[i]), " ", y
     if x != y:
       if winner == 0:
         if x > y: winner = 1
@@ -389,7 +390,16 @@ proc isConvertibleToRange(f, a: PType): bool =
   # be less picky for tyRange, as that it is used for array indexing:
   if f.kind in {tyInt..tyInt64, tyUInt..tyUInt64} and
      a.kind in {tyInt..tyInt64, tyUInt..tyUInt64}:
-    result = true
+    case f.kind
+    of tyInt, tyInt64: result = true
+    of tyInt8: result = a.kind in {tyInt8, tyInt}
+    of tyInt16: result = a.kind in {tyInt8, tyInt16, tyInt}
+    of tyInt32: result = a.kind in {tyInt8, tyInt16, tyInt32, tyInt}
+    of tyUInt, tyUInt64: result = true
+    of tyUInt8: result = a.kind in {tyUInt8, tyUInt}
+    of tyUInt16: result = a.kind in {tyUInt8, tyUInt16, tyUInt}
+    of tyUInt32: result = a.kind in {tyUInt8, tyUInt16, tyUInt32, tyUInt}
+    else: result = false
   elif f.kind in {tyFloat..tyFloat128} and
        a.kind in {tyFloat..tyFloat128}:
     result = true
@@ -1288,11 +1298,12 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
     of tyString: result = isConvertible
     of tyPtr:
       # ptr[Tag, char] is not convertible to 'cstring' for now:
-      if a.len == 1 and a.sons[0].kind == tyChar: result = isConvertible
+      if a.len == 1:
-    of tyArray:
+        let pointsTo = a.sons[0].skipTypes(abstractInst)
-      if (firstOrd(a.sons[0]) == 0) and
+        if pointsTo.kind == tyChar: result = isConvertible
-          (skipTypes(a.sons[0], {tyRange}).kind in {tyInt..tyInt64}) and
+        elif pointsTo.kind == tyArray and firstOrd(pointsTo.sons[0]) == 0 and
-          (a.sons[1].kind == tyChar):
+            skipTypes(pointsTo.sons[0], {tyRange}).kind in {tyInt..tyInt64} and
+            pointsTo.sons[1].kind == tyChar:
           result = isConvertible
     else: discard
 
@@ -1450,8 +1461,13 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
   of tyOr:
     considerPreviousT:
       result = isNone
+      let oldInheritancePenalty = c.inheritancePenalty
+      var maxInheritance = 0
       for branch in f.sons:
+        c.inheritancePenalty = 0
         let x = typeRel(c, branch, aOrig)
+        maxInheritance = max(maxInheritance, c.inheritancePenalty)
+
         # 'or' implies maximum matching result:
         if x > result: result = x
       if result >= isSubtype:
@@ -1459,6 +1475,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
         bindingRet result
       else:
         result = isNone
+      c.inheritancePenalty = oldInheritancePenalty + maxInheritance
 
   of tyNot:
     considerPreviousT:
@@ -1549,11 +1566,19 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
           result = isNone
       else:
         if f.sonsLen > 0 and f.sons[0].kind != tyNone:
+          let oldInheritancePenalty = c.inheritancePenalty
           result = typeRel(c, f.lastSon, a, flags + {trDontBind})
           if doBind and result notin {isNone, isGeneric}:
             let concrete = concreteType(c, a)
             if concrete == nil: return isNone
             put(c, f, concrete)
+          # bug #6526
+          if result in {isEqual, isSubtype}:
+            # 'T: Class' is a *better* match than just 'T'
+            # but 'T: Subclass' is even better:
+            c.inheritancePenalty = oldInheritancePenalty - c.inheritancePenalty -
+                                  100 * ord(result == isEqual)
+            result = isGeneric
         else:
           result = isGeneric
 
@@ -2287,6 +2312,7 @@ proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo;
     localError(info, errGenerated, "cannot instantiate '" & dc.name.s & "'")
   else:
     result = c.semGenerateInstance(c, dc, m.bindings, info)
+    if op == attachedDeepCopy:
       assert sfFromGeneric in result.flags
 
 include suggest

compiler/transf.nim

@@ -21,9 +21,7 @@
 import
   intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
   idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread,
-  lambdalifting, sempass2, lowerings, lookups
+  lambdalifting, sempass2, lowerings, lookups, destroyer, liftlocals
-
-# implementation
 
 type
   PTransNode* = distinct PNode
@@ -45,7 +43,7 @@ type
     inlining: int            # > 0 if we are in inlining context (copy vars)
     nestedProcs: int         # > 0 if we are in a nested proc
     contSyms, breakSyms: seq[PSym]  # to transform 'continue' and 'break'
-    deferDetected, tooEarly: bool
+    deferDetected, tooEarly, needsDestroyPass: bool
   PTransf = ref TTransfContext
 
 proc newTransNode(a: PNode): PTransNode {.inline.} =
@@ -277,7 +275,7 @@ proc transformWhile(c: PTransf; n: PNode): PTransNode =
     var body = newTransNode(n)
     for i in 0..n.len-2:
       body[i] = transform(c, n.sons[i])
-    body[<n.len] = transformLoopBody(c, n.sons[<n.len])
+    body[n.len-1] = transformLoopBody(c, n.sons[n.len-1])
     result[1] = body
     discard c.breakSyms.pop
 
@@ -367,16 +365,22 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode =
       # addr ( nkConv ( deref ( x ) ) ) --> nkConv(x)
       n.sons[0].sons[0] = m.sons[0]
       result = PTransNode(n.sons[0])
+      if n.typ.kind != tyOpenArray:
+        PNode(result).typ = n.typ
   of nkHiddenStdConv, nkHiddenSubConv, nkConv:
     var m = n.sons[0].sons[1]
     if m.kind == a or m.kind == b:
       # addr ( nkConv ( deref ( x ) ) ) --> nkConv(x)
       n.sons[0].sons[1] = m.sons[0]
       result = PTransNode(n.sons[0])
+      if n.typ.kind != tyOpenArray:
+        PNode(result).typ = n.typ
   else:
     if n.sons[0].kind == a or n.sons[0].kind == b:
       # addr ( deref ( x )) --> x
       result = PTransNode(n.sons[0].sons[0])
+      if n.typ.kind != tyOpenArray:
+        PNode(result).typ = n.typ
 
 proc generateThunk(prc: PNode, dest: PType): PNode =
   ## Converts 'prc' into '(thunk, nil)' so that it's compatible with
@@ -782,7 +786,8 @@ proc transform(c: PTransf, n: PNode): PTransNode =
                   nkBlockStmt, nkBlockExpr}:
       oldDeferAnchor = c.deferAnchor
       c.deferAnchor = n
-
+  if n.typ != nil and tfHasAsgn in n.typ.flags:
+    c.needsDestroyPass = true
   case n.kind
   of nkSym:
     result = transformSym(c, n)
@@ -972,8 +977,11 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
     result = processTransf(c, result, prc)
     liftDefer(c, result)
     #result = liftLambdas(prc, result)
-    incl(result.flags, nfTransf)
     when useEffectSystem: trackProc(prc, result)
+    liftLocalsIfRequested(prc)
+    if c.needsDestroyPass and newDestructors:
+      result = injectDestructorCalls(prc, result)
+    incl(result.flags, nfTransf)
       #if prc.name.s == "testbody":
     #  echo renderTree(result)
 
@@ -985,10 +993,12 @@ proc transformStmt*(module: PSym, n: PNode): PNode =
     result = processTransf(c, n, module)
     liftDefer(c, result)
     #result = liftLambdasForTopLevel(module, result)
-    incl(result.flags, nfTransf)
     when useEffectSystem: trackTopLevelStmt(module, result)
     #if n.info ?? "temp.nim":
     #  echo renderTree(result, {renderIds})
+    if c.needsDestroyPass and newDestructors:
+      result = injectDestructorCalls(module, result)
+    incl(result.flags, nfTransf)
 
 proc transformExpr*(module: PSym, n: PNode): PNode =
   if nfTransf in n.flags:
@@ -997,4 +1007,6 @@ proc transformExpr*(module: PSym, n: PNode): PNode =
     var c = openTransf(module, "")
     result = processTransf(c, n, module)
     liftDefer(c, result)
+    if c.needsDestroyPass and newDestructors:
+      result = injectDestructorCalls(module, result)
     incl(result.flags, nfTransf)

compiler/types.nim

@@ -667,14 +667,6 @@ proc lengthOrd*(t: PType): BiggestInt =
     else:
       result = lastOrd(t) - firstOrd(t) + 1
 
-proc isCompatibleToCString*(a: PType): bool =
-  if a.kind == tyArray:
-    if (firstOrd(a.sons[0]) == 0) and
-        (skipTypes(a.sons[0], {tyRange, tyGenericInst, tyAlias}).kind in
-            {tyInt..tyInt64, tyUInt..tyUInt64}) and
-        (a.sons[1].kind == tyChar):
-      result = true
-
 # -------------- type equality -----------------------------------------------
 
 type

compiler/typesrenderer.nim

@@ -20,7 +20,7 @@ proc renderPlainSymbolName*(n: PNode): string =
   result = ""
   case n.kind
   of nkPostfix, nkAccQuoted:
-    result = renderPlainSymbolName(n[<n.len])
+    result = renderPlainSymbolName(n[n.len-1])
   of nkIdent:
     result = n.ident.s
   of nkSym:
@@ -59,7 +59,7 @@ proc renderType(n: PNode): string =
       assert len(params) > 0
       result = "proc("
       for i in 1 ..< len(params): result.add(renderType(params[i]) & ',')
-      result[<len(result)] = ')'
+      result[len(result)-1] = ')'
     else:
       result = "proc"
   of nkIdentDefs:
@@ -73,12 +73,12 @@ proc renderType(n: PNode): string =
   of nkTupleTy:
     result = "tuple["
     for i in 0 ..< len(n): result.add(renderType(n[i]) & ',')
-    result[<len(result)] = ']'
+    result[len(result)-1] = ']'
   of nkBracketExpr:
     assert len(n) >= 2
     result = renderType(n[0]) & '['
     for i in 1 ..< len(n): result.add(renderType(n[i]) & ',')
-    result[<len(result)] = ']'
+    result[len(result)-1] = ']'
   else: result = ""
   assert(not result.isNil)
 

compiler/vm.nim

@@ -1688,8 +1688,7 @@ proc evalMacroCall*(module: PSym; cache: IdentCache, n, nOrig: PNode,
   # arity here too:
   if sym.typ.len > n.safeLen and sym.typ.len > 1:
     globalError(n.info, "in call '$#' got $#, but expected $# argument(s)" % [
-        n.renderTree,
+        n.renderTree, $(n.safeLen-1), $(sym.typ.len-1)])
-        $ <n.safeLen, $ <sym.typ.len])
 
   setupGlobalCtx(module, cache)
   var c = globalCtx

compiler/vmops.nim

@@ -47,6 +47,11 @@ template wrap1s_ospaths(op) {.dirty.} =
     setResult(a, op(getString(a, 0)))
   ospathsop op
 
+template wrap2s_ospaths(op) {.dirty.} =
+  proc `op Wrapper`(a: VmArgs) {.nimcall.} =
+    setResult(a, op(getString(a, 0), getString(a, 1)))
+  ospathsop op
+
 template wrap1s_system(op) {.dirty.} =
   proc `op Wrapper`(a: VmArgs) {.nimcall.} =
     setResult(a, op(getString(a, 0)))
@@ -96,7 +101,7 @@ proc registerAdditionalOps*(c: PCtx) =
   wrap1f_math(ceil)
   wrap2f_math(fmod)
 
-  wrap1s_ospaths(getEnv)
+  wrap2s_ospaths(getEnv)
   wrap1s_ospaths(existsEnv)
   wrap1s_os(dirExists)
   wrap1s_os(fileExists)

compiler/wordrecg.nim

@@ -55,7 +55,7 @@ type
     wFloatchecks, wNanChecks, wInfChecks,
     wAssertions, wPatterns, wWarnings,
     wHints, wOptimization, wRaises, wWrites, wReads, wSize, wEffects, wTags,
-    wDeadCodeElim, wSafecode, wNoForward, wReorder, wNoRewrite,
+    wDeadCodeElim, wSafecode, wPackage, wNoForward, wReorder, wNoRewrite,
     wPragma,
     wCompileTime, wNoInit,
     wPassc, wPassl, wBorrow, wDiscardable,
@@ -66,7 +66,7 @@ type
     wWrite, wGensym, wInject, wDirty, wInheritable, wThreadVar, wEmit,
     wAsmNoStackFrame,
     wImplicitStatic, wGlobal, wCodegenDecl, wUnchecked, wGuard, wLocks,
-    wPartial, wExplain,
+    wPartial, wExplain, wLiftLocals,
 
     wAuto, wBool, wCatch, wChar, wClass,
     wConst_cast, wDefault, wDelete, wDouble, wDynamic_cast,
@@ -143,7 +143,7 @@ const
 
     "assertions", "patterns", "warnings", "hints",
     "optimization", "raises", "writes", "reads", "size", "effects", "tags",
-    "deadcodeelim", "safecode", "noforward", "reorder", "norewrite",
+    "deadcodeelim", "safecode", "package", "noforward", "reorder", "norewrite",
     "pragma",
     "compiletime", "noinit",
     "passc", "passl", "borrow", "discardable", "fieldchecks",
@@ -152,7 +152,7 @@ const
     "computedgoto", "injectstmt", "experimental",
     "write", "gensym", "inject", "dirty", "inheritable", "threadvar", "emit",
     "asmnostackframe", "implicitstatic", "global", "codegendecl", "unchecked",
-    "guard", "locks", "partial", "explain",
+    "guard", "locks", "partial", "explain", "liftlocals",
 
     "auto", "bool", "catch", "char", "class",
     "const_cast", "default", "delete", "double",

config/nim.cfg

@@ -202,6 +202,11 @@ vcc.cpp.linkerexe = "vccexe.exe"
 
 # set the options for specific platforms:
 vcc.options.always =      "/nologo"
+@if release:
+  # no debug symbols in release builds
+@else:
+  vcc.options.always %= "${vcc.options.always} /Z7" # Get VCC to output full debug symbols in the obj file
+@end
 vcc.cpp.options.always %=  "${vcc.options.always} /EHsc"
 vcc.options.linker =      "/nologo /DEBUG /Zi /F33554432" # set the stack size to 32 MiB
 vcc.cpp.options.linker %=  "${vcc.options.linker}"
@@ -222,8 +227,8 @@ vcc.options.linker %=      "--platform:arm ${vcc.options.linker}"
 vcc.cpp.options.linker %=  "--platform:arm ${vcc.cpp.options.linker}"
 @end
 
-vcc.options.debug =     "/Zi /FS /Od"
+vcc.options.debug =     "/Od"
-vcc.cpp.options.debug = "/Zi /FS /Od"
+vcc.cpp.options.debug = "/Od"
 vcc.options.speed =     "/O2"
 vcc.cpp.options.speed = "/O2"
 vcc.options.size =     "/O1"

doc/backends.txt

@@ -20,10 +20,12 @@ to compile to C++, Objective-C or JavaScript. This document tries to
 concentrate in a single place all the backend and interfacing options.
 
 The Nim compiler supports mainly two backend families: the C, C++ and
-Objective-C targets and the JavaScript target. `The C like targets`_ creates
+Objective-C targets and the JavaScript target. `The C like targets
-source files which can be compiled into a library or a final executable. `The
+<#backends-the-c-like-targets>`_ creates source files which can be compiled
-JavaScript target`_ can generate a ``.js`` file which you reference from an
+into a library or a final executable. `The JavaScript target
-HTML file or create a `standalone nodejs program <http://nodejs.org>`_.
+<#backends-the-javascript-target>`_ can generate a ``.js`` file which you
+reference from an HTML file or create a `standalone nodejs program
+<http://nodejs.org>`_.
 
 On top of generating libraries or standalone applications, Nim offers
 bidirectional interfacing with the backend targets through generic and

doc/basicopt.txt

@@ -29,6 +29,7 @@ Options:
   --floatChecks:on|off      turn all floating point (NaN/Inf) checks on|off
   --nanChecks:on|off        turn NaN checks on|off
   --infChecks:on|off        turn Inf checks on|off
+  --nilChecks:on|off        turn nil checks on|off
   --deadCodeElim:on|off     whole program dead code elimination on|off
   --opt:none|speed|size     optimize not at all or for speed|size
                             Note: use -d:release for a release build!

doc/manual/ffi.txt

@@ -206,7 +206,7 @@ strings, because they are precompiled.
 **Note**: Passing variables to the ``dynlib`` pragma will fail at runtime
 because of order of initialization problems.
 
-**Note**: A ``dynlib`` import can be overriden with
+**Note**: A ``dynlib`` import can be overridden with
 the ``--dynlibOverride:name`` command line option. The Compiler User Guide
 contains further information.
 

doc/manual/locking.txt

@@ -2,7 +2,7 @@ Guards and locks
 ================
 
 Apart from ``spawn`` and ``parallel`` Nim also provides all the common low level
-concurrency mechanisms like locks, atomic intristics or condition variables.
+concurrency mechanisms like locks, atomic intrinsics or condition variables.
 
 Nim significantly improves on the safety of these features via additional
 pragmas:
@@ -74,7 +74,7 @@ model low level lockfree mechanisms:
 
 The ``locks`` pragma takes a list of lock expressions ``locks: [a, b, ...]``
 in order to support *multi lock* statements. Why these are essential is
-explained in the `lock levels`_ section.
+explained in the `lock levels <#guards-and-locks-lock-levels>`_ section.
 
 
 Protecting general locations

doc/manual/modules.txt

@@ -105,7 +105,7 @@ From import statement
 ~~~~~~~~~~~~~~~~~~~~~
 
 After the ``from`` statement a module name follows followed by
-an ``import`` to list the symbols one likes to use without explict
+an ``import`` to list the symbols one likes to use without explicit
 full qualification:
 
 .. code-block:: nim
@@ -123,7 +123,7 @@ in ``module``.
 Export statement
 ~~~~~~~~~~~~~~~~
 
-An ``export`` statement can be used for symbol fowarding so that client
+An ``export`` statement can be used for symbol forwarding so that client
 modules don't need to import a module's dependencies:
 
 .. code-block:: nim

doc/manual/pragmas.txt

@@ -70,7 +70,7 @@ compileTime pragma
 The ``compileTime`` pragma is used to mark a proc or variable to be used at
 compile time only. No code will be generated for it. Compile time procs are
 useful as helpers for macros. Since version 0.12.0 of the language, a proc
-that uses ``system.NimNode`` within its parameter types is implictly declared
+that uses ``system.NimNode`` within its parameter types is implicitly declared
 ``compileTime``:
 
 .. code-block:: nim
@@ -733,7 +733,8 @@ about the ``importcpp`` pragma pattern language. It is not necessary
 to know all the details described here.
 
 
-Similar to the `importc pragma for C <manual.html#importc-pragma>`_, the
+Similar to the `importc pragma for C 
+<#foreign-function-interface-importc-pragma>`_, the 
 ``importcpp`` pragma can be used to import `C++`:idx: methods or C++ symbols
 in general. The generated code then uses the C++ method calling
 syntax: ``obj->method(arg)``.  In combination with the ``header`` and ``emit``
@@ -955,10 +956,11 @@ Produces:
 
 ImportObjC pragma
 -----------------
-Similar to the `importc pragma for C <manual.html#importc-pragma>`_, the
+Similar to the `importc pragma for C 
-``importobjc`` pragma can be used to import `Objective C`:idx: methods.  The
+<#foreign-function-interface-importc-pragma>`_, the ``importobjc`` pragma can 
-generated code then uses the Objective C method calling syntax: ``[obj method
+be used to import `Objective C`:idx: methods.  The generated code then uses the
-param1: arg]``.  In addition with the ``header`` and ``emit`` pragmas this
+Objective C method calling syntax: ``[obj method param1: arg]``. 
+In addition with the ``header`` and ``emit`` pragmas this
 allows *sloppy* interfacing with libraries written in Objective C:
 
 .. code-block:: Nim

doc/manual/procs.txt

@@ -142,10 +142,11 @@ The method call syntax conflicts with explicit generic instantiations:
 parsed as ``(x.p)[T]``.
 
 **Future directions**: ``p[.T.]`` might be introduced as an alternative syntax
-to pass explict types to a generic and then ``x.p[.T.]`` can be parsed as
+to pass explicit types to a generic and then ``x.p[.T.]`` can be parsed as
 ``x.(p[.T.])``.
 
-See also: `Limitations of the method call syntax`_.
+See also: `Limitations of the method call syntax
+<#templates-limitations-of-the-method-call-syntax>`_.
 
 
 Properties
@@ -178,7 +179,7 @@ different; for this a special setter syntax is needed:
 Command invocation syntax
 -------------------------
 
-Routines can be invoked without the ``()`` if the call is syntatically
+Routines can be invoked without the ``()`` if the call is syntactically
 a statement. This command invocation syntax also works for
 expressions, but then only a single argument may follow. This restriction
 means ``echo f 1, f 2`` is parsed as ``echo(f(1), f(2))`` and not as

doc/manual/stmts.txt

@@ -199,7 +199,7 @@ The rules for compile-time computability are:
    (possibly empty) list of compile-time computable arguments.
 
 
-Constants cannot be of type ``ptr``, ``ref``, ``var`` or ``object``, nor can
+Constants cannot be of type ``ptr``, ``ref`` or ``var``, nor can
 they contain such a type.
 
 

doc/manual/templates.txt

@@ -293,7 +293,7 @@ Limitations of the method call syntax
 
 The expression ``x`` in ``x.f`` needs to be semantically checked (that means
 symbol lookup and type checking) before it can be decided that it needs to be
-rewritten to ``f(x)``. Therefore the dot syntax has some limiations when it
+rewritten to ``f(x)``. Therefore the dot syntax has some limitations when it
 is used to invoke templates/macros:
 
 .. code-block:: nim

doc/manual/threads.txt

@@ -5,7 +5,7 @@ To enable thread support the ``--threads:on`` command line switch needs to
 be used. The ``system`` module then contains several threading primitives.
 See the `threads <threads.html>`_ and `channels <channels.html>`_ modules
 for the low level thread API. There are also high level parallelism constructs
-available. See `spawn`_ for further details.
+available. See `spawn <#parallel-spawn>`_ for further details.
 
 Nim's memory model for threads is quite different than that of other common
 programming languages (C, Pascal, Java): Each thread has its own (garbage

doc/manual/type_bound_ops.txt

@@ -7,12 +7,12 @@ There are 3 operations that are bound to a type:
 2. Destruction
 3. Deep copying for communication between threads
 
-These operations can be *overriden* instead of *overloaded*. This means the
+These operations can be *overridden* instead of *overloaded*. This means the
 implementation is automatically lifted to structured types. For instance if type
-``T`` has an overriden assignment operator ``=`` this operator is also used
+``T`` has an overridden assignment operator ``=`` this operator is also used
 for assignments of the type ``seq[T]``. Since these operations are bound to a
 type they have to be bound to a nominal type for reasons of simplicity of
-implementation: This means an overriden ``deepCopy`` for ``ref T`` is really
+implementation: This means an overridden ``deepCopy`` for ``ref T`` is really
 bound to ``T`` and not to ``ref T``. This also means that one cannot override
 ``deepCopy`` for both ``ptr T`` and ``ref T`` at the same time; instead a
 helper distinct or object type has to be used for one pointer type.

doc/manual/type_rel.txt

@@ -491,4 +491,4 @@ metatypes ``typed`` and ``typedesc`` are not lazy.
 Varargs matching
 ----------------
 
-See `Varargs`_.
+See `Varargs <#types-varargs>`_.

doc/manual/types.txt

@@ -114,7 +114,8 @@ if the literal's value fits this smaller type and such a conversion is less
 expensive than other implicit conversions, so ``myInt16 + 34`` produces
 an ``int16`` result.
 
-For further details, see `Convertible relation`_.
+For further details, see `Convertible relation
+<#type-relations-convertible-relation>`_.
 
 
 Subrange types
@@ -677,6 +678,44 @@ branch switch ``system.reset`` has to be used. Also, when the fields of a
 particular branch are specified during object construction, the correct value
 for the discriminator must be supplied at compile-time.
 
+Package level objects
+---------------------
+
+Every Nim module resides in a (nimble) package. An object type can be attached
+to the package it resides in. If that is done, the type can be referenced from
+other modules as an `incomplete`:idx: object type. This features allows to
+break up recursive type dependencies accross module boundaries. Incomplete
+object types are always passed ``byref`` and can only be used in pointer like
+contexts (``var/ref/ptr IncompleteObject``) in general since the compiler does
+not yet know the size of the object. To complete an incomplete object
+the ``package`` pragma has to be used. ``package`` implies ``byref``.
+
+As long as a type ``T`` is incomplete ``sizeof(T)`` or "runtime type
+information" for ``T`` is not available.
+
+
+Example:
+
+.. code-block:: nim
+
+  # module A (in an arbitrary package)
+  type
+    Pack.SomeObject = object ## declare as incomplete object of package 'Pack'
+    Triple = object
+      a, b, c: ref SomeObject ## pointers to incomplete objects are allowed
+
+  ## Incomplete objects can be used as parameters:
+  proc myproc(x: SomeObject) = discard
+
+
+.. code-block:: nim
+
+  # module B (in package "Pack")
+  type
+    SomeObject* {.package.} = object ## Use 'package' to complete the object
+      s, t: string
+      x, y: int
+
 
 Set type
 --------
@@ -885,8 +924,8 @@ not compatible to ``pointer`` to prevent the following from compiling:
 Future directions:
 
 * Memory regions might become available for  ``string`` and ``seq`` too.
-* Builtin regions like ``private``, ``global`` and ``local`` will
+* Builtin regions like ``private``, ``global`` and ``local`` might be
-  prove very useful for the upcoming OpenCL target.
+  useful for an OpenCL target.
 * Builtin "regions" can model ``lent`` and ``unique`` pointers.
 * An assignment operator can be attached to a region so that proper write
   barriers can be generated. This would imply that the GC can be implemented

koch.nim

@@ -263,7 +263,7 @@ proc buildTools(latest: bool) =
   nimexec "c -o:" & nimgrepExe & " tools/nimgrep.nim"
   when defined(windows): buildVccTool()
 
-  nimexec "c -o:" & ("bin/nimresolve".exe) & " tools/nimresolve.nim"
+  #nimexec "c -o:" & ("bin/nimresolve".exe) & " tools/nimresolve.nim"
 
   buildNimble(latest)
 
@@ -472,7 +472,8 @@ proc temp(args: string) =
   # 125 is the magic number to tell git bisect to skip the current
   # commit.
   let (bootArgs, programArgs) = splitArgs(args)
-  exec("nim c " & bootArgs & " compiler" / "nim", 125)
+  let nimexec = findNim()
+  exec(nimexec & " c " & bootArgs & " compiler" / "nim", 125)
   copyExe(output, finalDest)
   if programArgs.len > 0: exec(finalDest & " " & programArgs)
 

lib/core/macros.nim

@@ -129,13 +129,6 @@ const
   nnkCallKinds* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand,
                    nnkCallStrLit}
 
-proc `[]`*(n: NimNode, i: int): NimNode {.magic: "NChild", noSideEffect.}
-  ## get `n`'s `i`'th child.
-
-proc `[]=`*(n: NimNode, i: int, child: NimNode) {.magic: "NSetChild",
-  noSideEffect.}
-  ## set `n`'s `i`'th child to `child`.
-
 proc `!`*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect.}
   ## constructs an identifier from the string `s`
 
@@ -162,6 +155,20 @@ proc sameType*(a, b: NimNode): bool {.magic: "SameNodeType", noSideEffect.} =
 proc len*(n: NimNode): int {.magic: "NLen", noSideEffect.}
   ## returns the number of children of `n`.
 
+proc `[]`*(n: NimNode, i: int): NimNode {.magic: "NChild", noSideEffect.}
+  ## get `n`'s `i`'th child.
+
+proc `[]`*(n: NimNode, i: BackwardsIndex): NimNode = n[n.len - i.int]
+  ## get `n`'s `i`'th child.
+
+proc `[]=`*(n: NimNode, i: int, child: NimNode) {.magic: "NSetChild",
+  noSideEffect.}
+  ## set `n`'s `i`'th child to `child`.
+
+proc `[]=`*(n: NimNode, i: BackwardsIndex, child: NimNode) =
+  ## set `n`'s `i`'th child to `child`.
+  n[n.len - i.int] = child
+
 proc add*(father, child: NimNode): NimNode {.magic: "NAdd", discardable,
   noSideEffect, locks: 0.}
   ## Adds the `child` to the `father` node. Returns the
@@ -839,7 +846,7 @@ proc newNilLit*(): NimNode {.compileTime.} =
   ## New nil literal shortcut
   result = newNimNode(nnkNilLit)
 
-proc last*(node: NimNode): NimNode {.compileTime.} = node[<node.len]
+proc last*(node: NimNode): NimNode {.compileTime.} = node[node.len-1]
   ## Return the last item in nodes children. Same as `node[^1]`
 
 
@@ -986,6 +993,11 @@ iterator items*(n: NimNode): NimNode {.inline.} =
   for i in 0 ..< n.len:
     yield n[i]
 
+iterator pairs*(n: NimNode): (int, NimNode) {.inline.} =
+  ## Iterates over the children of the NimNode ``n`` and its indices.
+  for i in 0 ..< n.len:
+    yield (i, n[i])
+
 iterator children*(n: NimNode): NimNode {.inline.} =
   ## Iterates over the children of the NimNode ``n``.
   for i in 0 ..< n.len:

lib/impure/db_postgres.nim

@@ -98,6 +98,9 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
   var a = 0
   if args.len > 0 and not string(formatstr).contains("?"):
     dbError("""parameter substitution expects "?" """)
+  if args.len == 0:
+    return string(formatstr)
+  else:
     for c in items(string(formatstr)):
       if c == '?':
         if args[a] == nil:

lib/impure/nre.nim

@@ -155,7 +155,7 @@ type
     ##     -  ``"abc".match(re"(?<letter>\w)").captures["letter"] == "a"``
     ##     -  ``"abc".match(re"(\w)\w").captures[-1] == "ab"``
     ##
-    ## ``captureBounds[]: Option[Slice[int]]``
+    ## ``captureBounds[]: Option[HSlice[int, int]]``
     ##     gets the bounds of the given capture according to the same rules as
     ##     the above. If the capture is not filled, then ``None`` is returned.
     ##     The bounds are both inclusive.
@@ -167,7 +167,7 @@ type
     ## ``match: string``
     ##     the full text of the match.
     ##
-    ## ``matchBounds: Slice[int]``
+    ## ``matchBounds: HSlice[int, int]``
     ##     the bounds of the match, as in ``captureBounds[]``
     ##
     ## ``(captureBounds|captures).toTable``
@@ -182,7 +182,7 @@ type
                      ## Not nil.
     str*: string  ## The string that was matched against.
                   ## Not nil.
-    pcreMatchBounds: seq[Slice[cint]] ## First item is the bounds of the match
+    pcreMatchBounds: seq[HSlice[cint, cint]] ## First item is the bounds of the match
                                             ## Other items are the captures
                                             ## `a` is inclusive start, `b` is exclusive end
 
@@ -251,13 +251,13 @@ proc captureBounds*(pattern: RegexMatch): CaptureBounds = return CaptureBounds(p
 
 proc captures*(pattern: RegexMatch): Captures = return Captures(pattern)
 
-proc `[]`*(pattern: CaptureBounds, i: int): Option[Slice[int]] =
+proc `[]`*(pattern: CaptureBounds, i: int): Option[HSlice[int, int]] =
   let pattern = RegexMatch(pattern)
   if pattern.pcreMatchBounds[i + 1].a != -1:
     let bounds = pattern.pcreMatchBounds[i + 1]
     return some(int(bounds.a) .. int(bounds.b-1))
   else:
-    return none(Slice[int])
+    return none(HSlice[int, int])
 
 proc `[]`*(pattern: Captures, i: int): string =
   let pattern = RegexMatch(pattern)
@@ -272,10 +272,10 @@ proc `[]`*(pattern: Captures, i: int): string =
 proc match*(pattern: RegexMatch): string =
   return pattern.captures[-1]
 
-proc matchBounds*(pattern: RegexMatch): Slice[int] =
+proc matchBounds*(pattern: RegexMatch): HSlice[int, int] =
   return pattern.captureBounds[-1].get
 
-proc `[]`*(pattern: CaptureBounds, name: string): Option[Slice[int]] =
+proc `[]`*(pattern: CaptureBounds, name: string): Option[HSlice[int, int]] =
   let pattern = RegexMatch(pattern)
   return pattern.captureBounds[pattern.pattern.captureNameToId.fget(name)]
 
@@ -295,9 +295,9 @@ proc toTable*(pattern: Captures, default: string = nil): Table[string, string] =
   result = initTable[string, string]()
   toTableImpl(nextVal == nil)
 
-proc toTable*(pattern: CaptureBounds, default = none(Slice[int])):
+proc toTable*(pattern: CaptureBounds, default = none(HSlice[int, int])):
-    Table[string, Option[Slice[int]]] =
+    Table[string, Option[HSlice[int, int]]] =
-  result = initTable[string, Option[Slice[int]]]()
+  result = initTable[string, Option[HSlice[int, int]]]()
   toTableImpl(nextVal.isNone)
 
 template itemsImpl(cond: untyped) {.dirty.} =
@@ -309,13 +309,13 @@ template itemsImpl(cond: untyped) {.dirty.} =
     yield nextYieldVal
 
 
-iterator items*(pattern: CaptureBounds, default = none(Slice[int])): Option[Slice[int]] =
+iterator items*(pattern: CaptureBounds, default = none(HSlice[int, int])): Option[HSlice[int, int]] =
   itemsImpl(nextVal.isNone)
 
 iterator items*(pattern: Captures, default: string = nil): string =
   itemsImpl(nextVal == nil)
 
-proc toSeq*(pattern: CaptureBounds, default = none(Slice[int])): seq[Option[Slice[int]]] =
+proc toSeq*(pattern: CaptureBounds, default = none(HSlice[int, int])): seq[Option[HSlice[int, int]]] =
   accumulateResult(pattern.items(default))
 
 proc toSeq*(pattern: Captures, default: string = nil): seq[string] =
@@ -396,8 +396,6 @@ proc extractOptions(pattern: string): tuple[pattern: string, flags: int, study:
 
 # }}}
 
-type UncheckedArray {.unchecked.}[T] = array[0 .. 0, T]
-
 proc destroyRegex(pattern: Regex) =
   pcre.free_substring(cast[cstring](pattern.pcreObj))
   pattern.pcreObj = nil
@@ -464,7 +462,7 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt
   # 1x capture count as slack space for PCRE
   let vecsize = (pattern.captureCount() + 1) * 3
   # div 2 because each element is 2 cints long
-  myResult.pcreMatchBounds = newSeq[Slice[cint]](ceil(vecsize / 2).int)
+  myResult.pcreMatchBounds = newSeq[HSlice[cint, cint]](ceil(vecsize / 2).int)
   myResult.pcreMatchBounds.setLen(vecsize div 3)
 
   let strlen = if endpos == int.high: str.len else: endpos+1

lib/impure/re.nim

@@ -13,10 +13,6 @@
 ## We had to de-deprecate this module since too much code relies on it
 ## and many people prefer its API over ``nre``'s.
 ##
-## **Note:** The 're' proc defaults to the **extended regular expression
-## syntax** which lets you use whitespace freely to make your regexes readable.
-## However, this means matching whitespace requires ``\s`` or something similar.
-##
 ## This module is implemented by providing a wrapper around the
 ## `PRCE (Perl-Compatible Regular Expressions) <http://www.pcre.org>`_
 ## C library. This means that your application will depend on the PRCE
@@ -78,7 +74,7 @@ proc finalizeRegEx(x: Regex) =
   if not isNil(x.e):
     pcre.free_substring(cast[cstring](x.e))
 
-proc re*(s: string, flags = {reExtended, reStudy}): Regex =
+proc re*(s: string, flags = {reStudy}): Regex =
   ## Constructor of regular expressions.
   ##
   ## Note that Nim's
@@ -96,6 +92,13 @@ proc re*(s: string, flags = {reExtended, reStudy}): Regex =
     result.e = pcre.study(result.h, options, addr msg)
     if not isNil(msg): raiseInvalidRegex($msg)
 
+proc rex*(s: string, flags = {reStudy, reExtended}): Regex =
+  ## Constructor for extended regular expressions.
+  ##
+  ## The extended means that comments starting with `#` and
+  ## whitespace are ignored.
+  result = re(s, flags)
+
 proc bufSubstr(b: cstring, sPos, ePos: int): string {.inline.} =
   ## Return a Nim string built from a slice of a cstring buffer.
   ## Don't assume cstring is '\0' terminated

lib/impure/ssl.nim

@@ -63,16 +63,16 @@ proc recvLine*(sock: SecureSocket, line: var TaintedString): bool =
   setLen(line.string, 0)
   while true:
     var c: array[0..0, char]
-    var n = BIO_read(sock.bio, c, c.len.cint)
+    var n = BIO_read(sock.bio, addr c, c.len.cint)
     if n <= 0: return false
     if c[0] == '\r':
-      n = BIO_read(sock.bio, c, c.len.cint)
+      n = BIO_read(sock.bio, addr c, c.len.cint)
       if n > 0 and c[0] == '\L':
         return true
       elif n <= 0:
         return false
     elif c[0] == '\L': return true
-    add(line.string, c)
+    add(line.string, c[0])
 
 
 proc send*(sock: SecureSocket, data: string) =

lib/nimbase.h

@@ -396,15 +396,13 @@ typedef struct TStringDesc* string;
 #define GenericSeqSize sizeof(TGenericSeq)
 #define paramCount() cmdCount
 
-#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__i386__)
+// NAN definition copied from math.h included in the Windows SDK version 10.0.14393.0
 #ifndef NAN
-static unsigned long nimNaN[2]={0xffffffff, 0x7fffffff};
+#ifndef _HUGE_ENUF
-#    define NAN (*(double*) nimNaN)
+#define _HUGE_ENUF  1e+300  // _HUGE_ENUF*_HUGE_ENUF must overflow
 #endif
-#endif
+#define NAN_INFINITY ((float)(_HUGE_ENUF * _HUGE_ENUF))
-
+#define NAN ((float)(NAN_INFINITY * 0.0F))
-#ifndef NAN
-#  define NAN (0.0 / 0.0)
 #endif
 
 #ifndef INF

lib/posix/linux.nim

@@ -26,3 +26,5 @@ const
 proc clone*(fn: pointer; child_stack: pointer; flags: cint;
             arg: pointer; ptid: ptr Pid; tls: pointer;
             ctid: ptr Pid): cint {.importc, header: "<sched.h>".}
+
+proc pipe2*(a: array[0..1, cint], flags: cint): cint {.importc, header: "<unistd.h>".}

lib/posix/posix_other.nim

@@ -146,7 +146,7 @@ type
   Mode* {.importc: "mode_t", header: "<sys/types.h>".} = cint
   Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = int
   Off* {.importc: "off_t", header: "<sys/types.h>".} = int64
-  Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int
+  Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32
   Pthread_attr* {.importc: "pthread_attr_t", header: "<sys/types.h>".} = int
   Pthread_barrier* {.importc: "pthread_barrier_t",
                       header: "<sys/types.h>".} = int

lib/pure/algorithm.nim

@@ -371,3 +371,126 @@ when isMainModule:
   for i in 0 .. high(arr1):
     assert arr1.reversed(0, i) == arr1.reversed()[high(arr1) - i .. high(arr1)]
     assert arr1.reversed(i, high(arr1)) == arr1.reversed()[0 .. high(arr1) - i]
+
+
+proc rotateInternal[T](arg: var openarray[T]; first, middle, last: int): int =
+  ## A port of std::rotate from c++. Ported from `this reference <http://www.cplusplus.com/reference/algorithm/rotate/>`_.
+  result = first + last - middle
+
+  if first == middle or middle == last:
+    return
+
+  assert first < middle
+  assert middle < last
+
+  # m prefix for mutable
+  var
+    mFirst = first
+    mMiddle = middle
+    next = middle
+
+  swap(arg[mFirst], arg[next])
+  mFirst += 1
+  next  += 1
+  if mFirst == mMiddle:
+    mMiddle = next
+
+  while next != last:
+    swap(arg[mFirst], arg[next])
+    mFirst += 1
+    next += 1
+    if mFirst == mMiddle:
+      mMiddle = next
+
+  next = mMiddle
+  while next != last:
+    swap(arg[mFirst], arg[next])
+    mFirst += 1
+    next += 1
+    if mFirst == mMiddle:
+      mMiddle = next
+    elif next == last:
+      next = mMiddle
+
+proc rotatedInternal[T](arg: openarray[T]; first, middle, last: int): seq[T] =
+  result = newSeq[T](arg.len)
+  for i in 0 ..< first:
+    result[i] = arg[i]
+  let N = last - middle
+  let M = middle - first
+  for i in 0 ..< N:
+    result[first+i] = arg[middle+i]
+  for i in 0 ..< M:
+    result[first+N+i] = arg[first+i]
+  for i in last ..< arg.len:
+    result[i] = arg[i]
+
+proc rotateLeft*[T](arg: var openarray[T]; slice: HSlice[int, int]; dist: int): int =
+  ## Performs a left rotation on a range of elements. If you want to rotate right, use a negative ``dist``.
+  ## Specifically, ``rotateLeft`` rotates the elements at ``slice`` by ``dist`` positions.
+  ## The element at index ``slice.a + dist`` will be at index ``slice.a``.
+  ## The element at index ``slice.b`` will be at ``slice.a + dist -1``.
+  ## The element at index ``slice.a`` will be at ``slice.b + 1 - dist``.
+  ## The element at index ``slice.a + dist - 1`` will be at ``slice.b``.
+  #
+  ## Elements outsize of ``slice`` will be left unchanged.
+  ## The time complexity is linear to ``slice.b - slice.a + 1``.
+  ##
+  ## ``slice``
+  ##   the indices of the element range that should be rotated.
+  ##
+  ## ``dist``
+  ##   the distance in amount of elements that the data should be rotated. Can be negative, can be any number.
+  ##
+  ## .. code-block:: nim
+  ##     var list     = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+  ##     list.rotateLeft(1 .. 8, 3)
+  ##     doAssert list == [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10]
+  let sliceLen = slice.b + 1 - slice.a
+  let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen
+  arg.rotateInternal(slice.a, slice.a+distLeft, slice.b + 1)
+
+proc rotateLeft*[T](arg: var openarray[T]; dist: int): int =
+  ## default arguments for slice, so that this procedure operates on the entire
+  ## ``arg``, and not just on a part of it.
+  let arglen = arg.len
+  let distLeft = ((dist mod arglen) + arglen) mod arglen
+  arg.rotateInternal(0, distLeft, arglen)
+
+proc rotatedLeft*[T](arg: openarray[T]; slice: HSlice[int, int], dist: int): seq[T] =
+  ## same as ``rotateLeft``, just with the difference that it does
+  ## not modify the argument. It creates a new ``seq`` instead
+  let sliceLen = slice.b + 1 - slice.a
+  let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen
+  arg.rotatedInternal(slice.a, slice.a+distLeft, slice.b+1)
+
+proc rotatedLeft*[T](arg: openarray[T]; dist: int): seq[T] =
+  ## same as ``rotateLeft``, just with the difference that it does
+  ## not modify the argument. It creates a new ``seq`` instead
+  let arglen = arg.len
+  let distLeft = ((dist mod arglen) + arglen) mod arglen
+  arg.rotatedInternal(0, distLeft, arg.len)
+
+when isMainModule:
+  var list     = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+  let list2    = list.rotatedLeft(1 ..< 9, 3)
+  let expected = [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10]
+
+  doAssert list.rotateLeft(1 ..< 9, 3) == 6
+  doAssert list  ==  expected
+  doAssert list2 == @expected
+
+  var s0,s1,s2,s3,s4,s5 = "xxxabcdefgxxx"
+
+  doAssert s0.rotateLeft(3 ..< 10, 3) == 7
+  doAssert s0 == "xxxdefgabcxxx"
+  doAssert s1.rotateLeft(3 ..< 10, 2) == 8
+  doAssert s1 == "xxxcdefgabxxx"
+  doAssert s2.rotateLeft(3 ..< 10, 4) == 6
+  doAssert s2 == "xxxefgabcdxxx"
+  doAssert s3.rotateLeft(3 ..< 10, -3) == 6
+  doAssert s3 == "xxxefgabcdxxx"
+  doAssert s4.rotateLeft(3 ..< 10, -10) == 6
+  doAssert s4 == "xxxefgabcdxxx"
+  doAssert s5.rotateLeft(3 ..< 10, 11) == 6
+  doAssert s5 == "xxxefgabcdxxx"

lib/pure/collections/deques.nim

@@ -207,9 +207,9 @@ when isMainModule:
   assert($deq == "[4, 56, 6, 789]")
 
   assert deq[0] == deq.peekFirst and deq.peekFirst == 4
-  assert deq[^1] == deq.peekLast and deq.peekLast == 789
+  #assert deq[^1] == deq.peekLast and deq.peekLast == 789
   deq[0] = 42
-  deq[^1] = 7
+  deq[deq.len - 1] = 7
 
   assert 6 in deq and 789 notin deq
   assert deq.find(6) >= 0

lib/pure/collections/lists.nim

@@ -37,6 +37,14 @@ type
   DoublyLinkedRing*[T] = object ## a doubly linked ring
     head*: DoublyLinkedNode[T]
 
+  SomeLinkedList*[T] = SinglyLinkedList[T] | DoublyLinkedList[T]
+
+  SomeLinkedRing*[T] = SinglyLinkedRing[T] | DoublyLinkedRing[T]
+
+  SomeLinkedCollection*[T] = SomeLinkedList[T] | SomeLinkedRing[T]
+
+  SomeLinkedNode*[T] = SinglyLinkedNode[T] | DoublyLinkedNode[T]
+
 {.deprecated: [TDoublyLinkedNode: DoublyLinkedNodeObj,
     PDoublyLinkedNode: DoublyLinkedNode,
     TSinglyLinkedNode: SinglyLinkedNodeObj,
@@ -86,14 +94,34 @@ template itemsRingImpl() {.dirty.} =
       it = it.next
       if it == L.head: break
 
-template nodesListImpl() {.dirty.} =
+iterator items*[T](L: SomeLinkedList[T]): T =
+  ## yields every value of `L`.
+  itemsListImpl()
+
+iterator items*[T](L: SomeLinkedRing[T]): T =
+  ## yields every value of `L`.
+  itemsRingImpl()
+
+iterator mitems*[T](L: var SomeLinkedList[T]): var T =
+  ## yields every value of `L` so that you can modify it.
+  itemsListImpl()
+
+iterator mitems*[T](L: var SomeLinkedRing[T]): var T =
+  ## yields every value of `L` so that you can modify it.
+  itemsRingImpl()
+
+iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] =
+  ## iterates over every node of `x`. Removing the current node from the
+  ## list during traversal is supported.
   var it = L.head
   while it != nil:
     var nxt = it.next
     yield it
     it = nxt
 
-template nodesRingImpl() {.dirty.} =
+iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] =
+  ## iterates over every node of `x`. Removing the current node from the
+  ## list during traversal is supported.
   var it = L.head
   if it != nil:
     while true:
@@ -102,121 +130,21 @@ template nodesRingImpl() {.dirty.} =
       it = nxt
       if it == L.head: break
 
-template findImpl() {.dirty.} =
+proc `$`*[T](L: SomeLinkedCollection[T]): string =
-  for x in nodes(L):
+  ## turns a list into its string representation.
-    if x.value == value: return x
-
-iterator items*[T](L: DoublyLinkedList[T]): T =
-  ## yields every value of `L`.
-  itemsListImpl()
-
-iterator items*[T](L: SinglyLinkedList[T]): T =
-  ## yields every value of `L`.
-  itemsListImpl()
-
-iterator items*[T](L: SinglyLinkedRing[T]): T =
-  ## yields every value of `L`.
-  itemsRingImpl()
-
-iterator items*[T](L: DoublyLinkedRing[T]): T =
-  ## yields every value of `L`.
-  itemsRingImpl()
-
-iterator mitems*[T](L: var DoublyLinkedList[T]): var T =
-  ## yields every value of `L` so that you can modify it.
-  itemsListImpl()
-
-iterator mitems*[T](L: var SinglyLinkedList[T]): var T =
-  ## yields every value of `L` so that you can modify it.
-  itemsListImpl()
-
-iterator mitems*[T](L: var SinglyLinkedRing[T]): var T =
-  ## yields every value of `L` so that you can modify it.
-  itemsRingImpl()
-
-iterator mitems*[T](L: var DoublyLinkedRing[T]): var T =
-  ## yields every value of `L` so that you can modify it.
-  itemsRingImpl()
-
-iterator nodes*[T](L: SinglyLinkedList[T]): SinglyLinkedNode[T] =
-  ## iterates over every node of `x`. Removing the current node from the
-  ## list during traversal is supported.
-  nodesListImpl()
-
-iterator nodes*[T](L: DoublyLinkedList[T]): DoublyLinkedNode[T] =
-  ## iterates over every node of `x`. Removing the current node from the
-  ## list during traversal is supported.
-  nodesListImpl()
-
-iterator nodes*[T](L: SinglyLinkedRing[T]): SinglyLinkedNode[T] =
-  ## iterates over every node of `x`. Removing the current node from the
-  ## list during traversal is supported.
-  nodesRingImpl()
-
-iterator nodes*[T](L: DoublyLinkedRing[T]): DoublyLinkedNode[T] =
-  ## iterates over every node of `x`. Removing the current node from the
-  ## list during traversal is supported.
-  nodesRingImpl()
-
-template dollarImpl() {.dirty.} =
   result = "["
   for x in nodes(L):
     if result.len > 1: result.add(", ")
     result.add($x.value)
   result.add("]")
 
-proc `$`*[T](L: SinglyLinkedList[T]): string =
+proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] =
-  ## turns a list into its string representation.
-  dollarImpl()
-
-proc `$`*[T](L: DoublyLinkedList[T]): string =
-  ## turns a list into its string representation.
-  dollarImpl()
-
-proc `$`*[T](L: SinglyLinkedRing[T]): string =
-  ## turns a list into its string representation.
-  dollarImpl()
-
-proc `$`*[T](L: DoublyLinkedRing[T]): string =
-  ## turns a list into its string representation.
-  dollarImpl()
-
-proc find*[T](L: SinglyLinkedList[T], value: T): SinglyLinkedNode[T] =
   ## searches in the list for a value. Returns nil if the value does not
   ## exist.
-  findImpl()
+  for x in nodes(L):
+    if x.value == value: return x
 
-proc find*[T](L: DoublyLinkedList[T], value: T): DoublyLinkedNode[T] =
+proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} =
-  ## searches in the list for a value. Returns nil if the value does not
-  ## exist.
-  findImpl()
-
-proc find*[T](L: SinglyLinkedRing[T], value: T): SinglyLinkedNode[T] =
-  ## searches in the list for a value. Returns nil if the value does not
-  ## exist.
-  findImpl()
-
-proc find*[T](L: DoublyLinkedRing[T], value: T): DoublyLinkedNode[T] =
-  ## searches in the list for a value. Returns nil if the value does not
-  ## exist.
-  findImpl()
-
-proc contains*[T](L: SinglyLinkedList[T], value: T): bool {.inline.} =
-  ## searches in the list for a value. Returns false if the value does not
-  ## exist, true otherwise.
-  result = find(L, value) != nil
-
-proc contains*[T](L: DoublyLinkedList[T], value: T): bool {.inline.} =
-  ## searches in the list for a value. Returns false if the value does not
-  ## exist, true otherwise.
-  result = find(L, value) != nil
-
-proc contains*[T](L: SinglyLinkedRing[T], value: T): bool {.inline.} =
-  ## searches in the list for a value. Returns false if the value does not
-  ## exist, true otherwise.
-  result = find(L, value) != nil
-
-proc contains*[T](L: DoublyLinkedRing[T], value: T): bool {.inline.} =
   ## searches in the list for a value. Returns false if the value does not
   ## exist, true otherwise.
   result = find(L, value) != nil
@@ -266,7 +194,6 @@ proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
   if n.next != nil: n.next.prev = n.prev
   if n.prev != nil: n.prev.next = n.next
 
-
 proc append*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
   ## appends a node `n` to `L`. Efficiency: O(1).
   if L.head != nil:

lib/pure/collections/queues.nim

@@ -198,9 +198,8 @@ when isMainModule:
   assert($q == "[4, 56, 6, 789]")
 
   assert q[0] == q.front and q.front == 4
-  assert q[^1] == q.back and q.back == 789
   q[0] = 42
-  q[^1] = 7
+  q[q.len - 1] = 7
 
   assert 6 in q and 789 notin q
   assert q.find(6) >= 0

lib/pure/collections/sequtils.nim

@@ -13,8 +13,9 @@
 ## were inspired by functional programming languages.
 ##
 ## For functional style programming you may want to pass `anonymous procs
-## <manual.html#anonymous-procs>`_ to procs like ``filter`` to reduce typing.
+## <manual.html#procedures-anonymous-procs>`_ to procs like ``filter`` to
-## Anonymous procs can use `the special do notation <manual.html#do-notation>`_
+## reduce typing. Anonymous procs can use `the special do notation
+## <manual.html#procedures-do-notation>`_
 ## which is more convenient in certain situations.
 
 include "system/inclrtl"
@@ -43,8 +44,23 @@ proc concat*[T](seqs: varargs[seq[T]]): seq[T] =
       result[i] = itm
       inc(i)
 
-proc cycle*[T](s: seq[T], n: Natural): seq[T] =
+proc count*[T](s: openArray[T], x: T): int =
-  ## Returns a new sequence with the items of `s` repeated `n` times.
+  ## Returns the number of occurrences of the item `x` in the container `s`.
+  ##
+  ## Example:
+  ##
+  ## .. code-block::
+  ##   let
+  ##     s = @[1, 2, 2, 3, 2, 4, 2]
+  ##     c = count(s, 2)
+  ##   assert c == 4
+  for itm in items(s):
+    if itm == x:
+      inc result
+
+proc cycle*[T](s: openArray[T], n: Natural): seq[T] =
+  ## Returns a new sequence with the items of the container `s` repeated
+  ## `n` times.
   ##
   ## Example:
   ##
@@ -75,9 +91,11 @@ proc repeat*[T](x: T, n: Natural): seq[T] =
   for i in 0 ..< n:
     result[i] = x
 
-proc deduplicate*[T](seq1: seq[T]): seq[T] =
+proc deduplicate*[T](s: openArray[T]): seq[T] =
   ## Returns a new sequence without duplicates.
   ##
+  ## Example:
+  ##
   ## .. code-block::
   ##   let
   ##     dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4]
@@ -87,17 +105,19 @@ proc deduplicate*[T](seq1: seq[T]): seq[T] =
   ##   assert unique1 == @[1, 3, 4, 2, 8]
   ##   assert unique2 == @["a", "c", "d"]
   result = @[]
-  for itm in items(seq1):
+  for itm in items(s):
     if not result.contains(itm): result.add(itm)
 
 {.deprecated: [distnct: deduplicate].}
 
-proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] =
+proc zip*[S, T](s1: openArray[S], s2: openArray[T]): seq[tuple[a: S, b: T]] =
-  ## Returns a new sequence with a combination of the two input sequences.
+  ## Returns a new sequence with a combination of the two input containers.
   ##
   ## For convenience you can access the returned tuples through the named
-  ## fields `a` and `b`. If one sequence is shorter, the remaining items in the
+  ## fields `a` and `b`. If one container is shorter, the remaining items in
-  ## longer sequence are discarded. Example:
+  ## the longer container are discarded.
+  ##
+  ## Example:
   ##
   ## .. code-block::
   ##   let
@@ -110,9 +130,10 @@ proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] =
   ##   assert zip2 == @[(1, "one"), (2, "two"), (3, "three")]
   ##   assert zip1[2].b == 4
   ##   assert zip2[2].b == "three"
-  var m = min(seq1.len, seq2.len)
+  var m = min(s1.len, s2.len)
   newSeq(result, m)
-  for i in 0 .. m-1: result[i] = (seq1[i], seq2[i])
+  for i in 0 ..< m:
+    result[i] = (s1[i], s2[i])
 
 proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] =
   ## Splits and distributes a sequence `s` into `num` sub sequences.
@@ -178,29 +199,33 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] =
         result[i].add(s[g])
       first = last
 
-
+proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}):
-proc map*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}):
                                                             seq[S]{.inline.} =
   ## Returns a new sequence with the results of `op` applied to every item in
-  ## `data`.
+  ## the container `s`.
   ##
   ## Since the input is not modified you can use this version of ``map`` to
-  ## transform the type of the elements in the input sequence. Example:
+  ## transform the type of the elements in the input container.
+  ##
+  ## Example:
   ##
   ## .. code-block:: nim
   ##   let
   ##     a = @[1, 2, 3, 4]
   ##     b = map(a, proc(x: int): string = $x)
   ##   assert b == @["1", "2", "3", "4"]
-  newSeq(result, data.len)
+  newSeq(result, s.len)
-  for i in 0..data.len-1: result[i] = op(data[i])
+  for i in 0 ..< s.len:
+    result[i] = op(s[i])
 
-proc map*[T](data: var openArray[T], op: proc (x: var T) {.closure.})
+proc map*[T](s: var openArray[T], op: proc (x: var T) {.closure.})
                                                               {.deprecated.} =
-  ## Applies `op` to every item in `data` modifying it directly.
+  ## Applies `op` to every item in `s` modifying it directly.
   ##
   ## Note that this version of ``map`` requires your input and output types to
-  ## be the same, since they are modified in-place. Example:
+  ## be the same, since they are modified in-place.
+  ##
+  ## Example:
   ##
   ## .. code-block:: nim
   ##   var a = @["1", "2", "3", "4"]
@@ -210,15 +235,16 @@ proc map*[T](data: var openArray[T], op: proc (x: var T) {.closure.})
   ##   echo repr(a)
   ##   # --> ["142", "242", "342", "442"]
   ## **Deprecated since version 0.12.0:** Use the ``apply`` proc instead.
-  for i in 0..data.len-1: op(data[i])
+  for i in 0 ..< s.len: op(s[i])
 
-proc apply*[T](data: var seq[T], op: proc (x: var T) {.closure.})
+proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.})
                                                               {.inline.} =
-  ## Applies `op` to every item in `data` modifying it directly.
+  ## Applies `op` to every item in `s` modifying it directly.
   ##
   ## Note that this requires your input and output types to
   ## be the same, since they are modified in-place.
   ## The parameter function takes a ``var T`` type parameter.
+  ##
   ## Example:
   ##
   ## .. code-block:: nim
@@ -229,15 +255,16 @@ proc apply*[T](data: var seq[T], op: proc (x: var T) {.closure.})
   ##   echo repr(a)
   ##   # --> ["142", "242", "342", "442"]
   ##
-  for i in 0..data.len-1: op(data[i])
+  for i in 0 ..< s.len: op(s[i])
 
-proc apply*[T](data: var seq[T], op: proc (x: T): T {.closure.})
+proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.})
                                                               {.inline.} =
-  ## Applies `op` to every item in `data` modifying it directly.
+  ## Applies `op` to every item in `s` modifying it directly.
   ##
   ## Note that this requires your input and output types to
   ## be the same, since they are modified in-place.
   ## The parameter function takes and returns a ``T`` type variable.
+  ##
   ## Example:
   ##
   ## .. code-block:: nim
@@ -248,11 +275,10 @@ proc apply*[T](data: var seq[T], op: proc (x: T): T {.closure.})
   ##   echo repr(a)
   ##   # --> ["142", "242", "342", "442"]
   ##
-  for i in 0..data.len-1: data[i] = op(data[i])
+  for i in 0 ..< s.len: s[i] = op(s[i])
 
-
+iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T =
-iterator filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): T =
+  ## Iterates through a container and yields every item that fulfills the
-  ## Iterates through a sequence and yields every item that fulfills the
   ## predicate.
   ##
   ## Example:
@@ -262,11 +288,11 @@ iterator filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): T =
   ##   for n in filter(numbers, proc (x: int): bool = x mod 2 == 0):
   ##     echo($n)
   ##   # echoes 4, 8, 4 in separate lines
-  for i in 0..<seq1.len:
+  for i in 0 ..< s.len:
-    if pred(seq1[i]):
+    if pred(s[i]):
-      yield seq1[i]
+      yield s[i]
 
-proc filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): seq[T]
+proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T]
                                                                   {.inline.} =
   ## Returns a new sequence with all the items that fulfilled the predicate.
   ##
@@ -280,11 +306,11 @@ proc filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): seq[T]
   ##   assert f1 == @["red", "black"]
   ##   assert f2 == @["yellow"]
   result = newSeq[T]()
-  for i in 0..<seq1.len:
+  for i in 0 ..< s.len:
-    if pred(seq1[i]):
+    if pred(s[i]):
-      result.add(seq1[i])
+      result.add(s[i])
 
-proc keepIf*[T](seq1: var seq[T], pred: proc(item: T): bool {.closure.})
+proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.})
                                                                 {.inline.} =
   ## Keeps the items in the passed sequence if they fulfilled the predicate.
   ## Same as the ``filter`` proc, but modifies the sequence directly.
@@ -296,12 +322,12 @@ proc keepIf*[T](seq1: var seq[T], pred: proc(item: T): bool {.closure.})
   ##   keepIf(floats, proc(x: float): bool = x > 10)
   ##   assert floats == @[13.0, 12.5, 10.1]
   var pos = 0
-  for i in 0 .. <len(seq1):
+  for i in 0 ..< len(s):
-    if pred(seq1[i]):
+    if pred(s[i]):
       if pos != i:
-        shallowCopy(seq1[pos], seq1[i])
+        shallowCopy(s[pos], s[i])
       inc(pos)
-  setLen(seq1, pos)
+  setLen(s, pos)
 
 proc delete*[T](s: var seq[T]; first, last: Natural) =
   ## Deletes in `s` the items at position `first` .. `last`. This modifies
@@ -354,11 +380,12 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos=0) =
     inc(j)
 
 
-template filterIt*(seq1, pred: untyped): untyped =
+template filterIt*(s, pred: untyped): untyped =
   ## Returns a new sequence with all the items that fulfilled the predicate.
   ##
   ## Unlike the `proc` version, the predicate needs to be an expression using
   ## the ``it`` variable for testing, like: ``filterIt("abcxyz", it == 'x')``.
+  ##
   ## Example:
   ##
   ## .. code-block::
@@ -368,8 +395,8 @@ template filterIt*(seq1, pred: untyped): untyped =
   ##      notAcceptable = filterIt(temperatures, it > 50 or it < -10)
   ##    assert acceptable == @[-2.0, 24.5, 44.31]
   ##    assert notAcceptable == @[-272.15, 99.9, -113.44]
-  var result = newSeq[type(seq1[0])]()
+  var result = newSeq[type(s[0])]()
-  for it {.inject.} in items(seq1):
+  for it {.inject.} in items(s):
     if pred: result.add(it)
   result
 
@@ -378,6 +405,7 @@ template keepItIf*(varSeq: seq, pred: untyped) =
   ##
   ## Unlike the `proc` version, the predicate needs to be an expression using
   ## the ``it`` variable for testing, like: ``keepItIf("abcxyz", it == 'x')``.
+  ##
   ## Example:
   ##
   ## .. code-block::
@@ -393,8 +421,8 @@ template keepItIf*(varSeq: seq, pred: untyped) =
       inc(pos)
   setLen(varSeq, pos)
 
-proc all*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool =
+proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool =
-  ## Iterates through a sequence and checks if every item fulfills the
+  ## Iterates through a container and checks if every item fulfills the
   ## predicate.
   ##
   ## Example:
@@ -403,12 +431,12 @@ proc all*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool =
   ##   let numbers = @[1, 4, 5, 8, 9, 7, 4]
   ##   assert all(numbers, proc (x: int): bool = return x < 10) == true
   ##   assert all(numbers, proc (x: int): bool = return x < 9) == false
-  for i in seq1:
+  for i in s:
     if not pred(i):
       return false
   return true
 
-template allIt*(seq1, pred: untyped): bool =
+template allIt*(s, pred: untyped): bool =
   ## Checks if every item fulfills the predicate.
   ##
   ## Example:
@@ -418,14 +446,14 @@ template allIt*(seq1, pred: untyped): bool =
   ##   assert allIt(numbers, it < 10) == true
   ##   assert allIt(numbers, it < 9) == false
   var result = true
-  for it {.inject.} in items(seq1):
+  for it {.inject.} in items(s):
     if not pred:
       result = false
       break
   result
 
-proc any*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool =
+proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool =
-  ## Iterates through a sequence and checks if some item fulfills the
+  ## Iterates through a container and checks if some item fulfills the
   ## predicate.
   ##
   ## Example:
@@ -434,12 +462,12 @@ proc any*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool =
   ##   let numbers = @[1, 4, 5, 8, 9, 7, 4]
   ##   assert any(numbers, proc (x: int): bool = return x > 8) == true
   ##   assert any(numbers, proc (x: int): bool = return x > 9) == false
-  for i in seq1:
+  for i in s:
     if pred(i):
       return true
   return false
 
-template anyIt*(seq1, pred: untyped): bool =
+template anyIt*(s, pred: untyped): bool =
   ## Checks if some item fulfills the predicate.
   ##
   ## Example:
@@ -449,7 +477,7 @@ template anyIt*(seq1, pred: untyped): bool =
   ##   assert anyIt(numbers, it > 8) == true
   ##   assert anyIt(numbers, it > 9) == false
   var result = false
-  for it {.inject.} in items(seq1):
+  for it {.inject.} in items(s):
     if pred:
       result = true
       break
@@ -493,7 +521,9 @@ template foldl*(sequence, operation: untyped): untyped =
   ## variables ``a`` and ``b`` for each step of the fold. Since this is a left
   ## fold, for non associative binary operations like subtraction think that
   ## the sequence of numbers 1, 2 and 3 will be parenthesized as (((1) - 2) -
-  ## 3).  Example:
+  ## 3).
+  ##
+  ## Example:
   ##
   ## .. code-block::
   ##   let
@@ -527,6 +557,7 @@ template foldl*(sequence, operation, first): untyped =
   ## The ``operation`` parameter should be an expression which uses the variables
   ## ``a`` and ``b`` for each step of the fold. The ``first`` parameter is the
   ## start value (the first ``a``) and therefor defines the type of the result.
+  ##
   ## Example:
   ##
   ## .. code-block::
@@ -555,7 +586,9 @@ template foldr*(sequence, operation: untyped): untyped =
   ## variables ``a`` and ``b`` for each step of the fold. Since this is a right
   ## fold, for non associative binary operations like subtraction think that
   ## the sequence of numbers 1, 2 and 3 will be parenthesized as (1 - (2 -
-  ## (3))). Example:
+  ## (3))).
+  ##
+  ## Example:
   ##
   ## .. code-block::
   ##   let
@@ -580,13 +613,15 @@ template foldr*(sequence, operation: untyped): untyped =
     result = operation
   result
 
-template mapIt*(seq1, typ, op: untyped): untyped =
+template mapIt*(s, typ, op: untyped): untyped =
   ## Convenience template around the ``map`` proc to reduce typing.
   ##
   ## The template injects the ``it`` variable which you can use directly in an
   ## expression. You also need to pass as `typ` the type of the expression,
   ## since the new returned sequence can have a different type than the
-  ## original.  Example:
+  ## original.
+  ##
+  ## Example:
   ##
   ## .. code-block::
   ##   let
@@ -596,16 +631,18 @@ template mapIt*(seq1, typ, op: untyped): untyped =
   ## **Deprecated since version 0.12.0:** Use the ``mapIt(seq1, op)``
   ##   template instead.
   var result: seq[typ] = @[]
-  for it {.inject.} in items(seq1):
+  for it {.inject.} in items(s):
     result.add(op)
   result
 
 
-template mapIt*(seq1, op: untyped): untyped =
+template mapIt*(s, op: untyped): untyped =
   ## Convenience template around the ``map`` proc to reduce typing.
   ##
   ## The template injects the ``it`` variable which you can use directly in an
-  ## expression. Example:
+  ## expression.
+  ##
+  ## Example:
   ##
   ## .. code-block::
   ##   let
@@ -614,19 +651,19 @@ template mapIt*(seq1, op: untyped): untyped =
   ##   assert strings == @["4", "8", "12", "16"]
   type outType = type((
     block:
-      var it{.inject.}: type(items(seq1));
+      var it{.inject.}: type(items(s));
       op))
   var result: seq[outType]
-  when compiles(seq1.len):
+  when compiles(s.len):
-    let s = seq1
+    let t = s
     var i = 0
     result = newSeq[outType](s.len)
-    for it {.inject.} in s:
+    for it {.inject.} in t:
       result[i] = op
       i += 1
   else:
     result = @[]
-    for it {.inject.} in seq1:
+    for it {.inject.} in s:
       result.add(op)
   result
 
@@ -635,7 +672,9 @@ template applyIt*(varSeq, op: untyped) =
   ##
   ## The template injects the ``it`` variable which you can use directly in an
   ## expression. The expression has to return the same type as the sequence you
-  ## are mutating. Example:
+  ## are mutating.
+  ##
+  ## Example:
   ##
   ## .. code-block::
   ##   var nums = @[1, 2, 3, 4]
@@ -646,9 +685,10 @@ template applyIt*(varSeq, op: untyped) =
     varSeq[i] = op
 
 
-
 template newSeqWith*(len: int, init: untyped): untyped =
-  ## creates a new sequence, calling `init` to initialize each value. Example:
+  ## creates a new sequence, calling `init` to initialize each value.
+  ##
+  ## Example:
   ##
   ## .. code-block::
   ##   var seq2D = newSeqWith(20, newSeq[bool](10))
@@ -674,45 +714,178 @@ when isMainModule:
       total = concat(s1, s2, s3)
     assert total == @[1, 2, 3, 4, 5, 6, 7]
 
-  block: # duplicates test
+  block: # count test
+    let
+      s1 = @[1, 2, 3, 2]
+      s2 = @['a', 'b', 'x', 'a']
+      a1 = [1, 2, 3, 2]
+      a2 = ['a', 'b', 'x', 'a']
+      r0 = count(s1, 0)
+      r1 = count(s1, 1)
+      r2 = count(s1, 2)
+      r3 = count(s2, 'y')
+      r4 = count(s2, 'x')
+      r5 = count(s2, 'a')
+      ar0 = count(a1, 0)
+      ar1 = count(a1, 1)
+      ar2 = count(a1, 2)
+      ar3 = count(a2, 'y')
+      ar4 = count(a2, 'x')
+      ar5 = count(a2, 'a')
+    assert r0 == 0
+    assert r1 == 1
+    assert r2 == 2
+    assert r3 == 0
+    assert r4 == 1
+    assert r5 == 2
+    assert ar0 == 0
+    assert ar1 == 1
+    assert ar2 == 2
+    assert ar3 == 0
+    assert ar4 == 1
+    assert ar5 == 2
+
+  block: # cycle tests
+    let
+      a = @[1, 2, 3]
+      b: seq[int] = @[]
+      c = [1, 2, 3]
+
+    doAssert a.cycle(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3]
+    doAssert a.cycle(0) == @[]
+    #doAssert a.cycle(-1) == @[] # will not compile!
+    doAssert b.cycle(3) == @[]
+    doAssert c.cycle(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3]
+    doAssert c.cycle(0) == @[]
+
+  block: # repeat tests
+    assert repeat(10, 5) == @[10, 10, 10, 10, 10]
+    assert repeat(@[1,2,3], 2) == @[@[1,2,3], @[1,2,3]]
+    assert repeat([1,2,3], 2) == @[[1,2,3], [1,2,3]]
+
+  block: # deduplicates test
     let
       dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4]
       dup2 = @["a", "a", "c", "d", "d"]
+      dup3 = [1, 1, 3, 4, 2, 2, 8, 1, 4]
+      dup4 = ["a", "a", "c", "d", "d"]
       unique1 = deduplicate(dup1)
       unique2 = deduplicate(dup2)
+      unique3 = deduplicate(dup3)
+      unique4 = deduplicate(dup4)
     assert unique1 == @[1, 3, 4, 2, 8]
     assert unique2 == @["a", "c", "d"]
+    assert unique3 == @[1, 3, 4, 2, 8]
+    assert unique4 == @["a", "c", "d"]
 
   block: # zip test
     let
       short = @[1, 2, 3]
       long = @[6, 5, 4, 3, 2, 1]
       words = @["one", "two", "three"]
+      ashort = [1, 2, 3]
+      along = [6, 5, 4, 3, 2, 1]
+      awords = ["one", "two", "three"]
       zip1 = zip(short, long)
       zip2 = zip(short, words)
+      zip3 = zip(ashort, along)
+      zip4 = zip(ashort, awords)
+      zip5 = zip(ashort, words)
     assert zip1 == @[(1, 6), (2, 5), (3, 4)]
     assert zip2 == @[(1, "one"), (2, "two"), (3, "three")]
+    assert zip3 == @[(1, 6), (2, 5), (3, 4)]
+    assert zip4 == @[(1, "one"), (2, "two"), (3, "three")]
+    assert zip5 == @[(1, "one"), (2, "two"), (3, "three")]
     assert zip1[2].b == 4
     assert zip2[2].b == "three"
+    assert zip3[2].b == 4
+    assert zip4[2].b == "three"
+    assert zip5[2].b == "three"
+
+  block: # distribute tests
+    let numbers = @[1, 2, 3, 4, 5, 6, 7]
+    doAssert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]]
+    doAssert numbers.distribute(6)[0] == @[1, 2]
+    doAssert numbers.distribute(6)[5] == @[7]
+    let a = @[1, 2, 3, 4, 5, 6, 7]
+    doAssert a.distribute(1, true)   == @[@[1, 2, 3, 4, 5, 6, 7]]
+    doAssert a.distribute(1, false)  == @[@[1, 2, 3, 4, 5, 6, 7]]
+    doAssert a.distribute(2, true)   == @[@[1, 2, 3, 4], @[5, 6, 7]]
+    doAssert a.distribute(2, false)  == @[@[1, 2, 3, 4], @[5, 6, 7]]
+    doAssert a.distribute(3, true)   == @[@[1, 2, 3], @[4, 5], @[6, 7]]
+    doAssert a.distribute(3, false)  == @[@[1, 2, 3], @[4, 5, 6], @[7]]
+    doAssert a.distribute(4, true)   == @[@[1, 2], @[3, 4], @[5, 6], @[7]]
+    doAssert a.distribute(4, false)  == @[@[1, 2], @[3, 4], @[5, 6], @[7]]
+    doAssert a.distribute(5, true)   == @[@[1, 2], @[3, 4], @[5], @[6], @[7]]
+    doAssert a.distribute(5, false)  == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]]
+    doAssert a.distribute(6, true)   == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]]
+    doAssert a.distribute(6, false)  == @[
+      @[1, 2], @[3, 4], @[5, 6], @[7], @[], @[]]
+    doAssert a.distribute(8, false)  == a.distribute(8, true)
+    doAssert a.distribute(90, false) == a.distribute(90, true)
+    var b = @[0]
+    for f in 1 .. 25: b.add(f)
+    doAssert b.distribute(5, true)[4].len == 5
+    doAssert b.distribute(5, false)[4].len == 2
+
+  block: # map test
+    let
+      numbers = @[1, 4, 5, 8, 9, 7, 4]
+      anumbers = [1, 4, 5, 8, 9, 7, 4]
+      m1 = map(numbers, proc(x: int): int = 2*x)
+      m2 = map(anumbers, proc(x: int): int = 2*x)
+    assert m1 == @[2, 8, 10, 16, 18, 14, 8]
+    assert m2 == @[2, 8, 10, 16, 18, 14, 8]
+
+  block: # apply test
+    var a = @["1", "2", "3", "4"]
+    apply(a, proc(x: var string) = x &= "42")
+    assert a == @["142", "242", "342", "442"]
 
   block: # filter proc test
     let
       colors = @["red", "yellow", "black"]
+      acolors = ["red", "yellow", "black"]
       f1 = filter(colors, proc(x: string): bool = x.len < 6)
       f2 = filter(colors) do (x: string) -> bool : x.len > 5
+      f3 = filter(acolors, proc(x: string): bool = x.len < 6)
+      f4 = filter(acolors) do (x: string) -> bool : x.len > 5
     assert f1 == @["red", "black"]
     assert f2 == @["yellow"]
+    assert f3 == @["red", "black"]
+    assert f4 == @["yellow"]
 
   block: # filter iterator test
     let numbers = @[1, 4, 5, 8, 9, 7, 4]
+    let anumbers = [1, 4, 5, 8, 9, 7, 4]
     assert toSeq(filter(numbers, proc (x: int): bool = x mod 2 == 0)) ==
       @[4, 8, 4]
+    assert toSeq(filter(anumbers, proc (x: int): bool = x mod 2 == 0)) ==
+      @[4, 8, 4]
 
   block: # keepIf test
     var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1]
     keepIf(floats, proc(x: float): bool = x > 10)
     assert floats == @[13.0, 12.5, 10.1]
 
+  block: # delete tests
+    let outcome = @[1,1,1,1,1,1,1,1]
+    var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1]
+    dest.delete(3, 8)
+    assert outcome == dest, """\
+    Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1]
+    is [1,1,1,1,1,1,1,1]"""
+
+  block: # insert tests
+    var dest = @[1,1,1,1,1,1,1,1]
+    let
+      src = @[2,2,2,2,2,2]
+      outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1]
+    dest.insert(src, 3)
+    assert dest == outcome, """\
+    Inserting [2,2,2,2,2,2] into [1,1,1,1,1,1,1,1]
+    at 3 is [1,1,1,2,2,2,2,2,2,1,1,1,1,1]"""
+
   block: # filterIt test
     let
       temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44]
@@ -726,37 +899,49 @@ when isMainModule:
     keepItIf(candidates, it.len == 3 and it[0] == 'b')
     assert candidates == @["bar", "baz"]
 
-  block: # any
-    let
-      numbers = @[1, 4, 5, 8, 9, 7, 4]
-      len0seq : seq[int] = @[]
-    assert any(numbers, proc (x: int): bool = return x > 8) == true
-    assert any(numbers, proc (x: int): bool = return x > 9) == false
-    assert any(len0seq, proc (x: int): bool = return true) == false
-
-  block: # anyIt
-    let
-      numbers = @[1, 4, 5, 8, 9, 7, 4]
-      len0seq : seq[int] = @[]
-    assert anyIt(numbers, it > 8) == true
-    assert anyIt(numbers, it > 9) == false
-    assert anyIt(len0seq, true) == false
-
   block: # all
     let
       numbers = @[1, 4, 5, 8, 9, 7, 4]
+      anumbers = [1, 4, 5, 8, 9, 7, 4]
       len0seq : seq[int] = @[]
     assert all(numbers, proc (x: int): bool = return x < 10) == true
     assert all(numbers, proc (x: int): bool = return x < 9) == false
     assert all(len0seq, proc (x: int): bool = return false) == true
+    assert all(anumbers, proc (x: int): bool = return x < 10) == true
+    assert all(anumbers, proc (x: int): bool = return x < 9) == false
 
   block: # allIt
     let
       numbers = @[1, 4, 5, 8, 9, 7, 4]
+      anumbers = [1, 4, 5, 8, 9, 7, 4]
       len0seq : seq[int] = @[]
     assert allIt(numbers, it < 10) == true
     assert allIt(numbers, it < 9) == false
     assert allIt(len0seq, false) == true
+    assert allIt(anumbers, it < 10) == true
+    assert allIt(anumbers, it < 9) == false
+
+  block: # any
+    let
+      numbers = @[1, 4, 5, 8, 9, 7, 4]
+      anumbers = [1, 4, 5, 8, 9, 7, 4]
+      len0seq : seq[int] = @[]
+    assert any(numbers, proc (x: int): bool = return x > 8) == true
+    assert any(numbers, proc (x: int): bool = return x > 9) == false
+    assert any(len0seq, proc (x: int): bool = return true) == false
+    assert any(anumbers, proc (x: int): bool = return x > 8) == true
+    assert any(anumbers, proc (x: int): bool = return x > 9) == false
+
+  block: # anyIt
+    let
+      numbers = @[1, 4, 5, 8, 9, 7, 4]
+      anumbers = [1, 4, 5, 8, 9, 7, 4]
+      len0seq : seq[int] = @[]
+    assert anyIt(numbers, it > 8) == true
+    assert anyIt(numbers, it > 9) == false
+    assert anyIt(len0seq, true) == false
+    assert anyIt(anumbers, it > 8) == true
+    assert anyIt(anumbers, it > 9) == false
 
   block: # toSeq test
     let
@@ -792,56 +977,13 @@ when isMainModule:
     assert multiplication == 495, "Multiplication is (5*(9*(11)))"
     assert concatenation == "nimiscool"
 
-  block: # delete tests
-    let outcome = @[1,1,1,1,1,1,1,1]
-    var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1]
-    dest.delete(3, 8)
-    assert outcome == dest, """\
-    Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1]
-    is [1,1,1,1,1,1,1,1]"""
-
-  block: # insert tests
-    var dest = @[1,1,1,1,1,1,1,1]
-    let
-      src = @[2,2,2,2,2,2]
-      outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1]
-    dest.insert(src, 3)
-    assert dest == outcome, """\
-    Inserting [2,2,2,2,2,2] into [1,1,1,1,1,1,1,1]
-    at 3 is [1,1,1,2,2,2,2,2,2,1,1,1,1,1]"""
-
   block: # mapIt tests
     var
       nums = @[1, 2, 3, 4]
       strings = nums.mapIt($(4 * it))
     nums.applyIt(it * 3)
     assert nums[0] + nums[3] == 15
-
+    assert strings[2] == "12"
-  block: # distribute tests
-    let numbers = @[1, 2, 3, 4, 5, 6, 7]
-    doAssert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]]
-    doAssert numbers.distribute(6)[0] == @[1, 2]
-    doAssert numbers.distribute(6)[5] == @[7]
-    let a = @[1, 2, 3, 4, 5, 6, 7]
-    doAssert a.distribute(1, true)   == @[@[1, 2, 3, 4, 5, 6, 7]]
-    doAssert a.distribute(1, false)  == @[@[1, 2, 3, 4, 5, 6, 7]]
-    doAssert a.distribute(2, true)   == @[@[1, 2, 3, 4], @[5, 6, 7]]
-    doAssert a.distribute(2, false)  == @[@[1, 2, 3, 4], @[5, 6, 7]]
-    doAssert a.distribute(3, true)   == @[@[1, 2, 3], @[4, 5], @[6, 7]]
-    doAssert a.distribute(3, false)  == @[@[1, 2, 3], @[4, 5, 6], @[7]]
-    doAssert a.distribute(4, true)   == @[@[1, 2], @[3, 4], @[5, 6], @[7]]
-    doAssert a.distribute(4, false)  == @[@[1, 2], @[3, 4], @[5, 6], @[7]]
-    doAssert a.distribute(5, true)   == @[@[1, 2], @[3, 4], @[5], @[6], @[7]]
-    doAssert a.distribute(5, false)  == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]]
-    doAssert a.distribute(6, true)   == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]]
-    doAssert a.distribute(6, false)  == @[
-      @[1, 2], @[3, 4], @[5, 6], @[7], @[], @[]]
-    doAssert a.distribute(8, false)  == a.distribute(8, true)
-    doAssert a.distribute(90, false) == a.distribute(90, true)
-    var b = @[0]
-    for f in 1 .. 25: b.add(f)
-    doAssert b.distribute(5, true)[4].len == 5
-    doAssert b.distribute(5, false)[4].len == 2
 
   block: # newSeqWith tests
     var seq2D = newSeqWith(4, newSeq[bool](2))
@@ -850,19 +992,5 @@ when isMainModule:
     seq2D[0][1] = true
     doAssert seq2D == @[@[true, true], @[true, false], @[false, false], @[false, false]]
 
-  block: # cycle tests
-    let
-      a = @[1, 2, 3]
-      b: seq[int] = @[]
-
-    doAssert a.cycle(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3]
-    doAssert a.cycle(0) == @[]
-    #doAssert a.cycle(-1) == @[] # will not compile!
-    doAssert b.cycle(3) == @[]
-
-  block: # repeat tests
-    assert repeat(10, 5) == @[10, 10, 10, 10, 10]
-    assert repeat(@[1,2,3], 2) == @[@[1,2,3], @[1,2,3]]
-
   when not defined(testing):
     echo "Finished doc tests"

lib/pure/collections/sharedstrings.nim

@@ -55,7 +55,7 @@ proc `[]=`*(s: var SharedString; i: Natural; value: char) =
   if i < s.len: s.buffer.data[i+s.first] = value
   else: raise newException(IndexError, "index out of bounds")
 
-proc `[]`*(s: SharedString; ab: Slice[int]): SharedString =
+proc `[]`*(s: SharedString; ab: HSlice[int, int]): SharedString =
   #incRef(src.buffer)
   if ab.a < s.len:
     result.buffer = s.buffer

lib/pure/concurrency/cpuinfo.nim

@@ -45,8 +45,25 @@ proc countProcessors*(): int {.rtl, extern: "ncpi$1".} =
   ## returns the numer of the processors/cores the machine has.
   ## Returns 0 if it cannot be detected.
   when defined(windows):
-    var x = getEnv("NUMBER_OF_PROCESSORS")
+    type
-    if x.len > 0: result = parseInt(x.string)
+      SYSTEM_INFO {.final, pure.} = object
+        u1: int32
+        dwPageSize: int32
+        lpMinimumApplicationAddress: pointer
+        lpMaximumApplicationAddress: pointer
+        dwActiveProcessorMask: ptr int32
+        dwNumberOfProcessors: int32
+        dwProcessorType: int32
+        dwAllocationGranularity: int32
+        wProcessorLevel: int16
+        wProcessorRevision: int16
+
+    proc GetSystemInfo(lpSystemInfo: var SYSTEM_INFO) {.stdcall, dynlib: "kernel32", importc: "GetSystemInfo".}
+
+    var
+      si: SYSTEM_INFO
+    GetSystemInfo(si)
+    result = si.dwNumberOfProcessors
   elif defined(macosx) or defined(bsd):
     var
       mib: array[0..3, cint]

lib/pure/concurrency/threadpool.nim

@@ -448,12 +448,22 @@ proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerProc.} =
     if selectWorker(readyWorker, fn, data): return
     for i in 0..<currentPoolSize:
       if selectWorker(addr(workersData[i]), fn, data): return
+
     # determine what to do, but keep in mind this is expensive too:
     # state.calls < maxPoolSize: warmup phase
     # (state.calls and 127) == 0: periodic check
     if state.calls < maxPoolSize or (state.calls and 127) == 0:
       # ensure the call to 'advice' is atomic:
       if tryAcquire(stateLock):
+        if currentPoolSize < minPoolSize:
+          if not workersData[currentPoolSize].initialized:
+            activateWorkerThread(currentPoolSize)
+          let w = addr(workersData[currentPoolSize])
+          atomicInc currentPoolSize
+          if selectWorker(w, fn, data):
+            release(stateLock)
+            return
+
         case advice(state)
         of doNothing: discard
         of doCreateThread:

lib/pure/httpclient.nim

@@ -883,7 +883,9 @@ proc recvFull(client: HttpClient | AsyncHttpClient, size: int, timeout: int,
       let data = client.socket.recv(sizeToRecv, timeout)
     else:
       let data = await client.socket.recv(sizeToRecv)
-    if data == "": break # We've been disconnected.
+    if data == "":
+      client.close()
+      break # We've been disconnected.
 
     readLen.inc(data.len)
     if keep:
@@ -950,6 +952,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient,
       if length > 0:
         let recvLen = await client.recvFull(length, client.timeout, true)
         if recvLen == 0:
+          client.close()
           httpError("Got disconnected while trying to read body.")
         if recvLen != length:
           httpError("Received length doesn't match expected length. Wanted " &
@@ -962,13 +965,20 @@ proc parseBody(client: HttpClient | AsyncHttpClient,
       if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0":
         while true:
           let recvLen = await client.recvFull(4000, client.timeout, true)
-          if recvLen == 0: break
+          if recvLen == 0:
+            client.close()
+            break
 
   when client is AsyncHttpClient:
     client.bodyStream.complete()
   else:
     client.bodyStream.setPosition(0)
 
+  # If the server will close our connection, then no matter the method of
+  # reading the body, we need to close our socket.
+  if headers.getOrDefault"Connection" == "close":
+    client.close()
+
 proc parseResponse(client: HttpClient | AsyncHttpClient,
                    getBody: bool): Future[Response | AsyncResponse]
                    {.multisync.} =
@@ -984,7 +994,10 @@ proc parseResponse(client: HttpClient | AsyncHttpClient,
       line = await client.socket.recvLine(client.timeout)
     else:
       line = await client.socket.recvLine()
-    if line == "": break # We've been disconnected.
+    if line == "":
+      # We've been disconnected.
+      client.close()
+      break
     if line == "\c\L":
       fullyRead = true
       break
@@ -1033,7 +1046,8 @@ proc newConnection(client: HttpClient | AsyncHttpClient,
                    url: Uri) {.multisync.} =
   if client.currentURL.hostname != url.hostname or
       client.currentURL.scheme != url.scheme or
-      client.currentURL.port != url.port:
+      client.currentURL.port != url.port or
+      (not client.connected):
     let isSsl = url.scheme.toLowerAscii() == "https"
 
     if isSsl and not defined(ssl):

lib/pure/httpcore.nim

@@ -113,6 +113,9 @@ proc newHttpHeaders*(keyValuePairs:
   new result
   result.table = newTable[string, seq[string]](pairs)
 
+proc `$`*(headers: HttpHeaders): string =
+  return $headers.table
+
 proc clear*(headers: HttpHeaders) =
   headers.table.clear()
 

lib/pure/includes/osenv.nim

@@ -94,7 +94,7 @@ proc findEnvVar(key: string): int =
     if startsWith(environment[i], temp): return i
   return -1
 
-proc getEnv*(key: string): TaintedString {.tags: [ReadEnvEffect].} =
+proc getEnv*(key: string, default = ""): TaintedString {.tags: [ReadEnvEffect].} =
   ## Returns the value of the `environment variable`:idx: named `key`.
   ##
   ## If the variable does not exist, "" is returned. To distinguish
@@ -108,7 +108,7 @@ proc getEnv*(key: string): TaintedString {.tags: [ReadEnvEffect].} =
       return TaintedString(substr(environment[i], find(environment[i], '=')+1))
     else:
       var env = c_getenv(key)
-      if env == nil: return TaintedString("")
+      if env == nil: return TaintedString(default)
       result = TaintedString($env)
 
 proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} =

lib/pure/json.nim

@@ -37,15 +37,15 @@
 ## Retrieving the value of a JSON node can then be achieved using one of the
 ## helper procedures, which include:
 ##
-## * ``getNum``
+## * ``getInt``
-## * ``getFNum``
+## * ``getFloat``
 ## * ``getStr``
-## * ``getBVal``
+## * ``getBool``
 ##
 ## To retrieve the value of ``"key"`` you can do the following:
 ##
 ## .. code-block:: Nim
-##   doAssert jsonNode["key"].getFNum() == 3.14
+##   doAssert jsonNode["key"].getFloat() == 3.14
 ##
 ## The ``[]`` operator will raise an exception when the specified field does
 ## not exist. If you wish to avoid this behaviour you can use the ``{}``
@@ -681,14 +681,25 @@ proc getStr*(n: JsonNode, default: string = ""): string =
   if n.isNil or n.kind != JString: return default
   else: return n.str
 
-proc getNum*(n: JsonNode, default: BiggestInt = 0): BiggestInt =
+proc getInt*(n: JsonNode, default: int = 0): int =
   ## Retrieves the int value of a `JInt JsonNode`.
   ##
   ## Returns ``default`` if ``n`` is not a ``JInt``, or if ``n`` is nil.
   if n.isNil or n.kind != JInt: return default
+  else: return int(n.num)
+
+proc getBiggestInt*(n: JsonNode, default: BiggestInt = 0): BiggestInt =
+  ## Retrieves the BiggestInt value of a `JInt JsonNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``JInt``, or if ``n`` is nil.
+  if n.isNil or n.kind != JInt: return default
   else: return n.num
 
-proc getFNum*(n: JsonNode, default: float = 0.0): float =
+proc getNum*(n: JsonNode, default: BiggestInt = 0): BiggestInt {.deprecated.} =
+  ## Deprecated - use getInt or getBiggestInt instead
+  getBiggestInt(n, default)
+
+proc getFloat*(n: JsonNode, default: float = 0.0): float =
   ## Retrieves the float value of a `JFloat JsonNode`.
   ##
   ## Returns ``default`` if ``n`` is not a ``JFloat`` or ``JInt``, or if ``n`` is nil.
@@ -698,13 +709,21 @@ proc getFNum*(n: JsonNode, default: float = 0.0): float =
   of JInt: return float(n.num)
   else: return default
 
-proc getBVal*(n: JsonNode, default: bool = false): bool =
+proc getFNum*(n: JsonNode, default: float = 0.0): float {.deprecated.} =
+  ## Deprecated - use getFloat instead
+  getFloat(n, default)
+
+proc getBool*(n: JsonNode, default: bool = false): bool =
   ## Retrieves the bool value of a `JBool JsonNode`.
   ##
   ## Returns ``default`` if ``n`` is not a ``JBool``, or if ``n`` is nil.
   if n.isNil or n.kind != JBool: return default
   else: return n.bval
 
+proc getBVal*(n: JsonNode, default: bool = false): bool {.deprecated.} =
+  ## Deprecated - use getBVal instead
+  getBool(n, default)
+
 proc getFields*(n: JsonNode,
     default = initOrderedTable[string, JsonNode](4)):
         OrderedTable[string, JsonNode] =
@@ -1342,7 +1361,7 @@ proc getEnum(node: JsonNode, ast: string, T: typedesc): T =
     # TODO: I shouldn't need this proc.
     proc convert[T](x: BiggestInt): T = T(x)
     verifyJsonKind(node, {JInt}, ast)
-    return convert[T](node.getNum())
+    return convert[T](node.getBiggestInt())
   else:
     verifyJsonKind(node, {JString}, ast)
     return parseEnum[T](node.getStr())

lib/pure/lexbase.nim

@@ -37,6 +37,7 @@ type
     lineNumber*: int          ## the current line number
     sentinel: int
     lineStart: int            # index of last line start in buffer
+    offsetBase*: int          # use ``offsetBase + bufpos`` to get the offset
     refillChars: set[char]
 
 {.deprecated: [TBaseLexer: BaseLexer].}
@@ -107,7 +108,8 @@ proc fillBaseLexer(L: var BaseLexer, pos: int): int =
     result = pos + 1          # nothing to do
   else:
     fillBuffer(L)
-    L.bufpos = 0              # XXX: is this really correct?
+    L.offsetBase += pos
+    L.bufpos = 0
     result = 0
 
 proc handleCR*(L: var BaseLexer, pos: int): int =
@@ -147,6 +149,7 @@ proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192;
   assert(input != nil)
   L.input = input
   L.bufpos = 0
+  L.offsetBase = 0
   L.bufLen = bufLen
   L.refillChars = refillChars
   when defined(js):

lib/pure/marshal.nim

@@ -283,7 +283,7 @@ proc to*[T](data: string): T =
   loadAny(newStringStream(data), toAny(result), tab)
 
 when not defined(testing) and isMainModule:
-  template testit(x: expr) = echo($$to[type(x)]($$x))
+  template testit(x: untyped) = echo($$to[type(x)]($$x))
 
   var x: array[0..4, array[0..4, string]] = [
     ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"],

lib/pure/matchers.nim

@@ -48,7 +48,7 @@ proc validEmailAddress*(s: string): bool {.noSideEffect,
      "aero", "jobs", "museum": return true
   else: return false
 
-proc parseInt*(s: string, value: var int, validRange: Slice[int]) {.
+proc parseInt*(s: string, value: var int, validRange: HSlice[int, int]) {.
   noSideEffect, rtl, extern: "nmatchParseInt".} =
   ## parses `s` into an integer in the range `validRange`. If successful,
   ## `value` is modified to contain the result. Otherwise no exception is

lib/pure/math.nim

@@ -184,6 +184,8 @@ when not defined(JS):
   proc pow*(x, y: float32): float32 {.importc: "powf", header: "<math.h>".}
   proc pow*(x, y: float64): float64 {.importc: "pow", header: "<math.h>".}
     ## computes x to power raised of y.
+    ##
+    ## To compute power between integers, use `^` e.g. 2 ^ 6
 
   proc erf*(x: float32): float32 {.importc: "erff", header: "<math.h>".}
   proc erf*(x: float64): float64 {.importc: "erf", header: "<math.h>".}

lib/pure/nativesockets.nim

@@ -496,11 +496,12 @@ proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) =
                    addr(namelen)) == -1'i32:
       raiseOSError(osLastError())
     # Cannot use INET6_ADDRSTRLEN here, because it's a C define.
-    var buf: array[64, char]
+    result[0] = newString(64)
     if inet_ntop(name.sin6_family.cint,
-                 addr name.sin6_addr, buf.cstring, sizeof(buf).int32).isNil:
+                 addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil:
       raiseOSError(osLastError())
-    result = ($buf, Port(nativesockets.ntohs(name.sin6_port)))
+    setLen(result[0], result[0].cstring.len)
+    result[1] = Port(nativesockets.ntohs(name.sin6_port))
   else:
     raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr")
 
@@ -532,11 +533,12 @@ proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) =
                    addr(namelen)) == -1'i32:
       raiseOSError(osLastError())
     # Cannot use INET6_ADDRSTRLEN here, because it's a C define.
-    var buf: array[64, char]
+    result[0] = newString(64)
     if inet_ntop(name.sin6_family.cint,
-                 addr name.sin6_addr, buf.cstring, sizeof(buf).int32).isNil:
+                 addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil:
       raiseOSError(osLastError())
-    result = ($buf, Port(nativesockets.ntohs(name.sin6_port)))
+    setLen(result[0], result[0].cstring.len)
+    result[1] = Port(nativesockets.ntohs(name.sin6_port))
   else:
     raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr")
 

lib/pure/net.nim

@@ -857,13 +857,16 @@ proc close*(socket: Socket) =
         # shutdown i.e not wait for the peers "close notify" alert with a second
         # call to SSLShutdown
         let res = SSLShutdown(socket.sslHandle)
-        SSLFree(socket.sslHandle)
-        socket.sslHandle = nil
         if res == 0:
           discard
         elif res != 1:
           socketError(socket, res)
   finally:
+    when defineSsl:
+      if socket.isSSL and socket.sslHandle != nil:
+        SSLFree(socket.sslHandle)
+        socket.sslHandle = nil
+
     socket.fd.close()
 
 proc toCInt*(opt: SOBool): cint =