Merge branch 'devel' into async-improvements

.gitlab-ci.yml

@@ -8,7 +8,7 @@ stages:
 
 .linux_set_path: &linux_set_path_def
   before_script:
-    - export PATH=$(pwd)/bin:$PATH
+    - export PATH=$(pwd)/bin${PATH:+:$PATH}
   tags:
     - linux
 

.travis.yml

@@ -30,7 +30,7 @@ before_script:
   - sh build.sh
   - cd ..
   - sed -i -e 's,cc = gcc,cc = clang,' config/nim.cfg
-  - export PATH=$(pwd)/bin:$PATH
+  - export PATH=$(pwd)/bin${PATH:+:$PATH}
 script:
   - nim c koch
   - ./koch boot
@@ -48,3 +48,5 @@ script:
   - ./koch csource
   - ./koch nimsuggest
 #  - nim c -r nimsuggest/tester
+  - ( ! grep -F '.. code-block' -l -r --include '*.html' --exclude contributing.html --exclude docgen.html --exclude tut2.html )
+  - ( ! grep -F '..code-block' -l -r --include '*.html' --exclude contributing.html --exclude docgen.html --exclude tut2.html )

changelog.md

@@ -53,36 +53,37 @@
   what to return if the environment variable does not exist.
 - Bodies of ``for`` loops now get their own scope:
 
-.. code-block:: nim
+```nim
   # now compiles:
   for i in 0..4:
     let i = i + 1
     echo i
+```
 
 - The parsing rules of ``if`` expressions were changed so that multiple
   statements are allowed in the branches. We found few code examples that
   now fail because of this change, but here is one:
 
-.. code-block:: nim
+```nim
-
   t[ti] = if exp_negative: '-' else: '+'; inc(ti)
+```
 
 This now needs to be written as:
 
-.. code-block:: nim
+```nim
-
   t[ti] = (if exp_negative: '-' else: '+'); inc(ti)
+```
 
 - To make Nim even more robust the system iterators ``..`` and ``countup``
   now only accept a single generic type ``T``. This means the following code
   doesn't die with an "out of range" error anymore:
 
-.. code-block:: nim
+```nim
-
   var b = 5.Natural
   var a = -5
   for i in a..b:
     echo i
+```
 
 - ``formatFloat``/``formatBiggestFloat`` now support formatting floats with zero
   precision digits. The previous ``precision = 0`` behavior (default formatting)
@@ -115,13 +116,17 @@ This now needs to be written as:
 - Nim's ``rst2html`` command now supports the testing of code snippets via an RST
   extension that we called ``:test:``::
 
+  ```rst
     .. code-block:: nim
         :test:
       # shows how the 'if' statement works
       if true: echo "yes"
+  ```
 - The ``[]`` proc for strings now raises an ``IndexError`` exception when
   the specified slice is out of bounds. See issue
   [#6223](https://github.com/nim-lang/Nim/issues/6223) for more details.
+  You can use ``substr(str, start, finish)`` to get the old behaviour back,
+  see [this commit](https://github.com/nim-lang/nimbot/commit/98cc031a27ea89947daa7f0bb536bcf86462941f) for an example.
 - ``strutils.split`` and ``strutils.rsplit`` with an empty string and a
   separator now returns that empty string.
   See issue [#4377](https://github.com/nim-lang/Nim/issues/4377).
@@ -137,3 +142,68 @@ This now needs to be written as:
   to [http://www.gii.upv.es/tlsf/](http://www.gii.upv.es/tlsf/) the maximum
   fragmentation measured is lower than 25%. As a nice bonus ``alloc`` and
   ``dealloc`` became O(1) operations.
+- The behavior of ``$`` has been changed for all standard library collections. The
+  collection-to-string implementations now perform proper quoting and escaping of
+  strings and chars.
+- The ``random`` procs in ``random.nim`` have all been deprecated. Instead use
+  the new ``rand`` procs. The module now exports the state of the random
+  number generator as type ``Rand`` so multiple threads can easily use their
+  own random number generators that do not require locking. For more information
+  about this rename see issue [#6934](https://github.com/nim-lang/Nim/issues/6934)
+- The compiler is now more consistent in its treatment of ambiguous symbols:
+  Types that shadow procs and vice versa are marked as ambiguous (bug #6693).
+- ``yield`` (or ``await`` which is mapped to ``yield``) never worked reliably
+  in an array, seq or object constructor and is now prevented at compile-time.
+- For string formatting / interpolation a new module
+  called [strformat](https://nim-lang.org/docs/strformat.html) has been added
+  to the stdlib.
+- codegenDecl pragma now works for the JavaScript backend. It returns an empty string for
+  function return type placeholders.
+- Asynchronous programming for the JavaScript backend using the `asyncjs` module.
+- Extra semantic checks for procs with noreturn pragma: return type is not allowed,
+  statements after call to noreturn procs are no longer allowed.
+- Noreturn proc calls and raising exceptions branches are now skipped during common type
+  deduction in if and case expressions. The following code snippets now compile:
+```nim
+import strutils
+let str = "Y"
+let a = case str:
+  of "Y": true
+  of "N": false
+  else: raise newException(ValueError, "Invalid boolean")
+let b = case str:
+  of nil, "": raise newException(ValueError, "Invalid boolean")
+  elif str.startsWith("Y"): true
+  elif str.startsWith("N"): false
+  else: false
+let c = if str == "Y": true
+  elif str == "N": false
+  else:
+    echo "invalid bool"
+    quit("this is the end")
+```
+- Proc [toCountTable](https://nim-lang.org/docs/tables.html#toCountTable,openArray[A]) now produces a `CountTable` with values correspoding to the number of occurrences of the key in the input. It used to produce a table with all values set to `1`.
+
+Counting occurrences in a sequence used to be:
+
+```nim
+let mySeq = @[1, 2, 1, 3, 1, 4]
+var myCounter = initCountTable[int]()
+
+for item in mySeq:
+  myCounter.inc item
+```
+
+Now, you can simply do:
+
+```nim
+let
+  mySeq = @[1, 2, 1, 3, 1, 4]
+  myCounter = mySeq.toCountTable()
+```
+
+- Added support for casting between integers of same bitsize in VM (compile time and nimscript).
+  This allow to among other things to reinterpret signed integers as unsigned.
+- Pragmas now support call syntax, for example: ``{.exportc"myname".}`` and ``{.exportc("myname").}``
+- Custom pragmas are now supported using pragma ``pragma``, please see language manual for details
+

ci/build.sh

@@ -6,7 +6,7 @@ cd csources
 sh build.sh
 cd ..
 # Add Nim to the PATH
-export PATH=$(pwd)/bin:$PATH
+export PATH=$(pwd)/bin${PATH:+:$PATH}
 # Bootstrap.
 nim -v
 nim c koch

ci/deps.sh

@@ -7,7 +7,7 @@ apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev li
 
 gcc -v
 
-export PATH=$(pwd)/bin:$PATH
+export PATH=$(pwd)/bin${PATH:+:$PATH}
 
 # Nimble deps
 nim e install_nimble.nims

compiler/aliases.nim

@@ -49,7 +49,7 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult =
     if a.sons[0] != nil:
       result = isPartOfAux(a.sons[0].skipTypes(skipPtrs), b, marker)
     if result == arNo: result = isPartOfAux(a.n, b, marker)
-  of tyGenericInst, tyDistinct, tyAlias:
+  of tyGenericInst, tyDistinct, tyAlias, tySink:
     result = isPartOfAux(lastSon(a), b, marker)
   of tyArray, tySet, tyTuple:
     for i in countup(0, sonsLen(a) - 1):
@@ -179,5 +179,11 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
           result = isPartOf(a[0], b)
           if result == arNo: result = arMaybe
       else: discard
+    of nkObjConstr:
+      result = arNo
+      for i in 1..<b.len:
+        let res = isPartOf(a, b[i][1])
+        if res != arNo:
+          result = res
+          if res == arYes: break
     else: discard
-

compiler/ast.nim

@@ -62,8 +62,8 @@ type
     nkTripleStrLit,       # a triple string literal """
     nkNilLit,             # the nil literal
                           # end of atoms
-    nkMetaNode_Obsolete,  # difficult to explain; represents itself
+    nkComesFrom,          # "comes from" template/macro information for
-                          # (used for macros)
+                          # better stack trace generation
     nkDotCall,            # used to temporarily flag a nkCall node;
                           # this is used
                           # for transforming ``s.len`` to ``len(s)``
@@ -305,6 +305,7 @@ const
   sfEscapes* = sfProcvar              # param escapes
   sfBase* = sfDiscriminant
   sfIsSelf* = sfOverriden             # param is 'self'
+  sfCustomPragma* = sfRegister        # symbol is custom pragma template
 
 const
   # getting ready for the future expr/stmt merge
@@ -354,7 +355,7 @@ type
     tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers
     tyFloat, tyFloat32, tyFloat64, tyFloat128,
     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64,
-    tyOptAsRef, tyUnused1, tyUnused2,
+    tyOptAsRef, tySink, tyLent,
     tyVarargs,
     tyUnused,
     tyProxy # used as errornous type (for idetools)
@@ -639,7 +640,8 @@ type
     mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl,
     mNHint, mNWarning, mNError,
     mInstantiationInfo, mGetTypeInfo, mNGenSym,
-    mNimvm, mIntDefine, mStrDefine, mRunnableExamples
+    mNimvm, mIntDefine, mStrDefine, mRunnableExamples,
+    mException, mBuiltinType
 
 # things that we can evaluate safely at compile time, even if not asked for it:
 const
@@ -939,13 +941,13 @@ const
     tyGenericParam}
 
   StructuralEquivTypes*: TTypeKinds = {tyNil, tyTuple, tyArray,
-    tySet, tyRange, tyPtr, tyRef, tyVar, tySequence, tyProc, tyOpenArray,
+    tySet, tyRange, tyPtr, tyRef, tyVar, tyLent, tySequence, tyProc, tyOpenArray,
     tyVarargs}
 
   ConcreteTypes*: TTypeKinds = { # types of the expr that may occur in::
                                  # var x = expr
     tyBool, tyChar, tyEnum, tyArray, tyObject,
-    tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tySequence, tyProc,
+    tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tyLent, tySequence, tyProc,
     tyPointer,
     tyOpenArray, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
     tyUInt..tyUInt64}
@@ -1426,7 +1428,7 @@ proc propagateToOwner*(owner, elem: PType) =
     owner.flags.incl tfHasMeta
 
   if tfHasAsgn in elem.flags:
-    let o2 = owner.skipTypes({tyGenericInst, tyAlias})
+    let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink})
     if o2.kind in {tyTuple, tyObject, tyArray,
                    tySequence, tyOpt, tySet, tyDistinct}:
       o2.flags.incl tfHasAsgn
@@ -1434,7 +1436,7 @@ proc propagateToOwner*(owner, elem: PType) =
 
   if owner.kind notin {tyProc, tyGenericInst, tyGenericBody,
                        tyGenericInvocation, tyPtr}:
-    let elemB = elem.skipTypes({tyGenericInst, tyAlias})
+    let elemB = elem.skipTypes({tyGenericInst, tyAlias, tySink})
     if elemB.isGCedMem or tfHasGCedMem in elemB.flags:
       # for simplicity, we propagate this flag even to generics. We then
       # ensure this doesn't bite us in sempass2.

compiler/ccgcalls.nim

@@ -71,7 +71,7 @@ proc isInCurrentFrame(p: BProc, n: PNode): bool =
     if n.sym.kind in {skVar, skResult, skTemp, skLet} and p.prc != nil:
       result = p.prc.id == n.sym.owner.id
   of nkDotExpr, nkBracketExpr:
-    if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyPtr,tyRef}:
+    if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyLent,tyPtr,tyRef}:
       result = isInCurrentFrame(p, n.sons[0])
   of nkHiddenStdConv, nkHiddenSubConv, nkConv:
     result = isInCurrentFrame(p, n.sons[1])
@@ -331,7 +331,7 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   # skip the deref:
   var ri = ri[i]
   while ri.kind == nkObjDownConv: ri = ri[0]
-  let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias})
+  let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias, tySink})
   if t.kind == tyVar:
     let x = if ri.kind == nkHiddenAddr: ri[0] else: ri
     if x.typ.kind == tyPtr:
@@ -527,7 +527,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     line(p, cpsStmts, pl)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias}).callConv == ccClosure:
+  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
     genClosureCall(p, nil, e, d)
   elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags:
     genInfixCall(p, nil, e, d)
@@ -538,7 +538,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) =
   postStmtActions(p)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias}).callConv == ccClosure:
+  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
   elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags:
     genInfixCall(p, le, ri, d)

compiler/ccgexprs.nim

@@ -149,7 +149,7 @@ proc getStorageLoc(n: PNode): TStorageLoc =
     else: result = OnUnknown
   of nkDerefExpr, nkHiddenDeref:
     case n.sons[0].typ.kind
-    of tyVar: result = OnUnknown
+    of tyVar, tyLent: result = OnUnknown
     of tyPtr: result = OnStack
     of tyRef: result = OnHeap
     else: internalError(n.info, "getStorageLoc")
@@ -368,7 +368,7 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     else:
       linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
   of tyPtr, tyPointer, tyChar, tyBool, tyEnum, tyCString,
-     tyInt..tyUInt64, tyRange, tyVar:
+     tyInt..tyUInt64, tyRange, tyVar, tyLent:
     linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
   else: internalError("genAssignment: " & $ty.kind)
 
@@ -683,9 +683,10 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) =
       d.storage = OnHeap
   else:
     var a: TLoc
-    var typ = skipTypes(e.sons[0].typ, abstractInst)
+    var typ = e.sons[0].typ
     if typ.kind in {tyUserTypeClass, tyUserTypeClassInst} and typ.isResolvedUserTypeClass:
       typ = typ.lastSon
+    typ = typ.skipTypes(abstractInst)
     if typ.kind == tyVar and tfVarIsPtr notin typ.flags and p.module.compileToCpp and e.sons[0].kind == nkHiddenAddr:
       initLocExprSingleUse(p, e[0][0], d)
       return
@@ -849,7 +850,7 @@ proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   var a, b: TLoc
   initLocExpr(p, x, a)
   initLocExpr(p, y, b)
-  var ty = skipTypes(skipTypes(a.t, abstractVarRange), abstractPtrs)
+  var ty = skipTypes(a.t, abstractVarRange + abstractPtrs + tyUserTypeClasses)
   var first = intLiteral(firstOrd(ty))
   # emit range check:
   if optBoundsCheck in p.options and tfUncheckedArray notin ty.flags:
@@ -1201,18 +1202,30 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
   # we skip this step here:
   if not p.module.compileToCpp:
     if handleConstExpr(p, e, d): return
-  var tmp: TLoc
   var t = e.typ.skipTypes(abstractInst)
-  getTemp(p, t, tmp)
   let isRef = t.kind == tyRef
-  var r = rdLoc(tmp)
+
-  if isRef:
+  # check if we need to construct the object in a temporary
-    rawGenNew(p, tmp, nil)
+  var useTemp =
-    t = t.lastSon.skipTypes(abstractInst)
+        isRef or
-    r = "(*$1)" % [r]
+        (d.k notin {locTemp,locLocalVar,locGlobalVar,locParam,locField}) or
-    gcUsage(e)
+        (isPartOf(d.lode, e) != arNo)
+
+  var tmp: TLoc
+  var r: Rope
+  if useTemp:
+    getTemp(p, t, tmp)
+    r = rdLoc(tmp)
+    if isRef:
+      rawGenNew(p, tmp, nil)
+      t = t.lastSon.skipTypes(abstractInst)
+      r = "(*$1)" % [r]
+      gcUsage(e)
+    else:
+      constructLoc(p, tmp)
   else:
-    constructLoc(p, tmp)
+    resetLoc(p, d)
+    r = rdLoc(d)
   discard getTypeDesc(p.module, t)
   let ty = getUniqueType(t)
   for i in 1 ..< e.len:
@@ -1226,28 +1239,46 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
       genFieldCheck(p, it.sons[2], r, field)
     add(tmp2.r, ".")
     add(tmp2.r, field.loc.r)
-    tmp2.k = locTemp
+    if useTemp:
+      tmp2.k = locTemp
+      tmp2.storage = if isRef: OnHeap else: OnStack
+    else:
+      tmp2.k = d.k
+      tmp2.storage = if isRef: OnHeap else: d.storage
     tmp2.lode = it.sons[1]
-    tmp2.storage = if isRef: OnHeap else: OnStack
     expr(p, it.sons[1], tmp2)
+  if useTemp:
+    if d.k == locNone:
+      d = tmp
+    else:
+      genAssignment(p, d, tmp, {})
 
-  if d.k == locNone:
+proc lhsDoesAlias(a, b: PNode): bool =
-    d = tmp
+  for y in b:
-  else:
+    if isPartOf(a, y) != arNo: return true
-    genAssignment(p, d, tmp, {})
 
 proc genSeqConstr(p: BProc, n: PNode, d: var TLoc) =
-  var arr: TLoc
+  var arr, tmp: TLoc
-  if d.k == locNone:
+  # bug #668
+  let doesAlias = lhsDoesAlias(d.lode, n)
+  let dest = if doesAlias: addr(tmp) else: addr(d)
+  if doesAlias:
+    getTemp(p, n.typ, tmp)
+  elif d.k == locNone:
     getTemp(p, n.typ, d)
   # generate call to newSeq before adding the elements per hand:
-  genNewSeqAux(p, d, intLiteral(sonsLen(n)))
+  genNewSeqAux(p, dest[], intLiteral(sonsLen(n)))
   for i in countup(0, sonsLen(n) - 1):
     initLoc(arr, locExpr, n[i], OnHeap)
-    arr.r = rfmt(nil, "$1->data[$2]", rdLoc(d), intLiteral(i))
+    arr.r = rfmt(nil, "$1->data[$2]", rdLoc(dest[]), intLiteral(i))
     arr.storage = OnHeap            # we know that sequences are on the heap
     expr(p, n[i], arr)
   gcUsage(n)
+  if doesAlias:
+    if d.k == locNone:
+      d = tmp
+    else:
+      genAssignment(p, d, tmp, {})
 
 proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) =
   var elem, a, arr: TLoc
@@ -1692,7 +1723,7 @@ proc genRangeChck(p: BProc, n: PNode, d: var TLoc, magic: string) =
         rope(magic)]), a.storage)
 
 proc genConv(p: BProc, e: PNode, d: var TLoc) =
-  let destType = e.typ.skipTypes({tyVar, tyGenericInst, tyAlias})
+  let destType = e.typ.skipTypes({tyVar, tyGenericInst, tyAlias, tySink})
   if sameBackendType(destType, e.sons[1].typ):
     expr(p, e.sons[1], d)
   else:
@@ -1768,7 +1799,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
                                                "$# = #subInt64($#, $#);$n"]
     const fun: array[mInc..mDec, string] = ["$# = #addInt($#, $#);$n",
                                              "$# = #subInt($#, $#);$n"]
-    let underlying = skipTypes(e.sons[1].typ, {tyGenericInst, tyAlias, tyVar, tyRange})
+    let underlying = skipTypes(e.sons[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyRange})
     if optOverflowCheck notin p.options or underlying.kind in {tyUInt..tyUInt64}:
       binaryStmt(p, e, d, opr[op])
     else:
@@ -1778,7 +1809,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       initLocExpr(p, e.sons[1], a)
       initLocExpr(p, e.sons[2], b)
 
-      let ranged = skipTypes(e.sons[1].typ, {tyGenericInst, tyAlias, tyVar})
+      let ranged = skipTypes(e.sons[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent})
       let res = binaryArithOverflowRaw(p, ranged, a, b,
         if underlying.kind == tyInt64: fun64[op] else: fun[op])
       putIntoDest(p, a, e.sons[1], "($#)($#)" % [
@@ -1968,10 +1999,35 @@ proc genComplexConst(p: BProc, sym: PSym, d: var TLoc) =
   assert((sym.loc.r != nil) and (sym.loc.t != nil))
   putLocIntoDest(p, d, sym.loc)
 
+template genStmtListExprImpl(exprOrStmt) {.dirty.} =
+  #let hasNimFrame = magicsys.getCompilerProc("nimFrame") != nil
+  let hasNimFrame = p.prc != nil and
+      sfSystemModule notin p.module.module.flags and
+      optStackTrace in p.prc.options
+  var frameName: Rope = nil
+  for i in 0 .. n.len - 2:
+    let it = n[i]
+    if it.kind == nkComesFrom:
+      if hasNimFrame and frameName == nil:
+        inc p.labels
+        frameName = "FR" & rope(p.labels) & "_"
+        let theMacro = it[0].sym
+        add p.s(cpsStmts), initFrameNoDebug(p, frameName,
+           makeCString theMacro.name.s,
+           theMacro.info.quotedFilename, it.info.line)
+    else:
+      genStmts(p, it)
+  if n.len > 0: exprOrStmt
+  if frameName != nil:
+    add p.s(cpsStmts), deinitFrameNoDebug(p, frameName)
+
 proc genStmtListExpr(p: BProc, n: PNode, d: var TLoc) =
-  var length = sonsLen(n)
+  genStmtListExprImpl:
-  for i in countup(0, length - 2): genStmts(p, n.sons[i])
+    expr(p, n[n.len - 1], d)
-  if length > 0: expr(p, n.sons[length - 1], d)
+
+proc genStmtList(p: BProc, n: PNode) =
+  genStmtListExprImpl:
+    genStmts(p, n[n.len - 1])
 
 proc upConv(p: BProc, n: PNode, d: var TLoc) =
   var a: TLoc
@@ -1981,9 +2037,9 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) =
     var r = rdLoc(a)
     var nilCheck: Rope = nil
     var t = skipTypes(a.t, abstractInst)
-    while t.kind in {tyVar, tyPtr, tyRef}:
+    while t.kind in {tyVar, tyLent, tyPtr, tyRef}:
-      if t.kind != tyVar: nilCheck = r
+      if t.kind notin {tyVar, tyLent}: nilCheck = r
-      if t.kind != tyVar or not p.module.compileToCpp:
+      if t.kind notin {tyVar, tyLent} or not p.module.compileToCpp:
         r = "(*$1)" % [r]
       t = skipTypes(t.lastSon, abstractInst)
     if not p.module.compileToCpp:
@@ -2016,7 +2072,7 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) =
     var a: TLoc
     initLocExpr(p, arg, a)
     var r = rdLoc(a)
-    let isRef = skipTypes(arg.typ, abstractInst).kind in {tyRef, tyPtr, tyVar}
+    let isRef = skipTypes(arg.typ, abstractInst).kind in {tyRef, tyPtr, tyVar, tyLent}
     if isRef:
       add(r, "->Sup")
     else:
@@ -2028,7 +2084,7 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) =
       # (see bug #837). However sometimes using a temporary is not correct:
       # init(TFigure(my)) # where it is passed to a 'var TFigure'. We test
       # this by ensuring the destination is also a pointer:
-      if d.k == locNone and skipTypes(n.typ, abstractInst).kind in {tyRef, tyPtr, tyVar}:
+      if d.k == locNone and skipTypes(n.typ, abstractInst).kind in {tyRef, tyPtr, tyVar, tyLent}:
         getTemp(p, n.typ, d)
         linefmt(p, cpsStmts, "$1 = &$2;$n", rdLoc(d), r)
       else:
@@ -2170,8 +2226,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkCheckedFieldExpr: genCheckedRecordField(p, n, d)
   of nkBlockExpr, nkBlockStmt: genBlock(p, n, d)
   of nkStmtListExpr: genStmtListExpr(p, n, d)
-  of nkStmtList:
+  of nkStmtList: genStmtList(p, n)
-    for i in countup(0, sonsLen(n) - 1): genStmts(p, n.sons[i])
   of nkIfExpr, nkIfStmt: genIf(p, n, d)
   of nkWhen:
     # This should be a "when nimvm" node.
@@ -2268,7 +2323,7 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope =
   of tyBool: result = rope"NIM_FALSE"
   of tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64: result = rope"0"
   of tyFloat..tyFloat128: result = rope"0.0"
-  of tyCString, tyString, tyVar, tyPointer, tyPtr, tySequence, tyExpr,
+  of tyCString, tyString, tyVar, tyLent, tyPointer, tyPtr, tySequence, tyExpr,
      tyStmt, tyTypeDesc, tyStatic, tyRef, tyNil:
     result = rope"NIM_NIL"
   of tyProc:

compiler/ccgmerge.nim

@@ -96,7 +96,7 @@ proc writeIntSet(a: IntSet, s: var string) =
   s.add('}')
 
 proc genMergeInfo*(m: BModule): Rope =
-  if optSymbolFiles notin gGlobalOptions: return nil
+  if not compilationCachePresent: return nil
   var s = "/*\tNIM_merge_INFO:"
   s.add(tnl)
   s.add("typeCache:{")

compiler/ccgstmts.nim

@@ -21,8 +21,12 @@ proc registerGcRoot(p: BProc, v: PSym) =
     # we register a specialized marked proc here; this has the advantage
     # that it works out of the box for thread local storage then :-)
     let prc = genTraverseProcForGlobal(p.module, v, v.info)
-    appcg(p.module, p.module.initProc.procSec(cpsInit),
+    if sfThread in v.flags:
-      "#nimRegisterGlobalMarker($1);$n", [prc])
+      appcg(p.module, p.module.initProc.procSec(cpsInit),
+        "#nimRegisterThreadLocalMarker($1);$n", [prc])
+    else:
+      appcg(p.module, p.module.initProc.procSec(cpsInit),
+        "#nimRegisterGlobalMarker($1);$n", [prc])
 
 proc isAssignedImmediately(n: PNode): bool {.inline.} =
   if n.kind == nkEmpty: return false
@@ -564,9 +568,6 @@ proc genBreakStmt(p: BProc, t: PNode) =
   genLineDir(p, t)
   lineF(p, cpsStmts, "goto $1;$n", [label])
 
-proc getRaiseFrmt(p: BProc): string =
-  result = "#raiseException((#Exception*)$1, $2);$n"
-
 proc genRaiseStmt(p: BProc, t: PNode) =
   if p.inExceptBlock > 0:
     # if the current try stmt have a finally block,
@@ -580,7 +581,8 @@ proc genRaiseStmt(p: BProc, t: PNode) =
     var e = rdLoc(a)
     var typ = skipTypes(t.sons[0].typ, abstractPtrs)
     genLineDir(p, t)
-    lineCg(p, cpsStmts, getRaiseFrmt(p), [e, makeCString(typ.sym.name.s)])
+    lineCg(p, cpsStmts, "#raiseException((#Exception*)$1, $2);$n",
+        [e, makeCString(typ.sym.name.s)])
   else:
     genLineDir(p, t)
     # reraise the last exception:

compiler/ccgtrav.nim

@@ -66,7 +66,8 @@ proc genTraverseProc(c: var TTraversalClosure, accessor: Rope, typ: PType) =
 
   var p = c.p
   case typ.kind
-  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred:
+  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
+     tySink:
     genTraverseProc(c, accessor, lastSon(typ))
   of tyArray:
     let arraySize = lengthOrd(typ.sons[0])

compiler/ccgtypes.nim

@@ -119,7 +119,7 @@ proc scopeMangledParam(p: BProc; param: PSym) =
 
 const
   irrelevantForBackend = {tyGenericBody, tyGenericInst, tyGenericInvocation,
-                          tyDistinct, tyRange, tyStatic, tyAlias, tyInferred}
+                          tyDistinct, tyRange, tyStatic, tyAlias, tySink, tyInferred}
 
 proc typeName(typ: PType): Rope =
   let typ = typ.skipTypes(irrelevantForBackend)
@@ -139,7 +139,7 @@ proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
       t = t.lastSon
     else:
       break
-  let typ = if typ.kind == tyAlias: typ.lastSon else: typ
+  let typ = if typ.kind in {tyAlias, tySink}: typ.lastSon else: typ
   if typ.loc.r == nil:
     typ.loc.r = typ.typeName & $sig
   else:
@@ -170,7 +170,7 @@ proc mapType(typ: PType): TCTypeKind =
     internalAssert typ.isResolvedUserTypeClass
     return mapType(typ.lastSon)
   of tyGenericBody, tyGenericInst, tyGenericParam, tyDistinct, tyOrdinal,
-     tyTypeDesc, tyAlias, tyInferred:
+     tyTypeDesc, tyAlias, tySink, tyInferred:
     result = mapType(lastSon(typ))
   of tyEnum:
     if firstOrd(typ) < 0:
@@ -183,7 +183,7 @@ proc mapType(typ: PType): TCTypeKind =
       of 8: result = ctInt64
       else: internalError("mapType")
   of tyRange: result = mapType(typ.sons[0])
-  of tyPtr, tyVar, tyRef, tyOptAsRef:
+  of tyPtr, tyVar, tyLent, tyRef, tyOptAsRef:
     var base = skipTypes(typ.lastSon, typedescInst)
     case base.kind
     of tyOpenArray, tyArray, tyVarargs: result = ctPtrToArray
@@ -242,7 +242,7 @@ proc isInvalidReturnType(rettype: PType): bool =
     case mapType(rettype)
     of ctArray:
       result = not (skipTypes(rettype, typedescInst).kind in
-          {tyVar, tyRef, tyPtr})
+          {tyVar, tyLent, tyRef, tyPtr})
     of ctStruct:
       let t = skipTypes(rettype, typedescInst)
       if rettype.isImportedCppType or t.isImportedCppType: return false
@@ -328,7 +328,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
   of tyStatic:
     if typ.n != nil: result = getSimpleTypeDesc(m, lastSon typ)
     else: internalError("tyStatic for getSimpleTypeDesc")
-  of tyGenericInst, tyAlias:
+  of tyGenericInst, tyAlias, tySink:
     result = getSimpleTypeDesc(m, lastSon typ)
   else: result = nil
 
@@ -348,7 +348,7 @@ proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope =
     if result == nil: result = cacheGetType(m.typeCache, sig)
 
 proc structOrUnion(t: PType): Rope =
-  let t = t.skipTypes({tyAlias})
+  let t = t.skipTypes({tyAlias, tySink})
   (if tfUnion in t.flags: rope("union") else: rope("struct"))
 
 proc getForwardStructFormat(m: BModule): string =
@@ -396,7 +396,7 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet): Rope =
     result = getTypeDescAux(m, t, check)
 
 proc paramStorageLoc(param: PSym): TStorageLoc =
-  if param.typ.skipTypes({tyVar, tyTypeDesc}).kind notin {
+  if param.typ.skipTypes({tyVar, tyLent, tyTypeDesc}).kind notin {
           tyArray, tyOpenArray, tyVarargs}:
     result = OnStack
   else:
@@ -430,11 +430,11 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
     add(params, param.loc.r)
     # declare the len field for open arrays:
     var arr = param.typ
-    if arr.kind == tyVar: arr = arr.sons[0]
+    if arr.kind in {tyVar, tyLent}: arr = arr.lastSon
     var j = 0
     while arr.kind in {tyOpenArray, tyVarargs}:
       # this fixes the 'sort' bug:
-      if param.typ.kind == tyVar: param.loc.storage = OnUnknown
+      if param.typ.kind in {tyVar, tyLent}: param.loc.storage = OnUnknown
       # need to pass hidden parameter:
       addf(params, ", NI $1Len_$2", [param.loc.r, j.rope])
       inc(j)
@@ -496,7 +496,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
               if hasAttribute in CC[cCompiler].props:
                 add(unionBody, "struct __attribute__((__packed__)){" )
               else:
-                addf(unionBody, "#pragma pack(1)$nstruct{", [])
+                addf(unionBody, "#pragma pack(push, 1)$nstruct{", [])
             add(unionBody, a)
             addf(unionBody, "} $1;$n", [sname])
             if tfPacked in rectype.flags and hasAttribute notin CC[cCompiler].props:
@@ -551,7 +551,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
     if hasAttribute in CC[cCompiler].props:
       result = structOrUnion(typ) & " __attribute__((__packed__))"
     else:
-      result = "#pragma pack(1)" & tnl & structOrUnion(typ)
+      result = "#pragma pack(push, 1)" & tnl & structOrUnion(typ)
   else:
     result = structOrUnion(typ)
 
@@ -641,7 +641,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
     excl(check, t.id)
     return
   case t.kind
-  of tyRef, tyOptAsRef, tyPtr, tyVar:
+  of tyRef, tyOptAsRef, tyPtr, tyVar, tyLent:
     var star = if t.kind == tyVar and tfVarIsPtr notin origTyp.flags and
                     compileToCpp(m): "&" else: "*"
     var et = origTyp.skipTypes(abstractInst).lastSon
@@ -872,7 +872,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       of 1, 2, 4, 8: addf(m.s[cfsTypes], "typedef NU$2 $1;$n", [result, rope(s*8)])
       else: addf(m.s[cfsTypes], "typedef NU8 $1[$2];$n",
              [result, rope(getSize(t))])
-  of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias,
+  of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink,
      tyUserTypeClass, tyUserTypeClassInst, tyInferred:
     result = getTypeDescAux(m, lastSon(t), check)
   else:
@@ -925,6 +925,8 @@ proc genProcHeader(m: BModule, prc: PSym): Rope =
       result.add "N_LIB_EXPORT "
   elif prc.typ.callConv == ccInline:
     result.add "static "
+  elif {sfImportc, sfExportc} * prc.flags == {}:
+    result.add "N_LIB_PRIVATE "
   var check = initIntSet()
   fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown)
   genProcParams(m, prc.typ, rettype, params, check)
@@ -1225,7 +1227,7 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
   m.g.typeInfoMarker[sig] = result
   case t.kind
   of tyEmpty, tyVoid: result = rope"0"
-  of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar:
+  of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent:
     genTypeInfoAuxBase(m, t, t, result, rope"0", info)
   of tyStatic:
     if t.n != nil: result = genTypeInfo(m, lastSon t, info)

compiler/ccgutils.nim

@@ -110,13 +110,13 @@ proc getUniqueType*(key: PType): PType =
     of tyDistinct:
       if key.deepCopy != nil: result = key
       else: result = getUniqueType(lastSon(key))
-    of tyGenericInst, tyOrdinal, tyStatic, tyAlias, tyInferred:
+    of tyGenericInst, tyOrdinal, tyStatic, tyAlias, tySink, tyInferred:
       result = getUniqueType(lastSon(key))
       #let obj = lastSon(key)
       #if obj.sym != nil and obj.sym.name.s == "TOption":
       #  echo "for ", typeToString(key), " I returned "
       #  debug result
-    of tyPtr, tyRef, tyVar:
+    of tyPtr, tyRef, tyVar, tyLent:
       let elemType = lastSon(key)
       if elemType.kind in {tyBool, tyChar, tyInt..tyUInt64}:
         # no canonicalization for integral types, so that e.g. ``ptr pid_t`` is

compiler/cgen.nim

@@ -493,7 +493,32 @@ proc initLocExprSingleUse(p: BProc, e: PNode, result: var TLoc) =
 proc lenField(p: BProc): Rope =
   result = rope(if p.module.compileToCpp: "len" else: "Sup.len")
 
-include ccgcalls, "ccgstmts.nim", "ccgexprs.nim"
+include ccgcalls, "ccgstmts.nim"
+
+proc initFrame(p: BProc, procname, filename: Rope): Rope =
+  discard cgsym(p.module, "nimFrame")
+  if p.maxFrameLen > 0:
+    discard cgsym(p.module, "VarSlot")
+    result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4);$n",
+                  procname, filename, p.maxFrameLen.rope,
+                  p.blocks[0].frameLen.rope)
+  else:
+    result = rfmt(nil, "\tnimfr_($1, $2);$n", procname, filename)
+
+proc initFrameNoDebug(p: BProc; frame, procname, filename: Rope; line: int): Rope =
+  discard cgsym(p.module, "nimFrame")
+  addf(p.blocks[0].sections[cpsLocals], "TFrame $1;$n", [frame])
+  result = rfmt(nil, "\t$1.procname = $2; $1.filename = $3; " &
+                      " $1.line = $4; $1.len = -1; nimFrame(&$1);$n",
+                      frame, procname, filename, rope(line))
+
+proc deinitFrameNoDebug(p: BProc; frame: Rope): Rope =
+  result = rfmt(p.module, "\t#popFrameOfAddr(&$1);$n", frame)
+
+proc deinitFrame(p: BProc): Rope =
+  result = rfmt(p.module, "\t#popFrame();$n")
+
+include ccgexprs
 
 # ----------------------------- dynamic library handling -----------------
 # We don't finalize dynamic libs as the OS does this for us.
@@ -600,7 +625,7 @@ proc symInDynamicLibPartial(m: BModule, sym: PSym) =
   sym.typ.sym = nil           # generate a new name
 
 proc cgsym(m: BModule, name: string): Rope =
-  var sym = magicsys.getCompilerProc(name)
+  let sym = magicsys.getCompilerProc(name)
   if sym != nil:
     case sym.kind
     of skProc, skFunc, skMethod, skConverter, skIterator: genProc(m, sym)
@@ -637,19 +662,6 @@ proc generateHeaders(m: BModule) =
   add(m.s[cfsHeaders], "#undef powerpc" & tnl)
   add(m.s[cfsHeaders], "#undef unix" & tnl)
 
-proc initFrame(p: BProc, procname, filename: Rope): Rope =
-  discard cgsym(p.module, "nimFrame")
-  if p.maxFrameLen > 0:
-    discard cgsym(p.module, "VarSlot")
-    result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4);$n",
-                  procname, filename, p.maxFrameLen.rope,
-                  p.blocks[0].frameLen.rope)
-  else:
-    result = rfmt(nil, "\tnimfr_($1, $2);$n", procname, filename)
-
-proc deinitFrame(p: BProc): Rope =
-  result = rfmt(p.module, "\t#popFrame();$n")
-
 proc closureSetup(p: BProc, prc: PSym) =
   if tfCapturesEnv notin prc.typ.flags: return
   # prc.ast[paramsPos].last contains the type we're after:
@@ -1246,7 +1258,7 @@ proc resetModule*(m: BModule) =
 
   # indicate that this is now cached module
   # the cache will be invalidated by nullifying gModules
-  m.fromCache = true
+  #m.fromCache = true
   m.g = nil
 
   # we keep only the "merge info" information for the module
@@ -1324,7 +1336,6 @@ proc getCFile(m: BModule): string =
 
 proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext =
   injectG(graph.config)
-  assert optSymbolFiles in gGlobalOptions
   var m = newModule(g, module)
   readMergeInfo(getCFile(m), m)
   result = m
@@ -1378,7 +1389,7 @@ proc writeModule(m: BModule, pending: bool) =
   # generate code for the init statements of the module:
   let cfile = getCFile(m)
 
-  if not m.fromCache or optForceFullMake in gGlobalOptions:
+  if m.rd == nil or optForceFullMake in gGlobalOptions:
     genInitCode(m)
     finishTypeDescriptions(m)
     if sfMainModule in m.module.flags:
@@ -1431,6 +1442,10 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
   result = n
   if b == nil or passes.skipCodegen(n): return
   var m = BModule(b)
+  # if the module is cached, we don't regenerate the main proc
+  # nor the dispatchers? But if the dispatchers changed?
+  # XXX emit the dispatchers into its own .c file?
+  if b.rd != nil: return
   if n != nil:
     m.initProc.options = initProcOptions(m)
     genStmts(m.initProc, n)
@@ -1453,10 +1468,10 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
   if g.generatedHeader != nil: finishModule(g.generatedHeader)
   while g.forwardedProcsCounter > 0:
     for m in cgenModules(g):
-      if not m.fromCache:
+      if m.rd == nil:
         finishModule(m)
   for m in cgenModules(g):
-    if m.fromCache:
+    if m.rd != nil:
       m.updateCachedModule
     else:
       m.writeModule(pending=true)

compiler/cgmeth.nim

@@ -68,7 +68,7 @@ proc sameMethodBucket(a, b: PSym): MethodResult =
     while true:
       aa = skipTypes(aa, {tyGenericInst, tyAlias})
       bb = skipTypes(bb, {tyGenericInst, tyAlias})
-      if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef}:
+      if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef, tyLent}:
         aa = aa.lastSon
         bb = bb.lastSon
       else:

compiler/commands.nim

@@ -261,7 +261,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
   of "assertions", "a": result = contains(gOptions, optAssert)
   of "deadcodeelim": result = contains(gGlobalOptions, optDeadCodeElim)
   of "run", "r": result = contains(gGlobalOptions, optRun)
-  of "symbolfiles": result = contains(gGlobalOptions, optSymbolFiles)
+  of "symbolfiles": result = gSymbolFiles != disabledSf
   of "genscript": result = contains(gGlobalOptions, optGenScript)
   of "threads": result = contains(gGlobalOptions, optThreads)
   of "taintmode": result = contains(gGlobalOptions, optTaintMode)
@@ -598,7 +598,13 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(switch, arg, pass, info)
     helpOnError(pass)
   of "symbolfiles":
-    processOnOffSwitchG({optSymbolFiles}, arg, pass, info)
+    case arg.normalize
+    of "on": gSymbolFiles = enabledSf
+    of "off": gSymbolFiles = disabledSf
+    of "writeonly": gSymbolFiles = writeOnlySf
+    of "readonly": gSymbolFiles = readOnlySf
+    of "v2": gSymbolFiles = v2Sf
+    else: localError(info, errOnOrOffExpectedButXFound, arg)
   of "skipcfg":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optSkipConfigFile)
@@ -611,7 +617,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "skipparentcfg":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optSkipParentConfigFiles)
-  of "genscript":
+  of "genscript", "gendeps":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optGenScript)
   of "colors": processOnOffSwitchG({optUseColors}, arg, pass, info)

compiler/condsyms.nim

@@ -112,3 +112,4 @@ proc initDefines*() =
   defineSymbol("nimNewRoof")
   defineSymbol("nimHasRunnableExamples")
   defineSymbol("nimNewDot")
+  defineSymbol("nimHasNilChecks")

compiler/destroyer.nim

@@ -174,7 +174,7 @@ proc patchHead(n: PNode) =
       if n[1].typ.isNil:
         # XXX toptree crashes without this workaround. Figure out why.
         return
-      let t = n[1].typ.skipTypes({tyVar, tyGenericInst, tyAlias, tyInferred})
+      let t = n[1].typ.skipTypes({tyVar, tyLent, tyGenericInst, tyAlias, tySink, tyInferred})
       template patch(op, field) =
         if s.name.s == op and field != nil and field != s:
           n.sons[0].sym = field
@@ -198,15 +198,15 @@ template genOp(opr, opname) =
   result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest))
 
 proc genSink(t: PType; dest: PNode): PNode =
-  let t = t.skipTypes({tyGenericInst, tyAlias})
+  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
   genOp(if t.sink != nil: t.sink else: t.assignment, "=sink")
 
 proc genCopy(t: PType; dest: PNode): PNode =
-  let t = t.skipTypes({tyGenericInst, tyAlias})
+  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
   genOp(t.assignment, "=")
 
 proc genDestroy(t: PType; dest: PNode): PNode =
-  let t = t.skipTypes({tyGenericInst, tyAlias})
+  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
   genOp(t.destructor, "=destroy")
 
 proc addTopVar(c: var Con; v: PNode) =
@@ -296,7 +296,8 @@ proc p(n: PNode; c: var Con): PNode =
     recurse(n, result)
 
 proc injectDestructorCalls*(owner: PSym; n: PNode): PNode =
-  echo "injecting into ", n
+  when defined(nimDebugDestroys):
+    echo "injecting into ", n
   var c: Con
   c.owner = owner
   c.tmp = newSym(skTemp, getIdent":d", owner, n.info)

compiler/dfa.nim

@@ -132,7 +132,7 @@ proc gen(c: var Con; n: PNode) # {.noSideEffect.}
 proc genWhile(c: var Con; n: PNode) =
   # L1:
   #   cond, tmp
-  #   fjmp tmp, L2
+  #   fork tmp, L2
   #   body
   #   jmp L1
   # L2:
@@ -168,15 +168,13 @@ proc genIf(c: var Con, n: PNode) =
   var endings: seq[TPosition] = @[]
   for i in countup(0, len(n) - 1):
     var it = n.sons[i]
+    c.gen(it.sons[0])
     if it.len == 2:
-      c.gen(it.sons[0].sons[1])
+      let elsePos = c.forkI(it.sons[1])
-      var elsePos = c.forkI(it.sons[0].sons[1])
       c.gen(it.sons[1])
       if i < sonsLen(n)-1:
         endings.add(c.gotoI(it.sons[1]))
       c.patch(elsePos)
-    else:
-      c.gen(it.sons[0])
   for endPos in endings: c.patch(endPos)
 
 proc genAndOr(c: var Con; n: PNode) =
@@ -337,100 +335,107 @@ proc gen(c: var Con; n: PNode) =
   else: discard
 
 proc dfa(code: seq[Instr]) =
-  # We aggressively push 'undef' values for every 'use v' instruction
+  var u = newSeq[IntSet](code.len) # usages
-  # until they are eliminated via a 'def v' instructions.
+  var d = newSeq[IntSet](code.len) # defs
-  # If we manage to push one 'undef' to a 'use' instruction, we produce
+  var c = newSeq[IntSet](code.len) # consumed
-  # an error:
+  var backrefs = initTable[int, int]()
-  var undef = initIntSet()
   for i in 0..<code.len:
-    if code[i].kind == use: undef.incl(code[i].sym.id)
+    u[i] = initIntSet()
+    d[i] = initIntSet()
+    c[i] = initIntSet()
+    case code[i].kind
+    of use, useWithinCall: u[i].incl(code[i].sym.id)
+    of def: d[i].incl(code[i].sym.id)
+    of fork, goto:
+      let d = i+code[i].dest
+      backrefs.add(d, i)
 
-  var s = newSeq[IntSet](code.len)
-  for i in 0..<code.len:
-    assign(s[i], undef)
-
-  # In the original paper, W := {0,...,n} is done. This is wasteful, we
-  # have no intention to analyse a program like
-  #
-  # return 3
-  # echo a + b
-  #
-  # any further than necessary.
   var w = @[0]
-  while w.len > 0:
+  var maxIters = 50
-    var pc = w[^1]
+  var someChange = true
+  var takenGotos = initIntSet()
+  var consuming = -1
+  while w.len > 0 and maxIters > 0: # and someChange:
+    dec maxIters
+    var pc = w.pop() # w[^1]
+    var prevPc = -1
     # this simulates a single linear control flow execution:
-    while true:
+    while pc < code.len:
-      # according to the paper, it is better to shrink the working set here
+      if prevPc >= 0:
-      # in this inner loop:
+        someChange = false
-      let widx = w.find(pc)
+        # merge step and test for changes (we compute the fixpoints here):
-      if widx >= 0: w.del(widx)
+        # 'u' needs to be the union of prevPc, pc
+        # 'd' needs to be the intersection of 'pc'
+        for id in u[prevPc]:
+          if not u[pc].containsOrIncl(id):
+            someChange = true
+        # in (a; b) if ``a`` sets ``v`` so does ``b``. The intersection
+        # is only interesting on merge points:
+        for id in d[prevPc]:
+          if not d[pc].containsOrIncl(id):
+            someChange = true
+        # if this is a merge point, we take the intersection of the 'd' sets:
+        if backrefs.hasKey(pc):
+          var intersect = initIntSet()
+          assign(intersect, d[pc])
+          var first = true
+          for prevPc in backrefs.allValues(pc):
+            for def in d[pc]:
+              if def notin d[prevPc]:
+                excl(intersect, def)
+                someChange = true
+                when defined(debugDfa):
+                  echo "Excluding ", pc, " prev ", prevPc
+          assign d[pc], intersect
+      if consuming >= 0:
+        if not c[pc].containsOrIncl(consuming):
+          someChange = true
+        consuming = -1
+
       # our interpretation ![I!]:
-      var sid = -1
+      prevPc = pc
       case code[pc].kind
-      of goto, fork: discard
-      of use, useWithinCall:
-        let sym = code[pc].sym
-        if s[pc].contains(sym.id):
-          localError(code[pc].n.info, "variable read before initialized: " & sym.name.s)
-      of def:
-        sid = code[pc].sym.id
-
-      var pc2: int
-      if code[pc].kind == goto:
-        pc2 = pc + code[pc].dest
-      else:
-        pc2 = pc + 1
-        if code[pc].kind == fork:
-          let l = pc + code[pc].dest
-          if sid >= 0 and s[l].missingOrExcl(sid):
-            w.add l
-
-      if sid >= 0 and s[pc2].missingOrExcl(sid):
-        pc = pc2
-      else:
-        break
-      if pc >= code.len: break
-
-    when false:
-      case code[pc].kind
-      of use:
-        let s = code[pc].sym
-        if undefB.contains(s.id):
-          localError(code[pc].n.info, "variable read before initialized: " & s.name.s)
-          break
-        inc pc
-      of def:
-        let s = code[pc].sym
-        # exclude 'undef' for s for this path through the graph.
-        if not undefB.missingOrExcl(s.id):
-          inc pc
-        else:
-          break
-        #undefB.excl s.id
-        #inc pc
-        when false:
-          let prev = bindings.getOrDefault(s.id)
-          if prev != value:
-            # well now it has a value and we made progress, so
-            bindings[s.id] = value
-            inc pc
-          else:
-            break
-      of fork:
-        let diff = code[pc].dest
-        # we follow pc + 1 and remember the label for later:
-        w.add pc+diff
-        inc pc
       of goto:
-        let diff = code[pc].dest
+        # we must leave endless loops eventually:
-        pc = pc + diff
+        if not takenGotos.containsOrIncl(pc) or someChange:
-      if pc >= code.len: break
+          pc = pc + code[pc].dest
+        else:
+          inc pc
+      of fork:
+        # we follow the next instruction but push the dest onto our "work" stack:
+        #if someChange:
+        w.add pc + code[pc].dest
+        inc pc
+      of use, useWithinCall:
+        #if not d[prevPc].missingOrExcl():
+        # someChange = true
+        consuming = code[pc].sym.id
+        inc pc
+      of def:
+        if not d[pc].containsOrIncl(code[pc].sym.id):
+          someChange = true
+        inc pc
+
+  when defined(useDfa) and defined(debugDfa):
+    for i in 0..<code.len:
+      echo "PC ", i, ": defs: ", d[i], "; uses ", u[i], "; consumes ", c[i]
+
+  # now check the condition we're interested in:
+  for i in 0..<code.len:
+    case code[i].kind
+    of use, useWithinCall:
+      let s = code[i].sym
+      if s.id notin d[i]:
+        localError(code[i].n.info, "usage of uninitialized variable: " & s.name.s)
+      if s.id in c[i]:
+        localError(code[i].n.info, "usage of an already consumed variable: " & s.name.s)
+
+    else: discard
 
 proc dataflowAnalysis*(s: PSym; body: PNode) =
   var c = Con(code: @[], blocks: @[])
   gen(c, body)
-  #echoCfg(c.code)
+  when defined(useDfa) and defined(debugDfa): echoCfg(c.code)
   dfa(c.code)
 
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =

compiler/evalffi.nim

@@ -86,10 +86,10 @@ proc mapType(t: ast.PType): ptr libffi.TType =
     else: result = nil
   of tyFloat, tyFloat64: result = addr libffi.type_double
   of tyFloat32: result = addr libffi.type_float
-  of tyVar, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr,
+  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr,
      tyStmt, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
-  of tyDistinct, tyAlias:
+  of tyDistinct, tyAlias, tySink:
     result = mapType(t.sons[0])
   else:
     result = nil
@@ -112,12 +112,12 @@ template `+!`(x, y: untyped): untyped =
 proc packSize(v: PNode, typ: PType): int =
   ## computes the size of the blob
   case typ.kind
-  of tyPtr, tyRef, tyVar:
+  of tyPtr, tyRef, tyVar, tyLent:
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     else:
       result = sizeof(pointer) + packSize(v.sons[0], typ.lastSon)
-  of tyDistinct, tyGenericInst, tyAlias:
+  of tyDistinct, tyGenericInst, tyAlias, tySink:
     result = packSize(v, typ.sons[0])
   of tyArray:
     # consider: ptr array[0..1000_000, int] which is common for interfacing;
@@ -209,7 +209,7 @@ proc pack(v: PNode, typ: PType, res: pointer) =
       awr(cstring, cstring(v.strVal))
     else:
       globalError(v.info, "cannot map pointer/proc value to FFI")
-  of tyPtr, tyRef, tyVar:
+  of tyPtr, tyRef, tyVar, tyLent:
     if v.kind == nkNilLit:
       # nothing to do since the memory is 0 initialized anyway
       discard
@@ -231,7 +231,7 @@ proc pack(v: PNode, typ: PType, res: pointer) =
     packObject(v, typ, res)
   of tyNil:
     discard
-  of tyDistinct, tyGenericInst, tyAlias:
+  of tyDistinct, tyGenericInst, tyAlias, tySink:
     pack(v, typ.sons[0], res)
   else:
     globalError(v.info, "cannot map value to FFI " & typeToString(v.typ))
@@ -364,7 +364,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
       result = n
     else:
       awi(nkPtrLit, cast[ByteAddress](p))
-  of tyPtr, tyRef, tyVar:
+  of tyPtr, tyRef, tyVar, tyLent:
     let p = rd(pointer, x)
     if p.isNil:
       setNil()
@@ -388,14 +388,14 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
       aws(nkStrLit, $p)
   of tyNil:
     setNil()
-  of tyDistinct, tyGenericInst, tyAlias:
+  of tyDistinct, tyGenericInst, tyAlias, tySink:
     result = unpack(x, typ.lastSon, n)
   else:
     # XXX what to do with 'array' here?
     globalError(n.info, "cannot map value from FFI " & typeToString(typ))
 
 proc fficast*(x: PNode, destTyp: PType): PNode =
-  if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyPointer,
+  if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer,
                                            tyProc, tyCString, tyString,
                                            tySequence}:
     result = newNodeIT(x.kind, x.info, destTyp)

compiler/evaltempl.nim

@@ -109,7 +109,7 @@ proc evalTemplateArgs(n: PNode, s: PSym; fromHlo: bool): PNode =
 var evalTemplateCounter* = 0
   # to prevent endless recursion in templates instantiation
 
-proc wrapInComesFrom*(info: TLineInfo; res: PNode): PNode =
+proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
   when true:
     result = res
     result.info = info
@@ -124,8 +124,12 @@ proc wrapInComesFrom*(info: TLineInfo; res: PNode): PNode =
           if x[i].kind in nkCallKinds:
             x.sons[i].info = info
   else:
-    result = newNodeI(nkPar, info)
+    result = newNodeI(nkStmtListExpr, info)
+    var d = newNodeI(nkComesFrom, info)
+    d.add newSymNode(sym, info)
+    result.add d
     result.add res
+    result.typ = res.typ
 
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; fromHlo=false): PNode =
   inc(evalTemplateCounter)
@@ -156,6 +160,6 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; fromHlo=false): PNode =
     for i in countup(0, safeLen(body) - 1):
       evalTemplateAux(body.sons[i], args, ctx, result)
   result.flags.incl nfFromTemplate
-  result = wrapInComesFrom(n.info, result)
+  result = wrapInComesFrom(n.info, tmpl, result)
   dec(evalTemplateCounter)
 

compiler/extccomp.nim

@@ -21,7 +21,7 @@ import
 type
   TSystemCC* = enum
     ccNone, ccGcc, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc,
-    ccTcc, ccPcc, ccUcc, ccIcl
+    ccTcc, ccPcc, ccUcc, ccIcl, ccIcc
   TInfoCCProp* = enum         # properties of the C compiler:
     hasSwitchRange,           # CC allows ranges in switch statements (GNU C)
     hasComputedGoto,          # CC has computed goto (GNU C extension)
@@ -135,16 +135,18 @@ compiler vcc:
 
 # Intel C/C++ Compiler
 compiler icl:
-  # Intel compilers try to imitate the native ones (gcc and msvc)
+  result = vcc()
-  when defined(windows):
-    result = vcc()
-  else:
-    result = gcc()
-
   result.name = "icl"
   result.compilerExe = "icl"
   result.linkerExe = "icl"
 
+# Intel compilers try to imitate the native ones (gcc and msvc)
+compiler icc:
+  result = gcc()
+  result.name = "icc"
+  result.compilerExe = "icc"
+  result.linkerExe = "icc"
+
 # Local C Compiler
 compiler lcc:
   result = (
@@ -251,7 +253,7 @@ compiler tcc:
     compilerExe: "tcc",
     cppCompiler: "",
     compileTmpl: "-c $options $include -o $objfile $file",
-    buildGui: "UNAVAILABLE!",
+    buildGui: "-Wl,-subsystem=gui",
     buildDll: " -shared",
     buildLib: "", # XXX: not supported yet
     linkerExe: "tcc",
@@ -327,7 +329,8 @@ const
     tcc(),
     pcc(),
     ucc(),
-    icl()]
+    icl(),
+    icc()]
 
   hExt* = ".h"
 
@@ -791,42 +794,40 @@ proc writeJsonBuildInstructions*(projectfile: string) =
     else:
       f.write escapeJson(x)
 
-  proc cfiles(f: File; buf: var string; list: CfileList, isExternal: bool) =
+  proc cfiles(f: File; buf: var string; clist: CfileList, isExternal: bool) =
-    var i = 0
+    var pastStart = false
-    for it in list:
+    for it in clist:
       if CfileFlag.Cached in it.flags: continue
       let compileCmd = getCompileCFileCmd(it)
+      if pastStart: lit "],\L"
       lit "["
       str it.cname
       lit ", "
       str compileCmd
-      inc i
+      pastStart = true
-      if i == list.len:
+    lit "]\L"
-        lit "]\L"
-      else:
-        lit "],\L"
 
-  proc linkfiles(f: File; buf, objfiles: var string) =
+  proc linkfiles(f: File; buf, objfiles: var string; clist: CfileList;
-    for i, it in externalToLink:
+                 llist: seq[string]) =
-      let
+    var pastStart = false
-        objFile = if noAbsolutePaths(): it.extractFilename else: it
+    for it in llist:
-        objStr = addFileExt(objFile, CC[cCompiler].objExt)
+      let objfile = if noAbsolutePaths(): it.extractFilename
+                    else: it
+      let objstr = addFileExt(objfile, CC[cCompiler].objExt)
       add(objfiles, ' ')
-      add(objfiles, objStr)
+      add(objfiles, objstr)
-      str objStr
+      if pastStart: lit ",\L"
-      if toCompile.len == 0 and i == externalToLink.high:
+      str objstr
-        lit "\L"
+      pastStart = true
-      else:
+
-        lit ",\L"
+    for it in clist:
-    for i, x in toCompile:
+      let objstr = quoteShell(it.obj)
-      let objStr = quoteShell(x.obj)
       add(objfiles, ' ')
-      add(objfiles, objStr)
+      add(objfiles, objstr)
-      str objStr
+      if pastStart: lit ",\L"
-      if i == toCompile.high:
+      str objstr
-        lit "\L"
+      pastStart = true
-      else:
+    lit "\L"
-        lit ",\L"
 
   var buf = newStringOfCap(50)
 
@@ -840,7 +841,7 @@ proc writeJsonBuildInstructions*(projectfile: string) =
     lit "],\L\"link\":[\L"
     var objfiles = ""
     # XXX add every file here that is to link
-    linkfiles(f, buf, objfiles)
+    linkfiles(f, buf, objfiles, toCompile, externalToLink)
 
     lit "],\L\"linkcmd\": "
     str getLinkCmd(projectfile, objfiles)

compiler/importer.nim

@@ -27,7 +27,7 @@ proc rawImportSymbol(c: PContext, s: PSym) =
   # check if we have already a symbol of the same name:
   var check = strTableGet(c.importTable.symbols, s.name)
   if check != nil and check.id != s.id:
-    if s.kind notin OverloadableSyms:
+    if s.kind notin OverloadableSyms or check.kind notin OverloadableSyms:
       # s and check need to be qualified:
       incl(c.ambiguousSymbols, s.id)
       incl(c.ambiguousSymbols, check.id)

compiler/jsgen.nim

@@ -173,7 +173,7 @@ const
 proc mapType(typ: PType): TJSTypeKind =
   let t = skipTypes(typ, abstractInst)
   case t.kind
-  of tyVar, tyRef, tyPtr:
+  of tyVar, tyRef, tyPtr, tyLent:
     if skipTypes(t.lastSon, abstractInst).kind in MappedToObject:
       result = etyObject
     else:
@@ -196,14 +196,15 @@ proc mapType(typ: PType): TJSTypeKind =
      tyExpr, tyStmt, tyTypeDesc, tyBuiltInTypeClass, tyCompositeTypeClass,
      tyAnd, tyOr, tyNot, tyAnything, tyVoid:
     result = etyNone
-  of tyGenericInst, tyInferred, tyAlias, tyUserTypeClass, tyUserTypeClassInst:
+  of tyGenericInst, tyInferred, tyAlias, tyUserTypeClass, tyUserTypeClassInst,
+     tySink:
     result = mapType(typ.lastSon)
   of tyStatic:
     if t.n != nil: result = mapType(lastSon t)
     else: result = etyNone
   of tyProc: result = etyProc
   of tyCString: result = etyString
-  of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("mapType")
+  of tyUnused, tyOptAsRef: internalError("mapType")
 
 proc mapType(p: PProc; typ: PType): TJSTypeKind =
   if p.target == targetPHP: result = etyObject
@@ -869,8 +870,8 @@ proc generateHeader(p: PProc, typ: PType): Rope =
         add(result, name)
         add(result, "_Idx")
     elif not (i == 1 and param.name.s == "this"):
-      let k = param.typ.skipTypes({tyGenericInst, tyAlias}).kind
+      let k = param.typ.skipTypes({tyGenericInst, tyAlias, tySink}).kind
-      if k in {tyVar, tyRef, tyPtr, tyPointer}:
+      if k in {tyVar, tyRef, tyPtr, tyLent, tyPointer}:
         add(result, "&")
       add(result, "$")
       add(result, name)
@@ -899,7 +900,7 @@ const
 
 proc needsNoCopy(p: PProc; y: PNode): bool =
   result = (y.kind in nodeKindsNeedNoCopy) or
-      (skipTypes(y.typ, abstractInst).kind in {tyRef, tyPtr, tyVar}) or
+      (skipTypes(y.typ, abstractInst).kind in {tyRef, tyPtr, tyLent, tyVar}) or
       p.target == targetPHP
 
 proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
@@ -1077,7 +1078,7 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) =
 
 proc genArrayAccess(p: PProc, n: PNode, r: var TCompRes) =
   var ty = skipTypes(n.sons[0].typ, abstractVarRange)
-  if ty.kind in {tyRef, tyPtr}: ty = skipTypes(ty.lastSon, abstractVarRange)
+  if ty.kind in {tyRef, tyPtr, tyLent}: ty = skipTypes(ty.lastSon, abstractVarRange)
   case ty.kind
   of tyArray, tyOpenArray, tySequence, tyString, tyCString, tyVarargs:
     genArrayAddr(p, n, r)
@@ -1300,7 +1301,7 @@ proc genArg(p: PProc, n: PNode, param: PSym, r: var TCompRes; emitted: ptr int =
     add(r.res, ", ")
     add(r.res, a.res)
     if emitted != nil: inc emitted[]
-  elif n.typ.kind == tyVar and n.kind in nkCallKinds and mapType(param.typ) == etyBaseIndex:
+  elif n.typ.kind in {tyVar, tyLent} and n.kind in nkCallKinds and mapType(param.typ) == etyBaseIndex:
     # this fixes bug #5608:
     let tmp = getTemp(p)
     add(r.res, "($1 = $2, $1[0]), $1[1]" % [tmp, a.rdLoc])
@@ -1499,7 +1500,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     result = putToSeq("0", indirect)
   of tyFloat..tyFloat128:
     result = putToSeq("0.0", indirect)
-  of tyRange, tyGenericInst, tyAlias:
+  of tyRange, tyGenericInst, tyAlias, tySink:
     result = createVar(p, lastSon(typ), indirect)
   of tySet:
     result = putToSeq("{}" | "array()", indirect)
@@ -1546,7 +1547,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     createObjInitList(p, t, initIntSet(), initList)
     result = ("{$1}" | "array($#)") % [initList]
     if indirect: result = "[$1]" % [result]
-  of tyVar, tyPtr, tyRef:
+  of tyVar, tyPtr, tyLent, tyRef:
     if mapType(p, t) == etyBaseIndex:
       result = putToSeq("[null, 0]", indirect)
     else:
@@ -1579,7 +1580,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     let mname = mangleName(v, p.target)
     lineF(p, varCode & " = $3;$n" | "$$$2 = $3;$n",
                [returnType, mname, createVar(p, v.typ, isIndirect(v))])
-    if v.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, v.typ) == etyBaseIndex:
+    if v.typ.kind in {tyVar, tyPtr, tyLent, tyRef} and mapType(p, v.typ) == etyBaseIndex:
       lineF(p, "var $1_Idx = 0;$n", [ mname ])
   else:
     discard mangleName(v, p.target)
@@ -1774,7 +1775,7 @@ proc genRepr(p: PProc, n: PNode, r: var TCompRes) =
 
 proc genOf(p: PProc, n: PNode, r: var TCompRes) =
   var x: TCompRes
-  let t = skipTypes(n.sons[2].typ, abstractVarRange+{tyRef, tyPtr, tyTypeDesc})
+  let t = skipTypes(n.sons[2].typ, abstractVarRange+{tyRef, tyPtr, tyLent, tyTypeDesc})
   gen(p, n.sons[1], x)
   if tfFinal in t.flags:
     r.res = "($1.m_type == $2)" % [x.res, genTypeInfo(p, t)]
@@ -2051,10 +2052,10 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) =
     return
   case dest.kind:
   of tyBool:
-    r.res = "(($1)? 1:0)" % [r.res]
+    r.res = "(!!($1))" % [r.res]
     r.kind = resExpr
   of tyInt:
-    r.res = "($1|0)" % [r.res]
+    r.res = "(($1)|0)" % [r.res]
   else:
     # TODO: What types must we handle here?
     discard
@@ -2161,7 +2162,8 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
     let mname = mangleName(resultSym, p.target)
     let resVar = createVar(p, resultSym.typ, isIndirect(resultSym))
     resultAsgn = p.indentLine(("var $# = $#;$n" | "$$$# = $#;$n") % [mname, resVar])
-    if resultSym.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, resultSym.typ) == etyBaseIndex:
+    if resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef} and
+        mapType(p, resultSym.typ) == etyBaseIndex:
       resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname])
     gen(p, prc.ast.sons[resultPos], a)
     if mapType(p, resultSym.typ) == etyBaseIndex:
@@ -2218,10 +2220,10 @@ proc genCast(p: PProc, n: PNode, r: var TCompRes) =
   if dest.kind == src.kind:
     # no-op conversion
     return
-  let toInt = (dest.kind in tyInt .. tyInt32)
+  let toInt = (dest.kind in tyInt..tyInt32)
-  let toUint = (dest.kind in tyUInt .. tyUInt32)
+  let toUint = (dest.kind in tyUInt..tyUInt32)
-  let fromInt = (src.kind in tyInt .. tyInt32)
+  let fromInt = (src.kind in tyInt..tyInt32)
-  let fromUint = (src.kind in tyUInt .. tyUInt32)
+  let fromUint = (src.kind in tyUInt..tyUInt32)
 
   if toUint and (fromInt or fromUint):
     let trimmer = unsignedTrimmer(dest.size)
@@ -2252,7 +2254,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   case n.kind
   of nkSym:
     genSym(p, n, r)
-  of nkCharLit..nkUInt32Lit:
+  of nkCharLit..nkUInt64Lit:
     if n.typ.kind == tyBool:
       r.res = if n.intVal == 0: rope"false" else: rope"true"
     else:
@@ -2369,6 +2371,8 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkGotoState, nkState:
     internalError(n.info, "first class iterators not implemented")
   of nkPragmaBlock: gen(p, n.lastSon, r)
+  of nkComesFrom:
+    discard "XXX to implement for better stack traces"
   else: internalError(n.info, "gen: unknown node type: " & $n.kind)
 
 var globals: PGlobals

compiler/jstypes.nim

@@ -122,7 +122,7 @@ proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
          [name, genTypeInfo(p, typ.sons[0])])
 
 proc genEnumInfoPHP(p: PProc; t: PType): Rope =
-  let t = t.skipTypes({tyGenericInst, tyDistinct, tyAlias})
+  let t = t.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink})
   result = "$$NTI$1" % [rope(t.id)]
   p.declareGlobal(t.id, result)
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
@@ -141,7 +141,7 @@ proc genEnumInfoPHP(p: PProc; t: PType): Rope =
 proc genTypeInfo(p: PProc, typ: PType): Rope =
   if p.target == targetPHP:
     return makeJSString(typeToString(typ, preferModuleInfo))
-  let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias})
+  let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink})
   result = "NTI$1" % [rope(t.id)]
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
   case t.kind
@@ -152,7 +152,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
       [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-  of tyVar, tyRef, tyPtr, tySequence, tyRange, tySet:
+  of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]

compiler/lambdalifting.nim

@@ -190,7 +190,7 @@ proc interestingVar(s: PSym): bool {.inline.} =
 
 proc illegalCapture(s: PSym): bool {.inline.} =
   result = skipTypes(s.typ, abstractInst).kind in
-                   {tyVar, tyOpenArray, tyVarargs} or
+                   {tyVar, tyOpenArray, tyVarargs, tyLent} or
       s.kind == skResult
 
 proc isInnerProc(s: PSym): bool =
@@ -455,6 +455,7 @@ type
   LiftingPass = object
     processed: IntSet
     envVars: Table[int, PNode]
+    inContainer: int
 
 proc initLiftingPass(fn: PSym): LiftingPass =
   result.processed = initIntSet()
@@ -597,6 +598,8 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
 
 proc transformYield(n: PNode; owner: PSym; d: DetectionPass;
                     c: var LiftingPass): PNode =
+  if c.inContainer > 0:
+    localError(n.info, "invalid control flow: 'yield' within a constructor")
   let state = getStateField(owner)
   assert state != nil
   assert state.typ != nil
@@ -703,11 +706,14 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       if not c.processed.containsOrIncl(s.id):
         #if s.name.s == "temp":
         #  echo renderTree(s.getBody, {renderIds})
+        let oldInContainer = c.inContainer
+        c.inContainer = 0
         let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s)
         if c.envvars.getOrDefault(s.id).isNil:
           s.ast.sons[bodyPos] = body
         else:
           s.ast.sons[bodyPos] = newTree(nkStmtList, rawClosureCreation(s, d, c), body)
+        c.inContainer = oldInContainer
       if s.typ.callConv == ccClosure:
         result = symToClosure(n, owner, d, c)
     elif s.id in d.capturedVars:
@@ -717,7 +723,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         result = accessViaEnvParam(n, owner)
       else:
         result = accessViaEnvVar(n, owner, d, c)
-  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
+  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom,
      nkTemplateDef, nkTypeSection:
     discard
   of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef:
@@ -733,9 +739,12 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         n.sons[1] = x.sons[1]
   of nkLambdaKinds, nkIteratorDef, nkFuncDef:
     if n.typ != nil and n[namePos].kind == nkSym:
+      let oldInContainer = c.inContainer
+      c.inContainer = 0
       let m = newSymNode(n[namePos].sym)
       m.typ = n.typ
       result = liftCapturedVars(m, owner, d, c)
+      c.inContainer = oldInContainer
   of nkHiddenStdConv:
     if n.len == 2:
       n.sons[1] = liftCapturedVars(n[1], owner, d, c)
@@ -750,8 +759,12 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         # special case 'when nimVm' due to bug #3636:
         n.sons[1] = liftCapturedVars(n[1], owner, d, c)
         return
+
+    let inContainer = n.kind in {nkObjConstr, nkBracket}
+    if inContainer: inc c.inContainer
     for i in 0..<n.len:
       n.sons[i] = liftCapturedVars(n[i], owner, d, c)
+    if inContainer: dec c.inContainer
 
 # ------------------ old stuff -------------------------------------------
 

compiler/lookups.nim

@@ -445,13 +445,14 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
 
   if result != nil and result.kind == skStub: loadStub(result)
 
-proc pickSym*(c: PContext, n: PNode; kind: TSymKind;
+proc pickSym*(c: PContext, n: PNode; kinds: set[TSymKind];
               flags: TSymFlags = {}): PSym =
   var o: TOverloadIter
   var a = initOverloadIter(o, c, n)
   while a != nil:
-    if a.kind == kind and flags <= a.flags:
+    if a.kind in kinds and flags <= a.flags:
-      return a
+      if result == nil: result = a
+      else: return nil # ambiguous
     a = nextOverloadIter(o, c, n)
 
 proc isInfixAs*(n: PNode): bool =

compiler/lowerings.nim

@@ -330,7 +330,7 @@ proc typeNeedsNoDeepCopy(t: PType): bool =
   # note that seq[T] is fine, but 'var seq[T]' is not, so we need to skip 'var'
   # for the stricter check and likewise we can skip 'seq' for a less
   # strict check:
-  if t.kind in {tyVar, tySequence}: t = t.sons[0]
+  if t.kind in {tyVar, tyLent, tySequence}: t = t.lastSon
   result = not containsGarbageCollectedRef(t)
 
 proc addLocalVar(varSection, varInit: PNode; owner: PSym; typ: PType;
@@ -469,7 +469,7 @@ proc setupArgsForConcurrency(n: PNode; objType: PType; scratchObj: PSym,
     # we pick n's type here, which hopefully is 'tyArray' and not
     # 'tyOpenArray':
     var argType = n[i].typ.skipTypes(abstractInst)
-    if i < formals.len and formals[i].typ.kind == tyVar:
+    if i < formals.len and formals[i].typ.kind in {tyVar, tyLent}:
       localError(n[i].info, "'spawn'ed function cannot have a 'var' parameter")
     #elif containsTyRef(argType):
     #  localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")

compiler/main.nim

@@ -16,12 +16,12 @@ import
   cgen, jsgen, json, nversion,
   platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
   docgen2, service, parser, modules, ccgutils, sigmatch, ropes,
-  modulegraphs
+  modulegraphs, tables
 
 from magicsys import systemModule, resetSysTypes
 
 proc rodPass =
-  if optSymbolFiles in gGlobalOptions:
+  if gSymbolFiles in {enabledSf, writeOnlySf}:
     registerPass(rodwritePass)
 
 proc codegenPass =
@@ -36,6 +36,9 @@ proc writeDepsFile(g: ModuleGraph; project: string) =
   for m in g.modules:
     if m != nil:
       f.writeLine(toFullPath(m.position.int32))
+  for k in g.inclToMod.keys:
+    if g.getModule(k).isNil:  # don't repeat includes which are also modules
+      f.writeLine(k.toFullPath)
   f.close()
 
 proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) =
@@ -77,6 +80,8 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) =
     let proj = changeFileExt(gProjectFull, "")
     extccomp.callCCompiler(proj)
     extccomp.writeJsonBuildInstructions(proj)
+    if optGenScript in gGlobalOptions:
+      writeDepsFile(graph, toGeneratedFile(proj, ""))
 
 proc commandJsonScript(graph: ModuleGraph; cache: IdentCache) =
   let proj = changeFileExt(gProjectFull, "")

compiler/msgs.nim

@@ -26,7 +26,8 @@ type
     errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation,
     errExceptionExpected, errExceptionAlreadyHandled,
     errYieldNotAllowedHere, errYieldNotAllowedInTryStmt,
-    errInvalidNumberOfYieldExpr, errCannotReturnExpr, errAttemptToRedefine,
+    errInvalidNumberOfYieldExpr, errCannotReturnExpr, 
+    errNoReturnWithReturnTypeNotAllowed, errAttemptToRedefine,
     errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel,
     errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected,
     errInvalidVarSubstitution, errUnknownVar, errUnknownCcompiler,
@@ -179,8 +180,9 @@ const
     errYieldNotAllowedInTryStmt: "'yield' cannot be used within 'try' in a non-inlined iterator",
     errInvalidNumberOfYieldExpr: "invalid number of \'yield\' expressions",
     errCannotReturnExpr: "current routine cannot return an expression",
+    errNoReturnWithReturnTypeNotAllowed: "routines with NoReturn pragma are not allowed to have return type",
     errAttemptToRedefine: "redefinition of \'$1\'",
-    errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\', \'raise\' or \'continue'",
+    errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\', \'raise\', \'continue\' or proc call with noreturn pragma",
     errStmtExpected: "statement expected",
     errInvalidLabel: "\'$1\' is no label",
     errInvalidCmdLineOption: "invalid command line option: \'$1\'",

compiler/nimblecmd.nim

@@ -28,6 +28,10 @@ proc newVersion*(ver: string): Version =
 proc isSpecial(ver: Version): bool =
   return ($ver).len > 0 and ($ver)[0] == '#'
 
+proc isValidVersion(v: string): bool =
+  if v.len > 0:
+    if v[0] in {'#'} + Digits: return true
+
 proc `<`*(ver: Version, ver2: Version): bool =
   ## This is synced from Nimble's version module.
 
@@ -72,15 +76,23 @@ proc getPathVersion*(p: string): tuple[name, version: string] =
     result.name = p
     return
 
+  for i in sepIdx..<p.len:
+    if p[i] in {DirSep, AltSep}:
+      result.name = p
+      return
+
   result.name = p[0 .. sepIdx - 1]
   result.version = p.substr(sepIdx + 1)
 
-proc addPackage(packages: StringTableRef, p: string) =
+proc addPackage(packages: StringTableRef, p: string; info: TLineInfo) =
   let (name, ver) = getPathVersion(p)
-  let version = newVersion(ver)
+  if isValidVersion(ver):
-  if packages.getOrDefault(name).newVersion < version or
+    let version = newVersion(ver)
-     (not packages.hasKey(name)):
+    if packages.getOrDefault(name).newVersion < version or
-    packages[name] = $version
+      (not packages.hasKey(name)):
+      packages[name] = $version
+  else:
+    localError(info, "invalid package name: " & p)
 
 iterator chosen(packages: StringTableRef): string =
   for key, val in pairs(packages):
@@ -109,7 +121,7 @@ proc addPathRec(dir: string, info: TLineInfo) =
   if dir[pos] in {DirSep, AltSep}: inc(pos)
   for k,p in os.walkDir(dir):
     if k == pcDir and p[pos] != '.':
-      addPackage(packages, p)
+      addPackage(packages, p, info)
   for p in packages.chosen:
     addNimblePath(p, info)
 

compiler/options.nim

@@ -48,7 +48,6 @@ type                          # please make sure we have under 32 options
     optGenScript,             # generate a script file to compile the *.c files
     optGenMapping,            # generate a mapping file
     optRun,                   # run the compiled project
-    optSymbolFiles,           # use symbol files for speeding up compilation
     optCaasEnabled            # compiler-as-a-service is running
     optSkipConfigFile,        # skip the general config file
     optSkipProjConfigFile,    # skip the project's config file
@@ -147,12 +146,19 @@ var
   newDestructors*: bool
   gDynlibOverrideAll*: bool
 
+type
+  SymbolFilesOption* = enum
+    disabledSf, enabledSf, writeOnlySf, readOnlySf, v2Sf
+
+var gSymbolFiles*: SymbolFilesOption
+
 proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools}
 proc usesNativeGC*(): bool {.inline.} = gSelectedGC >= gcRefc
 template preciseStack*(): bool = gPreciseStack
 
 template compilationCachePresent*: untyped =
-  {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {}
+  gSymbolFiles in {enabledSf, writeOnlySf}
+#  {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {}
 
 template optPreserveOrigSource*: untyped =
   optEmbedOrigSrc in gGlobalOptions

compiler/passes.nim

@@ -18,7 +18,7 @@ import
 
 type
   TPassContext* = object of RootObj # the pass's context
-    fromCache*: bool  # true if created by "openCached"
+    rd*: PRodReader  # != nil if created by "openCached"
 
   PPassContext* = ref TPassContext
 
@@ -118,7 +118,7 @@ proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym,
     if not isNil(gPasses[i].openCached):
       a[i] = gPasses[i].openCached(g, module, rd)
       if a[i] != nil:
-        a[i].fromCache = true
+        a[i].rd = rd
     else:
       a[i] = nil
 

compiler/pragmas.nim

@@ -17,11 +17,12 @@ import
 const
   FirstCallConv* = wNimcall
   LastCallConv* = wNoconv
+  nkPragmaCallKinds = {nkExprColonExpr, nkCall, nkCallStrLit}
 
 const
   procPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl,
     wMagic, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader,
-    wCompilerproc, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge,
+    wCompilerProc, wCore, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge,
     wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC,
     wAsmNoStackFrame, wError, wDiscardable, wNoInit, wCodegenDecl,
     wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe,
@@ -29,9 +30,9 @@ const
   converterPragmas* = procPragmas
   methodPragmas* = procPragmas+{wBase}-{wImportCpp}
   templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty,
-    wDelegator, wExportNims, wUsed}
+    wDelegator, wExportNims, wUsed, wPragma}
   macroPragmas* = {FirstCallConv..LastCallConv, wImmediate, wImportc, wExportc,
-    wNodecl, wMagic, wNosideeffect, wCompilerproc, wDeprecated, wExtern,
+    wNodecl, wMagic, wNosideeffect, wCompilerProc, wCore, wDeprecated, wExtern,
     wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wDelegator,
     wExportNims, wUsed}
   iteratorPragmas* = {FirstCallConv..LastCallConv, wNosideeffect, wSideeffect,
@@ -52,14 +53,14 @@ const
     wDeprecated, wExtern, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame,
     wRaises, wLocks, wTags, wGcSafe}
   typePragmas* = {wImportc, wExportc, wDeprecated, wMagic, wAcyclic, wNodecl,
-    wPure, wHeader, wCompilerproc, wFinal, wSize, wExtern, wShallow,
+    wPure, wHeader, wCompilerProc, wCore, wFinal, wSize, wExtern, wShallow,
     wImportCpp, wImportObjC, wError, wIncompleteStruct, wByCopy, wByRef,
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
     wBorrow, wGcSafe, wExportNims, wPartial, wUsed, wExplain, wPackage}
   fieldPragmas* = {wImportc, wExportc, wDeprecated, wExtern,
     wImportCpp, wImportObjC, wError, wGuard, wBitsize, wUsed}
   varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl,
-    wMagic, wHeader, wDeprecated, wCompilerproc, wDynlib, wExtern,
+    wMagic, wHeader, wDeprecated, wCompilerProc, wCore, wDynlib, wExtern,
     wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal,
     wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims, wUsed}
   constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl,
@@ -74,7 +75,7 @@ proc getPragmaVal*(procAst: PNode; name: TSpecialWord): PNode =
   let p = procAst[pragmasPos]
   if p.kind == nkEmpty: return nil
   for it in p:
-    if it.kind == nkExprColonExpr and it[0].kind == nkIdent and
+    if it.kind in nkPragmaCallKinds and it.len == 2 and it[0].kind == nkIdent and
         it[0].ident.id == ord(name):
       return it[1]
 
@@ -89,7 +90,7 @@ proc pragmaAsm*(c: PContext, n: PNode): char =
   if n != nil:
     for i in countup(0, sonsLen(n) - 1):
       let it = n.sons[i]
-      if it.kind == nkExprColonExpr and it.sons[0].kind == nkIdent:
+      if it.kind in nkPragmaCallKinds and it.len == 2 and it.sons[0].kind == nkIdent:
         case whichKeyword(it.sons[0].ident)
         of wSubsChar:
           if it.sons[1].kind == nkCharLit: result = chr(int(it.sons[1].intVal))
@@ -151,7 +152,7 @@ proc newEmptyStrNode(n: PNode): PNode {.noinline.} =
   result.strVal = ""
 
 proc getStrLitNode(c: PContext, n: PNode): PNode =
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errStringLiteralExpected)
     # error correction:
     result = newEmptyStrNode(n)
@@ -168,7 +169,7 @@ proc expectStrLit(c: PContext, n: PNode): string =
   result = getStrLitNode(c, n).strVal
 
 proc expectIntLit(c: PContext, n: PNode): int =
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errIntLiteralExpected)
   else:
     n.sons[1] = c.semConstExpr(c, n.sons[1])
@@ -177,7 +178,7 @@ proc expectIntLit(c: PContext, n: PNode): int =
     else: localError(n.info, errIntLiteralExpected)
 
 proc getOptionalStr(c: PContext, n: PNode, defaultStr: string): string =
-  if n.kind == nkExprColonExpr: result = expectStrLit(c, n)
+  if n.kind in nkPragmaCallKinds: result = expectStrLit(c, n)
   else: result = defaultStr
 
 proc processCodegenDecl(c: PContext, n: PNode, sym: PSym) =
@@ -186,7 +187,7 @@ proc processCodegenDecl(c: PContext, n: PNode, sym: PSym) =
 proc processMagic(c: PContext, n: PNode, s: PSym) =
   #if sfSystemModule notin c.module.flags:
   #  liMessage(n.info, errMagicOnlyInSystem)
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errStringLiteralExpected)
     return
   var v: string
@@ -204,7 +205,7 @@ proc wordToCallConv(sw: TSpecialWord): TCallingConvention =
   result = TCallingConvention(ord(ccDefault) + ord(sw) - ord(wNimcall))
 
 proc isTurnedOn(c: PContext, n: PNode): bool =
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     let x = c.semConstBoolExpr(c, n.sons[1])
     n.sons[1] = x
     if x.kind == nkIntLit: return x.intVal != 0
@@ -223,7 +224,7 @@ proc pragmaNoForward(c: PContext, n: PNode; flag=sfNoForward) =
   else: excl(c.module.flags, flag)
 
 proc processCallConv(c: PContext, n: PNode) =
-  if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent):
+  if n.kind in nkPragmaCallKinds and n.len == 2 and n.sons[1].kind == nkIdent:
     var sw = whichKeyword(n.sons[1].ident)
     case sw
     of FirstCallConv..LastCallConv:
@@ -244,7 +245,7 @@ proc getLib(c: PContext, kind: TLibKind, path: PNode): PLib =
     result.isOverriden = options.isDynlibOverride(path.strVal)
 
 proc expectDynlibNode(c: PContext, n: PNode): PNode =
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errStringLiteralExpected)
     # error correction:
     result = newEmptyStrNode(n)
@@ -264,7 +265,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) =
     if not lib.isOverriden:
       c.optionStack[^1].dynlib = lib
   else:
-    if n.kind == nkExprColonExpr:
+    if n.kind in nkPragmaCallKinds:
       var lib = getLib(c, libDynamic, expectDynlibNode(c, n))
       if not lib.isOverriden:
         addToLib(lib, sym)
@@ -279,7 +280,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) =
       sym.typ.callConv = ccCDecl
 
 proc processNote(c: PContext, n: PNode) =
-  if (n.kind == nkExprColonExpr) and (sonsLen(n) == 2) and
+  if (n.kind in nkPragmaCallKinds) and (sonsLen(n) == 2) and
       (n.sons[0].kind == nkBracketExpr) and
       (n.sons[0].sons.len == 2) and
       (n.sons[0].sons[1].kind == nkIdent) and
@@ -307,7 +308,7 @@ proc processNote(c: PContext, n: PNode) =
     invalidPragma(n)
 
 proc processOption(c: PContext, n: PNode): bool =
-  if n.kind != nkExprColonExpr: result = true
+  if n.kind notin nkPragmaCallKinds or n.len != 2: result = true
   elif n.sons[0].kind == nkBracketExpr: processNote(c, n)
   elif n.sons[0].kind != nkIdent: result = true
   else:
@@ -355,8 +356,8 @@ proc processOption(c: PContext, n: PNode): bool =
     else: result = true
 
 proc processPush(c: PContext, n: PNode, start: int) =
-  if n.sons[start-1].kind == nkExprColonExpr:
+  if n.sons[start-1].kind in nkPragmaCallKinds:
-    localError(n.info, errGenerated, "':' after 'push' not supported")
+    localError(n.info, errGenerated, "'push' can't have arguments")
   var x = newOptionEntry()
   var y = c.optionStack[^1]
   x.options = gOptions
@@ -381,14 +382,14 @@ proc processPop(c: PContext, n: PNode) =
     c.optionStack.setLen(c.optionStack.len - 1)
 
 proc processDefine(c: PContext, n: PNode) =
-  if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent):
+  if (n.kind in nkPragmaCallKinds and n.len == 2) and (n.sons[1].kind == nkIdent):
     defineSymbol(n.sons[1].ident.s)
     message(n.info, warnDeprecated, "define")
   else:
     invalidPragma(n)
 
 proc processUndef(c: PContext, n: PNode) =
-  if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent):
+  if (n.kind in nkPragmaCallKinds and n.len == 2) and (n.sons[1].kind == nkIdent):
     undefSymbol(n.sons[1].ident.s)
     message(n.info, warnDeprecated, "undef")
   else:
@@ -420,7 +421,7 @@ proc processCompile(c: PContext, n: PNode) =
       localError(n.info, errStringLiteralExpected)
       result = ""
 
-  let it = if n.kind == nkExprColonExpr: n.sons[1] else: n
+  let it = if n.kind in nkPragmaCallKinds and n.len == 2: n.sons[1] else: n
   if it.kind == nkPar and it.len == 2:
     let s = getStrLit(c, it, 0)
     let dest = getStrLit(c, it, 1)
@@ -453,7 +454,7 @@ proc pragmaBreakpoint(c: PContext, n: PNode) =
   discard getOptionalStr(c, n, "")
 
 proc pragmaWatchpoint(c: PContext, n: PNode) =
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     n.sons[1] = c.semExpr(c, n.sons[1])
   else:
     invalidPragma(n)
@@ -494,7 +495,7 @@ proc semAsmOrEmit*(con: PContext, n: PNode, marker: char): PNode =
     result = newNode(nkAsmStmt, n.info)
 
 proc pragmaEmit(c: PContext, n: PNode) =
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errStringLiteralExpected)
   else:
     let n1 = n[1]
@@ -512,12 +513,12 @@ proc pragmaEmit(c: PContext, n: PNode) =
         localError(n.info, errStringLiteralExpected)
 
 proc noVal(n: PNode) =
-  if n.kind == nkExprColonExpr: invalidPragma(n)
+  if n.kind in nkPragmaCallKinds and n.len > 1: invalidPragma(n)
 
 proc pragmaUnroll(c: PContext, n: PNode) =
   if c.p.nestedLoopCounter <= 0:
     invalidPragma(n)
-  elif n.kind == nkExprColonExpr:
+  elif n.kind in nkPragmaCallKinds and n.len == 2:
     var unrollFactor = expectIntLit(c, n)
     if unrollFactor <% 32:
       n.sons[1] = newIntNode(nkIntLit, unrollFactor)
@@ -525,10 +526,11 @@ proc pragmaUnroll(c: PContext, n: PNode) =
       invalidPragma(n)
 
 proc pragmaLine(c: PContext, n: PNode) =
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     n.sons[1] = c.semConstExpr(c, n.sons[1])
     let a = n.sons[1]
     if a.kind == nkPar:
+      # unpack the tuple
       var x = a.sons[0]
       var y = a.sons[1]
       if x.kind == nkExprColonExpr: x = x.sons[1]
@@ -549,7 +551,7 @@ proc pragmaLine(c: PContext, n: PNode) =
 
 proc processPragma(c: PContext, n: PNode, i: int) =
   var it = n.sons[i]
-  if it.kind != nkExprColonExpr: invalidPragma(n)
+  if it.kind notin nkPragmaCallKinds and it.len == 2: invalidPragma(n)
   elif it.sons[0].kind != nkIdent: invalidPragma(n)
   elif it.sons[1].kind != nkIdent: invalidPragma(n)
 
@@ -566,7 +568,7 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) =
       localError(x.info, errGenerated, "invalid type for raises/tags list")
     x.typ = t
 
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     let it = n.sons[1]
     if it.kind notin {nkCurly, nkBracket}:
       processExc(c, it)
@@ -576,7 +578,7 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) =
     invalidPragma(n)
 
 proc pragmaLockStmt(c: PContext; it: PNode) =
-  if it.kind != nkExprColonExpr:
+  if it.kind notin nkPragmaCallKinds or it.len != 2:
     invalidPragma(it)
   else:
     let n = it[1]
@@ -587,7 +589,7 @@ proc pragmaLockStmt(c: PContext; it: PNode) =
         n.sons[i] = c.semExpr(c, n.sons[i])
 
 proc pragmaLocks(c: PContext, it: PNode): TLockLevel =
-  if it.kind != nkExprColonExpr:
+  if it.kind notin nkPragmaCallKinds or it.len != 2:
     invalidPragma(it)
   else:
     case it[1].kind
@@ -604,7 +606,7 @@ proc pragmaLocks(c: PContext, it: PNode): TLockLevel =
         result = TLockLevel(x)
 
 proc typeBorrow(sym: PSym, n: PNode) =
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     let it = n.sons[1]
     if it.kind != nkAccQuoted:
       localError(n.info, "a type can only borrow `.` for now")
@@ -624,7 +626,7 @@ proc deprecatedStmt(c: PContext; pragma: PNode) =
   if pragma.kind != nkBracket:
     localError(pragma.info, "list of key:value pairs expected"); return
   for n in pragma:
-    if n.kind in {nkExprColonExpr, nkExprEqExpr}:
+    if n.kind in nkPragmaCallKinds and n.len == 2:
       let dest = qualifiedLookUp(c, n[1], {checkUndeclared})
       if dest == nil or dest.kind in routineKinds:
         localError(n.info, warnUser, "the .deprecated pragma is unreliable for routines")
@@ -638,7 +640,7 @@ proc deprecatedStmt(c: PContext; pragma: PNode) =
       localError(n.info, "key:value pair expected")
 
 proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym =
-  if it.kind != nkExprColonExpr:
+  if it.kind notin nkPragmaCallKinds or it.len != 2:
     invalidPragma(it); return
   let n = it[1]
   if n.kind == nkSym:
@@ -655,13 +657,36 @@ proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym =
   else:
     result = qualifiedLookUp(c, n, {checkUndeclared})
 
+proc semCustomPragma(c: PContext, n: PNode): PNode =
+  assert(n.kind in nkPragmaCallKinds + {nkIdent})
+  
+  if n.kind == nkIdent:
+    result = newTree(nkCall, n)
+  elif n.kind == nkExprColonExpr:
+    # pragma: arg -> pragma(arg)
+    result = newTree(nkCall, n[0], n[1])
+  else:
+    result = n
+
+  result = c.semOverloadedCall(c, result, n, {skTemplate}, {})
+  if sfCustomPragma notin result[0].sym.flags:
+    invalidPragma(n)
+
+  if n.kind == nkIdent:
+    result = result[0]
+  elif n.kind == nkExprColonExpr:
+    result.kind = n.kind # pragma(arg) -> pragma: arg
+
 proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
                   validPragmas: TSpecialWords): bool =
   var it = n.sons[i]
-  var key = if it.kind == nkExprColonExpr: it.sons[0] else: it
+  var key = if it.kind in nkPragmaCallKinds and it.len > 1: it.sons[0] else: it
   if key.kind == nkBracketExpr:
     processNote(c, it)
     return
+  elif key.kind notin nkIdentKinds:
+    n.sons[i] = semCustomPragma(c, it)
+    return
   let ident = considerQuotedIdent(key)
   var userPragma = strTableGet(c.userPragmas, ident)
   if userPragma != nil:
@@ -771,9 +796,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
       of wNoreturn:
         noVal(it)
         incl(sym.flags, sfNoReturn)
+        if sym.typ[0] != nil:
+          localError(sym.ast[paramsPos][0].info, errNoReturnWithReturnTypeNotAllowed)
       of wDynlib:
         processDynLib(c, it, sym)
-      of wCompilerproc:
+      of wCompilerProc, wCore:
         noVal(it)           # compilerproc may not get a string!
         cppDefine(c.graph.config, sym.name.s)
         if sfFromGeneric notin sym.flags: markCompilerProc(sym)
@@ -783,7 +810,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
       of wExplain:
         sym.flags.incl sfExplain
       of wDeprecated:
-        if it.kind == nkExprColonExpr: deprecatedStmt(c, it)
+        if it.kind in nkPragmaCallKinds: deprecatedStmt(c, it)
         elif sym != nil: incl(sym.flags, sfDeprecated)
         else: incl(c.module.flags, sfDeprecated)
       of wVarargs:
@@ -862,8 +889,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         result = true
       of wPop: processPop(c, it)
       of wPragma:
-        processPragma(c, n, i)
+        if not sym.isNil and sym.kind == skTemplate:
-        result = true
+          sym.flags.incl sfCustomPragma
+        else:
+          processPragma(c, n, i)
+          result = true
       of wDiscardable:
         noVal(it)
         if sym != nil: incl(sym.flags, sfDiscardable)
@@ -937,7 +967,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         elif sym.typ == nil: invalidPragma(it)
         else: sym.typ.lockLevel = pragmaLocks(c, it)
       of wBitsize:
-        if sym == nil or sym.kind != skField or it.kind != nkExprColonExpr:
+        if sym == nil or sym.kind != skField:
           invalidPragma(it)
         else:
           sym.bitsize = expectIntLit(c, it)
@@ -955,7 +985,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         if sym == nil: invalidPragma(it)
         else: magicsys.registerNimScriptSymbol(sym)
       of wInjectStmt:
-        if it.kind != nkExprColonExpr:
+        if it.kind notin nkPragmaCallKinds or it.len != 2:
           localError(it.info, errExprExpected)
         else:
           it.sons[1] = c.semExpr(c, it.sons[1])
@@ -966,10 +996,12 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         else:
           localError(it.info, "'experimental' pragma only valid as toplevel statement")
       of wThis:
-        if it.kind == nkExprColonExpr:
+        if it.kind in nkPragmaCallKinds and it.len == 2:
           c.selfName = considerQuotedIdent(it[1])
-        else:
+        elif it.kind == nkIdent or it.len == 1:
           c.selfName = getIdent("self")
+        else:
+          localError(it.info, "'this' pragma is allowed to have zero or one arguments")
       of wNoRewrite:
         noVal(it)
       of wBase:
@@ -985,7 +1017,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         else: sym.flags.incl sfUsed
       of wLiftLocals: discard
       else: invalidPragma(it)
-    else: invalidPragma(it)
+    else: 
+      n.sons[i] = semCustomPragma(c, it)
+
 
 proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
                       validPragmas: TSpecialWords) =
@@ -1013,7 +1047,7 @@ proc hasPragma*(n: PNode, pragma: TSpecialWord): bool =
     return false
 
   for p in n.sons:
-    var key = if p.kind == nkExprColonExpr: p[0] else: p
+    var key = if p.kind in nkPragmaCallKinds and p.len > 1: p[0] else: p
     if key.kind == nkIdent and whichKeyword(key.ident) == pragma:
       return true
 

compiler/renderer.nim

@@ -175,8 +175,17 @@ proc put(g: var TSrcGen, kind: TTokType, s: string) =
 
 proc toNimChar(c: char): string =
   case c
-  of '\0': result = "\\0"
+  of '\0': result = "\\x00" # not "\\0" to avoid ambiguous cases like "\\012".
-  of '\x01'..'\x1F', '\x80'..'\xFF': result = "\\x" & strutils.toHex(ord(c), 2)
+  of '\a': result = "\\a" # \x07
+  of '\b': result = "\\b" # \x08
+  of '\t': result = "\\t" # \x09
+  of '\L': result = "\\L" # \x0A
+  of '\v': result = "\\v" # \x0B
+  of '\f': result = "\\f" # \x0C
+  of '\c': result = "\\c" # \x0D
+  of '\e': result = "\\e" # \x1B
+  of '\x01'..'\x06', '\x0E'..'\x1A', '\x1C'..'\x1F', '\x80'..'\xFF':
+    result = "\\x" & strutils.toHex(ord(c), 2)
   of '\'', '\"', '\\': result = '\\' & c
   else: result = c & ""
 
@@ -316,8 +325,8 @@ proc lsub(g: TSrcGen; n: PNode): int
 proc litAux(g: TSrcGen; n: PNode, x: BiggestInt, size: int): string =
   proc skip(t: PType): PType =
     result = t
-    while result.kind in {tyGenericInst, tyRange, tyVar, tyDistinct,
+    while result.kind in {tyGenericInst, tyRange, tyVar, tyLent, tyDistinct,
-                          tyOrdinal, tyAlias}:
+                          tyOrdinal, tyAlias, tySink}:
       result = lastSon(result)
   if n.typ != nil and n.typ.skip.kind in {tyBool, tyEnum}:
     let enumfields = n.typ.skip.n
@@ -710,6 +719,7 @@ proc gcase(g: var TSrcGen, n: PNode) =
   var c: TContext
   initContext(c)
   var length = sonsLen(n)
+  if length == 0: return
   var last = if n.sons[length-1].kind == nkElse: -2 else: -1
   if longMode(g, n, 0, last): incl(c.flags, rfLongMode)
   putWithSpace(g, tkCase, "case")
@@ -860,7 +870,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     a: TContext
   if n.comment != nil: pushCom(g, n)
   case n.kind                 # atoms:
-  of nkTripleStrLit: putRawStr(g, tkTripleStrLit, n.strVal)
+  of nkTripleStrLit: put(g, tkTripleStrLit, atom(g, n))
   of nkEmpty: discard
   of nkType: put(g, tkInvalid, atom(g, n))
   of nkSym, nkIdent: gident(g, n)
@@ -888,6 +898,14 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
       put(g, tkBracketLe, "[")
       gcomma(g, n, 2)
       put(g, tkBracketRi, "]")
+    elif n.len > 1 and n.lastSon.kind == nkStmtList:
+      gsub(g, n[0])
+      if n.len > 2:
+        put(g, tkParLe, "(")
+        gcomma(g, n, 1, -2)
+        put(g, tkParRi, ")")
+      put(g, tkColon, ":")
+      gsub(g, n, n.len-1)
     else:
       if sonsLen(n) >= 1: gsub(g, n.sons[0])
       put(g, tkParLe, "(")
@@ -1397,8 +1415,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     put(g, tkBracketRi, "]")
   of nkTupleClassTy:
     put(g, tkTuple, "tuple")
-  of nkMetaNode_Obsolete:
+  of nkComesFrom:
-    put(g, tkParLe, "(META|")
+    put(g, tkParLe, "(ComesFrom|")
     gsub(g, n, 0)
     put(g, tkParRi, ")")
   of nkGotoState, nkState:

compiler/rodread.nim

@@ -861,12 +861,11 @@ proc loadMethods(r: PRodReader) =
     if r.s[r.pos] == ' ': inc(r.pos)
 
 proc getHash*(fileIdx: int32): SecureHash =
-  internalAssert fileIdx >= 0 and fileIdx < gMods.len
+  if fileIdx <% gMods.len and gMods[fileIdx].hashDone:
-
-  if gMods[fileIdx].hashDone:
     return gMods[fileIdx].hash
 
   result = secureHashFile(fileIdx.toFullPath)
+  if fileIdx >= gMods.len: setLen(gMods, fileIdx+1)
   gMods[fileIdx].hash = result
 
 template growCache*(cache, pos) =
@@ -912,7 +911,7 @@ proc checkDep(fileIdx: int32; cache: IdentCache): TReasonForRecompile =
 
 proc handleSymbolFile*(module: PSym; cache: IdentCache): PRodReader =
   let fileIdx = module.fileIdx
-  if optSymbolFiles notin gGlobalOptions:
+  if gSymbolFiles in {disabledSf, writeOnlySf}:
     module.id = getID()
     return nil
   idgen.loadMaxIds(options.gProjectPath / options.gProjectName)

compiler/scriptconfig.nim

@@ -73,7 +73,8 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
   cbos copyFile:
     os.copyFile(getString(a, 0), getString(a, 1))
   cbos getLastModificationTime:
-    setResult(a, toSeconds(getLastModificationTime(getString(a, 0))))
+    # depends on Time's implementation!
+    setResult(a, int64(getLastModificationTime(getString(a, 0))))
 
   cbos rawExec:
     setResult(a, osproc.execCmd getString(a, 0))

compiler/sem.nim

@@ -74,7 +74,7 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
     localError(arg.info, errExprXHasNoType,
                renderTree(arg, {renderNoComments}))
     # error correction:
-    result = copyNode(arg)
+    result = copyTree(arg)
     result.typ = formal
   else:
     result = indexTypesMatch(c, formal, arg.typ, arg)
@@ -102,8 +102,8 @@ proc commonType*(x, y: PType): PType =
   # if expressions, etc.:
   if x == nil: return x
   if y == nil: return y
-  var a = skipTypes(x, {tyGenericInst, tyAlias})
+  var a = skipTypes(x, {tyGenericInst, tyAlias, tySink})
-  var b = skipTypes(y, {tyGenericInst, tyAlias})
+  var b = skipTypes(y, {tyGenericInst, tyAlias, tySink})
   result = x
   if a.kind in {tyExpr, tyNil}: result = y
   elif b.kind in {tyExpr, tyNil}: result = x
@@ -165,6 +165,19 @@ proc commonType*(x, y: PType): PType =
         result = newType(k, r.owner)
         result.addSonSkipIntLit(r)
 
+proc endsInNoReturn(n: PNode): bool =
+  # check if expr ends in raise exception or call of noreturn proc
+  var it = n
+  while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0:
+    it = it.lastSon
+  result = it.kind == nkRaiseStmt or
+    it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags
+
+proc commonType*(x: PType, y: PNode): PType =
+  # ignore exception raising branches in case/if expressions
+  if endsInNoReturn(y): return x
+  commonType(x, y.typ)
+
 proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym =
   result = newSym(kind, considerQuotedIdent(n), getCurrOwner(c), n.info)
   when defined(nimsuggest):
@@ -423,7 +436,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
   result = evalMacroCall(c.module, c.cache, n, nOrig, sym)
   if efNoSemCheck notin flags:
     result = semAfterMacroCall(c, n, result, sym, flags)
-  result = wrapInComesFrom(nOrig.info, result)
+  result = wrapInComesFrom(nOrig.info, sym, result)
   popInfoContext()
 
 proc forceBool(c: PContext, n: PNode): PNode =
@@ -488,6 +501,8 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
 
 proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext =
   result = myOpen(graph, module, rd.cache)
+
+proc replayMethodDefs(graph: ModuleGraph; rd: PRodReader) =
   for m in items(rd.methods): methodDef(graph, m, true)
 
 proc isImportSystemStmt(n: PNode): bool =
@@ -594,6 +609,8 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   addCodeForGenerics(c, result)
   if c.module.ast != nil:
     result.add(c.module.ast)
+  if c.rd != nil:
+    replayMethodDefs(graph, c.rd)
   popOwner(c)
   popProcCon(c)
   if c.runnableExamples != nil: testExamples(c)

compiler/semasgn.nim

@@ -242,9 +242,9 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
      tyTypeDesc, tyGenericInvocation, tyForward:
     internalError(c.info, "assignment requested for type: " & typeToString(t))
   of tyOrdinal, tyRange, tyInferred,
-     tyGenericInst, tyStatic, tyVar, tyAlias:
+     tyGenericInst, tyStatic, tyVar, tyLent, tyAlias, tySink:
     liftBodyAux(c, lastSon(t), body, x, y)
-  of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("liftBodyAux")
+  of tyUnused, tyOptAsRef: internalError("liftBodyAux")
 
 proc newProcType(info: TLineInfo; owner: PSym): PType =
   result = newType(tyProc, owner)
@@ -261,7 +261,7 @@ proc addParam(procType: PType; param: PSym) =
   rawAddSon(procType, param.typ)
 
 proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
-              info: TLineInfo): PSym {.discardable.} =
+              info: TLineInfo): PSym =
   var a: TLiftCtx
   a.info = info
   a.c = c
@@ -306,7 +306,7 @@ proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
 
 
 proc getAsgnOrLiftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
-  let t = typ.skipTypes({tyGenericInst, tyVar, tyAlias})
+  let t = typ.skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink})
   result = t.assignment
   if result.isNil:
     result = liftBody(c, t, attachedAsgn, info)

compiler/semcall.nim

@@ -377,7 +377,7 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode =
 
 proc canDeref(n: PNode): bool {.inline.} =
   result = n.len >= 2 and (let t = n[1].typ;
-    t != nil and t.skipTypes({tyGenericInst, tyAlias}).kind in {tyPtr, tyRef})
+    t != nil and t.skipTypes({tyGenericInst, tyAlias, tySink}).kind in {tyPtr, tyRef})
 
 proc tryDeref(n: PNode): PNode =
   result = newNodeI(nkHiddenDeref, n.info)

compiler/semdata.nim

@@ -259,19 +259,19 @@ proc makePtrType*(c: PContext, baseType: PType): PType =
 proc makeTypeWithModifier*(c: PContext,
                            modifier: TTypeKind,
                            baseType: PType): PType =
-  assert modifier in {tyVar, tyPtr, tyRef, tyStatic, tyTypeDesc}
+  assert modifier in {tyVar, tyLent, tyPtr, tyRef, tyStatic, tyTypeDesc}
 
-  if modifier in {tyVar, tyTypeDesc} and baseType.kind == modifier:
+  if modifier in {tyVar, tyLent, tyTypeDesc} and baseType.kind == modifier:
     result = baseType
   else:
     result = newTypeS(modifier, c)
     addSonSkipIntLit(result, baseType.assertNotNil)
 
-proc makeVarType*(c: PContext, baseType: PType): PType =
+proc makeVarType*(c: PContext, baseType: PType; kind = tyVar): PType =
-  if baseType.kind == tyVar:
+  if baseType.kind == kind:
     result = baseType
   else:
-    result = newTypeS(tyVar, c)
+    result = newTypeS(kind, c)
     addSonSkipIntLit(result, baseType.assertNotNil)
 
 proc makeTypeDesc*(c: PContext, typ: PType): PType =

compiler/semexprs.nim

@@ -32,7 +32,7 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     # XXX tyGenericInst here?
     if result.typ.kind == tyProc and tfUnresolved in result.typ.flags:
       localError(n.info, errProcHasNoConcreteType, n.renderTree)
-    if result.typ.kind == tyVar: result = newDeref(result)
+    if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
   elif {efWantStmt, efAllowStmt} * flags != {}:
     result.typ = newTypeS(tyVoid, c)
   else:
@@ -52,7 +52,7 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     result.typ = errorType(c)
   else:
     if efNoProcvarCheck notin flags: semProcvarCheck(c, result)
-    if result.typ.kind == tyVar: result = newDeref(result)
+    if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
 
 proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   result = semExpr(c, n, flags)
@@ -181,9 +181,18 @@ proc semConv(c: PContext, n: PNode): PNode =
   result = newNodeI(nkConv, n.info)
   var targetType = semTypeNode(c, n.sons[0], nil).skipTypes({tyTypeDesc})
   maybeLiftType(targetType, c, n[0].info)
-  result.addSon copyTree(n.sons[0])
-  var op = semExprWithType(c, n.sons[1])
 
+  if targetType.kind in {tySink, tyLent}:
+    let baseType = semTypeNode(c, n.sons[1], nil).skipTypes({tyTypeDesc})
+    let t = newTypeS(targetType.kind, c)
+    t.rawAddSonNoPropagationOfTypeFlags baseType
+    result = newNodeI(nkType, n.info)
+    result.typ = makeTypeDesc(c, t)
+    return
+
+  result.addSon copyTree(n.sons[0])
+
+  var op = semExprWithType(c, n.sons[1])
   if targetType.isMetaType:
     let final = inferWithMetatype(c, targetType, op, true)
     result.addSon final
@@ -191,6 +200,8 @@ proc semConv(c: PContext, n: PNode): PNode =
     return
 
   result.typ = targetType
+  # XXX op is overwritten later on, this is likely added too early
+  # here or needs to be overwritten too then.
   addSon(result, op)
 
   if not isSymChoice(op):
@@ -350,7 +361,7 @@ proc changeType(n: PNode, newType: PType, check: bool) =
     for i in countup(0, sonsLen(n) - 1):
       changeType(n.sons[i], elemType(newType), check)
   of nkPar:
-    let tup = newType.skipTypes({tyGenericInst, tyAlias})
+    let tup = newType.skipTypes({tyGenericInst, tyAlias, tySink})
     if tup.kind != tyTuple:
       if tup.kind == tyObject: return
       globalError(n.info, "no tuple type for constructor")
@@ -393,7 +404,7 @@ proc arrayConstrType(c: PContext, n: PNode): PType =
   if sonsLen(n) == 0:
     rawAddSon(typ, newTypeS(tyEmpty, c)) # needs an empty basetype!
   else:
-    var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyOrdinal, tyAlias})
+    var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
     addSonSkipIntLit(typ, t)
   typ.sons[0] = makeRangeType(c, 0, sonsLen(n) - 1, n.info)
   result = typ
@@ -417,7 +428,7 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
     let yy = semExprWithType(c, x)
     var typ = yy.typ
     addSon(result, yy)
-    #var typ = skipTypes(result.sons[0].typ, {tyGenericInst, tyVar, tyOrdinal})
+    #var typ = skipTypes(result.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyOrdinal})
     for i in countup(1, sonsLen(n) - 1):
       x = n.sons[i]
       if x.kind == nkExprColonExpr and sonsLen(x) == 2:
@@ -471,7 +482,7 @@ proc analyseIfAddressTaken(c: PContext, n: PNode): PNode =
   of nkSym:
     # n.sym.typ can be nil in 'check' mode ...
     if n.sym.typ != nil and
-        skipTypes(n.sym.typ, abstractInst-{tyTypeDesc}).kind != tyVar:
+        skipTypes(n.sym.typ, abstractInst-{tyTypeDesc}).kind notin {tyVar, tyLent}:
       incl(n.sym.flags, sfAddrTaken)
       result = newHiddenAddrTaken(c, n)
   of nkDotExpr:
@@ -479,12 +490,12 @@ proc analyseIfAddressTaken(c: PContext, n: PNode): PNode =
     if n.sons[1].kind != nkSym:
       internalError(n.info, "analyseIfAddressTaken")
       return
-    if skipTypes(n.sons[1].sym.typ, abstractInst-{tyTypeDesc}).kind != tyVar:
+    if skipTypes(n.sons[1].sym.typ, abstractInst-{tyTypeDesc}).kind notin {tyVar, tyLent}:
       incl(n.sons[1].sym.flags, sfAddrTaken)
       result = newHiddenAddrTaken(c, n)
   of nkBracketExpr:
     checkMinSonsLen(n, 1)
-    if skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc}).kind != tyVar:
+    if skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc}).kind notin {tyVar, tyLent}:
       if n.sons[0].kind == nkSym: incl(n.sons[0].sym.flags, sfAddrTaken)
       result = newHiddenAddrTaken(c, n)
   else:
@@ -499,7 +510,7 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
 
   # get the real type of the callee
   # it may be a proc var with a generic alias type, so we skip over them
-  var t = n.sons[0].typ.skipTypes({tyGenericInst, tyAlias})
+  var t = n.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink})
 
   if n.sons[0].kind == nkSym and n.sons[0].sym.magic in FakeVarParams:
     # BUGFIX: check for L-Value still needs to be done for the arguments!
@@ -692,7 +703,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
   else:
     n.sons[0] = semExpr(c, n.sons[0], {efInCall})
     let t = n.sons[0].typ
-    if t != nil and t.kind == tyVar:
+    if t != nil and t.kind in {tyVar, tyLent}:
       n.sons[0] = newDeref(n.sons[0])
     elif n.sons[0].kind == nkBracketExpr:
       let s = bracketedMacro(n.sons[0])
@@ -780,6 +791,19 @@ proc buildEchoStmt(c: PContext, n: PNode): PNode =
 
 proc semExprNoType(c: PContext, n: PNode): PNode =
   result = semExpr(c, n, {efWantStmt})
+  # make an 'if' expression an 'if' statement again for backwards
+  # compatibility (.discardable was a bad idea!); bug #6980
+  var isStmt = false
+  if result.kind == nkIfExpr:
+    isStmt = true
+    for condActionPair in result:
+      let action = condActionPair.lastSon
+      if not implicitlyDiscardable(action) and not
+          endsInNoReturn(action):
+        isStmt = false
+    if isStmt:
+      result.kind = nkIfStmt
+      result.typ = nil
   discardCheck(c, result)
 
 proc isTypeExpr(n: PNode): bool =
@@ -852,7 +876,7 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent,
 
 const
   tyTypeParamsHolders = {tyGenericInst, tyCompositeTypeClass}
-  tyDotOpTransparent = {tyVar, tyPtr, tyRef, tyAlias}
+  tyDotOpTransparent = {tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink}
 
 proc readTypeParameter(c: PContext, typ: PType,
                        paramName: PIdent, info: TLineInfo): PNode =
@@ -927,7 +951,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     else:
       result = semMacroExpr(c, n, n, s, flags)
   of skTemplate:
-    if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0:
+    if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0 or 
+       sfCustomPragma in sym.flags:
       markUsed(n.info, s, c.graph.usageSym)
       styleCheckUse(n.info, s)
       result = newSymNode(s, n.info)
@@ -985,8 +1010,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     while p != nil and p.selfSym == nil:
       p = p.next
     if p != nil and p.selfSym != nil:
-      var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyPtr, tyRef,
+      var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef,
-                                         tyAlias})
+                                         tyAlias, tySink})
       while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct})
       var check: PNode = nil
       if ty.kind == tyObject:
@@ -1095,7 +1120,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
     return nil
   if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass:
     ty = ty.lastSon
-  ty = skipTypes(ty, {tyGenericInst, tyVar, tyPtr, tyRef, tyAlias})
+  ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink})
   while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct})
   var check: PNode = nil
   if ty.kind == tyObject:
@@ -1162,7 +1187,7 @@ proc semDeref(c: PContext, n: PNode): PNode =
   checkSonsLen(n, 1)
   n.sons[0] = semExprWithType(c, n.sons[0])
   result = n
-  var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyAlias})
+  var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyAlias, tySink})
   case t.kind
   of tyRef, tyPtr: n.typ = t.lastSon
   else: result = nil
@@ -1182,7 +1207,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
   n.sons[0] = semExprWithType(c, n.sons[0],
                               {efNoProcvarCheck, efNoEvaluateGeneric})
   let arr = skipTypes(n.sons[0].typ, {tyGenericInst,
-                                      tyVar, tyPtr, tyRef, tyAlias})
+                                      tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink})
   case arr.kind
   of tyArray, tyOpenArray, tyVarargs, tySequence, tyString,
      tyCString:
@@ -1210,7 +1235,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
     n.sons[0] = makeDeref(n.sons[0])
     # [] operator for tuples requires constant expression:
     n.sons[1] = semConstExpr(c, n.sons[1])
-    if skipTypes(n.sons[1].typ, {tyGenericInst, tyRange, tyOrdinal, tyAlias}).kind in
+    if skipTypes(n.sons[1].typ, {tyGenericInst, tyRange, tyOrdinal, tyAlias, tySink}).kind in
         {tyInt..tyInt64}:
       var idx = getOrdValue(n.sons[1])
       if idx >= 0 and idx < sonsLen(arr): n.typ = arr.sons[int(idx)]
@@ -1290,13 +1315,10 @@ proc takeImplicitAddr(c: PContext, n: PNode): PNode =
 proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
   if le.kind == nkHiddenDeref:
     var x = le.sons[0]
-    if x.typ.kind == tyVar and x.kind == nkSym:
+    if x.typ.kind == tyVar and x.kind == nkSym and x.sym.kind == skResult:
-      if x.sym.kind == skResult:
+      n.sons[0] = x # 'result[]' --> 'result'
-        n.sons[0] = x # 'result[]' --> 'result'
+      n.sons[1] = takeImplicitAddr(c, ri)
-        n.sons[1] = takeImplicitAddr(c, ri)
+      x.typ.flags.incl tfVarIsPtr
-      if x.sym.kind != skParam:
-        # XXX This is hacky. See bug #4910.
-        x.typ.flags.incl tfVarIsPtr
       #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info
 
 template resultTypeIsInferrable(typ: PType): untyped =
@@ -1352,7 +1374,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
   # a = b # both are vars, means: a[] = b[]
   # a = b # b no 'var T' means: a = addr(b)
   var le = a.typ
-  if (skipTypes(le, {tyGenericInst, tyAlias}).kind != tyVar and
+  if (skipTypes(le, {tyGenericInst, tyAlias, tySink}).kind != tyVar and
         isAssignable(c, a) == arNone) or
       skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs}:
     # Direct assignment to a discriminant is allowed!
@@ -1368,13 +1390,16 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
     if lhsIsResult:
       n.typ = enforceVoidContext
       if c.p.owner.kind != skMacro and resultTypeIsInferrable(lhs.sym.typ):
-        if cmpTypes(c, lhs.typ, rhs.typ) == isGeneric:
+        var rhsTyp = rhs.typ
+        if rhsTyp.kind in tyUserTypeClasses and rhsTyp.isResolvedUserTypeClass:
+          rhsTyp = rhsTyp.lastSon
+        if cmpTypes(c, lhs.typ, rhsTyp) in {isGeneric, isEqual}:
           internalAssert c.p.resultSym != nil
-          lhs.typ = rhs.typ
+          lhs.typ = rhsTyp
-          c.p.resultSym.typ = rhs.typ
+          c.p.resultSym.typ = rhsTyp
-          c.p.owner.typ.sons[0] = rhs.typ
+          c.p.owner.typ.sons[0] = rhsTyp
         else:
-          typeMismatch(n.info, lhs.typ, rhs.typ)
+          typeMismatch(n.info, lhs.typ, rhsTyp)
 
     n.sons[1] = fitNode(c, le, rhs, n.info)
     if not newDestructors:
@@ -1412,11 +1437,7 @@ proc semProcBody(c: PContext, n: PNode): PNode =
   openScope(c)
   result = semExpr(c, n)
   if c.p.resultSym != nil and not isEmptyType(result.typ):
-    # transform ``expr`` to ``result = expr``, but not if the expr is already
+    if result.kind == nkNilLit:
-    # ``result``:
-    if result.kind == nkSym and result.sym == c.p.resultSym:
-      discard
-    elif result.kind == nkNilLit:
       # or ImplicitlyDiscardable(result):
       # new semantic: 'result = x' triggers the void context
       result.typ = nil
@@ -1446,17 +1467,18 @@ proc semProcBody(c: PContext, n: PNode): PNode =
   closeScope(c)
 
 proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
-  var t = skipTypes(restype, {tyGenericInst, tyAlias})
+  var t = skipTypes(restype, {tyGenericInst, tyAlias, tySink})
   case t.kind
-  of tyVar:
+  of tyVar, tyLent:
+    if t.kind == tyVar: t.flags.incl tfVarIsPtr # bugfix for #4048, #4910, #6892
     if n.sons[0].kind in {nkHiddenStdConv, nkHiddenSubConv}:
       n.sons[0] = n.sons[0].sons[1]
-
     n.sons[0] = takeImplicitAddr(c, n.sons[0])
   of tyTuple:
     for i in 0..<t.sonsLen:
-      var e = skipTypes(t.sons[i], {tyGenericInst, tyAlias})
+      var e = skipTypes(t.sons[i], {tyGenericInst, tyAlias, tySink})
-      if e.kind == tyVar:
+      if e.kind in {tyVar, tyLent}:
+        if e.kind == tyVar: e.flags.incl tfVarIsPtr # bugfix for #4048, #4910, #6892
         if n.sons[0].kind == nkPar:
           n.sons[0].sons[i] = takeImplicitAddr(c, n.sons[0].sons[i])
         elif n.sons[0].kind in {nkHiddenStdConv, nkHiddenSubConv} and
@@ -1775,6 +1797,13 @@ proc setMs(n: PNode, s: PSym): PNode =
   n.sons[0] = newSymNode(s)
   n.sons[0].info = n.info
 
+proc extractImports(n: PNode; result: PNode) =
+  if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
+    result.add copyTree(n)
+    n.kind = nkEmpty
+    return
+  for i in 0..<n.safeLen: extractImports(n[i], result)
+
 proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
   # this is a hotspot in the compiler!
   # DON'T forget to update ast.SpecialSemMagics if you add a magic here!
@@ -1848,14 +1877,16 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
         result = magicsAfterOverloadResolution(c, result, flags)
   of mRunnableExamples:
     if gCmd == cmdDoc and n.len >= 2 and n.lastSon.kind == nkStmtList:
-      if n.sons[0].kind == nkIdent:
+      if sfMainModule in c.module.flags:
-        if sfMainModule in c.module.flags:
+        let inp = toFullPath(c.module.info)
-          let inp = toFullPath(c.module.info)
+        if c.runnableExamples == nil:
-          if c.runnableExamples == nil:
+          c.runnableExamples = newTree(nkStmtList,
-            c.runnableExamples = newTree(nkStmtList,
+            newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp))))
-              newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp))))
+        let imports = newTree(nkStmtList)
-          c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon)
+        extractImports(n.lastSon, imports)
-        result = setMs(n, s)
+        for imp in imports: c.runnableExamples.add imp
+        c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon)
+      result = setMs(n, s)
     else:
       result = emptyNode
   else:
@@ -1936,17 +1967,17 @@ proc semSetConstr(c: PContext, n: PNode): PNode =
         n.sons[i].sons[2] = semExprWithType(c, n.sons[i].sons[2])
         if typ == nil:
           typ = skipTypes(n.sons[i].sons[1].typ,
-                          {tyGenericInst, tyVar, tyOrdinal, tyAlias})
+                          {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
         n.sons[i].typ = n.sons[i].sons[2].typ # range node needs type too
       elif n.sons[i].kind == nkRange:
         # already semchecked
         if typ == nil:
           typ = skipTypes(n.sons[i].sons[0].typ,
-                          {tyGenericInst, tyVar, tyOrdinal, tyAlias})
+                          {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
       else:
         n.sons[i] = semExprWithType(c, n.sons[i])
         if typ == nil:
-          typ = skipTypes(n.sons[i].typ, {tyGenericInst, tyVar, tyOrdinal, tyAlias})
+          typ = skipTypes(n.sons[i].typ, {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
     if not isOrdinalType(typ):
       localError(n.info, errOrdinalTypeExpected)
       typ = makeRangeType(c, 0, MaxSetElements-1, n.info)
@@ -2123,6 +2154,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   of nkIdent, nkAccQuoted:
     let checks = if efNoEvaluateGeneric in flags:
         {checkUndeclared, checkPureEnumFields}
+      elif efInCall in flags:
+        {checkUndeclared, checkModule, checkPureEnumFields}
       else:
         {checkUndeclared, checkModule, checkAmbiguity, checkPureEnumFields}
     var s = qualifiedLookUp(c, n, checks)
@@ -2217,10 +2250,10 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
         # XXX think about this more (``set`` procs)
         if n.len == 2:
           result = semConv(c, n)
+        elif contains(c.ambiguousSymbols, s.id) and n.len == 1:
+          errorUseQualifier(c, n.info, s)
         elif n.len == 1:
           result = semObjConstr(c, n, flags)
-        elif contains(c.ambiguousSymbols, s.id):
-          errorUseQualifier(c, n.info, s)
         elif s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
       of skProc, skFunc, skMethod, skConverter, skIterator:
@@ -2370,6 +2403,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     if n.len != 1 and n.len != 2: illFormedAst(n)
     for i in 0 ..< n.len:
       n.sons[i] = semExpr(c, n.sons[i])
+  of nkComesFrom: discard "ignore the comes from information for now"
   else:
     localError(n.info, errInvalidExpressionX,
                renderTree(n, {renderNoComments}))

compiler/semfold.nim

@@ -408,7 +408,7 @@ proc getArrayConstr(m: PSym, n: PNode): PNode =
 
 proc foldArrayAccess(m: PSym, n: PNode): PNode =
   var x = getConstExpr(m, n.sons[0])
-  if x == nil or x.typ.skipTypes({tyGenericInst, tyAlias}).kind == tyTypeDesc:
+  if x == nil or x.typ.skipTypes({tyGenericInst, tyAlias, tySink}).kind == tyTypeDesc:
     return
 
   var y = getConstExpr(m, n.sons[1])
@@ -655,5 +655,8 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
       result.typ = n.typ
   of nkBracketExpr: result = foldArrayAccess(m, n)
   of nkDotExpr: result = foldFieldAccess(m, n)
+  of nkStmtListExpr:
+    if n.len == 2 and n[0].kind == nkComesFrom:
+      result = getConstExpr(m, n[1])
   else:
     discard

compiler/semgnrc.nim

@@ -186,7 +186,7 @@ proc semGenericStmt(c: PContext, n: PNode,
     let a = n.sym
     let b = getGenSym(c, a)
     if b != a: n.sym = b
-  of nkEmpty, succ(nkSym)..nkNilLit:
+  of nkEmpty, succ(nkSym)..nkNilLit, nkComesFrom:
     # see tests/compile/tgensymgeneric.nim:
     # We need to open the gensym'ed symbol again so that the instantiation
     # creates a fresh copy; but this is wrong the very first reason for gensym

compiler/semmagic.nim

@@ -108,7 +108,7 @@ proc uninstantiate(t: PType): PType =
     else: t
 
 proc evalTypeTrait(traitCall: PNode, operand: PType, context: PSym): PNode =
-  const skippedTypes = {tyTypeDesc, tyAlias}
+  const skippedTypes = {tyTypeDesc, tyAlias, tySink}
   let trait = traitCall[0]
   internalAssert trait.kind == nkSym
   var operand = operand.skipTypes(skippedTypes)
@@ -145,7 +145,7 @@ proc evalTypeTrait(traitCall: PNode, operand: PType, context: PSym): PNode =
   of "stripGenericParams":
     result = uninstantiate(operand).toNode(traitCall.info)
   of "supportsCopyMem":
-    let t = operand.skipTypes({tyVar, tyGenericInst, tyAlias, tyInferred})
+    let t = operand.skipTypes({tyVar, tyLent, tyGenericInst, tyAlias, tySink, tyInferred})
     let complexObj = containsGarbageCollectedRef(t) or
                      hasDestructor(t)
     result = newIntNodeT(ord(not complexObj), traitCall)

compiler/semobjconstr.nim

@@ -39,13 +39,19 @@ proc mergeInitStatus(existing: var InitStatus, newStatus: InitStatus) =
   of initUnknown:
     discard
 
+proc invalidObjConstr(n: PNode) =
+  if n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s[0] == ':':
+    localError(n.info, "incorrect object construction syntax; use a space after the colon")
+  else:
+    localError(n.info, "incorrect object construction syntax")
+
 proc locateFieldInInitExpr(field: PSym, initExpr: PNode): PNode =
   # Returns the assignment nkExprColonExpr node or nil
   let fieldId = field.name.id
   for i in 1 ..< initExpr.len:
     let assignment = initExpr[i]
     if assignment.kind != nkExprColonExpr:
-      localError(initExpr.info, "incorrect object construction syntax")
+      invalidObjConstr(assignment)
       continue
 
     if fieldId == considerQuotedIdent(assignment[0]).id:
@@ -254,8 +260,8 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
   result = newNodeIT(nkObjConstr, n.info, t)
   for child in n: result.add child
 
-  t = skipTypes(t, {tyGenericInst, tyAlias})
+  t = skipTypes(t, {tyGenericInst, tyAlias, tySink})
-  if t.kind == tyRef: t = skipTypes(t.sons[0], {tyGenericInst, tyAlias})
+  if t.kind == tyRef: t = skipTypes(t.sons[0], {tyGenericInst, tyAlias, tySink})
   if t.kind != tyObject:
     localError(n.info, errGenerated, "object constructor needs an object type")
     return
@@ -284,7 +290,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
     let field = result[i]
     if nfSem notin field.flags:
       if field.kind != nkExprColonExpr:
-        localError(n.info, "incorrect object construction syntax")
+        invalidObjConstr(field)
         continue
       let id = considerQuotedIdent(field[0])
       # This node was not processed. There are two possible reasons:

compiler/sempass2.nim

@@ -979,10 +979,10 @@ proc trackProc*(s: PSym, body: PNode) =
     message(s.info, warnLockLevel,
       "declared lock level is $1, but real lock level is $2" %
         [$s.typ.lockLevel, $t.maxLockLevel])
-  when false:
+  when defined(useDfa):
     if s.kind == skFunc:
-      when defined(dfa): dataflowAnalysis(s, body)
+      dataflowAnalysis(s, body)
-      trackWrites(s, body)
+      when false: trackWrites(s, body)
 
 proc trackTopLevelStmt*(module: PSym; n: PNode) =
   if n.kind in {nkPragma, nkMacroDef, nkTemplateDef, nkProcDef, nkFuncDef,

compiler/semstmts.nim

@@ -102,7 +102,7 @@ proc semExprBranch(c: PContext, n: PNode): PNode =
   if result.typ != nil:
     # XXX tyGenericInst here?
     semProcvarCheck(c, result)
-    if result.typ.kind == tyVar: result = newDeref(result)
+    if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
 
 proc semExprBranchScope(c: PContext, n: PNode): PNode =
   openScope(c)
@@ -165,14 +165,14 @@ proc semIf(c: PContext, n: PNode): PNode =
       it.sons[0] = forceBool(c, semExprWithType(c, it.sons[0]))
       when not newScopeForIf: openScope(c)
       it.sons[1] = semExprBranch(c, it.sons[1])
-      typ = commonType(typ, it.sons[1].typ)
+      typ = commonType(typ, it.sons[1])
       closeScope(c)
     elif it.len == 1:
       hasElse = true
       it.sons[0] = semExprBranchScope(c, it.sons[0])
-      typ = commonType(typ, it.sons[0].typ)
+      typ = commonType(typ, it.sons[0])
     else: illFormedAst(it)
-  if isEmptyType(typ) or typ.kind == tyNil or not hasElse:
+  if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or not hasElse:
     for it in n: discardCheck(c, it.lastSon)
     result.kind = nkIfStmt
     # propagate any enforced VoidContext:
@@ -180,7 +180,8 @@ proc semIf(c: PContext, n: PNode): PNode =
   else:
     for it in n:
       let j = it.len-1
-      it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
+      if not endsInNoReturn(it.sons[j]):
+        it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
     result.kind = nkIfExpr
     result.typ = typ
 
@@ -213,7 +214,7 @@ proc semCase(c: PContext, n: PNode): PNode =
       semCaseBranch(c, n, x, i, covered)
       var last = sonsLen(x)-1
       x.sons[last] = semExprBranchScope(c, x.sons[last])
-      typ = commonType(typ, x.sons[last].typ)
+      typ = commonType(typ, x.sons[last])
     of nkElifBranch:
       chckCovered = false
       checkSonsLen(x, 2)
@@ -221,13 +222,13 @@ proc semCase(c: PContext, n: PNode): PNode =
       x.sons[0] = forceBool(c, semExprWithType(c, x.sons[0]))
       when not newScopeForIf: openScope(c)
       x.sons[1] = semExprBranch(c, x.sons[1])
-      typ = commonType(typ, x.sons[1].typ)
+      typ = commonType(typ, x.sons[1])
       closeScope(c)
     of nkElse:
       chckCovered = false
       checkSonsLen(x, 1)
       x.sons[0] = semExprBranchScope(c, x.sons[0])
-      typ = commonType(typ, x.sons[0].typ)
+      typ = commonType(typ, x.sons[0])
       hasElse = true
     else:
       illFormedAst(x)
@@ -237,7 +238,7 @@ proc semCase(c: PContext, n: PNode): PNode =
     else:
       localError(n.info, errNotAllCasesCovered)
   closeScope(c)
-  if isEmptyType(typ) or typ.kind == tyNil or not hasElse:
+  if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or not hasElse:
     for i in 1..n.len-1: discardCheck(c, n.sons[i].lastSon)
     # propagate any enforced VoidContext:
     if typ == enforceVoidContext:
@@ -246,7 +247,8 @@ proc semCase(c: PContext, n: PNode): PNode =
     for i in 1..n.len-1:
       var it = n.sons[i]
       let j = it.len-1
-      it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
+      if not endsInNoReturn(it.sons[j]):
+        it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
     result.typ = typ
 
 proc semTry(c: PContext, n: PNode): PNode =
@@ -441,20 +443,21 @@ proc hasEmpty(typ: PType): bool =
       result = result or hasEmpty(s)
 
 proc makeDeref(n: PNode): PNode =
-  var t = skipTypes(n.typ, {tyGenericInst, tyAlias})
+  var t = n.typ
   if t.kind in tyUserTypeClasses and t.isResolvedUserTypeClass:
     t = t.lastSon
+  t = skipTypes(t, {tyGenericInst, tyAlias, tySink})
   result = n
-  if t.kind == tyVar:
+  if t.kind in {tyVar, tyLent}:
     result = newNodeIT(nkHiddenDeref, n.info, t.sons[0])
     addSon(result, n)
-    t = skipTypes(t.sons[0], {tyGenericInst, tyAlias})
+    t = skipTypes(t.sons[0], {tyGenericInst, tyAlias, tySink})
   while t.kind in {tyPtr, tyRef}:
     var a = result
     let baseTyp = t.lastSon
     result = newNodeIT(nkHiddenDeref, n.info, baseTyp)
     addSon(result, a)
-    t = skipTypes(baseTyp, {tyGenericInst, tyAlias})
+    t = skipTypes(baseTyp, {tyGenericInst, tyAlias, tySink})
 
 proc fillPartialObject(c: PContext; n: PNode; typ: PType) =
   if n.len == 2:
@@ -530,7 +533,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
     if typ == nil: continue
     typeAllowedCheck(a.info, typ, symkind)
     liftTypeBoundOps(c, typ, a.info)
-    var tup = skipTypes(typ, {tyGenericInst, tyAlias})
+    var tup = skipTypes(typ, {tyGenericInst, tyAlias, tySink})
     if a.kind == nkVarTuple:
       if tup.kind != tyTuple:
         localError(a.info, errXExpected, "tuple")
@@ -646,7 +649,7 @@ proc semForVars(c: PContext, n: PNode): PNode =
   result = n
   var length = sonsLen(n)
   let iterBase = n.sons[length-2].typ
-  var iter = skipTypes(iterBase, {tyGenericInst, tyAlias})
+  var iter = skipTypes(iterBase, {tyGenericInst, tyAlias, tySink})
   # length == 3 means that there is one for loop variable
   # and thus no tuple unpacking:
   if iter.kind != tyTuple or length == 3:
@@ -680,7 +683,7 @@ proc semForVars(c: PContext, n: PNode): PNode =
 proc implicitIterator(c: PContext, it: string, arg: PNode): PNode =
   result = newNodeI(nkCall, arg.info)
   result.add(newIdentNode(it.getIdent, arg.info))
-  if arg.typ != nil and arg.typ.kind == tyVar:
+  if arg.typ != nil and arg.typ.kind in {tyVar, tyLent}:
     result.add newDeref(arg)
   else:
     result.add arg
@@ -730,6 +733,18 @@ proc semRaise(c: PContext, n: PNode): PNode =
     if typ.kind != tyRef or typ.lastSon.kind != tyObject:
       localError(n.info, errExprCannotBeRaised)
 
+    # check if the given object inherits from Exception
+    var base = typ.lastSon
+    while true:
+      if base.sym.magic == mException:
+        break
+      if base.lastSon == nil:
+        localError(n.info,
+          "raised object of type $1 does not inherit from Exception",
+          [typeToString(typ)])
+        return
+      base = base.lastSon
+
 proc addGenericParamListToScope(c: PContext, n: PNode) =
   if n.kind != nkGenericParams: illFormedAst(n)
   for i in countup(0, sonsLen(n)-1):
@@ -865,11 +880,11 @@ proc checkCovariantParamsUsages(genericType: PType) =
       for fieldType in t.sons:
         subresult traverseSubTypes(fieldType)
 
-    of tyPtr, tyRef, tyVar:
+    of tyPtr, tyRef, tyVar, tyLent:
       if t.base.kind == tyGenericParam: return true
       return traverseSubTypes(t.base)
 
-    of tyDistinct, tyAlias:
+    of tyDistinct, tyAlias, tySink:
       return traverseSubTypes(t.lastSon)
 
     of tyGenericInst:
@@ -979,8 +994,8 @@ proc checkForMetaFields(n: PNode) =
   of nkSym:
     let t = n.sym.typ
     case t.kind
-    of tySequence, tySet, tyArray, tyOpenArray, tyVar, tyPtr, tyRef,
+    of tySequence, tySet, tyArray, tyOpenArray, tyVar, tyLent, tyPtr, tyRef,
-       tyProc, tyGenericInvocation, tyGenericInst, tyAlias:
+       tyProc, tyGenericInvocation, tyGenericInst, tyAlias, tySink:
       let start = int ord(t.kind in {tyGenericInvocation, tyGenericInst})
       for i in start ..< t.sons.len:
         checkMeta(t.sons[i])
@@ -1005,7 +1020,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
         # type aliases are hard:
         var t = semTypeNode(c, x, nil)
         assert t != nil
-        if s.typ != nil and s.typ.kind != tyAlias:
+        if s.typ != nil and s.typ.kind notin {tyAlias, tySink}:
           if t.kind in {tyProc, tyGenericInst} and not t.isMetaType:
             assignType(s.typ, t)
             s.typ.id = t.id
@@ -1138,6 +1153,9 @@ proc semProcAnnotation(c: PContext, prc: PNode;
         else:
           localError(prc.info, errOnlyACallOpCanBeDelegator)
       continue
+    elif sfCustomPragma in m.flags:
+      continue # semantic check for custom pragma happens later in semProcAux
+      
     # we transform ``proc p {.m, rest.}`` into ``m(do: proc p {.rest.})`` and
     # let the semantic checker deal with it:
     var x = newNodeI(nkCall, n.info)
@@ -1400,9 +1418,9 @@ proc semMethodPrototype(c: PContext; s: PSym; n: PNode) =
     for col in countup(1, sonsLen(tt)-1):
       let t = tt.sons[col]
       if t != nil and t.kind == tyGenericInvocation:
-        var x = skipTypes(t.sons[0], {tyVar, tyPtr, tyRef, tyGenericInst,
+        var x = skipTypes(t.sons[0], {tyVar, tyLent, tyPtr, tyRef, tyGenericInst,
                                       tyGenericInvocation, tyGenericBody,
-                                      tyAlias})
+                                      tyAlias, tySink})
         if x.kind == tyObject and t.len-1 == n.sons[genericParamsPos].len:
           foundObj = true
           x.methods.safeAdd((col,s))
@@ -1840,8 +1858,8 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
       else:
         n.typ = n.sons[i].typ
         if not isEmptyType(n.typ): n.kind = nkStmtListExpr
-      case n.sons[i].kind
+      if n.sons[i].kind in LastBlockStmts or
-      of LastBlockStmts:
+         n.sons[i].kind in nkCallKinds and n.sons[i][0].kind == nkSym and sfNoReturn in n.sons[i][0].sym.flags:
         for j in countup(i + 1, length - 1):
           case n.sons[j].kind
           of nkPragma, nkCommentStmt, nkNilLit, nkEmpty, nkBlockExpr,

compiler/semtempl.nim

@@ -331,7 +331,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   of nkMixinStmt:
     if c.scopeN > 0: result = semTemplBodySons(c, n)
     else: result = semMixinStmt(c.c, n, c.toMixin)
-  of nkEmpty, nkSym..nkNilLit:
+  of nkEmpty, nkSym..nkNilLit, nkComesFrom:
     discard
   of nkIfStmt:
     for i in countup(0, sonsLen(n)-1):
@@ -528,7 +528,7 @@ proc semTemplBodyDirty(c: var TemplCtx, n: PNode): PNode =
     result = semTemplBodyDirty(c, n.sons[0])
   of nkBindStmt:
     result = semBindStmt(c.c, n, c.toBind)
-  of nkEmpty, nkSym..nkNilLit:
+  of nkEmpty, nkSym..nkNilLit, nkComesFrom:
     discard
   else:
     # dotExpr is ambiguous: note that we explicitly allow 'x.TemplateParam',
@@ -608,7 +608,10 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
   popOwner(c)
   s.ast = n
   result = n
-  if n.sons[bodyPos].kind == nkEmpty:
+  if sfCustomPragma in s.flags:
+    if n.sons[bodyPos].kind != nkEmpty:
+      localError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s)
+  elif n.sons[bodyPos].kind == nkEmpty:
     localError(n.info, errImplOfXexpected, s.name.s)
   var proto = searchForProc(c, c.currentScope, s)
   if proto == nil:

compiler/semtypes.nim

@@ -101,7 +101,7 @@ proc semSet(c: PContext, n: PNode, prev: PType): PType =
   if sonsLen(n) == 2:
     var base = semTypeNode(c, n.sons[1], nil)
     addSonSkipIntLit(result, base)
-    if base.kind in {tyGenericInst, tyAlias}: base = lastSon(base)
+    if base.kind in {tyGenericInst, tyAlias, tySink}: base = lastSon(base)
     if base.kind != tyGenericParam:
       if not isOrdinalType(base):
         localError(n.info, errOrdinalTypeExpected)
@@ -153,7 +153,7 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType =
         isNilable = true
       else:
         let region = semTypeNode(c, ni, nil)
-        if region.skipTypes({tyGenericInst, tyAlias}).kind notin {
+        if region.skipTypes({tyGenericInst, tyAlias, tySink}).kind notin {
               tyError, tyObject}:
           message n[i].info, errGenerated, "region needs to be an object type"
         addSonSkipIntLit(result, region)
@@ -286,7 +286,7 @@ proc semArray(c: PContext, n: PNode, prev: PType): PType =
     # 3 = length(array indx base)
     let indx = semArrayIndex(c, n[1])
     var indxB = indx
-    if indxB.kind in {tyGenericInst, tyAlias}: indxB = lastSon(indxB)
+    if indxB.kind in {tyGenericInst, tyAlias, tySink}: indxB = lastSon(indxB)
     if indxB.kind notin {tyGenericParam, tyStatic, tyFromExpr}:
       if not isOrdinalType(indxB):
         localError(n.sons[1].info, errOrdinalTypeExpected)
@@ -320,11 +320,8 @@ proc semTypeIdent(c: PContext, n: PNode): PSym =
   if n.kind == nkSym:
     result = getGenSym(c, n.sym)
   else:
-    when defined(nimfix):
+    result = pickSym(c, n, {skType, skGenericParam})
-      result = pickSym(c, n, skType)
+    if result.isNil:
-      if result.isNil:
-        result = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared})
-    else:
       result = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared})
     if result != nil:
       markUsed(n.info, result, c.graph.usageSym)
@@ -676,7 +673,7 @@ proc skipGenericInvocation(t: PType): PType {.inline.} =
   result = t
   if result.kind == tyGenericInvocation:
     result = result.sons[0]
-  while result.kind in {tyGenericInst, tyGenericBody, tyRef, tyPtr, tyAlias}:
+  while result.kind in {tyGenericInst, tyGenericBody, tyRef, tyPtr, tyAlias, tySink}:
     result = lastSon(result)
 
 proc addInheritedFields(c: PContext, check: var IntSet, pos: var int,
@@ -839,7 +836,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
       result = liftingWalk(paramType.sons[0], true)
 
   of tySequence, tySet, tyArray, tyOpenArray,
-     tyVar, tyPtr, tyRef, tyProc:
+     tyVar, tyLent, tyPtr, tyRef, tyProc:
     # XXX: this is a bit strange, but proc(s: seq)
     # produces tySequence(tyGenericParam, tyNone).
     # This also seems to be true when creating aliases
@@ -995,7 +992,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
       if isType: localError(a.info, "':' expected")
       if kind in {skTemplate, skMacro}:
         typ = newTypeS(tyExpr, c)
-    elif skipTypes(typ, {tyGenericInst, tyAlias}).kind == tyVoid:
+    elif skipTypes(typ, {tyGenericInst, tyAlias, tySink}).kind == tyVoid:
       continue
     for j in countup(0, length-3):
       var arg = newSymG(skParam, a.sons[j], c)
@@ -1027,7 +1024,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
   if r != nil:
     # turn explicit 'void' return type into 'nil' because the rest of the
     # compiler only checks for 'nil':
-    if skipTypes(r, {tyGenericInst, tyAlias}).kind != tyVoid:
+    if skipTypes(r, {tyGenericInst, tyAlias, tySink}).kind != tyVoid:
       # 'auto' as a return type does not imply a generic:
       if r.kind == tyAnything:
         # 'p(): auto' and 'p(): expr' are equivalent, but the rest of the
@@ -1338,7 +1335,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       result = semRangeAux(c, n, prev)
     elif n[0].kind == nkNilLit and n.len == 2:
       result = semTypeNode(c, n.sons[1], prev)
-      if result.skipTypes({tyGenericInst, tyAlias}).kind in NilableTypes+GenericTypes:
+      if result.skipTypes({tyGenericInst, tyAlias, tySink}).kind in NilableTypes+GenericTypes:
         if tfNotNil in result.flags:
           result = freshType(result, prev)
           result.flags.excl(tfNotNil)
@@ -1366,7 +1363,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
         case n.len
         of 3:
           result = semTypeNode(c, n.sons[1], prev)
-          if result.skipTypes({tyGenericInst, tyAlias}).kind in NilableTypes+GenericTypes+{tyForward} and
+          if result.skipTypes({tyGenericInst, tyAlias, tySink}).kind in NilableTypes+GenericTypes+{tyForward} and
               n.sons[2].kind == nkNilLit:
             result = freshType(result, prev)
             result.flags.incl(tfNotNil)
@@ -1419,7 +1416,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     of mVar:
       result = newOrPrevType(tyVar, prev, c)
       var base = semTypeNode(c, n.sons[1], nil)
-      if base.kind == tyVar:
+      if base.kind in {tyVar, tyLent}:
         localError(n.info, errVarVarTypeNotAllowed)
         base = base.sons[0]
       addSonSkipIntLit(result, base)
@@ -1620,6 +1617,12 @@ proc processMagicType(c: PContext, m: PSym) =
     rawAddSon(m.typ, newTypeS(tyNone, c))
   of mPNimrodNode:
     incl m.typ.flags, tfTriggersCompileTime
+  of mException: discard
+  of mBuiltinType:
+    case m.name.s
+    of "lent": setMagicType(m, tyLent, ptrSize)
+    of "sink": setMagicType(m, tySink, 0)
+    else: localError(m.info, errTypeExpected)
   else: localError(m.info, errTypeExpected)
 
 proc semGenericConstraints(c: PContext, x: PType): PType =

compiler/semtypinst.nim

@@ -17,7 +17,7 @@ const
 proc checkPartialConstructedType(info: TLineInfo, t: PType) =
   if tfAcyclic in t.flags and skipTypes(t, abstractInst).kind != tyObject:
     localError(info, errInvalidPragmaX, "acyclic")
-  elif t.kind == tyVar and t.sons[0].kind == tyVar:
+  elif t.kind in {tyVar, tyLent} and t.sons[0].kind in {tyVar, tyLent}:
     localError(info, errVarVarTypeNotAllowed)
 
 proc checkConstructedType*(info: TLineInfo, typ: PType) =
@@ -25,7 +25,7 @@ proc checkConstructedType*(info: TLineInfo, typ: PType) =
   if t.kind in tyTypeClasses: discard
   elif tfAcyclic in t.flags and skipTypes(t, abstractInst).kind != tyObject:
     localError(info, errInvalidPragmaX, "acyclic")
-  elif t.kind == tyVar and t.sons[0].kind == tyVar:
+  elif t.kind in {tyVar, tyLent} and t.sons[0].kind in {tyVar, tyLent}:
     localError(info, errVarVarTypeNotAllowed)
   elif computeSize(t) == szIllegalRecursion:
     localError(info, errIllegalRecursionInTypeX, typeToString(t))
@@ -518,7 +518,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
           var r = replaceTypeVarsT(cl, result.sons[i])
           if result.kind == tyObject:
             # carefully coded to not skip the precious tyGenericInst:
-            let r2 = r.skipTypes({tyAlias})
+            let r2 = r.skipTypes({tyAlias, tySink})
             if r2.kind in {tyPtr, tyRef}:
               r = skipTypes(r2, {tyPtr, tyRef})
           result.sons[i] = r

compiler/sigmatch.nim

@@ -180,7 +180,8 @@ proc sumGeneric(t: PType): int =
   while true:
     case t.kind
     of tyGenericInst, tyArray, tyRef, tyPtr, tyDistinct,
-        tyOpenArray, tyVarargs, tySet, tyRange, tySequence, tyGenericBody:
+        tyOpenArray, tyVarargs, tySet, tyRange, tySequence, tyGenericBody,
+        tyLent:
       t = t.lastSon
       inc result
     of tyOr:
@@ -207,7 +208,7 @@ proc sumGeneric(t: PType): int =
     of tyStatic:
       return t.sons[0].sumGeneric + 1
     of tyGenericParam, tyExpr, tyStmt: break
-    of tyAlias: t = t.lastSon
+    of tyAlias, tySink: t = t.lastSon
     of tyBool, tyChar, tyEnum, tyObject, tyPointer,
         tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
         tyUInt..tyUInt64, tyCompositeTypeClass:
@@ -464,7 +465,7 @@ proc skipToObject(t: PType; skipped: var SkippedPtr): PType =
       inc ptrs
       skipped = skippedPtr
       r = r.lastSon
-    of tyGenericBody, tyGenericInst, tyAlias:
+    of tyGenericBody, tyGenericInst, tyAlias, tySink:
       r = r.lastSon
     else:
       break
@@ -524,7 +525,7 @@ proc allowsNil(f: PType): TTypeRelation {.inline.} =
   result = if tfNotNil notin f.flags: isSubtype else: isNone
 
 proc inconsistentVarTypes(f, a: PType): bool {.inline.} =
-  result = f.kind != a.kind and (f.kind == tyVar or a.kind == tyVar)
+  result = f.kind != a.kind and (f.kind in {tyVar, tyLent} or a.kind in {tyVar, tyLent})
 
 proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
   ## For example we have:
@@ -889,7 +890,7 @@ proc inferStaticsInRange(c: var TCandidate,
     doInferStatic(lowerBound, upperBound.intVal + 1 - lengthOrd(concrete))
 
 template subtypeCheck() =
-  if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in {tyRef, tyPtr, tyVar}:
+  if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in {tyRef, tyPtr, tyVar, tyLent}:
     result = isNone
 
 proc isCovariantPtr(c: var TCandidate, f, a: PType): bool =
@@ -897,7 +898,7 @@ proc isCovariantPtr(c: var TCandidate, f, a: PType): bool =
   assert f.kind == a.kind
 
   template baseTypesCheck(lhs, rhs: PType): bool =
-    lhs.kind notin {tyPtr, tyRef, tyVar} and
+    lhs.kind notin {tyPtr, tyRef, tyVar, tyLent} and
       typeRel(c, lhs, rhs, {trNoCovariance}) == isSubtype
 
   case f.kind
@@ -983,17 +984,17 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
   template doBind: bool = trDontBind notin flags
 
   # var and static arguments match regular modifier-free types
-  var a = aOrig.skipTypes({tyStatic, tyVar}).maybeSkipDistinct(c.calleeSym)
+  var a = aOrig.skipTypes({tyStatic, tyVar, tyLent}).maybeSkipDistinct(c.calleeSym)
   # XXX: Theoretically, maybeSkipDistinct could be called before we even
   # start the param matching process. This could be done in `prepareOperand`
   # for example, but unfortunately `prepareOperand` is not called in certain
   # situation when nkDotExpr are rotated to nkDotCalls
 
-  if aOrig.kind == tyAlias:
+  if aOrig.kind in {tyAlias, tySink}:
     return typeRel(c, f, lastSon(aOrig))
 
   if a.kind == tyGenericInst and
-      skipTypes(f, {tyVar}).kind notin {
+      skipTypes(f, {tyVar, tyLent}).kind notin {
         tyGenericBody, tyGenericInvocation,
         tyGenericInst, tyGenericParam} + tyTypeClasses:
     return typeRel(c, f, lastSon(a))
@@ -1105,8 +1106,8 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
   of tyFloat32:  result = handleFloatRange(f, a)
   of tyFloat64:  result = handleFloatRange(f, a)
   of tyFloat128: result = handleFloatRange(f, a)
-  of tyVar:
+  of tyVar, tyLent:
-    if aOrig.kind == tyVar: result = typeRel(c, f.base, aOrig.base)
+    if aOrig.kind == f.kind: result = typeRel(c, f.base, aOrig.base)
     else: result = typeRel(c, f.base, aOrig, flags + {trNoCovariance})
     subtypeCheck()
   of tyArray:
@@ -1311,7 +1312,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
   of tyEmpty, tyVoid:
     if a.kind == f.kind: result = isEqual
 
-  of tyAlias:
+  of tyAlias, tySink:
     result = typeRel(c, lastSon(f), a)
 
   of tyGenericInst:
@@ -1497,7 +1498,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
     considerPreviousT:
       let targetKind = f.sons[0].kind
       let effectiveArgType = a.skipTypes({tyRange, tyGenericInst,
-                                          tyBuiltInTypeClass, tyAlias})
+                                          tyBuiltInTypeClass, tyAlias, tySink})
       let typeClassMatches = targetKind == effectiveArgType.kind and
                              not effectiveArgType.isEmptyContainer
       if typeClassMatches or
@@ -2068,7 +2069,8 @@ proc prepareNamedParam(a: PNode) =
 proc arrayConstr(c: PContext, n: PNode): PType =
   result = newTypeS(tyArray, c)
   rawAddSon(result, makeRangeType(c, 0, 0, n.info))
-  addSonSkipIntLit(result, skipTypes(n.typ, {tyGenericInst, tyVar, tyOrdinal}))
+  addSonSkipIntLit(result, skipTypes(n.typ,
+      {tyGenericInst, tyVar, tyLent, tyOrdinal}))
 
 proc arrayConstr(c: PContext, info: TLineInfo): PType =
   result = newTypeS(tyArray, c)

compiler/suggest.nim

@@ -240,7 +240,7 @@ proc suggestField(c: PContext, s: PSym; f: PNode; info: TLineInfo; outputs: var
 
 proc getQuality(s: PSym): range[0..100] =
   if s.typ != nil and s.typ.len > 1:
-    var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyAlias})
+    var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink})
     if exp.kind == tyVarargs: exp = elemType(exp)
     if exp.kind in {tyExpr, tyStmt, tyGenericParam, tyAnything}: return 50
   return 100
@@ -309,7 +309,7 @@ proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} =
     let m = s.getModule()
     if m != nil and sfSystemModule in m.flags:
       if s.kind == skType: return
-      var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyAlias})
+      var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink})
       if exp.kind == tyVarargs: exp = elemType(exp)
       if exp.kind in {tyExpr, tyStmt, tyGenericParam, tyAnything}: return
     result = sigmatch.argtypeMatches(c, s.typ.sons[1], firstArg)
@@ -378,8 +378,8 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
       t = t.sons[0]
     suggestOperations(c, n, field, typ, outputs)
   else:
-    let orig = typ # skipTypes(typ, {tyGenericInst, tyAlias})
+    let orig = typ # skipTypes(typ, {tyGenericInst, tyAlias, tySink})
-    typ = skipTypes(typ, {tyGenericInst, tyVar, tyPtr, tyRef, tyAlias})
+    typ = skipTypes(typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink})
     if typ.kind == tyObject:
       var t = typ
       while true:

compiler/transf.nim

@@ -93,7 +93,7 @@ proc getCurrOwner(c: PTransf): PSym =
 
 proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode =
   let r = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info)
-  r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias})
+  r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias, tySink})
   incl(r.flags, sfFromGeneric)
   let owner = getCurrOwner(c)
   if owner.isIterator and not c.tooEarly:
@@ -331,7 +331,7 @@ proc transformYield(c: PTransf, n: PNode): PTransNode =
   # c.transCon.forStmt.len == 3 means that there is one for loop variable
   # and thus no tuple unpacking:
   if e.typ.isNil: return result # can happen in nimsuggest for unknown reasons
-  if skipTypes(e.typ, {tyGenericInst, tyAlias}).kind == tyTuple and
+  if skipTypes(e.typ, {tyGenericInst, tyAlias, tySink}).kind == tyTuple and
       c.transCon.forStmt.len != 3:
     e = skipConv(e)
     if e.kind == nkPar:
@@ -506,7 +506,7 @@ proc putArgInto(arg: PNode, formal: PType): TPutArgInto =
       if putArgInto(arg.sons[i], formal) != paDirectMapping: return
     result = paDirectMapping
   else:
-    if skipTypes(formal, abstractInst).kind == tyVar: result = paVarAsgn
+    if skipTypes(formal, abstractInst).kind in {tyVar, tyLent}: result = paVarAsgn
     else: result = paFastAsgn
 
 proc findWrongOwners(c: PTransf, n: PNode) =
@@ -791,7 +791,7 @@ proc transform(c: PTransf, n: PNode): PTransNode =
   case n.kind
   of nkSym:
     result = transformSym(c, n)
-  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit:
+  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom:
     # nothing to be done for leaves:
     result = PTransNode(n)
   of nkBracketExpr: result = transformArrayAccess(c, n)
@@ -914,7 +914,7 @@ proc processTransf(c: PTransf, n: PNode, owner: PSym): PNode =
   # Note: For interactive mode we cannot call 'passes.skipCodegen' and skip
   # this step! We have to rely that the semantic pass transforms too errornous
   # nodes into an empty node.
-  if c.fromCache or nfTransf in n.flags: return n
+  if c.rd != nil or nfTransf in n.flags: return n
   pushTransCon(c, newTransCon(owner))
   result = PNode(transform(c, n))
   popTransCon(c)

compiler/trees.nim

@@ -102,7 +102,7 @@ proc isDeepConstExpr*(n: PNode): bool =
       if not isDeepConstExpr(n.sons[i]): return false
     if n.typ.isNil: result = true
     else:
-      let t = n.typ.skipTypes({tyGenericInst, tyDistinct, tyAlias})
+      let t = n.typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink})
       if t.kind in {tyRef, tyPtr}: return false
       if t.kind != tyObject or not isCaseObj(t.n):
         result = true

compiler/types.nim

@@ -51,17 +51,17 @@ const
   # TODO: Remove tyTypeDesc from each abstractX and (where necessary)
   # replace with typedescX
   abstractPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyDistinct, tyOrdinal,
-                   tyTypeDesc, tyAlias, tyInferred}
+                   tyTypeDesc, tyAlias, tyInferred, tySink, tyLent}
   abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc,
-                  tyAlias, tyInferred}
+                  tyAlias, tyInferred, tySink, tyLent}
   abstractRange* = {tyGenericInst, tyRange, tyDistinct, tyOrdinal, tyTypeDesc,
-                    tyAlias, tyInferred}
+                    tyAlias, tyInferred, tySink}
   abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal,
-                       tyTypeDesc, tyAlias, tyInferred}
+                       tyTypeDesc, tyAlias, tyInferred, tySink}
   abstractInst* = {tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias,
-                   tyInferred}
+                   tyInferred, tySink}
   skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyTypeDesc, tyAlias,
-               tyInferred}
+               tyInferred, tySink, tyLent}
   # typedescX is used if we're sure tyTypeDesc should be included (or skipped)
   typedescPtrs* = abstractPtrs + {tyTypeDesc}
   typedescInst* = abstractInst + {tyTypeDesc}
@@ -388,8 +388,8 @@ const
     "int", "int8", "int16", "int32", "int64",
     "float", "float32", "float64", "float128",
     "uint", "uint8", "uint16", "uint32", "uint64",
-    "unused0", "unused1",
+    "opt", "sink",
-    "unused2", "varargs[$1]", "unused", "Error Type",
+    "lent", "varargs[$1]", "unused", "Error Type",
     "BuiltInTypeClass", "UserTypeClass",
     "UserTypeClassInst", "CompositeTypeClass", "inferred",
     "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor",
@@ -539,7 +539,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
         add(result, typeToString(t.sons[i]))
         if i < sonsLen(t) - 1: add(result, ", ")
       add(result, ')')
-  of tyPtr, tyRef, tyVar:
+  of tyPtr, tyRef, tyVar, tyLent:
     result = typeToStr[t.kind]
     if t.len >= 2:
       setLen(result, result.len-1)
@@ -581,6 +581,8 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     if len(prag) != 0: add(result, "{." & prag & ".}")
   of tyVarargs:
     result = typeToStr[t.kind] % typeToString(t.sons[0])
+  of tySink:
+    result = "sink " & typeToString(t.sons[0])
   else:
     result = typeToStr[t.kind]
   result.addTypeFlags(t)
@@ -968,7 +970,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     if result and ExactGenericParams in c.flags:
       result = a.sym.position == b.sym.position
   of tyGenericInvocation, tyGenericBody, tySequence,
-     tyOpenArray, tySet, tyRef, tyPtr, tyVar,
+     tyOpenArray, tySet, tyRef, tyPtr, tyVar, tyLent, tySink,
      tyArray, tyProc, tyVarargs, tyOrdinal, tyTypeClasses, tyOpt:
     cycleCheck()
     if a.kind == tyUserTypeClass and a.n != nil: return a.n == b.n
@@ -992,7 +994,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     cycleCheck()
     result = sameTypeAux(a.lastSon, b.lastSon, c)
   of tyNone: result = false
-  of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("sameFlags")
+  of tyUnused, tyOptAsRef: internalError("sameFlags")
 
 proc sameBackendType*(x, y: PType): bool =
   var c = initSameTypeClosure()
@@ -1101,11 +1103,11 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
   if containsOrIncl(marker, typ.id): return
   var t = skipTypes(typ, abstractInst-{tyTypeDesc})
   case t.kind
-  of tyVar:
+  of tyVar, tyLent:
     if kind in {skProc, skFunc, skConst}: return t
     var t2 = skipTypes(t.sons[0], abstractInst-{tyTypeDesc})
     case t2.kind
-    of tyVar:
+    of tyVar, tyLent:
       if taHeap notin flags: result = t2 # ``var var`` is illegal on the heap
     of tyOpenArray:
       if kind != skParam: result = t
@@ -1143,7 +1145,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
   of tyRange:
     if skipTypes(t.sons[0], abstractInst-{tyTypeDesc}).kind notin
         {tyChar, tyEnum, tyInt..tyFloat128, tyUInt8..tyUInt32}: result = t
-  of tyOpenArray, tyVarargs:
+  of tyOpenArray, tyVarargs, tySink:
     if kind != skParam: result = t
     else: result = typeAllowedAux(marker, t.sons[0], skVar, flags)
   of tySequence, tyOpt:
@@ -1174,7 +1176,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
     # for now same as error node; we say it's a valid type as it should
     # prevent cascading errors:
     result = nil
-  of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("typeAllowedAux")
+  of tyUnused, tyOptAsRef: internalError("typeAllowedAux")
 
 proc typeAllowed*(t: PType, kind: TSymKind): PType =
   # returns 'nil' on success and otherwise the part of the type that is
@@ -1322,7 +1324,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt =
     if typ.callConv == ccClosure: result = 2 * ptrSize
     else: result = ptrSize
     a = ptrSize
-  of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyOpenArray:
+  of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyLent, tyOpenArray:
     let base = typ.lastSon
     if base == typ or (base.kind == tyTuple and base.size==szIllegalRecursion):
       result = szIllegalRecursion

compiler/vmdeps.nim

@@ -209,6 +209,8 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
     else:
       result = mapTypeToBracket("ref", mRef, t, info)
   of tyVar: result = mapTypeToBracket("var", mVar, t, info)
+  of tyLent: result = mapTypeToBracket("lent", mBuiltinType, t, info)
+  of tySink: result = mapTypeToBracket("sink", mBuiltinType, t, info)
   of tySequence: result = mapTypeToBracket("seq", mSeq, t, info)
   of tyOpt: result = mapTypeToBracket("opt", mOpt, t, info)
   of tyProc:
@@ -274,7 +276,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
       result.add atomicType("static", mNone)
       if t.n != nil:
         result.add t.n.copyTree
-  of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("mapTypeToAstX")
+  of tyUnused, tyOptAsRef: internalError("mapTypeToAstX")
 
 proc opMapTypeToAst*(t: PType; info: TLineInfo): PNode =
   result = mapTypeToAstX(t, info, false, true)

compiler/vmgen.nim

@@ -30,7 +30,7 @@
 import
   strutils, ast, astalgo, types, msgs, renderer, vmdef,
   trees, intsets, rodread, magicsys, options, lowerings
-
+import platform
 from os import splitFile
 
 when hasFFI:
@@ -761,6 +761,49 @@ proc genCard(c: PCtx; n: PNode; dest: var TDest) =
   c.gABC(n, opcCard, dest, tmp)
   c.freeTemp(tmp)
 
+proc genIntCast(c: PCtx; n: PNode; dest: var TDest) =
+  const allowedIntegers = {tyInt..tyInt64, tyUInt..tyUInt64, tyChar}
+  var signedIntegers = {tyInt8..tyInt32}
+  var unsignedIntegers = {tyUInt8..tyUInt32, tyChar}
+  let src = n.sons[1].typ.skipTypes(abstractRange)#.kind
+  let dst = n.sons[0].typ.skipTypes(abstractRange)#.kind
+  let src_size = src.getSize
+
+  if platform.intSize < 8:
+    signedIntegers.incl(tyInt)
+    unsignedIntegers.incl(tyUInt)
+  if src_size == dst.getSize and src.kind in allowedIntegers and
+                                 dst.kind in allowedIntegers:
+    let tmp = c.genx(n.sons[1])
+    var tmp2 = c.getTemp(n.sons[1].typ)
+    let tmp3 = c.getTemp(n.sons[1].typ)
+    if dest < 0: dest = c.getTemp(n[0].typ)
+    proc mkIntLit(ival: int): int =
+      result = genLiteral(c, newIntTypeNode(nkIntLit, ival, getSysType(tyInt)))
+    if src.kind in unsignedIntegers and dst.kind in signedIntegers:
+      # cast unsigned to signed integer of same size
+      # signedVal = (unsignedVal xor offset) -% offset
+      let offset = 1 shl (src_size * 8 - 1)
+      c.gABx(n, opcLdConst, tmp2, mkIntLit(offset))
+      c.gABC(n, opcBitxorInt, tmp3, tmp, tmp2)
+      c.gABC(n, opcSubInt, dest, tmp3, tmp2)
+    elif src.kind in signedIntegers and dst.kind in unsignedIntegers:
+      # cast signed to unsigned integer of same size
+      # unsignedVal = (offset +% signedVal +% 1) and offset
+      let offset = (1 shl (src_size * 8))  - 1
+      c.gABx(n, opcLdConst, tmp2, mkIntLit(offset))
+      c.gABx(n, opcLdConst, dest, mkIntLit(offset+1))
+      c.gABC(n, opcAddu, tmp3, tmp, dest)
+      c.gABC(n, opcNarrowU, tmp3, TRegister(src_size*8))
+      c.gABC(n, opcBitandInt, dest, tmp3, tmp2)
+    else:
+      c.gABC(n, opcAsgnInt, dest, tmp)
+    c.freeTemp(tmp)
+    c.freeTemp(tmp2)
+    c.freeTemp(tmp3)
+  else:
+    globalError(n.info, errGenerated, "VM is only allowed to 'cast' between integers of same size")
+
 proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   case m
   of mAnd: c.genAndOr(n, opcFJmp, dest)
@@ -1245,7 +1288,7 @@ proc whichAsgnOpc(n: PNode): TOpcode =
     opcAsgnStr
   of tyFloat..tyFloat128:
     opcAsgnFloat
-  of tyRef, tyNil, tyVar:
+  of tyRef, tyNil, tyVar, tyLent:
     opcAsgnRef
   else:
     opcAsgnComplex
@@ -1438,7 +1481,7 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
       cannotEval(n)
 
 template needsRegLoad(): untyped =
-  gfAddrOf notin flags and fitsRegister(n.typ.skipTypes({tyVar}))
+  gfAddrOf notin flags and fitsRegister(n.typ.skipTypes({tyVar, tyLent}))
 
 proc genArrAccess2(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
                    flags: TGenFlags) =
@@ -1510,7 +1553,7 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
     result = newNodeIT(nkFloatLit, info, t)
   of tyCString, tyString:
     result = newNodeIT(nkStrLit, info, t)
-  of tyVar, tyPointer, tyPtr, tySequence, tyExpr,
+  of tyVar, tyLent, tyPointer, tyPtr, tySequence, tyExpr,
      tyStmt, tyTypeDesc, tyStatic, tyRef, tyNil:
     result = newNodeIT(nkNilLit, info, t)
   of tyProc:
@@ -1844,9 +1887,11 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     if allowCast in c.features:
       genConv(c, n, n.sons[1], dest, opcCast)
     else:
-      globalError(n.info, errGenerated, "VM is not allowed to 'cast'")
+      genIntCast(c, n, dest)
   of nkTypeOfExpr:
     genTypeLit(c, n.typ, dest)
+  of nkComesFrom:
+    discard "XXX to implement for better stack traces"
   else:
     globalError(n.info, errGenerated, "cannot generate VM code for " & $n)
 

compiler/vmmarshal.nim

@@ -102,7 +102,7 @@ proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet) =
       else:
         storeAny(s, t.lastSon, a[i], stored)
     s.add("]")
-  of tyRange, tyGenericInst, tyAlias: storeAny(s, t.lastSon, a, stored)
+  of tyRange, tyGenericInst, tyAlias, tySink: storeAny(s, t.lastSon, a, stored)
   of tyEnum:
     # we need a slow linear search because of enums with holes:
     for e in items(t.n):
@@ -275,7 +275,7 @@ proc loadAny(p: var JsonParser, t: PType,
       next(p)
       return
     raiseParseErr(p, "float expected")
-  of tyRange, tyGenericInst, tyAlias: result = loadAny(p, t.lastSon, tab)
+  of tyRange, tyGenericInst, tyAlias, tySink: result = loadAny(p, t.lastSon, tab)
   else:
     internalError "cannot marshal at compile-time " & t.typeToString
 

compiler/wordrecg.nim

@@ -45,7 +45,7 @@ type
     wImportc, wExportc, wExportNims, wIncompleteStruct, wRequiresInit,
     wAlign, wNodecl, wPure, wSideeffect, wHeader,
     wNosideeffect, wGcSafe, wNoreturn, wMerge, wLib, wDynlib,
-    wCompilerproc, wProcVar, wBase, wUsed,
+    wCompilerproc, wCore, wProcVar, wBase, wUsed,
     wFatal, wError, wWarning, wHint, wLine, wPush, wPop, wDefine, wUndef,
     wLinedir, wStacktrace, wLinetrace, wLink, wCompile,
     wLinksys, wDeprecated, wVarargs, wCallconv, wBreakpoint, wDebugger,
@@ -131,7 +131,7 @@ const
     "incompletestruct",
     "requiresinit", "align", "nodecl", "pure", "sideeffect",
     "header", "nosideeffect", "gcsafe", "noreturn", "merge", "lib", "dynlib",
-    "compilerproc", "procvar", "base", "used",
+    "compilerproc", "core", "procvar", "base", "used",
     "fatal", "error", "warning", "hint", "line",
     "push", "pop", "define", "undef", "linedir", "stacktrace", "linetrace",
     "link", "compile", "linksys", "deprecated", "varargs",

config/nim.cfg

@@ -59,6 +59,9 @@ path="$lib/pure"
   debugger:off
   line_dir:off
   dead_code_elim:on
+  @if nimHasNilChecks:
+    nilchecks:off
+  @end
 @end
 
 @if release:

doc/advopt.txt

@@ -37,6 +37,7 @@ Advanced options:
   --noMain                  do not generate a main procedure
   --genScript               generate a compile script (in the 'nimcache'
                             subdirectory named 'compile_$project$scriptext')
+  --genDeps                 generate a '.deps' file containing the dependencies
   --os:SYMBOL               set the target operating system (cross-compilation)
   --cpu:SYMBOL              set the target processor (cross-compilation)
   --debuginfo               enables debug information

doc/astspec.txt

@@ -918,7 +918,7 @@ This is equivalent to ``var``, but with ``nnkLetSection`` rather than
 Concrete syntax:
 
 .. code-block:: nim
-  let v = 3
+  let a = 3
 
 AST:
 

doc/lib.rst

@@ -92,6 +92,10 @@ Collections and algorithms
 * `sequtils <sequtils.html>`_
   This module implements operations for the built-in seq type
   which were inspired by functional programming languages.
+* `sharedtables <sharedtables.html>`_
+  Nim shared hash table support. Contains shared tables.
+* `sharedlist <sharedlist.html>`_
+  Nim shared linked list support. Contains shared singly linked list.
 
 
 String handling
@@ -102,6 +106,10 @@ String handling
   case of a string, splitting a string into substrings, searching for
   substrings, replacing substrings.
 
+* `strformat <strformat.html>`_
+  Macro based standard string interpolation / formatting. Inpired by
+  Python's ```f``-strings.
+
 * `strmisc <strmisc.html>`_
   This module contains uncommon string handling operations that do not
   fit with the commonly used operations in strutils.
@@ -379,6 +387,7 @@ Cryptography and Hashing
 * `securehash <securehash.html>`_
   This module implements a sha1 encoder and decoder.
 
+
 Multimedia support
 ------------------
 
@@ -431,6 +440,8 @@ Modules for JS backend
 * `jsffi <jsffi.html>`_
   Types and macros for easier interaction with JavaScript.
 
+* `asyncjs <asyncjs.html>`_
+  Types and macros for writing asynchronous procedures in JavaScript.
 
 Deprecated modules
 ------------------

doc/manual/pragmas.txt

@@ -1087,3 +1087,70 @@ In the above example, providing the -d flag causes the symbol
 ``FooBar`` to be overwritten at compile time, printing out 42. If the
 ``-d:FooBar=42`` were to be omitted, the default value of 5 would be
 used.
+
+
+Custom annotations 
+------------------
+It is possible to define custom typed pragmas. Custom pragmas do not effect 
+code generation directly, but their presence can be detected by macros.
+Custom pragmas are defined using templates annotated with pragma ``pragma``:
+
+.. code-block:: nim
+  template dbTable(name: string, table_space: string = nil) {.pragma.}
+  template dbKey(name: string = nil, primary_key: bool = false) {.pragma.}
+  template dbForeignKey(t: typedesc) {.pragma.}
+  template dbIgnore {.pragma.}
+
+
+Consider stylized example of possible Object Relation Mapping (ORM) implementation:
+
+.. code-block:: nim
+  const tblspace {.strdefine.} = "dev" # switch for dev, test and prod environments 
+
+  type
+    User {.dbTable("users", tblspace).} = object
+      id {.dbKey(primary_key = true).}: int
+      name {.dbKey"full_name".}: string
+      is_cached {.dbIgnore.}: bool
+      age: int
+
+    UserProfile {.dbTable("profiles", tblspace).} = object
+      id {.dbKey(primary_key = true).}: int
+      user_id {.dbForeignKey: User.}: int
+      read_access: bool
+      write_access: bool
+      admin_acess: bool
+
+In this example custom pragmas are used to describe how Nim objects are 
+mapped to the schema of the relational database. Custom pragmas can have 
+zero or more arguments. In order to pass multiple arguments use one of 
+template call syntaxes. All arguments are typed and follow standard 
+overload resolution rules for templates. Therefore, it is possible to have 
+default values for arguments, pass by name, varargs, etc. 
+
+Custom pragmas can be used in all locations where ordinary pragmas can be
+specified. It is possible to annotate procs, templates, type and variable 
+definitions, statements, etc.
+
+Macros module includes helpers which can be used to simplify custom pragma 
+access `hasCustomPragma`, `getCustomPragmaVal`. Please consult macros module 
+documentation for details. These macros are no magic, they don't do anything 
+you cannot do yourself by walking AST object representation.
+
+More examples with custom pragmas:
+  - Better serialization/deserialization control:
+
+  .. code-block:: nim
+    type MyObj = object
+      a {.dontSerialize.}: int
+      b {.defaultDeserialize: 5.}: int
+      c {.serializationKey: "_c".}: string
+
+  - Adopting type for gui inspector in a game engine:
+
+  .. code-block:: nim
+    type MyComponent = object
+      position {.editable, animatable.}: Vector3
+      alpha {.editRange: [0.0..1.0], animatable.}: float32
+
+

doc/manual/stmts.txt

@@ -296,6 +296,10 @@ empty ``discard`` statement should be used.
 For non ordinal types it is not possible to list every possible value and so
 these always require an ``else`` part.
 
+As case statements perform compile-time exhaustiveness checks, the value in 
+every ``of`` branch must be known at compile time. This fact is also exploited
+to generate more performant code.
+
 As a special semantic extension, an expression in an ``of`` branch of a case
 statement may evaluate to a set or array constructor; the set or array is then
 expanded into a list of its elements:

doc/manual/types.txt

@@ -41,7 +41,8 @@ These integer types are pre-defined:
 ``int``
   the generic signed integer type; its size is platform dependent and has the
   same size as a pointer. This type should be used in general. An integer
-  literal that has no type suffix is of this type.
+  literal that has no type suffix is of this type if it is in the range
+  ``low(int32)..high(int32)`` otherwise the literal's type is ``int64``.
 
 intXX
   additional signed integer types of XX bits use this naming scheme

doc/tut1.rst

@@ -41,7 +41,7 @@ Save this code to the file "greetings.nim". Now compile and run it::
 
   nim compile --run greetings.nim
 
-With the ``--run`` `switch <nimc.html#command-line-switches>`_ Nim
+With the ``--run`` `switch <nimc.html#compiler-usage-command-line-switches>`_ Nim
 executes the file automatically after compilation. You can give your program
 command line arguments by appending them after the filename::
 
@@ -58,7 +58,7 @@ To compile a release version use::
 By default the Nim compiler generates a large amount of runtime checks
 aiming for your debugging pleasure. With ``-d:release`` these checks are
 `turned off and optimizations are turned on
-<nimc.html#compile-time-symbols>`_.
+<nimc.html#compiler-usage-compile-time-symbols>`_.
 
 Though it should be pretty obvious what the program does, I will explain the
 syntax: statements which are not indented are executed when the program

koch.nim

@@ -97,7 +97,7 @@ proc exec(cmd: string, errorcode: int = QuitFailure, additionalPath = "") =
     if not absolute.isAbsolute:
       absolute = getCurrentDir() / absolute
     echo("Adding to $PATH: ", absolute)
-    putEnv("PATH", prevPath & PathSep & absolute)
+    putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute)
   echo(cmd)
   if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
   putEnv("PATH", prevPath)
@@ -260,7 +260,7 @@ proc buildTools(latest: bool) =
       " nimsuggest/nimsuggest.nim"
 
   let nimgrepExe = "bin/nimgrep".exe
-  nimexec "c -o:" & nimgrepExe & " tools/nimgrep.nim"
+  nimexec "c -d:release -o:" & nimgrepExe & " tools/nimgrep.nim"
   when defined(windows): buildVccTool()
 
   #nimexec "c -o:" & ("bin/nimresolve".exe) & " tools/nimresolve.nim"
@@ -402,7 +402,7 @@ proc winReleaseArch(arch: string) =
 
   template withMingw(path, body) =
     let prevPath = getEnv("PATH")
-    putEnv("PATH", path & PathSep & prevPath)
+    putEnv("PATH", (if path.len > 0: path & PathSep else: "") & prevPath)
     try:
       body
     finally:
@@ -441,7 +441,7 @@ template `|`(a, b): string = (if a.len > 0: a else: b)
 proc tests(args: string) =
   # we compile the tester with taintMode:on to have a basic
   # taint mode test :-)
-  nimexec "cc --taintMode:on tests/testament/tester"
+  nimexec "cc --taintMode:on --opt:speed tests/testament/tester"
   # Since tests take a long time (on my machine), and we want to defy Murhpys
   # law - lets make sure the compiler really is freshly compiled!
   nimexec "c --lib:lib -d:release --opt:speed compiler/nim.nim"

lib/core/allocators.nim

@@ -0,0 +1,35 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2017 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+type
+  Allocator* = ptr object {.inheritable.}
+    alloc*: proc (a: Allocator; size: int; alignment: int = 8): pointer {.nimcall.}
+    dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall.}
+    realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall.}
+
+var
+  currentAllocator {.threadvar.}: Allocator
+
+proc getCurrentAllocator*(): Allocator =
+  result = currentAllocator
+
+proc setCurrentAllocator*(a: Allocator) =
+  currentAllocator = a
+
+proc alloc*(size: int; alignment: int = 8): pointer =
+  let a = getCurrentAllocator()
+  result = a.alloc(a, size, alignment)
+
+proc dealloc*(p: pointer; size: int) =
+  let a = getCurrentAllocator()
+  a.dealloc(a, p, size)
+
+proc realloc*(p: pointer; oldSize, newSize: int): pointer =
+  let a = getCurrentAllocator()
+  result = a.realloc(a, p, oldSize, newSize)

lib/core/macros.nim

@@ -21,7 +21,7 @@ type
     nnkInt16Lit, nnkInt32Lit, nnkInt64Lit, nnkUIntLit, nnkUInt8Lit,
     nnkUInt16Lit, nnkUInt32Lit, nnkUInt64Lit, nnkFloatLit,
     nnkFloat32Lit, nnkFloat64Lit, nnkFloat128Lit, nnkStrLit, nnkRStrLit,
-    nnkTripleStrLit, nnkNilLit, nnkMetaNode, nnkDotCall,
+    nnkTripleStrLit, nnkNilLit, nnkComesFrom, nnkDotCall,
     nnkCommand, nnkCall, nnkCallStrLit, nnkInfix,
     nnkPrefix, nnkPostfix, nnkHiddenCallConv,
     nnkExprEqExpr,
@@ -130,6 +130,7 @@ const
   nnkLiterals* = {nnkCharLit..nnkNilLit}
   nnkCallKinds* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand,
                    nnkCallStrLit}
+  nnkPragmaCallKinds = {nnkExprColonExpr, nnkCall, nnkCallStrLit}
 
 proc `!`*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect, deprecated.}
   ## constructs an identifier from the string `s`
@@ -1213,6 +1214,59 @@ macro expandMacros*(body: typed): untyped =
   result = getAst(inner(body))
   echo result.toStrLit
 
+proc customPragmaNode(n: NimNode): NimNode =
+  expectKind(n, {nnkSym, nnkDotExpr})
+  if n.kind == nnkSym:
+    let sym = n.symbol.getImpl()
+    sym.expectRoutine()
+    result = sym.pragma
+  elif n.kind == nnkDotExpr:
+    let typDef = getImpl(getTypeInst(n[0]).symbol)
+    typDef.expectKind(nnkTypeDef)
+    typDef[2].expectKind(nnkObjectTy)
+    let recList = typDef[2][2]
+    for identDefs in recList:
+      for i in 0 .. identDefs.len - 3:
+        if identDefs[i].kind == nnkPragmaExpr and 
+           identDefs[i][0].kind == nnkIdent and $identDefs[i][0] == $n[1]:
+          return identDefs[i][1]
+
+macro hasCustomPragma*(n: typed, cp: typed{nkSym}): untyped =
+  ## Expands to `true` if expression `n` which is expected to be `nnkDotExpr`
+  ## has custom pragma `cp`.
+  ##
+  ## .. code-block:: nim
+  ##   template myAttr() {.pragma.}
+  ##   type 
+  ##     MyObj = object
+  ##       myField {.myAttr.}: int
+  ##   var o: MyObj
+  ##   assert(o.myField.hasCustomPragma(myAttr) == 0)
+  let pragmaNode = customPragmaNode(n)
+  for p in pragmaNode:
+    if (p.kind == nnkSym and p == cp) or
+        (p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp):
+      return newLit(true)
+  return newLit(false)
+
+macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped =
+  ## Expands to value of custom pragma `cp` of expression `n` which is expected
+  ## to be `nnkDotExpr`.
+  ##
+  ## .. code-block:: nim
+  ##   template serializationKey(key: string) {.pragma.}
+  ##   type 
+  ##     MyObj = object
+  ##       myField {.serializationKey: "mf".}: int
+  ##   var o: MyObj
+  ##   assert(o.myField.getCustomPragmaVal(serializationKey) == "mf")
+  let pragmaNode = customPragmaNode(n)
+  for p in pragmaNode:
+    if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp:
+      return p[1]
+  return newEmptyNode()
+
+
 when not defined(booting):
   template emit*(e: static[string]): untyped {.deprecated.} =
     ## accepts a single string argument and treats it as nim code

lib/core/refs.nim

@@ -0,0 +1,97 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2017 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Default ref implementation used by Nim's core.
+
+# We cannot use the allocator interface here as we require a heap walker to
+# exist. Thus we import 'alloc' directly here to get our own heap that is
+# all under the GC's control and can use the ``allObjects`` iterator which
+# is crucial for the "sweep" phase.
+import typelayouts, alloc
+
+type
+  TracingGc = ptr object of Allocator
+    visit*: proc (fieldAddr: ptr pointer; a: Allocator) {.nimcall.}
+
+  GcColor = enum
+    white = 0, black = 1, grey = 2 ## to flip the meaning of white/black
+                                   ## perform (1 - col)
+
+  GcHeader = object
+    t: ptr TypeLayout
+    color: GcColor
+  Cell = ptr GcHeader
+
+  GcFrame {.core.} = object
+    prev: ptr GcFrame
+    marker: proc (self: GcFrame; a: Allocator)
+
+  Phase = enum
+    None, Marking, Sweeping
+
+  GcHeap = object
+    r: MemRegion
+    phase: Phase
+    currBlack, currWhite: GcColor
+    greyStack: seq[Cell]
+
+var
+  gch {.threadvar.}: GcHeap
+
+proc `=trace`[T](a: ref T) =
+  if not marked(a):
+    mark(a)
+    `=trace`(a[])
+
+template usrToCell(p: pointer): Cell =
+
+template cellToUsr(cell: Cell): pointer =
+  cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(GcHeader)))
+
+template usrToCell(usr: pointer): Cell =
+  cast[Cell](cast[ByteAddress](usr)-%ByteAddress(sizeof(GcHeader)))
+
+template markGrey(x: Cell) =
+  if x.color == gch.currWhite and phase == Marking:
+    x.color = grey
+    add(gch.greyStack, x)
+
+proc `=`[T](dest: var ref T; src: ref T) =
+  ## full write barrier implementation.
+  if src != nil:
+    let s = usrToCell(src)
+    markGrey(s)
+  system.`=`(dest, src)
+
+proc linkGcFrame(f: ptr GcFrame) {.core.}
+proc unlinkGcFrame() {.core.}
+
+proc setGcFrame(f: ptr GcFrame) {.core.}
+
+proc registerGlobal(p: pointer; t: ptr TypeLayout) {.core.}
+proc unregisterGlobal(p: pointer; t: ptr TypeLayout) {.core.}
+
+proc registerThreadvar(p: pointer; t: ptr TypeLayout) {.core.}
+proc unregisterThreadvar(p: pointer; t: ptr TypeLayout) {.core.}
+
+proc newImpl(t: ptr TypeLayout): pointer =
+  let r = cast[Cell](rawAlloc(t.size + sizeof(GcHeader)))
+  r.typ = t
+  result = r +! sizeof(GcHeader)
+
+template new*[T](x: var ref T) =
+  x = newImpl(getTypeLayout(x))
+
+
+when false:
+  # implement these if your GC requires them:
+  proc writeBarrierLocal() {.core.}
+  proc writeBarrierGlobal() {.core.}
+
+  proc writeBarrierGeneric() {.core.}

lib/core/seqs.nim

@@ -0,0 +1,139 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2017 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import allocators, typetraits
+
+## Default seq implementation used by Nim's core.
+type
+  seq*[T] = object
+    len, cap: int
+    data: ptr UncheckedArray[T]
+
+template frees(s) = dealloc(s.data, s.cap * sizeof(T))
+
+# XXX make code memory safe for overflows in '*'
+proc nimSeqLiteral[T](x: openArray[T]): seq[T] {.core.} =
+  seq[T](len: x.len, cap: x.len, data: x)
+
+when defined(nimHasTrace):
+  proc `=trace`[T](s: seq[T]; a: Allocator) =
+    for i in 0 ..< s.len: `=trace`(s.data[i], a)
+
+proc `=destroy`[T](x: var seq[T]) =
+  if x.data != nil:
+    when not supportsCopyMem(T):
+      for i in 0..<x.len: `=destroy`(x[i])
+    frees(x)
+    x.data = nil
+    x.len = 0
+    x.cap = 0
+
+proc `=`[T](a: var seq[T]; b: seq[T]) =
+  if a.data == b.data: return
+  if a.data != nil:
+    frees(a)
+    a.data = nil
+  a.len = b.len
+  a.cap = b.cap
+  if b.data != nil:
+    a.data = cast[type(a.data)](alloc(a.cap * sizeof(T)))
+    when supportsCopyMem(T):
+      copyMem(a.data, b.data, a.cap * sizeof(T))
+    else:
+      for i in 0..<a.len:
+        a.data[i] = b.data[i]
+
+proc `=sink`[T](a: var seq[T]; b: seq[T]) =
+  if a.data != nil and a.data != b.data:
+    frees(a)
+  a.len = b.len
+  a.cap = b.cap
+  a.data = b.data
+
+proc resize[T](s: var seq[T]) =
+  let old = s.cap
+  if old == 0: s.cap = 8
+  else: s.cap = (s.cap * 3) shr 1
+  s.data = cast[type(s.data)](realloc(s.data, old * sizeof(T), s.cap * sizeof(T)))
+
+proc reserveSlot[T](x: var seq[T]): ptr T =
+  if x.len >= x.cap: resize(x)
+  result = addr(x.data[x.len])
+  inc x.len
+
+template add*[T](x: var seq[T]; y: T) =
+  reserveSlot(x)[] = y
+
+proc shrink*[T](x: var seq[T]; newLen: int) =
+  assert newLen <= x.len
+  assert newLen >= 0
+  when not supportsCopyMem(T):
+    for i in countdown(x.len - 1, newLen - 1):
+      `=destroy`(x.data[i])
+  x.len = newLen
+
+proc grow*[T](x: var seq[T]; newLen: int; value: T) =
+  if newLen <= x.len: return
+  assert newLen >= 0
+  if x.cap == 0: x.cap = newLen
+  else: x.cap = max(newLen, (x.cap * 3) shr 1)
+  x.data = cast[type(x.data)](realloc(x.data, x.cap * sizeof(T)))
+  for i in x.len..<newLen:
+    x.data[i] = value
+  x.len = newLen
+
+template default[T](t: typedesc[T]): T =
+  var v: T
+  v
+
+proc setLen*[T](x: var seq[T]; newLen: int) {.deprecated.} =
+  if newlen < x.len: shrink(x, newLen)
+  else: grow(x, newLen, default(T))
+
+template `[]`*[T](x: seq[T]; i: Natural): T =
+  assert i < x.len
+  x.data[i]
+
+template `[]=`*[T](x: seq[T]; i: Natural; y: T) =
+  assert i < x.len
+  x.data[i] = y
+
+proc `@`*[T](elems: openArray[T]): seq[T] =
+  result.cap = elems.len
+  result.len = elems.len
+  result.data = cast[type(result.data)](alloc(result.cap * sizeof(T)))
+  when supportsCopyMem(T):
+    copyMem(result.data, unsafeAddr(elems[0]), result.cap * sizeof(T))
+  else:
+    for i in 0..<result.len:
+      result.data[i] = elems[i]
+
+proc len*[T](x: seq[T]): int {.inline.} = x.len
+
+proc `$`*[T](x: seq[T]): string =
+  result = "@["
+  var firstElement = true
+  for i in 0..<x.len:
+    let 
+      value = x.data[i]
+    if firstElement:
+      firstElement = false
+    else:
+      result.add(", ")
+
+    when compiles(value.isNil):
+      # this branch should not be necessary
+      if value.isNil:
+        result.add "nil"
+      else:
+        result.addQuoted(value)
+    else:
+      result.addQuoted(value)
+
+  result.add("]")

lib/core/strs.nim

@@ -0,0 +1,111 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2017 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Default string implementation used by Nim's core.
+
+import allocators
+
+type
+  string {.core.} = object
+    len, cap: int
+    data: ptr UncheckedArray[char]
+
+proc nimStringLiteral(x: cstring; len: int): string {.core.} =
+  string(len: len, cap: len, data: x)
+
+template frees(s) = dealloc(s.data, s.cap + 1)
+
+proc `=destroy`(s: var string) =
+  if s.data != nil:
+    frees(s)
+    s.data = nil
+    s.len = 0
+    s.cap = 0
+
+proc `=sink`(a: var string, b: string) =
+  # we hope this is optimized away for not yet alive objects:
+  if a.data != nil and a.data != b.data:
+    frees(a)
+  a.len = b.len
+  a.cap = b.cap
+  a.data = b.data
+
+proc `=`(a: var string; b: string) =
+  if a.data != nil and a.data != b.data:
+    frees(a)
+    a.data = nil
+  a.len = b.len
+  a.cap = b.cap
+  if b.data != nil:
+    a.data = cast[type(a.data)](alloc(a.cap + 1))
+    copyMem(a.data, b.data, a.cap+1)
+
+proc resize(s: var string) =
+  let old = s.cap
+  if old == 0: s.cap = 8
+  else: s.cap = (s.cap * 3) shr 1
+  s.data = cast[type(s.data)](realloc(s.data, old + 1, s.cap + 1))
+
+proc add*(s: var string; c: char) =
+  if s.len >= s.cap: resize(s)
+  s.data[s.len] = c
+  s.data[s.len+1] = '\0'
+  inc s.len
+
+proc ensure(s: var string; newLen: int) =
+  let old = s.cap
+  if newLen >= old:
+    s.cap = max((old * 3) shr 1, newLen)
+    if s.cap > 0:
+      s.data = cast[type(s.data)](realloc(s.data, old + 1, s.cap + 1))
+
+proc add*(s: var string; y: string) =
+  if y.len != 0:
+    let newLen = s.len + y.len
+    ensure(s, newLen)
+    copyMem(addr s.data[len], y.data, y.data.len + 1)
+    s.len = newLen
+
+proc len*(s: string): int {.inline.} = s.len
+
+proc newString*(len: int): string =
+  result.len = len
+  result.cap = len
+  if len > 0:
+    result.data = alloc0(len+1)
+
+converter toCString(x: string): cstring {.core.} =
+  if x.len == 0: cstring"" else: cast[cstring](x.data)
+
+proc newStringOfCap*(cap: int): string =
+  result.len = 0
+  result.cap = cap
+  if cap > 0:
+    result.data = alloc(cap+1)
+
+proc `&`*(a, b: string): string =
+  let sum = a.len + b.len
+  result = newStringOfCap(sum)
+  result.len = sum
+  copyMem(addr result.data[0], a.data, a.len)
+  copyMem(addr result.data[a.len], b.data, b.len)
+  if sum > 0:
+    result.data[sum] = '\0'
+
+proc concat(x: openArray[string]): string {.core.} =
+  ## used be the code generator to optimize 'x & y & z ...'
+  var sum = 0
+  for i in 0 ..< x.len: inc(sum, x[i].len)
+  result = newStringOfCap(sum)
+  sum = 0
+  for i in 0 ..< x.len:
+    let L = x[i].len
+    copyMem(addr result.data[sum], x[i].data, L)
+    inc(sum, L)
+

lib/core/typelayouts.nim

@@ -0,0 +1,19 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2017 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+type
+  TypeLayout* = object
+    size*, alignment*: int
+    destructor*: proc (self: pointer; a: Allocator) {.nimcall.}
+    trace*: proc (self: pointer; a: Allocator) {.nimcall.}
+    when false:
+      construct*: proc (self: pointer; a: Allocator) {.nimcall.}
+      copy*, deepcopy*, sink*: proc (self, other: pointer; a: Allocator) {.nimcall.}
+
+proc getTypeLayout(t: typedesc): ptr TypeLayout {.magic: "getTypeLayout".}

lib/js/asyncjs.nim

@@ -0,0 +1,141 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2017 Nim Authors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+
+## This module implements types and macros for writing asynchronous code
+## for the JS backend. It provides tools for interaction with JavaScript async API-s
+## and libraries, writing async procedures in Nim and converting callback-based code
+## to promises.
+##
+## A Nim procedure is asynchronous when it includes the ``{.async.}`` pragma. It
+## should always have a ``Future[T]`` return type or not have a return type at all.
+## A ``Future[void]`` return type is assumed by default.
+##
+## This is roughly equivalent to the ``async`` keyword in JavaScript code.
+##
+## .. code-block:: nim
+##  proc loadGame(name: string): Future[Game] {.async.} =
+##    # code
+##
+## should be equivalent to
+##
+## .. code-block:: javascript
+##   async function loadGame(name) {
+##     // code
+##   }
+##
+## A call to an asynchronous procedure usually needs ``await`` to wait for
+## the completion of the ``Future``.
+##
+## .. code-block:: nim
+##   var game = await loadGame(name)
+##
+## Often, you might work with callback-based API-s. You can wrap them with
+## asynchronous procedures using promises and ``newPromise``:
+##
+## .. code-block:: nim
+##   proc loadGame(name: string): Future[Game] =
+##     var promise = newPromise() do (resolve: proc(response: Game)):
+##       cbBasedLoadGame(name) do (game: Game):
+##         resolve(game)
+##     return promise
+##
+## Forward definitions work properly, you just need to always add the ``{.async.}`` pragma:
+##
+## .. code-block:: nim
+##   proc loadGame(name: string): Future[Game] {.async.}
+##
+## JavaScript compatibility
+## ~~~~~~~~~~~~~~~~~~~~~~~~~
+##
+## Nim currently generates `async/await` JavaScript code which is supported in modern
+## EcmaScript and most modern versions of browsers, Node.js and Electron.
+## If you need to use this module with older versions of JavaScript, you can
+## use a tool that backports the resulting JavaScript code, as babel.
+
+import jsffi
+import macros
+
+when not defined(js) and not defined(nimdoc) and not defined(nimsuggest):
+  {.fatal: "Module asyncjs is designed to be used with the JavaScript backend.".}
+
+type
+  Future*[T] = ref object
+    future*: T
+  ## Wraps the return type of an asynchronous procedure.
+
+  PromiseJs* {.importcpp: "Promise".} = ref object
+  ## A JavaScript Promise
+
+proc replaceReturn(node: var NimNode) =
+  var z = 0
+  for s in node:
+    var son = node[z]
+    if son.kind == nnkReturnStmt:
+      node[z] = nnkReturnStmt.newTree(nnkCall.newTree(ident("jsResolve"), son[0]))
+    elif son.kind == nnkAsgn and son[0].kind == nnkIdent and $son[0] == "result":
+      node[z] = nnkAsgn.newTree(son[0], nnkCall.newTree(ident("jsResolve"), son[1]))
+    else:
+      replaceReturn(son)
+    inc z
+
+proc isFutureVoid(node: NimNode): bool =
+  result = node.kind == nnkBracketExpr and
+           node[0].kind == nnkIdent and $node[0] == "Future" and
+           node[1].kind == nnkIdent and $node[1] == "void"
+
+proc generateJsasync(arg: NimNode): NimNode =
+  assert arg.kind == nnkProcDef
+  result = arg
+  var isVoid = false
+  var jsResolveNode = ident("jsResolve")
+
+  if arg.params[0].kind == nnkEmpty:
+    result.params[0] = nnkBracketExpr.newTree(ident("Future"), ident("void"))
+    isVoid = true
+  elif isFutureVoid(arg.params[0]):
+    isVoid = true
+
+  var code = result.body
+  replaceReturn(code)
+  result.body = nnkStmtList.newTree()
+
+  if len(code) > 0:
+    var awaitFunction = quote:
+      proc await[T](f: Future[T]): T {.importcpp: "(await #)".}
+    result.body.add(awaitFunction)
+
+    var resolve: NimNode
+    if isVoid:
+      resolve = quote:
+        var `jsResolveNode` {.importcpp: "undefined".}: Future[void]
+    else:
+      resolve = quote:
+        proc jsResolve[T](a: T): Future[T] {.importcpp: "#".}
+    result.body.add(resolve)
+  else:
+    result.body = newEmptyNode()
+  for child in code:
+    result.body.add(child)
+
+  if len(code) > 0 and isVoid:
+    var voidFix = quote:
+      return `jsResolveNode`
+    result.body.add(voidFix)
+
+  result.pragma = quote:
+    {.codegenDecl: "async function $2($3)".}
+
+
+macro async*(arg: untyped): untyped =
+  ## Macro which converts normal procedures into
+  ## javascript-compatible async procedures
+  generateJsasync(arg)
+
+proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importcpp: "(new Promise(#))".}
+  ## A helper for wrapping callback-based functions
+  ## into promises and async procedures

lib/js/dom.nim

@@ -134,9 +134,9 @@ type
 
   # https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement
   HtmlElement* = ref object of Element
-    contentEditable*: string
+    contentEditable*: cstring
     isContentEditable*: bool
-    dir*: string
+    dir*: cstring
     offsetHeight*: int
     offsetWidth*: int
     offsetLeft*: int
@@ -405,7 +405,7 @@ type
 # EventTarget "methods"
 proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false)
 proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), options: AddEventListenerOptions)
-
+proc removeEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false)
 
 # Window "methods"
 proc alert*(w: Window, msg: cstring)
@@ -507,6 +507,7 @@ proc replace*(loc: Location, s: cstring)
 proc back*(h: History)
 proc forward*(h: History)
 proc go*(h: History, pagesToJump: int)
+proc pushState*[T](h: History, stateObject: T, title, url: cstring)
 
 # Navigator "methods"
 proc javaEnabled*(h: Navigator): bool

lib/nimbase.h

@@ -70,7 +70,7 @@ __clang__
 #if defined(_MSC_VER)
 #  pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309)
 #  pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706)
-#  pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090 4297)
+#  pragma warning(disable: 4710 4711 4774 4800 4809 4820 4996 4090 4297)
 #endif
 /* ------------------------------------------------------------------------- */
 
@@ -159,6 +159,7 @@ __clang__
 /* ------------------------------------------------------------------- */
 
 #if defined(WIN32) || defined(_WIN32) /* only Windows has this mess... */
+#  define N_LIB_PRIVATE
 #  define N_CDECL(rettype, name) rettype __cdecl name
 #  define N_STDCALL(rettype, name) rettype __stdcall name
 #  define N_SYSCALL(rettype, name) rettype __syscall name
@@ -178,6 +179,7 @@ __clang__
 #  endif
 #  define N_LIB_IMPORT  extern __declspec(dllimport)
 #else
+#  define N_LIB_PRIVATE __attribute__((visibility("hidden")))
 #  if defined(__GNUC__)
 #    define N_CDECL(rettype, name) rettype name
 #    define N_STDCALL(rettype, name) rettype name
@@ -398,11 +400,11 @@ typedef struct TStringDesc* string;
 
 // NAN definition copied from math.h included in the Windows SDK version 10.0.14393.0
 #ifndef NAN
-#ifndef _HUGE_ENUF
+#  ifndef _HUGE_ENUF
-#define _HUGE_ENUF  1e+300  // _HUGE_ENUF*_HUGE_ENUF must overflow
+#    define _HUGE_ENUF  1e+300  // _HUGE_ENUF*_HUGE_ENUF must overflow
-#endif
+#  endif
-#define NAN_INFINITY ((float)(_HUGE_ENUF * _HUGE_ENUF))
+#  define NAN_INFINITY ((float)(_HUGE_ENUF * _HUGE_ENUF))
-#define NAN ((float)(NAN_INFINITY * 0.0F))
+#  define NAN ((float)(NAN_INFINITY * 0.0F))
 #endif
 
 #ifndef INF
@@ -480,7 +482,6 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); }
    On disagreement, your C compiler will say something like:
    "error: 'Nim_and_C_compiler_disagree_on_target_architecture' declared as an array with a negative size" */
 typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1];
-#endif
 
 #ifdef  __cplusplus
 #  define NIM_EXTERNC extern "C"
@@ -507,3 +508,5 @@ extern Libc::Env *genodeEnv;
 /* Compile with -d:checkAbi and a sufficiently C11:ish compiler to enable */
 #define NIM_CHECK_SIZE(typ, sz) \
   _Static_assert(sizeof(typ) == sz, "Nim & C disagree on type size")
+
+#endif /* NIMBASE_H */

lib/posix/posix.nim

@@ -609,11 +609,12 @@ proc clock_nanosleep*(a1: ClockId, a2: cint, a3: var Timespec,
 proc clock_settime*(a1: ClockId, a2: var Timespec): cint {.
   importc, header: "<time.h>".}
 
+proc `==`*(a, b: Time): bool {.borrow.}
+proc `-`*(a, b: Time): Time {.borrow.}
 proc ctime*(a1: var Time): cstring {.importc, header: "<time.h>".}
 proc ctime_r*(a1: var Time, a2: cstring): cstring {.importc, header: "<time.h>".}
 proc difftime*(a1, a2: Time): cdouble {.importc, header: "<time.h>".}
 proc getdate*(a1: cstring): ptr Tm {.importc, header: "<time.h>".}
-
 proc gmtime*(a1: var Time): ptr Tm {.importc, header: "<time.h>".}
 proc gmtime_r*(a1: var Time, a2: var Tm): ptr Tm {.importc, header: "<time.h>".}
 proc localtime*(a1: var Time): ptr Tm {.importc, header: "<time.h>".}

lib/posix/posix_linux_amd64.nim

@@ -12,8 +12,6 @@
 
 # To be included from posix.nim!
 
-from times import Time
-
 const
   hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays
   hasAioH = defined(linux)
@@ -40,13 +38,15 @@ type
 const SIG_HOLD* = cast[SigHandler](2)
 
 type
+  Time* {.importc: "time_t", header: "<time.h>".} = distinct clong
+
   Timespec* {.importc: "struct timespec",
                header: "<time.h>", final, pure.} = object ## struct timespec
     tv_sec*: Time  ## Seconds.
     tv_nsec*: clong  ## Nanoseconds.
 
   Dirent* {.importc: "struct dirent",
-             header: "<dirent.h>", final, pure.} = object ## dirent_t struct
+            header: "<dirent.h>", final, pure.} = object ## dirent_t struct
     d_ino*: Ino
     d_off*: Off
     d_reclen*: cushort

lib/posix/posix_other.nim

@@ -9,8 +9,6 @@
 
 {.deadCodeElim:on.}
 
-from times import Time
-
 const
   hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays
   hasAioH = defined(linux)
@@ -36,6 +34,8 @@ type
 {.deprecated: [TSocketHandle: SocketHandle].}
 
 type
+  Time* {.importc: "time_t", header: "<time.h>".} = distinct clong
+
   Timespec* {.importc: "struct timespec",
                header: "<time.h>", final, pure.} = object ## struct timespec
     tv_sec*: Time  ## Seconds.
@@ -209,24 +209,24 @@ type
     st_gid*: Gid          ## Group ID of file.
     st_rdev*: Dev         ## Device ID (if file is character or block special).
     st_size*: Off         ## For regular files, the file size in bytes.
-                           ## For symbolic links, the length in bytes of the
+                          ## For symbolic links, the length in bytes of the
-                           ## pathname contained in the symbolic link.
+                          ## pathname contained in the symbolic link.
-                           ## For a shared memory object, the length in bytes.
+                          ## For a shared memory object, the length in bytes.
-                           ## For a typed memory object, the length in bytes.
+                          ## For a typed memory object, the length in bytes.
-                           ## For other file types, the use of this field is
+                          ## For other file types, the use of this field is
-                           ## unspecified.
+                          ## unspecified.
     when defined(macosx) or defined(android):
-      st_atime*: Time      ## Time of last access.
+      st_atime*: Time     ## Time of last access.
-      st_mtime*: Time      ## Time of last data modification.
+      st_mtime*: Time     ## Time of last data modification.
-      st_ctime*: Time      ## Time of last status change.
+      st_ctime*: Time     ## Time of last status change.
     else:
-      st_atim*: Timespec   ## Time of last access.
+      st_atim*: Timespec  ## Time of last access.
-      st_mtim*: Timespec   ## Time of last data modification.
+      st_mtim*: Timespec  ## Time of last data modification.
-      st_ctim*: Timespec   ## Time of last status change.
+      st_ctim*: Timespec  ## Time of last status change.
-    st_blksize*: Blksize   ## A file system-specific preferred I/O block size
+    st_blksize*: Blksize  ## A file system-specific preferred I/O block size
-                           ## for this object. In some file system types, this
+                          ## for this object. In some file system types, this
-                           ## may vary from file to file.
+                          ## may vary from file to file.
-    st_blocks*: Blkcnt     ## Number of blocks allocated for this object.
+    st_blocks*: Blkcnt    ## Number of blocks allocated for this object.
 
 
   Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>",

lib/pure/asyncdispatch.nim

@@ -168,18 +168,20 @@ type
     timers*: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
     callbacks*: Deque[proc ()]
 
-proc processTimers(p: PDispatcherBase) {.inline.} =
+proc processTimers(p: PDispatcherBase; didSomeWork: var bool) {.inline.} =
   #Process just part if timers at a step
   var count = p.timers.len
   let t = epochTime()
   while count > 0 and t >= p.timers[0].finishAt:
     p.timers.pop().fut.complete()
     dec count
+    didSomeWork = true
 
-proc processPendingCallbacks(p: PDispatcherBase) =
+proc processPendingCallbacks(p: PDispatcherBase; didSomeWork: var bool) =
   while p.callbacks.len > 0:
     var cb = p.callbacks.popFirst()
     cb()
+    didSomeWork = true
 
 proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
   # If dispatcher has active timers this proc returns the timeout
@@ -298,14 +300,13 @@ when defined(windows) or defined(nimdoc):
     let p = getGlobalDispatcher()
     p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0
 
-  proc poll*(timeout = 500) =
+  proc runOnce(timeout = 500): bool =
-    ## Waits for completion events and processes them. Raises ``ValueError``
-    ## if there are no pending operations.
     let p = getGlobalDispatcher()
     if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0:
       raise newException(ValueError,
         "No handles or timers registered in dispatcher.")
 
+    result = false
     if p.handles.len != 0:
       let at = p.adjustedTimeout(timeout)
       var llTimeout =
@@ -318,6 +319,7 @@ when defined(windows) or defined(nimdoc):
       let res = getQueuedCompletionStatus(p.ioPort,
           addr lpNumberOfBytesTransferred, addr lpCompletionKey,
           cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool
+      result = true
 
       # http://stackoverflow.com/a/12277264/492186
       # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html
@@ -347,13 +349,14 @@ when defined(windows) or defined(nimdoc):
         else:
           if errCode.int32 == WAIT_TIMEOUT:
             # Timed out
-            discard
+            result = false
           else: raiseOSError(errCode)
 
     # Timer processing.
-    processTimers(p)
+    processTimers(p, result)
     # Callback queue processing
-    processPendingCallbacks(p)
+    processPendingCallbacks(p, result)
+
 
   var acceptEx: WSAPROC_ACCEPTEX
   var connectEx: WSAPROC_CONNECTEX
@@ -1229,7 +1232,7 @@ else:
       # descriptor was unregistered in callback via `unregister()`.
       discard
 
-  proc poll*(timeout = 500) =
+  proc runOnce(timeout = 500): bool =
     let p = getGlobalDispatcher()
     when ioselSupportedPlatform:
       let customSet = {Event.Timer, Event.Signal, Event.Process,
@@ -1239,6 +1242,7 @@ else:
       raise newException(ValueError,
         "No handles or timers registered in dispatcher.")
 
+    result = false
     if not p.selector.isEmpty():
       var keys: array[64, ReadyKey]
       var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys)
@@ -1251,20 +1255,24 @@ else:
 
         if Event.Read in events or events == {Event.Error}:
           processBasicCallbacks(fd, readList)
+          result = true
 
         if Event.Write in events or events == {Event.Error}:
           processBasicCallbacks(fd, writeList)
+          result = true
 
-        if Event.User in events or events == {Event.Error}:
+        if Event.User in events:
           processBasicCallbacks(fd, readList)
           custom = true
           if rLength == 0:
             p.selector.unregister(fd)
+          result = true
 
         when ioselSupportedPlatform:
           if (customSet * events) != {}:
             custom = true
             processCustomCallbacks(fd)
+            result = true
 
         # because state `data` can be modified in callback we need to update
         # descriptor events with currently registered callbacks.
@@ -1276,9 +1284,9 @@ else:
             p.selector.updateHandle(SocketHandle(fd), newEvents)
 
     # Timer processing.
-    processTimers(p)
+    processTimers(p, result)
     # Callback queue processing
-    processPendingCallbacks(p)
+    processPendingCallbacks(p, result)
 
   proc recv*(socket: AsyncFD, size: int,
              flags = {SocketFlag.SafeDisconn}): Future[string] =
@@ -1501,6 +1509,19 @@ else:
     data.readList.add(cb)
     p.selector.registerEvent(SelectEvent(ev), data)
 
+proc drain*(timeout = 500) =
+  ## Waits for completion events and processes them. Raises ``ValueError``
+  ## if there are no pending operations. In contrast to ``poll`` this
+  ## processes as many events as are available.
+  if runOnce(timeout):
+    while hasPendingOperations() and runOnce(0): discard
+
+proc poll*(timeout = 500) =
+  ## Waits for completion events and processes them. Raises ``ValueError``
+  ## if there are no pending operations. This runs the underlying OS
+  ## `epoll`:idx: or `kqueue`:idx: primitive only once.
+  discard runOnce(timeout)
+
 # Common procedures between current and upcoming asyncdispatch
 include includes.asynccommon
 

lib/pure/asyncfutures.nim

@@ -1,4 +1,4 @@
-import os, tables, strutils, times, heapqueue, options, deques
+import os, tables, strutils, times, heapqueue, options, deques, cstrutils
 
 # TODO: This shouldn't need to be included, but should ideally be exported.
 type
@@ -217,17 +217,78 @@ proc `callback=`*[T](future: Future[T],
   ## If future has already completed then ``cb`` will be called immediately.
   future.callback = proc () = cb(future)
 
-proc injectStacktrace[T](future: Future[T]) =
+proc getHint(entry: StackTraceEntry): string =
-  # TODO: Come up with something better.
+  ## We try to provide some hints about stack trace entries that the user
-  when not defined(release):
+  ## may not be familiar with, in particular calls inside the stdlib.
-    var msg = ""
+  result = ""
-    msg.add("\n  " & future.fromProc & "'s lead up to read of failed Future:")
+  if entry.procname == "processPendingCallbacks":
+    if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
+      return "Executes pending callbacks"
+  elif entry.procname == "poll":
+    if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
+      return "Processes asynchronous completion events"
 
-    if not future.errorStackTrace.isNil and future.errorStackTrace != "":
+  if entry.procname.endsWith("_continue"):
-      msg.add("\n" & indent(future.errorStackTrace.strip(), 4))
+    if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0:
-    else:
+      return "Resumes an async procedure"
-      msg.add("\n    Empty or nil stack trace.")
+
-    future.error.msg.add(msg)
+proc `$`*(entries: seq[StackTraceEntry]): string =
+  result = ""
+  # Find longest filename & line number combo for alignment purposes.
+  var longestLeft = 0
+  for entry in entries:
+    if entry.procName.isNil: continue
+
+    let left = $entry.filename & $entry.line
+    if left.len > longestLeft:
+      longestLeft = left.len
+
+  var indent = 2
+  # Format the entries.
+  for entry in entries:
+    if entry.procName.isNil:
+      if entry.line == -10:
+        result.add(spaces(indent) & "#[\n")
+        indent.inc(2)
+      else:
+        indent.dec(2)
+        result.add(spaces(indent)& "]#\n")
+      continue
+
+    let left = "$#($#)" % [$entry.filename, $entry.line]
+    result.add((spaces(indent) & "$#$# $#\n") % [
+      left,
+      spaces(longestLeft - left.len + 2),
+      $entry.procName
+    ])
+    let hint = getHint(entry)
+    if hint.len > 0:
+      result.add(spaces(indent+2) & "## " & hint & "\n")
+
+proc injectStacktrace[T](future: Future[T]) =
+  when not defined(release):
+    const header = "\nAsync traceback:\n"
+
+    var exceptionMsg = future.error.msg
+    if header in exceptionMsg:
+      # This is messy: extract the original exception message from the msg
+      # containing the async traceback.
+      let start = exceptionMsg.find(header)
+      exceptionMsg = exceptionMsg[0..<start]
+
+
+    var newMsg = exceptionMsg & header
+
+    let entries = getStackTraceEntries(future.error)
+    newMsg.add($entries)
+
+    newMsg.add("Exception message: " & exceptionMsg & "\n")
+    newMsg.add("Exception type:")
+
+    # # For debugging purposes
+    # for entry in getStackTraceEntries(future.error):
+    #   newMsg.add "\n" & $entry
+    future.error.msg = newMsg
 
 proc read*[T](future: Future[T] | FutureVar[T]): T =
   ## Retrieves the value of ``future``. Future must be finished otherwise
@@ -263,12 +324,12 @@ proc mget*[T](future: FutureVar[T]): var T =
   ## Future has not been finished.
   result = Future[T](future).value
 
-proc finished*[T](future: Future[T] | FutureVar[T]): bool =
+proc finished*(future: FutureBase | FutureVar): bool =
   ## Determines whether ``future`` has completed.
   ##
   ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish.
-  when future is FutureVar[T]:
+  when future is FutureVar:
-    result = (Future[T](future)).finished
+    result = (FutureBase(future)).finished
   else:
     result = future.finished
 

lib/pure/asyncmacro.nim

@@ -25,22 +25,28 @@ proc skipStmtList(node: NimNode): NimNode {.compileTime.} =
     result = node[0]
 
 template createCb(retFutureSym, iteratorNameSym,
-                  name, futureVarCompletions: untyped) =
+                  strName, identName, futureVarCompletions: untyped) =
   var nameIterVar = iteratorNameSym
   #{.push stackTrace: off.}
-  proc cb0 {.closure.} =
+  proc identName {.closure.} =
     try:
       if not nameIterVar.finished:
         var next = nameIterVar()
+        # Continue while the yielded future is already finished.
+        while (not next.isNil) and next.finished:
+          next = nameIterVar()
+          if nameIterVar.finished:
+            break
+
         if next == nil:
           if not retFutureSym.finished:
             let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " &
                     "`nil` Future?"
-            raise newException(AssertionError, msg % name)
+            raise newException(AssertionError, msg % strName)
         else:
           {.gcsafe.}:
             {.push hint[ConvFromXtoItselfNotNeeded]: off.}
-            next.callback = (proc() {.closure, gcsafe.})(cb0)
+            next.callback = (proc() {.closure, gcsafe.})(identName)
             {.pop.}
     except:
       futureVarCompletions
@@ -52,7 +58,7 @@ template createCb(retFutureSym, iteratorNameSym,
       else:
         retFutureSym.fail(getCurrentException())
 
-  cb0()
+  identName()
   #{.pop.}
 proc generateExceptionCheck(futSym,
     tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} =
@@ -389,9 +395,12 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
     outerProcBody.add(closureIterator)
 
     # -> createCb(retFuture)
-    #var cbName = newIdentNode("cb")
+    # NOTE: The "_continue" suffix is checked for in asyncfutures.nim to produce
+    # friendlier stack traces:
+    var cbName = genSym(nskProc, prcName & "_continue")
     var procCb = getAst createCb(retFutureSym, iteratorNameSym,
                          newStrLitNode(prcName),
+                         cbName,
                          createFutureVarCompletions(futureVarIdents, nil))
     outerProcBody.add procCb
 

lib/pure/asyncnet.nim

@@ -286,6 +286,7 @@ template readInto(buf: pointer, size: int, socket: AsyncSocket,
                   flags: set[SocketFlag]): int =
   ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. Note that
   ## this is a template and not a proc.
+  assert(not socket.closed, "Cannot `recv` on a closed socket")
   var res = 0
   if socket.isSsl:
     when defineSsl:
@@ -412,6 +413,7 @@ proc send*(socket: AsyncSocket, buf: pointer, size: int,
   ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all
   ## data has been sent.
   assert socket != nil
+  assert(not socket.closed, "Cannot `send` on a closed socket")
   if socket.isSsl:
     when defineSsl:
       sslLoop(socket, flags,

lib/pure/collections/critbits.nim

@@ -141,8 +141,8 @@ proc excl*[T](c: var CritBitTree[T], key: string) =
 
 proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool =
   ## Returns true iff `c` does not contain the given `key`. If the key
-  ## does exist, c.excl(key) is performed. 
+  ## does exist, c.excl(key) is performed.
-  let oldCount = c.count 
+  let oldCount = c.count
   var n = exclImpl(c, key)
   result = c.count == oldCount
 
@@ -326,7 +326,7 @@ proc `$`*[T](c: CritBitTree[T]): string =
       result.add($key)
       when T isnot void:
         result.add(": ")
-        result.add($val)
+        result.addQuoted(val)
     result.add("}")
 
 when isMainModule:

lib/pure/collections/deques.nim

@@ -185,7 +185,7 @@ proc `$`*[T](deq: Deque[T]): string =
   result = "["
   for x in deq:
     if result.len > 1: result.add(", ")
-    result.add($x)
+    result.addQuoted(x)
   result.add("]")
 
 when isMainModule:

lib/pure/collections/lists.nim

@@ -135,7 +135,7 @@ proc `$`*[T](L: SomeLinkedCollection[T]): string =
   result = "["
   for x in nodes(L):
     if result.len > 1: result.add(", ")
-    result.add($x.value)
+    result.addQuoted(x.value)
   result.add("]")
 
 proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] =

lib/pure/collections/sets.nim

@@ -406,7 +406,7 @@ template dollarImpl() {.dirty.} =
   result = "{"
   for key in items(s):
     if result.len > 1: result.add(", ")
-    result.add($key)
+    result.addQuoted(key)
   result.add("}")
 
 proc `$`*[A](s: HashSet[A]): string =

lib/pure/collections/sharedlist.nim

@@ -73,10 +73,10 @@ proc add*[A](x: var SharedList[A]; y: A) =
     node.d[node.dataLen] = y
     inc(node.dataLen)
 
-proc initSharedList*[A](): SharedList[A] =
+proc init*[A](t: var SharedList[A]) =
-  initLock result.lock
+  initLock t.lock
-  result.head = nil
+  t.head = nil
-  result.tail = nil
+  t.tail = nil
 
 proc clear*[A](t: var SharedList[A]) =
   withLock(t):
@@ -92,4 +92,11 @@ proc deinitSharedList*[A](t: var SharedList[A]) =
   clear(t)
   deinitLock t.lock
 
+proc initSharedList*[A](): SharedList[A] {.deprecated.} =
+  ## Deprecated. Use `init` instead.
+  ## This is not posix compliant, may introduce undefined behavior.
+  initLock result.lock
+  result.head = nil
+  result.tail = nil
+
 {.pop.}

lib/pure/collections/sharedtables.nim

@@ -183,6 +183,7 @@ proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) =
 
 proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) =
   ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
+  ## This can introduce duplicate keys into the table!
   withLock t:
     addImpl(enlarge)
 
@@ -191,19 +192,29 @@ proc del*[A, B](t: var SharedTable[A, B], key: A) =
   withLock t:
     delImpl()
 
-proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] =
+proc init*[A, B](t: var SharedTable[A, B], initialSize=64) =
   ## creates a new hash table that is empty.
   ##
   ## `initialSize` needs to be a power of two. If you need to accept runtime
   ## values for this you could use the ``nextPowerOfTwo`` proc from the
   ## `math <math.html>`_ module or the ``rightSize`` proc from this module.
   assert isPowerOfTwo(initialSize)
-  result.counter = 0
+  t.counter = 0
-  result.dataLen = initialSize
+  t.dataLen = initialSize
-  result.data = cast[KeyValuePairSeq[A, B]](allocShared0(
+  t.data = cast[KeyValuePairSeq[A, B]](allocShared0(
                                       sizeof(KeyValuePair[A, B]) * initialSize))
-  initLock result.lock
+  initLock t.lock
 
 proc deinitSharedTable*[A, B](t: var SharedTable[A, B]) =
   deallocShared(t.data)
   deinitLock t.lock
+
+proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] {.deprecated.} =
+  ## Deprecated. Use `init` instead.
+  ## This is not posix compliant, may introduce undefined behavior.
+  assert isPowerOfTwo(initialSize)
+  result.counter = 0
+  result.dataLen = initialSize
+  result.data = cast[KeyValuePairSeq[A, B]](allocShared0(
+                                     sizeof(KeyValuePair[A, B]) * initialSize))
+  initLock result.lock

lib/pure/collections/tables.nim

@@ -308,6 +308,7 @@ proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) =
 
 proc add*[A, B](t: var Table[A, B], key: A, val: B) =
   ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
+  ## This can introduce duplicate keys into the table!
   addImpl(enlarge)
 
 proc len*[A, B](t: TableRef[A, B]): int =
@@ -337,9 +338,9 @@ template dollarImpl(): untyped {.dirty.} =
     result = "{"
     for key, val in pairs(t):
       if result.len > 1: result.add(", ")
-      result.add($key)
+      result.addQuoted(key)
       result.add(": ")
-      result.add($val)
+      result.addQuoted(val)
     result.add("}")
 
 proc `$`*[A, B](t: Table[A, B]): string =
@@ -430,6 +431,7 @@ proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) =
 
 proc add*[A, B](t: TableRef[A, B], key: A, val: B) =
   ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
+  ## This can introduce duplicate keys into the table!
   t[].add(key, val)
 
 proc del*[A, B](t: TableRef[A, B], key: A) =
@@ -604,6 +606,7 @@ proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) =
 
 proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) =
   ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
+  ## This can introduce duplicate keys into the table!
   addImpl(enlarge)
 
 proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B =
@@ -770,6 +773,7 @@ proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) =
 
 proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: B) =
   ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
+  ## This can introduce duplicate keys into the table!
   t[].add(key, val)
 
 proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] =
@@ -962,9 +966,10 @@ proc initCountTable*[A](initialSize=64): CountTable[A] =
   newSeq(result.data, initialSize)
 
 proc toCountTable*[A](keys: openArray[A]): CountTable[A] =
-  ## creates a new count table with every key in `keys` having a count of 1.
+  ## creates a new count table with every key in `keys` having a count
+  ## of how many times it occurs in `keys`.
   result = initCountTable[A](rightSize(keys.len))
-  for key in items(keys): result[key] = 1
+  for key in items(keys): result.inc key
 
 proc `$`*[A](t: CountTable[A]): string =
   ## The `$` operator for count tables.
@@ -989,9 +994,10 @@ proc inc*[A](t: var CountTable[A], key: A, val = 1) =
 proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] =
   ## returns the (key,val)-pair with the smallest `val`. Efficiency: O(n)
   assert t.len > 0
-  var minIdx = 0
+  var minIdx = -1
-  for h in 1..high(t.data):
+  for h in 0..high(t.data):
-    if t.data[h].val > 0 and t.data[minIdx].val > t.data[h].val: minIdx = h
+    if t.data[h].val > 0 and (minIdx == -1 or t.data[minIdx].val > t.data[h].val):
+      minIdx = h
   result.key = t.data[minIdx].key
   result.val = t.data[minIdx].val
 
@@ -1325,3 +1331,7 @@ when isMainModule:
     assert((a == b) == true)
     assert((b == a) == true)
 
+  block: # CountTable.smallest
+    var t = initCountTable[int]()
+    for v in items([0, 0, 5, 5, 5]): t.inc(v)
+    doAssert t.smallest == (0, 2)

lib/pure/cookies.nim

@@ -51,7 +51,7 @@ proc setCookie*(key, value: string, domain = "", path = "",
   if secure: result.add("; Secure")
   if httpOnly: result.add("; HttpOnly")
 
-proc setCookie*(key, value: string, expires: TimeInfo,
+proc setCookie*(key, value: string, expires: DateTime,
                 domain = "", path = "", noName = false,
                 secure = false, httpOnly = false): string =
   ## Creates a command in the format of
@@ -63,9 +63,9 @@ proc setCookie*(key, value: string, expires: TimeInfo,
                    noname, secure, httpOnly)
 
 when isMainModule:
-  var tim = Time(int(getTime()) + 76 * (60 * 60 * 24))
+  var tim = fromUnix(getTime().toUnix + 76 * (60 * 60 * 24))
 
-  let cookie = setCookie("test", "value", tim.getGMTime())
+  let cookie = setCookie("test", "value", tim.utc)
   when not defined(testing):
     echo cookie
   let start = "Set-Cookie: test=value; Expires="

lib/pure/cstrutils.nim

@@ -0,0 +1,79 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2017 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module supports helper routines for working with ``cstring``
+## without having to convert ``cstring`` to ``string`` in order to
+## save allocations.
+
+include "system/inclrtl"
+
+proc toLowerAscii(c: char): char {.inline.} =
+  if c in {'A'..'Z'}:
+    result = chr(ord(c) + (ord('a') - ord('A')))
+  else:
+    result = c
+
+proc startsWith*(s, prefix: cstring): bool {.noSideEffect,
+  rtl, extern: "csuStartsWith".} =
+  ## Returns true iff ``s`` starts with ``prefix``.
+  ##
+  ## If ``prefix == ""`` true is returned.
+  var i = 0
+  while true:
+    if prefix[i] == '\0': return true
+    if s[i] != prefix[i]: return false
+    inc(i)
+
+proc endsWith*(s, suffix: cstring): bool {.noSideEffect,
+  rtl, extern: "csuEndsWith".} =
+  ## Returns true iff ``s`` ends with ``suffix``.
+  ##
+  ## If ``suffix == ""`` true is returned.
+  let slen = s.len
+  var i = 0
+  var j = slen - len(suffix)
+  while i+j <% slen:
+    if s[i+j] != suffix[i]: return false
+    inc(i)
+  if suffix[i] == '\0': return true
+
+proc cmpIgnoreStyle*(a, b: cstring): int {.noSideEffect,
+  rtl, extern: "csuCmpIgnoreStyle".} =
+  ## Compares two strings normalized (i.e. case and
+  ## underscores do not matter). Returns:
+  ##
+  ## | 0 iff a == b
+  ## | < 0 iff a < b
+  ## | > 0 iff a > b
+  var i = 0
+  var j = 0
+  while true:
+    while a[i] == '_': inc(i)
+    while b[j] == '_': inc(j) # BUGFIX: typo
+    var aa = toLowerAscii(a[i])
+    var bb = toLowerAscii(b[j])
+    result = ord(aa) - ord(bb)
+    if result != 0 or aa == '\0': break
+    inc(i)
+    inc(j)
+
+proc cmpIgnoreCase*(a, b: cstring): int {.noSideEffect,
+  rtl, extern: "csuCmpIgnoreCase".} =
+  ## Compares two strings in a case insensitive manner. Returns:
+  ##
+  ## | 0 iff a == b
+  ## | < 0 iff a < b
+  ## | > 0 iff a > b
+  var i = 0
+  while true:
+    var aa = toLowerAscii(a[i])
+    var bb = toLowerAscii(b[i])
+    result = ord(aa) - ord(bb)
+    if result != 0 or aa == '\0': break
+    inc(i)

lib/pure/httpclient.nim

@@ -923,8 +923,14 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void]
     if chunkSize <= 0:
       discard await recvFull(client, 2, client.timeout, false) # Skip \c\L
       break
-    discard await recvFull(client, chunkSize, client.timeout, true)
+    var bytesRead = await recvFull(client, chunkSize, client.timeout, true)
-    discard await recvFull(client, 2, client.timeout, false) # Skip \c\L
+    if bytesRead != chunkSize:
+      httpError("Server terminated connection prematurely")
+
+    bytesRead = await recvFull(client, 2, client.timeout, false) # Skip \c\L
+    if bytesRead != 2:
+      httpError("Server terminated connection prematurely")
+
     # Trailer headers will only be sent if the request specifies that we want
     # them: http://tools.ietf.org/html/rfc2616#section-3.6.1
 
@@ -965,7 +971,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient,
       if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0":
         while true:
           let recvLen = await client.recvFull(4000, client.timeout, true)
-          if recvLen == 0:
+          if recvLen != 4000:
             client.close()
             break
 

lib/pure/ioselects/ioselectors_epoll.nim

@@ -277,15 +277,16 @@ proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool,
   var events = {Event.Timer}
   var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP)
   epv.data.u64 = fdi.uint
+
   if oneshot:
-    new_ts.it_interval.tv_sec = 0.Time
+    new_ts.it_interval.tv_sec = posix.Time(0)
     new_ts.it_interval.tv_nsec = 0
-    new_ts.it_value.tv_sec = (timeout div 1_000).Time
+    new_ts.it_value.tv_sec = posix.Time(timeout div 1_000)
     new_ts.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000
     incl(events, Event.Oneshot)
     epv.events = epv.events or EPOLLONESHOT
   else:
-    new_ts.it_interval.tv_sec = (timeout div 1000).Time
+    new_ts.it_interval.tv_sec = posix.Time(timeout div 1000)
     new_ts.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000
     new_ts.it_value.tv_sec = new_ts.it_interval.tv_sec
     new_ts.it_value.tv_nsec = new_ts.it_interval.tv_nsec

lib/pure/ioselects/ioselectors_kqueue.nim

@@ -452,10 +452,10 @@ proc selectInto*[T](s: Selector[T], timeout: int,
 
   if timeout != -1:
     if timeout >= 1000:
-      tv.tv_sec = (timeout div 1_000).Time
+      tv.tv_sec = posix.Time(timeout div 1_000)
       tv.tv_nsec = (timeout %% 1_000) * 1_000_000
     else:
-      tv.tv_sec = 0.Time
+      tv.tv_sec = posix.Time(0)
       tv.tv_nsec = timeout * 1_000_000
   else:
     ptv = nil

lib/pure/logging.nim

@@ -107,9 +107,14 @@ var
 proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): string =
   ## Format a log message using the ``frmt`` format string, ``level`` and varargs.
   ## See the module documentation for the format string syntax.
+  const nilString = "nil"
+
   var msgLen = 0
   for arg in args:
-    msgLen += arg.len
+    if arg.isNil:
+      msgLen += nilString.len
+    else:
+      msgLen += arg.len
   result = newStringOfCap(frmt.len + msgLen + 20)
   var i = 0
   while i < frmt.len:
@@ -136,7 +141,10 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str
       of "levelname": result.add(LevelNames[level])
       else: discard
   for arg in args:
-    result.add(arg)
+    if arg.isNil:
+      result.add(nilString)
+    else:
+      result.add(arg)
 
 method log*(logger: Logger, level: Level, args: varargs[string, `$`]) {.
             raises: [Exception], gcsafe,
@@ -361,3 +369,6 @@ when not defined(testing) and isMainModule:
   addHandler(L)
   for i in 0 .. 25:
     info("hello", i)
+
+  var nilString: string
+  info "hello ", nilString

lib/pure/math.nim

@@ -291,6 +291,8 @@ when not defined(JS):
     ##  echo fmod(-2.5, 0.3) ## -0.1
 
 else:
+  proc trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.}
+  proc trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.}
   proc floor*(x: float32): float32 {.importc: "Math.floor", nodecl.}
   proc floor*(x: float64): float64 {.importc: "Math.floor", nodecl.}
   proc ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.}
@@ -349,15 +351,19 @@ proc round*[T: float32|float64](x: T, places: int = 0): T =
     result = round0(x*mult)/mult
 
 when not defined(JS):
-  proc frexp*(x: float32, exponent: var int): float32 {.
+  proc c_frexp*(x: float32, exponent: var int32): float32 {.
     importc: "frexp", header: "<math.h>".}
-  proc frexp*(x: float64, exponent: var int): float64 {.
+  proc c_frexp*(x: float64, exponent: var int32): float64 {.
     importc: "frexp", header: "<math.h>".}
+  proc frexp*[T, U](x: T, exponent: var U): T =
     ## Split a number into mantissa and exponent.
     ## `frexp` calculates the mantissa m (a float greater than or equal to 0.5
     ## and less than 1) and the integer value n such that `x` (the original
     ## float value) equals m * 2**n. frexp stores n in `exponent` and returns
     ## m.
+    var exp: int32
+    result = c_frexp(x, exp)
+    exponent = exp
 else:
   proc frexp*[T: float32|float64](x: T, exponent: var int): T =
     if x == 0.0:
@@ -366,9 +372,14 @@ else:
     elif x < 0.0:
       result = -frexp(-x, exponent)
     else:
-      var ex = floor(log2(x))
+      var ex = trunc(log2(x))
-      exponent = round(ex)
+      exponent = int(ex)
       result = x / pow(2.0, ex)
+      if abs(result) >= 1:
+        inc(exponent)
+        result = result / 2
+      if exponent == 1024 and result == 0.0:
+        result = 0.99999999999999988898
 
 proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] =
   ## Breaks `x` into an integral and a fractional part.