Merge branch 'devel' into async-improvements

changelog.md

@@ -108,3 +108,32 @@ This now needs to be written as:
   - [``poly``](https://github.com/lcrees/polynumeric)
   - [``pdcurses``](https://github.com/lcrees/pdcurses)
   - [``romans``](https://github.com/lcrees/romans)
+
+- Added ``system.runnableExamples`` to make examples in Nim's documentation easier
+  to write and test. The examples are tested as the last step of
+  ``nim doc``.
+- Nim's ``rst2html`` command now supports the testing of code snippets via an RST
+  extension that we called ``:test:``::
+
+    .. code-block:: nim
+        :test:
+      # shows how the 'if' statement works
+      if true: echo "yes"
+- The ``[]`` proc for strings now raises an ``IndexError`` exception when
+  the specified slice is out of bounds. See issue
+  [#6223](https://github.com/nim-lang/Nim/issues/6223) for more details.
+- ``strutils.split`` and ``strutils.rsplit`` with an empty string and a
+  separator now returns that empty string.
+  See issue [#4377](https://github.com/nim-lang/Nim/issues/4377).
+- The experimental overloading of the dot ``.`` operators now take
+  an ``untyped``` parameter as the field name, it used to be
+  a ``static[string]``. You can use ``when defined(nimNewDot)`` to make
+  your code work with both old and new Nim versions.
+  See [special-operators](https://nim-lang.org/docs/manual.html#special-operators)
+  for more information.
+- Added ``macros.unpackVarargs``.
+- The memory manager now uses a variant of the TLSF algorithm that has much
+  better memory fragmentation behaviour. According
+  to [http://www.gii.upv.es/tlsf/](http://www.gii.upv.es/tlsf/) the maximum
+  fragmentation measured is lower than 25%. As a nice bonus ``alloc`` and
+  ``dealloc`` became O(1) operations.

compiler/ast.nim

@@ -639,7 +639,7 @@ type
     mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl,
     mNHint, mNWarning, mNError,
     mInstantiationInfo, mGetTypeInfo, mNGenSym,
-    mNimvm, mIntDefine, mStrDefine
+    mNimvm, mIntDefine, mStrDefine, mRunnableExamples
 
 # things that we can evaluate safely at compile time, even if not asked for it:
 const

compiler/ccgexprs.nim

@@ -1860,7 +1860,10 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     initLocExpr(p, e.sons[2], b)
     genDeepCopy(p, a, b)
   of mDotDot, mEqCString: genCall(p, e, d)
-  else: internalError(e.info, "genMagicExpr: " & $op)
+  else:
+    when defined(debugMagics):
+      echo p.prc.name.s, " ", p.prc.id, " ", p.prc.flags, " ", p.prc.ast[genericParamsPos].kind
+    internalError(e.info, "genMagicExpr: " & $op)
 
 proc genSetConstr(p: BProc, e: PNode, d: var TLoc) =
   # example: { a..b, c, d, e, f..g }

compiler/ccgtypes.nim

@@ -968,8 +968,11 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
     addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)])
   discard cgsym(m, "TNimType")
   if isDefined("nimTypeNames"):
+    var typename = typeToString(origType, preferName)
+    if typename == "ref object" and origType.skipTypes(skipPtrs).sym != nil:
+      typename = "anon ref object from " & $origType.skipTypes(skipPtrs).sym.info
     addf(m.s[cfsTypeInit3], "$1.name = $2;$n",
-        [name, makeCstring typeToString(origType, preferName)])
+        [name, makeCstring typename])
     discard cgsym(m, "nimTypeRoot")
     addf(m.s[cfsTypeInit3], "$1.nextType = nimTypeRoot; nimTypeRoot=&$1;$n",
          [name])

compiler/cgendata.nim

@@ -54,7 +54,7 @@ type
   TCProcSections* = array[TCProcSection, Rope] # represents a generated C proc
   BModule* = ref TCGen
   BProc* = ref TCProc
-  TBlock*{.final.} = object
+  TBlock* = object
     id*: int                  # the ID of the label; positive means that it
     label*: Rope              # generated text for the label
                               # nil if label is not used
@@ -64,7 +64,7 @@ type
     nestedExceptStmts*: int16 # how many except statements is it nested into
     frameLen*: int16
 
-  TCProc{.final.} = object    # represents C proc that is currently generated
+  TCProc = object             # represents C proc that is currently generated
     prc*: PSym                # the Nim proc that this C proc belongs to
     beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
     threadVarAccessed*: bool  # true if the proc already accessed some threadvar

compiler/commands.nim

@@ -654,6 +654,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     gListFullPaths = true
   of "dynliboverride":
     dynlibOverride(switch, arg, pass, info)
+  of "dynliboverrideall":
+    expectNoArg(switch, arg, pass, info)
+    gDynlibOverrideAll = true
   of "cs":
     # only supported for compatibility. Does nothing.
     expectArg(switch, arg, pass, info)

compiler/condsyms.nim

@@ -110,3 +110,5 @@ proc initDefines*() =
   when false: defineSymbol("nimHasOpt")
   defineSymbol("nimNoArrayToCstringConversion")
   defineSymbol("nimNewRoof")
+  defineSymbol("nimHasRunnableExamples")
+  defineSymbol("nimNewDot")

compiler/destroyer.nim

@@ -167,10 +167,13 @@ template interestingSym(s: PSym): bool =
 proc patchHead(n: PNode) =
   if n.kind in nkCallKinds and n[0].kind == nkSym and n.len > 1:
     let s = n[0].sym
-    if sfFromGeneric in s.flags and s.name.s[0] == '=' and
+    if s.name.s[0] == '=' and s.name.s in ["=sink", "=", "=destroy"]:
-        s.name.s in ["=sink", "=", "=destroy"]:
+      if sfFromGeneric in s.flags:
-      excl(s.flags, sfFromGeneric)
+        excl(s.flags, sfFromGeneric)
-      patchHead(s.getBody)
+        patchHead(s.getBody)
+      if n[1].typ.isNil:
+        # XXX toptree crashes without this workaround. Figure out why.
+        return
       let t = n[1].typ.skipTypes({tyVar, tyGenericInst, tyAlias, tyInferred})
       template patch(op, field) =
         if s.name.s == op and field != nil and field != s:
@@ -181,24 +184,30 @@ proc patchHead(n: PNode) =
   for x in n:
     patchHead(x)
 
+proc patchHead(s: PSym) =
+  if sfFromGeneric in s.flags:
+    patchHead(s.ast[bodyPos])
+
+template genOp(opr, opname) =
+  let op = opr
+  if op == nil:
+    globalError(dest.info, "internal error: '" & opname & "' operator not found for type " & typeToString(t))
+  elif op.ast[genericParamsPos].kind != nkEmpty:
+    globalError(dest.info, "internal error: '" & opname & "' operator is generic")
+  patchHead op
+  result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest))
+
 proc genSink(t: PType; dest: PNode): PNode =
   let t = t.skipTypes({tyGenericInst, tyAlias})
-  let op = if t.sink != nil: t.sink else: t.assignment
+  genOp(if t.sink != nil: t.sink else: t.assignment, "=sink")
-  assert op != nil
-  patchHead op.ast[bodyPos]
-  result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest))
 
 proc genCopy(t: PType; dest: PNode): PNode =
   let t = t.skipTypes({tyGenericInst, tyAlias})
-  assert t.assignment != nil
+  genOp(t.assignment, "=")
-  patchHead t.assignment.ast[bodyPos]
-  result = newTree(nkCall, newSymNode(t.assignment), newTree(nkHiddenAddr, dest))
 
 proc genDestroy(t: PType; dest: PNode): PNode =
   let t = t.skipTypes({tyGenericInst, tyAlias})
-  assert t.destructor != nil
+  genOp(t.destructor, "=destroy")
-  patchHead t.destructor.ast[bodyPos]
-  result = newTree(nkCall, newSymNode(t.destructor), newTree(nkHiddenAddr, dest))
 
 proc addTopVar(c: var Con; v: PNode) =
   c.topLevelVars.add newTree(nkIdentDefs, v, emptyNode, emptyNode)
@@ -210,7 +219,7 @@ template recurse(n, dest) =
     dest.add p(n[i], c)
 
 proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
-  if ri.kind in nkCallKinds:
+  if ri.kind in nkCallKinds+{nkObjConstr}:
     result = genSink(ri.typ, dest)
     # watch out and no not transform 'ri' twice if it's a call:
     let ri2 = copyNode(ri)
@@ -287,6 +296,7 @@ proc p(n: PNode; c: var Con): PNode =
     recurse(n, result)
 
 proc injectDestructorCalls*(owner: PSym; n: PNode): PNode =
+  echo "injecting into ", n
   var c: Con
   c.owner = owner
   c.tmp = newSym(skTemp, getIdent":d", owner, n.info)
@@ -312,7 +322,7 @@ proc injectDestructorCalls*(owner: PSym; n: PNode): PNode =
     result.add body
 
   when defined(nimDebugDestroys):
-    if owner.name.s == "createSeq":
+    if owner.name.s == "main" or true:
       echo "------------------------------------"
       echo owner.name.s, " transformed to: "
       echo result

compiler/docgen.nim

@@ -22,7 +22,6 @@ type
   TSections = array[TSymKind, Rope]
   TDocumentor = object of rstgen.RstGenerator
     modDesc: Rope           # module description
-    id: int                  # for generating IDs
     toc, section: TSections
     indexValFilename: string
     analytics: string  # Google Analytics javascript, "" if doesn't exist
@@ -109,6 +108,8 @@ proc newDocumentor*(filename: string, config: StringTableRef): PDoc =
   result.id = 100
   result.jArray = newJArray()
   initStrTable result.types
+  result.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string; status: int; content: string) =
+    localError(newLineInfo(d.filename, -1, -1), warnUser, "only 'rst2html' supports the ':test:' attribute")
 
 proc dispA(dest: var Rope, xml, tex: string, args: openArray[Rope]) =
   if gCmd != cmdRst2tex: addf(dest, xml, args)
@@ -204,10 +205,87 @@ proc getPlainDocstring(n: PNode): string =
   if n.comment != nil and startsWith(n.comment, "##"):
     result = n.comment
   if result.len < 1:
-    if n.kind notin {nkEmpty..nkNilLit}:
+    for i in countup(0, safeLen(n)-1):
-      for i in countup(0, len(n)-1):
+      result = getPlainDocstring(n.sons[i])
-        result = getPlainDocstring(n.sons[i])
+      if result.len > 0: return
-        if result.len > 0: return
+
+proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {}) =
+  var r: TSrcGen
+  var literal = ""
+  initTokRender(r, n, renderFlags)
+  var kind = tkEof
+  while true:
+    getNextTok(r, kind, literal)
+    case kind
+    of tkEof:
+      break
+    of tkComment:
+      dispA(result, "<span class=\"Comment\">$1</span>", "\\spanComment{$1}",
+            [rope(esc(d.target, literal))])
+    of tokKeywordLow..tokKeywordHigh:
+      dispA(result, "<span class=\"Keyword\">$1</span>", "\\spanKeyword{$1}",
+            [rope(literal)])
+    of tkOpr:
+      dispA(result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}",
+            [rope(esc(d.target, literal))])
+    of tkStrLit..tkTripleStrLit:
+      dispA(result, "<span class=\"StringLit\">$1</span>",
+            "\\spanStringLit{$1}", [rope(esc(d.target, literal))])
+    of tkCharLit:
+      dispA(result, "<span class=\"CharLit\">$1</span>", "\\spanCharLit{$1}",
+            [rope(esc(d.target, literal))])
+    of tkIntLit..tkUInt64Lit:
+      dispA(result, "<span class=\"DecNumber\">$1</span>",
+            "\\spanDecNumber{$1}", [rope(esc(d.target, literal))])
+    of tkFloatLit..tkFloat128Lit:
+      dispA(result, "<span class=\"FloatNumber\">$1</span>",
+            "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))])
+    of tkSymbol:
+      dispA(result, "<span class=\"Identifier\">$1</span>",
+            "\\spanIdentifier{$1}", [rope(esc(d.target, literal))])
+    of tkSpaces, tkInvalid:
+      add(result, literal)
+    of tkCurlyDotLe:
+      dispA(result, """<span class="Other pragmabegin">$1</span><div class="pragma">""",
+                    "\\spanOther{$1}",
+                  [rope(esc(d.target, literal))])
+    of tkCurlyDotRi:
+      dispA(result, "</div><span class=\"Other pragmaend\">$1</span>",
+                    "\\spanOther{$1}",
+                  [rope(esc(d.target, literal))])
+    of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi,
+       tkBracketDotLe, tkBracketDotRi, tkParDotLe,
+       tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot,
+       tkAccent, tkColonColon,
+       tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr:
+      dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
+            [rope(esc(d.target, literal))])
+
+proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
+  case n.kind
+  of nkCallKinds:
+    if n[0].kind == nkSym and n[0].sym.magic == mRunnableExamples and
+        n.len >= 2 and n.lastSon.kind == nkStmtList:
+      dispA(dest, "\n<strong class=\"examples_text\">$1</strong>\n",
+          "\n\\textbf{$1}\n", [rope"Examples:"])
+      inc d.listingCounter
+      let id = $d.listingCounter
+      dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
+      # this is a rather hacky way to get rid of the initial indentation
+      # that the renderer currently produces:
+      var i = 0
+      var body = n.lastSon
+      if body.len == 1 and body.kind == nkStmtList and
+          body.lastSon.kind == nkStmtList:
+        body = body.lastSon
+      for b in body:
+        if i > 0: dest.add "\n"
+        inc i
+        nodeToHighlightedHtml(d, b, dest, {})
+      dest.add(d.config.getOrDefault"doc.listing_end" % id)
+  else: discard
+  for i in 0 ..< n.safeLen:
+    getAllRunnableExamples(d, n[i], dest)
 
 when false:
   proc findDocComment(n: PNode): PNode =
@@ -379,11 +457,12 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
   let
     name = getName(d, nameNode)
     nameRope = name.rope
-    plainDocstring = getPlainDocstring(n) # call here before genRecComment!
+  var plainDocstring = getPlainDocstring(n) # call here before genRecComment!
   var result: Rope = nil
   var literal, plainName = ""
   var kind = tkEof
   var comm = genRecComment(d, n)  # call this here for the side-effect!
+  getAllRunnableExamples(d, n, comm)
   var r: TSrcGen
   # Obtain the plain rendered string for hyperlink titles.
   initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
@@ -395,53 +474,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
     plainName.add(literal)
 
   # Render the HTML hyperlink.
-  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments})
+  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, renderDocComments})
-  while true:
-    getNextTok(r, kind, literal)
-    case kind
-    of tkEof:
-      break
-    of tkComment:
-      dispA(result, "<span class=\"Comment\">$1</span>", "\\spanComment{$1}",
-            [rope(esc(d.target, literal))])
-    of tokKeywordLow..tokKeywordHigh:
-      dispA(result, "<span class=\"Keyword\">$1</span>", "\\spanKeyword{$1}",
-            [rope(literal)])
-    of tkOpr:
-      dispA(result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}",
-            [rope(esc(d.target, literal))])
-    of tkStrLit..tkTripleStrLit:
-      dispA(result, "<span class=\"StringLit\">$1</span>",
-            "\\spanStringLit{$1}", [rope(esc(d.target, literal))])
-    of tkCharLit:
-      dispA(result, "<span class=\"CharLit\">$1</span>", "\\spanCharLit{$1}",
-            [rope(esc(d.target, literal))])
-    of tkIntLit..tkUInt64Lit:
-      dispA(result, "<span class=\"DecNumber\">$1</span>",
-            "\\spanDecNumber{$1}", [rope(esc(d.target, literal))])
-    of tkFloatLit..tkFloat128Lit:
-      dispA(result, "<span class=\"FloatNumber\">$1</span>",
-            "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))])
-    of tkSymbol:
-      dispA(result, "<span class=\"Identifier\">$1</span>",
-            "\\spanIdentifier{$1}", [rope(esc(d.target, literal))])
-    of tkSpaces, tkInvalid:
-      add(result, literal)
-    of tkCurlyDotLe:
-      dispA(result, """<span class="Other pragmabegin">$1</span><div class="pragma">""",
-                    "\\spanOther{$1}",
-                  [rope(esc(d.target, literal))])
-    of tkCurlyDotRi:
-      dispA(result, "</div><span class=\"Other pragmaend\">$1</span>",
-                    "\\spanOther{$1}",
-                  [rope(esc(d.target, literal))])
-    of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi,
-       tkBracketDotLe, tkBracketDotRi, tkParDotLe,
-       tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot,
-       tkAccent, tkColonColon,
-       tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr:
-      dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
-            [rope(esc(d.target, literal))])
 
   inc(d.id)
   let
@@ -520,12 +553,24 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode =
 proc checkForFalse(n: PNode): bool =
   result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0
 
-proc traceDeps(d: PDoc, n: PNode) =
+proc traceDeps(d: PDoc, it: PNode) =
   const k = skModule
-  if d.section[k] != nil: add(d.section[k], ", ")
+
-  dispA(d.section[k],
+  if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
-        "<a class=\"reference external\" href=\"$1.html\">$1</a>",
+    let sep = it[0]
-        "$1", [rope(getModuleName(n))])
+    let dir = it[1]
+    let a = newNodeI(nkInfix, it.info)
+    a.add sep
+    a.add dir
+    a.add sep # dummy entry, replaced in the loop
+    for x in it[2]:
+      a.sons[2] = x
+      traceDeps(d, a)
+  else:
+    if d.section[k] != nil: add(d.section[k], ", ")
+    dispA(d.section[k],
+          "<a class=\"reference external\" href=\"$1.html\">$1</a>",
+          "$1", [rope(getModuleName(it))])
 
 proc generateDoc*(d: PDoc, n: PNode) =
   case n.kind
@@ -609,10 +654,7 @@ proc generateJson*(d: PDoc, n: PNode) =
   else: discard
 
 proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string =
-  var
+  result = getName(d, nameNode) & "\n"
-    name = getName(d, nameNode)
-
-  result = name & "\n"
 
 proc generateTags*(d: PDoc, n: PNode, r: var Rope) =
   case n.kind
@@ -758,6 +800,26 @@ proc commandDoc*() =
 proc commandRstAux(filename, outExt: string) =
   var filen = addFileExt(filename, "txt")
   var d = newDocumentor(filen, options.gConfigVars)
+  d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
+                          status: int; content: string) =
+    var outp: string
+    if filename.len == 0:
+      inc(d.id)
+      let nameOnly = splitFile(d.filename).name
+      let subdir = getNimcacheDir() / nameOnly
+      createDir(subdir)
+      outp = subdir / (nameOnly & "_snippet_" & $d.id & ".nim")
+    elif isAbsolute(filename):
+      outp = filename
+    else:
+      # Nim's convention: every path is relative to the file it was written in:
+      outp = splitFile(d.filename).dir / filename
+    writeFile(outp, content)
+    let cmd = unescape(cmd) % quoteShell(outp)
+    rawMessage(hintExecuting, cmd)
+    if execShellCmd(cmd) != status:
+      rawMessage(errExecutionOfProgramFailed, cmd)
+
   d.isPureRst = true
   var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc,
                      {roSupportRawDirective})

compiler/evaltempl.nim

@@ -42,7 +42,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
            s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam:
         handleParam actual.sons[s.owner.typ.len + s.position - 1]
       else:
-        internalAssert sfGenSym in s.flags
+        internalAssert sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
           x = copySym(s, false)

compiler/extccomp.nim

@@ -728,13 +728,13 @@ proc execCmdsInParallel(cmds: seq[string]; prettyCb: proc (idx: int)) =
   else:
     tryExceptOSErrorMessage("invocation of external compiler program failed."):
       if optListCmd in gGlobalOptions or gVerbosity > 1:
-        res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams},
+        res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath},
                             gNumberOfProcessors, afterRunEvent=runCb)
       elif gVerbosity == 1:
-        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams},
+        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath},
                             gNumberOfProcessors, prettyCb, afterRunEvent=runCb)
       else:
-        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams},
+        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath},
                             gNumberOfProcessors, afterRunEvent=runCb)
   if res != 0:
     if gNumberOfProcessors <= 1:
@@ -764,8 +764,9 @@ proc callCCompiler*(projectfile: string) =
       add(objfiles, quoteShell(
           addFileExt(objFile, CC[cCompiler].objExt)))
     for x in toCompile:
+      let objFile = if noAbsolutePaths(): x.obj.extractFilename else: x.obj
       add(objfiles, ' ')
-      add(objfiles, quoteShell(x.obj))
+      add(objfiles, quoteShell(objFile))
 
     linkCmd = getLinkCmd(projectfile, objfiles)
     if optCompileOnly notin gGlobalOptions:

compiler/jsgen.nim

@@ -1563,14 +1563,22 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     internalError("createVar: " & $t.kind)
     result = nil
 
+template returnType: untyped =
+  ~""
+
 proc genVarInit(p: PProc, v: PSym, n: PNode) =
   var
     a: TCompRes
     s: Rope
+    varCode: string
+  if v.constraint.isNil:
+    varCode = "var $2"
+  else:
+    varCode = v.constraint.strVal
   if n.kind == nkEmpty:
     let mname = mangleName(v, p.target)
-    lineF(p, "var $1 = $2;$n" | "$$$1 = $2;$n",
+    lineF(p, varCode & " = $3;$n" | "$$$2 = $3;$n",
-             [mname, createVar(p, v.typ, isIndirect(v))])
+               [returnType, mname, createVar(p, v.typ, isIndirect(v))])
     if v.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, v.typ) == etyBaseIndex:
       lineF(p, "var $1_Idx = 0;$n", [ mname ])
   else:
@@ -1587,25 +1595,25 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       let targetBaseIndex = {sfAddrTaken, sfGlobal} * v.flags == {}
       if a.typ == etyBaseIndex:
         if targetBaseIndex:
-          lineF(p, "var $1 = $2, $1_Idx = $3;$n",
+          lineF(p, varCode & " = $3, $2_Idx = $4;$n",
-                   [v.loc.r, a.address, a.res])
+                   [returnType, v.loc.r, a.address, a.res])
         else:
-          lineF(p, "var $1 = [$2, $3];$n",
+          lineF(p, varCode & " = [$3, $4];$n",
-                   [v.loc.r, a.address, a.res])
+                   [returnType, v.loc.r, a.address, a.res])
       else:
         if targetBaseIndex:
           let tmp = p.getTemp
           lineF(p, "var $1 = $2, $3 = $1[0], $3_Idx = $1[1];$n",
                    [tmp, a.res, v.loc.r])
         else:
-          lineF(p, "var $1 = $2;$n", [v.loc.r, a.res])
+          lineF(p, varCode & " = $3;$n", [returnType, v.loc.r, a.res])
       return
     else:
       s = a.res
     if isIndirect(v):
-      lineF(p, "var $1 = [$2];$n", [v.loc.r, s])
+      lineF(p, varCode & " = [$3];$n", [returnType, v.loc.r, s])
     else:
-      lineF(p, "var $1 = $2;$n" | "$$$1 = $2;$n", [v.loc.r, s])
+      lineF(p, varCode & " = $3;$n" | "$$$2 = $3;$n", [returnType, v.loc.r, s])
 
 proc genVarStmt(p: PProc, n: PNode) =
   for i in countup(0, sonsLen(n) - 1):
@@ -2162,8 +2170,22 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
       returnStmt = "return $#;$n" % [a.res]
 
   p.nested: genStmt(p, prc.getBody)
-  let def = "function $#($#) {$n$#$#$#$#$#" %
+
-            [name, header,
+  var def: Rope
+  if not prc.constraint.isNil:
+    def = (prc.constraint.strVal & " {$n$#$#$#$#$#") %
+            [ returnType,
+              name,
+              header,
+              optionaLine(p.globals),
+              optionaLine(p.locals),
+              optionaLine(resultAsgn),
+              optionaLine(genProcBody(p, prc)),
+              optionaLine(p.indentLine(returnStmt))]
+  else:
+    def = "function $#($#) {$n$#$#$#$#$#" %
+            [ name,
+              header,
               optionaLine(p.globals),
               optionaLine(p.locals),
               optionaLine(resultAsgn),

compiler/lexer.nim

@@ -860,6 +860,23 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
+proc newlineFollows*(L: var TLexer): bool =
+  var pos = L.bufpos
+  var buf = L.buf
+  while true:
+    case buf[pos]
+    of ' ', '\t':
+      inc(pos)
+    of CR, LF:
+      result = true
+      break
+    of '#':
+      inc(pos)
+      if buf[pos] == '#': inc(pos)
+      if buf[pos] != '[': return true
+    else:
+      break
+
 proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
                           isDoc: bool) =
   var pos = start

compiler/options.nim

@@ -145,6 +145,7 @@ var
   gNoNimblePath* = false
   gExperimentalMode*: bool
   newDestructors*: bool
+  gDynlibOverrideAll*: bool
 
 proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools}
 proc usesNativeGC*(): bool {.inline.} = gSelectedGC >= gcRefc
@@ -427,7 +428,7 @@ proc inclDynlibOverride*(lib: string) =
   gDllOverrides[lib.canonDynlibName] = "true"
 
 proc isDynlibOverride*(lib: string): bool =
-  result = gDllOverrides.hasKey(lib.canonDynlibName)
+  result = gDynlibOverrideAll or gDllOverrides.hasKey(lib.canonDynlibName)
 
 proc binaryStrSearch*(x: openArray[string], y: string): int =
   var a = 0

compiler/pragmas.nim

@@ -23,7 +23,7 @@ const
     wMagic, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader,
     wCompilerproc, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge,
     wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC,
-    wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
+    wAsmNoStackFrame, wError, wDiscardable, wNoInit, wCodegenDecl,
     wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe,
     wOverride, wConstructor, wExportNims, wUsed, wLiftLocals}
   converterPragmas* = procPragmas
@@ -759,10 +759,6 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         incl(sym.loc.flags, lfNoDecl)
         # implies nodecl, because otherwise header would not make sense
         if sym.loc.r == nil: sym.loc.r = rope(sym.name.s)
-      of wDestructor:
-        sym.flags.incl sfOverriden
-        if sym.name.s.normalize != "destroy":
-          localError(n.info, errGenerated, "destructor has to be named 'destroy'")
       of wOverride:
         sym.flags.incl sfOverriden
       of wNosideeffect:

compiler/renderer.nim

@@ -826,7 +826,10 @@ proc gident(g: var TSrcGen, n: PNode) =
     t = tkOpr
   put(g, t, s)
   if n.kind == nkSym and (renderIds in g.flags or sfGenSym in n.sym.flags):
-    put(g, tkIntLit, $n.sym.id)
+    when defined(debugMagics):
+      put(g, tkIntLit, $n.sym.id & $n.sym.magic)
+    else:
+      put(g, tkIntLit, $n.sym.id)
 
 proc doParamsAux(g: var TSrcGen, params: PNode) =
   if params.len > 1:

compiler/scriptconfig.nim

@@ -143,6 +143,7 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
 
 proc runNimScript*(cache: IdentCache; scriptName: string;
                    freshDefines=true; config: ConfigRef=nil) =
+  rawMessage(hintConf, scriptName)
   passes.gIncludeFile = includeModule
   passes.gImportModule = importModule
   let graph = newModuleGraph(config)

compiler/sem.nim

@@ -570,6 +570,18 @@ proc myProcess(context: PPassContext, n: PNode): PNode =
         result = ast.emptyNode
       #if gCmd == cmdIdeTools: findSuggest(c, n)
 
+proc testExamples(c: PContext) =
+  let inp = toFullPath(c.module.info)
+  let outp = inp.changeFileExt"" & "_examples.nim"
+  renderModule(c.runnableExamples, inp, outp)
+  let backend = if isDefined("js"): "js"
+                elif isDefined("cpp"): "cpp"
+                elif isDefined("objc"): "objc"
+                else: "c"
+  if os.execShellCmd("nim " & backend & " -r " & outp) != 0:
+    quit "[Examples] failed"
+  removeFile(outp)
+
 proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   var c = PContext(context)
   if gCmd == cmdIdeTools and not c.suggestionsMade:
@@ -584,5 +596,6 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
     result.add(c.module.ast)
   popOwner(c)
   popProcCon(c)
+  if c.runnableExamples != nil: testExamples(c)
 
 const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose)

compiler/semasgn.nim

@@ -7,8 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
-## This module implements lifting for assignments. Later versions of this code
+## This module implements lifting for type-bound operations
-## will be able to also lift ``=deepCopy`` and ``=destroy``.
+## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
 
 # included from sem.nim
 
@@ -302,6 +302,7 @@ proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
   n.sons[paramsPos] = result.typ.n
   n.sons[bodyPos] = body
   result.ast = n
+  incl result.flags, sfFromGeneric
 
 
 proc getAsgnOrLiftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
@@ -319,8 +320,10 @@ proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
   ## to ensure we lift assignment, destructors and moves properly.
   ## The later 'destroyer' pass depends on it.
   if not newDestructors or not hasDestructor(typ): return
-  # do not produce wrong liftings while we're still instantiating generics:
+  when false:
-  if c.typesWithOps.len > 0: return
+    # do not produce wrong liftings while we're still instantiating generics:
+    # now disabled; breaks topttree.nim!
+    if c.typesWithOps.len > 0: return
   let typ = typ.skipTypes({tyGenericInst, tyAlias})
   # we generate the destructor first so that other operators can depend on it:
   if typ.destructor == nil:
@@ -329,3 +332,6 @@ proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
     liftBody(c, typ, attachedAsgn, info)
   if typ.sink == nil:
     liftBody(c, typ, attachedSink, info)
+
+#proc patchResolvedTypeBoundOp*(c: PContext; n: PNode): PNode =
+#  if n.kind == nkCall and

compiler/semcall.nim

@@ -179,7 +179,7 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) =
   add(result, ')')
   if candidates != "":
     add(result, "\n" & msgKindToString(errButExpected) & "\n" & candidates)
-  localError(n.info, errGenerated, result)
+  localError(n.info, errGenerated, result & "\nexpression: " & $n)
 
 proc bracketNotFoundError(c: PContext; n: PNode) =
   var errors: CandidateErrors = @[]
@@ -235,12 +235,11 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
 
     if nfDotField in n.flags:
       internalAssert f.kind == nkIdent and n.sonsLen >= 2
-      let calleeName = newStrNode(nkStrLit, f.ident.s).withInfo(n.info)
 
       # leave the op head symbol empty,
       # we are going to try multiple variants
-      n.sons[0..1] = [nil, n[1], calleeName]
+      n.sons[0..1] = [nil, n[1], f]
-      orig.sons[0..1] = [nil, orig[1], calleeName]
+      orig.sons[0..1] = [nil, orig[1], f]
 
       template tryOp(x) =
         let op = newIdentNode(getIdent(x), n.info)
@@ -255,8 +254,8 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
         tryOp "."
 
     elif nfDotSetter in n.flags and f.kind == nkIdent and n.len == 3:
-      let calleeName = newStrNode(nkStrLit,
+      # we need to strip away the trailing '=' here:
-        f.ident.s[0..f.ident.s.len-2]).withInfo(n.info)
+      let calleeName = newIdentNode(getIdent(f.ident.s[0..f.ident.s.len-2]), n.info)
       let callOp = newIdentNode(getIdent".=", n.info)
       n.sons[0..1] = [callOp, n[1], calleeName]
       orig.sons[0..1] = [callOp, orig[1], calleeName]

compiler/semdata.nim

@@ -136,6 +136,7 @@ type
       # the generic type has been constructed completely. See
       # tests/destructor/topttree.nim for an example that
       # would otherwise fail.
+    runnableExamples*: PNode
 
 proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair =
   result.genericSym = s

compiler/semdestruct.nim

@@ -1,186 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2013 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements destructors.
-
-# included from sem.nim
-
-# special marker values that indicates that we are
-# 1) AnalyzingDestructor: currently analyzing the type for destructor
-# generation (needed for recursive types)
-# 2) DestructorIsTrivial: completed the analysis before and determined
-# that the type has a trivial destructor
-var analyzingDestructor, destructorIsTrivial: PSym
-new(analyzingDestructor)
-new(destructorIsTrivial)
-
-var
-  destructorName = getIdent"destroy_"
-  destructorParam = getIdent"this_"
-  destructorPragma = newIdentNode(getIdent"destructor", unknownLineInfo())
-
-proc instantiateDestructor(c: PContext, typ: PType): PType
-
-proc doDestructorStuff(c: PContext, s: PSym, n: PNode) =
-  var t = s.typ.sons[1].skipTypes({tyVar})
-  if t.kind == tyGenericInvocation:
-    for i in 1 ..< t.sonsLen:
-      if t.sons[i].kind != tyGenericParam:
-        localError(n.info, errDestructorNotGenericEnough)
-        return
-    t = t.base
-  elif t.kind == tyCompositeTypeClass:
-    t = t.base
-    if t.kind != tyGenericBody:
-      localError(n.info, errDestructorNotGenericEnough)
-      return
-
-  t.destructor = s
-  # automatically insert calls to base classes' destructors
-  if n.sons[bodyPos].kind != nkEmpty:
-    for i in countup(0, t.sonsLen - 1):
-      # when inheriting directly from object
-      # there will be a single nil son
-      if t.sons[i] == nil: continue
-      let destructableT = instantiateDestructor(c, t.sons[i])
-      if destructableT != nil:
-        n.sons[bodyPos].addSon(newNode(nkCall, t.sym.info, @[
-            useSym(destructableT.destructor, c.graph.usageSym),
-            n.sons[paramsPos][1][0]]))
-
-proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode
-
-proc destroySym(c: PContext, field: PSym, holder: PNode): PNode =
-  let destructableT = instantiateDestructor(c, field.typ)
-  if destructableT != nil:
-    result = newNode(nkCall, field.info, @[
-      useSym(destructableT.destructor, c.graph.usageSym),
-      newNode(nkDotExpr, field.info, @[holder, useSym(field, c.graph.usageSym)])])
-
-proc destroyCase(c: PContext, n: PNode, holder: PNode): PNode =
-  var nonTrivialFields = 0
-  result = newNode(nkCaseStmt, n.info, @[])
-  # case x.kind
-  result.addSon(newNode(nkDotExpr, n.info, @[holder, n.sons[0]]))
-  for i in countup(1, n.len - 1):
-    # of A, B:
-    let ni = n[i]
-    var caseBranch = newNode(ni.kind, ni.info, ni.sons[0..ni.len-2])
-
-    let stmt = destroyFieldOrFields(c, ni.lastSon, holder)
-    if stmt == nil:
-      caseBranch.addSon(newNode(nkStmtList, ni.info, @[]))
-    else:
-      caseBranch.addSon(stmt)
-      nonTrivialFields += stmt.len
-
-    result.addSon(caseBranch)
-
-  # maybe no fields were destroyed?
-  if nonTrivialFields == 0:
-    result = nil
-
-proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode =
-  template maybeAddLine(e) =
-    let stmt = e
-    if stmt != nil:
-      if result == nil: result = newNode(nkStmtList)
-      result.addSon(stmt)
-
-  case field.kind
-  of nkRecCase:
-    maybeAddLine destroyCase(c, field, holder)
-  of nkSym:
-    maybeAddLine destroySym(c, field.sym, holder)
-  of nkRecList:
-    for son in field:
-      maybeAddLine destroyFieldOrFields(c, son, holder)
-  else:
-    internalAssert false
-
-proc generateDestructor(c: PContext, t: PType): PNode =
-  ## generate a destructor for a user-defined object or tuple type
-  ## returns nil if the destructor turns out to be trivial
-
-  # XXX: This may be true for some C-imported types such as
-  # Tposix_spawnattr
-  if t.n == nil or t.n.sons == nil: return
-  internalAssert t.n.kind == nkRecList
-  let destructedObj = newIdentNode(destructorParam, unknownLineInfo())
-  # call the destructods of all fields
-  result = destroyFieldOrFields(c, t.n, destructedObj)
-  # base classes' destructors will be automatically called by
-  # semProcAux for both auto-generated and user-defined destructors
-
-proc instantiateDestructor(c: PContext, typ: PType): PType =
-  # returns nil if a variable of type `typ` doesn't require a
-  # destructor. Otherwise, returns the type, which holds the
-  # destructor that must be used for the varialbe.
-  # The destructor is either user-defined or automatically
-  # generated by the compiler in a member-wise fashion.
-  var t = typ.skipGenericAlias
-  let typeHoldingUserDefinition = if t.kind == tyGenericInst: t.base else: t
-
-  if typeHoldingUserDefinition.destructor != nil:
-    # XXX: This is not entirely correct for recursive types, but we need
-    # it temporarily to hide the "destroy is already defined" problem
-    if typeHoldingUserDefinition.destructor notin
-        [analyzingDestructor, destructorIsTrivial]:
-      return typeHoldingUserDefinition
-    else:
-      return nil
-
-  t = t.skipTypes({tyGenericInst, tyAlias})
-  case t.kind
-  of tySequence, tyArray, tyOpenArray, tyVarargs:
-    t.destructor = analyzingDestructor
-    if instantiateDestructor(c, t.sons[0]) != nil:
-      t.destructor = getCompilerProc"nimDestroyRange"
-      return t
-    else:
-      return nil
-  of tyTuple, tyObject:
-    t.destructor = analyzingDestructor
-    let generated = generateDestructor(c, t)
-    if generated != nil:
-      internalAssert t.sym != nil
-      var i = t.sym.info
-      let fullDef = newNode(nkProcDef, i, @[
-        newIdentNode(destructorName, i),
-        emptyNode,
-        emptyNode,
-        newNode(nkFormalParams, i, @[
-          emptyNode,
-          newNode(nkIdentDefs, i, @[
-            newIdentNode(destructorParam, i),
-            symNodeFromType(c, makeVarType(c, t), t.sym.info),
-            emptyNode]),
-          ]),
-        newNode(nkPragma, i, @[destructorPragma]),
-        emptyNode,
-        generated
-        ])
-      let semantizedDef = semProc(c, fullDef)
-      t.destructor = semantizedDef[namePos].sym
-      return t
-    else:
-      t.destructor = destructorIsTrivial
-      return nil
-  else:
-    return nil
-
-proc createDestructorCall(c: PContext, s: PSym): PNode =
-  let varTyp = s.typ
-  if varTyp == nil or sfGlobal in s.flags: return
-  let destructableT = instantiateDestructor(c, varTyp)
-  if destructableT != nil:
-    let call = semStmt(c, newNode(nkCall, s.info, @[
-      useSym(destructableT.destructor, c.graph.usageSym),
-      useSym(s, c.graph.usageSym)]))
-    result = newNode(nkDefer, s.info, @[call])

compiler/semexprs.nim

@@ -53,7 +53,6 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   else:
     if efNoProcvarCheck notin flags: semProcvarCheck(c, result)
     if result.typ.kind == tyVar: result = newDeref(result)
-    semDestructorCheck(c, result, flags)
 
 proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   result = semExpr(c, n, flags)
@@ -66,7 +65,6 @@ proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     result.typ = errorType(c)
   else:
     semProcvarCheck(c, result)
-    semDestructorCheck(c, result, flags)
 
 proc semSymGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
   result = symChoice(c, n, s, scClosed)
@@ -671,6 +669,7 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode =
     if callee.magic != mNone:
       result = magicsAfterOverloadResolution(c, result, flags)
     if result.typ != nil: liftTypeBoundOps(c, result.typ, n.info)
+    #result = patchResolvedTypeBoundOp(c, result)
   if c.matchedConcept == nil:
     result = evalAtCompileTime(c, result)
 
@@ -1847,6 +1846,18 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
       analyseIfAddressTakenInCall(c, result)
       if callee.magic != mNone:
         result = magicsAfterOverloadResolution(c, result, flags)
+  of mRunnableExamples:
+    if gCmd == cmdDoc and n.len >= 2 and n.lastSon.kind == nkStmtList:
+      if n.sons[0].kind == nkIdent:
+        if sfMainModule in c.module.flags:
+          let inp = toFullPath(c.module.info)
+          if c.runnableExamples == nil:
+            c.runnableExamples = newTree(nkStmtList,
+              newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp))))
+          c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon)
+        result = setMs(n, s)
+    else:
+      result = emptyNode
   else:
     result = semDirectOp(c, n, flags)
 

compiler/semstmts.nim

@@ -97,27 +97,12 @@ template semProcvarCheck(c: PContext, n: PNode) =
 
 proc semProc(c: PContext, n: PNode): PNode
 
-include semdestruct
-
-proc semDestructorCheck(c: PContext, n: PNode, flags: TExprFlags) {.inline.} =
-  if not newDestructors:
-    if efAllowDestructor notin flags and
-        n.kind in nkCallKinds+{nkObjConstr,nkBracket}:
-      if instantiateDestructor(c, n.typ) != nil:
-        localError(n.info, warnDestructor)
-    # This still breaks too many things:
-    when false:
-      if efDetermineType notin flags and n.typ.kind == tyTypeDesc and
-          c.p.owner.kind notin {skTemplate, skMacro}:
-        localError(n.info, errGenerated, "value expected, but got a type")
-
 proc semExprBranch(c: PContext, n: PNode): PNode =
   result = semExpr(c, n)
   if result.typ != nil:
     # XXX tyGenericInst here?
     semProcvarCheck(c, result)
     if result.typ.kind == tyVar: result = newDeref(result)
-    semDestructorCheck(c, result, {})
 
 proc semExprBranchScope(c: PContext, n: PNode): PNode =
   openScope(c)
@@ -421,15 +406,6 @@ proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) =
   else:
     result.add identDefs
 
-proc addDefer(c: PContext; result: var PNode; s: PSym) =
-  let deferDestructorCall = createDestructorCall(c, s)
-  if deferDestructorCall != nil:
-    if result.kind != nkStmtList:
-      let oldResult = result
-      result = newNodeI(nkStmtList, result.info)
-      result.add oldResult
-    result.add deferDestructorCall
-
 proc isDiscardUnderscore(v: PSym): bool =
   if v.name.s == "_":
     v.flags.incl(sfGenSym)
@@ -609,7 +585,6 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         if def.kind == nkPar: v.ast = def[j]
         setVarType(v, tup.sons[j])
         b.sons[j] = newSymNode(v)
-      if not newDestructors: addDefer(c, result, v)
       checkNilable(v)
       if sfCompileTime in v.flags: hasCompileTime = true
   if hasCompileTime: vm.setupCompileTimeVar(c.module, c.cache, result)
@@ -1041,6 +1016,8 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
       checkConstructedType(s.info, s.typ)
       if s.typ.kind in {tyObject, tyTuple} and not s.typ.n.isNil:
         checkForMetaFields(s.typ.n)
+  instAllTypeBoundOp(c, n.info)
+
 
 proc semAllTypeSections(c: PContext; n: PNode): PNode =
   proc gatherStmts(c: PContext; n: PNode; result: PNode) {.nimcall.} =
@@ -1095,9 +1072,11 @@ proc semTypeSection(c: PContext, n: PNode): PNode =
   ## to allow the type definitions in the section to reference each other
   ## without regard for the order of their definitions.
   if sfNoForward notin c.module.flags or nfSem notin n.flags:
+    inc c.inTypeContext
     typeSectionLeftSidePass(c, n)
     typeSectionRightSidePass(c, n)
     typeSectionFinalPass(c, n)
+    dec c.inTypeContext
   result = n
 
 proc semParamList(c: PContext, n, genericParams: PNode, s: PSym) =
@@ -1318,7 +1297,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
         var obj = t.sons[1].sons[0]
         while true:
           incl(obj.flags, tfHasAsgn)
-          if obj.kind == tyGenericBody: obj = obj.lastSon
+          if obj.kind in {tyGenericBody, tyGenericInst}: obj = obj.lastSon
           elif obj.kind == tyGenericInvocation: obj = obj.sons[0]
           else: break
         if obj.kind in {tyObject, tyDistinct}:
@@ -1331,10 +1310,6 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
       if not noError and sfSystemModule notin s.owner.flags:
         localError(n.info, errGenerated,
           "signature for '" & s.name.s & "' must be proc[T: object](x: var T)")
-    else:
-      doDestructorStuff(c, s, n)
-      if not experimentalMode(c):
-        localError n.info, "use the {.experimental.} pragma to enable destructors"
     incl(s.flags, sfUsed)
   of "deepcopy", "=deepcopy":
     if s.typ.len == 2 and
@@ -1561,8 +1536,11 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   s.options = gOptions
   if sfOverriden in s.flags or s.name.s[0] == '=': semOverride(c, s, n)
   if s.name.s[0] in {'.', '('}:
-    if s.name.s in [".", ".()", ".=", "()"] and not experimentalMode(c):
+    if s.name.s in [".", ".()", ".="] and not experimentalMode(c) and not newDestructors:
       message(n.info, warnDeprecated, "overloaded '.' and '()' operators are now .experimental; " & s.name.s)
+    elif s.name.s == "()" and not experimentalMode(c):
+      message(n.info, warnDeprecated, "overloaded '()' operators are now .experimental; " & s.name.s)
+
   if n.sons[bodyPos].kind != nkEmpty:
     # for DLL generation it is annoying to check for sfImportc!
     if sfBorrow in s.flags:

compiler/sighashes.nim

@@ -87,6 +87,7 @@ type
     CoProc
     CoType
     CoOwnerSig
+    CoIgnoreRange
 
 proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag])
 
@@ -159,14 +160,15 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
     return
   else:
     discard
-  c &= char(t.kind)
   case t.kind
   of tyBool, tyChar, tyInt..tyUInt64:
     # no canonicalization for integral types, so that e.g. ``pid_t`` is
     # produced instead of ``NI``:
+    c &= char(t.kind)
     if t.sym != nil and {sfImportc, sfExportc} * t.sym.flags != {}:
       c.hashSym(t.sym)
   of tyObject, tyEnum:
+    c &= char(t.kind)
     if t.typeInst != nil:
       assert t.typeInst.kind == tyGenericInst
       for i in countup(1, sonsLen(t.typeInst) - 2):
@@ -199,26 +201,35 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
     if t.len > 0 and t.sons[0] != nil:
       hashType c, t.sons[0], flags
   of tyRef, tyPtr, tyGenericBody, tyVar:
+    c &= char(t.kind)
     c.hashType t.lastSon, flags
     if tfVarIsPtr in t.flags: c &= ".varisptr"
   of tyFromExpr:
+    c &= char(t.kind)
     c.hashTree(t.n)
   of tyTuple:
+    c &= char(t.kind)
     if t.n != nil and CoType notin flags:
       assert(sonsLen(t.n) == sonsLen(t))
       for i in countup(0, sonsLen(t.n) - 1):
         assert(t.n.sons[i].kind == nkSym)
         c &= t.n.sons[i].sym.name.s
         c &= ':'
-        c.hashType(t.sons[i], flags)
+        c.hashType(t.sons[i], flags+{CoIgnoreRange})
         c &= ','
     else:
-      for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i], flags
+      for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i], flags+{CoIgnoreRange}
-  of tyRange, tyStatic:
+  of tyRange:
-    #if CoType notin flags:
+    if CoIgnoreRange notin flags:
+      c &= char(t.kind)
+      c.hashTree(t.n)
+    c.hashType(t.sons[0], flags)
+  of tyStatic:
+    c &= char(t.kind)
     c.hashTree(t.n)
     c.hashType(t.sons[0], flags)
   of tyProc:
+    c &= char(t.kind)
     c &= (if tfIterator in t.flags: "iterator " else: "proc ")
     if CoProc in flags and t.n != nil:
       let params = t.n
@@ -236,7 +247,11 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
       if tfNoSideEffect in t.flags: c &= ".noSideEffect"
       if tfThread in t.flags: c &= ".thread"
     if tfVarargs in t.flags: c &= ".varargs"
+  of tyArray:
+    c &= char(t.kind)
+    for i in 0..<t.len: c.hashType(t.sons[i], flags-{CoIgnoreRange})
   else:
+    c &= char(t.kind)
     for i in 0..<t.len: c.hashType(t.sons[i], flags)
   if tfNotNil in t.flags and CoType notin flags: c &= "not nil"
 

compiler/transf.nim

@@ -693,7 +693,7 @@ proc transformCall(c: PTransf, n: PNode): PTransNode =
           inc(j)
       add(result, a.PTransNode)
     if len(result) == 2: result = result[1]
-  elif magic in {mNBindSym, mTypeOf}:
+  elif magic in {mNBindSym, mTypeOf, mRunnableExamples}:
     # for bindSym(myconst) we MUST NOT perform constant folding:
     result = n.PTransNode
   elif magic == mProcCall:

compiler/vmdeps.nim

@@ -84,10 +84,10 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
 
   if inst:
     if t.sym != nil:  # if this node has a symbol
-      if allowRecursion:  # getTypeImpl behavior: turn off recursion
+      if not allowRecursion:  # getTypeInst behavior: return symbol
-        allowRecursion = false
-      else:  # getTypeInst behavior: return symbol
         return atomicType(t.sym)
+      #else:  # getTypeImpl behavior: turn off recursion
+      #  allowRecursion = false
 
   case t.kind
   of tyNone: result = atomicType("none", mNone)
@@ -121,22 +121,25 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
     result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     for i in 0 ..< t.len:
       result.add mapTypeToAst(t.sons[i], info)
-  of tyGenericInst, tyAlias:
+  of tyGenericInst:
     if inst:
       if allowRecursion:
         result = mapTypeToAstR(t.lastSon, info)
       else:
         result = newNodeX(nkBracketExpr)
-        result.add mapTypeToAst(t.lastSon, info)
+        #result.add mapTypeToAst(t.lastSon, info)
+        result.add mapTypeToAst(t[0], info)
         for i in 1 ..< t.len-1:
           result.add mapTypeToAst(t.sons[i], info)
     else:
       result = mapTypeToAstX(t.lastSon, info, inst, allowRecursion)
   of tyGenericBody:
     if inst:
-      result = mapTypeToAstX(t.lastSon, info, inst, true)
+      result = mapTypeToAstR(t.lastSon, info)
     else:
       result = mapTypeToAst(t.lastSon, info)
+  of tyAlias:
+    result = mapTypeToAstX(t.lastSon, info, inst, allowRecursion)
   of tyOrdinal:
     result = mapTypeToAst(t.lastSon, info)
   of tyDistinct:

compiler/vmgen.nim

@@ -1130,6 +1130,8 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
       # produces a value
     else:
       globalError(n.info, "expandToAst requires a call expression")
+  of mRunnableExamples:
+    discard "just ignore any call to runnableExamples"
   else:
     # mGCref, mGCunref,
     globalError(n.info, "cannot generate code for: " & $m)

config/nimdoc.cfg

@@ -88,10 +88,58 @@ doc.body_toc = """
 </div>
 """
 
+@if boot:
+# This is enabled with the "boot" directive to generate
+# the compiler documentation.
+# As a user, tweak the block below instead.
+# You can add your own global-links entries
 doc.body_toc_group = """
 <div class="row">
   <div class="three columns">
-  <div>
+  <div id="global-links">
+    <ul class="simple">
+      <li>
+        <a href="manual.html">Manual</a>
+      </li>
+      <li>
+        <a href="lib.html">Standard library</a>
+      </li>
+      <li>
+        <a href="theindex.html">Index</a>
+      </li>
+    </ul>
+  </div>
+  <div id="searchInput">
+    Search: <input type="text" id="searchInput"
+      onkeyup="search()" />
+  </div>
+  <div class="search-groupby">
+    Group by:
+    <select onchange="groupBy(this.value)">
+      <option value="section">Section</option>
+      <option value="type">Type</option>
+    </select>
+  </div>
+  $tableofcontents
+  </div>
+  <div class="nine columns" id="content">
+  <div id="tocRoot"></div>
+  <p class="module-desc">$moduledesc</p>
+  $content
+  </div>
+</div>
+"""
+
+@else
+
+doc.body_toc_group = """
+<div class="row">
+  <div class="three columns">
+  <div id="global-links">
+    <ul class="simple">
+    </ul>
+  </div>
+  <div id="searchInput">
     Search: <input type="text" id="searchInput"
       onkeyup="search()" />
   </div>
@@ -111,6 +159,7 @@ doc.body_toc_group = """
   </div>
 </div>
 """
+@end
 
 doc.body_no_toc = """
 $moduledesc
@@ -135,7 +184,7 @@ doc.file = """<?xml version="1.0" encoding="utf-8" ?>
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 
 <!-- Google fonts -->
-<link href='https://fonts.googleapis.com/css?family=Raleway:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
 <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 
 <!-- CSS -->
@@ -168,18 +217,19 @@ html {
 
 /* Where we want fancier font if available */
 h1, h2, h3, h4, h5, h6, p.module-desc, table.docinfo + blockquote p, table.docinfo blockquote p, h1 + blockquote p {
-  font-family: "Raleway", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif !important; }
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif !important; }
 
 h1.title {
   font-weight: 900; }
 
 body {
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif;
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif;
   font-weight: 400;
-  font-size: 14px;
+  font-size: 16px;
   line-height: 20px;
-  color: #666;
+  color: #444;
-  background-color: rgba(252, 248, 244, 0.75); }
+  letter-spacing: 0.15px;
+  background-color: rgba(252, 248, 244, 0.45); }
 
 /* Skeleton grid */
 .container {
@@ -295,8 +345,8 @@ cite {
   font-style: italic !important; }
 
 dt > pre {
-  border-color: rgba(0, 0, 0, 0.15);
+  border-color: rgba(0, 0, 0, 0.1);
-  background-color: transparent;
+  background-color: rgba(255, 255, 255, 0.3);
   margin: 15px 0px 5px; }
 
 dd > pre {
@@ -313,6 +363,17 @@ dd > pre {
   width: 100%;
   table-layout: fixed; }
 
+/* Nim search input */
+div#searchInput {
+  margin-bottom: 8px;
+}
+div#searchInput input#searchInput {
+  width: 10em;
+}
+div.search-groupby {
+  margin-bottom: 8px;
+}
+
 table.line-nums-table {
   border-radius: 4px;
   border: 1px solid #cccccc;
@@ -456,7 +517,7 @@ img {
   box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); }
 
 p {
-  margin: 0 0 12px; }
+  margin: 0 0 8px; }
 
 small {
   font-size: 85%; }
@@ -476,7 +537,7 @@ h3,
 h4,
 h5,
 h6 {
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif;
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif;
   font-weight: 600;
   line-height: 20px;
   color: inherit;
@@ -484,6 +545,7 @@ h6 {
 
 h1 {
   font-size: 2em;
+  font-weight: 400;
   padding-bottom: .15em;
   border-bottom: 1px solid #aaaaaa;
   margin-top: 1.0em;
@@ -614,13 +676,13 @@ pre {
   box-sizing: border-box;
   min-width: calc(100% - 19.5px);
   padding: 9.5px;
-  margin: 0.25em 10px 0.25em 10px;
+  margin: 0.25em 10px 10px 10px;
-  font-size: 14px;
+  font-size: 15px;
   line-height: 20px;
   white-space: pre !important;
   overflow-y: hidden;
   overflow-x: visible;
-  background-color: whitesmoke;
+  background-color: rgba(0, 0, 0, 0.01);
   border: 1px solid #cccccc;
   -webkit-border-radius: 4px;
   -moz-border-radius: 4px;
@@ -899,14 +961,14 @@ div.admonition p.admonition-title, div.hint p.admonition-title,
 div.important p.admonition-title, div.note p.admonition-title,
 div.tip p.admonition-title {
   font-weight: bold;
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; }
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; }
 
 div.attention p.admonition-title, div.caution p.admonition-title,
 div.danger p.admonition-title, div.error p.admonition-title,
 div.warning p.admonition-title, .code .error {
   color: #b30000;
   font-weight: bold;
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; }
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; }
 
 /* Uncomment (and remove this text!) to get reduced vertical space in
    compound paragraphs.
@@ -953,7 +1015,7 @@ div.sidebar {
   clear: right; }
 
 div.sidebar p.rubric {
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif;
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif;
   font-size: medium; }
 
 div.system-messages {
@@ -1060,12 +1122,12 @@ p.rubric {
   text-align: center; }
 
 p.sidebar-title {
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif;
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif;
   font-weight: bold;
   font-size: larger; }
 
 p.sidebar-subtitle {
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif;
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif;
   font-weight: bold; }
 
 p.topic-title {
@@ -1107,15 +1169,15 @@ pre.code .inserted, code .inserted {
   background-color: #A3D289; }
 
 span.classifier {
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif;
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif;
   font-style: oblique; }
 
 span.classifier-delimiter {
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif;
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif;
   font-weight: bold; }
 
 span.interpreted {
-  font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; }
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; }
 
 span.option {
   white-space: nowrap; }
@@ -1138,7 +1200,7 @@ table.docinfo {
   margin: 0em;
   margin-top: 2em;
   margin-bottom: 2em;
-  font-family: "Raleway", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif !important;
+  font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif !important;
   color: #444444; }
 
 table.docutils {
@@ -1268,15 +1330,15 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator
   background-repeat: no-repeat;
   background-image: url("data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA==");
   margin-bottom: -5px; }
-  div.pragma {
+div.pragma {
-    display: none;
+  display: none;
-  }
+}
-  span.pragmabegin {
+span.pragmabegin {
-    cursor: pointer;
+  cursor: pointer;
-  }
+}
-  span.pragmaend {
+span.pragmaend {
-    cursor: pointer;
+  cursor: pointer;
-  }
+}
 
 div.search_results {
   background-color: antiquewhite;
@@ -1284,6 +1346,11 @@ div.search_results {
   padding: 1em;
   border: 1px solid #4d4d4d;
 }
+
+div#global-links ul {
+  margin-left: 0;
+  list-style-type: none;
+}
 </style>
 
 <script type="text/javascript" src="../dochack.js"></script>

doc/advopt.txt

@@ -79,6 +79,7 @@ Advanced options:
                             symbol matching is fuzzy so
                             that --dynlibOverride:lua matches
                             dynlib: "liblua.so.3"
+  --dynlibOverrideAll       makes the dynlib pragma have no effect
   --listCmd                 list the commands used to execute external programs
   --parallelBuild:0|1|...   perform a parallel build
                             value = number of processors (0 for auto-detect)

doc/manual/special_ops.txt

@@ -17,8 +17,8 @@ or dynamic file formats such as JSON or XML.
 When Nim encounters an expression that cannot be resolved by the
 standard overload resolution rules, the current scope will be searched
 for a dot operator that can be matched against a re-written form of
-the expression, where the unknown field or proc name is converted to
+the expression, where the unknown field or proc name is passed to
-an additional static string parameter:
+an ``untyped`` parameter:
 
 .. code-block:: nim
   a.b # becomes `.`(a, "b")
@@ -28,7 +28,7 @@ The matched dot operators can be symbols of any callable kind (procs,
 templates and macros), depending on the desired effect:
 
 .. code-block:: nim
-  proc `.` (js: PJsonNode, field: string): JSON = js[field]
+  template `.` (js: PJsonNode, field: untyped): JSON = js[astToStr(field)]
 
   var js = parseJson("{ x: 1, y: 2}")
   echo js.x # outputs 1

doc/tut1.rst

@@ -30,6 +30,7 @@ The first program
 We start the tour with a modified "hello world" program:
 
 .. code-block:: Nim
+    :test: "nim c $1"
   # This is a comment
   echo "What's your name? "
   var name: string = readLine(stdin)
@@ -72,6 +73,7 @@ you can leave out the type in the declaration (this is called `local type
 inference`:idx:). So this will work too:
 
 .. code-block:: Nim
+    :test: "nim c $1"
   var name = readLine(stdin)
 
 Note that this is basically the only form of type inference that exists in
@@ -116,6 +118,7 @@ Comments start anywhere outside a string or character literal with the
 hash character ``#``. Documentation comments start with ``##``:
 
 .. code-block:: nim
+    :test: "nim c $1"
   # A comment.
 
   var myVariable: int ## a documentation comment
@@ -129,6 +132,7 @@ Multiline comments are started with ``#[`` and terminated with ``]#``.  Multilin
 comments can also be nested.
 
 .. code-block:: nim
+    :test: "nim c $1"
   #[
   You can have any Nim code text commented
   out inside this with no indentation restrictions.
@@ -142,6 +146,7 @@ You can also use the `discard statement <#procedures-discard-statement>`_ togeth
 literals* to create block comments:
 
 .. code-block:: nim
+    :test: "nim c $1"
   discard """ You can have any Nim code text commented
   out inside this with no indentation restrictions.
         yes("May I ask a pointless question?") """
@@ -169,6 +174,7 @@ Indentation can be used after the ``var`` keyword to list a whole section of
 variables:
 
 .. code-block::
+    :test: "nim c $1"
   var
     x, y: int
     # a comment can occur here too
@@ -190,6 +196,7 @@ overloaded. You can declare multiple variables with a single assignment
 statement and all the variables will have the same value:
 
 .. code-block::
+    :test: "nim c $1"
   var x, y = 3  # assigns 3 to the variables `x` and `y`
   echo "x ", x  # outputs "x 3"
   echo "y ", y  # outputs "y 3"
@@ -212,12 +219,14 @@ cannot change. The compiler must be able to evaluate the expression in a
 constant declaration at compile time:
 
 .. code-block:: nim
+    :test: "nim c $1"
   const x = "abc" # the constant x contains the string "abc"
 
 Indentation can be used after the ``const`` keyword to list a whole section of
 constants:
 
 .. code-block::
+    :test: "nim c $1"
   const
     x = 1
     # a comment can occur here too
@@ -243,6 +252,7 @@ and put it into a data section":
   const input = readLine(stdin) # Error: constant expression expected
 
 .. code-block::
+    :test: "nim c $1"
   let input = readLine(stdin)   # works
 
 
@@ -260,6 +270,7 @@ If statement
 The if statement is one way to branch the control flow:
 
 .. code-block:: nim
+    :test: "nim c $1"
   let name = readLine(stdin)
   if name == "":
     echo "Poor soul, you lost your name?"
@@ -281,6 +292,7 @@ Another way to branch is provided by the case statement. A case statement is
 a multi-branch:
 
 .. code-block:: nim
+    :test: "nim c $1"
   let name = readLine(stdin)
   case name
   of "":
@@ -338,6 +350,7 @@ While statement
 The while statement is a simple looping construct:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   echo "What's your name? "
   var name = readLine(stdin)
@@ -358,6 +371,7 @@ provides. The example uses the built-in `countup <system.html#countup>`_
 iterator:
 
 .. code-block:: nim
+    :test: "nim c $1"
   echo "Counting to ten: "
   for i in countup(1, 10):
     echo i
@@ -409,6 +423,7 @@ Other useful iterators for collections (like arrays and sequences) are
 * ``pairs`` and ``mpairs`` which provides the element and an index number (immutable and mutable respectively)
 
 .. code-block:: nim
+    :test: "nim c $1"
   for index, item in ["a","b"].pairs:
     echo item, " at index ", index
   # => a at index 0
@@ -421,6 +436,8 @@ new scope. This means that in the following example, ``x`` is not accessible
 outside the loop:
 
 .. code-block:: nim
+    :test: "nim c $1"
+    :status: 1
   while false:
     var x = "hi"
   echo x # does not work
@@ -430,6 +447,8 @@ are only visible within the block they have been declared. The ``block``
 statement can be used to open a new block explicitly:
 
 .. code-block:: nim
+    :test: "nim c $1"
+    :status: 1
   block myblock:
     var x = "hi"
   echo x # does not work either
@@ -444,6 +463,7 @@ can leave a ``while``, ``for``, or a ``block`` statement. It leaves the
 innermost construct, unless a label of a block is given:
 
 .. code-block:: nim
+    :test: "nim c $1"
   block myblock:
     echo "entering block"
     while true:
@@ -465,6 +485,7 @@ Like in many other programming languages, a ``continue`` statement starts
 the next iteration immediately:
 
 .. code-block:: nim
+    :test: "nim c $1"
   while true:
     let x = readLine(stdin)
     if x == "": continue
@@ -477,6 +498,7 @@ When statement
 Example:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   when system.hostOS == "windows":
     echo "running on Windows!"
@@ -549,6 +571,7 @@ an expression is allowed:
 
 .. code-block:: nim
   # computes fac(4) at compile time:
+    :test: "nim c $1"
   const fac4 = (var x = 1; for i in 1..4: x *= i; x)
 
 
@@ -561,6 +584,7 @@ is needed. (Some languages call them *methods* or *functions*.) In Nim new
 procedures are defined with the ``proc`` keyword:
 
 .. code-block:: nim
+    :test: "nim c $1"
   proc yes(question: string): bool =
     echo question, " (y/n)"
     while true:
@@ -597,6 +621,7 @@ automatically at the end of a procedure if there is no ``return`` statement at
 the exit.
 
 .. code-block:: nim
+    :test: "nim c $1"
   proc sumTillNegative(x: varargs[int]): int =
     for i in x:
       if i < 0:
@@ -624,6 +649,7 @@ to be declared with ``var`` in the procedure body. Shadowing the parameter name
 is possible, and actually an idiom:
 
 .. code-block:: nim
+    :test: "nim c $1"
   proc printSeq(s: seq, nprinted: int = -1) =
     var nprinted = if nprinted == -1: s.len else: min(nprinted, s.len)
     for i in 0 .. <nprinted:
@@ -633,6 +659,7 @@ If the procedure needs to modify the argument for the
 caller, a ``var`` parameter can be used:
 
 .. code-block:: nim
+    :test: "nim c $1"
   proc divmod(a, b: int; res, remainder: var int) =
     res = a div b        # integer division
     remainder = a mod b  # integer modulo operation
@@ -663,6 +690,7 @@ The return value can be ignored implicitly if the called proc/iterator has
 been declared with the ``discardable`` pragma:
 
 .. code-block:: nim
+    :test: "nim c $1"
   proc p(x, y: int): int {.discardable.} =
     return x + y
 
@@ -772,6 +800,7 @@ The "``" notation can also be used to call an operator just like any other
 procedure:
 
 .. code-block:: nim
+    :test: "nim c $1"
   if `==`( `+`(3, 4), 7): echo "True"
 
 
@@ -819,6 +848,7 @@ Iterators
 Let's return to the simple counting example:
 
 .. code-block:: nim
+    :test: "nim c $1"
   echo "Counting to ten: "
   for i in countup(1, 10):
     echo i
@@ -840,6 +870,7 @@ the only thing left to do is to replace the ``proc`` keyword by ``iterator``
 and here it is - our first iterator:
 
 .. code-block:: nim
+    :test: "nim c $1"
   iterator countup(a, b: int): int =
     var res = a
     while res <= b:
@@ -894,7 +925,8 @@ evaluation. For example:
 Characters
 ----------
 The `character type` is called ``char``. Its size is always one byte, so
-it cannot represent most UTF-8 characters; but it *can* represent one of the bytes that makes up a multi-byte UTF-8 character.
+it cannot represent most UTF-8 characters; but it *can* represent one of the bytes
+that makes up a multi-byte UTF-8 character.
 The reason for this is efficiency: for the overwhelming majority of use-cases,
 the resulting programs will still handle UTF-8 properly as UTF-8 was specially
 designed for this.
@@ -945,6 +977,7 @@ to specify a non-default integer type:
 
 
 .. code-block:: nim
+    :test: "nim c $1"
   let
     x = 0     # x is of type ``int``
     y = 0'i8  # y is of type ``int8``
@@ -981,6 +1014,7 @@ Float literals can have a *type suffix* to specify a non-default float
 type:
 
 .. code-block:: nim
+    :test: "nim c $1"
   var
     x = 0.0      # x is of type ``float``
     y = 0.0'f32  # y is of type ``float32``
@@ -1002,6 +1036,7 @@ Conversion between basic types is performed by using the
 type as a function:
 
 .. code-block:: nim
+    :test: "nim c $1"
   var
     x: int32 = 1.int32   # same as calling int32(1)
     y: int8  = int8('a') # 'a' == 97'i8
@@ -1023,6 +1058,7 @@ graphs with cycles. The following example shows that even for basic types
 there is a difference between the ``$`` and ``repr`` outputs:
 
 .. code-block:: nim
+    :test: "nim c $1"
   var
     myBool = true
     myCharacter = 'n'
@@ -1047,6 +1083,7 @@ Advanced types
 In Nim new types can be defined within a ``type`` statement:
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
     biggestInt = int64      # biggest integer type that is available
     biggestFloat = float64  # biggest float type that is available
@@ -1063,6 +1100,7 @@ to an integer value internally. The first symbol is represented
 at runtime by 0, the second by 1 and so on. For example:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   type
     Direction = enum
@@ -1087,6 +1125,7 @@ explicitly given is assigned the value of the previous symbol + 1.
 An explicit ordered enum can have *holes*:
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
     MyEnum = enum
       a = 2, b = 4, c = 89
@@ -1125,6 +1164,7 @@ A subrange type is a range of values from an integer or enumeration type
 (the base type). Example:
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
     MySubrange = range[0..5]
 
@@ -1155,6 +1195,7 @@ an array has the same type. The array's index type can be any ordinal type.
 Arrays can be constructed using ``[]``:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   type
     IntArray = array[0..5, int] # an array that is indexed with 0..5
@@ -1177,6 +1218,7 @@ length. `low(a) <system.html#low>`_ returns the lowest valid index for the
 array `a` and `high(a) <system.html#high>`_ the highest valid index.
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
     Direction = enum
       north, east, south, west
@@ -1228,6 +1270,7 @@ It is quite common to have arrays start at zero, so there's a shortcut syntax
 to specify a range from zero to the specified index minus one:
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
     IntArray = array[0..5, int] # an array that is indexed with 0..5
     QuickArray = array[6, int]  # an array that is indexed with 0..5
@@ -1260,6 +1303,7 @@ A sequence may be passed to an openarray parameter.
 Example:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   var
     x: seq[int] # a reference to a sequence of integers
@@ -1282,6 +1326,7 @@ value. Here the ``for`` statement is looping over the results from the
 <system.html>`_ module.  Examples:
 
 .. code-block:: nim
+    :test: "nim c $1"
   for value in @[3, 4, 5]:
     echo value
   # --> 3
@@ -1308,6 +1353,7 @@ with a compatible base type can be passed to an openarray parameter, the index
 type does not matter.
 
 .. code-block:: nim
+    :test: "nim c $1"
   var
     fruits:   seq[string]       # reference to a sequence of strings that is initialized with 'nil'
     capitals: array[3, string]  # array of strings with a fixed size
@@ -1337,6 +1383,7 @@ arguments to a procedure. The compiler converts the list of arguments
 to an array automatically:
 
 .. code-block:: nim
+    :test: "nim c $1"
   proc myWriteln(f: File, a: varargs[string]) =
     for s in items(a):
       write(f, s)
@@ -1351,6 +1398,7 @@ last parameter in the procedure header. It is also possible to perform
 type conversions in this context:
 
 .. code-block:: nim
+    :test: "nim c $1"
   proc myWriteln(f: File, a: varargs[string, `$`]) =
     for s in items(a):
       write(f, s)
@@ -1374,6 +1422,7 @@ context. A slice is just an object of type Slice which contains two bounds,
 define operators which accept Slice objects to define ranges.
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   var
     a = "Nim is a progamming language"
@@ -1388,27 +1437,31 @@ slice's bounds can hold any value supported by
 their type, but it is the proc using the slice object which defines what values
 are accepted.
 
- To understand some of the different ways of specifying the indices of strings, arrays, sequences, etc.,
+To understand some of the different ways of specifying the indices of
- it must be remembered that Nim uses zero-based indices.
+strings, arrays, sequences, etc., it must be remembered that Nim uses
+zero-based indices.
 
- So the string ``b`` is of length 19, and two different ways of specifying the indices are
+So the string ``b`` is of length 19, and two different ways of specifying the
+indices are
 
- .. code-block:: nim
+.. code-block:: nim
 
   "Slices are useless."
    |          |     |
    0         11    17   using indices
   ^19        ^8    ^2   using ^ syntax
 
- where ``b[0..^1]`` is equivalent to ``b[0..b.len-1]`` and ``b[0..<b.len]``, and it can be seen that the ``^1`` provides a short-hand way of specifying the ``b.len-1``
+where ``b[0..^1]`` is equivalent to ``b[0..b.len-1]`` and ``b[0..<b.len]``, and it
+can be seen that the ``^1`` provides a short-hand way of specifying the ``b.len-1``.
 
- In the above example, because the string ends in a period, to get the portion of the string that is "useless" and replace it with "useful"
+In the above example, because the string ends in a period, to get the portion of the
+string that is "useless" and replace it with "useful".
 
- ``b[11..^2]`` is the portion "useless", and
+``b[11..^2]`` is the portion "useless", and ``b[11..^2] = "useful"`` replaces the
- ``b[11..^2] = "useful"`` replaces the "useless" portion with "useful",
+"useless" portion with "useful", giving the result "Slices are useful."
- giving the result "Slices are useful."
 
- Note: alternate ways of writing this are ``b[^8..^2] = "useful"`` or as ``b[11..b.len-2] = "useful"`` or as ``b[11..<b.len-1] = "useful"`` or as ....
+Note: alternate ways of writing this are ``b[^8..^2] = "useful"`` or
+as ``b[11..b.len-2] = "useful"`` or as ``b[11..<b.len-1] = "useful"``.
 
 Tuples
 ------
@@ -1425,6 +1478,7 @@ The assignment operator for tuples copies each component. The notation
 integer.
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   type
     Person = tuple[name: string, age: int] # type representing a person:
@@ -1470,6 +1524,7 @@ otherwise you will be assigning the same value to all the individual
 variables! For example:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   import os
 
@@ -1509,6 +1564,7 @@ tuple/object field operator) and ``[]`` (array/string/sequence index operator)
 operators perform implicit dereferencing operations for reference types:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   type
     Node = ref object
@@ -1538,6 +1594,7 @@ techniques.
 Example:
 
 .. code-block:: nim
+    :test: "nim c $1"
   proc echoItem(x: int) = echo x
 
   proc forEach(action: proc (x: int)) =
@@ -1555,9 +1612,11 @@ listed in the `manual <manual.html#types-procedural-type>`_.
 
 Distinct type
 -------------
-A Distinct type allows for the creation of new type that "does not imply a subtype relationship between it and its base type".
+A Distinct type allows for the creation of new type that "does not imply a
+subtype relationship between it and its base type".
 You must **explicitly** define all behaviour for the distinct type.
-To help with this, both the distinct type and its base type can cast from one type to the other.
+To help with this, both the distinct type and its base type can cast from one
+type to the other.
 Examples are provided in the `manual <manual.html#types-distinct-type>`_.
 
 Modules
@@ -1592,39 +1651,6 @@ Each module has a special magic constant ``isMainModule`` that is true if the
 module is compiled as the main file. This is very useful to embed tests within
 the module as shown by the above example.
 
-Modules that depend on each other are possible, but strongly discouraged,
-because then one module cannot be reused without the other.
-
-The algorithm for compiling modules is:
-
-- Compile the whole module as usual, following import statements recursively.
-- If there is a cycle only import the already parsed symbols (that are
-  exported); if an unknown identifier occurs then abort.
-
-This is best illustrated by an example:
-
-.. code-block:: nim
-  # Module A
-  type
-    T1* = int  # Module A exports the type ``T1``
-  import B     # the compiler starts parsing B
-
-  proc main() =
-    var i = p(3) # works because B has been parsed completely here
-
-  main()
-
-.. code-block:: nim
-  # Module B
-  import A  # A is not parsed here! Only the already known symbols
-            # of A are imported.
-
-  proc p*(x: A.T1): A.T1 =
-    # this works because the compiler has already
-    # added T1 to A's interface symbol table
-    result = x + 1
-
-
 A symbol of a module *can* be *qualified* with the ``module.symbol`` syntax. And if
 a symbol is ambiguous, it *must* be qualified. A symbol is ambiguous
 if it is defined in two (or more) different modules and both modules are

doc/tut2.rst

@@ -55,6 +55,7 @@ Objects have access to their type at runtime. There is an
 ``of`` operator that can be used to check the object's type:
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
     Person = ref object of RootObj
       name*: string  # the * means that `name` is accessible from other modules
@@ -103,6 +104,7 @@ would require arbitrary symbol lookahead which slows down compilation.)
 Example:
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
     Node = ref object  # a reference to an object with the following field:
       le, ri: Node     # left and right subtrees
@@ -144,6 +146,7 @@ variant types are needed.
 An example:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   # This is an example how an abstract syntax tree could be modelled in Nim
   type
@@ -201,9 +204,11 @@ This method call syntax is not restricted to objects, it can be used
 for any type:
 
 .. code-block:: nim
+    :test: "nim c $1"
+  import strutils
 
   echo "abc".len # is the same as echo len("abc")
-  echo "abc".toUpper()
+  echo "abc".toUpperAscii()
   echo({'a', 'b', 'c'}.card)
   stdout.writeLine("Hallo") # the same as writeLine(stdout, "Hallo")
 
@@ -213,6 +218,7 @@ postfix notation.)
 So "pure object oriented" code is easy to write:
 
 .. code-block:: nim
+    :test: "nim c $1"
   import strutils, sequtils
 
   stdout.writeLine("Give a list of numbers (separated by spaces): ")
@@ -228,6 +234,7 @@ the same. But setting a value is different; for this a special setter syntax
 is needed:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   type
     Socket* = ref object of RootObj
@@ -252,6 +259,7 @@ The ``[]`` array access operator can be overloaded to provide
 `array properties`:idx:\ :
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
     Vector* = object
       x, y, z: float
@@ -283,23 +291,24 @@ Procedures always use static dispatch. For dynamic dispatch replace the
 ``proc`` keyword by ``method``:
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
-    PExpr = ref object of RootObj ## abstract base class for an expression
+    Expression = ref object of RootObj ## abstract base class for an expression
-    PLiteral = ref object of PExpr
+    Literal = ref object of Expression
       x: int
-    PPlusExpr = ref object of PExpr
+    PlusExpr = ref object of Expression
-      a, b: PExpr
+      a, b: Expression
 
   # watch out: 'eval' relies on dynamic binding
-  method eval(e: PExpr): int =
+  method eval(e: Expression): int =
     # override this base method
     quit "to override!"
 
-  method eval(e: PLiteral): int = e.x
+  method eval(e: Literal): int = e.x
-  method eval(e: PPlusExpr): int = eval(e.a) + eval(e.b)
+  method eval(e: PlusExpr): int = eval(e.a) + eval(e.b)
 
-  proc newLit(x: int): PLiteral = PLiteral(x: x)
+  proc newLit(x: int): Literal = Literal(x: x)
-  proc newPlus(a, b: PExpr): PPlusExpr = PPlusExpr(a: a, b: b)
+  proc newPlus(a, b: Expression): PlusExpr = PlusExpr(a: a, b: b)
 
   echo eval(newPlus(newPlus(newLit(1), newLit(2)), newLit(4)))
 
@@ -311,6 +320,7 @@ In a multi-method all parameters that have an object type are used for the
 dispatching:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   type
     Thing = ref object of RootObj
@@ -365,6 +375,7 @@ Raise statement
 Raising an exception is done with the ``raise`` statement:
 
 .. code-block:: nim
+    :test: "nim c $1"
   var
     e: ref OSError
   new(e)
@@ -385,6 +396,9 @@ Try statement
 The ``try`` statement handles exceptions:
 
 .. code-block:: nim
+    :test: "nim c $1"
+  from strutils import parseInt
+
   # read the first two lines of a text file that should contain numbers
   # and tries to add them
   var
@@ -479,6 +493,7 @@ with `type parameters`:idx:. They are most useful for efficient type safe
 containers:
 
 .. code-block:: nim
+    :test: "nim c $1"
   type
     BinaryTree*[T] = ref object # BinaryTree is a generic type with
                                 # generic param ``T``
@@ -573,6 +588,7 @@ Templates are especially useful for lazy evaluation purposes. Consider a
 simple proc for logging:
 
 .. code-block:: nim
+    :test: "nim c $1"
   const
     debug = true
 
@@ -590,6 +606,7 @@ evaluation for procedures is *eager*).
 Turning the ``log`` proc into a template solves this problem:
 
 .. code-block:: nim
+    :test: "nim c $1"
   const
     debug = true
 
@@ -611,6 +628,7 @@ If the template has no explicit return type,
 To pass a block of statements to a template, use 'untyped' for the last parameter:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   template withFile(f: untyped, filename: string, mode: FileMode,
                     body: untyped): typed =
@@ -665,6 +683,7 @@ The following example implements a powerful ``debug`` command that accepts a
 variable number of arguments:
 
 .. code-block:: nim
+    :test: "nim c $1"
   # to work with Nim syntax trees, we need an API that is defined in the
   # ``macros`` module:
   import macros
@@ -744,6 +763,7 @@ dynamic code into something that compiles statically. For the exercise we will
 use the following snippet of code as the starting point:
 
 .. code-block:: nim
+    :test: "nim c $1"
 
   import strutils, tables
 
@@ -863,9 +883,9 @@ variables with ``cfg``. In essence, what the compiler is doing is replacing
 the line calling the macro with the following snippet of code:
 
 .. code-block:: nim
-  const cfgversion= "1.1"
+  const cfgversion = "1.1"
-  const cfglicenseOwner= "Hyori Lee"
+  const cfglicenseOwner = "Hyori Lee"
-  const cfglicenseKey= "M1Tl3PjBWO2CC48m"
+  const cfglicenseKey = "M1Tl3PjBWO2CC48m"
 
 You can verify this yourself adding the line ``echo source`` somewhere at the
 end of the macro and compiling the program. Another difference is that instead
@@ -891,12 +911,13 @@ an expression macro.  Since we know that we want to generate a bunch of
 see what the compiler *expects* from us:
 
 .. code-block:: nim
+    :test: "nim c $1"
   import macros
 
   dumpTree:
     const cfgversion: string = "1.1"
-    const cfglicenseOwner= "Hyori Lee"
+    const cfglicenseOwner = "Hyori Lee"
-    const cfglicenseKey= "M1Tl3PjBWO2CC48m"
+    const cfglicenseKey = "M1Tl3PjBWO2CC48m"
 
 During compilation of the source code we should see the following lines in the
 output (again, since this is a macro, compilation is enough, you don't have to
@@ -996,6 +1017,7 @@ Lifting Procs
 +++++++++++++
 
 .. code-block:: nim
+    :test: "nim c $1"
   import math
 
   template liftScalarProc(fname) =

lib/core/macros.nim

@@ -1226,3 +1226,8 @@ when not defined(booting):
     macro payload: untyped {.gensym.} =
       result = parseStmt(e)
     payload()
+
+macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped =
+  result = newCall(callee)
+  for i in 0 ..< args.len:
+    result.add args[i]

lib/impure/re.nim

@@ -49,9 +49,6 @@ type
   RegexError* = object of ValueError
     ## is raised if the pattern is no valid regular expression.
 
-{.deprecated: [TRegexFlag: RegexFlag, TRegexDesc: RegexDesc, TRegex: Regex,
-    EInvalidRegEx: RegexError].}
-
 proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} =
   var e: ref RegexError
   new(e)
@@ -470,8 +467,8 @@ proc replacef*(s: string, sub: Regex, by: string): string =
     prev = match.last + 1
   add(result, substr(s, prev))
 
-proc parallelReplace*(s: string, subs: openArray[
+proc multiReplace*(s: string, subs: openArray[
-                      tuple[pattern: Regex, repl: string]]): string =
+                   tuple[pattern: Regex, repl: string]]): string =
   ## Returns a modified copy of ``s`` with the substitutions in ``subs``
   ## applied in parallel.
   result = ""
@@ -490,13 +487,20 @@ proc parallelReplace*(s: string, subs: openArray[
   # copy the rest:
   add(result, substr(s, i))
 
+proc parallelReplace*(s: string, subs: openArray[
+                      tuple[pattern: Regex, repl: string]]): string {.deprecated.} =
+  ## Returns a modified copy of ``s`` with the substitutions in ``subs``
+  ## applied in parallel.
+  ## **Deprecated since version 0.18.0**: Use ``multiReplace`` instead.
+  result = multiReplace(s, subs)
+
 proc transformFile*(infile, outfile: string,
                     subs: openArray[tuple[pattern: Regex, repl: string]]) =
   ## reads in the file ``infile``, performs a parallel replacement (calls
   ## ``parallelReplace``) and writes back to ``outfile``. Raises ``IOError`` if an
   ## error occurs. This is supposed to be used for quick scripting.
   var x = readFile(infile).string
-  writeFile(outfile, x.parallelReplace(subs))
+  writeFile(outfile, x.multiReplace(subs))
 
 iterator split*(s: string, sep: Regex): string =
   ## Splits the string ``s`` into substrings.
@@ -579,12 +583,12 @@ const ## common regular expressions
     ## describes an URL
 
 when isMainModule:
-  doAssert match("(a b c)", re"\( .* \)")
+  doAssert match("(a b c)", rex"\( .* \)")
   doAssert match("WHiLe", re("while", {reIgnoreCase}))
 
   doAssert "0158787".match(re"\d+")
   doAssert "ABC 0232".match(re"\w+\s+\d+")
-  doAssert "ABC".match(re"\d+ | \w+")
+  doAssert "ABC".match(rex"\d+ | \w+")
 
   {.push warnings:off.}
   doAssert matchLen("key", re(reIdentifier)) == 3

lib/js/jsffi.nim

@@ -177,7 +177,7 @@ proc `==`*(x, y: JsRoot): bool {. importcpp: "(# === #)" .}
   ## and not strings or numbers, this is a *comparison of references*.
 
 {. experimental .}
-macro `.`*(obj: JsObject, field: static[cstring]): JsObject =
+macro `.`*(obj: JsObject, field: untyped): JsObject =
   ## Experimental dot accessor (get) for type JsObject.
   ## Returns the value of a property of name `field` from a JsObject `x`.
   ##
@@ -196,14 +196,14 @@ macro `.`*(obj: JsObject, field: static[cstring]): JsObject =
       helper(`obj`)
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName(field)
+      mangledNames[$field] = $mangleJsName($field)
     let importString = "#." & mangledNames[$field]
     result = quote do:
       proc helper(o: JsObject): JsObject
         {. importcpp: `importString`, gensym .}
       helper(`obj`)
 
-macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped =
+macro `.=`*(obj: JsObject, field, value: untyped): untyped =
   ## Experimental dot accessor (set) for type JsObject.
   ## Sets the value of a property of name `field` in a JsObject `x` to `value`.
   if validJsName($field):
@@ -214,7 +214,7 @@ macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped =
       helper(`obj`, `value`)
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName(field)
+      mangledNames[$field] = $mangleJsName($field)
     let importString = "#." & mangledNames[$field] & " = #"
     result = quote do:
       proc helper(o: JsObject, v: auto)
@@ -222,7 +222,7 @@ macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped =
       helper(`obj`, `value`)
 
 macro `.()`*(obj: JsObject,
-             field: static[cstring],
+             field: untyped,
              args: varargs[JsObject, jsFromAst]): JsObject =
   ## Experimental "method call" operator for type JsObject.
   ## Takes the name of a method of the JavaScript object (`field`) and calls
@@ -245,7 +245,7 @@ macro `.()`*(obj: JsObject,
     importString = "#." & $field & "(@)"
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName(field)
+      mangledNames[$field] = $mangleJsName($field)
     importString = "#." & mangledNames[$field] & "(@)"
   result = quote:
     proc helper(o: JsObject): JsObject
@@ -257,7 +257,7 @@ macro `.()`*(obj: JsObject,
     result[1].add args[idx].copyNimTree
 
 macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V],
-                                   field: static[cstring]): V =
+                                   field: untyped): V =
   ## Experimental dot accessor (get) for type JsAssoc.
   ## Returns the value of a property of name `field` from a JsObject `x`.
   var importString: string
@@ -265,7 +265,7 @@ macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V],
     importString = "#." & $field
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName(field)
+      mangledNames[$field] = $mangleJsName($field)
     importString = "#." & mangledNames[$field]
   result = quote do:
     proc helper(o: type(`obj`)): `obj`.V
@@ -273,7 +273,7 @@ macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V],
     helper(`obj`)
 
 macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V],
-                                    field: static[cstring],
+                                    field: untyped,
                                     value: V): untyped =
   ## Experimental dot accessor (set) for type JsAssoc.
   ## Sets the value of a property of name `field` in a JsObject `x` to `value`.
@@ -282,7 +282,7 @@ macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V],
     importString = "#." & $field & " = #"
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName(field)
+      mangledNames[$field] = $mangleJsName($field)
     importString = "#." & mangledNames[$field] & " = #"
   result = quote do:
     proc helper(o: type(`obj`), v: `obj`.V)
@@ -290,7 +290,7 @@ macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V],
     helper(`obj`, `value`)
 
 macro `.()`*[K: string | cstring, V: proc](obj: JsAssoc[K, V],
-                                           field: static[cstring],
+                                           field: untyped,
                                            args: varargs[untyped]): auto =
   ## Experimental "method call" operator for type JsAssoc.
   ## Takes the name of a method of the JavaScript object (`field`) and calls

lib/nimbase.h

@@ -70,7 +70,7 @@ __clang__
 #if defined(_MSC_VER)
 #  pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309)
 #  pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706)
-#  pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090)
+#  pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090 4297)
 #endif
 /* ------------------------------------------------------------------------- */
 

lib/packages/docutils/highlite.nim

@@ -31,14 +31,12 @@ type
     state: TokenClass
 
   SourceLanguage* = enum
-    langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava,
+    langNone, langNim, langCpp, langCsharp, langC, langJava,
     langYaml
-{.deprecated: [TSourceLanguage: SourceLanguage, TTokenClass: TokenClass,
-              TGeneralTokenizer: GeneralTokenizer].}
 
 const
   sourceLanguageToStr*: array[SourceLanguage, string] = ["none",
-    "Nim", "Nimrod", "C++", "C#", "C", "Java", "Yaml"]
+    "Nim", "C++", "C#", "C", "Java", "Yaml"]
   tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace",
     "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber",
     "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit",
@@ -398,7 +396,6 @@ type
   TokenizerFlag = enum
     hasPreprocessor, hasNestedComments
   TokenizerFlags = set[TokenizerFlag]
-{.deprecated: [TTokenizerFlag: TokenizerFlag, TTokenizerFlags: TokenizerFlags].}
 
 proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string],
                     flags: TokenizerFlags) =
@@ -888,7 +885,7 @@ proc yamlNextToken(g: var GeneralTokenizer) =
 proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) =
   case lang
   of langNone: assert false
-  of langNim, langNimrod: nimNextToken(g)
+  of langNim: nimNextToken(g)
   of langCpp: cppNextToken(g)
   of langCsharp: csharpNextToken(g)
   of langC: cNextToken(g)

lib/packages/docutils/rst.nim

@@ -45,8 +45,6 @@ type
   MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind,
                        arg: string) {.nimcall.} ## what to do in case of an error
   FindFileHandler* = proc (filename: string): string {.nimcall.}
-{.deprecated: [TRstParseOptions: RstParseOptions, TRstParseOption: RstParseOption,
-              TMsgKind: MsgKind].}
 
 const
   messages: array[MsgKind, string] = [
@@ -127,8 +125,6 @@ type
     bufpos*: int
     line*, col*, baseIndent*: int
     skipPounds*: bool
-{.deprecated: [TTokType: TokType, TToken: Token, TTokenSeq: TokenSeq,
-              TLexer: Lexer].}
 
 proc getThing(L: var Lexer, tok: var Token, s: set[char]) =
   tok.kind = tkWord
@@ -288,10 +284,6 @@ type
     hasToc*: bool
 
   EParseError* = object of ValueError
-{.deprecated: [TLevelMap: LevelMap, TSubstitution: Substitution,
-              TSharedState: SharedState, TRstParser: RstParser,
-              TMsgHandler: MsgHandler, TFindFileHandler: FindFileHandler,
-              TMsgClass: MsgClass].}
 
 proc whichMsgClass*(k: MsgKind): MsgClass =
   ## returns which message class `k` belongs to.
@@ -341,11 +333,6 @@ proc rstMessage(p: RstParser, msgKind: MsgKind) =
                              p.col + p.tok[p.idx].col, msgKind,
                              p.tok[p.idx].symbol)
 
-when false:
-  proc corrupt(p: RstParser) =
-    assert p.indentStack[0] == 0
-    for i in 1 .. high(p.indentStack): assert p.indentStack[i] < 1_000
-
 proc currInd(p: RstParser): int =
   result = p.indentStack[high(p.indentStack)]
 

lib/packages/docutils/rstgen.nim

@@ -46,7 +46,7 @@ type
     target*: OutputTarget
     config*: StringTableRef
     splitAfter*: int          # split too long entries in the TOC
-    listingCounter: int
+    listingCounter*: int
     tocPart*: seq[TocEntry]
     hasToc*: bool
     theIndex: string # Contents of the index file to be dumped at the end.
@@ -61,6 +61,9 @@ type
     seenIndexTerms: Table[string, int] ## \
     ## Keeps count of same text index terms to generate different identifiers
     ## for hyperlinks. See renderIndexTerm proc for details.
+    id*: int               ## A counter useful for generating IDs.
+    onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int;
+                          content: string)
 
   PDoc = var RstGenerator ## Alias to type less.
 
@@ -69,8 +72,9 @@ type
     startLine: int ## The starting line of the code block, by default 1.
     langStr: string ## Input string used to specify the language.
     lang: SourceLanguage ## Type of highlighting, by default none.
-{.deprecated: [TRstGenerator: RstGenerator, TTocEntry: TocEntry,
+    filename: string
-              TOutputTarget: OutputTarget, TMetaEnum: MetaEnum].}
+    testCmd: string
+    status: int
 
 proc init(p: var CodeBlockParams) =
   ## Default initialisation of CodeBlockParams to sane values.
@@ -133,6 +137,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
   g.options = options
   g.findFile = findFile
   g.currentSection = ""
+  g.id = 0
   let fileParts = filename.splitFile
   if fileParts.ext == ".nim":
     g.currentSection = "Module " & fileParts.name
@@ -368,7 +373,6 @@ type
     ##
     ## The value indexed by this IndexEntry is a sequence with the real index
     ## entries found in the ``.idx`` file.
-{.deprecated: [TIndexEntry: IndexEntry, TIndexedDocs: IndexedDocs].}
 
 proc cmp(a, b: IndexEntry): int =
   ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`.
@@ -823,13 +827,20 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
     var number: int
     if parseInt(n.getFieldValue, number) > 0:
       params.startLine = number
-  of "file":
+  of "file", "filename":
     # The ``file`` option is a Nim extension to the official spec, it acts
     # like it would for other directives like ``raw`` or ``cvs-table``. This
     # field is dealt with in ``rst.nim`` which replaces the existing block with
     # the referenced file, so we only need to ignore it here to avoid incorrect
     # warning messages.
-    discard
+    params.filename = n.getFieldValue.strip
+  of "test":
+    params.testCmd = n.getFieldValue.strip
+    if params.testCmd.len == 0: params.testCmd = "nim c -r $1"
+  of "status":
+    var status: int
+    if parseInt(n.getFieldValue, status) > 0:
+      params.status = status
   of "default-language":
     params.langStr = n.getFieldValue.strip
     params.lang = params.langStr.getSourceLanguage
@@ -901,6 +912,9 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
   var m = n.sons[2].sons[0]
   assert m.kind == rnLeaf
 
+  if params.testCmd.len > 0 and d.onTestSnippet != nil:
+    d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text)
+
   let (blockStart, blockEnd) = buildLinesHTMLTable(d, params, m.text)
 
   dispA(d.target, result, blockStart, "\\begin{rstpre}\n", [])

lib/pure/asyncdispatch.nim

@@ -59,9 +59,10 @@ export asyncfutures, asyncstreams
 ##
 ##   .. code-block::nim
 ##      var future = socket.recv(100)
-##      future.callback =
+##      future.addCallback(
 ##        proc () =
 ##          echo(future.read)
+##      )
 ##
 ## All asynchronous functions returning a ``Future`` will not block. They
 ## will not however return immediately. An asynchronous function will have

lib/pure/asyncfutures.nim

@@ -334,7 +334,7 @@ proc all*[T](futs: varargs[Future[T]]): auto =
     let totalFutures = len(futs)
 
     for fut in futs:
-      fut.callback = proc(f: Future[T]) =
+      fut.addCallback proc (f: Future[T]) =
         inc(completedFutures)
         if not retFuture.finished:
           if f.failed:
@@ -356,7 +356,7 @@ proc all*[T](futs: varargs[Future[T]]): auto =
 
     for i, fut in futs:
       proc setCallback(i: int) =
-        fut.callback = proc(f: Future[T]) =
+        fut.addCallback proc (f: Future[T]) =
           inc(completedFutures)
           if not retFuture.finished:
             if f.failed:

lib/pure/asynchttpserver.nim

@@ -275,10 +275,7 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string
   lineFut.mget() = newStringOfCap(80)
 
   while not client.isClosed:
-    try:
+    await processRequest(server, request, client, address, lineFut, callback)
-      await processRequest(server, request, client, address, lineFut, callback)
-    except:
-      asyncCheck request.mget().respondError(Http500)
 
 proc serve*(server: AsyncHttpServer, port: Port,
             callback: proc (request: Request): Future[void] {.closure,gcsafe.},

lib/pure/bitops.nim

@@ -181,7 +181,7 @@ elif useICC_builtins:
 
 
 proc countSetBits*(x: SomeInteger): int {.inline, nosideeffect.} =
-  ## Counts the set bits in integer. (also called Hamming weight.)
+  ## Counts the set bits in integer. (also called `Hamming weight`:idx:.)
   # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT.
   # like GCC and MSVC
   when nimvm:

lib/pure/cgi.nim

@@ -29,21 +29,8 @@
 ##    writeLine(stdout, "your password: " & myData["password"])
 ##    writeLine(stdout, "</body></html>")
 
-import strutils, os, strtabs, cookies
+import strutils, os, strtabs, cookies, uri
-
+export uri.encodeUrl, uri.decodeUrl
-proc encodeUrl*(s: string): string =
-  ## Encodes a value to be HTTP safe: This means that characters in the set
-  ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result,
-  ## a space is converted to ``'+'`` and every other character is encoded as
-  ## ``'%xx'`` where ``xx`` denotes its hexadecimal value.
-  result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars
-  for i in 0..s.len-1:
-    case s[i]
-    of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i])
-    of ' ': add(result, '+')
-    else:
-      add(result, '%')
-      add(result, toHex(ord(s[i]), 2))
 
 proc handleHexChar(c: char, x: var int) {.inline.} =
   case c
@@ -52,28 +39,6 @@ proc handleHexChar(c: char, x: var int) {.inline.} =
   of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
   else: assert(false)
 
-proc decodeUrl*(s: string): string =
-  ## Decodes a value from its HTTP representation: This means that a ``'+'``
-  ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal
-  ## value) is converted to the character with ordinal number ``xx``, and
-  ## and every other character is carried over.
-  result = newString(s.len)
-  var i = 0
-  var j = 0
-  while i < s.len:
-    case s[i]
-    of '%':
-      var x = 0
-      handleHexChar(s[i+1], x)
-      handleHexChar(s[i+2], x)
-      inc(i, 2)
-      result[j] = chr(x)
-    of '+': result[j] = ' '
-    else: result[j] = s[i]
-    inc(i)
-    inc(j)
-  setLen(result, j)
-
 proc addXmlChar(dest: var string, c: char) {.inline.} =
   case c
   of '&': add(dest, "&amp;")
@@ -390,8 +355,3 @@ proc existsCookie*(name: string): bool =
   ## Checks if a cookie of `name` exists.
   if gcookies == nil: gcookies = parseCookies(getHttpCookie())
   result = hasKey(gcookies, name)
-
-when isMainModule:
-  const test1 = "abc\L+def xyz"
-  assert encodeUrl(test1) == "abc%0A%2Bdef+xyz"
-  assert decodeUrl(encodeUrl(test1)) == test1

lib/pure/collections/sequtils.nim

@@ -66,7 +66,7 @@ proc cycle*[T](s: openArray[T], n: Natural): seq[T] =
   ##
   ## Example:
   ##
-  ## .. code-block:
+  ## .. code-block::
   ##
   ##   let
   ##     s = @[1, 2, 3]
@@ -84,7 +84,7 @@ proc repeat*[T](x: T, n: Natural): seq[T] =
   ##
   ## Example:
   ##
-  ## .. code-block:
+  ## .. code-block::
   ##
   ##   let
   ##     total = repeat(5, 3)

lib/pure/htmlgen.nim

@@ -59,8 +59,8 @@ proc xmlCheckedTag*(e: NimNode, tag: string, optAttr = "", reqAttr = "",
 
   # copy the attributes; when iterating over them these lists
   # will be modified, so that each attribute is only given one value
-  var req = split(reqAttr)
+  var req = splitWhitespace(reqAttr)
-  var opt = split(optAttr)
+  var opt = splitWhitespace(optAttr)
   result = newNimNode(nnkBracket, e)
   result.add(newStrLitNode("<"))
   result.add(newStrLitNode(tag))

lib/pure/ioselects/ioselectors_select.nim

@@ -279,8 +279,9 @@ proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle,
         inc(s.count)
       pkey.events = events
 
-proc unregister*[T](s: Selector[T], fd: int | SocketHandle) =
+proc unregister*[T](s: Selector[T], fd: SocketHandle|int) =
   s.withSelectLock():
+    let fd = fd.SocketHandle
     var pkey = s.getKey(fd)
     if Event.Read in pkey.events:
       IOFD_CLR(fd, addr s.rSet)
@@ -438,18 +439,19 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value,
                       body1, body2: untyped) =
   mixin withSelectLock
   s.withSelectLock():
-    var value: ptr T
+    block:
-    let fdi = int(fd)
+      var value: ptr T
-    var i = 0
+      let fdi = int(fd)
-    while i < FD_SETSIZE:
+      var i = 0
-      if s.fds[i].ident == fdi:
+      while i < FD_SETSIZE:
-        value = addr(s.fds[i].data)
+        if s.fds[i].ident == fdi:
-        break
+          value = addr(s.fds[i].data)
-      inc(i)
+          break
-    if i != FD_SETSIZE:
+        inc(i)
-      body1
+      if i != FD_SETSIZE:
-    else:
+        body1
-      body2
+      else:
+        body2
 
 
 proc getFd*[T](s: Selector[T]): int =

lib/pure/json.nim

@@ -1346,6 +1346,16 @@ proc createJsonIndexer(jsonNode: NimNode,
     indexNode
   )
 
+proc transformJsonIndexer(jsonNode: NimNode): NimNode =
+  case jsonNode.kind
+  of nnkBracketExpr:
+    result = newNimNode(nnkCurlyExpr)
+  else:
+    result = jsonNode.copy()
+
+  for child in jsonNode:
+    result.add(transformJsonIndexer(child))
+
 template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind],
                         ast: string) =
   if node.kind notin kinds:
@@ -1524,6 +1534,35 @@ proc processObjField(field, jsonNode: NimNode): seq[NimNode] =
 
   doAssert result.len > 0
 
+proc processFields(obj: NimNode,
+                   jsonNode: NimNode): seq[NimNode] {.compileTime.} =
+  ## Process all the fields of an ``ObjectTy`` and any of its
+  ## parent type's fields (via inheritance).
+  result = @[]
+  case obj.kind
+  of nnkObjectTy:
+    expectKind(obj[2], nnkRecList)
+    for field in obj[2]:
+      let nodes = processObjField(field, jsonNode)
+      result.add(nodes)
+
+    # process parent type fields
+    case obj[1].kind
+    of nnkBracketExpr:
+      assert $obj[1][0] == "ref"
+      result.add(processFields(getType(obj[1][1]), jsonNode))
+    of nnkSym:
+      result.add(processFields(getType(obj[1]), jsonNode))
+    else:
+      discard
+  of nnkTupleTy:
+    for identDefs in obj:
+      expectKind(identDefs, nnkIdentDefs)
+      let nodes = processObjField(identDefs[0], jsonNode)
+      result.add(nodes)
+  else:
+    doAssert false, "Unable to process field type: " & $obj.kind
+
 proc processType(typeName: NimNode, obj: NimNode,
                  jsonNode: NimNode, isRef: bool): NimNode {.compileTime.} =
   ## Process a type such as ``Sym "float"`` or ``ObjectTy ...``.
@@ -1533,20 +1572,21 @@ proc processType(typeName: NimNode, obj: NimNode,
   ## .. code-block::plain
   ##     ObjectTy
   ##       Empty
-  ##       Empty
+  ##       InheritanceInformation
   ##       RecList
   ##         Sym "events"
   case obj.kind
-  of nnkObjectTy:
+  of nnkObjectTy, nnkTupleTy:
     # Create object constructor.
-    result = newNimNode(nnkObjConstr)
+    result =
-    result.add(typeName) # Name of the type to construct.
+      if obj.kind == nnkObjectTy: newNimNode(nnkObjConstr)
+      else: newNimNode(nnkPar)
 
-    # Process each object field and add it as an exprColonExpr
+    if obj.kind == nnkObjectTy:
-    expectKind(obj[2], nnkRecList)
+      result.add(typeName) # Name of the type to construct.
-    for field in obj[2]:
+
-      let nodes = processObjField(field, jsonNode)
+    # Process each object/tuple field and add it as an exprColonExpr
-      result.add(nodes)
+    result.add(processFields(obj, jsonNode))
 
     # Object might be null. So we need to check for that.
     if isRef:
@@ -1569,25 +1609,14 @@ proc processType(typeName: NimNode, obj: NimNode,
         `getEnumCall`
       )
   of nnkSym:
-    case ($typeName).normalize
+    let name = ($typeName).normalize
-    of "float":
+    case name
-      result = quote do:
-        (
-          verifyJsonKind(`jsonNode`, {JFloat, JInt}, astToStr(`jsonNode`));
-          if `jsonNode`.kind == JFloat: `jsonNode`.fnum else: `jsonNode`.num.float
-        )
     of "string":
       result = quote do:
         (
           verifyJsonKind(`jsonNode`, {JString, JNull}, astToStr(`jsonNode`));
           if `jsonNode`.kind == JNull: nil else: `jsonNode`.str
         )
-    of "int":
-      result = quote do:
-        (
-          verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`));
-          `jsonNode`.num.int
-        )
     of "biggestint":
       result = quote do:
         (
@@ -1601,12 +1630,36 @@ proc processType(typeName: NimNode, obj: NimNode,
           `jsonNode`.bval
         )
     else:
-      doAssert false, "Unable to process nnkSym " & $typeName
+      if name.startsWith("int") or name.startsWith("uint"):
+        result = quote do:
+          (
+            verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`));
+            `jsonNode`.num.`obj`
+          )
+      elif name.startsWith("float"):
+        result = quote do:
+          (
+            verifyJsonKind(`jsonNode`, {JInt, JFloat}, astToStr(`jsonNode`));
+            if `jsonNode`.kind == JFloat: `jsonNode`.fnum.`obj` else: `jsonNode`.num.`obj`
+          )
+      else:
+        doAssert false, "Unable to process nnkSym " & $typeName
   else:
     doAssert false, "Unable to process type: " & $obj.kind
 
   doAssert(not result.isNil(), "processType not initialised.")
 
+import options
+proc workaroundMacroNone[T](): Option[T] =
+  none(T)
+
+proc depth(n: NimNode, current = 0): int =
+  result = 1
+  for child in n:
+    let d = 1 + child.depth(current + 1)
+    if d > result:
+      result = d
+
 proc createConstructor(typeSym, jsonNode: NimNode): NimNode =
   ## Accepts a type description, i.e. "ref Type", "seq[Type]", "Type" etc.
   ##
@@ -1616,10 +1669,50 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode =
   # echo("--createConsuctor-- \n", treeRepr(typeSym))
   # echo()
 
+  if depth(jsonNode) > 150:
+    error("The `to` macro does not support ref objects with cycles.", jsonNode)
+
   case typeSym.kind
   of nnkBracketExpr:
     var bracketName = ($typeSym[0]).normalize
     case bracketName
+    of "option":
+      # TODO: Would be good to verify that this is Option[T] from
+      # options module I suppose.
+      let lenientJsonNode = transformJsonIndexer(jsonNode)
+
+      let optionGeneric = typeSym[1]
+      let value = createConstructor(typeSym[1], jsonNode)
+      let workaround = bindSym("workaroundMacroNone") # TODO: Nim Bug: This shouldn't be necessary.
+
+      result = quote do:
+        (
+          if `lenientJsonNode`.isNil: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`)
+        )
+    of "table", "orderedtable":
+      let tableKeyType = typeSym[1]
+      if ($tableKeyType).cmpIgnoreStyle("string") != 0:
+        error("JSON doesn't support keys of type " & $tableKeyType)
+      let tableValueType = typeSym[2]
+
+      let forLoopKey = genSym(nskForVar, "key")
+      let indexerNode = createJsonIndexer(jsonNode, forLoopKey)
+      let constructorNode = createConstructor(tableValueType, indexerNode)
+
+      let tableInit =
+        if bracketName == "table":
+          bindSym("initTable")
+        else:
+          bindSym("initOrderedTable")
+
+      # Create a statement expression containing a for loop.
+      result = quote do:
+        (
+          var map = `tableInit`[`tableKeyType`, `tableValueType`]();
+          verifyJsonKind(`jsonNode`, {JObject}, astToStr(`jsonNode`));
+          for `forLoopKey` in keys(`jsonNode`.fields): map[`forLoopKey`] = `constructorNode`;
+          map
+        )
     of "ref":
       # Ref type.
       var typeName = $typeSym[1]
@@ -1663,12 +1756,23 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode =
       let obj = getType(typeSym)
       result = processType(typeSym, obj, jsonNode, false)
   of nnkSym:
+    # Handle JsonNode.
+    if ($typeSym).cmpIgnoreStyle("jsonnode") == 0:
+      return jsonNode
+
+    # Handle all other types.
     let obj = getType(typeSym)
     if obj.kind == nnkBracketExpr:
       # When `Sym "Foo"` turns out to be a `ref object`.
       result = createConstructor(obj, jsonNode)
     else:
       result = processType(typeSym, obj, jsonNode, false)
+  of nnkTupleTy:
+    result = processType(typeSym, typeSym, jsonNode, false)
+  of nnkPar:
+    # TODO: The fact that `jsonNode` here works to give a good line number
+    # is weird. Specifying typeSym should work but doesn't.
+    error("Use a named tuple instead of: " & $toStrLit(typeSym), jsonNode)
   else:
     doAssert false, "Unable to create constructor for: " & $typeSym.kind
 
@@ -1796,10 +1900,18 @@ macro to*(node: JsonNode, T: typedesc): untyped =
   expectKind(typeNode, nnkBracketExpr)
   doAssert(($typeNode[0]).normalize == "typedesc")
 
-  result = createConstructor(typeNode[1], node)
+  # Create `temp` variable to store the result in case the user calls this
-  # TODO: Rename postProcessValue and move it (?)
+  # on `parseJson` (see bug #6604).
-  result = postProcessValue(result)
+  result = newNimNode(nnkStmtListExpr)
+  let temp = genSym(nskLet, "temp")
+  result.add quote do:
+    let `temp` = `node`
 
+  let constructor = createConstructor(typeNode[1], temp)
+  # TODO: Rename postProcessValue and move it (?)
+  result.add(postProcessValue(constructor))
+
+  # echo(treeRepr(result))
   # echo(toStrLit(result))
 
 when false:

lib/pure/logging.nim

@@ -202,13 +202,17 @@ when not defined(js):
 
   proc countLogLines(logger: RollingFileLogger): int =
     result = 0
-    for line in logger.file.lines():
+    let fp = open(logger.baseName, fmRead)
+    for line in fp.lines():
       result.inc()
+    fp.close()
 
   proc countFiles(filename: string): int =
     # Example: file.log.1
     result = 0
-    let (dir, name, ext) = splitFile(filename)
+    var (dir, name, ext) = splitFile(filename)
+    if dir == "":
+      dir = "."
     for kind, path in walkDir(dir):
       if kind == pcFile:
         let llfn = name & ext & ExtSep

lib/pure/mimetypes.nim

@@ -491,6 +491,8 @@ const mimes* = {
     "vrml": "x-world/x-vrml",
     "wrl": "x-world/x-vrml"}
 
+from strutils import startsWith
+
 proc newMimetypes*(): MimeDB =
   ## Creates a new Mimetypes database. The database will contain the most
   ## common mimetypes.
@@ -498,8 +500,11 @@ proc newMimetypes*(): MimeDB =
 
 proc getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string =
   ## Gets mimetype which corresponds to ``ext``. Returns ``default`` if ``ext``
-  ## could not be found.
+  ## could not be found. ``ext`` can start with an optional dot which is ignored.
-  result = mimedb.mimes.getOrDefault(ext)
+  if ext.startsWith("."):
+    result = mimedb.mimes.getOrDefault(ext.substr(1))
+  else:
+    result = mimedb.mimes.getOrDefault(ext)
   if result == "":
     return default
 

lib/pure/net.nim

@@ -145,7 +145,7 @@ type
 
   SOBool* = enum ## Boolean socket options.
     OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive,
-    OptOOBInline, OptReuseAddr, OptReusePort
+    OptOOBInline, OptReuseAddr, OptReusePort, OptNoDelay
 
   ReadLineResult* = enum ## result for readLineAsync
     ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone
@@ -865,6 +865,11 @@ proc close*(socket: Socket) =
 
     socket.fd.close()
 
+when defined(posix):
+  from posix import TCP_NODELAY
+else:
+  from winlean import TCP_NODELAY
+
 proc toCInt*(opt: SOBool): cint =
   ## Converts a ``SOBool`` into its Socket Option cint representation.
   case opt
@@ -876,6 +881,7 @@ proc toCInt*(opt: SOBool): cint =
   of OptOOBInline: SO_OOBINLINE
   of OptReuseAddr: SO_REUSEADDR
   of OptReusePort: SO_REUSEPORT
+  of OptNoDelay: TCP_NODELAY
 
 proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {.
   tags: [ReadIOEffect].} =
@@ -898,6 +904,12 @@ proc getPeerAddr*(socket: Socket): (string, Port) =
 proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {.
   tags: [WriteIOEffect].} =
   ## Sets option ``opt`` to a boolean value specified by ``value``.
+  ##
+  ## .. code-block:: Nim
+  ##   var socket = newSocket()
+  ##   socket.setSockOpt(OptReusePort, true)
+  ##   socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt)
+  ##
   var valuei = cint(if value: 1 else: 0)
   setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei)
 

lib/pure/os.nim

@@ -630,7 +630,7 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1",
   ## the process has finished. To execute a program without having a
   ## shell involved, use the `execProcess` proc of the `osproc`
   ## module.
-  when defined(linux):
+  when defined(posix):
     result = c_system(command) shr 8
   else:
     result = c_system(command)

lib/pure/ospaths.nim

@@ -602,14 +602,13 @@ proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".}
   else:
     return "'" & s.replace("'", "'\"'\"'") & "'"
 
-proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} =
+when defined(windows) or defined(posix):
-  ## Quote ``s``, so it can be safely passed to shell.
+  proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} =
-  when defined(Windows):
+    ## Quote ``s``, so it can be safely passed to shell.
-    return quoteShellWindows(s)
+    when defined(windows):
-  elif defined(posix):
+      return quoteShellWindows(s)
-    return quoteShellPosix(s)
+    else:
-  else:
+      return quoteShellPosix(s)
-    {.error:"quoteShell is not supported on your system".}
 
 when isMainModule:
   assert quoteShellWindows("aaa") == "aaa"

lib/pure/osproc.nim

@@ -41,6 +41,8 @@ type
                          ## Windows: Named pipes are used so that you can peek
                          ## at the process' output streams.
     poDemon              ## Windows: The program creates no Window.
+                         ## Unix: Start the program as a demon. This is still
+                         ## work in progress!
 
   ProcessObj = object of RootObj
     when defined(windows):
@@ -167,8 +169,7 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl,
   ## On posix, if the process has exited because of a signal, 128 + signal
   ## number will be returned.
 
-
+proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].}
-proc peekExitCode*(p: Process): int {.tags: [].}
   ## return -1 if the process is still running. Otherwise the process' exit code
   ##
   ## On posix, if the process has exited because of a signal, 128 + signal
@@ -231,55 +232,79 @@ proc execProcesses*(cmds: openArray[string],
   ## executes the commands `cmds` in parallel. Creates `n` processes
   ## that execute in parallel. The highest return value of all processes
   ## is returned. Runs `beforeRunEvent` before running each command.
-  when false:
-    # poParentStreams causes problems on Posix, so we simply disable it:
-    var options = options - {poParentStreams}
 
   assert n > 0
   if n > 1:
-    var q: seq[Process]
+    var i = 0
-    newSeq(q, n)
+    var q = newSeq[Process](n)
     var m = min(n, cmds.len)
-    for i in 0..m-1:
+
+    when defined(windows):
+      var w: WOHandleArray
+      var wcount = m
+
+    while i < m:
       if beforeRunEvent != nil:
         beforeRunEvent(i)
-      q[i] = startProcess(cmds[i], options=options + {poEvalCommand})
+      q[i] = startProcess(cmds[i], options = options + {poEvalCommand})
-    when defined(noBusyWaiting):
+      when defined(windows):
-      var r = 0
+        w[i] = q[i].fProcessHandle
-      for i in m..high(cmds):
+      inc(i)
-        when defined(debugExecProcesses):
+
-          var err = ""
+    var ecount = len(cmds)
-          var outp = outputStream(q[r])
+    while ecount > 0:
-          while running(q[r]) or not atEnd(outp):
+      when defined(windows):
-            err.add(outp.readLine())
+        # waiting for all children, get result if any child exits
-            err.add("\n")
+        var ret = waitForMultipleObjects(int32(wcount), addr(w), 0'i32,
-          echo(err)
+                                         INFINITE)
-        result = max(waitForExit(q[r]), result)
+        if ret == WAIT_TIMEOUT:
-        if afterRunEvent != nil: afterRunEvent(r, q[r])
+          # must not be happen
-        if q[r] != nil: close(q[r])
+          discard
-        if beforeRunEvent != nil:
+        elif ret == WAIT_FAILED:
-          beforeRunEvent(i)
+          raiseOSError(osLastError())
-        q[r] = startProcess(cmds[i], options=options + {poEvalCommand})
+      else:
-        r = (r + 1) mod n
+        var status : cint = 1
-    else:
+        # waiting for all children, get result if any child exits
-      var i = m
+        let res = waitpid(-1, status, 0)
-      while i <= high(cmds):
+        if res > 0:
-        sleep(50)
+          for r in 0..m-1:
-        for r in 0..n-1:
+            if not isNil(q[r]) and q[r].id == res:
+              # we updating `exitStatus` manually, so `running()` can work.
+              if WIFEXITED(status) or WIFSIGNALED(status):
+                q[r].exitStatus = status
+                break
+        else:
+          let err = osLastError()
+          if err == OSErrorCode(ECHILD):
+            # some child exits, we need to check our childs exit codes
+            discard
+          elif err == OSErrorCode(EINTR):
+            # signal interrupted our syscall, lets repeat it
+            continue
+          else:
+            # all other errors are exceptions
+            raiseOSError(err)
+
+      for r in 0..m-1:
+        if not isNil(q[r]):
           if not running(q[r]):
-            #echo(outputStream(q[r]).readLine())
+            result = max(result, q[r].peekExitCode())
-            result = max(waitForExit(q[r]), result)
             if afterRunEvent != nil: afterRunEvent(r, q[r])
-            if q[r] != nil: close(q[r])
+            close(q[r])
-            if beforeRunEvent != nil:
+            if i < len(cmds):
-              beforeRunEvent(i)
+              if beforeRunEvent != nil: beforeRunEvent(i)
-            q[r] = startProcess(cmds[i], options=options + {poEvalCommand})
+              q[r] = startProcess(cmds[i],
-            inc(i)
+                                  options = options + {poEvalCommand})
-            if i > high(cmds): break
+              when defined(windows):
-    for j in 0..m-1:
+                w[r] = q[r].fProcessHandle
-      result = max(waitForExit(q[j]), result)
+              inc(i)
-      if afterRunEvent != nil: afterRunEvent(j, q[j])
+            else:
-      if q[j] != nil: close(q[j])
+              q[r] = nil
+              when defined(windows):
+                for c in r..MAXIMUM_WAIT_OBJECTS - 2:
+                  w[c] = w[c + 1]
+                dec(wcount)
+            dec(ecount)
   else:
     for i in 0..high(cmds):
       if beforeRunEvent != nil:
@@ -321,6 +346,8 @@ when not defined(useNimRtl):
       elif not running(p): break
     close(p)
 
+template streamAccess(p) =
+  assert poParentStreams notin p.options, "API usage error: stream access not allowed when you use poParentStreams"
 
 when defined(Windows) and not defined(useNimRtl):
   # We need to implement a handle stream for Windows:
@@ -581,12 +608,15 @@ when defined(Windows) and not defined(useNimRtl):
       return res
 
   proc inputStream(p: Process): Stream =
+    streamAccess(p)
     result = newFileHandleStream(p.inHandle)
 
   proc outputStream(p: Process): Stream =
+    streamAccess(p)
     result = newFileHandleStream(p.outHandle)
 
   proc errorStream(p: Process): Stream =
+    streamAccess(p)
     result = newFileHandleStream(p.errHandle)
 
   proc execCmd(command: string): int =
@@ -682,9 +712,7 @@ elif not defined(useNimRtl):
     sysEnv: cstringArray
     workingDir: cstring
     pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint]
-    optionPoUsePath: bool
+    options: set[ProcessOption]
-    optionPoParentStreams: bool
-    optionPoStdErrToStdOut: bool
   {.deprecated: [TStartProcessData: StartProcessData].}
 
   const useProcessAuxSpawn = declared(posix_spawn) and not defined(useFork) and
@@ -749,10 +777,8 @@ elif not defined(useNimRtl):
     data.pStdin = pStdin
     data.pStdout = pStdout
     data.pStderr = pStderr
-    data.optionPoParentStreams = poParentStreams in options
-    data.optionPoUsePath = poUsePath in options
-    data.optionPoStdErrToStdOut = poStdErrToStdOut in options
     data.workingDir = workingDir
+    data.options = options
 
     when useProcessAuxSpawn:
       var currentDir = getCurrentDir()
@@ -801,19 +827,22 @@ elif not defined(useNimRtl):
       var mask: Sigset
       chck sigemptyset(mask)
       chck posix_spawnattr_setsigmask(attr, mask)
-      chck posix_spawnattr_setpgroup(attr, 0'i32)
+      if poDemon in data.options:
+        chck posix_spawnattr_setpgroup(attr, 0'i32)
 
-      chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or
+      var flags = POSIX_SPAWN_USEVFORK or
-                                          POSIX_SPAWN_SETSIGMASK or
+                  POSIX_SPAWN_SETSIGMASK
-                                          POSIX_SPAWN_SETPGROUP)
+      if poDemon in data.options:
+        flags = flags or POSIX_SPAWN_SETPGROUP
+      chck posix_spawnattr_setflags(attr, flags)
 
-      if not data.optionPoParentStreams:
+      if not (poParentStreams in data.options):
         chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx])
         chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx)
         chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx])
         chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx)
         chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx])
-        if data.optionPoStdErrToStdOut:
+        if (poStdErrToStdOut in data.options):
           chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2)
         else:
           chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2)
@@ -823,7 +852,7 @@ elif not defined(useNimRtl):
         setCurrentDir($data.workingDir)
       var pid: Pid
 
-      if data.optionPoUsePath:
+      if (poUsePath in data.options):
         res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
       else:
         res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
@@ -885,7 +914,7 @@ elif not defined(useNimRtl):
     # Warning: no GC here!
     # Or anything that touches global structures - all called nim procs
     # must be marked with stackTrace:off. Inspect C code after making changes.
-    if not data.optionPoParentStreams:
+    if not (poParentStreams in data.options):
       discard close(data.pStdin[writeIdx])
       if dup2(data.pStdin[readIdx], readIdx) < 0:
         startProcessFail(data)
@@ -893,7 +922,7 @@ elif not defined(useNimRtl):
       if dup2(data.pStdout[writeIdx], writeIdx) < 0:
         startProcessFail(data)
       discard close(data.pStderr[readIdx])
-      if data.optionPoStdErrToStdOut:
+      if (poStdErrToStdOut in data.options):
         if dup2(data.pStdout[writeIdx], 2) < 0:
           startProcessFail(data)
       else:
@@ -907,7 +936,7 @@ elif not defined(useNimRtl):
     discard close(data.pErrorPipe[readIdx])
     discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC)
 
-    if data.optionPoUsePath:
+    if (poUsePath in data.options):
       when defined(uClibc) or defined(linux):
         # uClibc environment (OpenWrt included) doesn't have the full execvpe
         let exe = findExe(data.sysCommand)
@@ -939,19 +968,22 @@ elif not defined(useNimRtl):
     if kill(p.id, SIGCONT) != 0'i32: raiseOsError(osLastError())
 
   proc running(p: Process): bool =
-    var ret : int
+    if p.exitStatus != -3:
-    var status : cint = 1
-    ret = waitpid(p.id, status, WNOHANG)
-    if ret == int(p.id):
-      if isExitStatus(status):
-        p.exitStatus = status
-        return false
-      else:
-        return true
-    elif ret == 0:
-      return true # Can't establish status. Assume running.
-    else:
       return false
+    else:
+      var ret : int
+      var status : cint = 1
+      ret = waitpid(p.id, status, WNOHANG)
+      if ret == int(p.id):
+        if isExitStatus(status):
+          p.exitStatus = status
+          return false
+        else:
+          return true
+      elif ret == 0:
+        return true # Can't establish status. Assume running.
+      else:
+        raiseOSError(osLastError())
 
   proc terminate(p: Process) =
     if kill(p.id, SIGTERM) != 0'i32:
@@ -1152,16 +1184,19 @@ elif not defined(useNimRtl):
     stream = newFileStream(f)
 
   proc inputStream(p: Process): Stream =
+    streamAccess(p)
     if p.inStream == nil:
       createStream(p.inStream, p.inHandle, fmWrite)
     return p.inStream
 
   proc outputStream(p: Process): Stream =
+    streamAccess(p)
     if p.outStream == nil:
       createStream(p.outStream, p.outHandle, fmRead)
     return p.outStream
 
   proc errorStream(p: Process): Stream =
+    streamAccess(p)
     if p.errStream == nil:
       createStream(p.errStream, p.errHandle, fmRead)
     return p.errStream

lib/pure/parsesql.nim

@@ -956,6 +956,7 @@ proc parseInsert(p: var SqlParser): SqlNode =
   if p.tok.kind == tkParLe:
     var n = newNode(nkColumnList)
     parseParIdentList(p, n)
+    result.add n
   else:
     result.add(nil)
   if isKeyw(p, "default"):
@@ -1160,7 +1161,7 @@ proc ra(n: SqlNode, s: var string, indent: int) =
     else:
       s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"")
   of nkStringLit:
-    s.add(escape(n.strVal, "e'", "'"))
+    s.add(escape(n.strVal, "'", "'"))
   of nkBitStringLit:
     s.add("b'" & n.strVal & "'")
   of nkHexStringLit:
@@ -1240,7 +1241,7 @@ proc ra(n: SqlNode, s: var string, indent: int) =
     if n.sons[2].kind == nkDefault:
       s.add("default values")
     else:
-      s.add("\nvalues ")
+      s.add("\n")
       ra(n.sons[2], s, indent)
     s.add(';')
   of nkUpdate:

lib/pure/selectors.nim

@@ -310,6 +310,8 @@ else:
     include ioselects/ioselectors_select
   elif defined(solaris):
     include ioselects/ioselectors_poll # need to replace it with event ports
+  elif defined(genode):
+    include ioselects/ioselectors_select # TODO: use the native VFS layer
   else:
     include ioselects/ioselectors_poll
 

lib/pure/strutils.nim

@@ -32,10 +32,6 @@ when defined(nimOldSplit):
 else:
   {.pragma: deprecatedSplit.}
 
-type
-  CharSet* {.deprecated.} = set[char] # for compatibility with Nim
-{.deprecated: [TCharSet: CharSet].}
-
 const
   Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'}
     ## All the characters that count as whitespace.
@@ -78,40 +74,40 @@ proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar,
   return c in Letters
 
 proc isAlphaNumeric*(c: char): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsAlphaNumericChar".}=
+  rtl, extern: "nsuIsAlphaNumericChar".} =
   ## Checks whether or not `c` is alphanumeric.
   ##
   ## This checks a-z, A-Z, 0-9 ASCII characters only.
-  return c in Letters or c in Digits
+  return c in Letters+Digits
 
 proc isDigit*(c: char): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsDigitChar".}=
+  rtl, extern: "nsuIsDigitChar".} =
   ## Checks whether or not `c` is a number.
   ##
   ## This checks 0-9 ASCII characters only.
   return c in Digits
 
 proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsSpaceAsciiChar".}=
+  rtl, extern: "nsuIsSpaceAsciiChar".} =
   ## Checks whether or not `c` is a whitespace character.
   return c in Whitespace
 
 proc isLowerAscii*(c: char): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsLowerAsciiChar".}=
+  rtl, extern: "nsuIsLowerAsciiChar".} =
   ## Checks whether or not `c` is a lower case character.
   ##
   ## This checks ASCII characters only.
   return c in {'a'..'z'}
 
 proc isUpperAscii*(c: char): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsUpperAsciiChar".}=
+  rtl, extern: "nsuIsUpperAsciiChar".} =
   ## Checks whether or not `c` is an upper case character.
   ##
   ## This checks ASCII characters only.
   return c in {'A'..'Z'}
 
 proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsAlphaAsciiStr".}=
+  rtl, extern: "nsuIsAlphaAsciiStr".} =
   ## Checks whether or not `s` is alphabetical.
   ##
   ## This checks a-z, A-Z ASCII characters only.
@@ -123,10 +119,10 @@ proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar,
 
   result = true
   for c in s:
-    result = c.isAlphaAscii() and result
+    if not c.isAlphaAscii(): return false
 
 proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsAlphaNumericStr".}=
+  rtl, extern: "nsuIsAlphaNumericStr".} =
   ## Checks whether or not `s` is alphanumeric.
   ##
   ## This checks a-z, A-Z, 0-9 ASCII characters only.
@@ -142,7 +138,7 @@ proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar,
       return false
 
 proc isDigit*(s: string): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsDigitStr".}=
+  rtl, extern: "nsuIsDigitStr".} =
   ## Checks whether or not `s` is a numeric value.
   ##
   ## This checks 0-9 ASCII characters only.
@@ -158,7 +154,7 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar,
       return false
 
 proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsSpaceAsciiStr".}=
+  rtl, extern: "nsuIsSpaceAsciiStr".} =
   ## Checks whether or not `s` is completely whitespace.
   ##
   ## Returns true if all characters in `s` are whitespace
@@ -172,7 +168,7 @@ proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar,
       return false
 
 proc isLowerAscii*(s: string): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsLowerAsciiStr".}=
+  rtl, extern: "nsuIsLowerAsciiStr".} =
   ## Checks whether or not `s` contains all lower case characters.
   ##
   ## This checks ASCII characters only.
@@ -187,7 +183,7 @@ proc isLowerAscii*(s: string): bool {.noSideEffect, procvar,
   true
 
 proc isUpperAscii*(s: string): bool {.noSideEffect, procvar,
-  rtl, extern: "nsuIsUpperAsciiStr".}=
+  rtl, extern: "nsuIsUpperAsciiStr".} =
   ## Checks whether or not `s` contains all upper case characters.
   ##
   ## This checks ASCII characters only.
@@ -506,16 +502,15 @@ template splitCommon(s, sep, maxsplit, sepLen) =
   var last = 0
   var splits = maxsplit
 
-  if len(s) > 0:
+  while last <= len(s):
-    while last <= len(s):
+    var first = last
-      var first = last
+    while last < len(s) and not stringHasSep(s, last, sep):
-      while last < len(s) and not stringHasSep(s, last, sep):
+      inc(last)
-        inc(last)
+    if splits == 0: last = len(s)
-      if splits == 0: last = len(s)
+    yield substr(s, first, last-1)
-      yield substr(s, first, last-1)
+    if splits == 0: break
-      if splits == 0: break
+    dec(splits)
-      dec(splits)
+    inc(last, sepLen)
-      inc(last, sepLen)
 
 template oldSplit(s, seps, maxsplit) =
   var last = 0
@@ -673,30 +668,29 @@ template rsplitCommon(s, sep, maxsplit, sepLen) =
     splits = maxsplit
     startPos = 0
 
-  if len(s) > 0:
+  # go to -1 in order to get separators at the beginning
-    # go to -1 in order to get separators at the beginning
+  while first >= -1:
-    while first >= -1:
+    while first >= 0 and not stringHasSep(s, first, sep):
-      while first >= 0 and not stringHasSep(s, first, sep):
-        dec(first)
-
-      if splits == 0:
-        # No more splits means set first to the beginning
-        first = -1
-
-      if first == -1:
-        startPos = 0
-      else:
-        startPos = first + sepLen
-
-      yield substr(s, startPos, last)
-
-      if splits == 0:
-        break
-
-      dec(splits)
       dec(first)
 
-      last = first
+    if splits == 0:
+      # No more splits means set first to the beginning
+      first = -1
+
+    if first == -1:
+      startPos = 0
+    else:
+      startPos = first + sepLen
+
+    yield substr(s, startPos, last)
+
+    if splits == 0:
+      break
+
+    dec(splits)
+    dec(first)
+
+    last = first
 
 iterator rsplit*(s: string, seps: set[char] = Whitespace,
                  maxsplit: int = -1): string =
@@ -824,12 +818,18 @@ proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[st
   noSideEffect, rtl, extern: "nsuSplitCharSet".} =
   ## The same as the `split iterator <#split.i,string,set[char],int>`_, but is a
   ## proc that returns a sequence of substrings.
+  runnableExamples:
+    doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"]
+    doAssert "".split({' '}) == @[""]
   accumulateResult(split(s, seps, maxsplit))
 
 proc split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect,
   rtl, extern: "nsuSplitChar".} =
   ## The same as the `split iterator <#split.i,string,char,int>`_, but is a proc
   ## that returns a sequence of substrings.
+  runnableExamples:
+    doAssert "a,b,c".split(',') == @["a", "b", "c"]
+    doAssert "".split(' ') == @[""]
   accumulateResult(split(s, sep, maxsplit))
 
 proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect,
@@ -838,6 +838,13 @@ proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEff
   ##
   ## Substrings are separated by the string `sep`. This is a wrapper around the
   ## `split iterator <#split.i,string,string,int>`_.
+  runnableExamples:
+    doAssert "a,b,c".split(",") == @["a", "b", "c"]
+    doAssert "a man a plan a canal panama".split("a ") == @["", "man ", "plan ", "canal panama"]
+    doAssert "".split("Elon Musk") == @[""]
+    doAssert "a  largely    spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"]
+
+    doAssert "a  largely    spaced sentence".split(" ", maxsplit=1) == @["a", " largely    spaced sentence"]
   doAssert(sep.len > 0)
 
   accumulateResult(split(s, sep, maxsplit))
@@ -906,6 +913,13 @@ proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string]
   ## .. code-block:: nim
   ##   @["Root#Object#Method", "Index"]
   ##
+  runnableExamples:
+    doAssert "a  largely    spaced sentence".rsplit(" ", maxsplit=1) == @["a  largely    spaced", "sentence"]
+
+    doAssert "a,b,c".rsplit(",") == @["a", "b", "c"]
+    doAssert "a man a plan a canal panama".rsplit("a ") == @["", "man ", "plan ", "canal panama"]
+    doAssert "".rsplit("Elon Musk") == @[""]
+    doAssert "a  largely    spaced sentence".rsplit(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"]
   accumulateResult(rsplit(s, sep, maxsplit))
   result.reverse()
 
@@ -1305,14 +1319,13 @@ proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0)
   ## This is often useful for generating some code where the items need to
   ## be *separated* by `sep`. `sep` is only added if `dest` is longer than
   ## `startLen`. The following example creates a string describing
-  ## an array of integers:
+  ## an array of integers.
-  ##
+  runnableExamples:
-  ## .. code-block:: nim
+     var arr = "["
-  ##   var arr = "["
+     for x in items([2, 3, 5, 7, 11]):
-  ##   for x in items([2, 3, 5, 7, 11]):
+       addSep(arr, startLen=len("["))
-  ##     addSep(arr, startLen=len("["))
+       add(arr, $x)
-  ##     add(arr, $x)
+     add(arr, "]")
-  ##   add(arr, "]")
   if dest.len > startLen: add(dest, sep)
 
 proc allCharsInSet*(s: string, theSet: set[char]): bool =
@@ -1730,7 +1743,9 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect,
   ##
   ## Even though the algorithm works with any string `s`, it is only useful
   ## if `s` contains a number.
-  ## Example: ``insertSep("1000000") == "1_000_000"``
+  runnableExamples:
+    doAssert insertSep("1000000") == "1_000_000"
+
   var L = (s.len-1) div digits + s.len
   result = newString(L)
   var j = 0
@@ -1818,6 +1833,8 @@ proc validIdentifier*(s: string): bool {.noSideEffect,
   ##
   ## A valid identifier starts with a character of the set `IdentStartChars`
   ## and is followed by any number of characters of the set `IdentChars`.
+  runnableExamples:
+    doAssert "abc_def08".validIdentifier
   if s[0] in IdentStartChars:
     for i in 1..s.len-1:
       if s[i] notin IdentChars: return false
@@ -1828,7 +1845,7 @@ proc editDistance*(a, b: string): int {.noSideEffect,
   ## Returns the edit distance between `a` and `b`.
   ##
   ## This uses the `Levenshtein`:idx: distance algorithm with only a linear
-  ## memory overhead.  This implementation is highly optimized!
+  ## memory overhead.
   var len1 = a.len
   var len2 = b.len
   if len1 > len2:
@@ -2007,16 +2024,11 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault,
   ## after the decimal point for Nim's ``float`` type.
   ##
   ## If ``precision == -1``, it tries to format it nicely.
-  ##
+  runnableExamples:
-  ## Examples:
+    let x = 123.456
-  ##
+    doAssert x.formatFloat() == "123.4560000000000"
-  ## .. code-block:: nim
+    doAssert x.formatFloat(ffDecimal, 4) == "123.4560"
-  ##
+    doAssert x.formatFloat(ffScientific, 2) == "1.23e+02"
-  ##    let x = 123.456
-  ##    doAssert x.formatFloat() == "123.4560000000000"
-  ##    doAssert x.formatFloat(ffDecimal, 4) == "123.4560"
-  ##    doAssert x.formatFloat(ffScientific, 2) == "1.23e+02"
-  ##
   result = formatBiggestFloat(f, format, precision, decimalSep)
 
 proc trimZeros*(x: var string) {.noSideEffect.} =
@@ -2051,18 +2063,13 @@ proc formatSize*(bytes: int64,
   ##
   ## `includeSpace` can be set to true to include the (SI preferred) space
   ## between the number and the unit (e.g. 1 KiB).
-  ##
+  runnableExamples:
-  ## Examples:
+    doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB"
-  ##
+    doAssert formatSize((2.234*1024*1024).int) == "2.234MiB"
-  ## .. code-block:: nim
+    doAssert formatSize(4096, includeSpace=true) == "4 KiB"
-  ##
+    doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB"
-  ##    formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB"
+    doAssert formatSize(4096) == "4KiB"
-  ##    formatSize((2.234*1024*1024).int) == "2.234MiB"
+    doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB"
-  ##    formatSize(4096, includeSpace=true) == "4 KiB"
-  ##    formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB"
-  ##    formatSize(4096) == "4KiB"
-  ##    formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB"
-  ##
   const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"]
   const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"]
   var
@@ -2156,7 +2163,7 @@ proc formatEng*(f: BiggestFloat,
   ##    formatEng(4100, unit="V") == "4.1e3 V"
   ##    formatEng(4100, unit="") == "4.1e3 " # Space with unit=""
   ##
-  ## `decimalSep` is used as the decimal separator
+  ## `decimalSep` is used as the decimal separator.
   var
     absolute: BiggestFloat
     significand: BiggestFloat
@@ -2369,17 +2376,16 @@ proc removeSuffix*(s: var string, chars: set[char] = Newlines) {.
   rtl, extern: "nsuRemoveSuffixCharSet".} =
   ## Removes all characters from `chars` from the end of the string `s`
   ## (in-place).
-  ##
+  runnableExamples:
-  ## .. code-block:: nim
+     var userInput = "Hello World!*~\r\n"
-  ##   var userInput = "Hello World!*~\r\n"
+     userInput.removeSuffix
-  ##   userInput.removeSuffix
+     doAssert userInput == "Hello World!*~"
-  ##   doAssert userInput == "Hello World!*~"
+     userInput.removeSuffix({'~', '*'})
-  ##   userInput.removeSuffix({'~', '*'})
+     doAssert userInput == "Hello World!"
-  ##   doAssert userInput == "Hello World!"
+
-  ##
+     var otherInput = "Hello!?!"
-  ##   var otherInput = "Hello!?!"
+     otherInput.removeSuffix({'!', '?'})
-  ##   otherInput.removeSuffix({'!', '?'})
+     doAssert otherInput == "Hello"
-  ##   doAssert otherInput == "Hello"
   if s.len == 0: return
   var last = s.high
   while last > -1 and s[last] in chars: last -= 1
@@ -2390,24 +2396,23 @@ proc removeSuffix*(s: var string, c: char) {.
   ## Removes all occurrences of a single character (in-place) from the end
   ## of a string.
   ##
-  ## .. code-block:: nim
+  runnableExamples:
-  ##   var table = "users"
+     var table = "users"
-  ##   table.removeSuffix('s')
+     table.removeSuffix('s')
-  ##   doAssert table == "user"
+     doAssert table == "user"
-  ##
+
-  ##   var dots = "Trailing dots......."
+     var dots = "Trailing dots......."
-  ##   dots.removeSuffix('.')
+     dots.removeSuffix('.')
-  ##   doAssert dots == "Trailing dots"
+     doAssert dots == "Trailing dots"
   removeSuffix(s, chars = {c})
 
 proc removeSuffix*(s: var string, suffix: string) {.
   rtl, extern: "nsuRemoveSuffixString".} =
   ## Remove the first matching suffix (in-place) from a string.
-  ##
+  runnableExamples:
-  ## .. code-block:: nim
+     var answers = "yeses"
-  ##   var answers = "yeses"
+     answers.removeSuffix("es")
-  ##   answers.removeSuffix("es")
+     doAssert answers == "yes"
-  ##   doAssert answers == "yes"
   var newLen = s.len
   if s.endsWith(suffix):
     newLen -= len(suffix)
@@ -2418,16 +2423,16 @@ proc removePrefix*(s: var string, chars: set[char] = Newlines) {.
   ## Removes all characters from `chars` from the start of the string `s`
   ## (in-place).
   ##
-  ## .. code-block:: nim
+  runnableExamples:
-  ##   var userInput = "\r\n*~Hello World!"
+     var userInput = "\r\n*~Hello World!"
-  ##   userInput.removePrefix
+     userInput.removePrefix
-  ##   doAssert userInput == "*~Hello World!"
+     doAssert userInput == "*~Hello World!"
-  ##   userInput.removePrefix({'~', '*'})
+     userInput.removePrefix({'~', '*'})
-  ##   doAssert userInput == "Hello World!"
+     doAssert userInput == "Hello World!"
-  ##
+
-  ##   var otherInput = "?!?Hello!?!"
+     var otherInput = "?!?Hello!?!"
-  ##   otherInput.removePrefix({'!', '?'})
+     otherInput.removePrefix({'!', '?'})
-  ##   doAssert otherInput == "Hello!?!"
+     doAssert otherInput == "Hello!?!"
   var start = 0
   while start < s.len and s[start] in chars: start += 1
   if start > 0: s.delete(0, start - 1)
@@ -2437,20 +2442,20 @@ proc removePrefix*(s: var string, c: char) {.
   ## Removes all occurrences of a single character (in-place) from the start
   ## of a string.
   ##
-  ## .. code-block:: nim
+  runnableExamples:
-  ##   var ident = "pControl"
+     var ident = "pControl"
-  ##   ident.removePrefix('p')
+     ident.removePrefix('p')
-  ##   doAssert ident == "Control"
+     doAssert ident == "Control"
   removePrefix(s, chars = {c})
 
 proc removePrefix*(s: var string, prefix: string) {.
   rtl, extern: "nsuRemovePrefixString".} =
   ## Remove the first matching prefix (in-place) from a string.
   ##
-  ## .. code-block:: nim
+  runnableExamples:
-  ##   var answers = "yesyes"
+     var answers = "yesyes"
-  ##   answers.removePrefix("yes")
+     answers.removePrefix("yes")
-  ##   doAssert answers == "yes"
+     doAssert answers == "yes"
   if s.startsWith(prefix):
     s.delete(0, prefix.len - 1)
 

lib/pure/unicode.nim

@@ -293,33 +293,33 @@ proc runeSubStr*(s: string, pos:int, len:int = int.high): string =
   if pos < 0:
     let (o, rl) = runeReverseOffset(s, -pos)
     if len >= rl:
-      result = s[o.. s.len-1]
+      result = s.substr(o, s.len-1)
     elif len < 0:
       let e = rl + len
       if e < 0:
         result = ""
       else:
-        result = s[o.. runeOffset(s, e-(rl+pos) , o)-1]
+        result = s.substr(o, runeOffset(s, e-(rl+pos) , o)-1)
     else:
-      result = s[o.. runeOffset(s, len, o)-1]
+      result = s.substr(o, runeOffset(s, len, o)-1)
   else:
     let o = runeOffset(s, pos)
     if o < 0:
       result = ""
     elif len == int.high:
-      result = s[o.. s.len-1]
+      result = s.substr(o, s.len-1)
     elif len < 0:
       let (e, rl) = runeReverseOffset(s, -len)
       discard rl
       if e <= 0:
         result = ""
       else:
-        result = s[o.. e-1]
+        result = s.substr(o, e-1)
     else:
       var e = runeOffset(s, len, o)
       if e < 0:
         e = s.len
-      result = s[o.. e-1]
+      result = s.substr(o, e-1)
 
 const
   alphaRanges = [

lib/pure/uri.nim

@@ -47,6 +47,49 @@ proc add*(url: var Url, a: Url) {.deprecated.} =
   url = url / a
 {.pop.}
 
+proc encodeUrl*(s: string): string =
+  ## Encodes a value to be HTTP safe: This means that characters in the set
+  ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result,
+  ## a space is converted to ``'+'`` and every other character is encoded as
+  ## ``'%xx'`` where ``xx`` denotes its hexadecimal value.
+  result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars
+  for i in 0..s.len-1:
+    case s[i]
+    of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i])
+    of ' ': add(result, '+')
+    else:
+      add(result, '%')
+      add(result, toHex(ord(s[i]), 2))
+      
+proc decodeUrl*(s: string): string =
+  ## Decodes a value from its HTTP representation: This means that a ``'+'``
+  ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal
+  ## value) is converted to the character with ordinal number ``xx``, and
+  ## and every other character is carried over.
+  proc handleHexChar(c: char, x: var int) {.inline.} =
+    case c
+    of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
+    of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
+    of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
+    else: assert(false)
+    
+  result = newString(s.len)
+  var i = 0
+  var j = 0
+  while i < s.len:
+    case s[i]
+    of '%':
+      var x = 0
+      handleHexChar(s[i+1], x)
+      handleHexChar(s[i+2], x)
+      inc(i, 2)
+      result[j] = chr(x)
+    of '+': result[j] = ' '
+    else: result[j] = s[i]
+    inc(i)
+    inc(j)
+  setLen(result, j)
+
 proc parseAuthority(authority: string, result: var Uri) =
   var i = 0
   var inPort = false
@@ -327,6 +370,11 @@ proc `$`*(u: Uri): string =
     result.add(u.anchor)
 
 when isMainModule:
+  block:
+    const test1 = "abc\L+def xyz"
+    doAssert encodeUrl(test1) == "abc%0A%2Bdef+xyz"
+    doAssert decodeUrl(encodeUrl(test1)) == test1
+    
   block:
     let str = "http://localhost"
     let test = parseUri(str)

lib/system.nim

@@ -1439,7 +1439,11 @@ const
     ## is the value that should be passed to `quit <#quit>`_ to indicate
     ## failure.
 
-var programResult* {.exportc: "nim_program_result".}: int
+when defined(nodejs):
+  var programResult* {.importc: "process.exitCode".}: int
+  programResult = 0
+else:
+  var programResult* {.exportc: "nim_program_result".}: int
   ## modify this variable to specify the exit code of the program
   ## under normal circumstances. When the program is terminated
   ## prematurely using ``quit``, this value is ignored.
@@ -3525,7 +3529,10 @@ when hasAlloc or defined(nimscript):
     ## .. code-block:: nim
     ##    var s = "abcdef"
     ##    assert s[1..3] == "bcd"
-    result = s.substr(s ^^ x.a, s ^^ x.b)
+    let a = s ^^ x.a
+    let L = (s ^^ x.b) - a + 1
+    result = newString(L)
+    for i in 0 ..< L: result[i] = s[i + a]
 
   proc `[]=`*[T, U](s: var string, x: HSlice[T, U], b: string) =
     ## slice assignment for strings. If
@@ -3752,6 +3759,7 @@ template assert*(cond: bool, msg = "") =
   ## that ``AssertionError`` is hidden from the effect system, so it doesn't
   ## produce ``{.raises: [AssertionError].}``. This exception is only supposed
   ## to be caught by unit testing frameworks.
+  ##
   ## The compiler may not generate any code at all for ``assert`` if it is
   ## advised to do so through the ``-d:release`` or ``--assertions:off``
   ## `command line switches <nimc.html#command-line-switches>`_.
@@ -3992,3 +4000,38 @@ when defined(windows) and appType == "console" and defined(nimSetUtf8CodePage):
   proc setConsoleOutputCP(codepage: cint): cint {.stdcall, dynlib: "kernel32",
     importc: "SetConsoleOutputCP".}
   discard setConsoleOutputCP(65001) # 65001 - utf-8 codepage
+
+
+when defined(nimHasRunnableExamples):
+  proc runnableExamples*(body: untyped) {.magic: "RunnableExamples".}
+    ## A section you should use to mark `runnable example`:idx: code with.
+    ##
+    ## - In normal debug and release builds code within
+    ##   a ``runnableExamples`` section is ignored.
+    ## - The documentation generator is aware of these examples and considers them
+    ##   part of the ``##`` doc comment. As the last step of documentation
+    ##   generation the examples are put into an ``$file_example.nim`` file,
+    ##   compiled and tested. The collected examples are
+    ##   put into their own module to ensure the examples do not refer to
+    ##   non-exported symbols.
+else:
+  template runnableExamples*(body: untyped) =
+    discard
+
+template doAssertRaises*(exception, code: untyped): typed =
+  ## Raises ``AssertionError`` if specified ``code`` does not raise the
+  ## specified exception.
+  runnableExamples:
+    doAssertRaises(ValueError):
+      raise newException(ValueError, "Hello World")
+
+  try:
+    block:
+      code
+    raiseAssert(astToStr(exception) & " wasn't raised by:\n" & astToStr(code))
+  except exception:
+    discard
+  except Exception as exc:
+    raiseAssert(astToStr(exception) &
+                " wasn't raised, another error was raised instead by:\n"&
+                astToStr(code))

lib/system/alloc.nim

@@ -8,8 +8,6 @@
 #
 
 # Low level allocator for Nim. Has been designed to support the GC.
-# TODO:
-# - make searching for block O(1)
 {.push profiler:off.}
 
 include osalloc
@@ -19,14 +17,17 @@ template track(op, address, size) =
     memTrackerOp(op, address, size)
 
 # We manage *chunks* of memory. Each chunk is a multiple of the page size.
-# Each chunk starts at an address that is divisible by the page size. Chunks
+# Each chunk starts at an address that is divisible by the page size.
-# that are bigger than ``ChunkOsReturn`` are returned back to the operating
-# system immediately.
 
 const
-  ChunkOsReturn = 256 * PageSize # 1 MB
+  InitialMemoryRequest = 128 * PageSize # 0.5 MB
-  InitialMemoryRequest = ChunkOsReturn div 2 # < ChunkOsReturn!
   SmallChunkSize = PageSize
+  MaxFli = 30
+  MaxLog2Sli = 5 # 32, this cannot be increased without changing 'uint32'
+                 # everywhere!
+  MaxSli = 1 shl MaxLog2Sli
+  FliOffset = 6
+  RealFli = MaxFli - FliOffset
 
 type
   PTrunk = ptr Trunk
@@ -99,10 +100,12 @@ type
   MemRegion = object
     minLargeObj, maxLargeObj: int
     freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk]
+    flBitmap: uint32
+    slBitmap: array[RealFli, uint32]
+    matrix: array[RealFli, array[MaxSli, PBigChunk]]
     llmem: PLLChunk
     currMem, maxMem, freeMem: int # memory sizes (allocated from OS)
     lastSize: int # needed for the case that OS gives us pages linearly
-    freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access
     chunkStarts: IntSet
     root, deleted, last, freeAvlNodes: PAvlNode
     locked, blockChunkSizeIncrease: bool # if locked, we cannot free pages.
@@ -110,7 +113,109 @@ type
     bottomData: AvlNode
     heapLinks: HeapLinks
 
-{.deprecated: [TMemRegion: MemRegion].}
+const
+  fsLookupTable: array[byte, int8] = [
+    -1'i8, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
+    4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
+    5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
+    5, 5, 5, 5, 5, 5, 5, 5,
+    6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
+    6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
+    6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
+    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+    7, 7, 7, 7, 7, 7, 7, 7
+  ]
+
+proc msbit(x: uint32): int {.inline.} =
+  let a = if x <= 0xff_ff:
+            (if x <= 0xff: 0 else: 8)
+          else:
+            (if x <= 0xff_ff_ff: 16 else: 24)
+  result = int(fsLookupTable[byte(x shr a)]) + a
+
+proc lsbit(x: uint32): int {.inline.} =
+  msbit(x and ((not x) + 1))
+
+proc setBit(nr: int; dest: var uint32) {.inline.} =
+  dest = dest or (1u32 shl (nr and 0x1f))
+
+proc clearBit(nr: int; dest: var uint32) {.inline.} =
+  dest = dest and not (1u32 shl (nr and 0x1f))
+
+proc mappingSearch(r, fl, sl: var int) {.inline.} =
+  #let t = (1 shl (msbit(uint32 r) - MaxLog2Sli)) - 1
+  # This diverges from the standard TLSF algorithm because we need to ensure
+  # PageSize alignment:
+  let t = roundup((1 shl (msbit(uint32 r) - MaxLog2Sli)), PageSize) - 1
+  r = r + t
+  fl = msbit(uint32 r)
+  sl = (r shr (fl - MaxLog2Sli)) - MaxSli
+  dec fl, FliOffset
+  r = r and not t
+  sysAssert((r and PageMask) == 0, "mappingSearch: still not aligned")
+
+# See http://www.gii.upv.es/tlsf/files/papers/tlsf_desc.pdf for details of
+# this algorithm.
+
+proc mappingInsert(r: int): tuple[fl, sl: int] {.inline.} =
+  sysAssert((r and PageMask) == 0, "mappingInsert: still not aligned")
+  result.fl = msbit(uint32 r)
+  result.sl = (r shr (result.fl - MaxLog2Sli)) - MaxSli
+  dec result.fl, FliOffset
+
+template mat(): untyped = a.matrix[fl][sl]
+
+proc findSuitableBlock(a: MemRegion; fl, sl: var int): PBigChunk {.inline.} =
+  let tmp = a.slBitmap[fl] and (not 0u32 shl sl)
+  result = nil
+  if tmp != 0:
+    sl = lsbit(tmp)
+    result = mat()
+  else:
+    fl = lsbit(a.flBitmap and (not 0u32 shl (fl + 1)))
+    if fl > 0:
+      sl = lsbit(a.slBitmap[fl])
+      result = mat()
+
+template clearBits(sl, fl) =
+  clearBit(sl, a.slBitmap[fl])
+  if a.slBitmap[fl] == 0u32:
+    # do not forget to cascade:
+    clearBit(fl, a.flBitmap)
+
+proc removeChunkFromMatrix(a: var MemRegion; b: PBigChunk) =
+  let (fl, sl) = mappingInsert(b.size)
+  if b.next != nil: b.next.prev = b.prev
+  if b.prev != nil: b.prev.next = b.next
+  if mat() == b:
+    mat() = b.next
+    if mat() == nil:
+      clearBits(sl, fl)
+  b.prev = nil
+  b.next = nil
+
+proc removeChunkFromMatrix2(a: var MemRegion; b: PBigChunk; fl, sl: int) =
+  mat() = b.next
+  if mat() != nil:
+    mat().prev = nil
+  else:
+    clearBits(sl, fl)
+  b.prev = nil
+  b.next = nil
+
+proc addChunkToMatrix(a: var MemRegion; b: PBigChunk) =
+  let (fl, sl) = mappingInsert(b.size)
+  b.prev = nil
+  b.next = mat()
+  if mat() != nil:
+    mat().prev = b
+  mat() = b
+  setBit(sl, a.slBitmap[fl])
+  setBit(fl, a.flBitmap)
 
 {.push stack_trace: off.}
 proc initAllocator() = discard "nothing to do anymore"
@@ -203,6 +308,7 @@ proc llDeallocAll(a: var MemRegion) =
     var next = it.next
     osDeallocPages(it, PageSize)
     it = next
+  a.llmem = nil
 
 proc intSetGet(t: IntSet, key: int): PTrunk =
   var it = t.data[key and high(t.data)]
@@ -369,6 +475,7 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk =
     result.prevSize = 0 or (result.prevSize and 1) # unknown
     # but do not overwrite 'used' field
   a.lastSize = size # for next request
+  sysAssert((cast[int](result) and PageMask) == 0, "requestOschunks: unaligned chunk")
 
 proc isAccessible(a: MemRegion, p: pointer): bool {.inline.} =
   result = contains(a.chunkStarts, pageIndex(p))
@@ -419,7 +526,7 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) =
     if isAccessible(a, ri) and chunkUnused(ri):
       sysAssert(not isSmallChunk(ri), "freeBigChunk 3")
       if not isSmallChunk(ri):
-        listRemove(a.freeChunksList, cast[PBigChunk](ri))
+        removeChunkFromMatrix(a, cast[PBigChunk](ri))
         inc(c.size, ri.size)
         excl(a.chunkStarts, pageIndex(ri))
   when coalescLeft:
@@ -430,49 +537,44 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) =
       if isAccessible(a, le) and chunkUnused(le):
         sysAssert(not isSmallChunk(le), "freeBigChunk 5")
         if not isSmallChunk(le):
-          listRemove(a.freeChunksList, cast[PBigChunk](le))
+          removeChunkFromMatrix(a, cast[PBigChunk](le))
           inc(le.size, c.size)
           excl(a.chunkStarts, pageIndex(c))
           c = cast[PBigChunk](le)
 
   incl(a, a.chunkStarts, pageIndex(c))
   updatePrevSize(a, c, c.size)
-  listAdd(a.freeChunksList, c)
+  addChunkToMatrix(a, c)
   # set 'used' to false:
   c.prevSize = c.prevSize and not 1
 
 proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) =
   var rest = cast[PBigChunk](cast[ByteAddress](c) +% size)
-  sysAssert(rest notin a.freeChunksList, "splitChunk")
   rest.size = c.size - size
   track("rest.origSize", addr rest.origSize, sizeof(int))
+  # XXX check if these two nil assignments are dead code given
+  # addChunkToMatrix's implementation:
   rest.next = nil
   rest.prev = nil
-  # size and not used
+  # size and not used:
   rest.prevSize = size
   sysAssert((size and 1) == 0, "splitChunk 2")
+  sysAssert((size and PageMask) == 0,
+      "splitChunk: size is not a multiple of the PageSize")
   updatePrevSize(a, c, rest.size)
   c.size = size
   incl(a, a.chunkStarts, pageIndex(rest))
-  listAdd(a.freeChunksList, rest)
+  addChunkToMatrix(a, rest)
 
 proc getBigChunk(a: var MemRegion, size: int): PBigChunk =
   # use first fit for now:
-  sysAssert((size and PageMask) == 0, "getBigChunk 1")
   sysAssert(size > 0, "getBigChunk 2")
-  result = a.freeChunksList
+  var size = size # roundup(size, PageSize)
-  block search:
+  var fl, sl: int
-    while result != nil:
+  mappingSearch(size, fl, sl)
-      sysAssert chunkUnused(result), "getBigChunk 3"
+  sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk")
-      if result.size == size:
+  result = findSuitableBlock(a, fl, sl)
-        listRemove(a.freeChunksList, result)
+  if result == nil:
-        break search
-      elif result.size > size:
-        listRemove(a.freeChunksList, result)
-        splitChunk(a, result, size)
-        break search
-      result = result.next
-      sysAssert result != a.freeChunksList, "getBigChunk 4"
     if size < InitialMemoryRequest:
       result = requestOsChunks(a, InitialMemoryRequest)
       splitChunk(a, result, size)
@@ -481,7 +583,10 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk =
       # if we over allocated split the chunk:
       if result.size > size:
         splitChunk(a, result, size)
-
+  else:
+    removeChunkFromMatrix2(a, result, fl, sl)
+    if result.size >= size + PageSize:
+      splitChunk(a, result, size)
   # set 'used' to to true:
   result.prevSize = 1
   track("setUsedToFalse", addr result.origSize, sizeof(int))
@@ -572,14 +677,14 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
                size == 0, "rawAlloc 21")
     sysAssert(allocInv(a), "rawAlloc: end small size")
   else:
-    size = roundup(requestedSize+bigChunkOverhead(), PageSize)
+    size = requestedSize + bigChunkOverhead() #  roundup(requestedSize+bigChunkOverhead(), PageSize)
     # allocate a large block
     var c = getBigChunk(a, size)
     sysAssert c.prev == nil, "rawAlloc 10"
     sysAssert c.next == nil, "rawAlloc 11"
-    sysAssert c.size == size, "rawAlloc 12"
     result = addr(c.data)
-    sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 13")
+    sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13")
+    sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary")
     if a.root == nil: a.root = getBottom(a)
     add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size)
   sysAssert(isAccessible(a, result), "rawAlloc 14")

lib/system/mmdisp.nim

@@ -42,7 +42,8 @@ type
 # Page size of the system; in most cases 4096 bytes. For exotic OS or
 # CPU this needs to be changed:
 const
-  PageShift = when defined(cpu16): 8 else: 12
+  PageShift = when defined(cpu16): 8 else: 12 # \
+    # my tests showed no improvments for using larger page sizes.
   PageSize = 1 shl PageShift
   PageMask = PageSize-1
 
@@ -343,7 +344,6 @@ elif defined(gogc):
 
   const goFlagNoZero: uint32 = 1 shl 3
   proc goRuntimeMallocGC(size: uint, typ: uint, flag: uint32): pointer {.importc: "runtime_mallocgc", dynlib: goLib.}
-  proc goFree(v: pointer) {.importc: "__go_free", dynlib: goLib.}
 
   proc goSetFinalizer(obj: pointer, f: pointer) {.importc: "set_finalizer", codegenDecl:"$1 $2$3 __asm__ (\"main.Set_finalizer\");\n$1 $2$3", dynlib: goLib.}
 
@@ -376,7 +376,6 @@ elif defined(gogc):
     result = goRuntimeMallocGC(roundup(newsize, sizeof(pointer)).uint, 0.uint, goFlagNoZero)
     copyMem(result, old, oldsize)
     zeroMem(cast[pointer](cast[ByteAddress](result) +% oldsize), newsize - oldsize)
-    goFree(old)
 
   proc nimGCref(p: pointer) {.compilerproc, inline.} = discard
   proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard
@@ -573,3 +572,11 @@ when not declared(nimNewSeqOfCap):
     cast[PGenericSeq](result).reserved = cap
 
 {.pop.}
+
+when not declared(ForeignCell):
+  type ForeignCell* = object
+    data*: pointer
+
+  proc protect*(x: pointer): ForeignCell = ForeignCell(data: x)
+  proc dispose*(x: ForeignCell) = discard
+  proc isNotForeign*(x: ForeignCell): bool = false

lib/system/osalloc.nim

@@ -166,7 +166,7 @@ elif defined(windows):
     # space heavily, so we now treat Windows as a strange unmap target.
     when reallyOsDealloc:
       if virtualFree(p, 0, MEM_RELEASE) == 0:
-        cprintf "yes, failing!"
+        cprintf "virtualFree failing!"
         quit 1
     #VirtualFree(p, size, MEM_DECOMMIT)
 

lib/system/sysstr.nim

@@ -259,7 +259,7 @@ proc incrSeqV2(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerProc.} =
     result.reserved = r
 
 proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {.
-    compilerRtl.} =
+    compilerRtl, inl.} =
   result = seq
   if result.space < newLen:
     let r = max(resize(result.space), newLen)
@@ -282,10 +282,11 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {.
             doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic)
           gch.tempStack.len = len0
       else:
-        for i in newLen..result.len-1:
+        if ntfNoRefs notin extGetCellType(result).base.flags:
-          forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +%
+          for i in newLen..result.len-1:
-                            GenericSeqSize +% (i*%elemSize)),
+            forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +%
-                            extGetCellType(result).base, waZctDecRef)
+                              GenericSeqSize +% (i*%elemSize)),
+                              extGetCellType(result).base, waZctDecRef)
 
     # XXX: zeroing out the memory can still result in crashes if a wiped-out
     # cell is aliased by another pointer (ie proc parameter or a let variable).

lib/system/threads.nim

@@ -255,9 +255,9 @@ when emulatedThreadVars:
   proc nimThreadVarsSize(): int {.noconv, importc: "NimThreadVarsSize".}
 
 # we preallocate a fixed size for thread local storage, so that no heap
-# allocations are needed. Currently less than 7K are used on a 64bit machine.
+# allocations are needed. Currently less than 16K are used on a 64bit machine.
 # We use ``float`` for proper alignment:
-const nimTlsSize {.intdefine.} = 8000
+const nimTlsSize {.intdefine.} = 16000
 type
   ThreadLocalStorage = array[0..(nimTlsSize div sizeof(float)), float]
 

lib/windows/winlean.nim

@@ -541,6 +541,7 @@ var
   SO_DONTLINGER* {.importc, header: "winsock2.h".}: cint
   SO_EXCLUSIVEADDRUSE* {.importc, header: "winsock2.h".}: cint # disallow local address reuse
   SO_ERROR* {.importc, header: "winsock2.h".}: cint
+  TCP_NODELAY* {.importc, header: "winsock2.h".}: cint
 
 proc `==`*(x, y: SocketHandle): bool {.borrow.}
 

nimsuggest/nimsuggest.nim

@@ -526,6 +526,9 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
     of cmdEnd: break
     of cmdLongoption, cmdShortOption:
       case p.key.normalize
+      of "help":
+        stdout.writeline(Usage)
+        quit()
       of "port":
         gPort = parseInt(p.val).Port
         gMode = mtcp

tests/async/tasyncall.nim

@@ -40,6 +40,16 @@ proc testVarargs(x, y, z: int): seq[int] =
 
   result = waitFor all(a, b, c)
 
+proc testWithDupes() =
+  var
+    tasks = newSeq[Future[void]](taskCount)
+    fut = futureWithoutValue()
+
+  for i in 0..<taskCount:
+    tasks[i] = fut
+
+  waitFor all(tasks)
+
 block:
     let
       startTime = cpuTime()
@@ -57,6 +67,13 @@ block:
 
     doAssert execTime * 1000 < taskCount * sleepDuration
 
+block:
+    let startTime = cpuTime()
+    testWithDupes()
+    let execTime = cpuTime() - startTime
+
+    doAssert execTime * 1000 < taskCount * sleepDuration
+
 block:
     let
       startTime = cpuTime()

tests/async/tasyncfile.nim

@@ -34,4 +34,19 @@ proc main() {.async.} =
     doAssert data == "foot\ntest2"
     file.close()
 
+  # Issue #5531
+  block:
+    removeFile(fn)
+    var file = openAsync(fn, fmWrite)
+    await file.write("test2")
+    file.close()
+    file = openAsync(fn, fmWrite)
+    await file.write("test3")
+    file.close()
+    file = openAsync(fn, fmRead)
+    let data = await file.readAll()
+    doAssert data == "test3"
+    file.close()
+
+
 waitFor main()

tests/ccgbugs/t6756.nim

@@ -0,0 +1,18 @@
+import typetraits
+type
+  A[T] = ref object
+    v: T
+
+template templ(o: A, op: untyped): untyped =
+  type T = type(o.v)
+
+  var res: A[T]
+
+  block:
+    var it {.inject.}: T
+    it = o.v
+    res = A[T](v: op)
+  res
+
+let a = A[int](v: 1)
+echo templ(a, it + 2)[]

tests/ccgbugs/tuple_canon.nim

@@ -85,3 +85,31 @@ proc go() =
     echo "vidx ", $vidx(hg, 1, 2, hiC)
 
 go()
+
+# another sighashes problem: In tuples we have to ignore ranges.
+
+type
+  Position = tuple[x, y: int16]
+  n16 = range[0'i16..high(int16)]
+
+proc print(pos: Position) =
+  echo $pos.x, ",", $pos.y
+
+var x = 0.n16
+var y = 0.n16
+print((x, y))
+
+
+# bug #6889
+proc createProgressSetterWithPropSetter[T](setter: proc(v: T)) = discard
+
+type A = distinct array[4, float32]
+type B = distinct array[3, float32]
+
+type Foo[T] = tuple
+    setter: proc(v: T)
+
+proc getFoo[T](): Foo[T] = discard
+
+createProgressSetterWithPropSetter(getFoo[A]().setter)
+createProgressSetterWithPropSetter(getFoo[B]().setter)

tests/concepts/texplain.nim

@@ -1,62 +1,66 @@
 discard """
   cmd: "nim c --verbosity:0 --colors:off $file"
   nimout: '''
-texplain.nim(99, 10) Hint: Non-matching candidates for e(y)
+texplain.nim(103, 10) Hint: Non-matching candidates for e(y)
 proc e(i: int): int
 
-texplain.nim(102, 7) Hint: Non-matching candidates for e(10)
+texplain.nim(106, 7) Hint: Non-matching candidates for e(10)
 proc e(o: ExplainedConcept): int
-texplain.nim(65, 6) ExplainedConcept: undeclared field: 'foo'
+texplain.nim(69, 6) ExplainedConcept: undeclared field: 'foo'
-texplain.nim(65, 6) ExplainedConcept: undeclared field: '.'
+texplain.nim(69, 6) ExplainedConcept: undeclared field: '.'
-texplain.nim(65, 6) ExplainedConcept: expression '.' cannot be called
+texplain.nim(69, 6) ExplainedConcept: expression '.' cannot be called
-texplain.nim(65, 5) ExplainedConcept: concept predicate failed
+texplain.nim(69, 5) ExplainedConcept: concept predicate failed
-texplain.nim(66, 6) ExplainedConcept: undeclared field: 'bar'
+texplain.nim(70, 6) ExplainedConcept: undeclared field: 'bar'
-texplain.nim(66, 6) ExplainedConcept: undeclared field: '.'
+texplain.nim(70, 6) ExplainedConcept: undeclared field: '.'
-texplain.nim(66, 6) ExplainedConcept: expression '.' cannot be called
+texplain.nim(70, 6) ExplainedConcept: expression '.' cannot be called
-texplain.nim(65, 5) ExplainedConcept: concept predicate failed
+texplain.nim(69, 5) ExplainedConcept: concept predicate failed
 
-texplain.nim(105, 10) Hint: Non-matching candidates for e(10)
+texplain.nim(109, 10) Hint: Non-matching candidates for e(10)
 proc e(o: ExplainedConcept): int
-texplain.nim(65, 6) ExplainedConcept: undeclared field: 'foo'
+texplain.nim(69, 6) ExplainedConcept: undeclared field: 'foo'
-texplain.nim(65, 6) ExplainedConcept: undeclared field: '.'
+texplain.nim(69, 6) ExplainedConcept: undeclared field: '.'
-texplain.nim(65, 6) ExplainedConcept: expression '.' cannot be called
+texplain.nim(69, 6) ExplainedConcept: expression '.' cannot be called
-texplain.nim(65, 5) ExplainedConcept: concept predicate failed
+texplain.nim(69, 5) ExplainedConcept: concept predicate failed
-texplain.nim(66, 6) ExplainedConcept: undeclared field: 'bar'
+texplain.nim(70, 6) ExplainedConcept: undeclared field: 'bar'
-texplain.nim(66, 6) ExplainedConcept: undeclared field: '.'
+texplain.nim(70, 6) ExplainedConcept: undeclared field: '.'
-texplain.nim(66, 6) ExplainedConcept: expression '.' cannot be called
+texplain.nim(70, 6) ExplainedConcept: expression '.' cannot be called
-texplain.nim(65, 5) ExplainedConcept: concept predicate failed
+texplain.nim(69, 5) ExplainedConcept: concept predicate failed
 
-texplain.nim(109, 20) Error: type mismatch: got (NonMatchingType)
+texplain.nim(113, 20) Error: type mismatch: got (NonMatchingType)
 but expected one of:
 proc e(o: ExplainedConcept): int
-texplain.nim(65, 5) ExplainedConcept: concept predicate failed
+texplain.nim(69, 5) ExplainedConcept: concept predicate failed
 proc e(i: int): int
 
-texplain.nim(110, 20) Error: type mismatch: got (NonMatchingType)
+expression: e(n)
+texplain.nim(114, 20) Error: type mismatch: got (NonMatchingType)
 but expected one of:
 proc r(o: RegularConcept): int
-texplain.nim(69, 5) RegularConcept: concept predicate failed
+texplain.nim(73, 5) RegularConcept: concept predicate failed
 proc r[T](a: SomeNumber; b: T; c: auto)
 proc r(i: string): int
 
-texplain.nim(111, 20) Hint: Non-matching candidates for r(y)
+expression: r(n)
+texplain.nim(115, 20) Hint: Non-matching candidates for r(y)
 proc r[T](a: SomeNumber; b: T; c: auto)
 proc r(i: string): int
 
-texplain.nim(119, 2) Error: type mismatch: got (MatchingType)
+texplain.nim(123, 2) Error: type mismatch: got (MatchingType)
 but expected one of:
 proc f(o: NestedConcept)
-texplain.nim(69, 6) RegularConcept: undeclared field: 'foo'
+texplain.nim(73, 6) RegularConcept: undeclared field: 'foo'
-texplain.nim(69, 6) RegularConcept: undeclared field: '.'
+texplain.nim(73, 6) RegularConcept: undeclared field: '.'
-texplain.nim(69, 6) RegularConcept: expression '.' cannot be called
+texplain.nim(73, 6) RegularConcept: expression '.' cannot be called
-texplain.nim(69, 5) RegularConcept: concept predicate failed
+texplain.nim(73, 5) RegularConcept: concept predicate failed
-texplain.nim(70, 6) RegularConcept: undeclared field: 'bar'
+texplain.nim(74, 6) RegularConcept: undeclared field: 'bar'
-texplain.nim(70, 6) RegularConcept: undeclared field: '.'
+texplain.nim(74, 6) RegularConcept: undeclared field: '.'
-texplain.nim(70, 6) RegularConcept: expression '.' cannot be called
+texplain.nim(74, 6) RegularConcept: expression '.' cannot be called
-texplain.nim(69, 5) RegularConcept: concept predicate failed
+texplain.nim(73, 5) RegularConcept: concept predicate failed
-texplain.nim(73, 5) NestedConcept: concept predicate failed
+texplain.nim(77, 5) NestedConcept: concept predicate failed
+
+expression: f(y)
 '''
-  line: 119
+  line: 123
   errormsg: "type mismatch: got (MatchingType)"
 """
 

tests/cpp/tcasts.nim

@@ -1,6 +1,7 @@
 discard """
   cmd: "nim cpp $file"
   output: ""
+  targets: "cpp"
 """
 
 block: #5979

tests/destructor/tatomicptrs.nim

@@ -0,0 +1,101 @@
+discard """
+  output: '''allocating
+allocating
+allocating
+55
+60
+99
+deallocating
+deallocating
+deallocating
+'''
+  cmd: '''nim c --newruntime $file'''
+"""
+
+type
+  SharedPtr*[T] = object
+    x: ptr T
+
+#proc isNil[T](s: SharedPtr[T]): bool {.inline.} = s.x.isNil
+
+template incRef(x) =
+  atomicInc(x.refcount)
+
+template decRef(x): untyped = atomicDec(x.refcount)
+
+proc makeShared*[T](x: T): SharedPtr[T] =
+  # XXX could benefit from 'sink' parameter.
+  # XXX could benefit from a macro that generates it.
+  result = cast[SharedPtr[T]](allocShared(sizeof(x)))
+  result.x[] = x
+  echo "allocating"
+
+proc `=destroy`*[T](dest: var SharedPtr[T]) =
+  var s = dest.x
+  if s != nil and decRef(s) == 0:
+    `=destroy`(s[])
+    deallocShared(s)
+    echo "deallocating"
+    dest.x = nil
+
+proc `=`*[T](dest: var SharedPtr[T]; src: SharedPtr[T]) =
+  var s = src.x
+  if s != nil: incRef(s)
+  #atomicSwap(dest, s)
+  # XXX use an atomic store here:
+  swap(dest.x, s)
+  if s != nil and decRef(s) == 0:
+    `=destroy`(s[])
+    deallocShared(s)
+    echo "deallocating"
+
+proc `=sink`*[T](dest: var SharedPtr[T]; src: SharedPtr[T]) =
+  ## XXX make this an atomic store:
+  if dest.x != src.x:
+    let s = dest.x
+    if s != nil:
+      `=destroy`(s[])
+      deallocShared(s)
+      echo "deallocating"
+    dest.x = src.x
+
+template `.`*[T](s: SharedPtr[T]; field: untyped): untyped =
+  s.x.field
+
+template `.=`*[T](s: SharedPtr[T]; field, value: untyped) =
+  s.x.field = value
+
+from macros import unpackVarargs
+
+template `.()`*[T](s: SharedPtr[T]; field: untyped, args: varargs[untyped]): untyped =
+  unpackVarargs(s.x.field, args)
+
+
+type
+  Tree = SharedPtr[TreeObj]
+  TreeObj = object
+    refcount: int
+    le, ri: Tree
+    data: int
+
+proc takesTree(a: Tree) =
+  if not a.isNil:
+    takesTree(a.le)
+    echo a.data
+    takesTree(a.ri)
+
+proc createTree(data: int): Tree =
+  result = makeShared(TreeObj(refcount: 1, data: data))
+
+proc createTree(data: int; le, ri: Tree): Tree =
+  result = makeShared(TreeObj(refcount: 1, le: le, ri: ri, data: data))
+
+
+proc main =
+  let le = createTree(55)
+  let ri = createTree(99)
+  let t = createTree(60, le, ri)
+  takesTree(t)
+
+main()
+

tests/destructor/tdestructor.nim

@@ -20,10 +20,10 @@ myobj destroyed
 ----
 myobj destroyed
 '''
+  cmd: '''nim c --newruntime $file'''
+  disabled: "true"
 """
 
-{.experimental.}
-
 type
   TMyObj = object
     x, y: int
@@ -61,7 +61,7 @@ proc `=destroy`(o: var TMyObj) =
   if o.p != nil: dealloc o.p
   echo "myobj destroyed"
 
-proc `=destroy`(o: var TMyGeneric1) =
+proc `=destroy`(o: var TMyGeneric1[int]) =
   echo "mygeneric1 destroyed"
 
 proc `=destroy`[A, B](o: var TMyGeneric2[A, B]) =

tests/destructor/tdestructor2.nim

@@ -1,27 +0,0 @@
-discard """
-  line: 23
-  nimout: " usage of a type with a destructor in a non destructible context"
-"""
-
-{.experimental.}
-
-type
-  TMyObj = object
-    x, y: int
-    p: pointer
-
-proc `=destroy`(o: var TMyObj) =
-  if o.p != nil: dealloc o.p
-
-proc open: TMyObj =
-  result = TMyObj(x: 1, y: 2, p: alloc(3))
-
-
-proc `$`(x: TMyObj): string = $x.y
-
-proc foo =
-  discard open()
-
-# XXX doesn't trigger this yet:
-#echo open()
-

tests/destructor/tdestructor3.nim

@@ -2,14 +2,14 @@ discard """
   output: '''assign
 destroy
 destroy
-destroy Foo: 5
 5
-destroy Foo: 123
+123
-123'''
+destroy Foo: 5
+destroy Foo: 123'''
+  cmd: '''nim c --newruntime $file'''
 """
 
 # bug #2821
-{.experimental.}
 
 type T = object
 

tests/destructor/tmove_objconstr.nim

@@ -0,0 +1,59 @@
+
+discard """
+output:  '''test created
+test destroyed 0
+1
+2
+3
+4
+Pony is dying!'''
+  cmd: '''nim c --newruntime $file'''
+"""
+
+# bug #4214
+type
+  Data = object
+    data: string
+    rc: int
+
+proc `=destroy`(d: var Data) =
+  dec d.rc
+  echo d.data, " destroyed ", d.rc
+
+proc `=`(dst: var Data, src: Data) =
+  echo src.data, " copied"
+  dst.data = src.data & " (copy)"
+  dec dst.rc
+  inc dst.rc
+
+proc initData(s: string): Data =
+  result = Data(data: s, rc: 1)
+  echo s, " created"
+
+proc pointlessWrapper(s: string): Data =
+  result = initData(s)
+
+proc main =
+  var x = pointlessWrapper"test"
+
+when isMainModule:
+  main()
+
+# bug #985
+
+type
+    Pony = object
+        name: string
+
+proc `=destroy`(o: var Pony) =
+  echo "Pony is dying!"
+
+proc getPony: Pony =
+    result.name = "Sparkles"
+
+iterator items(p: Pony): int =
+    for i in 1..4:
+        yield i
+
+for x in getPony():
+    echo x

tests/fragmentation/tfragment_alloc.nim

@@ -0,0 +1,27 @@
+
+discard """
+  output: '''occupied ok: true
+total ok: true'''
+"""
+
+import strutils, data
+
+proc main =
+  var m = 0
+  for i in 0..1000_000:
+    let size = sizes[i mod sizes.len]
+    let p = alloc(size)
+    if p == nil:
+      quit "could not serve request!"
+    dealloc p
+ #   c_fprintf(stdout, "iteration: %ld size: %ld\n", i, size)
+
+main()
+
+let occ = getOccupiedMem()
+let total = getTotalMem()
+
+# Current values on Win64: 824KiB / 106.191MiB
+
+echo "occupied ok: ", occ < 2 * 1024 * 1024
+echo "total ok: ", total < 120 * 1024 * 1024

tests/fragmentation/tfragment_gc.nim

@@ -0,0 +1,27 @@
+discard """
+  output: '''occupied ok: true
+total ok: true'''
+"""
+
+import strutils, data
+
+proc main =
+  var m = 0
+  # Since the GC test is slower than the alloc test, we only iterate 100_000 times here:
+  for i in 0..100_000:
+    let size = sizes[i mod sizes.len]
+    let p = newString(size)
+ #   c_fprintf(stdout, "iteration: %ld size: %ld\n", i, size)
+
+main()
+
+let occ = getOccupiedMem()
+let total = getTotalMem()
+
+# Concrete values on Win64: 58.152MiB / 188.285MiB
+
+echo "occupied ok: ", occ < 60 * 1024 * 1024
+let totalOk = total < 210 * 1024 * 1024
+if not totalOk:
+  echo "total peak memory ", formatSize(total)
+echo "total ok: ", totalOk

tests/js/tcodegendeclproc.nim

@@ -0,0 +1,11 @@
+discard """
+  output: '''
+-1
+8
+'''
+  ccodecheck: "'console.log(-1); function fac_' \\d+ '(n_' \\d+ ')'"
+"""
+proc fac(n: int): int {.codegenDecl: "console.log(-1); function $2($3)".} =
+  return n
+
+echo fac(8)

tests/js/tcodegendeclvar.nim

@@ -0,0 +1,10 @@
+discard """
+  output: '''
+-1
+2
+'''
+  ccodecheck: "'console.log(-1); var v_' \\d+ ' = [2]'"
+"""
+
+var v {.codegenDecl: "console.log(-1); var $2".} = 2
+echo v

tests/macros/tgettypeinst.nim

@@ -27,9 +27,10 @@ macro testX(x,inst0: typed; recurse: static[bool]; implX: typed): typed =
   let inst = x.getTypeInst
   let instr = inst.symToIdent.treeRepr
   let inst0r = inst0.symToIdent.treeRepr
-  #echo instr
+  if instr != inst0r:
-  #echo inst0r
+    echo "instr:\n", instr
-  doAssert(instr == inst0r)
+    echo "inst0r:\n", inst0r
+    doAssert(instr == inst0r)
 
   # check that getTypeImpl(x) is correct
   #  if implX is nil then compare to inst0
@@ -41,9 +42,10 @@ macro testX(x,inst0: typed; recurse: static[bool]; implX: typed): typed =
     else: implX[0][2]
   let implr = impl.symToIdent.treerepr
   let impl0r = impl0.symToIdent.treerepr
-  #echo implr
+  if implr != impl0r:
-  #echo impl0r
+    echo "implr:\n", implr
-  doAssert(implr == impl0r)
+    echo "impl0r:\n", impl0r
+    doAssert(implr == impl0r)
 
   result = newStmtList()
   #template echoString(s: string) = echo s.replace("\n","\n  ")
@@ -111,6 +113,18 @@ type
   Generic[T] = seq[int]
   Concrete = Generic[int]
 
+  Generic2[T1, T2] = seq[T1]
+  Concrete2 = Generic2[int, float]
+
+  Alias1 = float
+  Alias2 = Concrete
+  Alias3 = Concrete2
+
+  Vec[N: static[int],T] = object
+    arr: array[N,T]
+  Vec4[T] = Vec[4,T]
+
+
 test(bool)
 test(char)
 test(int)
@@ -144,11 +158,27 @@ test(Tree):
     left: ref Tree
     right: ref Tree
 test(Concrete):
-  type _ = Generic[int]
+  type _ = seq[int]
 test(Generic[int]):
   type _ = seq[int]
 test(Generic[float]):
   type _ = seq[int]
+test(Concrete2):
+  type _ = seq[int]
+test(Generic2[int,float]):
+  type _ = seq[int]
+test(Alias1):
+  type _ = float
+test(Alias2):
+  type _ = seq[int]
+test(Alias3):
+  type _ = seq[int]
+test(Vec[4,float32]):
+  type _ = object
+    arr: array[0..3,float32]
+test(Vec4[float32]):
+  type _ = object
+    arr: array[0..3,float32]
 
 # bug #4862
 static:

tests/osproc/texecps.nim

@@ -0,0 +1,32 @@
+discard """
+  file: "texecps.nim"
+  output: ""
+"""
+
+import osproc, streams, strutils, os
+
+const NumberOfProcesses = 13
+
+var gResults {.threadvar.}: seq[string]
+
+proc execCb(idx: int, p: Process) =
+  let exitCode = p.peekExitCode
+  if exitCode < len(gResults):
+    gResults[exitCode] = p.outputStream.readAll.strip
+
+when isMainModule:
+
+  if paramCount() == 0:
+    gResults = newSeq[string](NumberOfProcesses)
+    var checks = newSeq[string](NumberOfProcesses)
+    var commands = newSeq[string](NumberOfProcesses)
+    for i in 0..len(commands) - 1:
+      commands[i] = getAppFileName() & " " & $i
+      checks[i] = $i
+    let cres = execProcesses(commands, options = {poStdErrToStdOut},
+                             afterRunEvent = execCb)
+    doAssert(cres == len(commands) - 1)
+    doAssert(gResults == checks)
+  else:
+    echo paramStr(1)
+    programResult = parseInt(paramStr(1))

tests/parallel/tdisjoint_slice2.nim

@@ -11,12 +11,19 @@ discard """
   sortoutput: true
 """
 
-import threadpool
+import threadpool, locks
+
+var echoLock: Lock
+initLock echoLock
 
 proc f(a: openArray[int]) =
-  for x in a: echo x
+  for x in a:
+    withLock echoLock:
+      echo x
 
-proc f(a: int) = echo a
+proc f(a: int) =
+  withLock echoLock:
+    echo a
 
 proc main() =
   var a: array[0..9, int] = [0,1,2,3,4,5,6,7,8,9]

tests/specialops/tdotops.nim

@@ -11,7 +11,8 @@ no params call to a
 no params call to b
 100
 one param call to c with 10
-100'''
+100
+0 4'''
 """
 
 type
@@ -23,16 +24,16 @@ type
   T2 = object
     x: int
 
-proc `.`*(v: T1, f: string): int =
+template `.`*(v: T1, f: untyped): int =
-  echo "reading field ", f
+  echo "reading field ", astToStr(f)
-  return v.x
+  v.x
 
-proc `.=`(x: var T1, f: string{lit}, v: int) =
+template `.=`(t: var T1, f: untyped, v: int) =
-  echo "assigning ", f, " = ", v
+  echo "assigning ", astToStr(f), " = ", v
-  x.x = v
+  t.x = v
 
-template `.()`(x: T1, f: string, args: varargs[typed]): string =
+template `.()`(x: T1, f: untyped, args: varargs[typed]): string =
-  echo "call to ", f
+  echo "call to ", astToStr(f)
   "dot call"
 
 echo ""
@@ -47,13 +48,13 @@ echo t.y()
 var d = TD(t)
 assert(not compiles(d.y))
 
-proc `.`(v: T2, f: string): int =
+template `.`(v: T2, f: untyped): int =
-  echo "no params call to ", f
+  echo "no params call to ", astToStr(f)
-  return v.x
+  v.x
 
-proc `.`*(v: T2, f: string, a: int): int =
+template `.`*(v: T2, f: untyped, a: int): int =
-  echo "one param call to ", f, " with ", a
+  echo "one param call to ", astToStr(f), " with ", a
-  return v.x
+  v.x
 
 var tt = T2(x: 100)
 
@@ -63,3 +64,24 @@ echo tt.c(10)
 
 assert(not compiles(tt.d("x")))
 assert(not compiles(tt.d(1, 2)))
+
+# test simple usage that delegates fields:
+type
+  Other = object
+    a: int
+    b: string
+  MyObject = object
+    nested: Other
+    x, y: int
+
+template `.`(x: MyObject; field: untyped): untyped =
+  x.nested.field
+
+template `.=`(x: MyObject; field, value: untyped) =
+  x.nested.field = value
+
+var m: MyObject
+
+m.a = 4
+m.b = "foo"
+echo m.x, " ", m.a

tests/stdlib/tjsonmacro.nim

@@ -2,7 +2,7 @@ discard """
   file: "tjsonmacro.nim"
   output: ""
 """
-import json, strutils
+import json, strutils, options, tables
 
 when isMainModule:
   # Tests inspired by own use case (with some additional tests).
@@ -247,3 +247,137 @@ when isMainModule:
     let jnode = %b
     let data = jnode.to(Bird)
     doAssert data == b
+
+  block:
+    type
+      MsgBase = ref object of RootObj
+        name*: string
+
+      MsgChallenge = ref object of MsgBase
+        challenge*: string
+
+    let data = %*{"name": "foo", "challenge": "bar"}
+    let msg = data.to(MsgChallenge)
+    doAssert msg.name == "foo"
+    doAssert msg.challenge == "bar"
+
+  block:
+    type
+      Color = enum Red, Brown
+      Thing = object
+        animal: tuple[fur: bool, legs: int]
+        color: Color
+
+    var j = parseJson("""
+      {"animal":{"fur":true,"legs":6},"color":"Red"}
+    """)
+
+    let parsed = to(j, Thing)
+    doAssert parsed.animal.fur
+    doAssert parsed.animal.legs == 6
+    doAssert parsed.color == Red
+
+  block:
+    type
+      Car = object
+        engine: tuple[name: string, capacity: float]
+        model: string
+
+    let j = """
+      {"engine": {"name": "V8", "capacity": 5.5}, "model": "Skyline"}
+    """
+
+    var i = 0
+    proc mulTest: JsonNode =
+      i.inc()
+      return parseJson(j)
+
+    let parsed = mulTest().to(Car)
+    doAssert parsed.engine.name == "V8"
+
+    doAssert i == 1
+
+  block:
+    # Option[T] support!
+    type
+      Car1 = object # TODO: Codegen bug when `Car`
+        engine: tuple[name: string, capacity: Option[float]]
+        model: string
+        year: Option[int]
+
+    let noYear = """
+      {"engine": {"name": "V8", "capacity": 5.5}, "model": "Skyline"}
+    """
+
+    let noYearParsed = parseJson(noYear)
+    let noYearDeser = to(noYearParsed, Car1)
+    doAssert noYearDeser.engine.capacity == some(5.5)
+    doAssert noYearDeser.year.isNone
+    doAssert noYearDeser.engine.name == "V8"
+
+  # Table[T, Y] support.
+  block:
+    type
+      Friend = object
+        name: string
+        age: int
+
+      Dynamic = object
+        name: string
+        friends: Table[string, Friend]
+
+    let data = """
+      {"friends": {
+                    "John": {"name": "John", "age": 35},
+                    "Elizabeth": {"name": "Elizabeth", "age": 23}
+                  }, "name": "Dominik"}
+    """
+
+    let dataParsed = parseJson(data)
+    let dataDeser = to(dataParsed, Dynamic)
+    doAssert dataDeser.name == "Dominik"
+    doAssert dataDeser.friends["John"].age == 35
+    doAssert dataDeser.friends["Elizabeth"].age == 23
+
+  # JsonNode support
+  block:
+    type
+      Test = object
+        name: string
+        fallback: JsonNode
+
+    let data = """
+      {"name": "FooBar", "fallback": 56.42}
+    """
+
+    let dataParsed = parseJson(data)
+    let dataDeser = to(dataParsed, Test)
+    doAssert dataDeser.name == "FooBar"
+    doAssert dataDeser.fallback.kind == JFloat
+    doAssert dataDeser.fallback.getFloat() == 56.42
+
+  # int64, float64 etc support.
+  block:
+    type
+      Test1 = object
+        a: int8
+        b: int16
+        c: int32
+        d: int64
+        e: uint8
+        f: uint16
+        g: uint32
+        h: uint64
+        i: float32
+        j: float64
+
+    let data = """
+      {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7,
+       "h": 8, "i": 9.9, "j": 10.10}
+    """
+
+    let dataParsed = parseJson(data)
+    let dataDeser = to(dataParsed, Test1)
+    doAssert dataDeser.a == 1
+    doAssert dataDeser.f == 6
+    doAssert dataDeser.i == 9.9'f32

tests/stdlib/tjsonmacro_reject.nim

@@ -0,0 +1,18 @@
+discard """
+  file: "tjsonmacro_reject.nim"
+  line: 11
+  errormsg: "Use a named tuple instead of: (string, float)"
+"""
+
+import json
+
+type
+  Car = object
+    engine: (string, float)
+    model: string
+
+let j = """
+  {"engine": {"name": "V8", "capacity": 5.5}, model: "Skyline"}
+"""
+let parsed = parseJson(j)
+echo(to(parsed, Car))

tests/stdlib/tjsonmacro_reject2.nim

@@ -0,0 +1,21 @@
+discard """
+  file: "tjsonmacro_reject2.nim"
+  line: 10
+  errormsg: "The `to` macro does not support ref objects with cycles."
+"""
+import json
+
+type
+  Misdirection = object
+    cycle: Cycle
+
+  Cycle = ref object
+    foo: string
+    cycle: Misdirection
+
+let data = """
+  {"cycle": null}
+"""
+
+let dataParsed = parseJson(data)
+let dataDeser = to(dataParsed, Cycle)

tests/stdlib/tstring.nim

@@ -50,6 +50,10 @@ proc test_string_slice() =
   s[2..0] = numbers
   doAssert s == "ab1234567890cdefghijklmnopqrstuvwxyz"
 
+  # bug #6223
+  doAssertRaises(IndexError):
+    discard s[0..999]
+
   echo("OK")
 
 test_string_slice()

todo.txt

@@ -1,6 +1,9 @@
 version 1.0 battle plan
 =======================
 
+- introduce ``nkStmtListExpr`` for template/macro invokations to produce
+  better stack traces
+- let 'doAssert' analyse the expressions and produce more helpful output
 - fix "high priority" bugs
 - try to fix as many compiler crashes as reasonable
 
@@ -28,7 +31,6 @@ Not critical for 1.0
 - pragmas need 'bindSym' support
 - pragmas need re-work: 'push' is dangerous, 'hasPragma' does not work
   reliably with user-defined pragmas
-- memory manager: add a measure of fragmentation
 - we need a magic thisModule symbol
 - optimize 'genericReset'; 'newException' leads to code bloat
 
@@ -49,7 +51,6 @@ Bugs
 GC
 ==
 
-- use slightly bigger blocks in the allocator
 - resizing of strings/sequences could take into account the memory that
   is allocated
 

tools/finish.nim

@@ -32,7 +32,7 @@ proc downloadMingw(): DownloadResult =
   let curl = findExe"curl"
   var cmd: string
   if curl.len > 0:
-    cmd = curl & " --out " & "dist" / mingw & " " & url
+    cmd = quoteShell(curl) & " --out " & "dist" / mingw & " " & url
   elif fileExists"bin/nimgrab.exe":
     cmd = "bin/nimgrab.exe " & url & " dist" / mingw
   if cmd.len > 0:

tools/nimweb.nim

@@ -52,7 +52,7 @@ proc initConfigData(c: var TConfigData) =
   c.pdf = @[]
   c.infile = ""
   c.outdir = ""
-  c.nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off "
+  c.nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot "
   c.authors = ""
   c.projectTitle = ""
   c.projectName = ""