Merge branch 'devel' into sighashes

.gitignore

@@ -21,6 +21,7 @@ tags
 install.sh
 deinstall.sh
 
+doc/html/
 doc/*.html
 doc/*.pdf
 doc/*.idx
@@ -47,6 +48,7 @@ xcuserdata/
 /testresults.json
 testament.db
 /csources
+dist/
 
 # Private directories and files (IDEs)
 .*/

.travis.yml

@@ -22,10 +22,9 @@ script:
   - nim c koch
   - ./koch boot
   - ./koch boot -d:release
-  - nim e install_nimble.nims
+  - ./koch nimble
   - nim e tests/test_nimscript.nims
-  - nimble update
+  - nimble install zip -y
-  - nimble install zip
   - nimble install opengl
   - nimble install sdl1
   - nimble install jester@#head

compiler/ccgexprs.nim

@@ -354,6 +354,14 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
   else: internalError("genAssignment: " & $ty.kind)
 
+  if optMemTracker in p.options and dest.s in {OnHeap, OnUnknown}:
+    #writeStackTrace()
+    #echo p.currLineInfo, " requesting"
+    linefmt(p, cpsStmts, "#memTrackerWrite((void*)$1, $2, $3, $4);$n",
+            addrLoc(dest), rope getSize(dest.t),
+            makeCString(p.currLineInfo.toFullPath),
+            rope p.currLineInfo.safeLineNm)
+
 proc genDeepCopy(p: BProc; dest, src: TLoc) =
   var ty = skipTypes(dest.t, abstractVarRange)
   case ty.kind
@@ -1940,6 +1948,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) =
       d.s = OnStatic
 
 proc expr(p: BProc, n: PNode, d: var TLoc) =
+  p.currLineInfo = n.info
   case n.kind
   of nkSym:
     var sym = n.sym

compiler/ccgtypes.nim

@@ -221,6 +221,10 @@ proc cacheGetType(tab: TypeCache; sig: SigHash): Rope =
   # linear search is not necessary anymore:
   result = tab.getOrDefault(sig)
 
+proc addAbiCheck(m: BModule, t: PType, name: Rope) =
+  if isDefined("checkabi"):
+    addf(m.s[cfsTypeInfo], "NIM_CHECK_SIZE($1, $2);$n", [name, rope(getSize(t))])
+
 proc getTempName(m: BModule): Rope =
   result = m.tmpBase & rope(m.labels)
   inc m.labels
@@ -282,6 +286,11 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
     result = getSimpleTypeDesc(m, lastSon typ)
   else: result = nil
 
+  if result != nil and typ.isImportedType():
+    if cacheGetType(m.typeCache, typ) == nil:
+      idTablePut(m.typeCache, typ, result)
+      addAbiCheck(m, typ, result)
+
 proc pushType(m: BModule, typ: PType) =
   add(m.typeStack, typ)
 
@@ -678,6 +687,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       let foo = getTypeDescAux(m, t.sons[1], check)
       addf(m.s[cfsTypes], "typedef $1 $2[$3];$n",
            [foo, result, rope(n)])
+    else: addAbiCheck(m, t, result)
   of tyObject, tyTuple:
     if isImportedCppType(t) and origTyp.kind == tyGenericInst:
       # for instantiated templates we do not go through the type cache as the
@@ -739,7 +749,9 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       m.typeCache[sig] = result # always call for sideeffects:
       let recdesc = if t.kind != tyTuple: getRecordDesc(m, t, result, check)
                     else: getTupleDesc(m, t, result, check)
-      if not isImportedType(t): add(m.s[cfsTypes], recdesc)
+      if not isImportedType(t):
+        add(m.s[cfsTypes], recdesc)
+      elif tfIncompleteStruct notin t.flags: addAbiCheck(m, t, result)
   of tySet:
     result = getTypeName(m, t.lastSon, hashType t.lastSon) & "_Set"
     m.typeCache[sig] = result
@@ -841,6 +853,8 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) =
   #else MessageOut("can contain a cycle: " & typeToString(typ))
   if flags != 0:
     addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)])
+  if isDefined("nimTypeNames"):
+    addf(m.s[cfsTypeInit3], "$1.name = $2;$n", [name, makeCstring typeToString origType])
   discard cgsym(m, "TNimType")
   addf(m.s[cfsVars], "TNimType $1; /* $2 */$n",
        [name, rope(typeToString(typ))])

compiler/cgendata.nim

@@ -69,6 +69,7 @@ type
     beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
     threadVarAccessed*: bool  # true if the proc already accessed some threadvar
     lastLineInfo*: TLineInfo  # to avoid generating excessive 'nimln' statements
+    currLineInfo*: TLineInfo  # AST codegen will make this superfluous
     nestedTryStmts*: seq[PNode]   # in how many nested try statements we are
                                   # (the vars must be volatile then)
     inExceptBlock*: int       # are we currently inside an except block?

compiler/commands.nim

@@ -242,6 +242,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
   of "linetrace": result = contains(gOptions, optLineTrace)
   of "debugger": result = contains(gOptions, optEndb)
   of "profiler": result = contains(gOptions, optProfiler)
+  of "memtracker": result = contains(gOptions, optMemTracker)
   of "checks", "x": result = gOptions * ChecksOptions == ChecksOptions
   of "floatchecks":
     result = gOptions * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck}
@@ -264,6 +265,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
   of "implicitstatic": result = contains(gOptions, optImplicitStatic)
   of "patterns": result = contains(gOptions, optPatterns)
   of "experimental": result = gExperimentalMode
+  of "excessivestacktrace": result = contains(gGlobalOptions, optExcessiveStackTrace)
   else: invalidCmdLineOption(passCmd1, switch, info)
 
 proc processPath(path: string, info: TLineInfo,
@@ -445,6 +447,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
     processOnOffSwitch({optProfiler}, arg, pass, info)
     if optProfiler in gOptions: defineSymbol("profiler")
     else: undefSymbol("profiler")
+  of "memtracker":
+    processOnOffSwitch({optMemTracker}, arg, pass, info)
+    if optMemTracker in gOptions: defineSymbol("memtracker")
+    else: undefSymbol("memtracker")
   of "checks", "x": processOnOffSwitch(ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch({optNaNCheck, optInfCheck}, arg, pass, info)

compiler/importer.nim

@@ -100,7 +100,7 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   let ident = lookups.considerQuotedIdent(n)
   let s = strTableGet(fromMod.tab, ident)
   if s == nil:
-    localError(n.info, errUndeclaredIdentifier, ident.s)
+    errorUndeclaredIdentifier(c, n.info, ident.s)
   else:
     if s.kind == skStub: loadStub(s)
     if s.kind notin ExportableSymKinds:
@@ -162,12 +162,26 @@ proc importModuleAs(n: PNode, realModule: PSym): PSym =
 proc myImportModule(c: PContext, n: PNode): PSym =
   var f = checkModuleName(n)
   if f != InvalidFileIDX:
+    let L = c.graph.importStack.len
+    let recursion = c.graph.importStack.find(f)
+    c.graph.importStack.add f
+    #echo "adding ", toFullPath(f), " at ", L+1
+    if recursion >= 0:
+      var err = ""
+      for i in countup(recursion, L-1):
+        if i > recursion: err.add "\n"
+        err.add toFullPath(c.graph.importStack[i]) & " imports " &
+                toFullPath(c.graph.importStack[i+1])
+      c.recursiveDep = err
     result = importModuleAs(n, gImportModule(c.graph, c.module, f, c.cache))
+    #echo "set back to ", L
+    c.graph.importStack.setLen(L)
     # we cannot perform this check reliably because of
     # test: modules/import_in_config)
-    if result.info.fileIndex == c.module.info.fileIndex and
+    when true:
-        result.info.fileIndex == n.info.fileIndex:
+      if result.info.fileIndex == c.module.info.fileIndex and
-      localError(n.info, errGenerated, "A module cannot import itself")
+          result.info.fileIndex == n.info.fileIndex:
+        localError(n.info, errGenerated, "A module cannot import itself")
     if sfDeprecated in result.flags:
       message(n.info, warnDeprecated, result.name.s)
     #suggestSym(n.info, result, false)

compiler/lookups.nim

@@ -242,6 +242,15 @@ proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) =
     inc i
   localError(info, errGenerated, err)
 
+proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string) =
+  var err = "undeclared identifier: '" & name & "'"
+  if c.recursiveDep.len > 0:
+    err.add "\nThis might be caused by a recursive module dependency: "
+    err.add c.recursiveDep
+    # prevent excessive errors for 'nim check'
+    c.recursiveDep = nil
+  localError(info, errGenerated, err)
+
 proc lookUp*(c: PContext, n: PNode): PSym =
   # Looks up a symbol. Generates an error in case of nil.
   case n.kind
@@ -249,7 +258,7 @@ proc lookUp*(c: PContext, n: PNode): PSym =
     result = searchInScopes(c, n.ident).skipAlias(n)
     if result == nil:
       fixSpelling(n, n.ident, searchInScopes)
-      localError(n.info, errUndeclaredIdentifier, n.ident.s)
+      errorUndeclaredIdentifier(c, n.info, n.ident.s)
       result = errorSym(c, n)
   of nkSym:
     result = n.sym
@@ -258,7 +267,7 @@ proc lookUp*(c: PContext, n: PNode): PSym =
     result = searchInScopes(c, ident).skipAlias(n)
     if result == nil:
       fixSpelling(n, ident, searchInScopes)
-      localError(n.info, errUndeclaredIdentifier, ident.s)
+      errorUndeclaredIdentifier(c, n.info, ident.s)
       result = errorSym(c, n)
   else:
     internalError(n.info, "lookUp")
@@ -282,7 +291,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
       result = searchInScopes(c, ident, allExceptModule).skipAlias(n)
     if result == nil and checkUndeclared in flags:
       fixSpelling(n, ident, searchInScopes)
-      localError(n.info, errUndeclaredIdentifier, ident.s)
+      errorUndeclaredIdentifier(c, n.info, ident.s)
       result = errorSym(c, n)
     elif checkAmbiguity in flags and result != nil and
         contains(c.ambiguousSymbols, result.id):
@@ -307,7 +316,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
           result = strTableGet(m.tab, ident).skipAlias(n)
         if result == nil and checkUndeclared in flags:
           fixSpelling(n.sons[1], ident, searchInScopes)
-          localError(n.sons[1].info, errUndeclaredIdentifier, ident.s)
+          errorUndeclaredIdentifier(c, n.sons[1].info, ident.s)
           result = errorSym(c, n.sons[1])
       elif n.sons[1].kind == nkSym:
         result = n.sons[1].sym

compiler/modulegraphs.nim

@@ -36,6 +36,8 @@ type
     invalidTransitiveClosure: bool
     inclToMod*: Table[int32, int32] # mapping of include file to the
                                     # first module that included it
+    importStack*: seq[int32]  # The current import stack. Used for detecting recursive
+                              # module dependencies.
 
 {.this: g.}
 
@@ -44,12 +46,14 @@ proc newModuleGraph*(): ModuleGraph =
   initStrTable(result.packageSyms)
   result.deps = initIntSet()
   result.modules = @[]
+  result.importStack = @[]
   result.inclToMod = initTable[int32, int32]()
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(packageSyms)
   deps = initIntSet()
   modules = @[]
+  importStack = @[]
   inclToMod = initTable[int32, int32]()
 
 proc getModule*(g: ModuleGraph; fileIdx: int32): PSym =

compiler/modules.nim

@@ -231,6 +231,7 @@ proc compileProject*(graph: ModuleGraph; cache: IdentCache;
   wantMainModule()
   let systemFileIdx = fileInfoIdx(options.libpath / "system.nim")
   let projectFile = if projectFileIdx < 0: gProjectMainIdx else: projectFileIdx
+  graph.importStack.add projectFile
   if projectFile == systemFileIdx:
     discard graph.compileModule(projectFile, cache, {sfMainModule, sfSystemModule})
   else:

compiler/msgs.nim

@@ -35,7 +35,7 @@ type
     errNoneSpeedOrSizeExpectedButXFound, errGuiConsoleOrLibExpectedButXFound,
     errUnknownOS, errUnknownCPU, errGenOutExpectedButXFound,
     errArgsNeedRunOption, errInvalidMultipleAsgn, errColonOrEqualsExpected,
-    errExprExpected, errUndeclaredIdentifier, errUndeclaredField,
+    errExprExpected, errUndeclaredField,
     errUndeclaredRoutine, errUseQualifier,
     errTypeExpected,
     errSystemNeeds, errExecutionOfProgramFailed, errNotOverloadable,
@@ -197,7 +197,6 @@ const
     errInvalidMultipleAsgn: "multiple assignment is not allowed",
     errColonOrEqualsExpected: "\':\' or \'=\' expected, but found \'$1\'",
     errExprExpected: "expression expected, but found \'$1\'",
-    errUndeclaredIdentifier: "undeclared identifier: \'$1\'",
     errUndeclaredField: "undeclared field: \'$1\'",
     errUndeclaredRoutine: "attempting to call undeclared routine: \'$1\'",
     errUseQualifier: "ambiguous identifier: \'$1\' -- use a qualifier",
@@ -676,9 +675,8 @@ proc getInfoContext*(index: int): TLineInfo =
   if i >=% L: result = unknownLineInfo()
   else: result = msgContext[i]
 
-proc toFilename*(fileIdx: int32): string =
+template toFilename*(fileIdx: int32): string =
-  if fileIdx < 0: result = "???"
+  (if fileIdx < 0: "???" else: fileInfos[fileIdx].projPath)
-  else: result = fileInfos[fileIdx].projPath
 
 proc toFullPath*(fileIdx: int32): string =
   if fileIdx < 0: result = "???"

compiler/nim.nim

@@ -46,7 +46,7 @@ proc handleCmdLine(cache: IdentCache) =
     if gProjectName == "-":
       gProjectName = "stdinfile"
       gProjectFull = "stdinfile"
-      gProjectPath = getCurrentDir()
+      gProjectPath = canonicalizePath getCurrentDir()
       gProjectIsStdin = true
     elif gProjectName != "":
       try:
@@ -54,10 +54,10 @@ proc handleCmdLine(cache: IdentCache) =
       except OSError:
         gProjectFull = gProjectName
       let p = splitFile(gProjectFull)
-      gProjectPath = p.dir
+      gProjectPath = canonicalizePath p.dir
       gProjectName = p.name
     else:
-      gProjectPath = getCurrentDir()
+      gProjectPath = canonicalizePath getCurrentDir()
     loadConfigs(DefaultConfig) # load all config files
     let scriptFile = gProjectFull.changeFileExt("nims")
     if fileExists(scriptFile):

compiler/nimeval.nim

@@ -8,10 +8,9 @@
 #
 
 ## exposes the Nim VM to clients.
-
 import
   ast, modules, passes, passaux, condsyms,
-  options, nimconf, lists, sem, semdata, llstream, vm
+  options, nimconf, lists, sem, semdata, llstream, vm, modulegraphs, idents
 
 proc execute*(program: string) =
   passes.gIncludeFile = includeModule
@@ -27,7 +26,9 @@ proc execute*(program: string) =
   registerPass(evalPass)
 
   appendStr(searchPaths, options.libpath)
-  compileSystemModule()
+  var graph = newModuleGraph()
-  var m = makeStdinModule()
+  var cache = newIdentCache()
+  var m = makeStdinModule(graph)
   incl(m.flags, sfMainModule)
-  processModule(m, llStreamOpen(program), nil)
+  compileSystemModule(graph,cache)
+  processModule(graph,m, llStreamOpen(program), nil, cache)

compiler/options.nim

@@ -34,7 +34,8 @@ type                          # please make sure we have under 32 options
     optProfiler,              # profiler turned on
     optImplicitStatic,        # optimization: implicit at compile time
                               # evaluation
-    optPatterns               # en/disable pattern matching
+    optPatterns,              # en/disable pattern matching
+    optMemTracker
 
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**
@@ -231,10 +232,10 @@ proc canonicalizePath*(path: string): string =
 
 proc shortenDir*(dir: string): string =
   ## returns the interesting part of a dir
-  var prefix = getPrefixDir() & DirSep
+  var prefix = gProjectPath & DirSep
   if startsWith(dir, prefix):
     return substr(dir, len(prefix))
-  prefix = gProjectPath & DirSep
+  prefix = getPrefixDir() & DirSep
   if startsWith(dir, prefix):
     return substr(dir, len(prefix))
   result = dir

compiler/passes.nim

@@ -149,14 +149,25 @@ proc closePassesCached(a: var TPassContextArray) =
       m = gPasses[i].close(a[i], m)
     a[i] = nil                # free the memory here
 
+proc resolveMod(module, relativeTo: string): int32 =
+  let fullPath = findModule(module, relativeTo)
+  if fullPath.len == 0:
+    result = InvalidFileIDX
+  else:
+    result = fullPath.fileInfoIdx
+
 proc processImplicits(implicits: seq[string], nodeKind: TNodeKind,
-                      a: var TPassContextArray) =
+                      a: var TPassContextArray; m: PSym) =
+  # XXX fixme this should actually be relative to the config file!
+  let relativeTo = m.info.toFullPath
   for module in items(implicits):
-    var importStmt = newNodeI(nodeKind, gCmdLineInfo)
+    # implicit imports should not lead to a module importing itself
-    var str = newStrNode(nkStrLit, module)
+    if m.position != resolveMod(module, relativeTo):
-    str.info = gCmdLineInfo
+      var importStmt = newNodeI(nodeKind, gCmdLineInfo)
-    importStmt.addSon str
+      var str = newStrNode(nkStrLit, module)
-    if not processTopLevelStmt(importStmt, a): break
+      str.info = gCmdLineInfo
+      importStmt.addSon str
+      if not processTopLevelStmt(importStmt, a): break
 
 proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
                     rd: PRodReader; cache: IdentCache): bool {.discardable.} =
@@ -183,8 +194,8 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
         # modules to include between compilation runs? we'd need to track that
         # in ROD files. I think we should enable this feature only
         # for the interactive mode.
-        processImplicits implicitImports, nkImportStmt, a
+        processImplicits implicitImports, nkImportStmt, a, module
-        processImplicits implicitIncludes, nkIncludeStmt, a
+        processImplicits implicitIncludes, nkIncludeStmt, a, module
 
       while true:
         var n = parseTopLevelStmt(p)

compiler/pragmas.nim

@@ -323,7 +323,8 @@ proc processOption(c: PContext, n: PNode): bool =
     of wStacktrace: onOff(c, n, {optStackTrace})
     of wLinetrace: onOff(c, n, {optLineTrace})
     of wDebugger: onOff(c, n, {optEndb})
-    of wProfiler: onOff(c, n, {optProfiler})
+    of wProfiler: onOff(c, n, {optProfiler, optMemTracker})
+    of wMemTracker: onOff(c, n, {optMemTracker})
     of wByRef: onOff(c, n, {optByRef})
     of wDynlib: processDynLib(c, n, nil)
     of wOptimization:

compiler/semdata.nim

@@ -110,6 +110,7 @@ type
     cache*: IdentCache
     graph*: ModuleGraph
     signatures*: TStrTable
+    recursiveDep*: string
 
 proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair =
   result.genericSym = s

compiler/semexprs.nim

@@ -1553,7 +1553,7 @@ proc expectMacroOrTemplateCall(c: PContext, n: PNode): PSym =
   if isCallExpr(n):
     var expandedSym = qualifiedLookUp(c, n[0], {checkUndeclared})
     if expandedSym == nil:
-      localError(n.info, errUndeclaredIdentifier, n[0].renderTree)
+      errorUndeclaredIdentifier(c, n.info, n[0].renderTree)
       return errorSym(c, n[0])
 
     if expandedSym.kind notin {skMacro, skTemplate}:

compiler/semgnrc.nim

@@ -107,7 +107,7 @@ proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags,
   var s = searchInScopes(c, ident).skipAlias(n)
   if s == nil:
     if ident.id notin ctx.toMixin and withinMixin notin flags:
-      localError(n.info, errUndeclaredIdentifier, ident.s)
+      errorUndeclaredIdentifier(c, n.info, ident.s)
   else:
     if withinBind in flags:
       result = symChoice(c, n, s, scClosed)
@@ -195,7 +195,7 @@ proc semGenericStmt(c: PContext, n: PNode,
     if s == nil and withinMixin notin flags and
         fn.kind in {nkIdent, nkAccQuoted} and
         considerQuotedIdent(fn).id notin ctx.toMixin:
-      localError(n.info, errUndeclaredIdentifier, fn.renderTree)
+      errorUndeclaredIdentifier(c, n.info, fn.renderTree)
 
     var first = 0
     var mixinContext = false

compiler/semmagic.nim

@@ -143,7 +143,7 @@ proc semBindSym(c: PContext, n: PNode): PNode =
     var sc = symChoice(c, id, s, TSymChoiceRule(isMixin.intVal))
     result.add(sc)
   else:
-    localError(n.sons[1].info, errUndeclaredIdentifier, sl.strVal)
+    errorUndeclaredIdentifier(c, n.sons[1].info, sl.strVal)
 
 proc semShallowCopy(c: PContext, n: PNode, flags: TExprFlags): PNode
 

compiler/wordrecg.nim

@@ -34,7 +34,7 @@ type
 
     wColon, wColonColon, wEquals, wDot, wDotDot,
     wStar, wMinus,
-    wMagic, wThread, wFinal, wProfiler, wObjChecks,
+    wMagic, wThread, wFinal, wProfiler, wMemTracker, wObjChecks,
     wIntDefine, wStrDefine,
 
     wDestroy,
@@ -121,7 +121,7 @@ const
 
     ":", "::", "=", ".", "..",
     "*", "-",
-    "magic", "thread", "final", "profiler", "objchecks", "intdefine", "strdefine",
+    "magic", "thread", "final", "profiler", "memtracker", "objchecks", "intdefine", "strdefine",
 
     "destroy",
 

doc/advopt.txt

@@ -61,6 +61,9 @@ Advanced options:
   --taintMode:on|off        turn taint mode on|off
   --implicitStatic:on|off   turn implicit compile time evaluation on|off
   --patterns:on|off         turn pattern matching on|off
+  --memTracker:on|off       turn memory tracker on|off
+  --excessiveStackTrace:on|off
+                            stack traces use full file paths
   --skipCfg                 do not read the general configuration file
   --skipUserCfg             do not read the user's configuration file
   --skipParentCfg           do not read the parent dirs' configuration files

doc/lib.rst

@@ -77,8 +77,9 @@ Collections and algorithms
 * `lists <lists.html>`_
   Nim linked list support. Contains singly and doubly linked lists and
   circular lists ("rings").
-* `queues <queues.html>`_
+* `deques <deques.html>`_
-  Implementation of a queue. The underlying implementation uses a ``seq``.
+  Implementation of a double-ended queue.
+  The underlying implementation uses a ``seq``.
 * `intsets <intsets.html>`_
   Efficient implementation of a set of ints as a sparse bit set.
 * `critbits <critbits.html>`_

doc/nimc.rst

@@ -258,6 +258,10 @@ Define               Effect
 ``ssl``              Enables OpenSSL support for the sockets module.
 ``memProfiler``      Enables memory profiling for the native GC.
 ``uClibc``           Use uClibc instead of libc. (Relevant for Unix-like OSes)
+``checkAbi``         When using types from C headers, add checks that compare
+                     what's in the Nim file with what's in the C header
+                     (requires a C compiler with _Static_assert support, like
+                     any C11 compiler)
 ==================   =========================================================
 
 

doc/nims.rst

@@ -75,22 +75,8 @@ done:
 Nimble integration
 ==================
 
-A ``project.nims`` file can also be used as an alternative to
+See the `Nimble readme <https://github.com/nim-lang/nimble#readme>`_
-a ``project.nimble`` file to specify the meta information (for example, author,
+for more information.
-description) and dependencies of a Nimble package. This means you can easily
-have platform specific dependencies:
-
-.. code-block:: nim
-
-  version = "1.0"
-  author = "The green goo."
-  description = "Lexer generation and regex implementation for Nim."
-  license = "MIT"
-
-  when defined(windows):
-    requires "oldwinapi >= 1.0"
-  else:
-    requires "gtk2 >= 1.0"
 
 
 

lib/nimbase.h

@@ -459,3 +459,7 @@ typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == siz
 #elif defined(__FreeBSD__)
 #  include <sys/types.h>
 #endif
+
+/* Compile with -d:checkAbi and a sufficiently C11:ish compiler to enable */
+#define NIM_CHECK_SIZE(typ, sz) \
+  _Static_assert(sizeof(typ) == sz, "Nim & C disagree on type size")

lib/pure/asyncdispatch.nim

@@ -11,7 +11,7 @@ include "system/inclrtl"
 
 import os, oids, tables, strutils, times, heapqueue
 
-import nativesockets, net, queues
+import nativesockets, net, deques
 
 export Port, SocketFlag
 
@@ -164,7 +164,7 @@ include includes/asyncfutures
 type
   PDispatcherBase = ref object of RootRef
     timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
-    callbacks: Queue[proc ()]
+    callbacks: Deque[proc ()]
 
 proc processTimers(p: PDispatcherBase) {.inline.} =
   while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt:
@@ -172,7 +172,7 @@ proc processTimers(p: PDispatcherBase) {.inline.} =
 
 proc processPendingCallbacks(p: PDispatcherBase) =
   while p.callbacks.len > 0:
-    var cb = p.callbacks.dequeue()
+    var cb = p.callbacks.popFirst()
     cb()
 
 proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
@@ -230,7 +230,7 @@ when defined(windows) or defined(nimdoc):
     result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1)
     result.handles = initSet[AsyncFD]()
     result.timers.newHeapQueue()
-    result.callbacks = initQueue[proc ()](64)
+    result.callbacks = initDeque[proc ()](64)
 
   var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
   proc getGlobalDispatcher*(): PDispatcher =
@@ -987,7 +987,7 @@ else:
     new result
     result.selector = newSelector()
     result.timers.newHeapQueue()
-    result.callbacks = initQueue[proc ()](64)
+    result.callbacks = initDeque[proc ()](64)
 
   var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
   proc getGlobalDispatcher*(): PDispatcher =
@@ -1043,6 +1043,26 @@ else:
     p.selector[fd.SocketHandle].data.PData.writeCBs.add(cb)
     update(fd, p.selector[fd.SocketHandle].events + {EvWrite})
 
+  template processCallbacks(callbacks: expr) =
+    # Callback may add items to ``callbacks`` which causes issues if
+    # we are iterating over it at the same time. We therefore
+    # make a copy to iterate over.
+    let currentCBs = callbacks
+    callbacks = @[]
+    # Using another sequence because callbacks themselves can add
+    # other callbacks.
+    var newCBs: seq[Callback] = @[]
+    for cb in currentCBs:
+      if newCBs.len > 0:
+        # A callback has already returned with EAGAIN, don't call any
+        # others until next `poll`.
+        newCBs.add(cb)
+      else:
+        if not cb(data.fd):
+          # Callback wants to be called again.
+          newCBs.add(cb)
+    callbacks = newCBs & callbacks
+
   proc poll*(timeout = 500) =
     let p = getGlobalDispatcher()
 
@@ -1056,23 +1076,10 @@ else:
         # `recv(...)` routines.
 
         if EvRead in info.events or info.events == {EvError}:
-          # Callback may add items to ``data.readCBs`` which causes issues if
+          processCallbacks(data.readCBs)
-          # we are iterating over ``data.readCBs`` at the same time. We therefore
-          # make a copy to iterate over.
-          let currentCBs = data.readCBs
-          data.readCBs = @[]
-          for cb in currentCBs:
-            if not cb(data.fd):
-              # Callback wants to be called again.
-              data.readCBs.add(cb)
 
         if EvWrite in info.events or info.events == {EvError}:
-          let currentCBs = data.writeCBs
+          processCallbacks(data.writeCBs)
-          data.writeCBs = @[]
-          for cb in currentCBs:
-            if not cb(data.fd):
-              # Callback wants to be called again.
-              data.writeCBs.add(cb)
 
         if info.key in p.selector:
           var newEvents: set[Event]
@@ -1410,7 +1417,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} =
 proc callSoon*(cbproc: proc ()) =
   ## Schedule `cbproc` to be called as soon as possible.
   ## The callback is called when control returns to the event loop.
-  getGlobalDispatcher().callbacks.enqueue(cbproc)
+  getGlobalDispatcher().callbacks.addLast(cbproc)
 
 proc runForever*() =
   ## Begins a never ending global dispatcher poll loop.

lib/pure/asyncfile.nim

@@ -118,8 +118,8 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
   ## Read ``size`` bytes from the specified file asynchronously starting at
   ## the current position of the file pointer.
   ##
-  ## If the file pointer is past the end of the file then an empty string is
+  ## If the file pointer is past the end of the file then zero is returned
-  ## returned.
+  ## and no bytes are read into ``buf``
   var retFuture = newFuture[int]("asyncfile.readBuffer")
 
   when defined(windows) or defined(nimdoc):
@@ -149,7 +149,11 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
       let err = osLastError()
       if err.int32 != ERROR_IO_PENDING:
         GC_unref(ol)
-        retFuture.fail(newException(OSError, osErrorMsg(err)))
+        if err.int32 == ERROR_HANDLE_EOF:
+          # This happens in Windows Server 2003
+          retFuture.complete(0)
+        else:
+          retFuture.fail(newException(OSError, osErrorMsg(err)))
     else:
       # Request completed immediately.
       var bytesRead: DWord
@@ -233,7 +237,12 @@ proc read*(f: AsyncFile, size: int): Future[string] =
           dealloc buffer
           buffer = nil
         GC_unref(ol)
-        retFuture.fail(newException(OSError, osErrorMsg(err)))
+
+        if err.int32 == ERROR_HANDLE_EOF:
+          # This happens in Windows Server 2003
+          retFuture.complete("")
+        else:
+          retFuture.fail(newException(OSError, osErrorMsg(err)))
     else:
       # Request completed immediately.
       var bytesRead: DWord

lib/pure/collections/deques.nim

@@ -0,0 +1,266 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2012 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implementation of a `deque`:idx: (double-ended queue).
+## The underlying implementation uses a ``seq``.
+##
+## None of the procs that get an individual value from the deque can be used
+## on an empty deque.
+## If compiled with `boundChecks` option, those procs will raise an `IndexError`
+## on such access. This should not be relied upon, as `-d:release` will
+## disable those checks and may return garbage or crash the program.
+##
+## As such, a check to see if the deque is empty is needed before any
+## access, unless your program logic guarantees it indirectly.
+##
+## .. code-block:: Nim
+##   proc foo(a, b: Positive) =  # assume random positive values for `a` and `b`
+##     var deq = initDeque[int]()  # initializes the object
+##     for i in 1 ..< a: deq.addLast i  # populates the deque
+##
+##     if b < deq.len:  # checking before indexed access
+##       echo "The element at index position ", b, " is ", deq[b]
+##
+##     # The following two lines don't need any checking on access due to the
+##     # logic of the program, but that would not be the case if `a` could be 0.
+##     assert deq.peekFirst == 1
+##     assert deq.peekLast == a
+##
+##     while deq.len > 0:  # checking if the deque is empty
+##       echo deq.removeLast()
+##
+## Note: For inter thread communication use
+## a `Channel <channels.html>`_ instead.
+
+import math
+
+type
+  Deque*[T] = object
+    ## A double-ended queue backed with a ringed seq buffer.
+    data: seq[T]
+    head, tail, count, mask: int
+
+proc initDeque*[T](initialSize: int = 4): Deque[T] =
+  ## Create a new deque.
+  ## Optionally, the initial capacity can be reserved via `initialSize` as a
+  ## performance optimization. The length of a newly created deque will still
+  ## be 0.
+  ##
+  ## `initialSize` needs to be a power of two. If you need to accept runtime
+  ## values for this you could use the ``nextPowerOfTwo`` proc from the
+  ## `math <math.html>`_ module.
+  assert isPowerOfTwo(initialSize)
+  result.mask = initialSize-1
+  newSeq(result.data, initialSize)
+
+proc len*[T](deq: Deque[T]): int {.inline.} =
+  ## Return the number of elements of `deq`.
+  result = deq.count
+
+template emptyCheck(deq) =
+  # Bounds check for the regular deque access.
+  when compileOption("boundChecks"):
+    if unlikely(deq.count < 1):
+      raise newException(IndexError, "Empty deque.")
+
+template xBoundsCheck(deq, i) =
+  # Bounds check for the array like accesses.
+  when compileOption("boundChecks"):  # d:release should disable this.
+    if unlikely(i >= deq.count):  # x < deq.low is taken care by the Natural parameter
+      raise newException(IndexError,
+                         "Out of bounds: " & $i & " > " & $(deq.count - 1))
+
+proc `[]`*[T](deq: Deque[T], i: Natural) : T {.inline.} =
+  ## Access the i-th element of `deq` by order from first to last.
+  ## deq[0] is the first, deq[^1] is the last.
+  xBoundsCheck(deq, i)
+  return deq.data[(deq.first + i) and deq.mask]
+
+proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} =
+  ## Access the i-th element of `deq` and returns a mutable
+  ## reference to it.
+  xBoundsCheck(deq, i)
+  return deq.data[(deq.head + i) and deq.mask]
+
+proc `[]=`* [T] (deq: var Deque[T], i: Natural, val : T) {.inline.} =
+  ## Change the i-th element of `deq`.
+  xBoundsCheck(deq, i)
+  deq.data[(deq.head + i) and deq.mask] = val
+
+iterator items*[T](deq: Deque[T]): T =
+  ## Yield every element of `deq`.
+  var i = deq.head
+  for c in 0 ..< deq.count:
+    yield deq.data[i]
+    i = (i + 1) and deq.mask
+
+iterator mitems*[T](deq: var Deque[T]): var T =
+  ## Yield every element of `deq`.
+  var i = deq.head
+  for c in 0 ..< deq.count:
+    yield deq.data[i]
+    i = (i + 1) and deq.mask
+
+iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] =
+  ## Yield every (position, value) of `deq`.
+  var i = deq.head
+  for c in 0 ..< deq.count:
+    yield (c, deq.data[i])
+    i = (i + 1) and deq.mask
+
+proc contains*[T](deq: Deque[T], item: T): bool {.inline.} =
+  ## Return true if `item` is in `deq` or false if not found. Usually used
+  ## via the ``in`` operator. It is the equivalent of ``deq.find(item) >= 0``.
+  ##
+  ## .. code-block:: Nim
+  ##   if x in q:
+  ##     assert q.contains x
+  for e in deq:
+    if e == item: return true
+  return false
+
+proc expandIfNeeded[T](deq: var Deque[T]) =
+  var cap = deq.mask + 1
+  if unlikely(deq.count >= cap):
+    var n = newSeq[T](cap * 2)
+    for i, x in deq:  # don't use copyMem because the GC and because it's slower.
+      shallowCopy(n[i], x)
+    shallowCopy(deq.data, n)
+    deq.mask = cap * 2 - 1
+    deq.tail = deq.count
+    deq.head = 0
+
+proc addFirst*[T](deq: var Deque[T], item: T) =
+  ## Add an `item` to the beginning of the `deq`.
+  expandIfNeeded(deq)
+  inc deq.count
+  deq.head = (deq.head - 1) and deq.mask
+  deq.data[deq.head] = item
+
+proc addLast*[T](deq: var Deque[T], item: T) =
+  ## Add an `item` to the end of the `deq`.
+  expandIfNeeded(deq)
+  inc deq.count
+  deq.data[deq.tail] = item
+  deq.tail = (deq.tail + 1) and deq.mask
+
+proc peekFirst*[T](deq: Deque[T]): T {.inline.}=
+  ## Returns the first element of `deq`, but does not remove it from the deque.
+  emptyCheck(deq)
+  result = deq.data[deq.head]
+
+proc peekLast*[T](deq: Deque[T]): T {.inline.} =
+  ## Returns the last element of `deq`, but does not remove it from the deque.
+  emptyCheck(deq)
+  result = deq.data[(deq.tail - 1) and deq.mask]
+
+proc default[T](t: typedesc[T]): T {.inline.} = discard
+proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} =
+  ## Remove and returns the first element of the `deq`.
+  emptyCheck(deq)
+  dec deq.count
+  result = deq.data[deq.head]
+  deq.data[deq.head] = default(type(result))
+  deq.head = (deq.head + 1) and deq.mask
+
+proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} =
+  ## Remove and returns the last element of the `deq`.
+  emptyCheck(deq)
+  dec deq.count
+  deq.tail = (deq.tail - 1) and deq.mask
+  result = deq.data[deq.tail]
+  deq.data[deq.tail] = default(type(result))
+
+proc `$`*[T](deq: Deque[T]): string =
+  ## Turn a deque into its string representation.
+  result = "["
+  for x in deq:
+    if result.len > 1: result.add(", ")
+    result.add($x)
+  result.add("]")
+
+when isMainModule:
+  var deq = initDeque[int](1)
+  deq.addLast(4)
+  deq.addFirst(9)
+  deq.addFirst(123)
+  var first = deq.popFirst()
+  deq.addLast(56)
+  assert(deq.peekLast() == 56)
+  deq.addLast(6)
+  assert(deq.peekLast() == 6)
+  var second = deq.popFirst()
+  deq.addLast(789)
+  assert(deq.peekLast() == 789)
+
+  assert first == 123
+  assert second == 9
+  assert($deq == "[4, 56, 6, 789]")
+
+  assert deq[0] == deq.peekFirst and deq.peekFirst == 4
+  assert deq[^1] == deq.peekLast and deq.peekLast == 789
+  deq[0] = 42
+  deq[^1] = 7
+
+  assert 6 in deq and 789 notin deq
+  assert deq.find(6) >= 0
+  assert deq.find(789) < 0
+
+  for i in -2 .. 10:
+    if i in deq:
+      assert deq.contains(i) and deq.find(i) >= 0
+    else:
+      assert(not deq.contains(i) and deq.find(i) < 0)
+
+  when compileOption("boundChecks"):
+    try:
+      echo deq[99]
+      assert false
+    except IndexError:
+      discard
+
+    try:
+      assert deq.len == 4
+      for i in 0 ..< 5: deq.popFirst()
+      assert false
+    except IndexError:
+      discard
+
+  # grabs some types of resize error.
+  deq = initDeque[int]()
+  for i in 1 .. 4: deq.addLast i
+  deq.popFirst()
+  deq.popLast()
+  for i in 5 .. 8: deq.addFirst i
+  assert $deq == "[8, 7, 6, 5, 2, 3]"
+
+  # Similar to proc from the documentation example
+  proc foo(a, b: Positive) = # assume random positive values for `a` and `b`.
+    var deq = initDeque[int]()
+    assert deq.len == 0
+    for i in 1 .. a: deq.addLast i
+
+    if b < deq.len: # checking before indexed access.
+      assert deq[b] == b + 1
+
+    # The following two lines don't need any checking on access due to the logic
+    # of the program, but that would not be the case if `a` could be 0.
+    assert deq.peekFirst == 1
+    assert deq.peekLast == a
+
+    while deq.len > 0: # checking if the deque is empty
+      assert deq.popFirst() > 0
+
+  #foo(0,0)
+  foo(8,5)
+  foo(10,9)
+  foo(1,1)
+  foo(2,1)
+  foo(1,5)
+  foo(3,2)

lib/pure/collections/queues.nim

@@ -39,8 +39,10 @@
 
 import math
 
+{.warning: "`queues` module is deprecated - use `deques` instead".}
+
 type
-  Queue*[T] = object ## A queue.
+  Queue* {.deprecated.} [T] = object ## A queue.
     data: seq[T]
     rd, wr, count, mask: int
 

lib/pure/collections/tableimpl.nim

@@ -39,16 +39,22 @@ template rawGetKnownHCImpl() {.dirty.} =
     h = nextTry(h, maxHash(t))
   result = -1 - h                   # < 0 => MISSING; insert idx = -1 - result
 
-template rawGetImpl() {.dirty.} =
+template genHashImpl(key, hc: typed) =
   hc = hash(key)
   if hc == 0:       # This almost never taken branch should be very predictable.
     hc = 314159265  # Value doesn't matter; Any non-zero favorite is fine.
+
+template genHash(key: typed): Hash =
+  var res: Hash
+  genHashImpl(key, res)
+  res
+
+template rawGetImpl() {.dirty.} =
+  genHashImpl(key, hc)
   rawGetKnownHCImpl()
 
 template rawGetDeepImpl() {.dirty.} =   # Search algo for unconditional add
-  hc = hash(key)
+  genHashImpl(key, hc)
-  if hc == 0:
-    hc = 314159265
   var h: Hash = hc and maxHash(t)
   while isFilled(t.data[h].hcode):
     h = nextTry(h, maxHash(t))

lib/pure/collections/tables.nim

@@ -224,7 +224,7 @@ template withValue*[A, B](t: var Table[A, B], key: A,
 
 iterator allValues*[A, B](t: Table[A, B]; key: A): B =
   ## iterates over any value in the table `t` that belongs to the given `key`.
-  var h: Hash = hash(key) and high(t.data)
+  var h: Hash = genHash(key) and high(t.data)
   while isFilled(t.data[h].hcode):
     if t.data[h].key == key:
       yield t.data[h].val
@@ -479,7 +479,7 @@ proc clear*[A, B](t: var OrderedTableRef[A, B]) =
   ## Resets the table so that is is empty.
   clear(t[])
 
-template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} =
+template forAllOrderedPairs(yieldStmt: untyped): typed {.dirty.} =
   var h = t.first
   while h >= 0:
     var nxt = t.data[h].next
@@ -674,13 +674,6 @@ proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} =
   ## returns the number of keys in `t`.
   result = t.counter
 
-template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} =
-  var h = t.first
-  while h >= 0:
-    var nxt = t.data[h].next
-    if isFilled(t.data[h].hcode): yieldStmt
-    h = nxt
-
 iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) =
   ## iterates over any (key, value) pair in the table `t` in insertion
   ## order.
@@ -785,20 +778,22 @@ proc sort*[A, B](t: OrderedTableRef[A, B],
 
 proc del*[A, B](t: var OrderedTable[A, B], key: A) =
   ## deletes `key` from ordered hash table `t`. O(n) comlexity.
-  var prev = -1
+  var n: OrderedKeyValuePairSeq[A, B]
-  let hc = hash(key)
+  newSeq(n, len(t.data))
-  forAllOrderedPairs:
+  var h = t.first
-    if t.data[h].hcode == hc:
+  t.first = -1
-      if t.first == h:
+  t.last = -1
-        t.first = t.data[h].next
+  swap(t.data, n)
+  let hc = genHash(key)
+  while h >= 0:
+    var nxt = n[h].next
+    if isFilled(n[h].hcode):
+      if n[h].hcode == hc and n[h].key == key:
+        dec t.counter
       else:
-        t.data[prev].next = t.data[h].next
+        var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode)
-      var zeroValue : type(t.data[h])
+        rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j)
-      t.data[h] = zeroValue
+    h = nxt
-      dec t.counter
-      break
-    else:
-      prev = h
 
 proc del*[A, B](t: var OrderedTableRef[A, B], key: A) =
   ## deletes `key` from ordered hash table `t`. O(n) comlexity.
@@ -1164,6 +1159,20 @@ when isMainModule:
       doAssert(prev < i)
       prev = i
 
+  block: # Deletion from OrderedTable should account for collision groups. See issue #5057.
+    # The bug is reproducible only with exact keys
+    const key1 = "boy_jackpot.inGamma"
+    const key2 = "boy_jackpot.outBlack"
+
+    var t = {
+        key1: 0,
+        key2: 0
+    }.toOrderedTable()
+
+    t.del(key1)
+    assert(t.len == 1)
+    assert(key2 in t)
+
   var
     t1 = initCountTable[string]()
     t2 = initCountTable[string]()

lib/pure/includes/asyncfutures.nim

@@ -263,13 +263,13 @@ proc all*[T](futs: varargs[Future[T]]): auto =
 
     for fut in futs:
       fut.callback = proc(f: Future[T]) =
-        if f.failed:
+        inc(completedFutures)
-          retFuture.fail(f.error)
+        if not retFuture.finished:
-        elif not retFuture.finished:
+          if f.failed:
-          inc(completedFutures)
+            retFuture.fail(f.error)
-
+          else:
-          if completedFutures == totalFutures:
+            if completedFutures == totalFutures:
-            retFuture.complete()
+              retFuture.complete()
 
     if totalFutures == 0:
       retFuture.complete()
@@ -285,14 +285,15 @@ proc all*[T](futs: varargs[Future[T]]): auto =
     for i, fut in futs:
       proc setCallback(i: int) =
         fut.callback = proc(f: Future[T]) =
-          if f.failed:
+          inc(completedFutures)
-            retFuture.fail(f.error)
+          if not retFuture.finished:
-          elif not retFuture.finished:
+            if f.failed:
-            retValues[i] = f.read()
+              retFuture.fail(f.error)
-            inc(completedFutures)
+            else:
+              retValues[i] = f.read()
 
-            if completedFutures == len(retValues):
+              if completedFutures == len(retValues):
-              retFuture.complete(retValues)
+                retFuture.complete(retValues)
 
       setCallback(i)
 

lib/pure/json.nim

@@ -954,9 +954,11 @@ proc newIndent(curr, indent: int, ml: bool): int =
 proc nl(s: var string, ml: bool) =
   if ml: s.add("\n")
 
-proc escapeJson*(s: string): string =
+proc escapeJson*(s: string; result: var string) =
   ## Converts a string `s` to its JSON representation.
-  result = newStringOfCap(s.len + s.len shr 3)
+  ## Appends to ``result``.
+  const
+    HexChars = "0123456789ABCDEF"
   result.add("\"")
   for x in runes(s):
     var r = int(x)
@@ -967,10 +969,19 @@ proc escapeJson*(s: string): string =
       of '\\': result.add("\\\\")
       else: result.add(c)
     else:
-      result.add("\\u")
+      # toHex inlined for more speed (saves stupid string allocations):
-      result.add(toHex(r, 4))
+      result.add("\\u0000")
+      let start = result.len - 4
+      for j in countdown(3, 0):
+        result[j+start] = HexChars[r and 0xF]
+        r = r shr 4
   result.add("\"")
 
+proc escapeJson*(s: string): string =
+  ## Converts a string `s` to its JSON representation.
+  result = newStringOfCap(s.len + s.len shr 3)
+  escapeJson(s, result)
+
 proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
               lstArr = false, currIndent = 0) =
   case node.kind
@@ -988,7 +999,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
         inc i
         # Need to indent more than {
         result.indent(newIndent(currIndent, indent, ml))
-        result.add(escapeJson(key))
+        escapeJson(key, result)
         result.add(": ")
         toPretty(result, val, indent, ml, false,
                  newIndent(currIndent, indent, ml))
@@ -999,16 +1010,19 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
       result.add("{}")
   of JString:
     if lstArr: result.indent(currIndent)
-    result.add(escapeJson(node.str))
+    escapeJson(node.str, result)
   of JInt:
     if lstArr: result.indent(currIndent)
-    result.add($node.num)
+    when defined(js): result.add($node.num)
+    else: result.add(node.num)
   of JFloat:
     if lstArr: result.indent(currIndent)
-    result.add($node.fnum)
+    # Fixme: implement new system.add ops for the JS target
+    when defined(js): result.add($node.fnum)
+    else: result.add(node.fnum)
   of JBool:
     if lstArr: result.indent(currIndent)
-    result.add($node.bval)
+    result.add(if node.bval: "true" else: "false")
   of JArray:
     if lstArr: result.indent(currIndent)
     if len(node.elems) != 0:
@@ -1057,16 +1071,18 @@ proc toUgly*(result: var string, node: JsonNode) =
     for key, value in pairs(node.fields):
       if comma: result.add ","
       else:     comma = true
-      result.add key.escapeJson()
+      key.escapeJson(result)
       result.add ":"
       result.toUgly value
     result.add "}"
   of JString:
-    result.add node.str.escapeJson()
+    node.str.escapeJson(result)
   of JInt:
-    result.add($node.num)
+    when defined(js): result.add($node.num)
+    else: result.add(node.num)
   of JFloat:
-    result.add($node.fnum)
+    when defined(js): result.add($node.fnum)
+    else: result.add(node.fnum)
   of JBool:
     result.add(if node.bval: "true" else: "false")
   of JNull:
@@ -1394,4 +1410,6 @@ when isMainModule:
     var parsed2 = parseFile("tests/testdata/jsontest2.json")
     doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}")
 
+  doAssert escapeJson("\10FoobarÄ") == "\"\\u000AFoobar\\u00C4\""
+
   echo("Tests succeeded!")

lib/pure/logging.nim

@@ -172,18 +172,26 @@ when not defined(js):
     var (path, name, _) = splitFile(getAppFilename())
     result = changeFileExt(path / name, "log")
 
+  proc newFileLogger*(file: File,
+                      levelThreshold = lvlAll,
+                      fmtStr = defaultFmtStr): FileLogger =
+    ## Creates a new file logger. This logger logs to ``file``.
+    new(result)
+    result.file = file
+    result.levelThreshold = levelThreshold
+    result.fmtStr = fmtStr
+
   proc newFileLogger*(filename = defaultFilename(),
                       mode: FileMode = fmAppend,
                       levelThreshold = lvlAll,
                       fmtStr = defaultFmtStr,
                       bufSize: int = -1): FileLogger =
-    ## Creates a new file logger. This logger logs to a file.
+    ## Creates a new file logger. This logger logs to a file, specified
+    ## by ``fileName``.
     ## Use ``bufSize`` as size of the output buffer when writing the file
     ## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size).
-    new(result)
+    let file = open(filename, mode, bufSize = bufSize)
-    result.levelThreshold = levelThreshold
+    newFileLogger(file, levelThreshold, fmtStr)
-    result.file = open(filename, mode, bufSize = bufSize)
-    result.fmtStr = fmtStr
 
   # ------
 

lib/pure/marshal.nim

@@ -9,6 +9,7 @@
 
 ## This module contains procs for `serialization`:idx: and `deseralization`:idx:
 ## of arbitrary Nim data structures. The serialization format uses `JSON`:idx:.
+## Warning: The serialization format could change in future!
 ##
 ## **Restriction**: For objects their type is **not** serialized. This means
 ## essentially that it does not work if the object has some other runtime
@@ -29,6 +30,12 @@
 ##   a = b
 ##   echo($$a[]) # produces "{}", not "{f: 0}"
 ##
+##   # unmarshal
+##   let c = to[B]("""{"f": 2}""")
+##
+##   # marshal
+##   let s = $$c
+
 ## **Note**: The ``to`` and ``$$`` operations are available at compile-time!
 
 import streams, typeinfo, json, intsets, tables, unicode

lib/pure/net.nim

@@ -1250,7 +1250,7 @@ proc IPv6_loopback*(): IpAddress =
     address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1])
 
 proc `==`*(lhs, rhs: IpAddress): bool =
-  ## Compares two IpAddresses for Equality. Returns two if the addresses are equal
+  ## Compares two IpAddresses for Equality. Returns true if the addresses are equal
   if lhs.family != rhs.family: return false
   if lhs.family == IpAddressFamily.IPv4:
     for i in low(lhs.address_v4) .. high(lhs.address_v4):

lib/pure/nimtracker.nim

@@ -0,0 +1,80 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2016 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Memory tracking support for Nim.
+
+when not defined(memTracker) and not isMainModule:
+  {.error: "Memory tracking support is turned off!".}
+
+{.push memtracker: off.}
+# we import the low level wrapper and are careful not to use Nim's
+# memory manager for anything here.
+import sqlite3
+
+var
+  dbHandle: PSqlite3
+  insertStmt: Pstmt
+
+template sbind(x: int; value) =
+  when value is cstring:
+    let ret = insertStmt.bindText(x, value, value.len.int32, SQLITE_TRANSIENT)
+    if ret != SQLITE_OK:
+      quit "could not bind value"
+  else:
+    let ret = insertStmt.bindInt64(x, value)
+    if ret != SQLITE_OK:
+      quit "could not bind value"
+
+when defined(memTracker):
+  proc logEntries(log: TrackLog) {.nimcall, locks: 0, tags: [].} =
+    for i in 0..log.count-1:
+      var success = false
+      let e = log.data[i]
+      discard sqlite3.reset(insertStmt)
+      discard clearBindings(insertStmt)
+      sbind 1, e.op
+      sbind(2, cast[int](e.address))
+      sbind 3, e.size
+      sbind 4, e.file
+      sbind 5, e.line
+      if step(insertStmt) == SQLITE_DONE:
+        success = true
+      if not success:
+        quit "could not write to database!"
+
+proc execQuery(q: string) =
+  var s: Pstmt
+  if prepare_v2(dbHandle, q, q.len.int32, s, nil) == SQLITE_OK:
+    discard step(s)
+    if finalize(s) != SQLITE_OK:
+      quit "could not finalize " & $sqlite3.errmsg(dbHandle)
+  else:
+    quit "could not prepare statement " & $sqlite3.errmsg(dbHandle)
+
+proc setupDb() =
+  execQuery """create table if not exists tracking(
+       id integer primary key,
+       op varchar not null,
+       address integer not null,
+       size integer not null,
+       file varchar not null,
+       line integer not null
+     )"""
+  execQuery "delete from tracking"
+
+if sqlite3.open("memtrack.db", dbHandle) == SQLITE_OK:
+  setupDb()
+  const query = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)"
+  if prepare_v2(dbHandle, query,
+      query.len, insertStmt, nil) == SQLITE_OK:
+    when defined(memTracker):
+      setTrackLogger logEntries
+  else:
+    quit "could not prepare statement B " & $sqlite3.errmsg(dbHandle)
+{.pop.}

lib/pure/strutils.nim

@@ -1309,10 +1309,12 @@ proc join*[T: not string](a: openArray[T], sep: string = ""): string {.
 type
   SkipTable = array[char, int]
 
+{.push profiler: off.}
 proc preprocessSub(sub: string, a: var SkipTable) =
   var m = len(sub)
   for i in 0..0xff: a[chr(i)] = m+1
   for i in 0..m-1: a[sub[i]] = m-i
+{.pop.}
 
 proc findAux(s, sub: string, start: int, a: SkipTable): int =
   # Fast "quick search" algorithm:

lib/pure/times.nim

@@ -144,8 +144,9 @@ type
     yearday*: range[0..365]   ## The number of days since January 1,
                               ## in the range 0 to 365.
                               ## Always 0 if the target is JS.
-    isDST*: bool              ## Determines whether DST is in effect. Always
+    isDST*: bool              ## Determines whether DST is in effect.
-                              ## ``False`` if time is UTC.
+                              ## Semantically, this adds another negative hour
+                              ## offset to the time in addition to the timezone.
     timezone*: int            ## The offset of the (non-DST) timezone in seconds
                               ## west of UTC. Note that the sign of this number
                               ## is the opposite of the one in a formatted
@@ -278,16 +279,20 @@ proc `+`*(ti1, ti2: TimeInterval): TimeInterval =
   carryO = `div`(ti1.months + ti2.months, 12)
   result.years = carryO + ti1.years + ti2.years
 
+proc `-`*(ti: TimeInterval): TimeInterval =
+  result = TimeInterval(
+    milliseconds: -ti.milliseconds,
+    seconds: -ti.seconds,
+    minutes: -ti.minutes,
+    hours: -ti.hours,
+    days: -ti.days,
+    months: -ti.months,
+    years: -ti.years
+  )
+
 proc `-`*(ti1, ti2: TimeInterval): TimeInterval =
   ## Subtracts TimeInterval ``ti1`` from ``ti2``.
-  result = ti1
+  result = ti1 + (-ti2)
-  result.milliseconds -= ti2.milliseconds
-  result.seconds -= ti2.seconds
-  result.minutes -= ti2.minutes
-  result.hours -= ti2.hours
-  result.days -= ti2.days
-  result.months -= ti2.months
-  result.years -= ti2.years
 
 proc isLeapYear*(year: int): bool =
   ## returns true if ``year`` is a leap year
@@ -363,16 +368,9 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo =
   ##
   ## **Note:** This has been only briefly tested, it is inaccurate especially
   ## when you subtract so much that you reach the Julian calendar.
-  let t = toSeconds(toTime(a))
+  let
-  var intval: TimeInterval
+    t = toSeconds(toTime(a))
-  intval.milliseconds = - interval.milliseconds
+    secs = toSeconds(a, -interval)
-  intval.seconds = - interval.seconds
-  intval.minutes = - interval.minutes
-  intval.hours = - interval.hours
-  intval.days = - interval.days
-  intval.months = - interval.months
-  intval.years = - interval.years
-  let secs = toSeconds(a, intval)
   if a.timezone == 0:
     result = getGMTime(fromSeconds(t + secs))
   else:
@@ -732,6 +730,7 @@ const
   secondsInMin = 60
   secondsInHour = 60*60
   secondsInDay = 60*60*24
+  minutesInHour = 60
   epochStartYear = 1970
 
 proc formatToken(info: TimeInfo, token: string, buf: var string) =
@@ -823,28 +822,32 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) =
     if fyear.len != 5: fyear = repeat('0', 5-fyear.len()) & fyear
     buf.add(fyear)
   of "z":
-    let hours = abs(info.timezone) div secondsInHour
+    let
-    if info.timezone < 0: buf.add('-')
+      nonDstTz = info.timezone - int(info.isDst) * secondsInHour
-    else: buf.add('+')
+      hours = abs(nonDstTz) div secondsInHour
+    if nonDstTz <= 0: buf.add('+')
+    else: buf.add('-')
     buf.add($hours)
   of "zz":
-    let hours = abs(info.timezone) div secondsInHour
+    let
-    if info.timezone < 0: buf.add('-')
+      nonDstTz = info.timezone - int(info.isDst) * secondsInHour
-    else: buf.add('+')
+      hours = abs(nonDstTz) div secondsInHour
+    if nonDstTz <= 0: buf.add('+')
+    else: buf.add('-')
     if hours < 10: buf.add('0')
     buf.add($hours)
   of "zzz":
     let
-      hours = abs(info.timezone) div secondsInHour
+      nonDstTz = info.timezone - int(info.isDst) * secondsInHour
-      minutes = abs(info.timezone) mod 60
+      hours = abs(nonDstTz) div secondsInHour
-    if info.timezone < 0: buf.add('-')
+      minutes = (abs(nonDstTz) div secondsInMin) mod minutesInHour
-    else: buf.add('+')
+    if nonDstTz <= 0: buf.add('+')
+    else: buf.add('-')
     if hours < 10: buf.add('0')
     buf.add($hours)
     buf.add(':')
     if minutes < 10: buf.add('0')
     buf.add($minutes)
-
   of "":
     discard
   else:
@@ -999,7 +1002,6 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) =
   of "M":
     var pd = parseInt(value[j..j+1], sv)
     info.month = Month(sv-1)
-    info.monthday = sv
     j += pd
   of "MM":
     var month = value[j..j+1].parseInt()
@@ -1088,27 +1090,42 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) =
     info.year = value[j..j+3].parseInt()
     j += 4
   of "z":
+    info.isDST = false
     if value[j] == '+':
       info.timezone = 0 - parseInt($value[j+1]) * secondsInHour
     elif value[j] == '-':
       info.timezone = parseInt($value[j+1]) * secondsInHour
+    elif value[j] == 'Z':
+      info.timezone = 0
+      j += 1
+      return
     else:
       raise newException(ValueError,
         "Couldn't parse timezone offset (z), got: " & value[j])
     j += 2
   of "zz":
+    info.isDST = false
     if value[j] == '+':
       info.timezone = 0 - value[j+1..j+2].parseInt() * secondsInHour
     elif value[j] == '-':
       info.timezone = value[j+1..j+2].parseInt() * secondsInHour
+    elif value[j] == 'Z':
+      info.timezone = 0
+      j += 1
+      return
     else:
       raise newException(ValueError,
         "Couldn't parse timezone offset (zz), got: " & value[j])
     j += 3
   of "zzz":
+    info.isDST = false
     var factor = 0
     if value[j] == '+': factor = -1
     elif value[j] == '-': factor = 1
+    elif value[j] == 'Z':
+      info.timezone = 0
+      j += 1
+      return
     else:
       raise newException(ValueError,
         "Couldn't parse timezone offset (zzz), got: " & value[j])
@@ -1121,8 +1138,11 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) =
     j += token.len
 
 proc parse*(value, layout: string): TimeInfo =
-  ## This function parses a date/time string using the standard format identifiers (below)
+  ## This function parses a date/time string using the standard format
-  ## The function defaults information not provided in the format string from the running program (timezone, month, year, etc)
+  ## identifiers as listed below. The function defaults information not provided
+  ## in the format string from the running program (timezone, month, year, etc).
+  ## Daylight saving time is only set if no timezone is given and the given date
+  ## lies within the DST period of the current locale.
   ##
   ## ==========  =================================================================================  ================================================
   ## Specifier   Description                                                                        Example
@@ -1147,7 +1167,7 @@ proc parse*(value, layout: string): TimeInfo =
   ##    tt       Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively.
   ##    yy       Displays the year to two digits.                                                   ``2012 -> 12``
   ##    yyyy     Displays the year to four digits.                                                  ``2012 -> 2012``
-  ##    z        Displays the timezone offset from UTC.                                             ``GMT+7 -> +7``, ``GMT-5 -> -5``
+  ##    z        Displays the timezone offset from UTC. ``Z`` is parsed as ``+0``                   ``GMT+7 -> +7``, ``GMT-5 -> -5``
   ##    zz       Same as above but with leading 0.                                                  ``GMT+7 -> +07``, ``GMT-5 -> -05``
   ##    zzz      Same as above but with ``:mm`` where *mm* represents minutes.                      ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00``
   ## ==========  =================================================================================  ================================================
@@ -1165,6 +1185,8 @@ proc parse*(value, layout: string): TimeInfo =
   info.hour = 0
   info.minute = 0
   info.second = 0
+  info.isDST = true # using this is flag for checking whether a timezone has \
+      # been read (because DST is always false when a tz is parsed)
   while true:
     case layout[i]
     of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',':
@@ -1194,16 +1216,17 @@ proc parse*(value, layout: string): TimeInfo =
         parseToken(info, token, value, j)
         token = ""
 
-  # We are going to process the date to find out if we are in DST, because the
+  if info.isDST:
-  # default based on the current time may be wrong. Calling getLocalTime will
+    # means that no timezone has been parsed. In this case, we need to check
-  # set this correctly, but the actual time may be offset from when we called
+    # whether the date is within DST of the local time.
-  # toTime with a possibly incorrect DST setting, so we are only going to take
+    let tmp = getLocalTime(toTime(info))
-  # the isDST from this result.
+    # correctly set isDST so that the following step works on the correct time
-  let correctDST = getLocalTime(toTime(info))
+    info.isDST = tmp.isDST
-  info.isDST = correctDST.isDST
 
-  # Now we process it again with the correct isDST to correct things like
+  # Correct weekday and yearday; transform timestamp to local time.
-  # weekday and yearday.
+  # There currently is no way of returning this with the original (parsed)
+  # timezone while also setting weekday and yearday (we are depending on stdlib
+  # to provide this calculation).
   return getLocalTime(toTime(info))
 
 # Leap year calculations are adapted from:

lib/pure/unittest.nim

@@ -98,7 +98,7 @@ proc startSuite(name: string) =
   template rawPrint() = echo("\n[Suite] ", name) 
   when not defined(ECMAScript):
     if colorOutput:
-      styledEcho styleBright, fgBlue, "\n[Suite] ", fgWhite, name
+      styledEcho styleBright, fgBlue, "\n[Suite] ", resetStyle, name
     else: rawPrint()
   else: rawPrint()
 
@@ -159,7 +159,7 @@ proc testDone(name: string, s: TestStatus, indent: bool) =
                     of FAILED: fgRed
                     of SKIPPED: fgYellow
                     else: fgWhite
-        styledEcho styleBright, color, prefix, "[", $s, "] ", fgWhite, name
+        styledEcho styleBright, color, prefix, "[", $s, "] ", resetStyle, name
       else:
         rawPrint()
     else:

lib/system.nim

@@ -1272,12 +1272,14 @@ const
 
   seqShallowFlag = low(int)
 
+{.push profiler: off.}
 when defined(nimKnowsNimvm):
   let nimvm* {.magic: "Nimvm".}: bool = false
     ## may be used only in "when" expression.
     ## It is true in Nim VM context and false otherwise
 else:
   const nimvm*: bool = false
+{.pop.}
 
 proc compileOption*(option: string): bool {.
   magic: "CompileOption", noSideEffect.}
@@ -2544,6 +2546,7 @@ when hostOS == "standalone":
   include "$projectpath/panicoverride"
 
 when not declared(sysFatal):
+  {.push profiler: off.}
   when hostOS == "standalone":
     proc sysFatal(exceptn: typedesc, message: string) {.inline.} =
       panic(message)
@@ -2563,6 +2566,7 @@ when not declared(sysFatal):
       new(e)
       e.msg = message & arg
       raise e
+  {.pop.}
 
 proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.}
   ## get type information for `x`. Ordinary code should not use this, but
@@ -2616,8 +2620,10 @@ when not defined(JS): #and not defined(nimscript):
       when declared(setStackBottom):
         setStackBottom(locals)
 
+    {.push profiler: off.}
     var
       strDesc = TNimType(size: sizeof(string), kind: tyString, flags: {ntfAcyclic})
+    {.pop.}
 
 
   # ----------------- IO Part ------------------------------------------------
@@ -2950,6 +2956,8 @@ when not defined(JS): #and not defined(nimscript):
         ## lead to the ``raise`` statement. This only works for debug builds.
 
     {.push stack_trace: off, profiler:off.}
+    when defined(memtracker):
+      include "system/memtracker"
     when hostOS == "standalone":
       include "system/embedded"
     else:
@@ -2992,7 +3000,9 @@ when not defined(JS): #and not defined(nimscript):
       else:
         result = n.sons[n.len]
 
+    {.push profiler:off.}
     when hasAlloc: include "system/mmdisp"
+    {.pop.}
     {.push stack_trace: off, profiler:off.}
     when hasAlloc: include "system/sysstr"
     {.pop.}

lib/system/alloc.nim

@@ -15,6 +15,10 @@
 
 include osalloc
 
+template track(op, address, size) =
+  when defined(memTracker):
+    memTrackerOp(op, address, size)
+
 # We manage *chunks* of memory. Each chunk is a multiple of the page size.
 # Each chunk starts at an address that is divisible by the page size. Chunks
 # that are bigger than ``ChunkOsReturn`` are returned back to the operating
@@ -645,6 +649,7 @@ proc alloc(allocator: var MemRegion, size: Natural): pointer =
   cast[ptr FreeCell](result).zeroField = 1 # mark it as used
   sysAssert(not isAllocatedPtr(allocator, result), "alloc")
   result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell))
+  track("alloc", result, size)
 
 proc alloc0(allocator: var MemRegion, size: Natural): pointer =
   result = alloc(allocator, size)
@@ -658,6 +663,7 @@ proc dealloc(allocator: var MemRegion, p: pointer) =
   sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc 2")
   rawDealloc(allocator, x)
   sysAssert(not isAllocatedPtr(allocator, x), "dealloc 3")
+  track("dealloc", p, 0)
 
 proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer =
   if newsize > 0:

lib/system/deepcopy.nim

@@ -7,18 +7,66 @@
 #    distribution, for details about the copyright.
 #
 
-proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) {.benign.}
+type
-proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode) {.benign.} =
+  PtrTable = ptr object
+    counter, max: int
+    data: array[0..0xff_ffff, (pointer, pointer)]
+
+template hashPtr(key: pointer): int = cast[int](key) shr 8
+template allocPtrTable: untyped =
+  cast[PtrTable](alloc0(sizeof(int)*2 + sizeof(pointer)*2*cap))
+
+proc rehash(t: PtrTable): PtrTable =
+  let cap = (t.max+1) * 2
+  result = allocPtrTable()
+  result.counter = t.counter
+  result.max = cap-1
+  for i in 0..t.max:
+    let k = t.data[i][0]
+    if k != nil:
+      var h = hashPtr(k)
+      while result.data[h and result.max][0] != nil: inc h
+      result.data[h and result.max] = t.data[i]
+  dealloc t
+
+proc initPtrTable(): PtrTable =
+  const cap = 32
+  result = allocPtrTable()
+  result.counter = 0
+  result.max = cap-1
+
+template deinit(t: PtrTable) = dealloc(t)
+
+proc get(t: PtrTable; key: pointer): pointer =
+  var h = hashPtr(key)
+  while true:
+    let k = t.data[h and t.max][0]
+    if k == nil: break
+    if k == key:
+      return t.data[h and t.max][1]
+    inc h
+
+proc put(t: var PtrTable; key, val: pointer) =
+  if (t.max+1) * 2 < t.counter * 3: t = rehash(t)
+  var h = hashPtr(key)
+  while t.data[h and t.max][0] != nil: inc h
+  t.data[h and t.max] = (key, val)
+  inc t.counter
+
+proc genericDeepCopyAux(dest, src: pointer, mt: PNimType;
+                        tab: var PtrTable) {.benign.}
+proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode;
+                        tab: var PtrTable) {.benign.} =
   var
     d = cast[ByteAddress](dest)
     s = cast[ByteAddress](src)
   case n.kind
   of nkSlot:
     genericDeepCopyAux(cast[pointer](d +% n.offset),
-                       cast[pointer](s +% n.offset), n.typ)
+                       cast[pointer](s +% n.offset), n.typ, tab)
   of nkList:
     for i in 0..n.len-1:
-      genericDeepCopyAux(dest, src, n.sons[i])
+      genericDeepCopyAux(dest, src, n.sons[i], tab)
   of nkCase:
     var dd = selectBranch(dest, n)
     var m = selectBranch(src, n)
@@ -29,10 +77,10 @@ proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode) {.benign.} =
     copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset),
             n.typ.size)
     if m != nil:
-      genericDeepCopyAux(dest, src, m)
+      genericDeepCopyAux(dest, src, m, tab)
   of nkNone: sysAssert(false, "genericDeepCopyAux")
 
-proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) =
+proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) =
   var
     d = cast[ByteAddress](dest)
     s = cast[ByteAddress](src)
@@ -60,22 +108,22 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) =
         cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize),
         cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +%
                      GenericSeqSize),
-        mt.base)
+        mt.base, tab)
   of tyObject:
     # we need to copy m_type field for tyObject, as it could be empty for
     # sequence reallocations:
     if mt.base != nil:
-      genericDeepCopyAux(dest, src, mt.base)
+      genericDeepCopyAux(dest, src, mt.base, tab)
     else:
       var pint = cast[ptr PNimType](dest)
       pint[] = cast[ptr PNimType](src)[]
-    genericDeepCopyAux(dest, src, mt.node)
+    genericDeepCopyAux(dest, src, mt.node, tab)
   of tyTuple:
-    genericDeepCopyAux(dest, src, mt.node)
+    genericDeepCopyAux(dest, src, mt.node, tab)
   of tyArray, tyArrayConstr:
     for i in 0..(mt.size div mt.base.size)-1:
       genericDeepCopyAux(cast[pointer](d +% i*% mt.base.size),
-                         cast[pointer](s +% i*% mt.base.size), mt.base)
+                         cast[pointer](s +% i*% mt.base.size), mt.base, tab)
   of tyRef:
     let s2 = cast[PPointer](src)[]
     if s2 == nil:
@@ -84,30 +132,29 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) =
       let z = mt.base.deepcopy(s2)
       unsureAsgnRef(cast[PPointer](dest), z)
     else:
-      # we modify the header of the cell temporarily; instead of the type
+      let z = tab.get(s2)
-      # field we store a forwarding pointer. XXX This is bad when the cloning
+      if z == nil:
-      # fails due to OOM etc.
+        when declared(usrToCell):
-      when declared(usrToCell):
+          let x = usrToCell(s2)
-        # unfortunately we only have cycle detection for our native GCs.
-        let x = usrToCell(s2)
-        let forw = cast[int](x.typ)
-        if (forw and 1) == 1:
-          # we stored a forwarding pointer, so let's use that:
-          let z = cast[pointer](forw and not 1)
-          unsureAsgnRef(cast[PPointer](dest), z)
-        else:
           let realType = x.typ
           let z = newObj(realType, realType.base.size)
           unsureAsgnRef(cast[PPointer](dest), z)
-          x.typ = cast[PNimType](cast[int](z) or 1)
+          tab.put(s2, z)
-          genericDeepCopyAux(z, s2, realType.base)
+          genericDeepCopyAux(z, s2, realType.base, tab)
-          x.typ = realType
+        else:
+          when false:
+            # addition check disabled
+            let x = usrToCell(s2)
+            let realType = x.typ
+            sysAssert realType == mt, " types do differ"
+          # this version should work for any possible GC:
+          let size = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[].size else: mt.base.size
+          let z = newObj(mt, size)
+          unsureAsgnRef(cast[PPointer](dest), z)
+          tab.put(s2, z)
+          genericDeepCopyAux(z, s2, mt.base, tab)
       else:
-        let size = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[].size
-                   else: mt.base.size
-        let z = newObj(mt, size)
         unsureAsgnRef(cast[PPointer](dest), z)
-        genericDeepCopyAux(z, s2, mt.base)
   of tyPtr:
     # no cycle check here, but also not really required
     let s2 = cast[PPointer](src)[]
@@ -120,7 +167,9 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) =
 
 proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} =
   GC_disable()
-  genericDeepCopyAux(dest, src, mt)
+  var tab = initPtrTable()
+  genericDeepCopyAux(dest, src, mt, tab)
+  deinit tab
   GC_enable()
 
 proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} =

lib/system/excpt.nim

@@ -339,6 +339,8 @@ when not defined(noSignalHandler):
           action("unknown signal\n")
 
     # print stack trace and quit
+    when defined(memtracker):
+      logPendingOps()
     when hasSomeStackTrace:
       GC_disable()
       var buf = newStringOfCap(2000)

lib/system/gc.nim

@@ -445,6 +445,15 @@ proc gcInvariant*() =
     markForDebug(gch)
 {.pop.}
 
+template setFrameInfo(c: PCell) =
+  when leakDetector:
+    if framePtr != nil and framePtr.prev != nil:
+      c.filename = framePtr.prev.filename
+      c.line = framePtr.prev.line
+    else:
+      c.filename = nil
+      c.line = 0
+
 proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer =
   # generates a new object and sets its reference counter to 0
   sysAssert(allocInv(gch.region), "rawNewObj begin")
@@ -455,19 +464,14 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer =
   gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2")
   # now it is buffered in the ZCT
   res.typ = typ
-  when leakDetector:
+  setFrameInfo(res)
-    res.filename = nil
-    res.line = 0
-    when not hasThreadSupport:
-      if framePtr != nil and framePtr.prev != nil:
-        res.filename = framePtr.prev.filename
-        res.line = framePtr.prev.line
   # refcount is zero, color is black, but mark it to be in the ZCT
   res.refcount = ZctFlag
   sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3")
   # its refcount is zero, so add it to the ZCT:
   addNewObjToZCT(res, gch)
   when logGC: writeCell("new cell", res)
+  track("rawNewObj", res, size)
   gcTrace(res, csAllocated)
   release(gch)
   when useCellIds:
@@ -509,16 +513,11 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
   sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2")
   # now it is buffered in the ZCT
   res.typ = typ
-  when leakDetector:
+  setFrameInfo(res)
-    res.filename = nil
-    res.line = 0
-    when not hasThreadSupport:
-      if framePtr != nil and framePtr.prev != nil:
-        res.filename = framePtr.prev.filename
-        res.line = framePtr.prev.line
   res.refcount = rcIncrement # refcount is 1
   sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3")
   when logGC: writeCell("new cell", res)
+  track("newObjRC1", res, size)
   gcTrace(res, csAllocated)
   release(gch)
   when useCellIds:
@@ -561,6 +560,8 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer =
     writeCell("growObj new cell", res)
   gcTrace(ol, csZctFreed)
   gcTrace(res, csAllocated)
+  track("growObj old", ol, 0)
+  track("growObj new", res, newsize)
   when reallyDealloc:
     sysAssert(allocInv(gch.region), "growObj before dealloc")
     if ol.refcount shr rcShift <=% 1:
@@ -604,6 +605,7 @@ proc growObj(old: pointer, newsize: int): pointer {.rtl.} =
 proc freeCyclicCell(gch: var GcHeap, c: PCell) =
   prepareDealloc(c)
   gcTrace(c, csCycFreed)
+  track("cycle collector dealloc cell", c, 0)
   when logGC: writeCell("cycle collector dealloc cell", c)
   when reallyDealloc:
     sysAssert(allocInv(gch.region), "free cyclic cell")
@@ -673,6 +675,7 @@ proc doOperation(p: pointer, op: WalkOp) =
     gcAssert(c.refcount >=% rcIncrement, "doOperation 2")
     #c.refcount = c.refcount -% rcIncrement
     when logGC: writeCell("decref (from doOperation)", c)
+    track("waZctDecref", p, 0)
     decRef(c)
     #if c.refcount <% rcIncrement: addZCT(gch.zct, c)
   of waPush:
@@ -765,6 +768,7 @@ proc collectZCT(gch: var GcHeap): bool =
       # In any case, it should be removed from the ZCT. But not
       # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!**
       when logGC: writeCell("zct dealloc cell", c)
+      track("zct dealloc cell", c, 0)
       gcTrace(c, csZctFreed)
       # We are about to free the object, call the finalizer BEFORE its
       # children are deleted as well, because otherwise the finalizer may

lib/system/hti.nim

@@ -86,6 +86,8 @@ type
     finalizer: pointer # the finalizer for the type
     marker: proc (p: pointer, op: int) {.nimcall, benign.} # marker proc for GC
     deepcopy: proc (p: pointer): pointer {.nimcall, benign.}
+    when defined(nimTypeNames):
+      name: cstring
   PNimType = ptr TNimType
 
 # node.len may be the ``first`` element of a set

lib/system/memtracker.nim

@@ -0,0 +1,70 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2016 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Memory tracking support for Nim.
+
+when not defined(memTracker):
+  {.error: "Memory tracking support is turned off! Enable memory tracking by passing `--memtracker:on` to the compiler (see the Nim Compiler User Guide for more options).".}
+
+when defined(noSignalHandler):
+  {.error: "Memory tracking works better with the default signal handler.".}
+
+# We don't want to memtrack the tracking code ...
+{.push memtracker: off.}
+
+type
+  LogEntry* = object
+    op*: cstring
+    address*: pointer
+    size*: int
+    file*: cstring
+    line*: int
+  TrackLog* = object
+    count*: int
+    disabled: bool
+    data*: array[4000, LogEntry]
+  TrackLogger* = proc (log: TrackLog) {.nimcall, tags: [], locks: 0.}
+
+var
+  gLog*: TrackLog
+  gLogger*: TrackLogger = proc (log: TrackLog) = discard
+
+proc setTrackLogger*(logger: TrackLogger) =
+  gLogger = logger
+
+proc addEntry(entry: LogEntry) =
+  if not gLog.disabled:
+    if gLog.count > high(gLog.data):
+      gLogger(gLog)
+      gLog.count = 0
+    gLog.data[gLog.count] = entry
+    inc gLog.count
+
+proc memTrackerWrite(address: pointer; size: int; file: cstring; line: int) {.compilerProc.} =
+  addEntry LogEntry(op: "write", address: address,
+      size: size, file: file, line: line)
+
+proc memTrackerOp*(op: cstring; address: pointer; size: int) =
+  addEntry LogEntry(op: op, address: address, size: size,
+      file: "", line: 0)
+
+proc memTrackerDisable*() =
+  gLog.disabled = true
+
+proc memTrackerEnable*() =
+  gLog.disabled = false
+
+proc logPendingOps() {.noconv.} =
+  # forward declared and called from Nim's signal handler.
+  gLogger(gLog)
+  gLog.count = 0
+
+addQuitProc logPendingOps
+
+{.pop.}

lib/system/sysstr.nim

@@ -263,27 +263,32 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {.
   result.len = newLen
 
 # --------------- other string routines ----------------------------------
-proc nimIntToStr(x: int): string {.compilerRtl.} =
+proc add*(result: var string; x: int64) =
-  result = newString(sizeof(x)*4)
+  let base = result.len
+  setLen(result, base + sizeof(x)*4)
   var i = 0
   var y = x
   while true:
     var d = y div 10
-    result[i] = chr(abs(int(y - d*10)) + ord('0'))
+    result[base+i] = chr(abs(int(y - d*10)) + ord('0'))
     inc(i)
     y = d
     if y == 0: break
   if x < 0:
-    result[i] = '-'
+    result[base+i] = '-'
     inc(i)
-  setLen(result, i)
+  setLen(result, base+i)
   # mirror the string:
   for j in 0..i div 2 - 1:
-    swap(result[j], result[i-j-1])
+    swap(result[base+j], result[base+i-j-1])
 
-proc nimFloatToStr(f: float): string {.compilerproc.} =
+proc nimIntToStr(x: int): string {.compilerRtl.} =
+  result = newStringOfCap(sizeof(x)*4)
+  result.add x
+
+proc add*(result: var string; x: float) =
   var buf: array[0..64, char]
-  var n: int = c_sprintf(buf, "%.16g", f)
+  var n: int = c_sprintf(buf, "%.16g", x)
   var hasDot = false
   for i in 0..n-1:
     if buf[i] == ',':
@@ -298,14 +303,18 @@ proc nimFloatToStr(f: float): string {.compilerproc.} =
   # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' are produced.
   # We want to get rid of these here:
   if buf[n-1] in {'n', 'N'}:
-    result = "nan"
+    result.add "nan"
   elif buf[n-1] == 'F':
     if buf[0] == '-':
-      result = "-inf"
+      result.add "-inf"
     else:
-      result = "inf"
+      result.add "inf"
   else:
-    result = $buf
+    result.add buf
+
+proc nimFloatToStr(f: float): string {.compilerproc.} =
+  result = newStringOfCap(8)
+  result.add f
 
 proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {.
   importc: "strtod", header: "<stdlib.h>", noSideEffect.}
@@ -469,22 +478,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
   number = c_strtod(t, nil)
 
 proc nimInt64ToStr(x: int64): string {.compilerRtl.} =
-  result = newString(sizeof(x)*4)
+  result = newStringOfCap(sizeof(x)*4)
-  var i = 0
+  result.add x
-  var y = x
-  while true:
-    var d = y div 10
-    result[i] = chr(abs(int(y - d*10)) + ord('0'))
-    inc(i)
-    y = d
-    if y == 0: break
-  if x < 0:
-    result[i] = '-'
-    inc(i)
-  setLen(result, i)
-  # mirror the string:
-  for j in 0..i div 2 - 1:
-    swap(result[j], result[i-j-1])
 
 proc nimBoolToStr(x: bool): string {.compilerRtl.} =
   return if x: "true" else: "false"

lib/upcoming/asyncdispatch.nim

@@ -9,9 +9,9 @@
 
 include "system/inclrtl"
 
-import os, oids, tables, strutils, times, heapqueue
+import os, oids, tables, strutils, times, heapqueue, lists
 
-import nativesockets, net, queues
+import nativesockets, net, deques
 
 export Port, SocketFlag
 
@@ -135,7 +135,7 @@ include "../includes/asyncfutures"
 type
   PDispatcherBase = ref object of RootRef
     timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
-    callbacks: Queue[proc ()]
+    callbacks: Deque[proc ()]
 
 proc processTimers(p: PDispatcherBase) {.inline.} =
   while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt:
@@ -143,7 +143,7 @@ proc processTimers(p: PDispatcherBase) {.inline.} =
 
 proc processPendingCallbacks(p: PDispatcherBase) =
   while p.callbacks.len > 0:
-    var cb = p.callbacks.dequeue()
+    var cb = p.callbacks.popFirst()
     cb()
 
 proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
@@ -729,7 +729,7 @@ when defined(windows) or defined(nimdoc):
     var lpOutputBuf = newString(lpOutputLen)
     var dwBytesReceived: Dword
     let dwReceiveDataLength = 0.Dword # We don't want any data to be read.
-    let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16)
+    let dwLocalAddressLength = Dword(sizeof(Sockaddr_in) + 16)
     let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16)
 
     template completeAccept() {.dirty.} =
@@ -1095,9 +1095,11 @@ else:
     AsyncFD* = distinct cint
     Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.}
 
+    DoublyLinkedListRef = ref DoublyLinkedList[Callback]
+
     AsyncData = object
-      readCB: Callback
+      readCBs: DoublyLinkedListRef
-      writeCB: Callback
+      writeCBs: DoublyLinkedListRef
 
     AsyncEvent* = distinct SelectEvent
 
@@ -1112,7 +1114,7 @@ else:
     new result
     result.selector = newSelector[AsyncData]()
     result.timers.newHeapQueue()
-    result.callbacks = initQueue[proc ()](64)
+    result.callbacks = initDeque[proc ()](64)
 
   var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
   proc getGlobalDispatcher*(): PDispatcher =
@@ -1121,7 +1123,10 @@ else:
 
   proc register*(fd: AsyncFD) =
     let p = getGlobalDispatcher()
-    var data = AsyncData()
+    var data = AsyncData(
+      readCBs: DoublyLinkedListRef(),
+      writeCBs: DoublyLinkedListRef()
+    )
     p.selector.registerHandle(fd.SocketHandle, {}, data)
 
   proc newAsyncNativeSocket*(domain: cint, sockType: cint,
@@ -1156,8 +1161,9 @@ else:
     let p = getGlobalDispatcher()
     var newEvents = {Event.Read}
     withData(p.selector, fd.SocketHandle, adata) do:
-      adata.readCB = cb
+      adata.readCBs[].append(cb)
-      if adata.writeCB != nil:
+      newEvents.incl(Event.Read)
+      if not isNil(adata.writeCBs.head):
         newEvents.incl(Event.Write)
     do:
       raise newException(ValueError, "File descriptor not registered.")
@@ -1167,8 +1173,9 @@ else:
     let p = getGlobalDispatcher()
     var newEvents = {Event.Write}
     withData(p.selector, fd.SocketHandle, adata) do:
-      adata.writeCB = cb
+      adata.writeCBs[].append(cb)
-      if adata.readCB != nil:
+      newEvents.incl(Event.Write)
+      if not isNil(adata.readCBs.head):
         newEvents.incl(Event.Read)
     do:
       raise newException(ValueError, "File descriptor not registered.")
@@ -1195,31 +1202,32 @@ else:
         let events = keys[i].events
 
         if Event.Read in events or events == {Event.Error}:
-          let cb = keys[i].data.readCB
+          for node in keys[i].data.readCBs[].nodes():
-          if cb != nil:
+            let cb = node.value
-            if cb(fd.AsyncFD):
+            if cb != nil:
-              p.selector.withData(fd, adata) do:
+              if cb(fd.AsyncFD):
-                if adata.readCB == cb:
+                keys[i].data.readCBs[].remove(node)
-                  adata.readCB = nil
+              else:
+                break
 
         if Event.Write in events or events == {Event.Error}:
-          let cb = keys[i].data.writeCB
+          for node in keys[i].data.writeCBs[].nodes():
-          if cb != nil:
+            let cb = node.value
-            if cb(fd.AsyncFD):
+            if cb != nil:
-              p.selector.withData(fd, adata) do:
+              if cb(fd.AsyncFD):
-                if adata.writeCB == cb:
+                keys[i].data.writeCBs[].remove(node)
-                  adata.writeCB = nil
+              else:
+                break
 
         when supportedPlatform:
           if (customSet * events) != {}:
-            let cb = keys[i].data.readCB
+            for node in keys[i].data.readCBs[].nodes():
-            doAssert(cb != nil)
+              let cb = node.value
-            custom = true
+              doAssert(cb != nil)
-            if cb(fd.AsyncFD):
+              custom = true
-              p.selector.withData(fd, adata) do:
+              if cb(fd.AsyncFD):
-                if adata.readCB == cb:
+                keys[i].data.readCBs[].remove(node)
-                  adata.readCB = nil
+                p.selector.unregister(fd)
-                  p.selector.unregister(fd)
 
         # because state `data` can be modified in callback we need to update
         # descriptor events with currently registered callbacks.
@@ -1227,8 +1235,8 @@ else:
           var update = false
           var newEvents: set[Event] = {}
           p.selector.withData(fd, adata) do:
-            if adata.readCB != nil: incl(newEvents, Event.Read)
+            if not isNil(adata.readCBs.head): incl(newEvents, Event.Read)
-            if adata.writeCB != nil: incl(newEvents, Event.Write)
+            if not isNil(adata.writeCBs.head): incl(newEvents, Event.Write)
             update = true
           if update:
             p.selector.updateHandle(fd, newEvents)
@@ -1491,21 +1499,33 @@ else:
       ## ``oneshot`` - if ``true`` only one event will be dispatched,
       ## if ``false`` continuous events every ``timeout`` milliseconds.
       let p = getGlobalDispatcher()
-      var data = AsyncData(readCB: cb)
+      var data = AsyncData(
+        readCBs: DoublyLinkedListRef(),
+        writeCBs: DoublyLinkedListRef()
+      )
+      data.readCBs[].append(cb)
       p.selector.registerTimer(timeout, oneshot, data)
 
     proc addSignal*(signal: int, cb: Callback) =
       ## Start watching signal ``signal``, and when signal appears, call the
       ## callback ``cb``.
       let p = getGlobalDispatcher()
-      var data = AsyncData(readCB: cb)
+      var data = AsyncData(
+        readCBs: DoublyLinkedListRef(),
+        writeCBs: DoublyLinkedListRef()
+      )
+      data.readCBs[].append(cb)
       p.selector.registerSignal(signal, data)
 
     proc addProcess*(pid: int, cb: Callback) =
       ## Start watching for process exit with pid ``pid``, and then call
       ## the callback ``cb``.
       let p = getGlobalDispatcher()
-      var data = AsyncData(readCB: cb)
+      var data = AsyncData(
+        readCBs: DoublyLinkedListRef(),
+        writeCBs: DoublyLinkedListRef()
+      )
+      data.readCBs[].append(cb)
       p.selector.registerProcess(pid, data)
 
   proc newAsyncEvent*(): AsyncEvent =
@@ -1524,7 +1544,11 @@ else:
     ## Start watching for event ``ev``, and call callback ``cb``, when
     ## ev will be set to signaled state.
     let p = getGlobalDispatcher()
-    var data = AsyncData(readCB: cb)
+    var data = AsyncData(
+      readCBs: DoublyLinkedListRef(),
+      writeCBs: DoublyLinkedListRef()
+    )
+    data.readCBs[].append(cb)
     p.selector.registerEvent(SelectEvent(ev), data)
 
 proc sleepAsync*(ms: int): Future[void] =
@@ -1591,7 +1615,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
   ## **Note**: This procedure is mostly used for testing. You likely want to
   ## use ``asyncnet.recvLine`` instead.
 
-  template addNLIfEmpty(): stmt =
+  template addNLIfEmpty(): typed =
     if result.len == 0:
       result.add("\c\L")
 
@@ -1614,7 +1638,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
 proc callSoon*(cbproc: proc ()) =
   ## Schedule `cbproc` to be called as soon as possible.
   ## The callback is called when control returns to the event loop.
-  getGlobalDispatcher().callbacks.enqueue(cbproc)
+  getGlobalDispatcher().callbacks.addLast(cbproc)
 
 proc runForever*() =
   ## Begins a never ending global dispatcher poll loop.

lib/wrappers/openssl.nim

@@ -261,11 +261,14 @@ proc ERR_error_string*(e: cInt, buf: cstring): cstring{.cdecl,
 proc ERR_get_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.}
 proc ERR_peek_last_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.}
 
-proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSSL_add_all_algorithms_conf".}
+when defined(android):
+    template OpenSSL_add_all_algorithms*() = discard
+else:
+    proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSSL_add_all_algorithms_conf".}
 
 proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.}
 
-when not useWinVersion and not defined(macosx):
+when not useWinVersion and not defined(macosx) and not defined(android):
   proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl,
     dynlib: DLLUtilName, importc.}
 
@@ -279,7 +282,7 @@ when not useWinVersion and not defined(macosx):
     if p != nil: dealloc(p)
 
 proc CRYPTO_malloc_init*() =
-  when not useWinVersion and not defined(macosx):
+  when not useWinVersion and not defined(macosx) and not defined(android):
     CRYPTO_set_mem_functions(allocWrapper, reallocWrapper, deallocWrapper)
 
 proc SSL_CTX_ctrl*(ctx: SslCtx, cmd: cInt, larg: int, parg: pointer): int{.

lib/wrappers/sqlite3.nim

@@ -96,10 +96,6 @@ const
                               #define SQLITE_TRANSIENT   ((void(*)(void *))-1)
   SQLITE_DETERMINISTIC* = 0x800
 
-const
-  SQLITE_STATIC* = nil
-  SQLITE_TRANSIENT* = cast[pointer](- 1)
-
 type
   Sqlite3 {.pure, final.} = object
   PSqlite3* = ptr Sqlite3
@@ -114,7 +110,7 @@ type
 
   Callback* = proc (para1: pointer, para2: int32, para3,
                      para4: cstringArray): int32{.cdecl.}
-  Tbind_destructor_func* = proc (para1: pointer){.cdecl.}
+  Tbind_destructor_func* = proc (para1: pointer){.cdecl, locks: 0, tags: [].}
   Create_function_step_func* = proc (para1: Pcontext, para2: int32,
                                       para3: PValueArg){.cdecl.}
   Create_function_func_func* = proc (para1: Pcontext, para2: int32,
@@ -132,6 +128,10 @@ type
     Tresult_func: Result_func, Tcreate_collation_func: Create_collation_func,
     Tcollation_needed_func: Collation_needed_func].}
 
+const
+  SQLITE_STATIC* = nil
+  SQLITE_TRANSIENT* = cast[Tbind_destructor_func](-1)
+
 proc close*(para1: PSqlite3): int32{.cdecl, dynlib: Lib, importc: "sqlite3_close".}
 proc exec*(para1: PSqlite3, sql: cstring, para3: Callback, para4: pointer,
            errmsg: var cstring): int32{.cdecl, dynlib: Lib,
@@ -234,7 +234,8 @@ proc bind_parameter_name*(para1: Pstmt, para2: int32): cstring{.cdecl,
     dynlib: Lib, importc: "sqlite3_bind_parameter_name".}
 proc bind_parameter_index*(para1: Pstmt, zName: cstring): int32{.cdecl,
     dynlib: Lib, importc: "sqlite3_bind_parameter_index".}
-  #function sqlite3_clear_bindings(_para1:Psqlite3_stmt):longint;cdecl; external Sqlite3Lib name 'sqlite3_clear_bindings';
+proc clear_bindings*(para1: Pstmt): int32 {.cdecl,
+    dynlib: Lib, importc: "sqlite3_clear_bindings".}
 proc column_count*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib,
     importc: "sqlite3_column_count".}
 proc column_name*(para1: Pstmt, para2: int32): cstring{.cdecl, dynlib: Lib,

tests/assert/tfailedassert.nim

@@ -3,7 +3,7 @@ discard """
 WARNING: false first assertion from bar
 ERROR: false second assertion from bar
 -1
-tests/assert/tfailedassert.nim:27 false assertion from foo
+tfailedassert.nim:27 false assertion from foo
 '''
 """
 

tests/collections/ttables.nim

@@ -112,7 +112,7 @@ block orderedTableTest2:
 block countTableTest1:
   var s = data.toTable
   var t = initCountTable[string]()
-  
+
   for k in s.keys: t.inc(k)
   for k in t.keys: assert t[k] == 1
   t.inc("90", 3)
@@ -167,6 +167,29 @@ block mpairsTableTest1:
 block SyntaxTest:
   var x = toTable[int, string]({:})
 
+block zeroHashKeysTest:
+  proc doZeroHashValueTest[T, K, V](t: T, nullHashKey: K, value: V) =
+    let initialLen = t.len
+    var testTable = t
+    testTable[nullHashKey] = value
+    assert testTable[nullHashKey] == value
+    assert testTable.len == initialLen + 1
+    testTable.del(nullHashKey)
+    assert testTable.len == initialLen
+
+  # with empty table
+  doZeroHashValueTest(toTable[int,int]({:}), 0, 42)
+  doZeroHashValueTest(toTable[string,int]({:}), "", 23)
+  doZeroHashValueTest(toOrderedTable[int,int]({:}), 0, 42)
+  doZeroHashValueTest(toOrderedTable[string,int]({:}), "", 23)
+
+  # with non-empty table
+  doZeroHashValueTest(toTable[int,int]({1:2}), 0, 42)
+  doZeroHashValueTest(toTable[string,string]({"foo": "bar"}), "", "zero")
+  doZeroHashValueTest(toOrderedTable[int,int]({3:4}), 0, 42)
+  doZeroHashValueTest(toOrderedTable[string,string]({"egg": "sausage"}),
+      "", "spam")
+
 # Until #4448 is fixed, these tests will fail
 when false:
   block clearTableTest:

tests/manyloc/keineschweine/keineschweine.nim

@@ -40,7 +40,7 @@ type
     trailDelay*: float
     body: chipmunk.PBody
     shape: chipmunk.PShape
-import vehicles
+include vehicles
 const
   LGrabbable*  = (1 shl 0).TLayers
   LBorders*    = (1 shl 1).TLayers

tests/manyloc/keineschweine/lib/vehicles.nim

@@ -1,6 +1,6 @@
 import
   sfml, chipmunk,
-  sg_assets, sfml_stuff, "../keineschweine"
+  sg_assets, sfml_stuff#, "../keineschweine"
 
 
 proc accel*(obj: PVehicle, dt: float) =

tests/method/tmapper.nim

@@ -1,5 +1,5 @@
 discard """
-  errormsg: "invalid declaration order; cannot attach 'step' to method defined here: tests/method/tmapper.nim(22,7)"
+  errormsg: "invalid declaration order; cannot attach 'step' to method defined here: tmapper.nim(22,7)"
   line: 25
 """
 

tests/misc/tvarious1.nim

@@ -18,15 +18,15 @@ echo v[2]
 
 # bug #569
 
-import queues
+import deques
 
 type
   TWidget = object
-    names: Queue[string]
+    names: Deque[string]
 
-var w = TWidget(names: initQueue[string]())
+var w = TWidget(names: initDeque[string]())
 
-add(w.names, "Whopie")
+addLast(w.names, "Whopie")
 
 for n in w.names: echo(n)
 

tests/modules/trecinca.nim

@@ -1,7 +1,7 @@
 discard """
-  file: "tests/reject/trecincb.nim"
+  file: "trecincb.nim"
   line: 9
-  errormsg: "recursive dependency: 'tests/modules/trecincb.nim'"
+  errormsg: "recursive dependency: 'trecincb.nim'"
 """
 # Test recursive includes
 

tests/modules/trecincb.nim

@@ -1,7 +1,7 @@
 discard """
   file: "trecincb.nim"
   line: 9
-  errormsg: "recursive dependency: 'tests/modules/trecincb.nim'"
+  errormsg: "recursive dependency: 'trecincb.nim'"
 """
 # Test recursive includes
 

tests/modules/trecmod.nim

@@ -1,2 +1,8 @@
+discard """
+  file: "mrecmod.nim"
+  line: 1
+  errormsg: "recursive module dependency detected"
+  disabled: true
+"""
 # recursive module
 import mrecmod

tests/modules/trecmod2.nim

@@ -1,10 +1,13 @@
+discard """
+  output: "4"
+"""
 type
   T1* = int  # Module A exports the type ``T1``
 
 import mrecmod2   # the compiler starts parsing B
-
+# the manual says this should work
 proc main() =
-  var i = p(3) # works because B has been parsed completely here
+  echo p(3) # works because B has been parsed completely here
 
 main()
 

tests/stdlib/tmitems.nim

@@ -98,13 +98,13 @@ block:
     x += 10
   echo sl
 
-import queues
+import deques
 
 block:
-  var q = initQueue[int]()
+  var q = initDeque[int]()
-  q.add(1)
+  q.addLast(1)
-  q.add(2)
+  q.addLast(2)
-  q.add(3)
+  q.addLast(3)
   for x in q.mitems:
     x += 10
   echo q

tests/stdlib/ttime.nim

@@ -96,6 +96,10 @@ parseTest("2006-01-12T15:04:05Z-07:00", "yyyy-MM-dd'T'HH:mm:ss'Z'zzz",
 # RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00"
 parseTest("2006-01-12T15:04:05.999999999Z-07:00",
     "yyyy-MM-ddTHH:mm:ss.999999999Zzzz", "2006-01-12T22:04:05+00:00", 11)
+for tzFormat in ["z", "zz", "zzz"]:
+  # formatting timezone as 'Z' for UTC
+  parseTest("2001-01-12T22:04:05Z", "yyyy-MM-dd'T'HH:mm:ss" & tzFormat,
+      "2001-01-12T22:04:05+00:00", 11)
 # Kitchen     = "3:04PM"
 parseTestTimeOnly("3:04PM", "h:mmtt", "15:04:00")
 #when not defined(testing):
@@ -190,3 +194,36 @@ doAssert cmpTimeNoSideEffect(0.fromSeconds, 0.fromSeconds)
 let seqA: seq[Time] = @[]
 let seqB: seq[Time] = @[]
 doAssert seqA == seqB
+
+for tz in [
+    (0, "+0", "+00", "+00:00"), # UTC
+    (-3600, "+1", "+01", "+01:00"), # CET
+    (-39600, "+11", "+11", "+11:00"), # two digits
+    (-1800, "+0", "+00", "+00:30"), # half an hour
+    (7200, "-2", "-02", "-02:00"), # positive
+    (38700, "-10", "-10", "-10:45")]: # positive with three quaters hour
+  let ti = TimeInfo(monthday: 1, timezone: tz[0])
+  doAssert ti.format("z") == tz[1]
+  doAssert ti.format("zz") == tz[2]
+  doAssert ti.format("zzz") == tz[3]
+
+block dstTest:
+  let nonDst = TimeInfo(year: 2015, month: mJan, monthday: 01, yearday: 0,
+      weekday: dThu, hour: 00, minute: 00, second: 00, isDST: false, timezone: 0)
+  var dst = nonDst
+  dst.isDst = true
+  # note that both isDST == true and isDST == false are valid here because
+  # DST is in effect on January 1st in some southern parts of Australia.
+
+  doAssert nonDst.toTime() - dst.toTime() == 3600
+  doAssert nonDst.format("z") == "+0"
+  doAssert dst.format("z") == "+1"
+
+  # parsing will set isDST in relation to the local time. We take a date in
+  # January and one in July to maximize the probability to hit one date with DST
+  # and one without on the local machine. However, this is not guaranteed.
+  let
+    parsedJan = parse("2016-01-05 04:00:00+01:00", "yyyy-MM-dd HH:mm:sszzz")
+    parsedJul = parse("2016-07-01 04:00:00+01:00", "yyyy-MM-dd HH:mm:sszzz")
+  doAssert toTime(parsedJan) == fromSeconds(1451962800)
+  doAssert toTime(parsedJul) == fromSeconds(1467342000)

tests/system/tdeepcopy.nim

@@ -0,0 +1,95 @@
+discard """
+  output: "ok"
+  disabled: "true"
+"""
+
+import tables, lists
+
+type
+  ListTable[K, V] = object
+    valList: DoublyLinkedList[V]
+    table: Table[K, DoublyLinkedNode[V]]
+
+  ListTableRef*[K, V] = ref ListTable[K, V]
+
+proc initListTable*[K, V](initialSize = 64): ListTable[K, V] =
+  result.valList = initDoublyLinkedList[V]()
+  result.table = initTable[K, DoublyLinkedNode[V]]()
+
+proc newListTable*[K, V](initialSize = 64): ListTableRef[K, V] =
+  new(result)
+  result[] = initListTable[K, V](initialSize)
+
+proc `[]=`*[K, V](t: var ListTable[K, V], key: K, val: V) =
+  if key in t.table:
+    t.table[key].value = val
+  else:
+    let node = newDoublyLinkedNode(val)
+    t.valList.append(node)
+    t.table[key] = node
+
+proc `[]`*[K, V](t: ListTable[K, V], key: K): var V {.inline.} =
+  result = t.table[key].value
+
+proc len*[K, V](t: ListTable[K, V]): Natural {.inline.} =
+  result = t.table.len
+
+iterator values*[K, V](t: ListTable[K, V]): V =
+  for val in t.valList.items():
+    yield val
+
+proc `[]=`*[K, V](t: ListTableRef[K, V], key: K, val: V) =
+  t[][key] = val
+
+proc `[]`*[K, V](t: ListTableRef[K, V], key: K): var V {.inline.} =
+  t[][key]
+
+proc len*[K, V](t: ListTableRef[K, V]): Natural {.inline.} =
+  t[].len
+
+iterator values*[K, V](t: ListTableRef[K, V]): V =
+  for val in t[].values:
+    yield val
+
+proc main() =
+  type SomeObj = ref object
+
+  for outer in 0..10_000:
+    let myObj = new(SomeObj)
+    let table = newListTable[int, SomeObj]()
+
+    table[0] = myObj
+    for i in 1..100:
+      table[i] = new(SomeObj)
+
+    var myObj2: SomeObj
+    for val in table.values():
+      if myObj2.isNil:
+        myObj2 = val
+    assert(myObj == myObj2) # passes
+
+    var tableCopy: ListTableRef[int, SomeObj]
+    deepCopy(tableCopy, table)
+
+    let myObjCopy = tableCopy[0]
+    var myObjCopy2: SomeObj = nil
+    for val in tableCopy.values():
+      if myObjCopy2.isNil:
+        myObjCopy2 = val
+
+    #echo cast[int](myObj)
+    #echo cast[int](myObjCopy)
+    #echo cast[int](myObjCopy2)
+
+    assert(myObjCopy == myObjCopy2) # fails
+
+
+type
+  PtrTable = object
+    counter, max: int
+    data: array[0..99, (pointer, pointer)]
+
+assert(sizeof(PtrTable) == 2*sizeof(int)+sizeof(pointer)*2*100)
+
+main()
+echo "ok"

tests/test_nimscript.nims

@@ -14,7 +14,7 @@ import ospaths
 # import parseopt
 import parseutils
 # import pegs
-import queues
+import deques
 import sequtils
 import strutils
 import subexes

tools/nimsuggest/nimsuggest.nim

@@ -431,10 +431,10 @@ proc handleCmdLine(cache: IdentCache) =
       except OSError:
         gProjectFull = gProjectName
       var p = splitFile(gProjectFull)
-      gProjectPath = p.dir
+      gProjectPath = canonicalizePath p.dir
       gProjectName = p.name
     else:
-      gProjectPath = getCurrentDir()
+      gProjectPath = canonicalizePath getCurrentDir()
 
     # Find Nim's prefix dir.
     let binaryPath = findExe("nim")

tools/website.tmpl

@@ -60,17 +60,7 @@
 #  if currentTab == "index":
         <div id="slideshow">
           <!-- slides -->
-          <div id="slide0" class="active niaslide">
+          <div id="slide0" class="active codeslide2">
-            <a href="sponsors.html">
-              <img src="${rootDir}assets/bountysource/meet_sponsors.png" alt="Meet our BountySource sponsors!"/>
-            </a>
-          </div>
-          <div id="slide1" class="niaslide">
-            <a href="news/2016_01_27_nim_in_action_is_now_available.html">
-              <img src="${rootDir}assets/niminaction/banner.jpg" alt="New in Manning Early Access Program: Nim in Action!"/>
-            </a>
-          </div>
-          <div id="slide2" class="codeslide2">
             <div>
               <h2>Nim is simple..</h2>
 <pre>
@@ -104,7 +94,7 @@ p.greet() <span class="cmt"># or greet(p)</span>
 </pre>
              </div>
           </div>
-          <div id="slide3" class="codeslide3">
+          <div id="slide1" class="codeslide3">
             <div>
               <h2>C FFI is easy in Nim..</h2>
 <pre>
@@ -136,6 +126,16 @@ runForever()
               <p><span class="desc"><b>View in browser at:</b><br>&nbsp;&nbsp;&nbsp;&nbsp;localhost:5000</span></p>
             </div>
           </div>
+          <div id="slide2" class="niaslide">
+            <a href="news/e030_nim_in_action_in_production.html">
+              <img src="${rootDir}assets/niminaction/banner2.png" alt="A printed copy of Nim in Action should be available in March 2017!"/>
+            </a>
+          </div>
+          <div id="slide3" class="niaslide">
+            <a href="sponsors.html">
+              <img src="${rootDir}assets/bountysource/meet_sponsors.png" alt="Meet our BountySource sponsors!"/>
+            </a>
+          </div>
         </div>
         <div id="slideshow-nav">
           <div id="slideControl0" onclick="slideshow_click(0)" class="active"></div>

web/index.rst

@@ -5,7 +5,7 @@ Home
 Welcome to Nim
 --------------
 
-**Nim** (formerly known as "Nimrod") is a statically typed, imperative
+**Nim** is a statically typed, imperative
 programming language that tries to give the programmer ultimate power without
 compromises on runtime efficiency. This means it focuses on compile-time
 mechanisms in all their various forms.

web/news.rst

@@ -2,6 +2,9 @@
 News
 ====
 
+`2016-11-20 Nim in Action is going into production! <news/e030_nim_in_action_in_production.html>`_
+===================================
+
 `2016-10-23 Nim Version 0.15.2 released <news/e028_version_0_15_2.html>`_
 ===================================
 

web/news/e029_version_0_16_0.rst

@@ -35,6 +35,9 @@ Library Additions
 
 - Added new parameter to ``error`` proc of ``macro`` module to provide better
   error message
+- Added new ``deques`` module intended to replace ``queues``.
+  ``deques`` provides a superset of ``queues`` API with clear naming.
+  ``queues`` module is now deprecated and will be removed in the future.
 
 Tool Additions
 --------------

web/news/e030_nim_in_action_in_production.rst

@@ -0,0 +1,53 @@
+Nim in Action is going into production!
+=======================================
+
+.. container:: metadata
+
+  Posted by Dominik Picheta on 20/11/2016
+
+.. raw::html
+
+  <a href="https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81">
+    <img src="../assets/niminaction/banner2.png" alt="A printed copy of Nim in Action should be available in March 2017!" width="682"/>
+  </a>
+
+I am very happy to say that just last week I have put the finishing touches
+on Nim in Action. The final manuscript has been submitted to Manning (the book's
+publisher), and the printed version is expected to start shipping in March
+2017 (give or take 1 month).
+
+The eBook is still available and now contains all of the book's chapters,
+including new ones dealing with the foreign function interface and
+metaprogramming.
+That said, it may still take some time before the eBook is updated with the
+latest corrections.
+
+I am incredibly thankful to everyone that purchased the book already. Many of
+you have also given me a lot of `brilliant <http://forum.nim-lang.org/t/1978>`_
+`feedback <https://forums.manning.com/forums/nim-in-action>`_,
+thank you very much for
+taking the time to do so. I have done my best to act on this
+feedback and I hope you will agree that the book has risen in quality as a
+result.
+
+Writing this book has been both exhausting and incredible at the same time.
+I look forward
+to having a physical copy of it in my hands, and I'm sure many of you do as
+well. I can safely say that without your support this book would not have
+happened, even if you did not purchase a copy your interest in Nim has made it
+possible and I thank you for that.
+
+As always, you can make a purchase on
+`Manning's website <https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81>`_.
+Both eBook's and printed books are available, and purchasing a printed book will
+get you an eBook for free.
+You can now also pre-order Nim in Action on
+`Amazon <https://www.amazon.co.uk/Nim-Action-Dominik-Picheta/dp/1617293431/ref=sr_1_1?ie=UTF8&qid=1479663850&sr=8-1&keywords=nim+in+action>`_!
+
+If you would like updates about the book then please feel free to
+follow either `myself <https://twitter.com/d0m96>`_ or
+`@nim_lang <https://twitter.com/nim_lang>`_ on Twitter. Finally, if you have any
+questions, do get in touch via `Twitter, NimForum,
+IRC or Gitter <http://nim-lang.org/community.html>`_.
+
+Thanks for reading!

web/ticker.html

@@ -1,8 +1,18 @@
+<a class="news" href="$1news/e030_nim_in_action_in_production.html">
+  <h4>November 20, 2016</h4>
+  <p>Nim in Action is going into production!</p>
+</a>
+
 <a class="news" href="$1news/e028_version_0_15_2.html">
   <h4>October 23, 2016</h4>
   <p>Nim version 0.15.2 has been released!</p>
 </a>
 
+<a class="news" href="$1news/e027_version_0_15_0.html">
+  <h4>September 30, 2016</h4>
+  <p>Nim version 0.15.0 has been released!</p>
+</a>
+
 <a class="news" href="$1news/e026_survey_results.html">
   <h4>September 3, 2016</h4>
   <p>Nim Community Survey results</p>
@@ -13,14 +23,4 @@
   <p>BountySource Update: The Road to v1.0</p>
 </a>
 
-<a class="news" href="$1news/e024_survey.html">
-  <h4>June 23, 2016</h4>
-  <p>Launching the 2016 Nim community survey!</p>
-</a>
-
-<a class="news" href="$1news/e023_version_0_14_2.html">
-  <h4>June 11, 2016</h4>
-  <p>Nim version 0.14.2 has been released!</p>
-</a>
-
 <a href="$1news.html" class="blue">See All News...</a>

web/website.ini

@@ -51,7 +51,7 @@ srcdoc2: "pure/ropes;pure/unidecode/unidecode;pure/xmldom;pure/xmldomparser"
 srcdoc2: "pure/xmlparser;pure/htmlparser;pure/xmltree;pure/colors;pure/mimetypes"
 srcdoc2: "pure/json;pure/base64;pure/scgi"
 srcdoc2: "pure/collections/tables;pure/collections/sets;pure/collections/lists"
-srcdoc2: "pure/collections/intsets;pure/collections/queues;pure/encodings"
+srcdoc2: "pure/collections/intsets;pure/collections/queues;pure/collections/deques;pure/encodings"
 srcdoc2: "pure/events;pure/collections/sequtils;pure/cookies"
 srcdoc2: "pure/memfiles;pure/subexes;pure/collections/critbits"
 srcdoc2: "deprecated/pure/asyncio;deprecated/pure/actors;core/locks;core/rlocks;pure/oids;pure/endians;pure/uri"