Merge branch 'devel' into feature/async-streams

.travis.yml

@@ -2,14 +2,24 @@ sudo: false
 language: c
 os:
   - linux
+  - osx
 dist: trusty
 
+matrix:
+  allow_failures:
+    - os: osx
+
 addons:
   apt:
     packages:
     - libcurl4-openssl-dev
     - libsdl1.2-dev
     - libgc-dev
+  brew:
+    packages:
+    - boehmgc
+    - node
+
 before_script:
   - set -e
   - curl --out fasm-1.71.39.tgz https://nim-lang.org/download/fasm-1.71.39.tgz

appveyor.yml

@@ -7,10 +7,7 @@ cache:
 # - i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
 
 matrix:
-  fast_finish: true     # set this flag to immediately finish build once one of the jobs fails.
+  fast_finish: true
-  allow_failures:
-    - platform: x64
-#   - platform: x86
 
 environment:
   matrix:

compiler/ast.nim

@@ -10,7 +10,7 @@
 # abstract syntax tree + symbol table
 
 import
-  msgs, hashes, nversion, options, strutils, securehash, ropes, idents, lists,
+  msgs, hashes, nversion, options, strutils, securehash, ropes, idents,
   intsets, idgen
 
 type
@@ -736,13 +736,15 @@ type
 
   TLibKind* = enum
     libHeader, libDynamic
-  TLib* = object of lists.TListEntry # also misused for headers!
+    
+  TLib* = object              # also misused for headers!
     kind*: TLibKind
     generated*: bool          # needed for the backends:
     isOverriden*: bool
     name*: Rope
     path*: PNode              # can be a string literal!
 
+    
   CompilesId* = int ## id that is used for the caching logic within
                     ## ``system.compiles``. See the seminst module.
   TInstantiation* = object
@@ -1541,8 +1543,7 @@ proc skipGenericOwner*(s: PSym): PSym =
   ## Generic instantiations are owned by their originating generic
   ## symbol. This proc skips such owners and goes straight to the owner
   ## of the generic itself (the module or the enclosing proc).
-  result = if s.kind in skProcKinds and {sfGenSym, sfFromGeneric} * s.flags ==
+  result = if s.kind in skProcKinds and sfFromGeneric in s.flags:
-                                                  {sfFromGeneric}:
              s.owner.owner
            else:
              s.owner

compiler/astalgo.nim

@@ -388,6 +388,7 @@ proc debugTree(n: PNode, indent: int, maxRecDepth: int;
              [istr, makeYamlString($n.kind)]
     addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(n.info)])
     if maxRecDepth != 0:
+      addf(result, ",$N$1\"flags\": $2", [istr, rope($n.flags)])
       case n.kind
       of nkCharLit..nkUInt64Lit:
         addf(result, ",$N$1\"intVal\": $2", [istr, rope(n.intVal)])

compiler/ccgexprs.nim

@@ -220,7 +220,7 @@ proc genOptAsgnTuple(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
                      optAsgnLoc(src, t, field), newflags)
 
 proc genOptAsgnObject(p: BProc, dest, src: TLoc, flags: TAssignmentFlags,
-                      t: PNode) =
+                      t: PNode, typ: PType) =
   if t == nil: return
   let newflags =
     if src.s == OnStatic:
@@ -232,10 +232,11 @@ proc genOptAsgnObject(p: BProc, dest, src: TLoc, flags: TAssignmentFlags,
   case t.kind
   of nkSym:
     let field = t.sym
+    if field.loc.r == nil: fillObjectFields(p.module, typ)
     genAssignment(p, optAsgnLoc(dest, field.typ, field.loc.r),
                      optAsgnLoc(src, field.typ, field.loc.r), newflags)
   of nkRecList:
-    for child in items(t): genOptAsgnObject(p, dest, src, newflags, child)
+    for child in items(t): genOptAsgnObject(p, dest, src, newflags, child, typ)
   else: discard
 
 proc genGenericAsgn(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
@@ -315,9 +316,9 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
       genGenericAsgn(p, dest, src, flags)
     elif needsComplexAssignment(ty):
       if ty.sons[0].isNil and asgnComplexity(ty.n) <= 4:
-        discard getTypeDesc(p.module, dest.t)
+        discard getTypeDesc(p.module, ty)
         internalAssert ty.n != nil
-        genOptAsgnObject(p, dest, src, flags, ty.n)
+        genOptAsgnObject(p, dest, src, flags, ty.n, ty)
       else:
         genGenericAsgn(p, dest, src, flags)
     else:
@@ -943,7 +944,7 @@ proc genEcho(p: BProc, n: PNode) =
   # this unusal way of implementing it ensures that e.g. ``echo("hallo", 45)``
   # is threadsafe.
   internalAssert n.kind == nkBracket
-  discard lists.includeStr(p.module.headerFiles, "<stdio.h>")
+  p.module.includeHeader("<stdio.h>")
   var args: Rope = nil
   var a: TLoc
   for i in countup(0, n.len-1):

compiler/ccgstmts.nim

@@ -540,7 +540,7 @@ proc genBreakStmt(p: BProc, t: PNode) =
     # named break?
     assert(t.sons[0].kind == nkSym)
     var sym = t.sons[0].sym
-    assert(sym.loc.k == locOther)
+    doAssert(sym.loc.k == locOther)
     idx = sym.position-1
   else:
     # an unnamed 'break' can only break a loop after 'transf' pass:
@@ -884,7 +884,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
   #
   if not isEmptyType(t.typ) and d.k == locNone:
     getTemp(p, t.typ, d)
-  discard lists.includeStr(p.module.headerFiles, "<setjmp.h>")
+  p.module.includeHeader("<setjmp.h>")
   genLineDir(p, t)
   var safePoint = getTempName(p.module)
   if getCompilerProc("Exception") != nil:
@@ -1103,8 +1103,8 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   genAssignment(p, a, tmp, {})
 
 proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
-  genLineDir(p, e)
   if e.sons[0].kind == nkSym and sfGoto in e.sons[0].sym.flags:
+    genLineDir(p, e)
     genGotoVar(p, e.sons[1])
   elif not fieldDiscriminantCheckNeeded(p, e):
     var a: TLoc
@@ -1114,8 +1114,11 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
       initLocExpr(p, e.sons[0], a)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)
     assert(a.t != nil)
-    loadInto(p, e.sons[0], e.sons[1], a)
+    let ri = e.sons[1]
+    genLineDir(p, ri)
+    loadInto(p, e.sons[0], ri, a)
   else:
+    genLineDir(p, e)
     asgnFieldDiscriminant(p, e)
 
 proc genStmts(p: BProc, t: PNode) =

compiler/ccgthreadvars.nim

@@ -57,8 +57,8 @@ proc generateThreadLocalStorage(m: BModule) =
 
 proc generateThreadVarsSize(m: BModule) =
   if nimtv != nil:
-    let externc = if gCmd != cmdCompileToCpp and
+    let externc = if gCmd == cmdCompileToCpp or
-                       sfCompileToCpp in m.module.flags: "extern \"C\""
+                       sfCompileToCpp in m.module.flags: "extern \"C\" "
                   else: ""
     addf(m.s[cfsProcs],
       "$#NI NimThreadVarsSize(){return (NI)sizeof(NimThreadVars);}$n",

compiler/ccgtypes.nim

@@ -24,7 +24,13 @@ proc isKeyword(w: PIdent): bool =
 
 proc mangleField(m: BModule; name: PIdent): string =
   result = mangle(name.s)
-  if isKeyword(name) or m.g.config.cppDefines.contains(result):
+  # fields are tricky to get right and thanks to generic types producing
+  # duplicates we can end up mangling the same field multiple times. However
+  # if we do so, the 'cppDefines' table might be modified in the meantime
+  # meaning we produce inconsistent field names (see bug #5404).
+  # Hence we do not check for ``m.g.config.cppDefines.contains(result)`` here
+  # anymore:
+  if isKeyword(name):
     result.add "_0"
 
 when false:

compiler/ccgutils.nim

@@ -10,7 +10,7 @@
 # This module declares some helpers for the C code generator.
 
 import
-  ast, astalgo, ropes, lists, hashes, strutils, types, msgs, wordrecg,
+  ast, astalgo, ropes, hashes, strutils, types, msgs, wordrecg,
   platform, trees
 
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =

compiler/cgen.nim

@@ -11,7 +11,7 @@
 
 import
   ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets,
-  nversion, nimsets, msgs, securehash, bitsets, idents, lists, types,
+  nversion, nimsets, msgs, securehash, bitsets, idents, types,
   ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth,
   condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
   lowerings, semparallel, tables, sets, ndi
@@ -75,12 +75,13 @@ proc isSimpleConst(typ: PType): bool =
 proc useStringh(m: BModule) =
   if includesStringh notin m.flags:
     incl m.flags, includesStringh
-    discard lists.includeStr(m.headerFiles, "<string.h>")
+    m.includeHeader("<string.h>")
 
 proc useHeader(m: BModule, sym: PSym) =
   if lfHeader in sym.loc.flags:
     assert(sym.annex != nil)
-    discard lists.includeStr(m.headerFiles, getStr(sym.annex.path))
+    let str = getStr(sym.annex.path)
+    m.includeHeader(str)
 
 proc cgsym(m: BModule, name: string): Rope
 
@@ -594,15 +595,14 @@ proc cgsym(m: BModule, name: string): Rope =
 
 proc generateHeaders(m: BModule) =
   add(m.s[cfsHeaders], tnl & "#include \"nimbase.h\"" & tnl)
-  var it = PStrEntry(m.headerFiles.head)
+  
-  while it != nil:
+  for it in m.headerFiles:
-    if it.data[0] == '#':
+    if it[0] == '#':
-      add(m.s[cfsHeaders], rope(it.data.replace('`', '"') & tnl))
+      add(m.s[cfsHeaders], rope(it.replace('`', '"') & tnl))
-    elif it.data[0] notin {'\"', '<'}:
+    elif it[0] notin {'\"', '<'}:
-      addf(m.s[cfsHeaders], "#include \"$1\"$N", [rope(it.data)])
+      addf(m.s[cfsHeaders], "#include \"$1\"$N", [rope(it)])
     else:
-      addf(m.s[cfsHeaders], "#include $1$N", [rope(it.data)])
+      addf(m.s[cfsHeaders], "#include $1$N", [rope(it)])
-    it = PStrEntry(it.next)
   add(m.s[cfsHeaders], "#undef linux" & tnl)
 
 proc initFrame(p: BProc, procname, filename: Rope): Rope =
@@ -715,10 +715,13 @@ proc genProcAux(m: BModule, prc: PSym) =
     add(generatedProc, ~"}$N")
   add(m.s[cfsProcs], generatedProc)
 
-proc crossesCppBoundary(m: BModule; sym: PSym): bool {.inline.} =
+proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} =
-  result = sfCompileToCpp in m.module.flags and
+  result = (sfCompileToCpp in m.module.flags and
            sfCompileToCpp notin sym.getModule().flags and
-           gCmd != cmdCompileToCpp
+           gCmd != cmdCompileToCpp) or (
+           sym.flags * {sfImportc, sfInfixCall, sfCompilerProc} == {sfImportc} and
+           sym.magic == mNone and
+           gCmd == cmdCompileToCpp)
 
 proc genProcPrototype(m: BModule, sym: PSym) =
   useHeader(m, sym)
@@ -732,7 +735,7 @@ proc genProcPrototype(m: BModule, sym: PSym) =
     var header = genProcHeader(m, sym)
     if sfNoReturn in sym.flags and hasDeclspec in extccomp.CC[cCompiler].props:
       header = "__declspec(noreturn) " & header
-    if sym.typ.callConv != ccInline and crossesCppBoundary(m, sym):
+    if sym.typ.callConv != ccInline and requiresExternC(m, sym):
       header = "extern \"C\" " & header
     if sfPure in sym.flags and hasAttribute in CC[cCompiler].props:
       header.add(" __attribute__((naked))")
@@ -971,7 +974,7 @@ proc genMainProc(m: BModule) =
     else:
       nimMain = WinNimDllMain
       otherMain = WinCDllMain
-    discard lists.includeStr(m.headerFiles, "<windows.h>")
+    m.includeHeader("<windows.h>")
   elif optGenDynLib in gGlobalOptions:
     nimMain = PosixNimDllMain
     otherMain = PosixCDllMain
@@ -1126,7 +1129,7 @@ proc initProcOptions(m: BModule): TOptions =
 proc rawNewModule(g: BModuleList; module: PSym, filename: string): BModule =
   new(result)
   result.tmpBase = rope("TM" & $hashOwner(module) & "_")
-  initLinkedList(result.headerFiles)
+  result.headerFiles = @[]
   result.declaredThings = initIntSet()
   result.declaredProtos = initIntSet()
   result.cfilename = filename
@@ -1163,7 +1166,7 @@ proc nullify[T](arr: var T) =
 proc resetModule*(m: BModule) =
   # between two compilations in CAAS mode, we can throw
   # away all the data that was written to disk
-  initLinkedList(m.headerFiles)
+  m.headerFiles = @[]
   m.declaredProtos = initIntSet()
   m.forwTypeCache = initTable[SigHash, Rope]()
   m.initProc = newProc(nil, m)
@@ -1363,7 +1366,7 @@ proc updateCachedModule(m: BModule) =
     cf.flags = {CfileFlag.Cached}
   addFileToCompile(cf)
 
-proc myClose(b: PPassContext, n: PNode): PNode =
+proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
   result = n
   if b == nil or passes.skipCodegen(n): return
   var m = BModule(b)
@@ -1375,7 +1378,7 @@ proc myClose(b: PPassContext, n: PNode): PNode =
 
   if sfMainModule in m.module.flags:
     incl m.flags, objHasKidsValid
-    var disp = generateMethodDispatchers()
+    var disp = generateMethodDispatchers(graph)
     for i in 0..sonsLen(disp)-1: genProcAux(m, disp.sons[i].sym)
     genMainProc(m)
 

compiler/cgendata.nim

@@ -10,7 +10,7 @@
 ## This module contains the data structures for the C code generation phase.
 
 import
-  ast, astalgo, ropes, passes, options, intsets, lists, platform, sighashes,
+  ast, astalgo, ropes, passes, options, intsets, platform, sighashes,
   tables, ndi
 
 from msgs import TLineInfo
@@ -130,7 +130,7 @@ type
     forwTypeCache*: TypeCache # cache for forward declarations of types
     declaredThings*: IntSet   # things we have declared in this .c file
     declaredProtos*: IntSet   # prototypes we have declared in this .c file
-    headerFiles*: TLinkedList # needed headers to include
+    headerFiles*: seq[string] # needed headers to include
     typeInfoMarker*: TypeCache # needed for generating type information
     initProc*: BProc          # code for init procedure
     postInitProc*: BProc      # code to be executed after the init proc
@@ -148,9 +148,13 @@ type
     g*: BModuleList
     ndi*: NdiFile
 
+proc includeHeader*(this: BModule; header: string) =
+  if not this.headerFiles.contains header:
+    this.headerFiles.add header
+
 proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # section in the current block
-  result = p.blocks[p.blocks.len - 1].sections[s]
+  result = p.blocks[^1].sections[s]
 
 proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # top level proc sections

compiler/cgmeth.nim

@@ -11,7 +11,7 @@
 
 import
   intsets, options, ast, astalgo, msgs, idents, renderer, types, magicsys,
-  sempass2, strutils
+  sempass2, strutils, modulegraphs
 
 proc genConv(n: PNode, d: PType, downcast: bool): PNode =
   var dest = skipTypes(d, abstractPtrs)
@@ -35,19 +35,24 @@ proc genConv(n: PNode, d: PType, downcast: bool): PNode =
   else:
     result = n
 
+proc getDispatcher*(s: PSym): PSym =
+  ## can return nil if is has no dispatcher.
+  let dispn = lastSon(s.ast)
+  if dispn.kind == nkSym:
+    let disp = dispn.sym
+    if sfDispatcher in disp.flags: result = disp
+
 proc methodCall*(n: PNode): PNode =
   result = n
   # replace ordinary method by dispatcher method:
-  var disp = lastSon(result.sons[0].sym.ast).sym
+  let disp = getDispatcher(result.sons[0].sym)
-  assert sfDispatcher in disp.flags
+  if disp != nil:
-  result.sons[0].sym = disp
+    result.sons[0].sym = disp
-  # change the arguments to up/downcasts to fit the dispatcher's parameters:
+    # change the arguments to up/downcasts to fit the dispatcher's parameters:
-  for i in countup(1, sonsLen(result)-1):
+    for i in countup(1, sonsLen(result)-1):
-    result.sons[i] = genConv(result.sons[i], disp.typ.sons[i], true)
+      result.sons[i] = genConv(result.sons[i], disp.typ.sons[i], true)
-
+  else:
-# save for incremental compilation:
+    localError(n.info, "'" & $result.sons[0] & "' lacks a dispatcher")
-var
-  gMethods: seq[tuple[methods: TSymSeq, dispatcher: PSym]] = @[]
 
 type
   MethodResult = enum No, Invalid, Yes
@@ -55,8 +60,8 @@ type
 proc sameMethodBucket(a, b: PSym): MethodResult =
   if a.name.id != b.name.id: return
   if sonsLen(a.typ) != sonsLen(b.typ):
-    return                    # check for return type:
+    return
-  if not sameTypeOrNil(a.typ.sons[0], b.typ.sons[0]): return
+
   for i in countup(1, sonsLen(a.typ) - 1):
     var aa = a.typ.sons[i]
     var bb = b.typ.sons[i]
@@ -84,6 +89,14 @@ proc sameMethodBucket(a, b: PSym): MethodResult =
         return No
     else:
       return No
+  if result == Yes:
+    # check for return type:
+    if not sameTypeOrNil(a.typ.sons[0], b.typ.sons[0]):
+      if b.typ.sons[0] != nil and b.typ.sons[0].kind == tyExpr:
+        # infer 'auto' from the base to make it consistent:
+        b.typ.sons[0] = a.typ.sons[0]
+      else:
+        return No
 
 proc attachDispatcher(s: PSym, dispatcher: PNode) =
   var L = s.ast.len-1
@@ -144,27 +157,27 @@ proc fixupDispatcher(meth, disp: PSym) =
       if disp.typ.lockLevel < meth.typ.lockLevel:
         disp.typ.lockLevel = meth.typ.lockLevel
 
-proc methodDef*(s: PSym, fromCache: bool) =
+proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
-  let L = len(gMethods)
+  let L = len(g.methods)
   var witness: PSym
   for i in countup(0, L - 1):
-    let disp = gMethods[i].dispatcher
+    let disp = g.methods[i].dispatcher
     case sameMethodBucket(disp, s)
     of Yes:
-      add(gMethods[i].methods, s)
+      add(g.methods[i].methods, s)
       attachDispatcher(s, lastSon(disp.ast))
       fixupDispatcher(s, disp)
       #echo "fixup ", disp.name.s, " ", disp.id
       when useEffectSystem: checkMethodEffects(disp, s)
-      if sfBase in s.flags and gMethods[i].methods[0] != s:
+      if sfBase in s.flags and g.methods[i].methods[0] != s:
         # already exists due to forwarding definition?
         localError(s.info, "method is not a base")
       return
     of No: discard
     of Invalid:
-      if witness.isNil: witness = gMethods[i].methods[0]
+      if witness.isNil: witness = g.methods[i].methods[0]
   # create a new dispatcher:
-  add(gMethods, (methods: @[s], dispatcher: createDispatcher(s)))
+  add(g.methods, (methods: @[s], dispatcher: createDispatcher(s)))
   #echo "adding ", s.info
   #if fromCache:
   #  internalError(s.info, "no method dispatcher found")
@@ -258,12 +271,12 @@ proc genDispatcher(methods: TSymSeq, relevantCols: IntSet): PSym =
       disp = ret
   result.ast.sons[bodyPos] = disp
 
-proc generateMethodDispatchers*(): PNode =
+proc generateMethodDispatchers*(g: ModuleGraph): PNode =
   result = newNode(nkStmtList)
-  for bucket in countup(0, len(gMethods) - 1):
+  for bucket in countup(0, len(g.methods) - 1):
     var relevantCols = initIntSet()
-    for col in countup(1, sonsLen(gMethods[bucket].methods[0].typ) - 1):
+    for col in countup(1, sonsLen(g.methods[bucket].methods[0].typ) - 1):
-      if relevantCol(gMethods[bucket].methods, col): incl(relevantCols, col)
+      if relevantCol(g.methods[bucket].methods, col): incl(relevantCols, col)
-    sortBucket(gMethods[bucket].methods, relevantCols)
+    sortBucket(g.methods[bucket].methods, relevantCols)
     addSon(result,
-           newSymNode(genDispatcher(gMethods[bucket].methods, relevantCols)))
+           newSymNode(genDispatcher(g.methods[bucket].methods, relevantCols)))

compiler/commands.nim

@@ -26,8 +26,8 @@ bootSwitch(usedGoGC, defined(gogc), "--gc:go")
 bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
 import
-  os, msgs, options, nversion, condsyms, strutils, extccomp, platform, lists,
+  os, msgs, options, nversion, condsyms, strutils, extccomp, platform,
-  wordrecg, parseutils, nimblecmd, idents, parseopt
+  wordrecg, parseutils, nimblecmd, idents, parseopt, sequtils
 
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
@@ -335,12 +335,14 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "excludepath":
     expectArg(switch, arg, pass, info)
     let path = processPath(arg, info)
-    lists.excludePath(options.searchPaths, path)
+    
-    lists.excludePath(options.lazyPaths, path)
+    options.searchPaths.keepItIf( cmpPaths(it, path) != 0 )
+    options.lazyPaths.keepItIf( cmpPaths(it, path) != 0 )
+    
     if (len(path) > 0) and (path[len(path) - 1] == DirSep):
       let strippedPath = path[0 .. (len(path) - 2)]
-      lists.excludePath(options.searchPaths, strippedPath)
+      options.searchPaths.keepItIf( cmpPaths(it, strippedPath) != 0 )
-      lists.excludePath(options.lazyPaths, strippedPath)
+      options.lazyPaths.keepItIf( cmpPaths(it, strippedPath) != 0 )
   of "nimcache":
     expectArg(switch, arg, pass, info)
     options.nimcacheDir = processPath(arg, info, true)

compiler/compilerlog.nim

@@ -0,0 +1,9 @@
+
+from os import getHomeDir, `/`
+
+proc logStr*(line: string) =
+  var f: File
+  if open(f, getHomeDir() / "nimsuggest.log", fmAppend):
+    f.writeLine(line)
+    f.close()
+

compiler/docgen2.nim

@@ -33,11 +33,11 @@ template closeImpl(body: untyped) {.dirty.} =
     except IOError:
       discard
 
-proc close(p: PPassContext, n: PNode): PNode =
+proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
     writeOutput(g.doc, g.module.filename, HtmlExt, useWarning)
 
-proc closeJson(p: PPassContext, n: PNode): PNode =
+proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
     writeOutputJson(g.doc, g.module.filename, ".json", useWarning)
 

compiler/extccomp.nim

@@ -13,7 +13,7 @@
 # nim files.
 
 import
-  lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs,
+  ropes, os, strutils, osproc, platform, condsyms, options, msgs,
   securehash, streams
 
 #from debuginfo import writeDebugInfo
@@ -390,8 +390,8 @@ type
   CfileList = seq[Cfile]
 
 var
-  externalToLink: TLinkedList # files to link in addition to the file
+  externalToLink: seq[string] = @[] # files to link in addition to the file
-                              # we compiled
+                                    # we compiled
   linkOptionsCmd: string = ""
   compileOptionsCmd: seq[string] = @[]
   linkOptions: string = ""
@@ -486,11 +486,10 @@ proc resetCompilationLists* =
   # when the module is loaded/unloaded it adds/removes its items
   # That's because we still need to hash check the external files
   # Maybe we can do that in checkDep on the other hand?
-  initLinkedList(externalToLink)
+  externalToLink.setLen 0
 
 proc addExternalFileToLink*(filename: string) =
-  prependStr(externalToLink, filename)
+  externalToLink.insert(filename, 0)
-  # BUGFIX: was ``appendStr``
 
 proc execWithEcho(cmd: string, msg = hintExecuting): int =
   rawMessage(msg, cmd)
@@ -527,9 +526,6 @@ proc noAbsolutePaths: bool {.inline.} =
   # `optGenMapping` is included here for niminst.
   result = gGlobalOptions * {optGenScript, optGenMapping} != {}
 
-proc add(s: var string, many: openArray[string]) =
-  s.add many.join
-
 proc cFileSpecificOptions(cfilename: string): string =
   result = compileOptions
   for option in compileOptionsCmd:
@@ -560,22 +556,24 @@ proc getLinkOptions: string =
   for linkedLib in items(cLinkedLibs):
     result.add(CC[cCompiler].linkLibCmd % linkedLib.quoteShell)
   for libDir in items(cLibs):
-    result.add([CC[cCompiler].linkDirCmd, libDir.quoteShell])
+    result.add(join([CC[cCompiler].linkDirCmd, libDir.quoteShell]))
 
 proc needsExeExt(): bool {.inline.} =
   result = (optGenScript in gGlobalOptions and targetOS == osWindows) or
            (platform.hostOS == osWindows)
 
-proc getCompilerExe(compiler: TSystemCC): string =
+proc getCompilerExe(compiler: TSystemCC; cfile: string): string =
-  result = if gCmd == cmdCompileToCpp: CC[compiler].cppCompiler
+  result = if gCmd == cmdCompileToCpp and not cfile.endsWith(".c"):
-           else: CC[compiler].compilerExe
+             CC[compiler].cppCompiler
+           else:
+             CC[compiler].compilerExe
   if result.len == 0:
     rawMessage(errCompilerDoesntSupportTarget, CC[compiler].name)
 
 proc getLinkerExe(compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
            elif gMixedMode and gCmd != cmdCompileToCpp: CC[compiler].cppCompiler
-           else: compiler.getCompilerExe
+           else: compiler.getCompilerExe("")
 
 proc getCompileCFileCmd*(cfile: Cfile): string =
   var c = cCompiler
@@ -596,7 +594,7 @@ proc getCompileCFileCmd*(cfile: Cfile): string =
 
   var options = cFileSpecificOptions(cfile.cname)
   var exe = getConfigVar(c, ".exe")
-  if exe.len == 0: exe = c.getCompilerExe
+  if exe.len == 0: exe = c.getCompilerExe(cfile.cname)
 
   if needsExeExt(): exe = addFileExt(exe, "exe")
   if optGenDynLib in gGlobalOptions and
@@ -609,12 +607,12 @@ proc getCompileCFileCmd*(cfile: Cfile): string =
     includeCmd = CC[c].includeCmd & quoteShell(libpath)
 
     for includeDir in items(cIncludes):
-      includeCmd.add([CC[c].includeCmd, includeDir.quoteShell])
+      includeCmd.add(join([CC[c].includeCmd, includeDir.quoteShell]))
 
     compilePattern = joinPath(ccompilerpath, exe)
   else:
     includeCmd = ""
-    compilePattern = c.getCompilerExe
+    compilePattern = c.getCompilerExe(cfile.cname)
 
   var cf = if noAbsolutePaths(): extractFilename(cfile.cname)
            else: cfile.cname
@@ -784,14 +782,12 @@ proc callCCompiler*(projectfile: string) =
         rawMessage(errExecutionOfProgramFailed, cmds.join())
   if optNoLinking notin gGlobalOptions:
     # call the linker:
-    var it = PStrEntry(externalToLink.head)
     var objfiles = ""
-    while it != nil:
+    for it in externalToLink:
-      let objFile = if noAbsolutePaths(): it.data.extractFilename else: it.data
+      let objFile = if noAbsolutePaths(): it.extractFilename else: it
       add(objfiles, ' ')
       add(objfiles, quoteShell(
           addFileExt(objFile, CC[cCompiler].objExt)))
-      it = PStrEntry(it.next)
     for x in toCompile:
       add(objfiles, ' ')
       add(objfiles, quoteShell(x.obj))
@@ -834,15 +830,14 @@ proc writeJsonBuildInstructions*(projectfile: string) =
       else:
         lit "],\L"
 
-  proc linkfiles(f: File; buf, objfiles: var string; toLink: TLinkedList) =
+  proc linkfiles(f: File; buf, objfiles: var string; toLink: seq[string]) =
-    var it = PStrEntry(toLink.head)
+    for i, it in toLink:
-    while it != nil:
+      let objfile = addFileExt(it, CC[cCompiler].objExt)
-      let objfile = addFileExt(it.data, CC[cCompiler].objExt)
+      str(objfile)
-      str objfile
       add(objfiles, ' ')
       add(objfiles, quoteShell(objfile))
-      it = PStrEntry(it.next)
+      
-      if it == nil:
+      if i == toLink.high:
         lit "\L"
       else:
         lit ",\L"

compiler/jsgen.nim

@@ -31,7 +31,7 @@ implements the required case distinction.
 
 import
   ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options,
-  nversion, nimsets, msgs, securehash, bitsets, idents, lists, types, os,
+  nversion, nimsets, msgs, securehash, bitsets, idents, types, os,
   times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils,
   intsets, cgmeth, lowerings
 
@@ -2237,13 +2237,13 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
   add(p.g.code, p.body)
   globals.unique = p.unique
 
-proc wholeCode*(m: BModule): Rope =
+proc wholeCode(graph: ModuleGraph; m: BModule): Rope =
   for prc in globals.forwarded:
     if not globals.generatedSyms.containsOrIncl(prc.id):
       var p = newProc(globals, m, nil, m.module.options)
       attachProc(p, prc)
 
-  var disp = generateMethodDispatchers()
+  var disp = generateMethodDispatchers(graph)
   for i in 0..sonsLen(disp)-1:
     let prc = disp.sons[i].sym
     if not globals.generatedSyms.containsOrIncl(prc.id):
@@ -2277,7 +2277,7 @@ proc genClass(obj: PType; content: Rope; ext: string) =
   let outfile = changeFileExt(completeCFilePath($cls), ext)
   discard writeRopeIfNotEqual(result, outfile)
 
-proc myClose(b: PPassContext, n: PNode): PNode =
+proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
   if passes.skipCodegen(n): return n
   result = myProcess(b, n)
   var m = BModule(b)
@@ -2285,7 +2285,7 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     let ext = if m.target == targetJS: "js" else: "php"
     let f = if globals.classes.len == 0: m.module.filename
             else: "nimsystem"
-    let code = wholeCode(m)
+    let code = wholeCode(graph, m)
     let outfile =
       if options.outFile.len > 0:
         if options.outFile.isAbsolute: options.outFile

compiler/lambdalifting.nim

@@ -10,7 +10,7 @@
 # This include file implements lambda lifting for the transformator.
 
 import
-  intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os,
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
   idents, renderer, types, magicsys, rodread, lowerings, tables
 
 discard """

compiler/lists.nim

@@ -7,113 +7,22 @@
 #    distribution, for details about the copyright.
 #
 
-# This module implements a generic doubled linked list.
+# This module is deprecated, don't use it.
-# TODO Remove this and replace it with something sensible
+# TODO Remove this
+
 import os
-type
-  PListEntry* = ref TListEntry
-  TListEntry* = object of RootObj
-    prev*, next*: PListEntry
 
-  TStrEntry* = object of TListEntry
+static:
-    data*: string
+  echo "WARNING: imported deprecated module compiler/lists.nim, use seq ore lists from the standard library"
 
-  PStrEntry* = ref TStrEntry
+proc appendStr*(list: var seq[string]; data: string) {.deprecated.} =
-  TLinkedList* = object       # for the "find" operation:
+  # just use system.add
-    head*, tail*: PListEntry
+  list.add(data)
-    counter*: int
 
-  TCompareProc* = proc (entry: PListEntry, closure: pointer): bool {.nimcall.}
+proc includeStr(list: var seq[string]; data: string): bool {.deprecated.} =
-
+  if list.contains(data):
-proc initLinkedList*(list: var TLinkedList) =
+    result = true
-  list.counter = 0
-  list.head = nil
-  list.tail = nil
-
-proc append*(list: var TLinkedList, entry: PListEntry) =
-  inc(list.counter)
-  entry.next = nil
-  entry.prev = list.tail
-  if list.tail != nil:
-    assert(list.tail.next == nil)
-    list.tail.next = entry
-  list.tail = entry
-  if list.head == nil: list.head = entry
-
-proc contains*(list: TLinkedList, data: string): bool =
-  var it = list.head
-  while it != nil:
-    if PStrEntry(it).data == data:
-      return true
-    it = it.next
-
-proc newStrEntry(data: string): PStrEntry =
-  new(result)
-  result.data = data
-
-proc appendStr*(list: var TLinkedList, data: string) =
-  append(list, newStrEntry(data))
-
-proc includeStr*(list: var TLinkedList, data: string): bool =
-  if contains(list, data): return true
-  appendStr(list, data)       # else: add to list
-
-proc prepend*(list: var TLinkedList, entry: PListEntry) =
-  inc(list.counter)
-  entry.prev = nil
-  entry.next = list.head
-  if list.head != nil:
-    assert(list.head.prev == nil)
-    list.head.prev = entry
-  list.head = entry
-  if list.tail == nil: list.tail = entry
-
-proc prependStr*(list: var TLinkedList, data: string) =
-  prepend(list, newStrEntry(data))
-
-proc insertBefore*(list: var TLinkedList, pos, entry: PListEntry) =
-  assert(pos != nil)
-  if pos == list.head:
-    prepend(list, entry)
   else:
-    inc(list.counter)
+    result = false
-    entry.next = pos
+    list.add data
-    entry.prev = pos.prev
-    if pos.prev != nil: pos.prev.next = entry
-    pos.prev = entry
 
-proc remove*(list: var TLinkedList, entry: PListEntry) =
-  dec(list.counter)
-  if entry == list.tail:
-    list.tail = entry.prev
-  if entry == list.head:
-    list.head = entry.next
-  if entry.next != nil: entry.next.prev = entry.prev
-  if entry.prev != nil: entry.prev.next = entry.next
-
-proc bringToFront*(list: var TLinkedList, entry: PListEntry) =
-  when true:
-    list.remove entry
-    list.prepend entry
-  else:
-    if entry == list.head: return
-    if entry == list.tail: list.tail = entry.prev
-    if entry.next != nil: entry.next.prev = entry.prev
-    if entry.prev != nil: entry.prev.next = entry.next
-    entry.prev = nil
-    entry.next = list.head
-    list.head = entry
-
-proc excludePath*(list: var TLinkedList, data: string) =
-  var it = list.head
-  while it != nil:
-    let nxt = it.next
-    if cmpPaths(PStrEntry(it).data, data) == 0:
-      remove(list, it)
-    it = nxt
-
-proc find*(list: TLinkedList, fn: TCompareProc, closure: pointer): PListEntry =
-  result = list.head
-  while result != nil:
-    if fn(result, closure): return
-    result = result.next

compiler/lookups.nim

@@ -115,7 +115,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
       considerQuotedIdent(m)
     else:
       getIdent("err:" & renderTree(m))
-  result = newSym(skError, ident, getCurrOwner(), n.info)
+  result = newSym(skError, ident, getCurrOwner(c), n.info)
   result.typ = errorType(c)
   incl(result.flags, sfDiscardable)
   # pretend it's imported from some unknown module to prevent cascading errors:

compiler/main.nim

@@ -15,7 +15,7 @@ import
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
   cgen, jsgen, json, nversion,
   platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
-  docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists,
+  docgen2, service, parser, modules, ccgutils, sigmatch, ropes,
   modulegraphs
 
 from magicsys import systemModule, resetSysTypes
@@ -151,7 +151,7 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
   # In "nim serve" scenario, each command must reset the registered passes
   clearPasses()
   gLastCmdTime = epochTime()
-  appendStr(searchPaths, options.libpath)
+  searchPaths.add(options.libpath)
   when false: # gProjectFull.len != 0:
     # current path is always looked first for modules
     prependStr(searchPaths, gProjectPath)
@@ -229,7 +229,7 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
       for s in definedSymbolNames(): definedSymbols.elems.add(%s)
 
       var libpaths = newJArray()
-      for dir in iterSearchPath(searchPaths): libpaths.elems.add(%dir)
+      for dir in searchPaths: libpaths.elems.add(%dir)
 
       var dumpdata = % [
         (key: "version", val: %VersionAsString),
@@ -245,7 +245,7 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
       for s in definedSymbolNames(): msgWriteln(s, {msgStdout, msgSkipHook})
       msgWriteln("-- end of list --", {msgStdout, msgSkipHook})
 
-      for it in iterSearchPath(searchPaths): msgWriteln(it)
+      for it in searchPaths: msgWriteln(it)
   of "check":
     gCmd = cmdCheck
     commandCheck(graph, cache)

compiler/modulegraphs.nim

@@ -40,9 +40,16 @@ type
                               # module dependencies.
     backend*: RootRef # minor hack so that a backend can extend this easily
     config*: ConfigRef
+    doStopCompile*: proc(): bool {.closure.}
+    usageSym*: PSym # for nimsuggest
+    owners*: seq[PSym]
+    methods*: seq[tuple[methods: TSymSeq, dispatcher: PSym]]
 
 {.this: g.}
 
+proc stopCompile*(g: ModuleGraph): bool {.inline.} =
+  result = doStopCompile != nil and doStopCompile()
+
 proc newModuleGraph*(config: ConfigRef = nil): ModuleGraph =
   result = ModuleGraph()
   initStrTable(result.packageSyms)
@@ -54,6 +61,8 @@ proc newModuleGraph*(config: ConfigRef = nil): ModuleGraph =
     result.config = newConfigRef()
   else:
     result.config = config
+  result.owners = @[]
+  result.methods = @[]
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(packageSyms)
@@ -61,6 +70,9 @@ proc resetAllModules*(g: ModuleGraph) =
   modules = @[]
   importStack = @[]
   inclToMod = initTable[int32, int32]()
+  usageSym = nil
+  owners = @[]
+  methods = @[]
 
 proc getModule*(g: ModuleGraph; fileIdx: int32): PSym =
   if fileIdx >= 0 and fileIdx < modules.len:
@@ -73,7 +85,7 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: int32) =
     deps.incl m.position.dependsOn(dep)
     # we compute the transitive closure later when quering the graph lazily.
     # this improve efficiency quite a lot:
-    invalidTransitiveClosure = true
+    #invalidTransitiveClosure = true
 
 proc addIncludeDep*(g: ModuleGraph; module, includeFile: int32) =
   discard hasKeyOrPut(inclToMod, includeFile, module)

compiler/msgs.nim

@@ -567,6 +567,15 @@ proc newFileInfo(fullPath, projPath: string): TFileInfo =
   if optEmbedOrigSrc in gGlobalOptions or true:
     result.lines = @[]
 
+proc fileInfoKnown*(filename: string): bool =
+  var
+    canon: string
+  try:
+    canon = canonicalizePath(filename)
+  except:
+    canon = filename
+  result = filenameToIndexTbl.hasKey(canon)
+
 proc fileInfoIdx*(filename: string; isKnownFile: var bool): int32 =
   var
     canon: string

compiler/nimblecmd.nim

@@ -9,11 +9,11 @@
 
 ## Implements some helper procs for Nimble (Nim's package manager) support.
 
-import parseutils, strutils, strtabs, os, options, msgs, lists
+import parseutils, strutils, strtabs, os, options, msgs
 
 proc addPath*(path: string, info: TLineInfo) =
-  if not contains(options.searchPaths, path):
+  if not options.searchPaths.contains(path):
-    lists.prependStr(options.searchPaths, path)
+    options.searchPaths.insert(path, 0)
 
 proc versionSplitPos(s: string): int =
   result = s.len-2
@@ -61,7 +61,7 @@ iterator chosen(packages: StringTableRef): string =
 proc addNimblePath(p: string, info: TLineInfo) =
   if not contains(options.searchPaths, p):
     message(info, hintPath, p)
-    lists.prependStr(options.lazyPaths, p)
+    options.lazyPaths.insert(p, 0)
 
 proc addPathRec(dir: string, info: TLineInfo) =
   var packages = newStringTable(modeStyleInsensitive)

compiler/nimeval.nim

@@ -10,7 +10,7 @@
 ## exposes the Nim VM to clients.
 import
   ast, modules, passes, passaux, condsyms,
-  options, nimconf, lists, sem, semdata, llstream, vm, modulegraphs, idents
+  options, nimconf, sem, semdata, llstream, vm, modulegraphs, idents
 
 proc execute*(program: string) =
   passes.gIncludeFile = includeModule
@@ -25,7 +25,7 @@ proc execute*(program: string) =
   registerPass(semPass)
   registerPass(evalPass)
 
-  appendStr(searchPaths, options.libpath)
+  searchPaths.add options.libpath
   var graph = newModuleGraph()
   var cache = newIdentCache()
   var m = makeStdinModule(graph)

compiler/nimfix/nimfix.nim

@@ -13,7 +13,7 @@ import strutils, os, parseopt
 import compiler/[options, commands, modules, sem,
   passes, passaux, nimfix/pretty,
   msgs, nimconf,
-  extccomp, condsyms, lists,
+  extccomp, condsyms,
   modulegraphs, idents]
 
 const Usage = """
@@ -39,10 +39,10 @@ proc mainCommand =
   registerPass verbosePass
   registerPass semPass
   gCmd = cmdPretty
-  appendStr(searchPaths, options.libpath)
+  searchPaths.add options.libpath
   if gProjectFull.len != 0:
     # current path is always looked first for modules
-    prependStr(searchPaths, gProjectPath)
+    searchPaths.insert(gProjectPath, 0)
 
   compileProject(newModuleGraph(), newIdentCache())
   pretty.overwriteFiles()

compiler/options.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  os, lists, strutils, strtabs, osproc, sets
+  os, strutils, strtabs, osproc, sets
 
 const
   hasTinyCBackend* = defined(tinyc)
@@ -100,7 +100,7 @@ type
 
   IdeCmd* = enum
     ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod,
-    ideHighlight, ideOutline
+    ideHighlight, ideOutline, ideKnown
 
   ConfigRef* = ref object ## eventually all global configuration should be moved here
     cppDefines*: HashSet[string]
@@ -129,7 +129,8 @@ var
   gExitcode*: int8
   gCmd*: TCommands = cmdNone  # the command
   gSelectedGC* = gcRefc       # the selected GC
-  searchPaths*, lazyPaths*: TLinkedList
+  searchPaths*: seq[string] = @[]
+  lazyPaths*: seq[string]   = @[]
   outFile*: string = ""
   docSeeSrcUrl*: string = ""  # if empty, no seeSrc will be generated. \
   # The string uses the formatting variables `path` and `line`.
@@ -267,9 +268,7 @@ proc removeTrailingDirSep*(path: string): string =
 
 proc disableNimblePath*() =
   gNoNimblePath = true
-  lazyPaths.head = nil
+  lazyPaths.setLen(0)
-  lazyPaths.tail = nil
-  lazyPaths.counter = 0
 
 include packagehandling
 
@@ -336,27 +335,22 @@ proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string =
   result = joinPath(subdir, tail)
   #echo "completeGeneratedFilePath(", f, ") = ", result
 
-iterator iterSearchPath*(searchPaths: TLinkedList): string =
-  var it = PStrEntry(searchPaths.head)
-  while it != nil:
-    yield it.data
-    it = PStrEntry(it.next)
-
 proc rawFindFile(f: string): string =
-  for it in iterSearchPath(searchPaths):
+  for it in searchPaths:
     result = joinPath(it, f)
     if existsFile(result):
       return result.canonicalizePath
   result = ""
 
 proc rawFindFile2(f: string): string =
-  var it = PStrEntry(lazyPaths.head)
+  for i, it in lazyPaths:
-  while it != nil:
+    result = joinPath(it, f)
-    result = joinPath(it.data, f)
     if existsFile(result):
-      bringToFront(lazyPaths, it)
+      # bring to front
+      for j in countDown(i,1):
+        swap(lazyPaths[j], lazyPaths[j-1])
+      
       return result.canonicalizePath
-    it = PStrEntry(it.next)
   result = ""
 
 template patchModule() {.dirty.} =
@@ -442,6 +436,7 @@ proc parseIdeCmd*(s: string): IdeCmd =
   of "mod": ideMod
   of "highlight": ideHighlight
   of "outline": ideOutline
+  of "known": ideKnown
   else: ideNone
 
 proc `$`*(c: IdeCmd): string =
@@ -456,3 +451,4 @@ proc `$`*(c: IdeCmd): string =
   of ideNone: "none"
   of ideHighlight: "highlight"
   of ideOutline: "outline"
+  of ideKnown: "known"

compiler/passaux.nim

@@ -29,21 +29,3 @@ proc verboseProcess(context: PPassContext, n: PNode): PNode =
     message(n.info, hintProcessing, $idgen.gFrontendId)
 
 const verbosePass* = makePass(open = verboseOpen, process = verboseProcess)
-
-proc cleanUp(c: PPassContext, n: PNode): PNode =
-  result = n
-  # we cannot clean up if dead code elimination is activated
-  if optDeadCodeElim in gGlobalOptions or n == nil: return
-  case n.kind
-  of nkStmtList:
-    for i in countup(0, sonsLen(n) - 1): discard cleanUp(c, n.sons[i])
-  of nkProcDef, nkMethodDef:
-    if n.sons[namePos].kind == nkSym:
-      var s = n.sons[namePos].sym
-      if sfDeadCodeElim notin getModule(s).flags and not astNeeded(s):
-        s.ast.sons[bodyPos] = ast.emptyNode # free the memory
-  else:
-    discard
-
-const cleanupPass* = makePass(process = cleanUp, close = cleanUp)
-

compiler/passes.nim

@@ -11,7 +11,7 @@
 # `TPass` interface.
 
 import
-  strutils, lists, options, ast, astalgo, llstream, msgs, platform, os,
+  strutils, options, ast, astalgo, llstream, msgs, platform, os,
   condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
   nimsets, syntaxes, times, rodread, idgen, modulegraphs
 
@@ -24,7 +24,7 @@ type
   TPassOpen* = proc (graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext {.nimcall.}
   TPassOpenCached* =
     proc (graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext {.nimcall.}
-  TPassClose* = proc (p: PPassContext, n: PNode): PNode {.nimcall.}
+  TPassClose* = proc (graph: ModuleGraph; p: PPassContext, n: PNode): PNode {.nimcall.}
   TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
 
   TPass* = tuple[open: TPassOpen, openCached: TPassOpenCached,
@@ -94,7 +94,7 @@ proc carryPass*(g: ModuleGraph; p: TPass, module: PSym; cache: IdentCache;
                 m: TPassData): TPassData =
   var c = p.open(g, module, cache)
   result.input = p.process(c, m.input)
-  result.closeOutput = if p.close != nil: p.close(c, m.closeOutput)
+  result.closeOutput = if p.close != nil: p.close(g, c, m.closeOutput)
                        else: m.closeOutput
 
 proc carryPasses*(g: ModuleGraph; nodes: PNode, module: PSym;
@@ -121,10 +121,10 @@ proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym,
     else:
       a[i] = nil
 
-proc closePasses(a: var TPassContextArray) =
+proc closePasses(graph: ModuleGraph; a: var TPassContextArray) =
   var m: PNode = nil
   for i in countup(0, gPassesLen - 1):
-    if not isNil(gPasses[i].close): m = gPasses[i].close(a[i], m)
+    if not isNil(gPasses[i].close): m = gPasses[i].close(graph, a[i], m)
     a[i] = nil                # free the memory here
 
 proc processTopLevelStmt(n: PNode, a: var TPassContextArray): bool =
@@ -142,11 +142,11 @@ proc processTopLevelStmtCached(n: PNode, a: var TPassContextArray) =
   for i in countup(0, gPassesLen - 1):
     if not isNil(gPasses[i].openCached): m = gPasses[i].process(a[i], m)
 
-proc closePassesCached(a: var TPassContextArray) =
+proc closePassesCached(graph: ModuleGraph; a: var TPassContextArray) =
   var m: PNode = nil
   for i in countup(0, gPassesLen - 1):
     if not isNil(gPasses[i].openCached) and not isNil(gPasses[i].close):
-      m = gPasses[i].close(a[i], m)
+      m = gPasses[i].close(graph, a[i], m)
     a[i] = nil                # free the memory here
 
 proc resolveMod(module, relativeTo: string): int32 =
@@ -171,6 +171,7 @@ proc processImplicits(implicits: seq[string], nodeKind: TNodeKind,
 
 proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
                     rd: PRodReader; cache: IdentCache): bool {.discardable.} =
+  if graph.stopCompile(): return true
   var
     p: TParsers
     a: TPassContextArray
@@ -198,6 +199,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
         processImplicits implicitIncludes, nkIncludeStmt, a, module
 
       while true:
+        if graph.stopCompile(): break
         var n = parseTopLevelStmt(p)
         if n.kind == nkEmpty: break
         if sfNoForward in module.flags:
@@ -213,12 +215,14 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
         elif not processTopLevelStmt(n, a): break
       closeParsers(p)
       if s.kind != llsStdIn: break
-    closePasses(a)
+    closePasses(graph, a)
     # id synchronization point for more consistent code generation:
     idSynchronizationPoint(1000)
   else:
     openPassesCached(graph, a, module, rd)
     var n = loadInitSection(rd)
-    for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a)
+    for i in countup(0, sonsLen(n) - 1):
-    closePassesCached(a)
+      if graph.stopCompile(): break
+      processTopLevelStmtCached(n.sons[i], a)
+    closePassesCached(graph, a)
   result = true

compiler/patterns.nim

@@ -289,7 +289,7 @@ proc applyRule*(c: PContext, s: PSym, n: PNode): PNode =
         # constraint not fulfilled:
         if not ok: return nil
 
-  markUsed(n.info, s)
+  markUsed(n.info, s, c.graph.usageSym)
   if ctx.subMatch:
     assert m.len == 3
     m.sons[1] = result

compiler/plugins/locals/locals.nim

@@ -28,7 +28,7 @@ proc semLocals(c: PContext, n: PNode): PNode =
           it.typ.skipTypes({tyGenericInst, tyVar}).kind notin
             {tyVarargs, tyOpenArray, tyTypeDesc, tyStatic, tyExpr, tyStmt, tyEmpty}:
 
-        var field = newSym(skField, it.name, getCurrOwner(), n.info)
+        var field = newSym(skField, it.name, getCurrOwner(c), n.info)
         field.typ = it.typ.skipTypes({tyGenericInst, tyVar})
         field.position = counter
         inc(counter)

compiler/pragmas.nim

@@ -11,7 +11,7 @@
 
 import
   os, platform, condsyms, ast, astalgo, idents, semdata, msgs, renderer,
-  wordrecg, ropes, options, strutils, lists, extccomp, math, magicsys, trees,
+  wordrecg, ropes, options, strutils, extccomp, math, magicsys, trees,
   rodread, types, lookups
 
 const
@@ -25,19 +25,19 @@ const
     wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC,
     wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
     wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe,
-    wOverride, wConstructor, wExportNims}
+    wOverride, wConstructor, wExportNims, wUsed}
   converterPragmas* = procPragmas
   methodPragmas* = procPragmas+{wBase}-{wImportCpp}
   templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty,
-    wDelegator, wExportNims}
+    wDelegator, wExportNims, wUsed}
   macroPragmas* = {FirstCallConv..LastCallConv, wImmediate, wImportc, wExportc,
     wNodecl, wMagic, wNosideeffect, wCompilerproc, wDeprecated, wExtern,
     wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wDelegator,
-    wExportNims}
+    wExportNims, wUsed}
   iteratorPragmas* = {FirstCallConv..LastCallConv, wNosideeffect, wSideeffect,
     wImportc, wExportc, wNodecl, wMagic, wDeprecated, wBorrow, wExtern,
     wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wRaises,
-    wTags, wLocks, wGcSafe, wExportNims}
+    wTags, wLocks, wGcSafe, wExportNims, wUsed}
   exprPragmas* = {wLine, wLocks, wNoRewrite, wGcSafe}
   stmtPragmas* = {wChecks, wObjChecks, wFieldChecks, wRangechecks,
     wBoundchecks, wOverflowchecks, wNilchecks, wAssertions, wWarnings, wHints,
@@ -55,16 +55,16 @@ const
     wPure, wHeader, wCompilerproc, wFinal, wSize, wExtern, wShallow,
     wImportCpp, wImportObjC, wError, wIncompleteStruct, wByCopy, wByRef,
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
-    wBorrow, wGcSafe, wExportNims, wPartial}
+    wBorrow, wGcSafe, wExportNims, wPartial, wUsed}
   fieldPragmas* = {wImportc, wExportc, wDeprecated, wExtern,
-    wImportCpp, wImportObjC, wError, wGuard, wBitsize}
+    wImportCpp, wImportObjC, wError, wGuard, wBitsize, wUsed}
   varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl,
     wMagic, wHeader, wDeprecated, wCompilerproc, wDynlib, wExtern,
     wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal,
-    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims}
+    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims, wUsed}
   constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl,
     wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims,
-    wIntDefine, wStrDefine}
+    wIntDefine, wStrDefine, wUsed}
   letPragmas* = varPragmas
   procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNosideeffect,
                       wThread, wRaises, wLocks, wTags, wGcSafe}
@@ -218,20 +218,19 @@ proc processCallConv(c: PContext, n: PNode) =
     var sw = whichKeyword(n.sons[1].ident)
     case sw
     of FirstCallConv..LastCallConv:
-      POptionEntry(c.optionStack.tail).defaultCC = wordToCallConv(sw)
+      c.optionStack[^1].defaultCC = wordToCallConv(sw)
     else: localError(n.info, errCallConvExpected)
   else:
     localError(n.info, errCallConvExpected)
 
 proc getLib(c: PContext, kind: TLibKind, path: PNode): PLib =
-  var it = PLib(c.libs.head)
+  for it in c.libs:
-  while it != nil:
+    if it.kind == kind and trees.exprStructuralEquivalent(it.path, path):
-    if it.kind == kind:
+      return it
-      if trees.exprStructuralEquivalent(it.path, path): return it
+      
-    it = PLib(it.next)
   result = newLib(kind)
   result.path = path
-  append(c.libs, result)
+  c.libs.add result
   if path.kind in {nkStrLit..nkTripleStrLit}:
     result.isOverriden = options.isDynlibOverride(path.strVal)
 
@@ -254,7 +253,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) =
   if (sym == nil) or (sym.kind == skModule):
     let lib = getLib(c, libDynamic, expectDynlibNode(c, n))
     if not lib.isOverriden:
-      POptionEntry(c.optionStack.tail).dynlib = lib
+      c.optionStack[^1].dynlib = lib
   else:
     if n.kind == nkExprColonExpr:
       var lib = getLib(c, libDynamic, expectDynlibNode(c, n))
@@ -350,12 +349,12 @@ proc processPush(c: PContext, n: PNode, start: int) =
   if n.sons[start-1].kind == nkExprColonExpr:
     localError(n.info, errGenerated, "':' after 'push' not supported")
   var x = newOptionEntry()
-  var y = POptionEntry(c.optionStack.tail)
+  var y = c.optionStack[^1]
   x.options = gOptions
   x.defaultCC = y.defaultCC
   x.dynlib = y.dynlib
   x.notes = gNotes
-  append(c.optionStack, x)
+  c.optionStack.add(x)
   for i in countup(start, sonsLen(n) - 1):
     if processOption(c, n.sons[i]):
       # simply store it somewhere:
@@ -365,12 +364,12 @@ proc processPush(c: PContext, n: PNode, start: int) =
     #localError(n.info, errOptionExpected)
 
 proc processPop(c: PContext, n: PNode) =
-  if c.optionStack.counter <= 1:
+  if c.optionStack.len <= 1:
     localError(n.info, errAtPopWithoutPush)
   else:
-    gOptions = POptionEntry(c.optionStack.tail).options
+    gOptions = c.optionStack[^1].options
-    gNotes = POptionEntry(c.optionStack.tail).notes
+    gNotes = c.optionStack[^1].notes
-    remove(c.optionStack, c.optionStack.tail)
+    c.optionStack.setLen(c.optionStack.len - 1)
 
 proc processDefine(c: PContext, n: PNode) =
   if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent):
@@ -582,7 +581,13 @@ proc pragmaLocks(c: PContext, it: PNode): TLockLevel =
   if it.kind != nkExprColonExpr:
     invalidPragma(it)
   else:
-    if it[1].kind != nkNilLit:
+    case it[1].kind
+    of nkStrLit, nkRStrLit, nkTripleStrLit:
+      if it[1].strVal == "unknown":
+        result = UnknownLockLevel
+      else:
+        localError(it[1].info, "invalid string literal for locks pragma (only allowed string is \"unknown\")")
+    else:
       let x = expectIntLit(c, it)
       if x < 0 or x > MaxLockLevel:
         localError(it[1].info, "integer must be within 0.." & $MaxLockLevel)
@@ -961,14 +966,17 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         sym.magic = mIntDefine
       of wStrDefine:
         sym.magic = mStrDefine
+      of wUsed:
+        noVal(it)
+        if sym == nil: invalidPragma(it)
+        else: sym.flags.incl sfUsed
       else: invalidPragma(it)
     else: invalidPragma(it)
 
 proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
                       validPragmas: TSpecialWords) =
   if sym != nil and sym.kind != skModule:
-    var it = POptionEntry(c.optionStack.head)
+    for it in c.optionStack:
-    while it != nil:
       let o = it.otherPragmas
       if not o.isNil:
         pushInfoContext(n.info)
@@ -976,11 +984,10 @@ proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
           if singlePragma(c, sym, o, i, validPragmas):
             internalError(n.info, "implicitPragmas")
         popInfoContext()
-      it = it.next.POptionEntry
 
     if lfExportLib in sym.loc.flags and sfExportc notin sym.flags:
       localError(n.info, errDynlibRequiresExportc)
-    var lib = POptionEntry(c.optionStack.tail).dynlib
+    var lib = c.optionStack[^1].dynlib
     if {lfDynamicLib, lfHeader} * sym.loc.flags == {} and
         sfImportc in sym.flags and lib != nil:
       incl(sym.loc.flags, lfDynamicLib)

compiler/renderer.nim

@@ -10,7 +10,7 @@
 # This module implements the renderer of the standard Nim representation.
 
 import
-  lexer, options, idents, strutils, ast, msgs, lists
+  lexer, options, idents, strutils, ast, msgs
 
 type
   TRenderFlag* = enum

compiler/rodwrite.nim

@@ -641,7 +641,7 @@ proc myOpen(g: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
   rawAddInterfaceSym(w, module)
   result = w
 
-proc myClose(c: PPassContext, n: PNode): PNode =
+proc myClose(graph: ModuleGraph; c: PPassContext, n: PNode): PNode =
   result = process(c, n)
   var w = PRodWriter(c)
   writeRod(w)

compiler/scriptconfig.nim

@@ -12,7 +12,7 @@
 
 import
   ast, modules, idents, passes, passaux, condsyms,
-  options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs,
+  options, nimconf, sem, semdata, llstream, vm, vmdef, commands, msgs,
   os, times, osproc, wordrecg, strtabs, modulegraphs
 
 # we support 'cmpIgnoreStyle' natively for efficiency:
@@ -153,7 +153,7 @@ proc runNimScript*(cache: IdentCache; scriptName: string;
   registerPass(semPass)
   registerPass(evalPass)
 
-  appendStr(searchPaths, options.libpath)
+  searchPaths.add(options.libpath)
 
   var m = graph.makeModule(scriptName)
   incl(m.flags, sfMainModule)

compiler/sem.nim

@@ -10,7 +10,7 @@
 # This module implements the semantic checking pass.
 
 import
-  ast, strutils, hashes, lists, options, lexer, astalgo, trees, treetab,
+  ast, strutils, hashes, options, lexer, astalgo, trees, treetab,
   wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
   magicsys, parser, nversion, nimsets, semfold, importer,
   procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
@@ -166,7 +166,7 @@ proc commonType*(x, y: PType): PType =
         result.addSonSkipIntLit(r)
 
 proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym =
-  result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info)
+  result = newSym(kind, considerQuotedIdent(n), getCurrOwner(c), n.info)
 
 proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
   proc `$`(kind: TSymKind): string = substr(system.`$`(kind), 2).toLowerAscii
@@ -178,12 +178,17 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
     if result.kind != kind:
       localError(n.info, "cannot use symbol of kind '" &
                  $result.kind & "' as a '" & $kind & "'")
+    if sfGenSym in result.flags and result.kind notin {skTemplate, skMacro, skParam}:
+      # declarative context, so produce a fresh gensym:
+      result = copySym(result)
+      result.ast = n.sym.ast
+      put(c.p, n.sym, result)
     # when there is a nested proc inside a template, semtmpl
     # will assign a wrong owner during the first pass over the
     # template; we must fix it here: see #909
-    result.owner = getCurrOwner()
+    result.owner = getCurrOwner(c)
   else:
-    result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info)
+    result = newSym(kind, considerQuotedIdent(n), getCurrOwner(c), n.info)
   #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule:
   #  incl(result.flags, sfGlobal)
 
@@ -371,7 +376,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
                   flags: TExprFlags = {}): PNode =
   pushInfoContext(nOrig.info)
 
-  markUsed(n.info, sym)
+  markUsed(n.info, sym, c.graph.usageSym)
   styleCheckUse(n.info, sym)
   if sym == c.p.owner:
     globalError(n.info, errRecursiveDependencyX, sym.name.s)
@@ -429,7 +434,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
   c.instTypeBoundOp = sigmatch.instTypeBoundOp
 
   pushProcCon(c, module)
-  pushOwner(c.module)
+  pushOwner(c, c.module)
   c.importTable = openScope(c)
   c.importTable.addSym(module) # a module knows itself
   if sfSystemModule in module.flags:
@@ -445,7 +450,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
 
 proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext =
   result = myOpen(graph, module, rd.cache)
-  for m in items(rd.methods): methodDef(m, true)
+  for m in items(rd.methods): methodDef(graph, m, true)
 
 proc isImportSystemStmt(n: PNode): bool =
   if magicsys.systemModule == nil: return false
@@ -497,7 +502,7 @@ proc recoverContext(c: PContext) =
   # faster than wrapping every stack operation in a 'try finally' block and
   # requires far less code.
   c.currentScope = c.topLevelScope
-  while getCurrOwner().kind != skModule: popOwner()
+  while getCurrOwner(c).kind != skModule: popOwner(c)
   while c.p != nil and c.p.owner.kind != skModule: c.p = c.p.next
 
 proc myProcess(context: PPassContext, n: PNode): PNode =
@@ -518,7 +523,7 @@ proc myProcess(context: PPassContext, n: PNode): PNode =
       else: result = ast.emptyNode
       #if gCmd == cmdIdeTools: findSuggest(c, n)
 
-proc myClose(context: PPassContext, n: PNode): PNode =
+proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   var c = PContext(context)
   closeScope(c)         # close module's scope
   rawCloseScope(c)      # imported symbols; don't check for unused ones!
@@ -528,7 +533,7 @@ proc myClose(context: PPassContext, n: PNode): PNode =
   addCodeForGenerics(c, result)
   if c.module.ast != nil:
     result.add(c.module.ast)
-  popOwner()
+  popOwner(c)
   popProcCon(c)
 
 const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose)

compiler/semasgn.nim

@@ -105,7 +105,7 @@ proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
   of attachedDestructor:
     let op = t.destructor
     if op != nil:
-      markUsed(c.info, op)
+      markUsed(c.info, op, c.c.graph.usageSym)
       styleCheckUse(c.info, op)
       body.add newDestructorCall(op, x)
       result = true
@@ -123,14 +123,14 @@ proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
         op = t.assignment
         if op == nil:
           op = liftBody(c.c, t, c.info)
-      markUsed(c.info, op)
+      markUsed(c.info, op, c.c.graph.usageSym)
       styleCheckUse(c.info, op)
       body.add newAsgnCall(c.c, op, x, y)
       result = true
   of attachedDeepCopy:
     let op = t.deepCopy
     if op != nil:
-      markUsed(c.info, op)
+      markUsed(c.info, op, c.c.graph.usageSym)
       styleCheckUse(c.info, op)
       body.add newDeepCopyCall(op, x, y)
       result = true
@@ -248,6 +248,7 @@ proc addParam(procType: PType; param: PSym) =
 proc liftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
   var a: TLiftCtx
   a.info = info
+  a.c = c
   let body = newNodeI(nkStmtList, info)
   result = newSym(skProc, getIdent":lifted=", typ.owner, info)
   a.fn = result

compiler/semcall.nim

@@ -335,7 +335,7 @@ proc inferWithMetatype(c: PContext, formal: PType,
 proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode =
   assert x.state == csMatch
   var finalCallee = x.calleeSym
-  markUsed(n.sons[0].info, finalCallee)
+  markUsed(n.sons[0].info, finalCallee, c.graph.usageSym)
   styleCheckUse(n.sons[0].info, finalCallee)
   assert finalCallee.ast != nil
   if x.hasFauxMatch:
@@ -411,7 +411,7 @@ proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode =
     let tm = typeRel(m, formal, arg, true)
     if tm in {isNone, isConvertible}: return nil
   var newInst = generateInstance(c, s, m.bindings, n.info)
-  markUsed(n.info, s)
+  markUsed(n.info, s, c.graph.usageSym)
   styleCheckUse(n.info, s)
   result = newSymNode(newInst, n.info)
 

compiler/semdata.nim

@@ -10,15 +10,14 @@
 ## This module contains the data structures for the semantic checking phase.
 
 import
-  strutils, lists, intsets, options, lexer, ast, astalgo, trees, treetab,
+  strutils, intsets, options, lexer, ast, astalgo, trees, treetab,
   wordrecg,
   ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math,
   magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef,
   modulegraphs
 
 type
-  TOptionEntry* = object of lists.TListEntry # entries to put on a
+  TOptionEntry* = object      # entries to put on a stack for pragma parsing
-                                             # stack for pragma parsing
     options*: TOptions
     defaultCC*: TCallingConvention
     dynlib*: PLib
@@ -39,6 +38,7 @@ type
     next*: PProcCon           # used for stacking procedure contexts
     wasForwarded*: bool       # whether the current proc has a separate header
     bracketExpr*: PNode       # current bracket expression (for ^ support)
+    mapping*: TIdTable
 
   TInstantiationPair* = object
     genericSym*: PSym
@@ -78,10 +78,10 @@ type
     inGenericInst*: int        # > 0 if we are instantiating a generic
     converters*: TSymSeq       # sequence of converters
     patterns*: TSymSeq         # sequence of pattern matchers
-    optionStack*: TLinkedList
+    optionStack*: seq[POptionEntry]
     symMapping*: TIdTable      # every gensym'ed symbol needs to be mapped
                                # to some new symbol in a generic instantiation
-    libs*: TLinkedList         # all libs used by this module
+    libs*: seq[PLib]           # all libs used by this module
     semConstExpr*: proc (c: PContext, n: PNode): PNode {.nimcall.} # for the pragmas
     semExpr*: proc (c: PContext, n: PNode, flags: TExprFlags = {}): PNode {.nimcall.}
     semTryExpr*: proc (c: PContext, n: PNode,flags: TExprFlags = {}): PNode {.nimcall.}
@@ -124,29 +124,55 @@ proc scopeDepth*(c: PContext): int {.inline.} =
   result = if c.currentScope != nil: c.currentScope.depthLevel
            else: 0
 
-var gOwners*: seq[PSym] = @[]
+proc getCurrOwner*(c: PContext): PSym =
-
-proc getCurrOwner*(): PSym =
   # owner stack (used for initializing the
   # owner field of syms)
   # the documentation comment always gets
   # assigned to the current owner
-  # BUGFIX: global array is needed!
+  result = c.graph.owners[^1]
-  result = gOwners[high(gOwners)]
 
-proc pushOwner*(owner: PSym) =
+proc pushOwner*(c: PContext; owner: PSym) =
-  add(gOwners, owner)
+  add(c.graph.owners, owner)
 
-proc popOwner*() =
+proc popOwner*(c: PContext) =
-  var length = len(gOwners)
+  var length = len(c.graph.owners)
-  if length > 0: setLen(gOwners, length - 1)
+  if length > 0: setLen(c.graph.owners, length - 1)
   else: internalError("popOwner")
 
 proc lastOptionEntry*(c: PContext): POptionEntry =
-  result = POptionEntry(c.optionStack.tail)
+  result = c.optionStack[^1]
 
 proc popProcCon*(c: PContext) {.inline.} = c.p = c.p.next
 
+proc put*(p: PProcCon; key, val: PSym) =
+  if p.mapping.data == nil: initIdTable(p.mapping)
+  #echo "put into table ", key.info
+  p.mapping.idTablePut(key, val)
+
+proc get*(p: PProcCon; key: PSym): PSym =
+  if p.mapping.data == nil: return nil
+  result = PSym(p.mapping.idTableGet(key))
+
+proc getGenSym*(c: PContext; s: PSym): PSym =
+  if sfGenSym notin s.flags: return s
+  var it = c.p
+  while it != nil:
+    result = get(it, s)
+    if result != nil:
+      #echo "got from table ", result.name.s, " ", result.info
+      return result
+    it = it.next
+  result = s
+
+proc considerGenSyms*(c: PContext; n: PNode) =
+  if n.kind == nkSym:
+    let s = getGenSym(c, n.sym)
+    if n.sym != s:
+      n.sym = s
+  else:
+    for i in 0..<n.safeLen:
+      considerGenSyms(c, n.sons[i])
+
 proc newOptionEntry*(): POptionEntry =
   new(result)
   result.options = gOptions
@@ -157,9 +183,9 @@ proc newOptionEntry*(): POptionEntry =
 proc newContext*(graph: ModuleGraph; module: PSym; cache: IdentCache): PContext =
   new(result)
   result.ambiguousSymbols = initIntSet()
-  initLinkedList(result.optionStack)
+  result.optionStack = @[]
-  initLinkedList(result.libs)
+  result.libs = @[]
-  append(result.optionStack, newOptionEntry())
+  result.optionStack.add(newOptionEntry())
   result.module = module
   result.friendModules = @[module]
   result.converters = @[]
@@ -196,7 +222,7 @@ proc addToLib*(lib: PLib, sym: PSym) =
   sym.annex = lib
 
 proc newTypeS*(kind: TTypeKind, c: PContext): PType =
-  result = newType(kind, getCurrOwner())
+  result = newType(kind, getCurrOwner(c))
 
 proc makePtrType*(c: PContext, baseType: PType): PType =
   result = newTypeS(tyPtr, c)
@@ -215,7 +241,7 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType =
 
 proc makeTypeSymNode*(c: PContext, typ: PType, info: TLineInfo): PNode =
   let typedesc = makeTypeDesc(c, typ)
-  let sym = newSym(skType, c.cache.idAnon, getCurrOwner(), info).linkTo(typedesc)
+  let sym = newSym(skType, c.cache.idAnon, getCurrOwner(c), info).linkTo(typedesc)
   return newSymNode(sym, info)
 
 proc makeTypeFromExpr*(c: PContext, n: PNode): PType =
@@ -225,7 +251,7 @@ proc makeTypeFromExpr*(c: PContext, n: PNode): PType =
 
 proc newTypeWithSons*(c: PContext, kind: TTypeKind,
                       sons: seq[PType]): PType =
-  result = newType(kind, getCurrOwner())
+  result = newType(kind, getCurrOwner(c))
   result.sons = sons
 
 proc makeStaticExpr*(c: PContext, n: PNode): PNode =
@@ -296,7 +322,7 @@ proc errorNode*(c: PContext, n: PNode): PNode =
 
 proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) =
   dest.kind = kind
-  dest.owner = getCurrOwner()
+  dest.owner = getCurrOwner(c)
   dest.size = - 1
 
 proc makeRangeType*(c: PContext; first, last: BiggestInt;

compiler/semdestruct.nim

@@ -51,7 +51,7 @@ proc doDestructorStuff(c: PContext, s: PSym, n: PNode) =
       let destructableT = instantiateDestructor(c, t.sons[i])
       if destructableT != nil:
         n.sons[bodyPos].addSon(newNode(nkCall, t.sym.info, @[
-            useSym(destructableT.destructor),
+            useSym(destructableT.destructor, c.graph.usageSym),
             n.sons[paramsPos][1][0]]))
 
 proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode
@@ -60,8 +60,8 @@ proc destroySym(c: PContext, field: PSym, holder: PNode): PNode =
   let destructableT = instantiateDestructor(c, field.typ)
   if destructableT != nil:
     result = newNode(nkCall, field.info, @[
-      useSym(destructableT.destructor),
+      useSym(destructableT.destructor, c.graph.usageSym),
-      newNode(nkDotExpr, field.info, @[holder, useSym(field)])])
+      newNode(nkDotExpr, field.info, @[holder, useSym(field, c.graph.usageSym)])])
 
 proc destroyCase(c: PContext, n: PNode, holder: PNode): PNode =
   var nonTrivialFields = 0
@@ -181,7 +181,8 @@ proc createDestructorCall(c: PContext, s: PSym): PNode =
   let destructableT = instantiateDestructor(c, varTyp)
   if destructableT != nil:
     let call = semStmt(c, newNode(nkCall, s.info, @[
-      useSym(destructableT.destructor), useSym(s)]))
+      useSym(destructableT.destructor, c.graph.usageSym),
+      useSym(s, c.graph.usageSym)]))
     result = newNode(nkDefer, s.info, @[call])
 
 proc insertDestructors(c: PContext,
@@ -233,8 +234,8 @@ proc insertDestructors(c: PContext,
       tryStmt.addSon(
         newNode(nkFinally, info, @[
           semStmt(c, newNode(nkCall, info, @[
-            useSym(destructableT.destructor),
+            useSym(destructableT.destructor, c.graph.usageSym),
-            useSym(varId.sym)]))]))
+            useSym(varId.sym, c.graph.usageSym)]))]))
 
       result.outer = newNodeI(nkStmtList, info)
       varSection.sons.setLen(j+1)

compiler/semexprs.nim

@@ -12,10 +12,10 @@
 
 proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
                      flags: TExprFlags = {}): PNode =
-  markUsed(n.info, s)
+  markUsed(n.info, s, c.graph.usageSym)
   styleCheckUse(n.info, s)
   pushInfoContext(n.info)
-  result = evalTemplate(n, s, getCurrOwner(), efFromHlo in flags)
+  result = evalTemplate(n, s, getCurrOwner(c), efFromHlo in flags)
   if efNoSemCheck notin flags: result = semAfterMacroCall(c, result, s, flags)
   popInfoContext()
 
@@ -209,7 +209,7 @@ proc semConv(c: PContext, n: PNode): PNode =
       let it = op.sons[i]
       let status = checkConvertible(c, result.typ, it.typ)
       if status in {convOK, convNotNeedeed}:
-        markUsed(n.info, it.sym)
+        markUsed(n.info, it.sym, c.graph.usageSym)
         styleCheckUse(n.info, it.sym)
         markIndirect(c, it.sym)
         return it
@@ -922,10 +922,11 @@ proc readTypeParameter(c: PContext, typ: PType,
         return newSymNode(copySym(tParam.sym).linkTo(foundTyp), info)
   #echo "came here: returned nil"
 
-proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
+proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
+  let s = getGenSym(c, sym)
   case s.kind
   of skConst:
-    markUsed(n.info, s)
+    markUsed(n.info, s, c.graph.usageSym)
     styleCheckUse(n.info, s)
     case skipTypes(s.typ, abstractInst-{tyTypeDesc}).kind
     of  tyNil, tyChar, tyInt..tyInt64, tyFloat..tyFloat128,
@@ -949,20 +950,20 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
       result = newSymNode(s, n.info)
   of skMacro:
     if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0:
-      markUsed(n.info, s)
+      markUsed(n.info, s, c.graph.usageSym)
       styleCheckUse(n.info, s)
       result = newSymNode(s, n.info)
     else:
       result = semMacroExpr(c, n, n, s, flags)
   of skTemplate:
     if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0:
-      markUsed(n.info, s)
+      markUsed(n.info, s, c.graph.usageSym)
       styleCheckUse(n.info, s)
       result = newSymNode(s, n.info)
     else:
       result = semTemplateExpr(c, n, s, flags)
   of skParam:
-    markUsed(n.info, s)
+    markUsed(n.info, s, c.graph.usageSym)
     styleCheckUse(n.info, s)
     if s.typ.kind == tyStatic and s.typ.n != nil:
       # XXX see the hack in sigmatch.nim ...
@@ -984,7 +985,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     if s.magic == mNimvm:
       localError(n.info, "illegal context for 'nimvm' magic")
 
-    markUsed(n.info, s)
+    markUsed(n.info, s, c.graph.usageSym)
     styleCheckUse(n.info, s)
     result = newSymNode(s, n.info)
     # We cannot check for access to outer vars for example because it's still
@@ -1002,7 +1003,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
       n.typ = s.typ
       return n
   of skType:
-    markUsed(n.info, s)
+    markUsed(n.info, s, c.graph.usageSym)
     styleCheckUse(n.info, s)
     if s.typ.kind == tyStatic and s.typ.n != nil:
       return s.typ.n
@@ -1024,7 +1025,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
           if f != nil and fieldVisible(c, f):
             # is the access to a public field or in the same module or in a friend?
             doAssert f == s
-            markUsed(n.info, f)
+            markUsed(n.info, f, c.graph.usageSym)
             styleCheckUse(n.info, f)
             result = newNodeIT(nkDotExpr, n.info, f.typ)
             result.add makeDeref(newSymNode(p.selfSym))
@@ -1037,11 +1038,11 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
           if ty.sons[0] == nil: break
           ty = skipTypes(ty.sons[0], skipPtrs)
     # old code, not sure if it's live code:
-    markUsed(n.info, s)
+    markUsed(n.info, s, c.graph.usageSym)
     styleCheckUse(n.info, s)
     result = newSymNode(s, n.info)
   else:
-    markUsed(n.info, s)
+    markUsed(n.info, s, c.graph.usageSym)
     styleCheckUse(n.info, s)
     result = newSymNode(s, n.info)
 
@@ -1061,7 +1062,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
       result = symChoice(c, n, s, scClosed)
       if result.kind == nkSym: result = semSym(c, n, s, flags)
     else:
-      markUsed(n.sons[1].info, s)
+      markUsed(n.sons[1].info, s, c.graph.usageSym)
       result = semSym(c, n, s, flags)
     styleCheckUse(n.sons[1].info, s)
     return
@@ -1086,18 +1087,18 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
         result = newSymNode(f)
         result.info = n.info
         result.typ = ty
-        markUsed(n.info, f)
+        markUsed(n.info, f, c.graph.usageSym)
         styleCheckUse(n.info, f)
         return
     of tyTypeParamsHolders:
       return readTypeParameter(c, ty, i, n.info)
     of tyObject, tyTuple:
       if ty.n != nil and ty.n.kind == nkRecList:
-        for field in ty.n:
+        let field = lookupInRecord(ty.n, i)
-          if field.sym.name == i:
+        if field != nil:
-            n.typ = newTypeWithSons(c, tyFieldAccessor, @[ty, field.sym.typ])
+          n.typ = newTypeWithSons(c, tyFieldAccessor, @[ty, field.typ])
-            n.typ.n = copyTree(n)
+          n.typ.n = copyTree(n)
-            return n
+          return n
     else:
       # echo "TYPE FIELD ACCESS"
       # debug ty
@@ -1119,7 +1120,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
     if f != nil:
       if fieldVisible(c, f):
         # is the access to a public field or in the same module or in a friend?
-        markUsed(n.sons[1].info, f)
+        markUsed(n.sons[1].info, f, c.graph.usageSym)
         styleCheckUse(n.sons[1].info, f)
         n.sons[0] = makeDeref(n.sons[0])
         n.sons[1] = newSymNode(f) # we now have the correct field
@@ -1133,7 +1134,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   elif ty.kind == tyTuple and ty.n != nil:
     f = getSymFromList(ty.n, i)
     if f != nil:
-      markUsed(n.sons[1].info, f)
+      markUsed(n.sons[1].info, f, c.graph.usageSym)
       styleCheckUse(n.sons[1].info, f)
       n.sons[0] = makeDeref(n.sons[0])
       n.sons[1] = newSymNode(f)
@@ -1580,9 +1581,8 @@ proc getMagicSym(magic: TMagic): PSym =
   result = newSym(skProc, getIdent($magic), systemModule, gCodegenLineInfo)
   result.magic = magic
 
-proc newAnonSym(c: PContext; kind: TSymKind, info: TLineInfo,
+proc newAnonSym(c: PContext; kind: TSymKind, info: TLineInfo): PSym =
-                owner = getCurrOwner()): PSym =
+  result = newSym(kind, c.cache.idAnon, getCurrOwner(c), info)
-  result = newSym(kind, c.cache.idAnon, owner, info)
   result.flags = {sfGenSym}
 
 proc semExpandToAst(c: PContext, n: PNode): PNode =
@@ -1591,7 +1591,7 @@ proc semExpandToAst(c: PContext, n: PNode): PNode =
   if expandedSym.kind == skError: return n
 
   macroCall.sons[0] = newSymNode(expandedSym, macroCall.info)
-  markUsed(n.info, expandedSym)
+  markUsed(n.info, expandedSym, c.graph.usageSym)
   styleCheckUse(n.info, expandedSym)
 
   for i in countup(1, macroCall.len-1):
@@ -1682,7 +1682,7 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   # open a scope for temporary symbol inclusions:
   let oldScope = c.currentScope
   openScope(c)
-  let oldOwnerLen = len(gOwners)
+  let oldOwnerLen = len(c.graph.owners)
   let oldGenerics = c.generics
   let oldErrorOutputs = errorOutputs
   errorOutputs = {}
@@ -1708,7 +1708,7 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   c.inGenericInst = oldInGenericInst
   c.p = oldProcCon
   msgs.setInfoContextLen(oldContextLen)
-  setLen(gOwners, oldOwnerLen)
+  setLen(c.graph.owners, oldOwnerLen)
   c.currentScope = oldScope
   errorOutputs = oldErrorOutputs
   msgs.gErrorCounter = oldErrorCount
@@ -2106,8 +2106,8 @@ proc semBlock(c: PContext, n: PNode): PNode =
     var labl = newSymG(skLabel, n.sons[0], c)
     if sfGenSym notin labl.flags:
       addDecl(c, labl)
-      n.sons[0] = newSymNode(labl, n.sons[0].info)
+    n.sons[0] = newSymNode(labl, n.sons[0].info)
-    suggestSym(n.sons[0].info, labl)
+    suggestSym(n.sons[0].info, labl, c.graph.usageSym)
     styleCheckDef(labl)
   n.sons[1] = semExpr(c, n.sons[1])
   n.typ = n.sons[1].typ

compiler/semfields.nim

@@ -108,7 +108,7 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode =
   var trueSymbol = strTableGet(magicsys.systemModule.tab, getIdent"true")
   if trueSymbol == nil:
     localError(n.info, errSystemNeeds, "true")
-    trueSymbol = newSym(skUnknown, getIdent"true", getCurrOwner(), n.info)
+    trueSymbol = newSym(skUnknown, getIdent"true", getCurrOwner(c), n.info)
     trueSymbol.typ = getSysType(tyBool)
 
   result.sons[0] = newSymNode(trueSymbol, n.info)

compiler/semfold.nim

@@ -11,7 +11,7 @@
 # and evaluation phase
 
 import
-  strutils, lists, options, ast, astalgo, trees, treetab, nimsets, times,
+  strutils, options, ast, astalgo, trees, treetab, nimsets, times,
   nversion, platform, math, msgs, os, condsyms, idents, renderer, types,
   commands, magicsys, saturate
 
@@ -628,7 +628,10 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
     of {skProc, skMethod}:
       result = n
     of skType:
-      result = newSymNodeTypeDesc(s, n.info)
+      # XXX gensym'ed symbols can come here and cannot be resolved. This is
+      # dirty, but correct.
+      if s.typ != nil:
+        result = newSymNodeTypeDesc(s, n.info)
     of skGenericParam:
       if s.typ.kind == tyStatic:
         if s.typ.n != nil:
@@ -656,7 +659,7 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
           localError(a.info, errCannotEvalXBecauseIncompletelyDefined,
                      "sizeof")
           result = nil
-        elif skipTypes(a.typ, typedescInst).kind in
+        elif skipTypes(a.typ, typedescInst+{tyRange}).kind in
              IntegralTypes+NilableTypes+{tySet}:
           #{tyArray,tyObject,tyTuple}:
           result = newIntNodeT(getSize(a.typ), n)

compiler/semgnrc.nim

@@ -174,7 +174,11 @@ proc semGenericStmt(c: PContext, n: PNode,
     # XXX for example: ``result.add`` -- ``add`` needs to be looked up here...
     var dummy: bool
     result = fuzzyLookup(c, n, flags, ctx, dummy)
-  of nkEmpty, nkSym..nkNilLit:
+  of nkSym:
+    let a = n.sym
+    let b = getGenSym(c, a)
+    if b != a: n.sym = b
+  of nkEmpty, succ(nkSym)..nkNilLit:
     # see tests/compile/tgensymgeneric.nim:
     # We need to open the gensym'ed symbol again so that the instantiation
     # creates a fresh copy; but this is wrong the very first reason for gensym

compiler/seminst.nim

@@ -61,7 +61,7 @@ iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable): PSym
     if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic}+tyTypeClasses:
       continue
     let symKind = if q.typ.kind == tyStatic: skConst else: skType
-    var s = newSym(symKind, q.name, getCurrOwner(), q.info)
+    var s = newSym(symKind, q.name, getCurrOwner(c), q.info)
     s.flags = s.flags + {sfUsed, sfFromGeneric}
     var t = PType(idTableGet(pt, q.typ))
     if t == nil:
@@ -106,15 +106,18 @@ proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) =
   #if n.kind == nkSym and sfGenSym in n.sym.flags:
   #  if n.sym.owner != orig:
   #    echo "symbol ", n.sym.name.s, " orig ", orig, " owner ", n.sym.owner
-  if n.kind == nkSym and {sfGenSym, sfFromGeneric} * n.sym.flags == {sfGenSym}: # and
+  if n.kind == nkSym and sfGenSym in n.sym.flags: # and
     #  (n.sym.owner == orig or n.sym.owner.kind in {skPackage}):
     let s = n.sym
     var x = PSym(idTableGet(symMap, s))
-    if x == nil:
+    if x != nil:
+      n.sym = x
+    elif s.owner.kind == skPackage:
+      #echo "copied this ", s.name.s
       x = copySym(s, false)
       x.owner = owner
       idTablePut(symMap, s, x)
-    n.sym = x
+      n.sym = x
   else:
     for i in 0 .. <safeLen(n): freshGenSyms(n.sons[i], owner, orig, symMap)
 
@@ -252,7 +255,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   incl(result.flags, sfFromGeneric)
   result.owner = fn
   result.ast = n
-  pushOwner(result)
+  pushOwner(c, result)
 
   openScope(c)
   let gp = n.sons[genericParamsPos]
@@ -301,7 +304,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   popProcCon(c)
   popInfoContext()
   closeScope(c)           # close scope for parameters
-  popOwner()
+  popOwner(c)
   c.currentScope = oldScope
   discard c.friendModules.pop()
   dec(c.instCounter)

compiler/semmagic.nim

@@ -107,7 +107,7 @@ proc semTypeTraits(c: PContext, n: PNode): PNode =
   if t.sonsLen > 0:
     # This is either a type known to sem or a typedesc
     # param to a regular proc (again, known at instantiation)
-    result = evalTypeTrait(n[0], t, getCurrOwner())
+    result = evalTypeTrait(n[0], t, getCurrOwner(c))
   else:
     # a typedesc variable, pass unmodified to evals
     result = n

compiler/semstmts.nim

@@ -30,12 +30,13 @@ proc semBreakOrContinue(c: PContext, n: PNode): PNode =
       of nkIdent: s = lookUp(c, n.sons[0])
       of nkSym: s = n.sons[0].sym
       else: illFormedAst(n)
+      s = getGenSym(c, s)
       if s.kind == skLabel and s.owner.id == c.p.owner.id:
         var x = newSymNode(s)
         x.info = n.info
         incl(s.flags, sfUsed)
         n.sons[0] = x
-        suggestSym(x.info, s)
+        suggestSym(x.info, s, c.graph.usageSym)
         styleCheckUse(x.info, s)
       else:
         localError(n.info, errInvalidControlFlowX, s.name.s)
@@ -366,11 +367,13 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym =
   if isTopLevel(c):
     result = semIdentWithPragma(c, kind, n, {sfExported})
     incl(result.flags, sfGlobal)
+    #if kind in {skVar, skLet}:
+    #  echo "global variable here ", n.info, " ", result.name.s
   else:
     result = semIdentWithPragma(c, kind, n, {})
     if result.owner.kind == skModule:
       incl(result.flags, sfGlobal)
-  suggestSym(n.info, result)
+  suggestSym(n.info, result, c.graph.usageSym)
   styleCheckDef(result)
 
 proc checkNilable(v: PSym) =
@@ -621,7 +624,7 @@ proc semForVars(c: PContext, n: PNode): PNode =
   if iter.kind != tyTuple or length == 3:
     if length == 3:
       var v = symForVar(c, n.sons[0])
-      if getCurrOwner().kind == skModule: incl(v.flags, sfGlobal)
+      if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal)
       # BUGFIX: don't use `iter` here as that would strip away
       # the ``tyGenericInst``! See ``tests/compile/tgeneric.nim``
       # for an example:
@@ -635,7 +638,7 @@ proc semForVars(c: PContext, n: PNode): PNode =
   else:
     for i in countup(0, length - 3):
       var v = symForVar(c, n.sons[i])
-      if getCurrOwner().kind == skModule: incl(v.flags, sfGlobal)
+      if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal)
       v.typ = iter.sons[i]
       n.sons[i] = newSymNode(v)
       if sfGenSym notin v.flags and not isDiscardUnderscore(v):
@@ -745,7 +748,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
       # We have a generic type declaration here. In generic types,
       # symbol lookup needs to be done here.
       openScope(c)
-      pushOwner(s)
+      pushOwner(c, s)
       if s.magic == mNone: s.typ.kind = tyGenericBody
       # XXX for generic type aliases this is not correct! We need the
       # underlying Id really:
@@ -769,11 +772,11 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
         body.sym = s
         body.size = -1 # could not be computed properly
         s.typ.sons[sonsLen(s.typ) - 1] = body
-      popOwner()
+      popOwner(c)
       closeScope(c)
     elif a.sons[2].kind != nkEmpty:
       # process the type's body:
-      pushOwner(s)
+      pushOwner(c, s)
       var t = semTypeNode(c, a.sons[2], s.typ)
       if s.typ == nil:
         s.typ = t
@@ -782,7 +785,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
         assignType(s.typ, t)
         #debug s.typ
       s.ast = a
-      popOwner()
+      popOwner(c)
     let aa = a.sons[2]
     if aa.kind in {nkRefTy, nkPtrTy} and aa.len == 1 and
        aa.sons[0].kind == nkObjectTy:
@@ -793,7 +796,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
       internalAssert st.lastSon.sym == nil
       incl st.flags, tfRefsAnonObj
       let obj = newSym(skType, getIdent(s.name.s & ":ObjectType"),
-                              getCurrOwner(), s.info)
+                              getCurrOwner(c), s.info)
       obj.typ = st.lastSon
       st.lastSon.sym = obj
 
@@ -927,7 +930,7 @@ proc semBorrow(c: PContext, n: PNode, s: PSym) =
 
 proc addResult(c: PContext, t: PType, info: TLineInfo, owner: TSymKind) =
   if t != nil:
-    var s = newSym(skResult, getIdent"result", getCurrOwner(), info)
+    var s = newSym(skResult, getIdent"result", getCurrOwner(c), info)
     s.typ = t
     incl(s.flags, sfUsed)
     addParamOrResult(c, s, owner)
@@ -1002,12 +1005,12 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
   checkSonsLen(n, bodyPos + 1)
   var s: PSym
   if n[namePos].kind != nkSym:
-    s = newSym(skProc, c.cache.idAnon, getCurrOwner(), n.info)
+    s = newSym(skProc, c.cache.idAnon, getCurrOwner(c), n.info)
     s.ast = n
     n.sons[namePos] = newSymNode(s)
   else:
     s = n[namePos].sym
-  pushOwner(s)
+  pushOwner(c, s)
   openScope(c)
   var gp: PNode
   if n.sons[genericParamsPos].kind != nkEmpty:
@@ -1047,7 +1050,7 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
   else:
     localError(n.info, errImplOfXexpected, s.name.s)
   closeScope(c)           # close scope for parameters
-  popOwner()
+  popOwner(c)
   result.typ = s.typ
 
 proc semDo(c: PContext, n: PNode, flags: TExprFlags): PNode =
@@ -1081,7 +1084,7 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
                  params[i].sym.name.s)
     #params[i].sym.owner = s
   openScope(c)
-  pushOwner(s)
+  pushOwner(c, s)
   addParams(c, params, skProc)
   pushProcCon(c, s)
   addResult(c, n.typ.sons[0], n.info, skProc)
@@ -1089,7 +1092,7 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
   let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
   n.sons[bodyPos] = transformBody(c.module, semBody, s)
   popProcCon(c)
-  popOwner()
+  popOwner(c)
   closeScope(c)
 
   # alternative variant (not quite working):
@@ -1177,11 +1180,58 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
       localError(n.info, errGenerated,
                  "'destroy' or 'deepCopy' expected for 'override'")
 
+proc cursorInProcAux(n: PNode): bool =
+  if inCheckpoint(n.info) != cpNone: return true
+  for i in 0..<n.safeLen:
+    if cursorInProcAux(n[i]): return true
+
+proc cursorInProc(n: PNode): bool =
+  if n.info.fileIndex == gTrackPos.fileIndex:
+    result = cursorInProcAux(n)
+
 type
   TProcCompilationSteps = enum
     stepRegisterSymbol,
     stepDetermineType,
 
+import compilerlog
+
+proc hasObjParam(s: PSym): bool =
+  var t = s.typ
+  for col in countup(1, sonsLen(t)-1):
+    if skipTypes(t.sons[col], skipPtrs).kind == tyObject:
+      return true
+
+proc finishMethod(c: PContext, s: PSym) =
+  if hasObjParam(s):
+    methodDef(c.graph, s, false)
+
+proc semMethodPrototype(c: PContext; s: PSym; n: PNode) =
+  if isGenericRoutine(s):
+    let tt = s.typ
+    var foundObj = false
+    # we start at 1 for now so that tparsecombnum continues to compile.
+    # XXX Revisit this problem later.
+    for col in countup(1, sonsLen(tt)-1):
+      let t = tt.sons[col]
+      if t != nil and t.kind == tyGenericInvocation:
+        var x = skipTypes(t.sons[0], {tyVar, tyPtr, tyRef, tyGenericInst,
+                                      tyGenericInvocation, tyGenericBody,
+                                      tyAlias})
+        if x.kind == tyObject and t.len-1 == n.sons[genericParamsPos].len:
+          foundObj = true
+          x.methods.safeAdd((col,s))
+    if not foundObj:
+      message(n.info, warnDeprecated, "generic method not attachable to object type")
+  else:
+    # why check for the body? bug #2400 has none. Checking for sfForward makes
+    # no sense either.
+    # and result.sons[bodyPos].kind != nkEmpty:
+    if hasObjParam(s):
+      methodDef(c.graph, s, fromCache=false)
+    else:
+      localError(n.info, errXNeedsParamObjectType, "method")
+
 proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
                 validPragmas: TSpecialWords,
                 phase = stepRegisterSymbol): PNode =
@@ -1196,7 +1246,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     assert phase == stepRegisterSymbol
 
     if n[namePos].kind == nkEmpty:
-      s = newSym(kind, c.cache.idAnon, getCurrOwner(), n.info)
+      s = newSym(kind, c.cache.idAnon, getCurrOwner(c), n.info)
       incl(s.flags, sfUsed)
       isAnon = true
     else:
@@ -1213,7 +1263,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         return
   else:
     s = n[namePos].sym
-    s.owner = getCurrOwner()
+    s.owner = getCurrOwner(c)
     typeIsDetermined = s.typ == nil
     s.ast = n
     #s.scope = c.currentScope
@@ -1222,7 +1272,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   # where the proc was declared
   let oldScope = c.currentScope
   #c.currentScope = s.scope
-  pushOwner(s)
+  pushOwner(c, s)
   openScope(c)
   var gp: PNode
   if n.sons[genericParamsPos].kind != nkEmpty:
@@ -1288,8 +1338,8 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     if importantComments() and not isNil(proto.ast.comment):
       n.comment = proto.ast.comment
     proto.ast = n             # needed for code generation
-    popOwner()
+    popOwner(c)
-    pushOwner(s)
+    pushOwner(c, s)
   s.options = gOptions
   if sfOverriden in s.flags or s.name.s[0] == '=': semOverride(c, s, n)
   if s.name.s[0] in {'.', '('}:
@@ -1303,30 +1353,40 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     # Macros and Templates can have generic parameters, but they are
     # only used for overload resolution (there is no instantiation of
     # the symbol, so we must process the body now)
-    pushProcCon(c, s)
+    if not usePseudoGenerics and gIdeCmd in {ideSug, ideCon} and not
-    if n.sons[genericParamsPos].kind == nkEmpty or usePseudoGenerics:
+        cursorInProc(n.sons[bodyPos]):
-      if not usePseudoGenerics: paramsTypeCheck(c, s.typ)
+      discard "speed up nimsuggest"
-
+      if s.kind == skMethod: semMethodPrototype(c, s, n)
-      c.p.wasForwarded = proto != nil
-      maybeAddResult(c, s, n)
-      if lfDynamicLib notin s.loc.flags:
-        # no semantic checking for importc:
-        let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
-        # unfortunately we cannot skip this step when in 'system.compiles'
-        # context as it may even be evaluated in 'system.compiles':
-        n.sons[bodyPos] = transformBody(c.module, semBody, s)
     else:
-      if s.typ.sons[0] != nil and kind != skIterator:
+      pushProcCon(c, s)
-        addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info))
+      if n.sons[genericParamsPos].kind == nkEmpty or usePseudoGenerics:
-      openScope(c)
+        if not usePseudoGenerics: paramsTypeCheck(c, s.typ)
-      n.sons[bodyPos] = semGenericStmt(c, n.sons[bodyPos])
+
-      closeScope(c)
+        c.p.wasForwarded = proto != nil
-      fixupInstantiatedSymbols(c, s)
+        maybeAddResult(c, s, n)
-    if sfImportc in s.flags:
+        if s.kind == skMethod: semMethodPrototype(c, s, n)
-      # so we just ignore the body after semantic checking for importc:
+
-      n.sons[bodyPos] = ast.emptyNode
+        if lfDynamicLib notin s.loc.flags:
-    popProcCon(c)
+          # no semantic checking for importc:
+          let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
+          # unfortunately we cannot skip this step when in 'system.compiles'
+          # context as it may even be evaluated in 'system.compiles':
+          n.sons[bodyPos] = transformBody(c.module, semBody, s)
+      else:
+        if s.typ.sons[0] != nil and kind != skIterator:
+          addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info))
+
+        openScope(c)
+        n.sons[bodyPos] = semGenericStmt(c, n.sons[bodyPos])
+        closeScope(c)
+        fixupInstantiatedSymbols(c, s)
+        if s.kind == skMethod: semMethodPrototype(c, s, n)
+      if sfImportc in s.flags:
+        # so we just ignore the body after semantic checking for importc:
+        n.sons[bodyPos] = ast.emptyNode
+      popProcCon(c)
   else:
+    if s.kind == skMethod: semMethodPrototype(c, s, n)
     if proto != nil: localError(n.info, errImplOfXexpected, proto.name.s)
     if {sfImportc, sfBorrow} * s.flags == {} and s.magic == mNone:
       incl(s.flags, sfForward)
@@ -1334,7 +1394,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   sideEffectsCheck(c, s)
   closeScope(c)           # close scope for parameters
   # c.currentScope = oldScope
-  popOwner()
+  popOwner(c)
   if n.sons[patternPos].kind != nkEmpty:
     c.patterns.add(s)
   if isAnon: result.typ = s.typ
@@ -1345,6 +1405,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
 proc determineType(c: PContext, s: PSym) =
   if s.typ != nil: return
   #if s.magic != mNone: return
+  #if s.ast.isNil: return
   discard semProcAux(c, s.ast, s.kind, {}, stepDetermineType)
 
 proc semIterator(c: PContext, n: PNode): PNode =
@@ -1352,7 +1413,7 @@ proc semIterator(c: PContext, n: PNode): PNode =
   let isAnon = n[namePos].kind == nkEmpty
   if n[namePos].kind == nkSym:
     # gensym'ed iterators might need to become closure iterators:
-    n[namePos].sym.owner = getCurrOwner()
+    n[namePos].sym.owner = getCurrOwner(c)
     n[namePos].sym.kind = skIterator
   result = semProcAux(c, n, skIterator, iteratorPragmas)
   var s = result.sons[namePos].sym
@@ -1379,46 +1440,22 @@ proc semIterator(c: PContext, n: PNode): PNode =
 proc semProc(c: PContext, n: PNode): PNode =
   result = semProcAux(c, n, skProc, procPragmas)
 
-proc hasObjParam(s: PSym): bool =
-  var t = s.typ
-  for col in countup(1, sonsLen(t)-1):
-    if skipTypes(t.sons[col], skipPtrs).kind == tyObject:
-      return true
-
-proc finishMethod(c: PContext, s: PSym) =
-  if hasObjParam(s):
-    methodDef(s, false)
-
 proc semMethod(c: PContext, n: PNode): PNode =
   if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "method")
   result = semProcAux(c, n, skMethod, methodPragmas)
-  # macros can transform methods to nothing:
+  # macros can transform converters to nothing:
   if namePos >= result.safeLen: return result
   var s = result.sons[namePos].sym
-  if isGenericRoutine(s):
+  # we need to fix the 'auto' return type for the dispatcher here (see tautonotgeneric
-    let tt = s.typ
+  # test case):
-    var foundObj = false
+  let disp = getDispatcher(s)
-    # we start at 1 for now so that tparsecombnum continues to compile.
+  # auto return type?
-    # XXX Revisit this problem later.
+  if disp != nil and disp.typ.sons[0] != nil and disp.typ.sons[0].kind == tyExpr:
-    for col in countup(1, sonsLen(tt)-1):
+    let ret = s.typ.sons[0]
-      let t = tt.sons[col]
+    disp.typ.sons[0] = ret
-      if t != nil and t.kind == tyGenericInvocation:
+    if disp.ast[resultPos].kind == nkSym:
-        var x = skipTypes(t.sons[0], {tyVar, tyPtr, tyRef, tyGenericInst,
+      if isEmptyType(ret): disp.ast.sons[resultPos] = emptyNode
-                                      tyGenericInvocation, tyGenericBody,
+      else: disp.ast[resultPos].sym.typ = ret
-                                      tyAlias})
-        if x.kind == tyObject and t.len-1 == result.sons[genericParamsPos].len:
-          foundObj = true
-          x.methods.safeAdd((col,s))
-    if not foundObj:
-      message(n.info, warnDeprecated, "generic method not attachable to object type")
-  else:
-    # why check for the body? bug #2400 has none. Checking for sfForward makes
-    # no sense either.
-    # and result.sons[bodyPos].kind != nkEmpty:
-    if hasObjParam(s):
-      methodDef(s, fromCache=false)
-    else:
-      localError(n.info, errXNeedsParamObjectType, "method")
 
 proc semConverterDef(c: PContext, n: PNode): PNode =
   if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "converter")

compiler/semtempl.nim

@@ -60,7 +60,7 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode =
     # (s.kind notin routineKinds or s.magic != mNone):
     # for instance 'nextTry' is both in tables.nim and astalgo.nim ...
     result = newSymNode(s, n.info)
-    markUsed(n.info, s)
+    markUsed(n.info, s, c.graph.usageSym)
   else:
     # semantic checking requires a type; ``fitNode`` deals with it
     # appropriately
@@ -172,7 +172,7 @@ proc newGenSym(kind: TSymKind, n: PNode, c: var TemplCtx): PSym =
   result = newSym(kind, considerQuotedIdent(n), c.owner, n.info)
   incl(result.flags, sfGenSym)
   incl(result.flags, sfShadowed)
-  if c.scopeN == 0: incl(result.flags, sfFromGeneric)
+  #if c.scopeN == 0: incl(result.flags, sfFromGeneric)
 
 proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
   # locals default to 'gensym':
@@ -384,11 +384,13 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
     checkSonsLen(n, 2)
     openScope(c)
     if n.sons[0].kind != nkEmpty:
-      # labels are always 'gensym'ed:
+      addLocalDecl(c, n.sons[0], skLabel)
-      let s = newGenSym(skLabel, n.sons[0], c)
+      when false:
-      addPrelimDecl(c.c, s)
+        # labels are always 'gensym'ed:
-      styleCheckDef(s)
+        let s = newGenSym(skLabel, n.sons[0], c)
-      n.sons[0] = newSymNode(s, n.sons[0].info)
+        addPrelimDecl(c.c, s)
+        styleCheckDef(s)
+        n.sons[0] = newSymNode(s, n.sons[0].info)
     n.sons[1] = semTemplBody(c, n.sons[1])
     closeScope(c)
   of nkTryStmt:
@@ -559,7 +561,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
   styleCheckDef(s)
   # check parameter list:
   #s.scope = c.currentScope
-  pushOwner(s)
+  pushOwner(c, s)
   openScope(c)
   n.sons[namePos] = newSymNode(s, n.sons[namePos].info)
   if n.sons[pragmasPos].kind != nkEmpty:
@@ -613,7 +615,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
   # only parameters are resolved, no type checking is performed
   semIdeForTemplateOrGeneric(c, n.sons[bodyPos], ctx.cursorInBody)
   closeScope(c)
-  popOwner()
+  popOwner(c)
   s.ast = n
   result = n
   if n.sons[bodyPos].kind == nkEmpty:
@@ -758,7 +760,7 @@ proc semPattern(c: PContext, n: PNode): PNode =
   ctx.toMixin = initIntSet()
   ctx.toInject = initIntSet()
   ctx.c = c
-  ctx.owner = getCurrOwner()
+  ctx.owner = getCurrOwner(c)
   result = flattenStmts(semPatternBody(ctx, n))
   if result.kind in {nkStmtList, nkStmtListExpr}:
     if result.len == 1:

compiler/semtypes.nim

@@ -297,7 +297,7 @@ proc semOrdinal(c: PContext, n: PNode, prev: PType): PType =
 
 proc semTypeIdent(c: PContext, n: PNode): PSym =
   if n.kind == nkSym:
-    result = n.sym
+    result = getGenSym(c, n.sym)
   else:
     when defined(nimfix):
       result = pickSym(c, n, skType)
@@ -306,7 +306,7 @@ proc semTypeIdent(c: PContext, n: PNode): PSym =
     else:
       result = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared})
     if result != nil:
-      markUsed(n.info, result)
+      markUsed(n.info, result, c.graph.usageSym)
       styleCheckUse(n.info, result)
       if result.kind == skParam and result.typ.kind == tyTypeDesc:
         # This is a typedesc param. is it already bound?
@@ -608,7 +608,7 @@ proc semRecordNodeAux(c: PContext, n: PNode, check: var IntSet, pos: var int,
     let rec = rectype.sym
     for i in countup(0, sonsLen(n)-3):
       var f = semIdentWithPragma(c, skField, n.sons[i], {sfExported})
-      suggestSym(n.sons[i].info, f)
+      suggestSym(n.sons[i].info, f, c.graph.usageSym)
       f.typ = typ
       f.position = pos
       if (rec != nil) and ({sfImportc, sfExportc} * rec.flags != {}) and
@@ -750,7 +750,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
         return genericParams.sons[i].typ
 
     let owner = if typeClass.sym != nil: typeClass.sym
-                else: getCurrOwner()
+                else: getCurrOwner(c)
     var s = newSym(skType, finalTypId, owner, info)
     if typId == nil: s.flags.incl(sfAnon)
     s.linkTo(typeClass)
@@ -850,7 +850,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
 
   of tyGenericInst:
     if paramType.lastSon.kind == tyUserTypeClass:
-      var cp = copyType(paramType, getCurrOwner(), false)
+      var cp = copyType(paramType, getCurrOwner(c), false)
       cp.kind = tyUserTypeClassInst
       return addImplicitGeneric(cp)
 
@@ -876,10 +876,10 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
       result = liftingWalk(expanded, true)
 
   of tyUserTypeClass, tyBuiltInTypeClass, tyAnd, tyOr, tyNot:
-    result = addImplicitGeneric(copyType(paramType, getCurrOwner(), true))
+    result = addImplicitGeneric(copyType(paramType, getCurrOwner(c), true))
 
   of tyGenericParam:
-    markUsed(info, paramType.sym)
+    markUsed(info, paramType.sym, c.graph.usageSym)
     styleCheckUse(info, paramType.sym)
     if tfWildcard in paramType.flags:
       paramType.flags.excl tfWildcard
@@ -1267,7 +1267,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     of mExpr:
       result = semTypeNode(c, n.sons[0], nil)
       if result != nil:
-        result = copyType(result, getCurrOwner(), false)
+        result = copyType(result, getCurrOwner(c), false)
         for i in countup(1, n.len - 1):
           result.rawAddSon(semTypeNode(c, n.sons[i], nil))
     of mDistinct:
@@ -1319,8 +1319,9 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
           prev.id = s.typ.id
         result = prev
   of nkSym:
-    if n.sym.kind == skType and n.sym.typ != nil:
+    let s = getGenSym(c, n.sym)
-      var t = n.sym.typ
+    if s.kind == skType and s.typ != nil:
+      var t = s.typ
       let alias = maybeAliasType(c, t, prev)
       if alias != nil:
         result = alias
@@ -1329,10 +1330,10 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       else:
         assignType(prev, t)
         result = prev
-      markUsed(n.info, n.sym)
+      markUsed(n.info, n.sym, c.graph.usageSym)
       styleCheckUse(n.info, n.sym)
     else:
-      if n.sym.kind != skError: localError(n.info, errTypeExpected)
+      if s.kind != skError: localError(n.info, errTypeExpected)
       result = newOrPrevType(tyError, prev, c)
   of nkObjectTy: result = semObjectNode(c, n, prev)
   of nkTupleTy: result = semTuple(c, n, prev)

compiler/sigmatch.nim

@@ -73,7 +73,7 @@ type
 const
   isNilConversion = isConvertible # maybe 'isIntConv' fits better?
 
-proc markUsed*(info: TLineInfo, s: PSym)
+proc markUsed*(info: TLineInfo, s: PSym; usageSym: var PSym)
 
 template hasFauxMatch*(c: TCandidate): bool = c.fauxMatch != tyNone
 
@@ -543,7 +543,7 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
       else:
         return isNone
     when useEffectSystem:
-      if not compatibleEffects(f, a): return isNone
+      if compatibleEffects(f, a) != efCompat: return isNone
 
   of tyNil:
     result = f.allowsNil
@@ -1291,7 +1291,7 @@ proc userConvMatch(c: PContext, m: var TCandidate, f, a: PType,
       dest = generateTypeInstance(c, m.bindings, arg, dest)
     let fdest = typeRel(m, f, dest)
     if fdest in {isEqual, isGeneric}:
-      markUsed(arg.info, c.converters[i])
+      markUsed(arg.info, c.converters[i], c.graph.usageSym)
       var s = newSymNode(c.converters[i])
       s.typ = c.converters[i].typ
       s.info = arg.info
@@ -1551,7 +1551,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
       else: result = nil
     else:
       # only one valid interpretation found:
-      markUsed(arg.info, arg.sons[best].sym)
+      markUsed(arg.info, arg.sons[best].sym, m.c.graph.usageSym)
       styleCheckUse(arg.info, arg.sons[best].sym)
       result = paramTypesMatchAux(m, f, arg.sons[best].typ, arg.sons[best],
                                   argOrig)
@@ -1583,12 +1583,14 @@ proc prepareOperand(c: PContext; formal: PType; a: PNode): PNode =
     result = c.semOperand(c, a, flags)
   else:
     result = a
+    considerGenSyms(c, result)
 
 proc prepareOperand(c: PContext; a: PNode): PNode =
   if a.typ.isNil:
     result = c.semOperand(c, a, {efDetermineType})
   else:
     result = a
+    considerGenSyms(c, result)
 
 proc prepareNamedParam(a: PNode) =
   if a.sons[0].kind != nkIdent:

compiler/suggest.nim

@@ -28,7 +28,7 @@ type
     column*: int                 # Starts at 0
     doc*: string           # Not escaped (yet)
     symkind*: TSymKind
-    forth*: string               # XXX TODO object on symkind
+    forth*: string               # type
     quality*: range[0..100]   # matching quality
     isGlobal*: bool # is a global variable
     tokenLen*: int
@@ -88,7 +88,7 @@ proc symToSuggest(s: PSym, isLocal: bool, section: string, li: TLineInfo;
     when not defined(noDocgen):
       result.doc = s.extractDocComment
 
-proc `$`(suggest: Suggest): string =
+proc `$`*(suggest: Suggest): string =
   result = $suggest.section
   result.add(sep)
   if suggest.section == ideHighlight:
@@ -131,7 +131,7 @@ proc suggestResult(s: Suggest) =
   if not isNil(suggestionResultHook):
     suggestionResultHook(s)
   else:
-    suggestWriteln($(s))
+    suggestWriteln($s)
 
 proc filterSym(s: PSym): bool {.inline.} =
   result = s.kind != skModule
@@ -297,10 +297,10 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
       suggestOperations(c, n, typ, outputs)
 
 type
-  TCheckPointResult = enum
+  TCheckPointResult* = enum
     cpNone, cpFuzzy, cpExact
 
-proc inCheckpoint(current: TLineInfo): TCheckPointResult =
+proc inCheckpoint*(current: TLineInfo): TCheckPointResult =
   if current.fileIndex == gTrackPos.fileIndex:
     if current.line == gTrackPos.line and
         abs(current.col-gTrackPos.col) < 4:
@@ -353,10 +353,10 @@ when defined(nimsuggest):
     s.allUsages.add(info)
 
 var
-  usageSym*: PSym
+  #usageSym*: PSym
   lastLineInfo*: TLineInfo
 
-proc findUsages(info: TLineInfo; s: PSym) =
+proc findUsages(info: TLineInfo; s: PSym; usageSym: var PSym) =
   if suggestVersion < 2:
     if usageSym == nil and isTracked(info, s.name.s.len):
       usageSym = s
@@ -385,7 +385,7 @@ proc ensureIdx[T](x: var T, y: int) =
 proc ensureSeq[T](x: var seq[T]) =
   if x == nil: newSeq(x, 0)
 
-proc suggestSym*(info: TLineInfo; s: PSym; isDecl=true) {.inline.} =
+proc suggestSym*(info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.inline.} =
   ## misnamed: should be 'symDeclared'
   when defined(nimsuggest):
     if suggestVersion == 2:
@@ -395,20 +395,20 @@ proc suggestSym*(info: TLineInfo; s: PSym; isDecl=true) {.inline.} =
         s.addNoDup(info)
 
     if gIdeCmd == ideUse:
-      findUsages(info, s)
+      findUsages(info, s, usageSym)
     elif gIdeCmd == ideDef:
       findDefinition(info, s)
     elif gIdeCmd == ideDus and s != nil:
       if isTracked(info, s.name.s.len):
         suggestResult(symToSuggest(s, isLocal=false, $ideDef, 100))
-      findUsages(info, s)
+      findUsages(info, s, usageSym)
     elif gIdeCmd == ideHighlight and info.fileIndex == gTrackPos.fileIndex:
       suggestResult(symToSuggest(s, isLocal=false, $ideHighlight, info, 100))
     elif gIdeCmd == ideOutline and info.fileIndex == gTrackPos.fileIndex and
         isDecl:
       suggestResult(symToSuggest(s, isLocal=false, $ideOutline, info, 100))
 
-proc markUsed(info: TLineInfo; s: PSym) =
+proc markUsed(info: TLineInfo; s: PSym; usageSym: var PSym) =
   incl(s.flags, sfUsed)
   if s.kind == skEnumField and s.owner != nil:
     incl(s.owner.flags, sfUsed)
@@ -416,11 +416,11 @@ proc markUsed(info: TLineInfo; s: PSym) =
     if sfDeprecated in s.flags: message(info, warnDeprecated, s.name.s)
     if sfError in s.flags: localError(info, errWrongSymbolX, s.name.s)
   when defined(nimsuggest):
-    suggestSym(info, s, false)
+    suggestSym(info, s, usageSym, false)
 
-proc useSym*(sym: PSym): PNode =
+proc useSym*(sym: PSym; usageSym: var PSym): PNode =
   result = newSymNode(sym)
-  markUsed(result.info, sym)
+  markUsed(result.info, sym, usageSym)
 
 proc safeSemExpr*(c: PContext, n: PNode): PNode =
   # use only for idetools support!

compiler/transf.nim

@@ -19,7 +19,7 @@
 # * transforms 'defer' into a 'try finally' statement
 
 import
-  intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os,
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
   idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread,
   lambdalifting, sempass2, lowerings, lookups
 
@@ -694,9 +694,10 @@ proc transformCall(c: PTransf, n: PNode): PTransNode =
     # bugfix: check after 'transformSons' if it's still a method call:
     # use the dispatcher for the call:
     if s.sons[0].kind == nkSym and s.sons[0].sym.kind == skMethod:
-      let t = lastSon(s.sons[0].sym.ast)
+      when false:
-      if t.kind != nkSym or sfDispatcher notin t.sym.flags:
+        let t = lastSon(s.sons[0].sym.ast)
-        methodDef(s.sons[0].sym, false)
+        if t.kind != nkSym or sfDispatcher notin t.sym.flags:
+          methodDef(s.sons[0].sym, false)
       result = methodCall(s).PTransNode
     else:
       result = s.PTransNode
@@ -869,7 +870,12 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     else:
       result = transformSons(c, n)
   of nkIdentDefs, nkConstDef:
-    result = transformSons(c, n)
+    when true:
+      result = transformSons(c, n)
+    else:
+      result = n.PTransNode
+      let L = n.len-1
+      result[L] = transform(c, n.sons[L])
     # XXX comment handling really sucks:
     if importantComments():
       PNode(result).comment = n.comment

compiler/trees.nim

@@ -38,7 +38,7 @@ proc exprStructuralEquivalent*(a, b: PNode; strictSymEquality=false): bool =
         # don't go nuts here: same symbol as string is enough:
         result = a.sym.name.id == b.sym.name.id
     of nkIdent: result = a.ident.id == b.ident.id
-    of nkCharLit..nkInt64Lit: result = a.intVal == b.intVal
+    of nkCharLit..nkUInt64Lit: result = a.intVal == b.intVal
     of nkFloatLit..nkFloat64Lit: result = a.floatVal == b.floatVal
     of nkStrLit..nkTripleStrLit: result = a.strVal == b.strVal
     of nkEmpty, nkNilLit, nkType: result = true
@@ -62,7 +62,7 @@ proc sameTree*(a, b: PNode): bool =
       # don't go nuts here: same symbol as string is enough:
       result = a.sym.name.id == b.sym.name.id
     of nkIdent: result = a.ident.id == b.ident.id
-    of nkCharLit..nkInt64Lit: result = a.intVal == b.intVal
+    of nkCharLit..nkUInt64Lit: result = a.intVal == b.intVal
     of nkFloatLit..nkFloat64Lit: result = a.floatVal == b.floatVal
     of nkStrLit..nkTripleStrLit: result = a.strVal == b.strVal
     of nkEmpty, nkNilLit, nkType: result = true

compiler/types.nim

@@ -1377,7 +1377,16 @@ proc compatibleEffectsAux(se, re: PNode): bool =
       return false
   result = true
 
-proc compatibleEffects*(formal, actual: PType): bool =
+type
+  EffectsCompat* = enum
+    efCompat
+    efRaisesDiffer
+    efRaisesUnknown
+    efTagsDiffer
+    efTagsUnknown
+    efLockLevelsDiffer
+
+proc compatibleEffects*(formal, actual: PType): EffectsCompat =
   # for proc type compatibility checking:
   assert formal.kind == tyProc and actual.kind == tyProc
   internalAssert formal.n.sons[0].kind == nkEffectList
@@ -1393,18 +1402,21 @@ proc compatibleEffects*(formal, actual: PType): bool =
     # 'r.msgHandler = if isNil(msgHandler): defaultMsgHandler else: msgHandler'
     if not isNil(se) and se.kind != nkArgList:
       # spec requires some exception or tag, but we don't know anything:
-      if real.len == 0: return false
+      if real.len == 0: return efRaisesUnknown
-      result = compatibleEffectsAux(se, real.sons[exceptionEffects])
+      let res = compatibleEffectsAux(se, real.sons[exceptionEffects])
-      if not result: return
+      if not res: return efRaisesDiffer
 
     let st = spec.sons[tagEffects]
     if not isNil(st) and st.kind != nkArgList:
       # spec requires some exception or tag, but we don't know anything:
-      if real.len == 0: return false
+      if real.len == 0: return efTagsUnknown
-      result = compatibleEffectsAux(st, real.sons[tagEffects])
+      let res = compatibleEffectsAux(st, real.sons[tagEffects])
-      if not result: return
+      if not res: return efTagsDiffer
-  result = formal.lockLevel.ord < 0 or
+  if formal.lockLevel.ord < 0 or
-      actual.lockLevel.ord <= formal.lockLevel.ord
+      actual.lockLevel.ord <= formal.lockLevel.ord:
+    result = efCompat
+  else:
+    result = efLockLevelsDiffer
 
 proc isCompileTimeOnly*(t: PType): bool {.inline.} =
   result = t.kind in {tyTypeDesc, tyStatic}
@@ -1508,6 +1520,21 @@ proc typeMismatch*(n: PNode, formal, actual: PType) =
     let named = typeToString(formal)
     let desc = typeToString(formal, preferDesc)
     let x = if named == desc: named else: named & " = " & desc
-    localError(n.info, errGenerated, msgKindToString(errTypeMismatch) &
+    var msg = msgKindToString(errTypeMismatch) &
-        typeToString(actual) & ") " &
+              typeToString(actual) & ") " &
-        `%`(msgKindToString(errButExpectedX), [x]))
+              msgKindToString(errButExpectedX) % [x]
+
+    if formal.kind == tyProc and actual.kind == tyProc:
+      case compatibleEffects(formal, actual)
+      of efCompat: discard
+      of efRaisesDiffer:
+        msg.add "\n.raise effects differ"
+      of efRaisesUnknown:
+        msg.add "\n.raise effect is 'can raise any'"
+      of efTagsDiffer:
+        msg.add "\n.tag effects differ"
+      of efTagsUnknown:
+        msg.add "\n.tag effect is 'any tag allowed'"
+      of efLockLevelsDiffer:
+        msg.add "\nlock levels differ"
+    localError(n.info, errGenerated, msg)

compiler/vm.nim

@@ -1556,7 +1556,10 @@ proc myProcess(c: PPassContext, n: PNode): PNode =
     result = n
   oldErrorCount = msgs.gErrorCounter
 
-const evalPass* = makePass(myOpen, nil, myProcess, myProcess)
+proc myClose(graph: ModuleGraph; c: PPassContext, n: PNode): PNode =
+  myProcess(c, n)
+
+const evalPass* = makePass(myOpen, nil, myProcess, myClose)
 
 proc evalConstExprAux(module: PSym; cache: IdentCache; prc: PSym, n: PNode,
                       mode: TEvalMode): PNode =

compiler/wordrecg.nim

@@ -45,7 +45,7 @@ type
     wImportc, wExportc, wExportNims, wIncompleteStruct, wRequiresInit,
     wAlign, wNodecl, wPure, wSideeffect, wHeader,
     wNosideeffect, wGcSafe, wNoreturn, wMerge, wLib, wDynlib,
-    wCompilerproc, wProcVar, wBase,
+    wCompilerproc, wProcVar, wBase, wUsed,
     wFatal, wError, wWarning, wHint, wLine, wPush, wPop, wDefine, wUndef,
     wLinedir, wStacktrace, wLinetrace, wLink, wCompile,
     wLinksys, wDeprecated, wVarargs, wCallconv, wBreakpoint, wDebugger,
@@ -131,7 +131,7 @@ const
     "incompletestruct",
     "requiresinit", "align", "nodecl", "pure", "sideeffect",
     "header", "nosideeffect", "gcsafe", "noreturn", "merge", "lib", "dynlib",
-    "compilerproc", "procvar", "base",
+    "compilerproc", "procvar", "base", "used",
     "fatal", "error", "warning", "hint", "line",
     "push", "pop", "define", "undef", "linedir", "stacktrace", "linetrace",
     "link", "compile", "linksys", "deprecated", "varargs",

doc/backends.txt

@@ -461,3 +461,11 @@ can then attach a GC to this thread via
 It is **not** safe to disable the garbage collector and enable it after the
 call from your background thread even if the code you are calling is short
 lived.
+
+Before the thread exits, you should tear down the thread's GC to prevent memory
+leaks by calling
+
+.. code-block:: nim
+
+  system.tearDownForeignThreadGc()
+

doc/manual/locking.txt

@@ -198,3 +198,22 @@ This is essential so that procs can be called within a ``locks`` section:
 As usual ``locks`` is an inferred effect and there is a subtype
 relation: ``proc () {.locks: N.}`` is a subtype of ``proc () {.locks: M.}``
 iff (M <= N).
+
+The ``locks`` pragma can also take the special value ``"unknown"``. This
+is useful in the context of dynamic method dispatching. In the following
+example, the compiler can infer a lock level of 0 for the ``base`` case.
+However, one of the overloaded methods calls a procvar which is
+potentially locking. Thus, the lock level of calling ``g.testMethod``
+cannot be inferred statically, leading to compiler warnings. By using
+``{.locks: "unknown".}``, the base method can be marked explicitly as
+having unknown lock level as well:
+
+.. code-block:: nim
+  type SomeBase* = ref object of RootObj
+  type SomeDerived* = ref object of SomeBase
+    memberProc*: proc ()
+
+  method testMethod(g: SomeBase) {.base, locks: "unknown".} = discard
+  method testMethod(g: SomeDerived) =
+    if g.memberProc != nil:
+      g.memberProc()

doc/manual/pragmas.txt

@@ -503,6 +503,26 @@ identifier that can be used to enable or disable it:
 This is often better than disabling all warnings at once.
 
 
+used pragma
+-----------
+
+Nim produces a warning for symbols that are not exported and not used either.
+The ``used`` pragma can be attached to a symbol to suppress this warning. This
+is particularly useful when the symbol was generated by a macro:
+
+.. code-block:: nim
+  template implementArithOps(T) =
+    proc echoAdd(a, b: T) {.used.} =
+      echo a + b
+    proc echoSub(a, b: T) {.used.} =
+      echo a - b
+
+  # no warning produced for the unused 'echoSub'
+  implementArithOps(int)
+  echoAdd 3, 5
+
+
+
 experimental pragma
 -------------------
 
@@ -1018,12 +1038,12 @@ the -d/--define option at compile time.
 The implementation currently provides the following possible options (various
 others may be added later).
 
-===============  ============================================
+=================  ============================================
-pragma           description
+pragma             description
-===============  ============================================
+=================  ============================================
-intdefine        Reads in a build-time define as an integer
+`intdefine`:idx:   Reads in a build-time define as an integer
-strdefine        Reads in a build-time define as a string
+`strdefine`:idx:   Reads in a build-time define as a string
-===============  ============================================
+=================  ============================================
 
 .. code-block:: nim
    const FooBar {.intdefine.}: int = 5

doc/tut1.rst

@@ -361,7 +361,7 @@ iterator:
 .. code-block:: nim
   echo "Counting to ten: "
   for i in countup(1, 10):
-    echo $i
+    echo i
   # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines
 
 The built-in `$ <system.html#$>`_ operator turns an integer (``int``) and many
@@ -374,7 +374,7 @@ Each value is ``echo``-ed. This code does the same:
   echo "Counting to 10: "
   var i = 1
   while i <= 10:
-    echo $i
+    echo i
     inc(i) # increment i by 1
   # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines
 
@@ -383,7 +383,7 @@ Counting down can be achieved as easily (but is less often needed):
 .. code-block:: nim
   echo "Counting down from 10 to 1: "
   for i in countdown(10, 1):
-    echo $i
+    echo i
   # --> Outputs 10 9 8 7 6 5 4 3 2 1 on different lines
 
 Since counting up occurs so often in programs, Nim also has a `..
@@ -827,7 +827,7 @@ Let's return to the boring counting example:
 .. code-block:: nim
   echo "Counting to ten: "
   for i in countup(1, 10):
-    echo $i
+    echo i
 
 Can a `countup <system.html#countup>`_ proc be written that supports this
 loop? Lets try:
@@ -1035,15 +1035,15 @@ there is a difference between the ``$`` and ``repr`` outputs:
     myString = "nim"
     myInteger = 42
     myFloat = 3.14
-  echo $myBool, ":", repr(myBool)
+  echo myBool, ":", repr(myBool)
   # --> true:true
-  echo $myCharacter, ":", repr(myCharacter)
+  echo myCharacter, ":", repr(myCharacter)
   # --> n:'n'
-  echo $myString, ":", repr(myString)
+  echo myString, ":", repr(myString)
   # --> nim:0x10fa8c050"nim"
-  echo $myInteger, ":", repr(myInteger)
+  echo myInteger, ":", repr(myInteger)
   # --> 42:42
-  echo $myFloat, ":", repr(myFloat)
+  echo myFloat, ":", repr(myFloat)
   # --> 3.1400000000000001e+00:3.1400000000000001e+00
 
 
@@ -1075,7 +1075,7 @@ at runtime by 0, the second by 1 and so on. Example:
       north, east, south, west
 
   var x = south      # `x` is of type `Direction`; its value is `south`
-  echo $x           # writes "south" to `stdout`
+  echo x           # writes "south" to `stdout`
 
 All comparison operators can be used with enumeration types.
 
@@ -1289,7 +1289,7 @@ value. Here the ``for`` statement is looping over the results from the
 
 .. code-block:: nim
   for i in @[3, 4, 5]:
-    echo $i
+    echo i
   # --> 3
   # --> 4
   # --> 5

koch.nim

@@ -223,8 +223,9 @@ proc bundleWinTools() =
   copyExe("tools/finish".exe, "finish".exe)
   removeFile("tools/finish".exe)
   nimexec("c -o:bin/vccexe.exe tools/vccenv/vccexe")
-  nimexec(r"c --cc:vcc --app:gui -o:bin\downloader.exe -d:ssl --noNimblePath " &
+  when false:
-          r"--path:..\ui tools\downloader.nim")
+    nimexec(r"c --cc:vcc --app:gui -o:bin\downloader.exe -d:ssl --noNimblePath " &
+            r"--path:..\ui tools\downloader.nim")
 
 proc zip(args: string) =
   bundleNimbleSrc()

lib/core/macros.nim

@@ -742,6 +742,8 @@ proc `$`*(node: NimNode): string {.compileTime.} =
     result = $node.symbol
   of nnkOpenSymChoice, nnkClosedSymChoice:
     result = $node[0]
+  of nnkAccQuoted:
+    result = $node[0]
   else:
     badNodeKind node.kind, "$"
 

lib/impure/db_postgres.nim

@@ -69,16 +69,14 @@ import db_common
 export db_common
 
 type
-  DbConn* = PPGconn   ## encapsulates a database connection
+  DbConn* = PPGconn    ## encapsulates a database connection
-  Row* = seq[string]  ## a row of a dataset. NULL database values will be
+  Row* = seq[string]   ## a row of a dataset. NULL database values will be
-                      ## converted to nil.
+                       ## converted to nil.
-  InstantRow* = tuple[res: PPGresult, line: int32]  ## a handle that can be
+  InstantRow* = object ## a handle that can be
-                                                    ## used to get a row's
+    res: PPGresult     ## used to get a row's
-                                                    ## column text on demand
+    line: int          ## column text on demand             
   SqlPrepared* = distinct string ## a identifier for the prepared queries
-
+{.deprecated: [TRow: Row, TDbConn: DbConn, TSqlPrepared: SqlPrepared].}
-{.deprecated: [TRow: Row, TDbConn: DbConn,
-              TSqlPrepared: SqlPrepared].}
 
 proc dbError*(db: DbConn) {.noreturn.} =
   ## raises a DbError exception.
@@ -213,7 +211,7 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
   ## on demand using []. Returned handle is valid only within iterator body.
   var res = setupQuery(db, query, args)
   for i in 0..pqNtuples(res)-1:
-    yield (res: res, line: i)
+    yield InstantRow(res: res, line: i)
   pqClear(res)
 
 iterator instantRows*(db: DbConn, stmtName: SqlPrepared,
@@ -223,16 +221,170 @@ iterator instantRows*(db: DbConn, stmtName: SqlPrepared,
   ## on demand using []. Returned handle is valid only within iterator body.
   var res = setupQuery(db, stmtName, args)
   for i in 0..pqNtuples(res)-1:
-    yield (res: res, line: i)
+    yield InstantRow(res: res, line: i)
   pqClear(res)
 
-proc `[]`*(row: InstantRow, col: int32): string {.inline.} =
+proc getColumnType(res: PPGresult, col: int) : DbType =
-  ## returns text for given column of the row
+  ## returns DbType for given column in the row
-  $pqgetvalue(row.res, row.line, col)
+  ## defined in pg_type.h file in the postgres source code
+  ## Wire representation for types: http://www.npgsql.org/dev/types.html
+  var oid = pqftype(res, int32(col))
+  ## The integer returned is the internal OID number of the type
+  case oid
+  of 16: return DbType(kind: DbTypeKind.dbBool, name: "bool")
+  of 17: return DbType(kind: DbTypeKind.dbBlob, name: "bytea")
 
-proc len*(row: InstantRow): int32 {.inline.} =
+  of 21:   return DbType(kind: DbTypeKind.dbInt, name: "int2", size: 2)
+  of 23:   return DbType(kind: DbTypeKind.dbInt, name: "int4", size: 4)
+  of 20:   return DbType(kind: DbTypeKind.dbInt, name: "int8", size: 8)
+  of 1560: return DbType(kind: DbTypeKind.dbBit, name: "bit")  
+  of 1562: return DbType(kind: DbTypeKind.dbInt, name: "varbit")
+
+  of 18:   return DbType(kind: DbTypeKind.dbFixedChar, name: "char")
+  of 19:   return DbType(kind: DbTypeKind.dbFixedChar, name: "name")
+  of 1042: return DbType(kind: DbTypeKind.dbFixedChar, name: "bpchar")
+
+  of 25:   return DbType(kind: DbTypeKind.dbVarchar, name: "text")
+  of 1043: return DbType(kind: DbTypeKind.dbVarChar, name: "varchar")
+  of 2275: return DbType(kind: DbTypeKind.dbVarchar, name: "cstring")
+
+  of 700: return DbType(kind: DbTypeKind.dbFloat, name: "float4")
+  of 701: return DbType(kind: DbTypeKind.dbFloat, name: "float8")
+
+  of 790:  return DbType(kind: DbTypeKind.dbDecimal, name: "money")  
+  of 1700: return DbType(kind: DbTypeKind.dbDecimal, name: "numeric")
+
+  of 704:  return DbType(kind: DbTypeKind.dbTimeInterval, name: "tinterval")
+  of 702:  return DbType(kind: DbTypeKind.dbTimestamp, name: "abstime")
+  of 703:  return DbType(kind: DbTypeKind.dbTimeInterval, name: "reltime")
+  of 1082: return DbType(kind: DbTypeKind.dbDate, name: "date")
+  of 1083: return DbType(kind: DbTypeKind.dbTime, name: "time")
+  of 1114: return DbType(kind: DbTypeKind.dbTimestamp, name: "timestamp")
+  of 1184: return DbType(kind: DbTypeKind.dbTimestamp, name: "timestamptz")
+  of 1186: return DbType(kind: DbTypeKind.dbTimeInterval, name: "interval")
+  of 1266: return DbType(kind: DbTypeKind.dbTime, name: "timetz")
+
+  of 114:  return DbType(kind: DbTypeKind.dbJson, name: "json")
+  of 142:  return DbType(kind: DbTypeKind.dbXml, name: "xml")
+  of 3802: return DbType(kind: DbTypeKind.dbJson, name: "jsonb")
+
+  of 600: return DbType(kind: DbTypeKind.dbPoint, name: "point")
+  of 601: return DbType(kind: DbTypeKind.dbLseg, name: "lseg")
+  of 602: return DbType(kind: DbTypeKind.dbPath, name: "path")
+  of 603: return DbType(kind: DbTypeKind.dbBox, name: "box")
+  of 604: return DbType(kind: DbTypeKind.dbPolygon, name: "polygon")
+  of 628: return DbType(kind: DbTypeKind.dbLine, name: "line")
+  of 718: return DbType(kind: DbTypeKind.dbCircle, name: "circle")  
+
+  of 650: return DbType(kind: DbTypeKind.dbInet, name: "cidr")
+  of 829: return DbType(kind: DbTypeKind.dbMacAddress, name: "macaddr")
+  of 869: return DbType(kind: DbTypeKind.dbInet, name: "inet")
+  
+  of 2950: return DbType(kind: DbTypeKind.dbVarchar, name: "uuid")
+  of 3614: return DbType(kind: DbTypeKind.dbVarchar, name: "tsvector")
+  of 3615: return DbType(kind: DbTypeKind.dbVarchar, name: "tsquery")
+  of 2970: return DbType(kind: DbTypeKind.dbVarchar, name: "txid_snapshot")
+
+  of 27:   return DbType(kind: DbTypeKind.dbComposite, name: "tid")
+  of 1790: return DbType(kind: DbTypeKind.dbComposite, name: "refcursor")
+  of 2249: return DbType(kind: DbTypeKind.dbComposite, name: "record")
+  of 3904: return DbType(kind: DbTypeKind.dbComposite, name: "int4range")
+  of 3906: return DbType(kind: DbTypeKind.dbComposite, name: "numrange")
+  of 3908: return DbType(kind: DbTypeKind.dbComposite, name: "tsrange")
+  of 3910: return DbType(kind: DbTypeKind.dbComposite, name: "tstzrange")
+  of 3912: return DbType(kind: DbTypeKind.dbComposite, name: "daterange")
+  of 3926: return DbType(kind: DbTypeKind.dbComposite, name: "int8range")
+
+  of 22:   return DbType(kind: DbTypeKind.dbArray, name: "int2vector")
+  of 30:   return DbType(kind: DbTypeKind.dbArray, name: "oidvector")
+  of 143:  return DbType(kind: DbTypeKind.dbArray, name: "xml[]")
+  of 199:  return DbType(kind: DbTypeKind.dbArray, name: "json[]")
+  of 629:  return DbType(kind: DbTypeKind.dbArray, name: "line[]")
+  of 651:  return DbType(kind: DbTypeKind.dbArray, name: "cidr[]")
+  of 719:  return DbType(kind: DbTypeKind.dbArray, name: "circle[]")
+  of 791:  return DbType(kind: DbTypeKind.dbArray, name: "money[]")
+  of 1000: return DbType(kind: DbTypeKind.dbArray, name: "bool[]")
+  of 1001: return DbType(kind: DbTypeKind.dbArray, name: "bytea[]")
+  of 1002: return DbType(kind: DbTypeKind.dbArray, name: "char[]")
+  of 1003: return DbType(kind: DbTypeKind.dbArray, name: "name[]")
+  of 1005: return DbType(kind: DbTypeKind.dbArray, name: "int2[]")
+  of 1006: return DbType(kind: DbTypeKind.dbArray, name: "int2vector[]")
+  of 1007: return DbType(kind: DbTypeKind.dbArray, name: "int4[]")
+  of 1008: return DbType(kind: DbTypeKind.dbArray, name: "regproc[]")
+  of 1009: return DbType(kind: DbTypeKind.dbArray, name: "text[]")
+  of 1028: return DbType(kind: DbTypeKind.dbArray, name: "oid[]")
+  of 1010: return DbType(kind: DbTypeKind.dbArray, name: "tid[]")
+  of 1011: return DbType(kind: DbTypeKind.dbArray, name: "xid[]")
+  of 1012: return DbType(kind: DbTypeKind.dbArray, name: "cid[]")
+  of 1013: return DbType(kind: DbTypeKind.dbArray, name: "oidvector[]")
+  of 1014: return DbType(kind: DbTypeKind.dbArray, name: "bpchar[]")
+  of 1015: return DbType(kind: DbTypeKind.dbArray, name: "varchar[]")
+  of 1016: return DbType(kind: DbTypeKind.dbArray, name: "int8[]")
+  of 1017: return DbType(kind: DbTypeKind.dbArray, name: "point[]")
+  of 1018: return DbType(kind: DbTypeKind.dbArray, name: "lseg[]")
+  of 1019: return DbType(kind: DbTypeKind.dbArray, name: "path[]")
+  of 1020: return DbType(kind: DbTypeKind.dbArray, name: "box[]")
+  of 1021: return DbType(kind: DbTypeKind.dbArray, name: "float4[]")
+  of 1022: return DbType(kind: DbTypeKind.dbArray, name: "float8[]")
+  of 1023: return DbType(kind: DbTypeKind.dbArray, name: "abstime[]")
+  of 1024: return DbType(kind: DbTypeKind.dbArray, name: "reltime[]")
+  of 1025: return DbType(kind: DbTypeKind.dbArray, name: "tinterval[]")
+  of 1027: return DbType(kind: DbTypeKind.dbArray, name: "polygon[]")
+  of 1040: return DbType(kind: DbTypeKind.dbArray, name: "macaddr[]")
+  of 1041: return DbType(kind: DbTypeKind.dbArray, name: "inet[]")
+  of 1263: return DbType(kind: DbTypeKind.dbArray, name: "cstring[]")
+  of 1115: return DbType(kind: DbTypeKind.dbArray, name: "timestamp[]")
+  of 1182: return DbType(kind: DbTypeKind.dbArray, name: "date[]")
+  of 1183: return DbType(kind: DbTypeKind.dbArray, name: "time[]")
+  of 1185: return DbType(kind: DbTypeKind.dbArray, name: "timestamptz[]")
+  of 1187: return DbType(kind: DbTypeKind.dbArray, name: "interval[]")
+  of 1231: return DbType(kind: DbTypeKind.dbArray, name: "numeric[]")
+  of 1270: return DbType(kind: DbTypeKind.dbArray, name: "timetz[]")
+  of 1561: return DbType(kind: DbTypeKind.dbArray, name: "bit[]")
+  of 1563: return DbType(kind: DbTypeKind.dbArray, name: "varbit[]")
+  of 2201: return DbType(kind: DbTypeKind.dbArray, name: "refcursor[]")
+  of 2951: return DbType(kind: DbTypeKind.dbArray, name: "uuid[]")
+  of 3643: return DbType(kind: DbTypeKind.dbArray, name: "tsvector[]")
+  of 3645: return DbType(kind: DbTypeKind.dbArray, name: "tsquery[]")
+  of 3807: return DbType(kind: DbTypeKind.dbArray, name: "jsonb[]")
+  of 2949: return DbType(kind: DbTypeKind.dbArray, name: "txid_snapshot[]")
+  of 3905: return DbType(kind: DbTypeKind.dbArray, name: "int4range[]")
+  of 3907: return DbType(kind: DbTypeKind.dbArray, name: "numrange[]")
+  of 3909: return DbType(kind: DbTypeKind.dbArray, name: "tsrange[]")
+  of 3911: return DbType(kind: DbTypeKind.dbArray, name: "tstzrange[]")
+  of 3913: return DbType(kind: DbTypeKind.dbArray, name: "daterange[]")
+  of 3927: return DbType(kind: DbTypeKind.dbArray, name: "int8range[]")
+  of 2287: return DbType(kind: DbTypeKind.dbArray, name: "record[]")
+
+  of 705:  return DbType(kind: DbTypeKind.dbUnknown, name: "unknown")
+  else: return DbType(kind: DbTypeKind.dbUnknown, name: $oid) ## Query the system table pg_type to determine exactly which type is referenced.
+
+proc setColumnInfo(columns: var DbColumns; res: PPGresult; L: int32) =
+  setLen(columns, L)
+  for i in 0..<L:
+    columns[i].name = $pqfname(res, i)
+    columns[i].typ = getColumnType(res, i)
+    columns[i].tableName = $(pqftable(res, i)) ## Returns the OID of the table from which the given column was fetched.
+                                               ## Query the system table pg_class to determine exactly which table is referenced.  
+    #columns[i].primaryKey = libpq does not have a function for that
+    #columns[i].foreignKey = libpq does not have a function for that
+
+iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery;
+                      args: varargs[string, `$`]): InstantRow
+                      {.tags: [ReadDbEffect].} =
+  var res = setupQuery(db, query, args)
+  setColumnInfo(columns, res, pqnfields(res))
+  for i in 0..<pqntuples(res):
+    yield InstantRow(res: res, line: i)
+  pqClear(res)
+
+proc `[]`*(row: InstantRow; col: int): string {.inline.} =
+  ## returns text for given column of the row
+  $pqgetvalue(row.res, int32(row.line), int32(col))
+
+proc len*(row: InstantRow): int {.inline.} =
   ## returns number of columns in the row
-  pqNfields(row.res)
+  int(pqNfields(row.res))
 
 proc getRow*(db: DbConn, query: SqlQuery,
              args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =

lib/impure/re.nim

@@ -290,7 +290,7 @@ proc find*(buf: cstring, pattern: Regex, matches: var openArray[string],
   for i in 1..int(res)-1:
     var a = rawMatches[i * 2]
     var b = rawMatches[i * 2 + 1]
-    if a >= 0'i32: matches[i-1] = bufSubstr(buf, int(a), int(b)-1)
+    if a >= 0'i32: matches[i-1] = bufSubstr(buf, int(a), int(b))
     else: matches[i-1] = nil
   return rawMatches[0]
 

lib/pure/algorithm.nim

@@ -46,7 +46,7 @@ proc reverse*[T](a: var openArray[T], first, last: Natural) =
 
 proc reverse*[T](a: var openArray[T]) =
   ## reverses the array `a`.
-  reverse(a, 0, a.high)
+  reverse(a, 0, max(0, a.high))
 
 proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] =
   ## returns the reverse of the array `a[first..last]`.

lib/pure/asynchttpserver.nim

@@ -31,7 +31,7 @@
 ##
 ##    waitFor server.serve(Port(8080), cb)
 
-import tables, asyncnet, asyncdispatch, parseutils, uri, strutils
+import tables, asyncnet, asyncdispatch, parseutils, uri, strutils, nativesockets
 import httpcore
 
 export httpcore except parseHeader
@@ -241,7 +241,7 @@ proc serve*(server: AsyncHttpServer, port: Port,
   ## specified address and port.
   ##
   ## When a request is made by a client the specified callback will be called.
-  server.socket = newAsyncSocket()
+  server.socket = newAsyncSocket(AF_INET6)
   if server.reuseAddr:
     server.socket.setSockOpt(OptReuseAddr, true)
   if server.reusePort:

lib/pure/collections/heapqueue.nim

@@ -77,8 +77,10 @@ proc pop*[T](heap: var HeapQueue[T]): T =
 proc del*[T](heap: var HeapQueue[T], index: int) =
   ## Removes element at `index`, maintaining the heap invariant.
   swap(seq[T](heap)[^1], seq[T](heap)[index])
-  seq[T](heap).setLen(heap.len - 1)
+  let newLen = heap.len - 1
-  heap.siftup(index)
+  seq[T](heap).setLen(newLen)
+  if index < newLen:
+    heap.siftup(index)
 
 proc replace*[T](heap: var HeapQueue[T], item: T): T =
   ## Pop and return the current smallest value, and add the new item.
@@ -101,16 +103,19 @@ proc pushpop*[T](heap: var HeapQueue[T], item: T): T =
   return item
 
 when isMainModule:
+  proc toSortedSeq[T](h: HeapQueue[T]): seq[T] =
+    var tmp = h
+    result = @[]
+    while tmp.len > 0:
+      result.add(pop(tmp))
+
   block: # Simple sanity test
     var heap = newHeapQueue[int]()
     let data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
     for item in data:
       push(heap, item)
     doAssert(heap[0] == 0)
-    var sort = newSeq[int]()
+    doAssert(heap.toSortedSeq == @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
-    while heap.len > 0:
-      sort.add(pop(heap))
-    doAssert(sort == @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
 
   block: # Test del
     var heap = newHeapQueue[int]()
@@ -121,11 +126,27 @@ when isMainModule:
     doAssert(heap[0] == 1)
 
     heap.del(seq[int](heap).find(7))
-    heap.del(seq[int](heap).find(5))
+    doAssert(heap.toSortedSeq == @[1, 2, 3, 4, 5, 6, 8, 9])
-    heap.del(seq[int](heap).find(6))
-    heap.del(seq[int](heap).find(2))
 
-    var sort = newSeq[int]()
+    heap.del(seq[int](heap).find(5))
-    while heap.len > 0:
+    doAssert(heap.toSortedSeq == @[1, 2, 3, 4, 6, 8, 9])
-      sort.add(pop(heap))
+
-    doAssert(sort == @[1, 3, 4, 8, 9])
+    heap.del(seq[int](heap).find(6))
+    doAssert(heap.toSortedSeq == @[1, 2, 3, 4, 8, 9])
+
+    heap.del(seq[int](heap).find(2))
+    doAssert(heap.toSortedSeq == @[1, 3, 4, 8, 9])
+
+  block: # Test del last
+    var heap = newHeapQueue[int]()
+    let data = [1, 2, 3]
+    for item in data: push(heap, item)
+
+    heap.del(2)
+    doAssert(heap.toSortedSeq == @[1, 2])
+
+    heap.del(1)
+    doAssert(heap.toSortedSeq == @[1])
+
+    heap.del(0)
+    doAssert(heap.toSortedSeq == @[])

lib/pure/distros.nim

@@ -225,7 +225,7 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) =
     else:
       result = ("<your package manager here> install " & p, true)
   else:
-    result = ("brew install " & p, true)
+    result = ("brew install " & p, false)
 
 proc foreignDep*(foreignPackageName: string) =
   ## Registers 'foreignPackageName' to the internal list of foreign deps.

lib/pure/hashes.nim

@@ -127,6 +127,15 @@ proc hash*(x: string): Hash =
     h = h !& ord(x[i])
   result = !$h
 
+proc hash*(x: cstring): Hash =
+  ## efficient hashing of null-terminated strings
+  var h: Hash = 0
+  var i = 0
+  while x[i] != 0.char:
+    h = h !& ord(x[i])
+    inc i
+  result = !$h
+
 proc hash*(sBuf: string, sPos, ePos: int): Hash =
   ## efficient hashing of a string buffer, from starting
   ## position `sPos` to ending position `ePos`
@@ -239,6 +248,7 @@ proc hash*[A](x: set[A]): Hash =
 
 when isMainModule:
   doAssert( hash("aa bb aaaa1234") == hash("aa bb aaaa1234", 0, 13) )
+  doAssert( hash("aa bb aaaa1234") == hash(cstring("aa bb aaaa1234")) )
   doAssert( hashIgnoreCase("aa bb aaaa1234") == hash("aa bb aaaa1234") )
   doAssert( hashIgnoreStyle("aa bb aaaa1234") == hashIgnoreCase("aa bb aaaa1234") )
   let xx = @['H','e','l','l','o']

lib/pure/httpcore.nim

@@ -206,6 +206,8 @@ proc parseHeader*(line: string): tuple[key: string, value: seq[string]] =
   inc(i) # skip :
   if i < len(line):
     i += parseList(line, result.value, i)
+  elif result.key.len > 0:
+    result.value = @[""]
   else:
     result.value = @[]
 
@@ -318,4 +320,6 @@ when isMainModule:
   let (key, value) = parseHeader("foobar: ")
   test = newHttpHeaders()
   test[key] = value
-  doAssert test["foobar"] == ""
+  doAssert test["foobar"] == ""
+
+  doAssert parseHeader("foobar:") == ("foobar", @[""])

lib/pure/json.nim

@@ -952,7 +952,7 @@ proc newIndent(curr, indent: int, ml: bool): int =
   else: return indent
 
 proc nl(s: var string, ml: bool) =
-  if ml: s.add("\n")
+  s.add(if ml: "\n" else: " ")
 
 proc escapeJson*(s: string; result: var string) =
   ## Converts a string `s` to its JSON representation.
@@ -986,15 +986,14 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
               lstArr = false, currIndent = 0) =
   case node.kind
   of JObject:
-    if currIndent != 0 and not lstArr: result.nl(ml)
+    if lstArr: result.indent(currIndent) # Indentation
-    result.indent(currIndent) # Indentation
     if node.fields.len > 0:
       result.add("{")
       result.nl(ml) # New line
       var i = 0
       for key, val in pairs(node.fields):
         if i > 0:
-          result.add(", ")
+          result.add(",")
           result.nl(ml) # New Line
         inc i
         # Need to indent more than {
@@ -1030,7 +1029,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
       result.nl(ml)
       for i in 0..len(node.elems)-1:
         if i > 0:
-          result.add(", ")
+          result.add(",")
           result.nl(ml) # New Line
         toPretty(result, node.elems[i], indent, ml,
             true, newIndent(currIndent, indent, ml))

lib/pure/logging.nim

@@ -46,6 +46,8 @@
 ##
 ## **Warning:** The global list of handlers is a thread var, this means that
 ## the handlers must be re-added in each thread.
+## **Warning:** When logging on disk or console, only error and fatal messages
+## are flushed out immediately. Use flushFile() where needed.
 
 import strutils, times
 when not defined(js):

lib/pure/os.nim

@@ -1002,8 +1002,8 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path:
 
 iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string {.
   tags: [ReadDirEffect].} =
-  ## walks over the directory `dir` and yields for each file in `dir`. The
+  ## Recursively walks over the directory `dir` and yields for each file in `dir`.
-  ## full path for each file is returned.
+  ## The full path for each file is returned. Directories are not returned.
   ## **Warning**:
   ## Modifying the directory structure while the iterator
   ## is traversing may result in undefined behavior!
@@ -1185,7 +1185,7 @@ proc createHardlink*(src, dest: string) =
 
 proc parseCmdLine*(c: string): seq[string] {.
   noSideEffect, rtl, extern: "nos$1".} =
-  ## Splits a command line into several components;
+  ## Splits a `command line`:idx: into several components;
   ## This proc is only occasionally useful, better use the `parseopt` module.
   ##
   ## On Windows, it uses the following parsing rules

lib/pure/parseutils.nim

@@ -250,18 +250,18 @@ proc parseInt*(s: string, number: var int, start = 0): int {.
   elif result != 0:
     number = int(res)
 
-# overflowChecks doesn't work with uint64
+# overflowChecks doesn't work with BiggestUInt
-proc rawParseUInt(s: string, b: var uint64, start = 0): int =
+proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int =
   var
-    res = 0'u64
+    res = 0.BiggestUInt
-    prev = 0'u64
+    prev = 0.BiggestUInt
     i = start
   if s[i] == '+': inc(i) # Allow
   if s[i] in {'0'..'9'}:
     b = 0
     while s[i] in {'0'..'9'}:
       prev = res
-      res = res * 10 + (ord(s[i]) - ord('0')).uint64
+      res = res * 10 + (ord(s[i]) - ord('0')).BiggestUInt
       if prev > res:
         return 0 # overflowChecks emulation
       inc(i)
@@ -269,13 +269,13 @@ proc rawParseUInt(s: string, b: var uint64, start = 0): int =
     b = res
     result = i - start
 
-proc parseBiggestUInt*(s: string, number: var uint64, start = 0): int {.
+proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {.
   rtl, extern: "npuParseBiggestUInt", noSideEffect.} =
   ## parses an unsigned integer starting at `start` and stores the value
   ## into `number`.
   ## Result is the number of processed chars or 0 if there is no integer
   ## or overflow detected.
-  var res: uint64
+  var res: BiggestUInt
   # use 'res' for exception safety (don't write to 'number' in case of an
   # overflow exception):
   result = rawParseUInt(s, res, start)
@@ -287,12 +287,12 @@ proc parseUInt*(s: string, number: var uint, start = 0): int {.
   ## into `number`.
   ## Result is the number of processed chars or 0 if there is no integer or
   ## overflow detected.
-  var res: uint64
+  var res: BiggestUInt
   result = parseBiggestUInt(s, res, start)
-  if (sizeof(uint) <= 4) and
+  when sizeof(BiggestUInt) > sizeof(uint) and sizeof(uint) <= 4:
-      (res > 0xFFFF_FFFF'u64):
+    if res > 0xFFFF_FFFF'u64:
-    raise newException(OverflowError, "overflow")
+      raise newException(OverflowError, "overflow")
-  elif result != 0:
+  if result != 0:
     number = uint(res)
 
 proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {.

lib/pure/strutils.nim

@@ -898,7 +898,7 @@ proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect,
 
 proc toHex*[T](x: T): string =
   ## Shortcut for ``toHex(x, T.sizeOf * 2)``
-  toHex(x, T.sizeOf * 2)
+  toHex(BiggestInt(x), T.sizeOf * 2)
 
 proc intToStr*(x: int, minchars: Positive = 1): string {.noSideEffect,
   rtl, extern: "nsuIntToStr".} =
@@ -939,7 +939,7 @@ proc parseUInt*(s: string): uint {.noSideEffect, procvar,
   if L != s.len or L == 0:
     raise newException(ValueError, "invalid unsigned integer: " & s)
 
-proc parseBiggestUInt*(s: string): uint64 {.noSideEffect, procvar,
+proc parseBiggestUInt*(s: string): BiggestUInt {.noSideEffect, procvar,
   rtl, extern: "nsuParseBiggestUInt".} =
   ## Parses a decimal unsigned integer value contained in `s`.
   ##

lib/pure/unittest.nim

@@ -24,7 +24,7 @@
 ##       echo "run before each test"
 ##
 ##     teardown:
-##       echo "run after each test":
+##       echo "run after each test"
 ##
 ##     test "essential truths":
 ##       # give up and stop if this fails

lib/system.nim

@@ -1512,6 +1512,17 @@ type
     ## compiler supports. Currently this is ``float64``, but it is
     ## platform-dependant in general.
 
+when defined(JS):
+  type BiggestUInt* = uint32
+    ## is an alias for the biggest unsigned integer type the Nim compiler
+    ## supports. Currently this is ``uint32`` for JS and ``uint64`` for other
+    ## targets.
+else:
+  type BiggestUInt* = uint64
+    ## is an alias for the biggest unsigned integer type the Nim compiler
+    ## supports. Currently this is ``uint32`` for JS and ``uint64`` for other
+    ## targets.
+
 {.deprecated: [TAddress: ByteAddress].}
 
 when defined(windows):
@@ -2732,6 +2743,7 @@ when not defined(JS): #and not defined(nimscript):
       # we use binary mode on Windows:
       c_setmode(c_fileno(stdin), O_BINARY)
       c_setmode(c_fileno(stdout), O_BINARY)
+      c_setmode(c_fileno(stderr), O_BINARY)
 
     when defined(endb):
       proc endbStep()

lib/system/alloc.nim

@@ -292,10 +292,15 @@ proc writeFreeList(a: MemRegion) =
               it, it.next, it.prev, it.size)
     it = it.next
 
+const nimMaxHeap {.intdefine.} = 0
+
 proc requestOsChunks(a: var MemRegion, size: int): PBigChunk =
   when not defined(emscripten):
     if not a.blockChunkSizeIncrease:
       let usedMem = a.currMem # - a.freeMem
+      when nimMaxHeap != 0:
+        if usedMem > nimMaxHeap * 1024 * 1024:
+          raiseOutOfMem()
       if usedMem < 64 * 1024:
         a.nextChunkSize = PageSize*4
       else:

lib/system/gc_common.nim

@@ -128,13 +128,7 @@ iterator items(stack: ptr GcStack): ptr GcStack =
     yield s
     s = s.next
 
-# There will be problems with GC in foreign threads if `threads` option is off or TLS emulation is enabled
+when declared(threadType):
-const allowForeignThreadGc = compileOption("threads") and not compileOption("tlsEmulation")
-
-when allowForeignThreadGc:
-  var
-    localGcInitialized {.rtlThreadVar.}: bool
-
   proc setupForeignThreadGc*() {.gcsafe.} =
     ## Call this if you registered a callback that will be run from a thread not
     ## under your control. This has a cheap thread-local guard, so the GC for
@@ -143,16 +137,33 @@ when allowForeignThreadGc:
     ##
     ## This function is available only when ``--threads:on`` and ``--tlsEmulation:off``
     ## switches are used
-    if not localGcInitialized:
+    if threadType == ThreadType.None:
-      localGcInitialized = true
       initAllocator()
       var stackTop {.volatile.}: pointer
       setStackBottom(addr(stackTop))
       initGC()
+      threadType = ThreadType.ForeignThread
+
+  proc tearDownForeignThreadGc*() {.gcsafe.} =
+    ## Call this to tear down the GC, previously initialized by ``setupForeignThreadGc``.
+    ## If GC has not been previously initialized, or has already been torn down, the
+    ## call does nothing.
+    ##
+    ## This function is available only when ``--threads:on`` and ``--tlsEmulation:off``
+    ## switches are used
+    if threadType != ThreadType.ForeignThread:
+      return
+    when declared(deallocOsPages): deallocOsPages()
+    threadType = ThreadType.None
+    when declared(gch): zeroMem(addr gch, sizeof(gch))
+
 else:
   template setupForeignThreadGc*() =
     {.error: "setupForeignThreadGc is available only when ``--threads:on`` and ``--tlsEmulation:off`` are used".}
 
+  template tearDownForeignThreadGc*() =
+    {.error: "tearDownForeignThreadGc is available only when ``--threads:on`` and ``--tlsEmulation:off`` are used".}
+
 # ----------------- stack management --------------------------------------
 #  inspired from Smart Eiffel
 

lib/system/sysio.nim

@@ -171,21 +171,23 @@ proc readLine(f: File): TaintedString =
 
 proc write(f: File, i: int) =
   when sizeof(int) == 8:
-    c_fprintf(f, "%lld", i)
+    if c_fprintf(f, "%lld", i) < 0: checkErr(f)
   else:
-    c_fprintf(f, "%ld", i)
+    if c_fprintf(f, "%ld", i) < 0: checkErr(f)
 
 proc write(f: File, i: BiggestInt) =
   when sizeof(BiggestInt) == 8:
-    c_fprintf(f, "%lld", i)
+    if c_fprintf(f, "%lld", i) < 0: checkErr(f)
   else:
-    c_fprintf(f, "%ld", i)
+    if c_fprintf(f, "%ld", i) < 0: checkErr(f)
 
 proc write(f: File, b: bool) =
   if b: write(f, "true")
   else: write(f, "false")
-proc write(f: File, r: float32) = c_fprintf(f, "%g", r)
+proc write(f: File, r: float32) =
-proc write(f: File, r: BiggestFloat) = c_fprintf(f, "%g", r)
+  if c_fprintf(f, "%g", r) < 0: checkErr(f)
+proc write(f: File, r: BiggestFloat) =
+  if c_fprintf(f, "%g", r) < 0: checkErr(f)
 
 proc write(f: File, c: char) = discard c_putc(ord(c), f)
 proc write(f: File, a: varargs[string, `$`]) =
@@ -213,7 +215,10 @@ proc rawFileSize(file: File): int =
   discard c_fseek(file, clong(oldPos), 0)
 
 proc endOfFile(f: File): bool =
-  result = c_feof(f) != 0
+  var c = c_fgetc(f)
+  discard c_ungetc(c, f)
+  return c < 0'i32
+  #result = c_feof(f) != 0
 
 proc readAllFile(file: File, len: int): string =
   # We acquire the filesize beforehand and hope it doesn't change.

lib/system/threads.nim

@@ -285,7 +285,19 @@ when useStackMaskHack:
 when not defined(useNimRtl):
   when not useStackMaskHack:
     #when not defined(createNimRtl): initStackBottom()
-    when declared(initGC): initGC()
+    when declared(initGC):
+      initGC()
+      when not emulatedThreadVars:
+        type ThreadType {.pure.} = enum
+          None = 0,
+          NimThread = 1,
+          ForeignThread = 2
+        var
+          threadType {.rtlThreadVar.}: ThreadType
+
+        threadType = ThreadType.NimThread
+
+
 
   when emulatedThreadVars:
     if nimThreadVarsSize() > sizeof(ThreadLocalStorage):
@@ -442,6 +454,8 @@ proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) =
       # init the GC for refc/markandsweep
       setStackBottom(addr(p))
       initGC()
+      when declared(threadType):
+        threadType = ThreadType.NimThread
     when declared(registerThread):
       thrd.stackBottom = addr(thrd)
       registerThread(thrd)

readme.md

@@ -1,146 +1,175 @@
-# <img src="https://raw.githubusercontent.com/nim-lang/assets/master/Art/logo-crown.png" width="36"> Nim [![Build Status](https://travis-ci.org/nim-lang/Nim.svg?branch=devel)](https://travis-ci.org/nim-lang/Nim)
+# ![Logo][image-nim-logo] Nim [![Build Status][badge-nim-travisci]][nim-travisci]
 
-This repo contains the Nim compiler, Nim's stdlib, tools and
+This repository contains the Nim compiler, Nim's stdlib, tools and documentation.
-documentation. For more information about Nim, including downloads
+For more information about Nim, including downloads and documentation for
-and documentation for the latest release, check out
+the latest release, check out [Nim's website][nim-site].
-[Nim's website](http://nim-lang.org).
+
+## Community
+[![Join the IRC chat][badge-nim-irc]][nim-irc]
+[![Join the Gitter chat][badge-nim-gitter]][nim-gitter]
+[![Get help][badge-nim-forum-gethelp]][nim-forum]
+[![View Nim posts on Stack Overflow][badge-nim-stackoverflow]][nim-stackoverflow-newest]
+[![Follow @nim_lang on Twitter][badge-nim-twitter]][nim-twitter]
+
+* The [forum][nim-forum] - the best place to ask questions and to discuss Nim.
+* [#nim IRC Channel (Freenode)][nim-irc] - a place to discuss Nim in real-time.
+  Also where most development decisions get made.
+* [Gitter][nim-gitter] - an additional place to discuss Nim in real-time. There
+  is a bridge between Gitter and the IRC channel.
+* [Stack Overflow][nim-stackoverflow] - a popular Q/A site for programming related
+  topics that includes posts about Nim.
 
 ## Compiling
-Compiling the Nim compiler is quite straightforward. Because
+The compiler currently officially supports the following platform and
-the Nim compiler itself is written in the Nim programming language
+architecture combinations:
-the C source of an older version of the compiler are needed to bootstrap the
-latest version. The C sources are available in a separate
-repo [here](http://github.com/nim-lang/csources).
-
-The compiler currently supports the following platform and architecture
-combinations:
 
   * Windows (Windows XP or greater) - x86 and x86_64
   * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l
-  * Mac OS X 10.04 or higher - x86, x86_64 and ppc64
+  * Mac OS X (10.04 or greater) - x86, x86_64 and ppc64
 
-In reality a lot more are supported, however they are not tested regularly.
+More platforms are supported, however they are not tested regularly and they
+may not be as stable as the above-listed platforms.
 
-To build from source you will need:
+Compiling the Nim compiler is quite straightforward if you follow these steps:
 
-  * gcc 3.x or later recommended. Other alternatives which may work
+First, the C source of an older version of the Nim compiler is needed to
-    are: clang, Visual C++, Intel's C++ compiler
+bootstrap the latest version because the Nim compiler itself is written in the
-  * git or wget
+Nim programming language. Those C sources are available within the 
+[``nim-lang/csources``][csources-repo] repository.
 
-**Note:** When installing ``gcc`` on Ubuntu (and likely other distros) ensure
+Next, to build from source you will need:
-that the ``build-essentials`` package is installed also.
 
-If you are on a fairly modern *nix system, the following steps should work:
+  * A C compiler such as ``gcc`` 3.x/later or an alternative such as ``clang``,
+    ``Visual C++`` or ``Intel C++``. It is recommended to use ``gcc`` 3.x or
+    later.
+  * Either ``git`` or ``wget`` to download the needed source repositories.
+  * The ``build-essentials`` package when using ``gcc`` on Ubuntu (and likely
+    other distros as well). 
+
+Then, if you are on a \*nix system or Windows, the following steps should compile
+Nim from source using ``gcc``, ``git`` and the ``koch`` build tool (in the place
+of ``sh build.sh`` you should substitute ``build.bat`` on x86 Windows or
+``build64.bat`` on x86_64 Windows):
 
 ```
 $ git clone https://github.com/nim-lang/Nim.git
 $ cd Nim
-$ git clone --depth 1 https://github.com/nim-lang/csources
+$ git clone --depth 1 https://github.com/nim-lang/csources.git
-$ cd csources && sh build.sh
+$ cd csources
-$ cd ..
+$ sh build.sh
+$ cd ../
 $ bin/nim c koch
 $ ./koch boot -d:release
 ```
 
-You should then add the ``bin`` directory to your PATH, to make it easily
+Finally, once you have finished the build steps (on Windows, Mac or Linux) you
-executable on your system.
+should add the ``bin`` directory to your PATH.
 
-The above steps can be performed on Windows in a similar fashion, the
+## Koch
-``build.bat`` and ``build64.bat`` (for x86_64 systems) are provided to be used
+``koch`` is the build tool used to build various parts of Nim and to generate
-instead of ``build.sh``.
+documentation and the website, among other things. The ``koch`` tool can also
+be used to run the Nim test suite. 
 
-The ``koch`` tool is the Nim build tool, more ``koch`` related options are
+Assuming that you added Nim's ``bin`` directory to your PATH, you may execute
-documented in [doc/koch.rst](doc/koch.rst).
+the tests using ``./koch tests``. The tests take a while to run, but you
+can run a subset of tests by specifying a category (for example 
+``./koch tests cat async``).
 
+For more information on the ``koch`` build tool please see the documentation
+within the [doc/koch.rst](doc/koch.rst) file.
 
 ## Nimble
-[Nimble](https://github.com/nim-lang/nimble) is Nim's package manager. For the
+``nimble`` is Nim's package manager and it can be acquired from the
-source based installations, where you added Nim's ``bin`` directory to your PATH,
+[``nim-lang/nimble``][nimble-repo] repository. Assuming that you added Nim's
-the easiest way of installing Nimble is via:
+``bin`` directory to your PATH, you may install Nimble from source by running
-
+``koch nimble`` within the root of the cloned repository.
-```
-$ koch nimble
-```
-
-## Community
-[![Join the Chat at irc.freenode.net#nim](https://img.shields.io/badge/IRC-join_chat_in_%23nim-blue.svg)](https://webchat.freenode.net/?channels=nim)
-[![Join the Gitter channel](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/nim-lang/Nim)
-[![Get help](https://img.shields.io/badge/Forum-get%20help-4eb899.svg)](http://forum.nim-lang.org)
-[![Stackoverflow](https://img.shields.io/badge/stackoverflow-use_%23nim_tag-yellow.svg)](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15)
-[![Follow @nim_lang!](https://img.shields.io/twitter/follow/nim_lang.svg?style=social)](https://twitter.com/nim_lang)
-
-* The [forum](http://forum.nim-lang.org/) - the best place to ask questions and to discuss Nim.
-* [IRC (Freenode#nim)](https://webchat.freenode.net/?channels=nim) - a place to discuss
-  Nim in real-time, this is also where most development decision get made!
-* [Gitter](https://gitter.im/nim-lang/Nim) allows to discuss Nim from your browser, one click to join.
-  There is a bridge between Gitter and IRC channels.
-* [Stackoverflow](http://stackoverflow.com/questions/tagged/nim)
 
 ## Contributing
+[![Contribute to Nim via Gratipay][badge-nim-gratipay]][nim-gratipay]
+[![Setup a bounty via Bountysource][badge-nim-bountysource]][nim-bountysource]
+[![Donate Bitcoins][badge-nim-bitcoin]][nim-bitcoin]
 
-[![Contribute to Nim via Gratipay!](https://img.shields.io/gratipay/team/nim.svg)](https://gratipay.com/nim/)
+We welcome everyone's contributions to Nim independent of how small or how large
-[![Bountysource](https://img.shields.io/bountysource/team/nim/activity.svg)](https://www.bountysource.com/teams/nim)
+they are. Anything from small spelling fixes to large modules intended to
-
+be included in the standard library are welcome and appreciated. Before you get
-We welcome everyone's contributions to Nim. No matter how small or large
+started contributing, you should familiarize yourself with the repository structure:
-the contribution is, anything from small spelling fixes to large modules
-intended to be included in the standard library are accepted. Before
-you get started, you should know the following about this repositories
-structure:
 
 * ``bin/``, ``build/`` - these directories are empty, but are used when Nim is built.
-* ``compiler/`` - the compiler source code, all the Nim source code files in this
+* ``compiler/`` - the compiler source code. Also includes nimfix, and plugins within
-  directory implement the compiler. This also includes nimfix, and plugins
+  ``compiler/nimfix`` and ``compiler/plugins`` respectively. Nimsuggest was moved to
-  which live in ``compiler/nimfix`` and ``compiler/plugins``
+  the [``nim-lang/nimsuggest``][nimsuggest-repo] repository, though it previously also 
-  respectively. Nimsuggest used to live in the ``compiler`` directory also,
+  lived within the ``compiler/`` directory.
-  but was moved to https://github.com/nim-lang/nimsuggest.
 * ``config/`` - the configuration for the compiler and documentation generator.
 * ``doc/`` - the documentation files in reStructuredText format.
-* ``lib/`` - where the standard library lives.
+* ``lib/`` - the standard library, including:
     * ``pure/`` - modules in the standard library written in pure Nim.
-    * ``impure/`` - modules in the standard library written in pure Nim which
+    * ``impure/`` - modules in the standard library written in pure Nim with
-      depend on libraries written in other languages.
+    dependencies written in other languages.
-    * ``wrappers/`` - modules which wrap libraries written in other languages.
+    * ``wrappers/`` - modules which wrap dependencies written in other languages.
-* ``tests/`` - contains tests for the compiler and standard library, organised by
+* ``tests/`` - contains categorized tests for the compiler and standard library.
-    category.
+* ``tools/`` - the tools including ``niminst`` and ``nimweb`` (mostly invoked via
-* ``tools/`` - the tools including ``niminst`` and ``nimweb``, most of these are invoked
+  ``koch``).
-    via ``koch``.
+* ``web/`` - [the Nim website][nim-site].
-* ``web/`` - the Nim website (http://nim-lang.org).
+* ``koch.nim`` - tool used to bootstrap Nim, generate C sources, build the website,
-* ``koch.nim`` - tool used to bootstrap Nim, generate C sources, build the website, documentation
+  and generate the documentation.
-  and more.
 
-Most importantly, the ``koch`` tool can be used to run the test suite. To do so compile it first
+If you are not familiar with making a pull request using GitHub and/or git, please
-by executing ``nim c koch``, then execute ``./koch tests``. The test suite takes a while to run,
+read [this guide][pull-request-instructions].
-but you can run specific tests by specifying a category to run, for example ``./koch tests cat async``.
 
-Make sure that the tests all pass before
+Ideally you should make sure that all tests pass before submitting a pull request.
-[submitting your pull request](https://help.github.com/articles/using-pull-requests/).
+However, if you are short on time, you can just run the tests specific to your
-If you're short on time, you can
+changes by only running the corresponding categories of tests. Travis CI verifies
-just run the tests specific to your change. Just run the category which corresponds to the change
+that all tests pass before allowing the pull request to be accepted, so only
-you've made. When you create your pull request, Travis CI will verify that all the tests pass
+running specific tests should be harmless.
-anyway.
 
-If you're looking for things to do, take a look at our
+If you're looking for ways to contribute, please look at our [issue tracker][nim-issues].
-[issue tracker](https://github.com/nim-lang/Nim/issues). There is always plenty of issues
+There are always plenty of issues labelled [``Easy``][nim-issues-easy]; these should
-labelled [``Easy``](https://github.com/nim-lang/Nim/labels/Easy), these should be a good
+be a good starting point for an initial contribution to Nim.
-starting point if this is your first contribution to Nim.
 
-You can also help with the development of Nim by making donations. You can do so
+You can also help with the development of Nim by making donations. Donations can be
-in many ways:
+made using:
 
-* [Gratipay](https://gratipay.com/nim/)
+* [Gratipay][nim-gratipay]
-* [Bountysource](https://www.bountysource.com/teams/nim)
+* [Bountysource][nim-bountysource]
-* Bitcoin - 1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ
+* [Bitcoin][nim-bitcoin]
 
-Finally, if you have any questions feel free to submit a question on the issue tracker,
+If you have any questions feel free to submit a question on the
-on the [Nim forum](http://forum.nim-lang.org), or on IRC.
+[Nim forum][nim-forum], or via IRC on [the \#nim channel][nim-irc].
 
 ## License
-The compiler and the standard library are licensed under the MIT license,
+The compiler and the standard library are licensed under the MIT license, except
-except for some modules where the documentation suggests otherwise. This means
+for some modules which explicitly state otherwise. As a result you may use any
-that you can use any license for your own programs developed with Nim,
+compatible license (essentially any license) for your own programs developed with
-allowing you to create commercial applications.
+Nim. You are explicitly permitted to develop commercial applications using Nim.
 
-Read copying.txt for more details.
+Please read the [copying.txt](copying.txt) file for more details.
 
-Copyright (c) 2006-2017 Andreas Rumpf.
+Copyright © 2006-2017 Andreas Rumpf, all rights reserved.
-All rights reserved.
+
+[nim-site]: https://nim-lang.org
+[nim-forum]: https://forum.nim-lang.org
+[nim-issues]: https://github.com/nim-lang/Nim/issues
+[nim-issues-easy]: https://github.com/nim-lang/Nim/labels/Easy
+[nim-irc]: https://webchat.freenode.net/?channels=nim
+[nim-travisci]: https://travis-ci.org/nim-lang/Nim
+[nim-twitter]: https://twitter.com/nim_lang
+[nim-stackoverflow]: https://stackoverflow.com/questions/tagged/nim
+[nim-stackoverflow-newest]: https://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15
+[nim-gitter]: https://gitter.im/nim-lang/Nim
+[nim-gratipay]: https://gratipay.com/nim/
+[nim-bountysource]: https://www.bountysource.com/teams/nim
+[nim-bitcoin]: https://blockchain.info/address/1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ
+[nimble-repo]: https://github.com/nim-lang/nimble
+[nimsuggest-repo]: https://github.com/nim-lang/nimsuggest
+[csources-repo]: https://github.com/nim-lang/csources
+[badge-nim-travisci]: https://img.shields.io/travis/nim-lang/Nim/devel.svg?style=flat-square
+[badge-nim-irc]: https://img.shields.io/badge/chat-on_irc-blue.svg?style=flat-square
+[badge-nim-gitter]: https://img.shields.io/badge/chat-on_gitter-blue.svg?style=flat-square
+[badge-nim-forum-gethelp]: https://img.shields.io/badge/Forum-get%20help-4eb899.svg?style=flat-square
+[badge-nim-twitter]: https://img.shields.io/twitter/follow/nim_lang.svg?style=social
+[badge-nim-stackoverflow]: https://img.shields.io/badge/stackoverflow-nim_tag-yellow.svg?style=flat-square
+[badge-nim-gratipay]: https://img.shields.io/gratipay/team/nim.svg?style=flat-square
+[badge-nim-bountysource]: https://img.shields.io/bountysource/team/nim/activity.svg?style=flat-square
+[badge-nim-bitcoin]: https://img.shields.io/badge/bitcoin-1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ-D69134.svg?style=flat-square
+[image-nim-logo]: https://images1-focus-opensocial.googleusercontent.com/gadgets/proxy?url=https://raw.githubusercontent.com/nim-lang/assets/master/Art/logo-crown.png&container=focus&resize_w=36&refresh=21600
+[pull-request-instructions]: https://help.github.com/articles/using-pull-requests/

tests/ccgbugs/tgeneric_smallobj_asgn_opt.nim

@@ -0,0 +1,26 @@
+discard """
+  output: '''false'''
+"""
+
+# bug #5402
+
+import lists
+
+type
+  Container[T] = ref object
+    obj: T
+
+  ListOfContainers[T] = ref object
+    list: DoublyLinkedList[Container[T]]
+
+proc contains[T](this: ListOfContainers[T], obj: T): bool =
+  for item in this.list.items():
+    if item.obj == obj: return true
+  return false
+
+proc newListOfContainers[T](): ListOfContainers[T] =
+  new(result)
+  result.list = initDoublyLinkedList[Container[T]]()
+
+let q = newListOfContainers[int64]()
+echo q.contains(123)

tests/ccgbugs/tmangle_field.nim

@@ -0,0 +1,16 @@
+discard """
+"""
+
+# bug #5404
+
+import parseopt2
+
+{.emit: """typedef struct {
+    int key;
+} foo;""".}
+
+type foo* {.importc: "foo", nodecl.} = object
+  key* {.importc: "key".}: cint
+
+for kind, key, value in parseopt2.getopt():
+  discard

tests/errmsgs/tproper_stacktrace3.nim

@@ -0,0 +1,23 @@
+discard """
+  outputsub: '''tproper_stacktrace3.nim(21) main'''
+  exitcode: 1
+"""
+
+# bug #5400
+
+type Container = object
+  val: int
+
+proc actualResolver(x: ptr Container): ptr Container = x
+
+template resolve(): untyped = actualResolver(db)
+
+proc myfail(): int =
+  doAssert false
+
+proc main() =
+  var db: ptr Container = nil
+  # actualResolver(db).val = myfail() # actualResolver is not included in stack trace.
+  resolve().val = myfail() # resolve template is included in stack trace.
+
+main()

tests/gc/foreign_thr.nim

@@ -0,0 +1,88 @@
+discard """
+  output: '''
+Hello from thread
+Hello from thread
+Hello from thread
+Hello from thread
+'''
+  cmd: "nim $target --hints:on --threads:on --tlsEmulation:off $options $file"
+"""
+# Copied from stdlib
+import strutils
+
+const
+  StackGuardSize = 4096
+  ThreadStackMask = 1024*256*sizeof(int)-1
+  ThreadStackSize = ThreadStackMask+1 - StackGuardSize
+
+type ThreadFunc = proc() {.thread.}
+
+when defined(posix):
+  import posix
+
+  proc runInForeignThread(f: ThreadFunc) =
+    proc wrapper(p: pointer): pointer {.noconv.} =
+      let thr = cast[ThreadFunc](p)
+      setupForeignThreadGc()
+      thr()
+      tearDownForeignThreadGc()
+      setupForeignThreadGc()
+      thr()
+      tearDownForeignThreadGc()
+      result = nil
+
+    var attrs {.noinit.}: PthreadAttr
+    doAssert pthread_attr_init(addr attrs) == 0
+    doAssert pthread_attr_setstacksize(addr attrs, ThreadStackSize) == 0
+    var tid: Pthread
+    doAssert pthread_create(addr tid, addr attrs, wrapper, f) == 0
+    doAssert pthread_join(tid, nil) == 0
+
+elif defined(windows):
+  import winlean
+  type
+    WinThreadProc = proc (x: pointer): int32 {.stdcall.}
+
+  proc createThread(lpThreadAttributes: pointer, dwStackSize: DWORD,
+                     lpStartAddress: WinThreadProc,
+                     lpParameter: pointer,
+                     dwCreationFlags: DWORD,
+                     lpThreadId: var DWORD): Handle {.
+    stdcall, dynlib: "kernel32", importc: "CreateThread".}
+
+  proc wrapper(p: pointer): int32 {.stdcall.} =
+    let thr = cast[ThreadFunc](p)
+    setupForeignThreadGc()
+    thr()
+    tearDownForeignThreadGc()
+    setupForeignThreadGc()
+    thr()
+    tearDownForeignThreadGc()
+    result = 0'i32
+
+  proc runInForeignThread(f: ThreadFunc) =
+    var dummyThreadId: DWORD
+    var h = createThread(nil, ThreadStackSize.int32, wrapper.WinThreadProc, cast[pointer](f), 0, dummyThreadId)
+    doAssert h != 0.Handle
+    doAssert waitForSingleObject(h, -1'i32) == 0.DWORD
+
+else:
+  {.fatal: "Unknown system".}
+
+proc runInNativeThread(f: ThreadFunc) =
+  proc wrapper(f: ThreadFunc) {.thread.} =
+    # These operations must be NOP
+    setupForeignThreadGc()
+    tearDownForeignThreadGc()
+    f()
+    f()
+  var thr: Thread[ThreadFunc]
+  createThread(thr, wrapper, f)
+  joinThread(thr)
+
+proc f {.thread.} =
+  var msg = "Hello " & "from thread"
+  echo msg
+
+runInForeignThread(f)
+runInNativeThread(f)

tests/metatype/tfieldaccessor.nim

@@ -0,0 +1,17 @@
+type
+  Test = object
+    x: int
+    case p: bool
+    of true:
+      a: int
+    else:
+      case q: bool
+      of true:
+        b: int
+      else:
+        discard
+
+proc f[T](t: typedesc[T]): int =
+  1
+
+assert Test.f == 1

tests/method/tautonotgeneric.nim

@@ -1,15 +1,24 @@
 discard """
-  output: "wof!"
+  output: '''wof!
+wof!'''
 """
 
 # bug #1659
 type Animal = ref object {.inheritable.}
 type Dog = ref object of Animal
 
-method say(a: Animal): auto = "wat!"
+method say(a: Animal): auto {.base.} = "wat!"
 method say(a: Dog): auto = "wof!"
 
 proc saySomething(a: Animal): auto = a.say()
 
+
+method ec(a: Animal): auto {.base.} = echo "wat!"
+method ec(a: Dog): auto = echo "wof!"
+
+proc ech(a: Animal): auto = a.ec()
+
+
 var a = Dog()
 echo saySomething(a)
+ech a

tests/misc/parsecomb.nim

@@ -29,10 +29,10 @@ method runInput[T, O](self: Parser[T, O], inp: Input[T]): Result[T, O] =
   # XXX: above needed for now, as without the `tmp` bit below, it compiles to invalid C.
   tmp(self)(inp)
 
-method run*[T, O](self: Parser[T, O], toks: seq[T]): Result[T, O] =
+proc run*[T, O](self: Parser[T, O], toks: seq[T]): Result[T, O] =
   self.runInput(Input[T](toks: toks, index: 0))
 
-method chain*[T, O1, O2](self: Parser[T, O1], nextp: proc (v: O1): Parser[T, O2]): Parser[T, O2] =
+proc chain*[T, O1, O2](self: Parser[T, O1], nextp: proc (v: O1): Parser[T, O2]): Parser[T, O2] =
   result = proc (inp: Input[T]): Result[T, O2] =
     let r = self.runInput(inp)
     case r.kind:
@@ -41,7 +41,7 @@ method chain*[T, O1, O2](self: Parser[T, O1], nextp: proc (v: O1): Parser[T, O2]
     of rkFailure:
       Result[T, O2](kind: rkFailure)
 
-method skip[T](self: Input[T], n: int): Input[T] =
+method skip[T](self: Input[T], n: int): Input[T] {.base.} =
   Input[T](toks: self.toks, index: self.index + n)
 
 proc pskip*[T](n: int): Parser[T, tuple[]] =
@@ -69,11 +69,11 @@ proc `+`*[T, O](first: Parser[T, O], second: Parser[T, O]): Parser[T, O] =
 
 # end of primitives (definitions involving Parser(..))
 
-method map*[T, O1, O2](self: Parser[T, O1], p: proc (v: O1): O2): Parser[T, O2] =
+proc map*[T, O1, O2](self: Parser[T, O1], p: proc (v: O1): O2): Parser[T, O2] =
   self.chain(proc (v: O1): Parser[T, O2] =
     unit[T, O2](p(v)))
 
-method then*[T, O1, O2](self: Parser[T, O1], next: Parser[T, O2]): Parser[T, O2] =
+proc then*[T, O1, O2](self: Parser[T, O1], next: Parser[T, O2]): Parser[T, O2] =
   self.chain(proc (v: O1): Parser[T, O2] =
     next)
 

tests/osproc/ta.nim

@@ -1,3 +0,0 @@
-import strutils
-let x = stdin.readLine()
-echo x.parseInt + 5

tests/osproc/ta_in.nim

@@ -0,0 +1,5 @@
+# This file is prefixed with an "a", because other tests
+# depend on it and it must be compiled first.
+import strutils
+let x = stdin.readLine()
+echo x.parseInt + 5

tests/osproc/ta_out.nim

@@ -0,0 +1,16 @@
+# This file is prefixed with an "a", because other tests
+# depend on it and it must be compiled first.
+stdout.writeLine("to stdout")
+stdout.flushFile()
+stdout.writeLine("to stdout")
+stdout.flushFile()
+
+stderr.writeLine("to stderr")
+stderr.flushFile()
+stderr.writeLine("to stderr")
+stderr.flushFile()
+
+stdout.writeLine("to stdout")
+stdout.flushFile()
+stdout.writeLine("to stdout")
+stdout.flushFile()

tests/osproc/tstdin.nim

@@ -4,7 +4,7 @@ discard """
 """
 import osproc, os, streams
 
-const filename = when defined(Windows): "ta.exe" else: "ta"
+const filename = when defined(Windows): "ta_in.exe" else: "ta_in"
 
 doAssert fileExists(getCurrentDir() / "tests" / "osproc" / filename)
 

tests/osproc/tstdout.nim

@@ -0,0 +1,29 @@
+discard """
+  output: '''--------------------------------------
+to stdout
+to stdout
+to stderr
+to stderr
+to stdout
+to stdout
+--------------------------------------
+'''
+"""
+import osproc, os, streams
+
+const filename = when defined(Windows): "ta_out.exe" else: "ta_out"
+
+doAssert fileExists(getCurrentDir() / "tests" / "osproc" / filename)
+
+var p = startProcess(filename, getCurrentDir() / "tests" / "osproc",
+                     options={poStdErrToStdOut})
+
+let outputStream = p.outputStream
+var x = newStringOfCap(120)
+var output = ""
+while outputStream.readLine(x.TaintedString):
+  output.add(x & "\n")
+
+echo "--------------------------------------"
+stdout.write output
+echo "--------------------------------------"

tests/overload/tprefer_tygenericinst.nim

@@ -19,7 +19,7 @@ when true:
   q(B()) # This call reported as ambiguous.
 
 # bug #2219
-template testPred(a: expr) =
+template testPred(a: untyped) =
   block:
     type A = object of RootObj
     type B = object of A

tests/pragmas/tlocks.nim

@@ -0,0 +1,13 @@
+
+type SomeBase* = ref object of RootObj
+type SomeDerived* = ref object of SomeBase
+  memberProc*: proc ()
+
+method testMethod(g: SomeBase) {.base, locks: "unknown".} = discard
+method testMethod(g: SomeDerived) =
+  if g.memberProc != nil:
+    g.memberProc()
+
+# ensure int literals still work
+proc plain*() {.locks: 0.} =
+  discard

tests/pragmas/tused.nim

@@ -0,0 +1,13 @@
+discard """
+  output: '''8'''
+"""
+
+template implementArithOps(T) =
+  proc echoAdd(a, b: T) {.used.} =
+    echo a + b
+  proc echoSub(a, b: T) {.used.} =
+    echo a - b
+
+# no warning produced for the unused 'echoSub'
+implementArithOps(int)
+echoAdd 3, 5

tests/stdlib/tbug5382.nim

@@ -0,0 +1,11 @@
+discard """
+  output: '''
+02
+'''
+"""
+import re
+
+let regexp = re"^\/([0-9]{2})\.html$"
+var matches: array[1, string]
+discard "/02.html".find(regexp, matches)
+echo matches[0]

tests/template/mgensym_generic_cross_module.nim

@@ -0,0 +1,14 @@
+
+template makeDomElement(x: untyped, name: string = nil) =
+  const tag {.gensym.} = if name == nil: astToStr(x) else: name
+
+  proc x*(p: int|float) =
+    echo tag, p
+
+  proc x*(p: string|cstring) =
+    echo tag, p
+
+#proc wrappedUp[T](x: T) =
+#  mixin foo, bar
+makeDomElement(foo, "foo")
+makeDomElement(bar)

tests/template/tgensym_generic_cross_module.nim

@@ -0,0 +1,14 @@
+discard """
+  output: '''foo55
+foo8.0
+fooaha
+bar7'''
+"""
+# bug #5419
+import mgensym_generic_cross_module
+
+foo(55)
+foo 8.0
+foo "aha"
+bar 7
+