Merge branch 'devel' into feature/3691

appveyor.yml

@@ -34,21 +34,20 @@ environment:
     #   platform: x86
 
 install:
-  - MKDIR %CD%\PCRE
+  - MKDIR %CD%\DIST
-  - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\pcre
+  - MKDIR %CD%\DIST\PCRE
+  - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\DIST\PCRE
   - IF not exist "%SQLITE_ARCHIVE%" appveyor DownloadFile "%SQLITE_URL%" -FileName "%SQLITE_ARCHIVE%"
-  - 7z x -y "%SQLITE_ARCHIVE%" > nul
+  - 7z x -y "%SQLITE_ARCHIVE%" -o"%CD%\DIST"> nul
   - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
-  - 7z x -y "%MINGW_ARCHIVE%" > nul
+  - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul
   - IF not exist "%FASM_ARCHIVE%" appveyor DownloadFile "%FASM_URL%" -FileName "%FASM_ARCHIVE%"
-  - 7z x -y "%FASM_ARCHIVE%" -o"%CD%\%FASM_DIR%" > nul
+  - 7z x -y "%FASM_ARCHIVE%" -o"%CD%\DIST\%FASM_DIR%" > nul
-  - SET PATH=%CD%\%MINGW_DIR%\bin;%CD%\Nim\bin;%CD%\%FASM_DIR%;%PATH%
+  - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%CD%\DIST\%FASM_DIR%;%PATH%
-  - git clone https://github.com/nim-lang/Nim.git %CD%\Nim
+  - IF "%PLATFORM%" == "x64" ( copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay64.dll & copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl64.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl32.dll )
-  - IF "%PLATFORM%" == "x64" ( copy C:\OpenSSL-Win64\libeay32.dll %CD%\Nim\bin\libeay64.dll & copy C:\OpenSSL-Win64\libeay32.dll %CD%\Nim\bin\libeay32.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\Nim\bin\libssl64.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\Nim\bin\libssl32.dll )
+    ELSE ( copy C:\OpenSSL-Win32\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win32\libssl32.dll %CD%\BIN\libssl32.dll )
-    ELSE ( copy C:\OpenSSL-Win32\libeay32.dll %CD%\Nim\bin\libeay32.dll & copy C:\OpenSSL-Win32\libssl32.dll %CD%\Nim\bin\libssl32.dll )
+  - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_64.dll ) ELSE ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_32.dll )
-  - IF "%PLATFORM%" == "x64" ( copy %CD%\sqlite3.dll %CD%\Nim\bin\sqlite3_64.dll ) ELSE ( copy %CD%\sqlite3.dll %CD%\Nim\bin\sqlite3_32.dll )
+  - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre64.dll ) ELSE ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre32.dll )
-  - IF "%PLATFORM%" == "x64" ( copy %CD%\pcre\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\Nim\bin\pcre64.dll ) ELSE ( copy %CD%\pcre\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\Nim\bin\pcre32.dll )
-  - cd %CD%\Nim
   - git clone --depth 1 https://github.com/nim-lang/csources
   - cd csources
   - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat )
@@ -70,6 +69,6 @@ build_script:
 test_script:
   - tests\testament\tester --pedantic all
   - koch csource
-  - koch xz
+  - koch zip
 
 deploy: off

compiler/ast.nim

@@ -1541,7 +1541,8 @@ proc skipGenericOwner*(s: PSym): PSym =
   ## Generic instantiations are owned by their originating generic
   ## symbol. This proc skips such owners and goes straight to the owner
   ## of the generic itself (the module or the enclosing proc).
-  result = if s.kind in skProcKinds and sfFromGeneric in s.flags:
+  result = if s.kind in skProcKinds and {sfGenSym, sfFromGeneric} * s.flags ==
+                                                  {sfFromGeneric}:
              s.owner.owner
            else:
              s.owner

compiler/ccgexprs.nim

@@ -1210,7 +1210,7 @@ proc genSeqConstr(p: BProc, t: PNode, d: var TLoc) =
 
 proc genArrToSeq(p: BProc, t: PNode, d: var TLoc) =
   var elem, a, arr: TLoc
-  if t.kind == nkBracket:
+  if t.sons[1].kind == nkBracket:
     t.sons[1].typ = t.typ
     genSeqConstr(p, t.sons[1], d)
     return
@@ -1383,7 +1383,9 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
 proc genSetLengthSeq(p: BProc, e: PNode, d: var TLoc) =
   var a, b: TLoc
   assert(d.k == locNone)
-  initLocExpr(p, e.sons[1], a)
+  var x = e.sons[1]
+  if x.kind in {nkAddr, nkHiddenAddr}: x = x[0]
+  initLocExpr(p, x, a)
   initLocExpr(p, e.sons[2], b)
   let t = skipTypes(e.sons[1].typ, {tyVar})
   let setLenPattern = if not p.module.compileToCpp:
@@ -2001,7 +2003,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       if sfThread in sym.flags:
         accessThreadLocalVar(p, sym)
         if emulatedThreadVars():
-          putIntoDest(p, d, sym.loc.t, "NimTV->" & sym.loc.r)
+          putIntoDest(p, d, sym.loc.t, "NimTV_->" & sym.loc.r)
         else:
           putLocIntoDest(p, d, sym.loc)
       else:

compiler/ccgstmts.nim

@@ -64,7 +64,7 @@ proc genVarTuple(p: BProc, n: PNode) =
       field.r = "$1.Field$2" % [rdLoc(tup), rope(i)]
     else:
       if t.n.sons[i].kind != nkSym: internalError(n.info, "genVarTuple")
-      field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(t.n.sons[i].sym, t)]
+      field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n.sons[i].sym, t)]
     putLocIntoDest(p, v.loc, field)
 
 proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false)
@@ -102,7 +102,7 @@ proc assignLabel(b: var TBlock): Rope {.inline.} =
 proc blockBody(b: var TBlock): Rope =
   result = b.sections[cpsLocals]
   if b.frameLen > 0:
-    result.addf("FR.len+=$1;$n", [b.frameLen.rope])
+    result.addf("FR_.len+=$1;$n", [b.frameLen.rope])
   result.add(b.sections[cpsInit])
   result.add(b.sections[cpsStmts])
 
@@ -123,7 +123,7 @@ proc endBlock(p: BProc) =
       ~"}$n"
   let frameLen = p.blocks[topBlock].frameLen
   if frameLen > 0:
-    blockEnd.addf("FR.len-=$1;$n", [frameLen.rope])
+    blockEnd.addf("FR_.len-=$1;$n", [frameLen.rope])
   endBlock(p, blockEnd)
 
 proc genSimpleBlock(p: BProc, stmts: PNode) {.inline.} =
@@ -156,7 +156,7 @@ proc genGotoState(p: BProc, n: PNode) =
   initLocExpr(p, n.sons[0], a)
   lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)])
   p.beforeRetNeeded = true
-  lineF(p, cpsStmts, "case -1: goto BeforeRet;$n", [])
+  lineF(p, cpsStmts, "case -1: goto BeforeRet_;$n", [])
   for i in 0 .. lastOrd(n.sons[0].typ):
     lineF(p, cpsStmts, "case $1: goto STATE$1;$n", [rope(i)])
   lineF(p, cpsStmts, "}$n", [])
@@ -373,7 +373,7 @@ proc genReturnStmt(p: BProc, t: PNode) =
     # consume it before we return.
     var safePoint = p.finallySafePoints[p.finallySafePoints.len-1]
     linefmt(p, cpsStmts, "if ($1.status != 0) #popCurrentException();$n", safePoint)
-  lineF(p, cpsStmts, "goto BeforeRet;$n", [])
+  lineF(p, cpsStmts, "goto BeforeRet_;$n", [])
 
 proc genGotoForCase(p: BProc; caseStmt: PNode) =
   for i in 1 .. <caseStmt.len:
@@ -411,11 +411,11 @@ proc genComputedGoto(p: BProc; n: PNode) =
     localError(n.info, "no case statement found for computed goto"); return
   var id = p.labels+1
   inc p.labels, arraySize+1
-  let tmp = "TMP$1" % [id.rope]
+  let tmp = "TMP$1_" % [id.rope]
   var gotoArray = "static void* $#[$#] = {" % [tmp, arraySize.rope]
   for i in 1..arraySize-1:
-    gotoArray.addf("&&TMP$#, ", [(id+i).rope])
+    gotoArray.addf("&&TMP$#_, ", [(id+i).rope])
-  gotoArray.addf("&&TMP$#};$n", [(id+arraySize).rope])
+  gotoArray.addf("&&TMP$#_};$n", [(id+arraySize).rope])
   line(p, cpsLocals, gotoArray)
 
   let topBlock = p.blocks.len-1
@@ -445,7 +445,7 @@ proc genComputedGoto(p: BProc; n: PNode) =
         localError(it.info, "range notation not available for computed goto")
         return
       let val = getOrdValue(it.sons[j])
-      lineF(p, cpsStmts, "TMP$#:$n", [intLiteral(val+id+1)])
+      lineF(p, cpsStmts, "TMP$#_:$n", [intLiteral(val+id+1)])
     genStmts(p, it.lastSon)
     #for j in casePos+1 .. <n.len: genStmts(p, n.sons[j]) # tailB
     #for j in 0 .. casePos-1: genStmts(p, n.sons[j])  # tailA
@@ -600,7 +600,7 @@ proc genCaseSecondPass(p: BProc, t: PNode, d: var TLoc,
   for i in 1..until:
     # bug #4230: avoid false sharing between branches:
     if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
-    lineF(p, cpsStmts, "LA$1: ;$n", [rope(labId + i)])
+    lineF(p, cpsStmts, "LA$1_: ;$n", [rope(labId + i)])
     if t.sons[i].kind == nkOfBranch:
       var length = sonsLen(t.sons[i])
       exprBlock(p, t.sons[i].sons[length - 1], d)
@@ -618,15 +618,15 @@ proc genIfForCaseUntil(p: BProc, t: PNode, d: var TLoc,
     inc(p.labels)
     if t.sons[i].kind == nkOfBranch: # else statement
       genCaseGenericBranch(p, t.sons[i], a, rangeFormat, eqFormat,
-                           "LA" & rope(p.labels))
+                           "LA" & rope(p.labels) & "_")
     else:
-      lineF(p, cpsStmts, "goto LA$1;$n", [rope(p.labels)])
+      lineF(p, cpsStmts, "goto LA$1_;$n", [rope(p.labels)])
   if until < t.len-1:
     inc(p.labels)
     var gotoTarget = p.labels
-    lineF(p, cpsStmts, "goto LA$1;$n", [rope(gotoTarget)])
+    lineF(p, cpsStmts, "goto LA$1_;$n", [rope(gotoTarget)])
     result = genCaseSecondPass(p, t, d, labId, until)
-    lineF(p, cpsStmts, "LA$1: ;$n", [rope(gotoTarget)])
+    lineF(p, cpsStmts, "LA$1_: ;$n", [rope(gotoTarget)])
   else:
     result = genCaseSecondPass(p, t, d, labId, until)
 
@@ -664,7 +664,7 @@ proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
     for i in countup(1, sonsLen(t) - 1):
       inc(p.labels)
       if t.sons[i].kind == nkOfBranch:
-        genCaseStringBranch(p, t.sons[i], a, "LA" & rope(p.labels),
+        genCaseStringBranch(p, t.sons[i], a, "LA" & rope(p.labels) & "_",
                             branches)
       else:
         # else statement: nothing to do yet
@@ -678,7 +678,7 @@ proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
              [intLiteral(j), branches[j]])
     lineF(p, cpsStmts, "}$n", []) # else statement:
     if t.sons[sonsLen(t)-1].kind != nkOfBranch:
-      lineF(p, cpsStmts, "goto LA$1;$n", [rope(p.labels)])
+      lineF(p, cpsStmts, "goto LA$1_;$n", [rope(p.labels)])
     # third pass: generate statements
     var lend = genCaseSecondPass(p, t, d, labId, sonsLen(t)-1)
     fixLabel(p, lend)
@@ -802,7 +802,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   let length = sonsLen(t)
   endBlock(p, ropecg(p.module, "} catch (NimException& $1) {$n", [exc]))
   if optStackTrace in p.options:
-    linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR);$n")
+    linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR_);$n")
   inc p.inExceptBlock
   var i = 1
   var catchAllPresent = false
@@ -910,7 +910,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
   startBlock(p, "else {$n")
   linefmt(p, cpsStmts, "#popSafePoint();$n")
   if optStackTrace in p.options:
-    linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR);$n")
+    linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR_);$n")
   inc p.inExceptBlock
   var i = 1
   while (i < length) and (t.sons[i].kind == nkExceptBranch):

compiler/ccgthreadvars.nim

@@ -19,9 +19,9 @@ proc accessThreadLocalVar(p: BProc, s: PSym) =
   if emulatedThreadVars() and not p.threadVarAccessed:
     p.threadVarAccessed = true
     incl p.module.flags, usesThreadVars
-    addf(p.procSec(cpsLocals), "\tNimThreadVars* NimTV;$n", [])
+    addf(p.procSec(cpsLocals), "\tNimThreadVars* NimTV_;$n", [])
     add(p.procSec(cpsInit),
-      ropecg(p.module, "\tNimTV = (NimThreadVars*) #GetThreadLocalVars();$n"))
+      ropecg(p.module, "\tNimTV_ = (NimThreadVars*) #GetThreadLocalVars();$n"))
 
 var
   nimtv: Rope                 # Nim thread vars; the struct body

compiler/ccgtrav.nim

@@ -145,7 +145,7 @@ proc genTraverseProcForGlobal(m: BModule, s: PSym): Rope =
 
   if sfThread in s.flags and emulatedThreadVars():
     accessThreadLocalVar(p, s)
-    sLoc = "NimTV->" & sLoc
+    sLoc = "NimTV_->" & sLoc
 
   c.visitorFrmt = "#nimGCvisit((void*)$1, 0);$n"
   c.p = p

compiler/ccgtypes.nim

@@ -22,12 +22,10 @@ proc isKeyword(w: PIdent): bool =
      ord(wInline): return true
   else: return false
 
-proc mangleField(name: PIdent): string =
+proc mangleField(m: BModule; name: PIdent): string =
   result = mangle(name.s)
-  if isKeyword(name):
+  if isKeyword(name) or m.g.config.cppDefines.contains(result):
-    result[0] = result[0].toUpperAscii
+    result.add "_0"
-    # Mangling makes everything lowercase,
-    # but some identifiers are C keywords
 
 when false:
   proc hashOwner(s: PSym): SigHash =
@@ -67,55 +65,50 @@ proc idOrSig(m: BModule; s: PSym): Rope =
 proc mangleName(m: BModule; s: PSym): Rope =
   result = s.loc.r
   if result == nil:
-    let keepOrigName = s.kind in skLocalVars - {skForVar} and
-      {sfFromGeneric, sfGlobal, sfShadowed, sfGenSym} * s.flags == {} and
-      not isKeyword(s.name)
-    # Even with all these inefficient checks, the bootstrap
-    # time is actually improved. This is probably because so many
-    # rope concatenations are now eliminated.
-    #
-    # sfFromGeneric is needed in order to avoid multiple
-    # definitions of certain variables generated in transf with
-    # names such as:
-    # `r`, `res`
-    # I need to study where these come from.
-    #
-    # about sfShadowed:
-    # consider the following Nim code:
-    #   var x = 10
-    #   block:
-    #     var x = something(x)
-    # The generated C code will be:
-    #   NI x;
-    #   x = 10;
-    #   {
-    #     NI x;
-    #     x = something(x); // Oops, x is already shadowed here
-    #   }
-    # Right now, we work-around by not keeping the original name
-    # of the shadowed variable, but we can do better - we can
-    # create an alternative reference to it in the outer scope and
-    # use that in the inner scope.
-    #
-    # about isCKeyword:
-    # Nim variable names can be C keywords.
-    # We need to avoid such names in the generated code.
-    #
-    # about sfGlobal:
-    # This seems to be harder - a top level extern variable from
-    # another modules can have the same name as a local one.
-    # Maybe we should just implement sfShadowed for them too.
-    #
-    # about skForVar:
-    # These are not properly scoped now - we need to add blocks
-    # around for loops in transf
     result = s.name.s.mangle.rope
-    if keepOrigName:
+    add(result, m.idOrSig(s))
-      result.add "0"
-    else:
-      add(result, m.idOrSig(s))
     s.loc.r = result
+    writeMangledName(m.ndi, s)
 
+proc mangleParamName(m: BModule; s: PSym): Rope =
+  ## we cannot use 'sigConflicts' here since we have a BModule, not a BProc.
+  ## Fortunately C's scoping rules are sane enough so that that doesn't
+  ## cause any trouble.
+  result = s.loc.r
+  if result == nil:
+    var res = s.name.s.mangle
+    if isKeyword(s.name) or m.g.config.cppDefines.contains(res):
+      res.add "_0"
+    result = res.rope
+    s.loc.r = result
+    writeMangledName(m.ndi, s)
+
+proc mangleLocalName(p: BProc; s: PSym): Rope =
+  assert s.kind in skLocalVars+{skTemp}
+  assert sfGlobal notin s.flags
+  result = s.loc.r
+  if result == nil:
+    var key = s.name.s.mangle
+    shallow(key)
+    let counter = p.sigConflicts.getOrDefault(key)
+    result = key.rope
+    if s.kind == skTemp:
+      # speed up conflict search for temps (these are quite common):
+      if counter != 0: result.add "_" & rope(counter+1)
+    elif counter != 0 or isKeyword(s.name) or p.module.g.config.cppDefines.contains(key):
+      result.add "_" & rope(counter+1)
+    p.sigConflicts.inc(key)
+    s.loc.r = result
+    if s.kind != skTemp: writeMangledName(p.module.ndi, s)
+
+proc scopeMangledParam(p: BProc; param: PSym) =
+  ## parameter generation only takes BModule, not a BProc, so we have to
+  ## remember these parameter names are already in scope to be able to
+  ## generate unique identifiers reliably (consider that ``var a = a`` is
+  ## even an idiom in Nim).
+  var key = param.name.s.mangle
+  shallow(key)
+  p.sigConflicts.inc(key)
 
 const
   irrelevantForBackend = {tyGenericBody, tyGenericInst, tyGenericInvocation,
@@ -393,7 +386,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
     var param = t.n.sons[i].sym
     if isCompileTimeOnly(param.typ): continue
     if params != nil: add(params, ~", ")
-    fillLoc(param.loc, locParam, param.typ, mangleName(m, param),
+    fillLoc(param.loc, locParam, param.typ, mangleParamName(m, param),
             param.paramStorageLoc)
     if ccgIntroducedPtr(param):
       add(params, getTypeDescWeak(m, param.typ, check))
@@ -436,12 +429,12 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
   else: add(params, ")")
   params = "(" & params
 
-proc mangleRecFieldName(field: PSym, rectype: PType): Rope =
+proc mangleRecFieldName(m: BModule; field: PSym, rectype: PType): Rope =
   if (rectype.sym != nil) and
       ({sfImportc, sfExportc} * rectype.sym.flags != {}):
     result = field.loc.r
   else:
-    result = rope(mangleField(field.name))
+    result = rope(mangleField(m, field.name))
   if result == nil: internalError(field.info, "mangleRecFieldName")
 
 proc genRecordFieldsAux(m: BModule, n: PNode,
@@ -480,7 +473,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
     let field = n.sym
     if field.typ.kind == tyVoid: return
     #assert(field.ast == nil)
-    let sname = mangleRecFieldName(field, rectype)
+    let sname = mangleRecFieldName(m, field, rectype)
     let ae = if accessExpr != nil: "$1.$2" % [accessExpr, sname]
              else: sname
     fillLoc(field.loc, locField, field.typ, ae, OnUnknown)
@@ -1103,13 +1096,11 @@ proc genTypeInfo(m: BModule, t: PType): Rope =
     discard cgsym(m, "TNimType")
     discard cgsym(m, "TNimNode")
     addf(m.s[cfsVars], "extern TNimType $1;$n", [result])
-    #return "(&".rope & result & ")".rope
-    #result = "NTI$1" % [rope($sig)]
     # also store in local type section:
     m.typeInfoMarker[sig] = result
     return "(&".rope & result & ")".rope
 
-  result = "NTI$1" % [rope($sig)]
+  result = "NTI$1_" % [rope($sig)]
   m.typeInfoMarker[sig] = result
 
   let owner = t.skipTypes(typedescPtrs).owner.getModule

compiler/ccgutils.nim

@@ -164,28 +164,50 @@ proc makeSingleLineCString*(s: string): string =
   result.add('\"')
 
 proc mangle*(name: string): string =
-  ## Lowercases the given name and manges any non-alphanumeric characters
-  ## so they are represented as `HEX____`. If the name starts with a number,
-  ## `N` is prepended
   result = newStringOfCap(name.len)
-  case name[0]
+  var start = 0
-  of Letters:
+  if name[0] in Digits:
-    result.add(name[0])
+    result.add("X" & name[0])
-  of Digits:
+    start = 1
-    result.add("N" & name[0])
+  var requiresUnderscore = false
-  else:
+  template special(x) =
-    result = "HEX" & toHex(ord(name[0]), 2)
+    result.add x
-  for i in 1..(name.len-1):
+    requiresUnderscore = true
+  for i in start..(name.len-1):
     let c = name[i]
     case c
-    of 'A'..'Z':
+    of 'a'..'z', '0'..'9', 'A'..'Z':
-      add(result, c.toLowerAscii)
-    of '_':
-      discard
-    of 'a'..'z', '0'..'9':
       add(result, c)
+    of '_':
+      # we generate names like 'foo_9' for scope disambiguations and so
+      # disallow this here:
+      if i < name.len-1 and name[i] in Digits:
+        discard
+      else:
+        add(result, c)
+    of '$': special "dollar"
+    of '%': special "percent"
+    of '&': special "amp"
+    of '^': special "roof"
+    of '!': special "emark"
+    of '?': special "qmark"
+    of '*': special "star"
+    of '+': special "plus"
+    of '-': special "minus"
+    of '/': special "slash"
+    of '=': special "eq"
+    of '<': special "lt"
+    of '>': special "gt"
+    of '~': special "tilde"
+    of ':': special "colon"
+    of '.': special "dot"
+    of '@': special "at"
+    of '|': special "bar"
     else:
-      add(result, "HEX" & toHex(ord(c), 2))
+      add(result, "X" & toHex(ord(c), 2))
+      requiresUnderscore = true
+  if requiresUnderscore:
+    result.add "_"
 
 proc makeLLVMString*(s: string): Rope =
   const MaxLineLength = 64

compiler/cgen.nim

@@ -14,7 +14,7 @@ import
   nversion, nimsets, msgs, securehash, bitsets, idents, lists, types,
   ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth,
   condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
-  lowerings, semparallel, tables
+  lowerings, semparallel, tables, sets, ndi
 
 import strutils except `%` # collides with ropes.`%`
 
@@ -216,7 +216,7 @@ proc genLineDir(p: BProc, t: PNode) =
       {optLineTrace, optStackTrace}) and
       (p.prc == nil or sfPure notin p.prc.flags) and tt.info.fileIndex >= 0:
     if freshLineInfo(p, tt.info):
-      linefmt(p, cpsStmts, "nimln($1, $2);$n",
+      linefmt(p, cpsStmts, "nimln_($1, $2);$n",
               line.rope, tt.info.quotedFilename)
 
 proc postStmtActions(p: BProc) {.inline.} =
@@ -338,7 +338,7 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) =
 
 proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
   inc(p.labels)
-  result.r = "LOC" & rope(p.labels)
+  result.r = "T" & rope(p.labels) & "_"
   linefmt(p, cpsLocals, "$1 $2;$n", getTypeDesc(p.module, t), result.r)
   result.k = locTemp
   result.t = t
@@ -347,12 +347,12 @@ proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
   constructLoc(p, result, not needsInit)
 
 proc initGCFrame(p: BProc): Rope =
-  if p.gcFrameId > 0: result = "struct {$1} GCFRAME;$n" % [p.gcFrameType]
+  if p.gcFrameId > 0: result = "struct {$1} GCFRAME_;$n" % [p.gcFrameType]
 
 proc deinitGCFrame(p: BProc): Rope =
   if p.gcFrameId > 0:
     result = ropecg(p.module,
-                    "if (((NU)&GCFRAME) < 4096) #nimGCFrame(&GCFRAME);$n")
+                    "if (((NU)&GCFRAME_) < 4096) #nimGCFrame(&GCFRAME_);$n")
 
 proc localDebugInfo(p: BProc, s: PSym) =
   if {optStackTrace, optEndb} * p.options != {optStackTrace, optEndb}: return
@@ -361,7 +361,7 @@ proc localDebugInfo(p: BProc, s: PSym) =
   var a = "&" & s.loc.r
   if s.kind == skParam and ccgIntroducedPtr(s): a = s.loc.r
   lineF(p, cpsInit,
-       "FR.s[$1].address = (void*)$3; FR.s[$1].typ = $4; FR.s[$1].name = $2;$n",
+       "FR_.s[$1].address = (void*)$3; FR_.s[$1].typ = $4; FR_.s[$1].name = $2;$n",
        [p.maxFrameLen.rope, makeCString(normalize(s.name.s)), a,
         genTypeInfo(p.module, s.loc.t)])
   inc(p.maxFrameLen)
@@ -369,7 +369,7 @@ proc localDebugInfo(p: BProc, s: PSym) =
 
 proc localVarDecl(p: BProc; s: PSym): Rope =
   if s.loc.k == locNone:
-    fillLoc(s.loc, locLocalVar, s.typ, mangleName(p.module, s), OnStack)
+    fillLoc(s.loc, locLocalVar, s.typ, mangleLocalName(p, s), OnStack)
     if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy)
   result = getTypeDesc(p.module, s.typ)
   if s.constraint.isNil:
@@ -434,6 +434,7 @@ proc assignGlobalVar(p: BProc, s: PSym) =
 
 proc assignParam(p: BProc, s: PSym) =
   assert(s.loc.r != nil)
+  scopeMangledParam(p, s)
   localDebugInfo(p, s)
 
 proc fillProcLoc(m: BModule; sym: PSym) =
@@ -442,7 +443,7 @@ proc fillProcLoc(m: BModule; sym: PSym) =
 
 proc getLabel(p: BProc): TLabel =
   inc(p.labels)
-  result = "LA" & rope(p.labels)
+  result = "LA" & rope(p.labels) & "_"
 
 proc fixLabel(p: BProc, labl: TLabel) =
   lineF(p, cpsStmts, "$1: ;$n", [labl])
@@ -520,7 +521,7 @@ proc mangleDynLibProc(sym: PSym): Rope =
     # NOTE: sym.loc.r is the external name!
     result = rope(sym.name.s)
   else:
-    result = "Dl_$1" % [rope(sym.id)]
+    result = "Dl_$1_" % [rope(sym.id)]
 
 proc symInDynamicLib(m: BModule, sym: PSym) =
   var lib = sym.annex
@@ -608,11 +609,11 @@ proc initFrame(p: BProc, procname, filename: Rope): Rope =
   discard cgsym(p.module, "nimFrame")
   if p.maxFrameLen > 0:
     discard cgsym(p.module, "VarSlot")
-    result = rfmt(nil, "\tnimfrs($1, $2, $3, $4)$N",
+    result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4)$N",
                   procname, filename, p.maxFrameLen.rope,
                   p.blocks[0].frameLen.rope)
   else:
-    result = rfmt(nil, "\tnimfr($1, $2)$N", procname, filename)
+    result = rfmt(nil, "\tnimfr_($1, $2)$N", procname, filename)
 
 proc deinitFrame(p: BProc): Rope =
   result = rfmt(p.module, "\t#popFrame();$n")
@@ -707,7 +708,7 @@ proc genProcAux(m: BModule, prc: PSym) =
     if p.beforeRetNeeded: add(generatedProc, "{")
     add(generatedProc, p.s(cpsInit))
     add(generatedProc, p.s(cpsStmts))
-    if p.beforeRetNeeded: add(generatedProc, ~"\t}BeforeRet: ;$n")
+    if p.beforeRetNeeded: add(generatedProc, ~"\t}BeforeRet_: ;$n")
     add(generatedProc, deinitGCFrame(p))
     if optStackTrace in prc.options: add(generatedProc, deinitFrame(p))
     add(generatedProc, returnStmt)
@@ -846,7 +847,8 @@ proc genVarPrototype(m: BModule, sym: PSym) =
   genVarPrototypeAux(m, sym)
 
 proc addIntTypes(result: var Rope) {.inline.} =
-  addf(result, "#define NIM_INTBITS $1" & tnl, [
+  addf(result, "#define NIM_NEW_MANGLING_RULES" & tnl &
+               "#define NIM_INTBITS $1" & tnl, [
     platform.CPU[targetCPU].intSize.rope])
 
 proc getCopyright(cfile: Cfile): Rope =
@@ -1058,7 +1060,7 @@ proc genInitCode(m: BModule) =
       var procname = makeCString(m.module.name.s)
       add(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename))
     else:
-      add(prc, ~"\tTFrame FR; FR.len = 0;$N")
+      add(prc, ~"\tTFrame FR_; FR_.len = 0;$N")
 
   add(prc, genSectionStart(cpsInit))
   add(prc, m.preInitProc.s(cpsInit))
@@ -1123,7 +1125,7 @@ proc initProcOptions(m: BModule): TOptions =
 
 proc rawNewModule(g: BModuleList; module: PSym, filename: string): BModule =
   new(result)
-  result.tmpBase = rope("T" & $hashOwner(module) & "_")
+  result.tmpBase = rope("TM" & $hashOwner(module) & "_")
   initLinkedList(result.headerFiles)
   result.declaredThings = initIntSet()
   result.declaredProtos = initIntSet()
@@ -1150,6 +1152,9 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: string): BModule =
     incl result.flags, preventStackTrace
     excl(result.preInitProc.options, optStackTrace)
     excl(result.postInitProc.options, optStackTrace)
+  let ndiName = if optCDebug in gGlobalOptions: changeFileExt(completeCFilePath(filename), "ndi")
+                else: ""
+  open(result.ndi, ndiName)
 
 proc nullify[T](arr: var T) =
   for i in low(arr)..high(arr):
@@ -1212,16 +1217,16 @@ proc newModule(g: BModuleList; module: PSym): BModule =
     if (sfDeadCodeElim in module.flags):
       internalError("added pending module twice: " & module.filename)
 
-template injectG() {.dirty.} =
+template injectG(config) {.dirty.} =
   if graph.backend == nil:
-    graph.backend = newModuleList()
+    graph.backend = newModuleList(config)
   let g = BModuleList(graph.backend)
 
 proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
-  injectG()
+  injectG(graph.config)
   result = newModule(g, module)
   if optGenIndex in gGlobalOptions and g.generatedHeader == nil:
-    let f = if headerFile.len > 0: headerFile else: gProjectFull
+    let f = if graph.config.headerFile.len > 0: graph.config.headerFile else: gProjectFull
     g.generatedHeader = rawNewModule(g, module,
       changeFileExt(completeCFilePath(f), hExt))
     incl g.generatedHeader.flags, isHeaderFile
@@ -1258,7 +1263,7 @@ proc getCFile(m: BModule): string =
   result = changeFileExt(completeCFilePath(m.cfilename.withPackageName), ext)
 
 proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext =
-  injectG()
+  injectG(graph.config)
   assert optSymbolFiles in gGlobalOptions
   var m = newModule(g, module)
   readMergeInfo(getCFile(m), m)
@@ -1341,6 +1346,7 @@ proc writeModule(m: BModule, pending: bool) =
     var cf = Cfile(cname: cfile, obj: completeCFilePath(toObjFile(cfile)), flags: {})
     if not existsFile(cf.obj): cf.flags = {CfileFlag.Cached}
     addFileToCompile(cf)
+  close(m.ndi)
 
 proc updateCachedModule(m: BModule) =
   let cfile = getCFile(m)
@@ -1373,11 +1379,12 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     for i in 0..sonsLen(disp)-1: genProcAux(m, disp.sons[i].sym)
     genMainProc(m)
 
-proc cgenWriteModules*(backend: RootRef) =
+proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
   let g = BModuleList(backend)
   # we need to process the transitive closure because recursive module
   # deps are allowed (and the system module is processed in the wrong
   # order anyway)
+  g.config = config
   if g.generatedHeader != nil: finishModule(g.generatedHeader)
   while g.forwardedProcsCounter > 0:
     for m in cgenModules(g):

compiler/cgendata.nim

@@ -11,7 +11,7 @@
 
 import
   ast, astalgo, ropes, passes, options, intsets, lists, platform, sighashes,
-  tables
+  tables, ndi
 
 from msgs import TLineInfo
 
@@ -56,7 +56,7 @@ type
   BProc* = ref TCProc
   TBlock*{.final.} = object
     id*: int                  # the ID of the label; positive means that it
-    label*: Rope             # generated text for the label
+    label*: Rope              # generated text for the label
                               # nil if label is not used
     sections*: TCProcSections # the code beloging
     isLoop*: bool             # whether block is a loop
@@ -76,7 +76,7 @@ type
                               # leaving such scopes by raise or by return must
                               # execute any applicable finally blocks
     finallySafePoints*: seq[Rope]  # For correctly cleaning up exceptions when
-                                    # using return in finally statements
+                                   # using return in finally statements
     labels*: Natural          # for generating unique labels in the C proc
     blocks*: seq[TBlock]      # nested blocks
     breakIdx*: int            # the block that will be exited
@@ -92,6 +92,7 @@ type
                               # (yes, C++ is weird like that)
     gcFrameId*: Natural       # for the GC stack marking
     gcFrameType*: Rope        # the struct {} we put the GC markers into
+    sigConflicts*: CountTable[string]
 
   TTypeSeq* = seq[PType]
   TypeCache* = Table[SigHash, Rope]
@@ -115,6 +116,7 @@ type
     breakPointId*: int
     breakpoints*: Rope # later the breakpoints are inserted into the main proc
     typeInfoMarker*: TypeCache
+    config*: ConfigRef
 
   TCGen = object of TPassContext # represents a C source file
     s*: TCFileSections        # sections of the C file
@@ -144,6 +146,7 @@ type
     injectStmt*: Rope
     sigConflicts*: CountTable[SigHash]
     g*: BModuleList
+    ndi*: NdiFile
 
 proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # section in the current block
@@ -162,9 +165,10 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   newSeq(result.blocks, 1)
   result.nestedTryStmts = @[]
   result.finallySafePoints = @[]
+  result.sigConflicts = initCountTable[string]()
 
-proc newModuleList*(): BModuleList =
+proc newModuleList*(config: ConfigRef): BModuleList =
-  BModuleList(modules: @[], typeInfoMarker: initTable[SigHash, Rope]())
+  BModuleList(modules: @[], typeInfoMarker: initTable[SigHash, Rope](), config: config)
 
 iterator cgenModules*(g: BModuleList): BModule =
   for i in 0..high(g.modules):

compiler/commands.nim

@@ -47,7 +47,8 @@ type
     passPP                    # preprocessor called processCommand()
 
 proc processCommand*(switch: string, pass: TCmdLinePass)
-proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo)
+proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
+                    config: ConfigRef = nil)
 
 # implementation
 
@@ -312,7 +313,8 @@ proc dynlibOverride(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
     expectArg(switch, arg, pass, info)
     options.inclDynlibOverride(arg)
 
-proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
+proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
+                   config: ConfigRef = nil) =
   var
     theOS: TSystemOS
     cpu: TSystemCPU
@@ -509,10 +511,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
     else: localError(info, errGuiConsoleOrLibExpectedButXFound, arg)
   of "passc", "t":
     expectArg(switch, arg, pass, info)
-    if pass in {passCmd2, passPP}: extccomp.addCompileOption(arg)
+    if pass in {passCmd2, passPP}: extccomp.addCompileOptionCmd(arg)
   of "passl", "l":
     expectArg(switch, arg, pass, info)
-    if pass in {passCmd2, passPP}: extccomp.addLinkOption(arg)
+    if pass in {passCmd2, passPP}: extccomp.addLinkOptionCmd(arg)
   of "cincludes":
     expectArg(switch, arg, pass, info)
     if pass in {passCmd2, passPP}: cIncludes.add arg.processPath(info)
@@ -523,7 +525,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
     expectArg(switch, arg, pass, info)
     if pass in {passCmd2, passPP}: cLinkedLibs.add arg.processPath(info)
   of "header":
-    headerFile = arg
+    if config != nil: config.headerFile = arg
     incl(gGlobalOptions, optGenIndex)
   of "index":
     processOnOffSwitchG({optGenIndex}, arg, pass, info)
@@ -646,6 +648,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optNoCppExceptions)
     defineSymbol("noCppExceptions")
+  of "cppdefine":
+    expectArg(switch, arg, pass, info)
+    if config != nil:
+      config.cppDefine(arg)
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(switch, arg)
     else: invalidCmdLineOption(pass, switch, info)

compiler/condsyms.nim

@@ -101,3 +101,4 @@ proc initDefines*() =
   defineSymbol("nimImmediateDeprecated")
   defineSymbol("nimNewShiftOps")
   defineSymbol("nimDistros")
+  defineSymbol("nimHasCppDefine")

compiler/docgen.nim

@@ -209,26 +209,26 @@ proc getPlainDocstring(n: PNode): string =
         result = getPlainDocstring(n.sons[i])
         if result.len > 0: return
 
+when false:
+  proc findDocComment(n: PNode): PNode =
+    if n == nil: return nil
+    if not isNil(n.comment) and startsWith(n.comment, "##"): return n
+    for i in countup(0, safeLen(n)-1):
+      result = findDocComment(n.sons[i])
+      if result != nil: return
 
-proc findDocComment(n: PNode): PNode =
+  proc extractDocComment*(s: PSym, d: PDoc = nil): string =
-  if n == nil: return nil
+    let n = findDocComment(s.ast)
-  if not isNil(n.comment) and startsWith(n.comment, "##"): return n
+    result = ""
-  for i in countup(0, safeLen(n)-1):
+    if not n.isNil:
-    result = findDocComment(n.sons[i])
+      if not d.isNil:
-    if result != nil: return
+        var dummyHasToc: bool
-
+        renderRstToOut(d[], parseRst(n.comment, toFilename(n.info),
-proc extractDocComment*(s: PSym, d: PDoc = nil): string =
+                                     toLinenumber(n.info), toColumn(n.info),
-  let n = findDocComment(s.ast)
+                                     dummyHasToc, d.options + {roSkipPounds}),
-  result = ""
+                       result)
-  if not n.isNil:
+      else:
-    if not d.isNil:
+        result = n.comment.substr(2).replace("\n##", "\n").strip
-      var dummyHasToc: bool
-      renderRstToOut(d[], parseRst(n.comment, toFilename(n.info),
-                                   toLinenumber(n.info), toColumn(n.info),
-                                   dummyHasToc, d.options + {roSkipPounds}),
-                     result)
-    else:
-      result = n.comment.substr(2).replace("\n##", "\n").strip
 
 proc isVisible(n: PNode): bool =
   result = false

compiler/extccomp.nim

@@ -392,6 +392,8 @@ type
 var
   externalToLink: TLinkedList # files to link in addition to the file
                               # we compiled
+  linkOptionsCmd: string = ""
+  compileOptionsCmd: seq[string] = @[]
   linkOptions: string = ""
   compileOptions: string = ""
   ccompilerpath: string = ""
@@ -450,6 +452,12 @@ proc addCompileOption*(option: string) =
   if strutils.find(compileOptions, option, 0) < 0:
     addOpt(compileOptions, option)
 
+proc addLinkOptionCmd*(option: string) =
+  addOpt(linkOptionsCmd, option)
+
+proc addCompileOptionCmd*(option: string) =
+  compileOptionsCmd.add(option)
+
 proc initVars*() =
   # we need to define the symbol here, because ``CC`` may have never been set!
   for i in countup(low(CC), high(CC)): undefSymbol(CC[i].name)
@@ -524,6 +532,10 @@ proc add(s: var string, many: openArray[string]) =
 
 proc cFileSpecificOptions(cfilename: string): string =
   result = compileOptions
+  for option in compileOptionsCmd:
+    if strutils.find(result, option, 0) < 0:
+      addOpt(result, option)
+
   var trunk = splitFile(cfilename).name
   if optCDebug in gGlobalOptions:
     var key = trunk & ".debug"
@@ -544,7 +556,7 @@ proc getCompileOptions: string =
   result = cFileSpecificOptions("__dummy__")
 
 proc getLinkOptions: string =
-  result = linkOptions
+  result = linkOptions & " " & linkOptionsCmd & " "
   for linkedLib in items(cLinkedLibs):
     result.add(CC[cCompiler].linkLibCmd % linkedLib.quoteShell)
   for libDir in items(cLibs):

compiler/installer.ini

@@ -46,7 +46,7 @@ Start: "doc/html/overview.html"
 
 
 [Other]
-Files: "readme.txt;install.txt;contributors.txt;copying.txt"
+Files: "readme.txt;copying.txt"
 Files: "makefile"
 Files: "koch.nim"
 Files: "install_nimble.nims"
@@ -94,15 +94,17 @@ Files: "bin/vccexe.exe"
 
 Files: "koch.exe"
 Files: "finish.exe"
+Files: "downloader.exe"
+
 ; Files: "dist/mingw"
 Files: r"tools\start.bat"
 BinPath: r"bin;dist\mingw\bin;dist"
 
 ;           Section | dir | zipFile | size hint (in KB) | url | exe start menu entry
-Download: r"Documentation|doc|docs.zip|13824|http://nim-lang.org/download/docs-${version}.zip|overview.html"
+Download: r"Documentation|doc|docs.zip|13824|https://nim-lang.org/download/docs-${version}.zip|overview.html"
-Download: r"C Compiler (MingW)|dist|mingw.zip|82944|http://nim-lang.org/download/${mingw}.zip"
+Download: r"C Compiler (MingW)|dist|mingw.zip|82944|https://nim-lang.org/download/${mingw}.zip"
-Download: r"Support DLLs|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip"
+Download: r"Support DLLs|bin|nim_dlls.zip|479|https://nim-lang.org/download/dlls.zip"
-Download: r"Aporia Text Editor|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe"
+Download: r"Aporia Text Editor|dist|aporia.zip|97997|https://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe"
 ; for now only NSIS supports optional downloads
 
 [WinBin]

compiler/main.nim

@@ -72,7 +72,7 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) =
   #registerPass(cleanupPass())
 
   compileProject(graph, cache)
-  cgenWriteModules(graph.backend)
+  cgenWriteModules(graph.backend, graph.config)
   if gCmd != cmdRun:
     let proj = changeFileExt(gProjectFull, "")
     extccomp.callCCompiler(proj)
@@ -294,4 +294,4 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
 
   resetAttributes()
 
-proc mainCommand*() = mainCommand(newModuleGraph(), newIdentCache())
+proc mainCommand*() = mainCommand(newModuleGraph(newConfigRef()), newIdentCache())

compiler/modulegraphs.nim

@@ -25,7 +25,7 @@
 ## - Its dependent module stays the same.
 ##
 
-import ast, intsets, tables
+import ast, intsets, tables, options
 
 type
   ModuleGraph* = ref object
@@ -39,16 +39,21 @@ type
     importStack*: seq[int32]  # The current import stack. Used for detecting recursive
                               # module dependencies.
     backend*: RootRef # minor hack so that a backend can extend this easily
+    config*: ConfigRef
 
 {.this: g.}
 
-proc newModuleGraph*(): ModuleGraph =
+proc newModuleGraph*(config: ConfigRef = nil): ModuleGraph =
   result = ModuleGraph()
   initStrTable(result.packageSyms)
   result.deps = initIntSet()
   result.modules = @[]
   result.importStack = @[]
   result.inclToMod = initTable[int32, int32]()
+  if config.isNil:
+    result.config = newConfigRef()
+  else:
+    result.config = config
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(packageSyms)

compiler/msgs.nim

@@ -12,7 +12,7 @@ import
 
 type
   TMsgKind* = enum
-    errUnknown, errIllFormedAstX, errInternal, errCannotOpenFile, errGenerated,
+    errUnknown, errInternal, errIllFormedAstX, errCannotOpenFile, errGenerated,
     errXCompilerDoesNotSupportCpp, errStringLiteralExpected,
     errIntLiteralExpected, errInvalidCharacterConstant,
     errClosingTripleQuoteExpected, errClosingQuoteExpected,
@@ -135,8 +135,8 @@ type
 const
   MsgKindToStr*: array[TMsgKind, string] = [
     errUnknown: "unknown error",
-    errIllFormedAstX: "illformed AST: $1",
     errInternal: "internal error: $1",
+    errIllFormedAstX: "illformed AST: $1",
     errCannotOpenFile: "cannot open \'$1\'",
     errGenerated: "$1",
     errXCompilerDoesNotSupportCpp: "\'$1\' compiler does not support C++",

compiler/ndi.nim

@@ -0,0 +1,40 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements the generation of ``.ndi`` files for better debugging
+## support of Nim code. "ndi" stands for "Nim debug info".
+
+import ast, msgs, ropes
+
+type
+  NdiFile* = object
+    enabled: bool
+    f: File
+    buf: string
+
+proc doWrite(f: var NdiFile; s: PSym) =
+  f.buf.setLen 0
+  f.buf.add s.info.line.int
+  f.buf.add "\t"
+  f.buf.add s.info.col.int
+  f.f.write(s.name.s, "\t")
+  f.f.writeRope(s.loc.r)
+  f.f.writeLine("\t", s.info.toFullPath, "\t", f.buf)
+
+template writeMangledName*(f: NdiFile; s: PSym) =
+  if f.enabled: doWrite(f, s)
+
+proc open*(f: var NdiFile; filename: string) =
+  f.enabled = filename.len > 0
+  if f.enabled:
+    f.f = open(filename, fmWrite, 8000)
+    f.buf = newStringOfCap(20)
+
+proc close*(f: var NdiFile) =
+  if f.enabled: close(f.f)

compiler/nim.nim

@@ -37,7 +37,7 @@ proc prependCurDir(f: string): string =
   else:
     result = f
 
-proc handleCmdLine(cache: IdentCache) =
+proc handleCmdLine(cache: IdentCache; config: ConfigRef) =
   if paramCount() == 0:
     writeCommandLineUsage()
   else:
@@ -59,22 +59,22 @@ proc handleCmdLine(cache: IdentCache) =
       gProjectName = p.name
     else:
       gProjectPath = canonicalizePath getCurrentDir()
-    loadConfigs(DefaultConfig) # load all config files
+    loadConfigs(DefaultConfig, config) # load all config files
     let scriptFile = gProjectFull.changeFileExt("nims")
     if fileExists(scriptFile):
-      runNimScript(cache, scriptFile, freshDefines=false)
+      runNimScript(cache, scriptFile, freshDefines=false, config)
       # 'nim foo.nims' means to just run the NimScript file and do nothing more:
       if scriptFile == gProjectFull: return
     elif fileExists(gProjectPath / "config.nims"):
       # directory wide NimScript file
-      runNimScript(cache, gProjectPath / "config.nims", freshDefines=false)
+      runNimScript(cache, gProjectPath / "config.nims", freshDefines=false, config)
     # now process command line arguments again, because some options in the
     # command line can overwite the config file's settings
     extccomp.initVars()
     processCmdLine(passCmd2, "")
     if options.command == "":
       rawMessage(errNoCommand, command)
-    mainCommand(newModuleGraph(), cache)
+    mainCommand(newModuleGraph(config), cache)
     if optHints in gOptions and hintGCStats in gNotes: echo(GC_getStatistics())
     #echo(GC_getStatistics())
     if msgs.gErrorCounter == 0:
@@ -118,5 +118,5 @@ when compileOption("gc", "v2") or compileOption("gc", "refc"):
 condsyms.initDefines()
 
 when not defined(selftest):
-  handleCmdLine(newIdentCache())
+  handleCmdLine(newIdentCache(), newConfigRef())
   msgQuit(int8(msgs.gErrorCounter > 0))

compiler/nimconf.nim

@@ -21,37 +21,37 @@ proc ppGetTok(L: var TLexer, tok: var TToken) =
   rawGetTok(L, tok)
   while tok.tokType in {tkComment}: rawGetTok(L, tok)
 
-proc parseExpr(L: var TLexer, tok: var TToken): bool
+proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool
-proc parseAtom(L: var TLexer, tok: var TToken): bool =
+proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
   if tok.tokType == tkParLe:
     ppGetTok(L, tok)
-    result = parseExpr(L, tok)
+    result = parseExpr(L, tok, config)
     if tok.tokType == tkParRi: ppGetTok(L, tok)
     else: lexMessage(L, errTokenExpected, "\')\'")
   elif tok.ident.id == ord(wNot):
     ppGetTok(L, tok)
-    result = not parseAtom(L, tok)
+    result = not parseAtom(L, tok, config)
   else:
     result = isDefined(tok.ident)
     ppGetTok(L, tok)
 
-proc parseAndExpr(L: var TLexer, tok: var TToken): bool =
+proc parseAndExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
-  result = parseAtom(L, tok)
+  result = parseAtom(L, tok, config)
   while tok.ident.id == ord(wAnd):
     ppGetTok(L, tok)          # skip "and"
-    var b = parseAtom(L, tok)
+    var b = parseAtom(L, tok, config)
     result = result and b
 
-proc parseExpr(L: var TLexer, tok: var TToken): bool =
+proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
-  result = parseAndExpr(L, tok)
+  result = parseAndExpr(L, tok, config)
   while tok.ident.id == ord(wOr):
     ppGetTok(L, tok)          # skip "or"
-    var b = parseAndExpr(L, tok)
+    var b = parseAndExpr(L, tok, config)
     result = result or b
 
-proc evalppIf(L: var TLexer, tok: var TToken): bool =
+proc evalppIf(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
   ppGetTok(L, tok)            # skip 'if' or 'elif'
-  result = parseExpr(L, tok)
+  result = parseExpr(L, tok, config)
   if tok.tokType == tkColon: ppGetTok(L, tok)
   else: lexMessage(L, errTokenExpected, "\':\'")
 
@@ -66,20 +66,20 @@ type
   TJumpDest = enum
     jdEndif, jdElseEndif
 
-proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest)
+proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: ConfigRef)
-proc doElse(L: var TLexer, tok: var TToken) =
+proc doElse(L: var TLexer, tok: var TToken; config: ConfigRef) =
   if high(condStack) < 0: lexMessage(L, errTokenExpected, "@if")
   ppGetTok(L, tok)
   if tok.tokType == tkColon: ppGetTok(L, tok)
-  if condStack[high(condStack)]: jumpToDirective(L, tok, jdEndif)
+  if condStack[high(condStack)]: jumpToDirective(L, tok, jdEndif, config)
 
-proc doElif(L: var TLexer, tok: var TToken) =
+proc doElif(L: var TLexer, tok: var TToken; config: ConfigRef) =
   if high(condStack) < 0: lexMessage(L, errTokenExpected, "@if")
-  var res = evalppIf(L, tok)
+  var res = evalppIf(L, tok, config)
-  if condStack[high(condStack)] or not res: jumpToDirective(L, tok, jdElseEndif)
+  if condStack[high(condStack)] or not res: jumpToDirective(L, tok, jdElseEndif, config)
   else: condStack[high(condStack)] = true
 
-proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest) =
+proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: ConfigRef) =
   var nestedIfs = 0
   while true:
     if tok.ident != nil and tok.ident.s == "@":
@@ -89,11 +89,11 @@ proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest) =
         inc(nestedIfs)
       of wElse:
         if dest == jdElseEndif and nestedIfs == 0:
-          doElse(L, tok)
+          doElse(L, tok, config)
           break
       of wElif:
         if dest == jdElseEndif and nestedIfs == 0:
-          doElif(L, tok)
+          doElif(L, tok, config)
           break
       of wEnd:
         if nestedIfs == 0:
@@ -108,16 +108,16 @@ proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest) =
     else:
       ppGetTok(L, tok)
 
-proc parseDirective(L: var TLexer, tok: var TToken) =
+proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef) =
   ppGetTok(L, tok)            # skip @
   case whichKeyword(tok.ident)
   of wIf:
     setLen(condStack, len(condStack) + 1)
-    let res = evalppIf(L, tok)
+    let res = evalppIf(L, tok, config)
     condStack[high(condStack)] = res
-    if not res: jumpToDirective(L, tok, jdElseEndif)
+    if not res: jumpToDirective(L, tok, jdElseEndif, config)
-  of wElif: doElif(L, tok)
+  of wElif: doElif(L, tok, config)
-  of wElse: doElse(L, tok)
+  of wElse: doElse(L, tok, config)
   of wEnd: doEnd(L, tok)
   of wWrite:
     ppGetTok(L, tok)
@@ -146,58 +146,58 @@ proc parseDirective(L: var TLexer, tok: var TToken) =
       ppGetTok(L, tok)
     else: lexMessage(L, errInvalidDirectiveX, tokToStr(tok))
 
-proc confTok(L: var TLexer, tok: var TToken) =
+proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef) =
   ppGetTok(L, tok)
   while tok.ident != nil and tok.ident.s == "@":
-    parseDirective(L, tok)    # else: give the token to the parser
+    parseDirective(L, tok, config)    # else: give the token to the parser
 
 proc checkSymbol(L: TLexer, tok: TToken) =
   if tok.tokType notin {tkSymbol..pred(tkIntLit), tkStrLit..tkTripleStrLit}:
     lexMessage(L, errIdentifierExpected, tokToStr(tok))
 
-proc parseAssignment(L: var TLexer, tok: var TToken) =
+proc parseAssignment(L: var TLexer, tok: var TToken; config: ConfigRef) =
   if tok.ident.s == "-" or tok.ident.s == "--":
-    confTok(L, tok)           # skip unnecessary prefix
+    confTok(L, tok, config)           # skip unnecessary prefix
   var info = getLineInfo(L, tok) # save for later in case of an error
   checkSymbol(L, tok)
   var s = tokToStr(tok)
-  confTok(L, tok)             # skip symbol
+  confTok(L, tok, config)             # skip symbol
   var val = ""
   while tok.tokType == tkDot:
     add(s, '.')
-    confTok(L, tok)
+    confTok(L, tok, config)
     checkSymbol(L, tok)
     add(s, tokToStr(tok))
-    confTok(L, tok)
+    confTok(L, tok, config)
   if tok.tokType == tkBracketLe:
     # BUGFIX: val, not s!
     # BUGFIX: do not copy '['!
-    confTok(L, tok)
+    confTok(L, tok, config)
     checkSymbol(L, tok)
     add(val, tokToStr(tok))
-    confTok(L, tok)
+    confTok(L, tok, config)
-    if tok.tokType == tkBracketRi: confTok(L, tok)
+    if tok.tokType == tkBracketRi: confTok(L, tok, config)
     else: lexMessage(L, errTokenExpected, "']'")
     add(val, ']')
   let percent = tok.ident != nil and tok.ident.s == "%="
   if tok.tokType in {tkColon, tkEquals} or percent:
     if len(val) > 0: add(val, ':')
-    confTok(L, tok)           # skip ':' or '=' or '%'
+    confTok(L, tok, config)           # skip ':' or '=' or '%'
     checkSymbol(L, tok)
     add(val, tokToStr(tok))
-    confTok(L, tok)           # skip symbol
+    confTok(L, tok, config)           # skip symbol
     while tok.ident != nil and tok.ident.s == "&":
-      confTok(L, tok)
+      confTok(L, tok, config)
       checkSymbol(L, tok)
       add(val, tokToStr(tok))
-      confTok(L, tok)
+      confTok(L, tok, config)
   if percent:
     processSwitch(s, strtabs.`%`(val, options.gConfigVars,
-                                {useEnvironment, useEmpty}), passPP, info)
+                                {useEnvironment, useEmpty}), passPP, info, config)
   else:
-    processSwitch(s, val, passPP, info)
+    processSwitch(s, val, passPP, info, config)
 
-proc readConfigFile(filename: string; cache: IdentCache) =
+proc readConfigFile(filename: string; cache: IdentCache; config: ConfigRef) =
   var
     L: TLexer
     tok: TToken
@@ -207,8 +207,8 @@ proc readConfigFile(filename: string; cache: IdentCache) =
     initToken(tok)
     openLexer(L, filename, stream, cache)
     tok.tokType = tkEof       # to avoid a pointless warning
-    confTok(L, tok)           # read in the first token
+    confTok(L, tok, config)           # read in the first token
-    while tok.tokType != tkEof: parseAssignment(L, tok)
+    while tok.tokType != tkEof: parseAssignment(L, tok, config)
     if len(condStack) > 0: lexMessage(L, errTokenExpected, "@end")
     closeLexer(L)
     rawMessage(hintConf, filename)
@@ -225,22 +225,22 @@ proc getSystemConfigPath(filename: string): string =
     if not existsFile(result): result = joinPath([p, "etc", filename])
     if not existsFile(result): result = "/etc/" & filename
 
-proc loadConfigs*(cfg: string; cache: IdentCache) =
+proc loadConfigs*(cfg: string; cache: IdentCache; config: ConfigRef = nil) =
   setDefaultLibpath()
 
   if optSkipConfigFile notin gGlobalOptions:
-    readConfigFile(getSystemConfigPath(cfg), cache)
+    readConfigFile(getSystemConfigPath(cfg), cache, config)
 
   if optSkipUserConfigFile notin gGlobalOptions:
-    readConfigFile(getUserConfigPath(cfg), cache)
+    readConfigFile(getUserConfigPath(cfg), cache, config)
 
   var pd = if gProjectPath.len > 0: gProjectPath else: getCurrentDir()
   if optSkipParentConfigFiles notin gGlobalOptions:
     for dir in parentDirs(pd, fromRoot=true, inclusive=false):
-      readConfigFile(dir / cfg, cache)
+      readConfigFile(dir / cfg, cache, config)
 
   if optSkipProjConfigFile notin gGlobalOptions:
-    readConfigFile(pd / cfg, cache)
+    readConfigFile(pd / cfg, cache, config)
 
     if gProjectName.len != 0:
       # new project wide config file:
@@ -251,8 +251,8 @@ proc loadConfigs*(cfg: string; cache: IdentCache) =
         projectConfig = changeFileExt(gProjectFull, "nimrod.cfg")
         if fileExists(projectConfig):
           rawMessage(warnDeprecated, projectConfig)
-      readConfigFile(projectConfig, cache)
+      readConfigFile(projectConfig, cache, config)
 
-proc loadConfigs*(cfg: string) =
+proc loadConfigs*(cfg: string; config: ConfigRef = nil) =
   # for backwards compatibility only.
-  loadConfigs(cfg, newIdentCache())
+  loadConfigs(cfg, newIdentCache(), config)

compiler/options.nim

@@ -102,6 +102,17 @@ type
     ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod,
     ideHighlight, ideOutline
 
+  ConfigRef* = ref object ## eventually all global configuration should be moved here
+    cppDefines*: HashSet[string]
+    headerFile*: string
+
+proc newConfigRef*(): ConfigRef =
+  result = ConfigRef(cppDefines: initSet[string](),
+    headerFile: "")
+
+proc cppDefine*(c: ConfigRef; define: string) =
+  c.cppDefines.incl define
+
 var
   gIdeCmd*: IdeCmd
 
@@ -122,7 +133,7 @@ var
   outFile*: string = ""
   docSeeSrcUrl*: string = ""  # if empty, no seeSrc will be generated. \
   # The string uses the formatting variables `path` and `line`.
-  headerFile*: string = ""
+  #headerFile*: string = ""
   gVerbosity* = 1             # how verbose the compiler is
   gNumberOfProcessors*: int   # number of processors
   gWholeProject*: bool        # for 'doc2': output any dependency

compiler/pragmas.nim

@@ -665,9 +665,14 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
       of wExportc:
         makeExternExport(sym, getOptionalStr(c, it, "$1"), it.info)
         incl(sym.flags, sfUsed) # avoid wrong hints
-      of wImportc: makeExternImport(sym, getOptionalStr(c, it, "$1"), it.info)
+      of wImportc:
+        let name = getOptionalStr(c, it, "$1")
+        cppDefine(c.graph.config, name)
+        makeExternImport(sym, name, it.info)
       of wImportCompilerProc:
-        processImportCompilerProc(sym, getOptionalStr(c, it, "$1"), it.info)
+        let name = getOptionalStr(c, it, "$1")
+        cppDefine(c.graph.config, name)
+        processImportCompilerProc(sym, name, it.info)
       of wExtern: setExternName(sym, expectStrLit(c, it), it.info)
       of wImmediate:
         if sym.kind in {skTemplate, skMacro}:
@@ -758,6 +763,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         processDynLib(c, it, sym)
       of wCompilerproc:
         noVal(it)           # compilerproc may not get a string!
+        cppDefine(c.graph.config, sym.name.s)
         if sfFromGeneric notin sym.flags: markCompilerProc(sym)
       of wProcVar:
         noVal(it)

compiler/scriptconfig.nim

@@ -25,7 +25,8 @@ proc listDirs(a: VmArgs, filter: set[PathComponent]) =
     if kind in filter: result.add path
   setResult(a, result)
 
-proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext =
+proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
+              config: ConfigRef = nil): PEvalContext =
   # For Nimble we need to export 'setupVM'.
   result = newCtx(module, cache)
   result.mode = emRepl
@@ -109,10 +110,13 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext
     let arg = a.getString 1
     if arg.len > 0:
       gProjectName = arg
+      let path =
+        if gProjectName.isAbsolute: gProjectName
+        else: gProjectPath / gProjectName
       try:
-        gProjectFull = canonicalizePath(gProjectPath / gProjectName)
+        gProjectFull = canonicalizePath(path)
       except OSError:
-        gProjectFull = gProjectName
+        gProjectFull = path
   cbconf getCommand:
     setResult(a, options.command)
   cbconf switch:
@@ -133,12 +137,15 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext
     gModuleOverrides[key] = val
   cbconf selfExe:
     setResult(a, os.getAppFilename())
+  cbconf cppDefine:
+    if config != nil:
+      options.cppDefine(config, a.getString(0))
 
 proc runNimScript*(cache: IdentCache; scriptName: string;
-                   freshDefines=true) =
+                   freshDefines=true; config: ConfigRef=nil) =
   passes.gIncludeFile = includeModule
   passes.gImportModule = importModule
-  let graph = newModuleGraph()
+  let graph = newModuleGraph(config)
   if freshDefines: initDefines()
 
   defineSymbol("nimscript")
@@ -150,7 +157,7 @@ proc runNimScript*(cache: IdentCache; scriptName: string;
 
   var m = graph.makeModule(scriptName)
   incl(m.flags, sfMainModule)
-  vm.globalCtx = setupVM(m, cache, scriptName)
+  vm.globalCtx = setupVM(m, cache, scriptName, config)
 
   graph.compileSystemModule(cache)
   discard graph.processModule(m, llStreamOpen(scriptName, fmRead), nil, cache)

compiler/semexprs.nim

@@ -1052,6 +1052,8 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   # work without now. template/tsymchoicefield doesn't like an early exit
   # here at all!
   #if isSymChoice(n.sons[1]): return
+  when defined(nimsuggest):
+    if gCmd == cmdIdeTools: suggestExpr(c, n)
 
   var s = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared, checkModule})
   if s != nil:

compiler/seminst.nim

@@ -97,10 +97,17 @@ proc genericCacheGet(genericSym: PSym, entry: TInstantiation;
       if inst.compilesId == id and sameInstantiation(entry, inst[]):
         return inst.sym
 
+when false:
+  proc `$`(x: PSym): string =
+    result = x.name.s & " " & " id " & $x.id
+
 proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) =
   # we need to create a fresh set of gensym'ed symbols:
-  if n.kind == nkSym and sfGenSym in n.sym.flags and
+  #if n.kind == nkSym and sfGenSym in n.sym.flags:
-      (n.sym.owner == orig or n.sym.owner.kind == skPackage):
+  #  if n.sym.owner != orig:
+  #    echo "symbol ", n.sym.name.s, " orig ", orig, " owner ", n.sym.owner
+  if n.kind == nkSym and {sfGenSym, sfFromGeneric} * n.sym.flags == {sfGenSym}: # and
+    #  (n.sym.owner == orig or n.sym.owner.kind in {skPackage}):
     let s = n.sym
     var x = PSym(idTableGet(symMap, s))
     if x == nil:

compiler/semtempl.nim

@@ -112,6 +112,7 @@ type
     toBind, toMixin, toInject: IntSet
     owner: PSym
     cursorInBody: bool # only for nimsuggest
+    scopeN: int
     bracketExpr: PNode
 
 template withBracketExpr(ctx, x, body: untyped) =
@@ -141,8 +142,13 @@ proc isTemplParam(c: TemplCtx, n: PNode): bool {.inline.} =
 
 proc semTemplBody(c: var TemplCtx, n: PNode): PNode
 
-proc openScope(c: var TemplCtx) = openScope(c.c)
+proc openScope(c: var TemplCtx) =
-proc closeScope(c: var TemplCtx) = closeScope(c.c)
+  openScope(c.c)
+  inc c.scopeN
+
+proc closeScope(c: var TemplCtx) =
+  dec c.scopeN
+  closeScope(c.c)
 
 proc semTemplBodyScope(c: var TemplCtx, n: PNode): PNode =
   openScope(c)
@@ -166,6 +172,7 @@ proc newGenSym(kind: TSymKind, n: PNode, c: var TemplCtx): PSym =
   result = newSym(kind, considerQuotedIdent(n), c.owner, n.info)
   incl(result.flags, sfGenSym)
   incl(result.flags, sfShadowed)
+  if c.scopeN == 0: incl(result.flags, sfFromGeneric)
 
 proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
   # locals default to 'gensym':

compiler/semtypes.nim

@@ -659,7 +659,8 @@ proc addInheritedFields(c: PContext, check: var IntSet, pos: var int,
   addInheritedFieldsAux(c, check, pos, obj.n)
 
 proc semObjectNode(c: PContext, n: PNode, prev: PType): PType =
-  if n.sonsLen == 0: return newConstraint(c, tyObject)
+  if n.sonsLen == 0:
+    return newConstraint(c, tyObject)
   var check = initIntSet()
   var pos = 0
   var base, realBase: PType = nil
@@ -1159,8 +1160,16 @@ proc maybeAliasType(c: PContext; typeExpr, prev: PType): PType =
     result.sym = prev.sym
     assignType(prev, result)
 
+proc fixupTypeOf(c: PContext, prev: PType, typExpr: PNode) =
+  if prev != nil:
+    let result = newTypeS(tyAlias, c)
+    result.rawAddSon typExpr.typ
+    result.sym = prev.sym
+    assignType(prev, result)
+
 proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
   result = nil
+
   if gCmd == cmdIdeTools: suggestExpr(c, n)
   case n.kind
   of nkEmpty: discard
@@ -1168,6 +1177,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     checkSonsLen(n, 1)
     let typExpr = semExprWithType(c, n.sons[0], {efInTypeof})
+    fixupTypeOf(c, prev, typExpr)
     result = typExpr.typ
   of nkPar:
     if sonsLen(n) == 1: result = semTypeNode(c, n.sons[0], prev)
@@ -1234,6 +1244,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       elif op.id == ord(wType):
         checkSonsLen(n, 2)
         let typExpr = semExprWithType(c, n.sons[1], {efInTypeof})
+        fixupTypeOf(c, prev, typExpr)
         result = typExpr.typ
       else:
         result = semTypeExpr(c, n, prev)

compiler/suggest.nim

@@ -41,6 +41,20 @@ var
 
 template origModuleName(m: PSym): string = m.name.s
 
+proc findDocComment(n: PNode): PNode =
+  if n == nil: return nil
+  if not isNil(n.comment): return n
+  for i in countup(0, safeLen(n)-1):
+    result = findDocComment(n.sons[i])
+    if result != nil: return
+
+proc extractDocComment(s: PSym): string =
+  let n = findDocComment(s.ast)
+  if not n.isNil:
+    result = n.comment.replace("\n##", "\n").strip
+  else:
+    result = ""
+
 proc symToSuggest(s: PSym, isLocal: bool, section: string, li: TLineInfo;
                   quality: range[0..100]): Suggest =
   result.section = parseIdeCmd(section)

compiler/vm.nim

@@ -559,7 +559,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         if regs[rb].node.kind == nkRefTy:
           regs[ra].node = regs[rb].node.sons[0]
         else:
-          stackTrace(c, tos, pc, errGenerated, "limited VM support for pointers")
+          ensureKind(rkNode)
+          regs[ra].node = regs[rb].node
       else:
         stackTrace(c, tos, pc, errNilAccess)
     of opcWrDeref:
@@ -932,7 +933,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
                             c.module
         var macroCall = newNodeI(nkCall, c.debug[pc])
         macroCall.add(newSymNode(prc))
-        for i in 1 .. rc-1: macroCall.add(regs[rb+i].regToNode)
+        for i in 1 .. rc-1:
+          let node = regs[rb+i].regToNode
+          node.info = c.debug[pc]
+          macroCall.add(node)
         let a = evalTemplate(macroCall, prc, genSymOwner)
         a.recSetFlagIsRef
         ensureKind(rkNode)

compiler/vmgen.nim

@@ -1259,6 +1259,13 @@ proc isTemp(c: PCtx; dest: TDest): bool =
 template needsAdditionalCopy(n): untyped =
   not c.isTemp(dest) and not fitsRegister(n.typ)
 
+proc genAdditionalCopy(c: PCtx; n: PNode; opc: TOpcode;
+                       dest, idx, value: TRegister) =
+  var cc = c.getTemp(n.typ)
+  c.gABC(n, whichAsgnOpc(n), cc, value, 0)
+  c.gABC(n, opc, dest, idx, cc)
+  c.freeTemp(cc)
+
 proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode;
                        dest, idx, value: TRegister) =
   # opcLdObj et al really means "load address". We sometimes have to create a
@@ -1266,10 +1273,7 @@ proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode;
   # mylocal = a.b  # needs a copy of the data!
   assert n.typ != nil
   if needsAdditionalCopy(n):
-    var cc = c.getTemp(n.typ)
+    genAdditionalCopy(c, n, opc, dest, idx, value)
-    c.gABC(n, whichAsgnOpc(n), cc, value, 0)
-    c.gABC(n, opc, dest, idx, cc)
-    c.freeTemp(cc)
   else:
     c.gABC(n, opc, dest, idx, value)
 
@@ -1352,7 +1356,7 @@ proc genGlobalInit(c: PCtx; n: PNode; s: PSym) =
   c.gABx(n, opcLdGlobal, dest, s.position)
   if s.ast != nil:
     let tmp = c.genx(s.ast)
-    c.preventFalseAlias(n, opcWrDeref, dest, 0, tmp)
+    c.genAdditionalCopy(n, opcWrDeref, dest, 0, tmp)
     c.freeTemp(dest)
     c.freeTemp(tmp)
 
@@ -1506,7 +1510,7 @@ proc genVarSection(c: PCtx; n: PNode) =
     #assert(a.sons[0].kind == nkSym) can happen for transformed vars
     if a.kind == nkVarTuple:
       for i in 0 .. a.len-3:
-        setSlot(c, a[i].sym)
+        if not a[i].sym.isGlobal: setSlot(c, a[i].sym)
         checkCanEval(c, a[i])
       c.gen(lowerTupleUnpacking(a, c.getOwner))
     elif a.sons[0].kind == nkSym:
@@ -1525,7 +1529,7 @@ proc genVarSection(c: PCtx; n: PNode) =
         if a.sons[2].kind != nkEmpty:
           let tmp = c.genx(a.sons[0], {gfAddrOf})
           let val = c.genx(a.sons[2])
-          c.preventFalseAlias(a.sons[2], opcWrDeref, tmp, 0, val)
+          c.genAdditionalCopy(a.sons[2], opcWrDeref, tmp, 0, val)
           c.freeTemp(val)
           c.freeTemp(tmp)
       else:

config/nim.cfg

@@ -1,5 +1,5 @@
 # Configuration file for the Nim Compiler.
-# (c) 2015 Andreas Rumpf
+# (c) 2017 Andreas Rumpf
 
 # Feel free to edit the default values as you need.
 

copying.txt

@@ -1,7 +1,7 @@
 =====================================================
-Nim -- a Compiler for Nim. http://nim-lang.org/
+Nim -- a Compiler for Nim. https://nim-lang.org/
 
-Copyright (C) 2006-2015 Andreas Rumpf. All rights reserved.
+Copyright (C) 2006-2017 Andreas Rumpf. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

doc/contributing.rst

@@ -137,7 +137,7 @@ etc. Note that currently the ``deprecated`` statement does not work well with
 overloading so for routines the latter variant is better.
 
 
-`Deprecated <http://nim-lang.org/docs/manual.html#pragmas-deprecated-pragma>`_
+`Deprecated <https://nim-lang.org/docs/manual.html#pragmas-deprecated-pragma>`_
 pragma in the manual.
 
 

doc/koch.rst

@@ -96,7 +96,7 @@ web command
 The `web`:idx: command converts the documentation in the ``doc`` directory
 from rst to HTML. It also repeats the same operation but places the result in
 the ``web/upload`` which can be used to update the website at
-http://nim-lang.org.
+https://nim-lang.org.
 
 By default the documentation will be built in parallel using the number of
 available CPU cores. If any documentation build sub commands fail, they will

doc/lib.rst

@@ -586,4 +586,4 @@ Nim programming language.
   nimblepkglist.js or have javascript disabled in your browser.</b></div>
 
   <script type="text/javascript" src="nimblepkglist.js"></script>
-  <script type="text/javascript" src="http://irclogs.nim-lang.org/packages?callback=gotPackageList" async></script>
+  <script type="text/javascript" src="https://irclogs.nim-lang.org/packages?callback=gotPackageList" async></script>

koch.nim

@@ -70,6 +70,8 @@ Web options:
                            build the official docs, use UA-48159761-1
 """
 
+const gaCode = " --googleAnalytics:UA-48159761-1"
+
 proc exe(f: string): string =
   result = addFileExt(f, ExeExt)
   when defined(windows):
@@ -221,6 +223,8 @@ proc bundleWinTools() =
   copyExe("tools/finish".exe, "finish".exe)
   removeFile("tools/finish".exe)
   nimexec("c -o:bin/vccexe.exe tools/vccenv/vccexe")
+  nimexec(r"c --cc:vcc --app:gui -o:bin\downloader.exe -d:ssl --noNimblePath " &
+          r"--path:..\ui tools\downloader.nim")
 
 proc zip(args: string) =
   bundleNimbleSrc()
@@ -319,7 +323,8 @@ proc boot(args: string) =
   var finalDest = "bin" / "nim".exe
   # default to use the 'c' command:
   let bootOptions = if args.len == 0 or args.startsWith("-"): "c" else: ""
-  let smartNimcache = if "release" in args: "nimcache/release" else: "nimcache/debug"
+  let smartNimcache = (if "release" in args: "nimcache/r_" else: "nimcache/d_") &
+                      hostOs & "_" & hostCpu
 
   copyExe(findStartNim(), 0.thVersion)
   for i in 0..2:
@@ -380,8 +385,64 @@ proc clean(args: string) =
 
 # -------------- builds a release ---------------------------------------------
 
+proc patchConfig(lookFor, replaceBy: string) =
+  const
+    cfgFile = "config/nim.cfg"
+  try:
+    let cfg = readFile(cfgFile)
+    let newCfg = cfg.replace(lookFor, replaceBy)
+    if newCfg == cfg:
+      echo "Could not patch 'config/nim.cfg' [Error]"
+      echo "Reason: patch substring not found:"
+      echo lookFor
+    else:
+      writeFile(cfgFile, newCfg)
+  except IOError:
+    quit "Could not access 'config/nim.cfg' [Error]"
+
+proc winReleaseArch(arch: string) =
+  doAssert arch in ["32", "64"]
+  let cpu = if arch == "32": "i386" else: "amd64"
+
+  template withMingw(path, body) =
+    const orig = """#gcc.path = r"$nim\dist\mingw\bin""""
+    let replacePattern = """gcc.path = r"..\mingw$1\bin" # winrelease""" % arch
+    patchConfig(orig, replacePattern)
+    try:
+      body
+    finally:
+      patchConfig(replacePattern, orig)
+
+  withMingw r"..\mingw" & arch & r"\bin":
+    # Rebuilding koch is necessary because it uses its pointer size to
+    # determine which mingw link to put in the NSIS installer.
+    nimexec "c --out:koch_temp --cpu:$# koch" % cpu
+    exec "koch_temp boot -d:release --cpu:$#" % cpu
+    exec "koch_temp nsis -d:release"
+    exec "koch_temp zip -d:release"
+
+    when false:
+      # we now disable the NSIS installer as it cannot download from https
+      # and is broken in so many different ways it's not funny anymore:
+      moveFile r"build\nim_$#.exe" % VersionAsString,
+               r"web\upload\download\nim-$#_x$#.exe" % [VersionAsString, arch]
+    moveFile r"build\nim-$#.zip" % VersionAsString,
+             r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch]
+
 proc winRelease() =
-  exec(r"call ci\nsis_build.bat " & VersionAsString)
+  # Build -docs file:
+  when true:
+    web(gaCode)
+    withDir "web/upload/" & VersionAsString:
+      exec "7z a -tzip docs-$#.zip *.html" % VersionAsString
+    moveFile "web/upload/$1/docs-$1.zip" % VersionAsString,
+             "web/upload/download/docs-$1.zip" % VersionAsString
+  when true:
+    csource("-d:release")
+  when true:
+    winReleaseArch "32"
+  when true:
+    winReleaseArch "64"
 
 # -------------- tests --------------------------------------------------------
 
@@ -463,10 +524,10 @@ of cmdArgument:
   of "web": web(op.cmdLineRest)
   of "doc", "docs": web("--onlyDocs " & op.cmdLineRest)
   of "json2": web("--json2 " & op.cmdLineRest)
-  of "website": website(op.cmdLineRest & " --googleAnalytics:UA-48159761-1")
+  of "website": website(op.cmdLineRest & gaCode)
   of "web0":
     # undocumented command for Araq-the-merciful:
-    web(op.cmdLineRest & " --googleAnalytics:UA-48159761-1")
+    web(op.cmdLineRest & gaCode)
   of "pdf": pdf()
   of "csource", "csources": csource(op.cmdLineRest)
   of "zip": zip(op.cmdLineRest)

lib/impure/nre.nim

@@ -23,6 +23,15 @@ export options
 ##
 ## A regular expression library for Nim using PCRE to do the hard work.
 ##
+## **Note**: If you love ``sequtils.toSeq`` we have bad news for you. This
+## library doesn't work with it due to documented compiler limitations. As
+## a workaround, use this:
+##
+## .. code-block:: nim
+##
+##    import nre except toSeq
+##
+##
 ## Licencing
 ## ---------
 ##

lib/nimbase.h

@@ -402,16 +402,29 @@ struct TFrame {
   NI16 calldepth;
 };
 
-#define nimfr(proc, file) \
+#ifdef NIM_NEW_MANGLING_RULES
-  TFrame FR; \
+  #define nimfr_(proc, file) \
-  FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = 0; nimFrame(&FR);
+    TFrame FR_; \
+    FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = 0; nimFrame(&FR_);
 
-#define nimfrs(proc, file, slots, length) \
+  #define nimfrs_(proc, file, slots, length) \
-  struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR; \
+    struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR_; \
-  FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = length; nimFrame((TFrame*)&FR);
+    FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = length; nimFrame((TFrame*)&FR_);
 
-#define nimln(n, file) \
+  #define nimln_(n, file) \
-  FR.line = n; FR.filename = file;
+    FR_.line = n; FR_.filename = file;
+#else
+  #define nimfr(proc, file) \
+    TFrame FR; \
+    FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = 0; nimFrame(&FR);
+
+  #define nimfrs(proc, file, slots, length) \
+    struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR; \
+    FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = length; nimFrame((TFrame*)&FR);
+
+  #define nimln(n, file) \
+    FR.line = n; FR.filename = file;
+#endif
 
 #define NIM_POSIX_INIT  __attribute__((constructor))
 

lib/pure/asyncdispatch.nim

@@ -753,26 +753,6 @@ when defined(windows) or defined(nimdoc):
     let dwLocalAddressLength = Dword(sizeof(Sockaddr_in) + 16)
     let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16)
 
-    template completeAccept() {.dirty.} =
-      var listenSock = socket
-      let setoptRet = setsockopt(clientSock, SOL_SOCKET,
-          SO_UPDATE_ACCEPT_CONTEXT, addr listenSock,
-          sizeof(listenSock).SockLen)
-      if setoptRet != 0: raiseOSError(osLastError())
-
-      var localSockaddr, remoteSockaddr: ptr SockAddr
-      var localLen, remoteLen: int32
-      getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength,
-                           dwLocalAddressLength, dwRemoteAddressLength,
-                           addr localSockaddr, addr localLen,
-                           addr remoteSockaddr, addr remoteLen)
-      register(clientSock.AsyncFD)
-      # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186
-      retFuture.complete(
-        (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr),
-         client: clientSock.AsyncFD)
-      )
-
     template failAccept(errcode) =
       if flags.isDisconnectionError(errcode):
         var newAcceptFut = acceptAddr(socket, flags)
@@ -785,6 +765,29 @@ when defined(windows) or defined(nimdoc):
       else:
         retFuture.fail(newException(OSError, osErrorMsg(errcode)))
 
+    template completeAccept() {.dirty.} =
+      var listenSock = socket
+      let setoptRet = setsockopt(clientSock, SOL_SOCKET,
+          SO_UPDATE_ACCEPT_CONTEXT, addr listenSock,
+          sizeof(listenSock).SockLen)
+      if setoptRet != 0:
+        let errcode = osLastError()
+        discard clientSock.closeSocket()
+        failAccept(errcode)
+      else:
+        var localSockaddr, remoteSockaddr: ptr SockAddr
+        var localLen, remoteLen: int32
+        getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength,
+                             dwLocalAddressLength, dwRemoteAddressLength,
+                             addr localSockaddr, addr localLen,
+                             addr remoteSockaddr, addr remoteLen)
+        register(clientSock.AsyncFD)
+        # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186
+        retFuture.complete(
+          (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr),
+          client: clientSock.AsyncFD)
+        )
+
     var ol = PCustomOverlapped()
     GC_ref(ol)
     ol.data = CompletionData(fd: socket, cb:

lib/pure/asyncmacro.nim

@@ -284,9 +284,9 @@ proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} =
 proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
   ## This macro transforms a single procedure into a closure iterator.
   ## The ``async`` macro supports a stmtList holding multiple async procedures.
-  if prc.kind notin {nnkProcDef, nnkLambda}:
+  if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef}:
       error("Cannot transform this node kind into an async proc." &
-            " Proc definition or lambda node expected.")
+            " proc/method definition or lambda node expected.")
 
   hint("Processing " & prc[0].getName & " as an async proc.")
 

lib/pure/collections/tables.nim

@@ -814,11 +814,14 @@ proc len*[A](t: CountTable[A]): int =
   ## returns the number of keys in `t`.
   result = t.counter
 
-proc clear*[A](t: var CountTable[A] | CountTableRef[A]) =
+proc clear*[A](t: CountTableRef[A]) =
   ## Resets the table so that it is empty.
   clearImpl()
-  t.counter = 0
 
+proc clear*[A](t: var CountTable[A]) =
+  ## Resets the table so that it is empty.
+  clearImpl()
+  
 iterator pairs*[A](t: CountTable[A]): (A, int) =
   ## iterates over any (key, value) pair in the table `t`.
   for h in 0..high(t.data):

lib/pure/httpcore.nim

@@ -312,3 +312,10 @@ when isMainModule:
   test.add("Connection", "Test")
   doAssert test["Connection", 2] == "Test"
   doAssert "upgrade" in test["Connection"]
+
+  # Bug #5344.
+  doAssert parseHeader("foobar: ") == ("foobar", @[""])
+  let (key, value) = parseHeader("foobar: ")
+  test = newHttpHeaders()
+  test[key] = value
+  doAssert test["foobar"] == ""

lib/pure/ioselects/ioselectors_epoll.nim

@@ -165,7 +165,7 @@ proc close*(ev: SelectEvent) =
 
 template checkFd(s, f) =
   if f >= s.maxFD:
-    raiseIOSelectorsError("Maximum file descriptors exceeded")
+    raiseIOSelectorsError("Maximum number of descriptors is exhausted!")
 
 proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
                         events: set[Event], data: T) =
@@ -188,7 +188,8 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) =
   let fdi = int(fd)
   s.checkFd(fdi)
   var pkey = addr(s.fds[fdi])
-  doAssert(pkey.ident != 0)
+  doAssert(pkey.ident != 0,
+           "Descriptor [" & $fdi & "] is not registered in the queue!")
   doAssert(pkey.events * maskEvents == {})
   if pkey.events != events:
     var epv = epoll_event(events: EPOLLRDHUP)
@@ -215,8 +216,8 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) =
   let fdi = int(fd)
   s.checkFd(fdi)
   var pkey = addr(s.fds[fdi])
-  doAssert(pkey.ident != 0)
+  doAssert(pkey.ident != 0,
-
+           "Descriptor [" & $fdi & "] is not registered in the queue!")
   if pkey.events != {}:
     when not defined(android):
       if pkey.events * {Event.Read, Event.Write} != {}:
@@ -277,7 +278,7 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) =
   let fdi = int(ev.efd)
   s.checkFd(fdi)
   var pkey = addr(s.fds[fdi])
-  doAssert(pkey.ident != 0)
+  doAssert(pkey.ident != 0, "Event is not registered in the queue!")
   doAssert(Event.User in pkey.events)
   var epv = epoll_event()
   if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0:
@@ -380,7 +381,7 @@ when not defined(android):
 
 proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) =
   let fdi = int(ev.efd)
-  doAssert(s.fds[fdi].ident == 0)
+  doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!")
   s.setKey(fdi, {Event.User}, 0, data)
   var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP)
   epv.data.u64 = ev.efd.uint

lib/pure/ioselects/ioselectors_kqueue.nim

@@ -119,12 +119,13 @@ proc newSelector*[T](): Selector[T] =
   result.maxFD = maxFD.int
 
 proc close*[T](s: Selector[T]) =
-  let res = posix.close(s.kqFD)
+  let res1 = posix.close(s.kqFD)
+  let res2 = posix.close(s.sock)
   when hasThreadSupport:
     deinitLock(s.changesLock)
     deallocSharedArray(s.fds)
     deallocShared(cast[pointer](s))
-  if res != 0:
+  if res1 != 0 or res2 != 0:
     raiseIOSelectorsError(osLastError())
 
 template clearKey[T](key: ptr SelectorKey[T]) =
@@ -157,7 +158,7 @@ proc close*(ev: SelectEvent) =
 
 template checkFd(s, f) =
   if f >= s.maxFD:
-    raiseIOSelectorsError("Maximum file descriptors exceeded!")
+    raiseIOSelectorsError("Maximum number of descriptors is exhausted!")
 
 when hasThreadSupport:
   template withChangeLock[T](s: Selector[T], body: untyped) =
@@ -241,7 +242,8 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle,
   let fdi = int(fd)
   s.checkFd(fdi)
   var pkey = addr(s.fds[fdi])
-  doAssert(pkey.ident != 0)
+  doAssert(pkey.ident != 0,
+           "Descriptor [" & $fdi & "] is not registered in the queue!")
   doAssert(pkey.events * maskEvents == {})
 
   if pkey.events != events:
@@ -329,7 +331,7 @@ proc registerProcess*[T](s: Selector[T], pid: int,
 
 proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) =
   let fdi = ev.rfd.int
-  doAssert(s.fds[fdi].ident == 0)
+  doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!")
   setKey(s, fdi, {Event.User}, 0, data)
 
   modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil)
@@ -372,7 +374,8 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) =
   let fdi = int(fd)
   s.checkFd(fdi)
   var pkey = addr(s.fds[fdi])
-  doAssert(pkey.ident != 0)
+  doAssert(pkey.ident != 0,
+           "Descriptor [" & $fdi & "] is not registered in the queue!")
 
   if pkey.events != {}:
     if pkey.events * {Event.Read, Event.Write} != {}:
@@ -431,9 +434,8 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) =
   let fdi = int(ev.rfd)
   s.checkFd(fdi)
   var pkey = addr(s.fds[fdi])
-  doAssert(pkey.ident != 0)
+  doAssert(pkey.ident != 0, "Event is not registered in the queue!")
   doAssert(Event.User in pkey.events)
-
   modifyKQueue(s, uint(fdi), EVFILT_READ, EV_DELETE, 0, 0, nil)
   when not declared(CACHE_EVENTS):
     flushKQueue(s)
@@ -564,8 +566,7 @@ proc selectInto*[T](s: Selector[T], timeout: int,
         pkey.events.incl(Event.Finished)
         rkey.events.incl(Event.Process)
       else:
-        pkey = addr(s.fds[cast[int](kevent.udata)])
+        doAssert(true, "Unsupported kqueue filter in the queue!")
-        raiseIOSelectorsError("Unsupported kqueue filter in queue!")
 
       if (kevent.flags and EV_EOF) != 0:
         rkey.events.incl(Event.Error)

lib/pure/ioselects/ioselectors_poll.nim

@@ -115,9 +115,8 @@ template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) =
         s.pollfds[i].events = pollev
         break
       inc(i)
-
+    doAssert(i < s.pollcnt,
-    if i == s.pollcnt:
+             "Descriptor [" & $sock & "] is not registered in the queue!")
-      raiseIOSelectorsError("Descriptor is not registered in queue")
 
 template pollRemove[T](s: Selector[T], sock: cint) =
   withPollLock(s):
@@ -140,7 +139,7 @@ template pollRemove[T](s: Selector[T], sock: cint) =
 
 template checkFd(s, f) =
   if f >= s.maxFD:
-    raiseIOSelectorsError("Descriptor is not registered in queue")
+    raiseIOSelectorsError("Maximum number of descriptors is exhausted!")
 
 proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
                         events: set[Event], data: T) =
@@ -157,7 +156,8 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle,
   let fdi = int(fd)
   s.checkFd(fdi)
   var pkey = addr(s.fds[fdi])
-  doAssert(pkey.ident != 0)
+  doAssert(pkey.ident != 0,
+           "Descriptor [" & $fdi & "] is not registered in the queue!")
   doAssert(pkey.events * maskEvents == {})
 
   if pkey.events != events:
@@ -172,7 +172,7 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle,
 
 proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) =
   var fdi = int(ev.rfd)
-  doAssert(s.fds[fdi].ident == 0)
+  doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!")
   var events = {Event.User}
   setKey(s, fdi, events, 0, data)
   events.incl(Event.Read)
@@ -182,7 +182,8 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) =
   let fdi = int(fd)
   s.checkFd(fdi)
   var pkey = addr(s.fds[fdi])
-  doAssert(pkey.ident != 0)
+  doAssert(pkey.ident != 0,
+           "Descriptor [" & $fdi & "] is not registered in the queue!")
   pkey.ident = 0
   pkey.events = {}
   s.pollRemove(fdi.cint)
@@ -191,7 +192,7 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) =
   let fdi = int(ev.rfd)
   s.checkFd(fdi)
   var pkey = addr(s.fds[fdi])
-  doAssert(pkey.ident != 0)
+  doAssert(pkey.ident != 0, "Event is not registered in the queue!")
   doAssert(Event.User in pkey.events)
   pkey.ident = 0
   pkey.events = {}

lib/pure/ioselects/ioselectors_select.nim

@@ -202,8 +202,8 @@ proc setSelectKey[T](s: Selector[T], fd: SocketHandle, events: set[Event],
       pkey.data = data
       break
     inc(i)
-  if i == FD_SETSIZE:
+  if i >= FD_SETSIZE:
-    raiseIOSelectorsError("Maximum numbers of fds exceeded")
+    raiseIOSelectorsError("Maximum number of descriptors is exhausted!")
 
 proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] =
   var i = 0
@@ -213,8 +213,8 @@ proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] =
       result = addr(s.fds[i])
       break
     inc(i)
-  if i == FD_SETSIZE:
+  doAssert(i < FD_SETSIZE,
-    raiseIOSelectorsError("Descriptor not registered in queue")
+           "Descriptor [" & $int(fd) & "] is not registered in the queue!")
 
 proc delKey[T](s: Selector[T], fd: SocketHandle) =
   var empty: T
@@ -226,8 +226,8 @@ proc delKey[T](s: Selector[T], fd: SocketHandle) =
       s.fds[i].data = empty
       break
     inc(i)
-  if i == FD_SETSIZE:
+  doAssert(i < FD_SETSIZE,
-    raiseIOSelectorsError("Descriptor not registered in queue")
+           "Descriptor [" & $int(fd) & "] is not registered in the queue!")
 
 proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
                         events: set[Event], data: T) =
@@ -294,6 +294,7 @@ proc unregister*[T](s: Selector[T], fd: SocketHandle) =
 proc unregister*[T](s: Selector[T], ev: SelectEvent) =
   let fd = ev.rsock
   s.withSelectLock():
+    var pkey = s.getKey(fd)
     IOFD_CLR(fd, addr s.rSet)
     dec(s.count)
     s.delKey(fd)

lib/pure/nativesockets.nim

@@ -22,11 +22,12 @@ const useWinVersion = defined(Windows) or defined(nimdoc)
 when useWinVersion:
   import winlean
   export WSAEWOULDBLOCK, WSAECONNRESET, WSAECONNABORTED, WSAENETRESET,
+         WSANOTINITIALISED, WSAENOTSOCK, WSAEINPROGRESS, WSAEINTR,
          WSAEDISCON, ERROR_NETNAME_DELETED
 else:
   import posix
   export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL,
-    EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET
+    EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET, EBADF
   export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length
 
 export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen,

lib/pure/terminal.nim

@@ -630,7 +630,7 @@ proc getch*(): char =
   when defined(windows):
     let fd = getStdHandle(STD_INPUT_HANDLE)
     var keyEvent = KEY_EVENT_RECORD()
-    var numRead: cint 
+    var numRead: cint
     while true:
       # Block until character is entered
       doAssert(waitForSingleObject(fd, INFINITE) == WAIT_OBJECT_0)

lib/system.nim

@@ -3252,19 +3252,18 @@ proc `/`*(x, y: int): float {.inline, noSideEffect.} =
 
 template spliceImpl(s, a, L, b: untyped): untyped =
   # make room for additional elements or cut:
-  var slen = s.len
+  var shift = b.len - max(0,L)  # ignore negative slice size
-  var shift = b.len - L
+  var newLen = s.len + shift
-  var newLen = slen + shift
   if shift > 0:
     # enlarge:
     setLen(s, newLen)
-    for i in countdown(newLen-1, a+shift+1): shallowCopy(s[i], s[i-shift])
+    for i in countdown(newLen-1, a+b.len): shallowCopy(s[i], s[i-shift])
   else:
-    for i in countup(a+b.len, s.len-1+shift): shallowCopy(s[i], s[i-shift])
+    for i in countup(a+b.len, newLen-1): shallowCopy(s[i], s[i-shift])
     # cut down:
     setLen(s, newLen)
   # fill the hole:
-  for i in 0 .. <b.len: s[i+a] = b[i]
+  for i in 0 .. <b.len: s[a+i] = b[i]
 
 when hasAlloc or defined(nimscript):
   proc `[]`*(s: string, x: Slice[int]): string {.inline.} =

lib/system/alloc.nim

@@ -295,10 +295,11 @@ proc writeFreeList(a: MemRegion) =
 proc requestOsChunks(a: var MemRegion, size: int): PBigChunk =
   when not defined(emscripten):
     if not a.blockChunkSizeIncrease:
-      if a.currMem < 64 * 1024:
+      let usedMem = a.currMem # - a.freeMem
+      if usedMem < 64 * 1024:
         a.nextChunkSize = PageSize*4
       else:
-        a.nextChunkSize = min(roundup(a.currMem shr 2, PageSize), a.nextChunkSize * 2)
+        a.nextChunkSize = min(roundup(usedMem shr 2, PageSize), a.nextChunkSize * 2)
   var size = size
 
   if size > a.nextChunkSize:
@@ -708,7 +709,7 @@ proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer =
   if newsize > 0:
     result = alloc0(allocator, newsize)
     if p != nil:
-      copyMem(result, p, ptrSize(p))
+      copyMem(result, p, min(ptrSize(p), newsize))
       dealloc(allocator, p)
   elif p != nil:
     dealloc(allocator, p)

lib/system/nimscript.nim

@@ -293,6 +293,11 @@ template task*(name: untyped; description: string; body: untyped): untyped =
     setCommand "nop"
     `name Task`()
 
+proc cppDefine*(define: string) =
+  ## tell Nim that ``define`` is a C preprocessor ``#define`` and so always
+  ## needs to be mangled.
+  builtin
+
 when not defined(nimble):
   # nimble has its own implementation for these things.
   var

lib/system/threads.nim

@@ -195,15 +195,15 @@ else:
     importc: "pthread_setaffinity_np", header: pthreadh.}
 
   when defined(linux):
-    proc syscall(arg: int): int {.varargs, importc: "syscall", header: "<unistd.h>".}
+    proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".}
-    var SYS_gettid {.importc, header: "<sys/syscall.h>".}: int
+    var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: int
 
     #type Pid {.importc: "pid_t", header: "<sys/types.h>".} = distinct int
     #proc gettid(): Pid {.importc, header: "<sys/types.h>".}
 
     proc getThreadId*(): int =
       ## get the ID of the currently running thread.
-      result = int(syscall(SYS_gettid))
+      result = int(syscall(NR_gettid))
   elif defined(macosx) or defined(bsd):
     proc pthread_threadid_np(y: pointer; x: var uint64): cint {.importc, header: "pthread.h".}
 

lib/upcoming/asyncdispatch.nim

@@ -738,26 +738,6 @@ when defined(windows) or defined(nimdoc):
     let dwLocalAddressLength = Dword(sizeof(Sockaddr_in) + 16)
     let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16)
 
-    template completeAccept() {.dirty.} =
-      var listenSock = socket
-      let setoptRet = setsockopt(clientSock, SOL_SOCKET,
-          SO_UPDATE_ACCEPT_CONTEXT, addr listenSock,
-          sizeof(listenSock).SockLen)
-      if setoptRet != 0: raiseOSError(osLastError())
-
-      var localSockaddr, remoteSockaddr: ptr SockAddr
-      var localLen, remoteLen: int32
-      getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength,
-                           dwLocalAddressLength, dwRemoteAddressLength,
-                           addr localSockaddr, addr localLen,
-                           addr remoteSockaddr, addr remoteLen)
-      register(clientSock.AsyncFD)
-      # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186
-      retFuture.complete(
-        (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr),
-         client: clientSock.AsyncFD)
-      )
-
     template failAccept(errcode) =
       if flags.isDisconnectionError(errcode):
         var newAcceptFut = acceptAddr(socket, flags)
@@ -770,6 +750,29 @@ when defined(windows) or defined(nimdoc):
       else:
         retFuture.fail(newException(OSError, osErrorMsg(errcode)))
 
+    template completeAccept() {.dirty.} =
+      var listenSock = socket
+      let setoptRet = setsockopt(clientSock, SOL_SOCKET,
+          SO_UPDATE_ACCEPT_CONTEXT, addr listenSock,
+          sizeof(listenSock).SockLen)
+      if setoptRet != 0:
+        let errcode = osLastError()
+        discard clientSock.closeSocket()
+        failAccept(errcode)
+      else:
+        var localSockaddr, remoteSockaddr: ptr SockAddr
+        var localLen, remoteLen: int32
+        getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength,
+                             dwLocalAddressLength, dwRemoteAddressLength,
+                             addr localSockaddr, addr localLen,
+                             addr remoteSockaddr, addr remoteLen)
+        register(clientSock.AsyncFD)
+        # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186
+        retFuture.complete(
+          (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr),
+          client: clientSock.AsyncFD)
+        )
+
     var ol = PCustomOverlapped()
     GC_ref(ol)
     ol.data = CompletionData(fd: socket, cb:
@@ -1056,16 +1059,14 @@ when defined(windows) or defined(nimdoc):
 
   proc unregister*(ev: AsyncEvent) =
     ## Unregisters event ``ev``.
-    if ev.hWaiter != 0:
+    doAssert(ev.hWaiter != 0, "Event is not registered in the queue!")
-      let p = getGlobalDispatcher()
+    let p = getGlobalDispatcher()
-      p.handles.excl(AsyncFD(ev.hEvent))
+    p.handles.excl(AsyncFD(ev.hEvent))
-      if unregisterWait(ev.hWaiter) == 0:
+    if unregisterWait(ev.hWaiter) == 0:
-        let err = osLastError()
+      let err = osLastError()
-        if err.int32 != ERROR_IO_PENDING:
+      if err.int32 != ERROR_IO_PENDING:
-          raiseOSError(err)
+        raiseOSError(err)
-      ev.hWaiter = 0
+    ev.hWaiter = 0
-    else:
-      raise newException(ValueError, "Event is not registered!")
 
   proc close*(ev: AsyncEvent) =
     ## Closes event ``ev``.
@@ -1076,8 +1077,7 @@ when defined(windows) or defined(nimdoc):
 
   proc addEvent*(ev: AsyncEvent, cb: Callback) =
     ## Registers callback ``cb`` to be called when ``ev`` will be signaled
-    if ev.hWaiter != 0:
+    doAssert(ev.hWaiter == 0, "Event is already registered in the queue!")
-      raise newException(ValueError, "Event is already registered!")
 
     let p = getGlobalDispatcher()
     let hEvent = ev.hEvent
@@ -1086,17 +1086,22 @@ when defined(windows) or defined(nimdoc):
     var flags = WT_EXECUTEINWAITTHREAD.Dword
 
     proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) =
-      if cb(fd):
+      if ev.hWaiter != 0:
-        # we need this check to avoid exception, if `unregister(event)` was
+        if cb(fd):
-        # called in callback.
+          # we need this check to avoid exception, if `unregister(event)` was
-        deallocShared(cast[pointer](pcd))
+          # called in callback.
-        if ev.hWaiter != 0: unregister(ev)
+          deallocShared(cast[pointer](pcd))
+          if ev.hWaiter != 0:
+            unregister(ev)
+        else:
+          # if callback returned `false`, then it wants to be called again, so
+          # we need to ref and protect `pcd.ovl` again, because it will be
+          # unrefed and disposed in `poll()`.
+          GC_ref(pcd.ovl)
+          pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb))
       else:
-        # if callback returned `false`, then it wants to be called again, so
+        # if ev.hWaiter == 0, then event was unregistered before `poll()` call.
-        # we need to ref and protect `pcd.ovl` again, because it will be
+        deallocShared(cast[pointer](pcd))
-        # unrefed and disposed in `poll()`.
-        GC_ref(pcd.ovl)
-        pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb))
 
     registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb)
     ev.hWaiter = pcd.waitFd
@@ -1205,7 +1210,7 @@ else:
     not p.selector.isEmpty() or p.timers.len != 0 or p.callbacks.len != 0
 
   template processBasicCallbacks(ident, rwlist: untyped) =
-    # Process pending descriptor's callbacks.
+    # Process pending descriptor's and AsyncEvent callbacks.
     # Invoke every callback stored in `rwlist`, until first one
     # returned `false`, which means callback wants to stay
     # alive. In such case all remaining callbacks will be added
@@ -1231,7 +1236,14 @@ else:
           newList.add(cb)
 
     withData(p.selector, ident, adata) do:
+      # descriptor still present in queue.
       adata.rwlist = newList & adata.rwlist
+      rLength = len(adata.readList)
+      wLength = len(adata.writeList)
+    do:
+      # descriptor was unregistered in callback via `unregister()`.
+      rLength = -1
+      wLength = -1
 
   template processCustomCallbacks(ident: untyped) =
     # Process pending custom event callbacks. Custom events are
@@ -1250,11 +1262,16 @@ else:
     var cb = curList[0]
     if not cb(fd.AsyncFD):
       newList.add(cb)
-    else:
-      p.selector.unregister(fd)
 
     withData(p.selector, ident, adata) do:
+      # descriptor still present in queue.
       adata.readList = newList & adata.readList
+      if len(adata.readList) == 0:
+        # if no callbacks registered with descriptor, unregister it.
+        p.selector.unregister(fd)
+    do:
+      # descriptor was unregistered in callback via `unregister()`.
+      discard
 
   proc poll*(timeout = 500) =
     var keys: array[64, ReadyKey]
@@ -1275,6 +1292,8 @@ else:
         var custom = false
         let fd = keys[i].fd
         let events = keys[i].events
+        var rLength = 0 # len(data.readList) after callback
+        var wLength = 0 # len(data.writeList) after callback
 
         if Event.Read in events or events == {Event.Error}:
           processBasicCallbacks(fd, readList)
@@ -1283,8 +1302,10 @@ else:
           processBasicCallbacks(fd, writeList)
 
         if Event.User in events or events == {Event.Error}:
-          custom = true
           processBasicCallbacks(fd, readList)
+          custom = true
+          if rLength == 0:
+            p.selector.unregister(fd)
 
         when ioselSupportedPlatform:
           if (customSet * events) != {}:
@@ -1294,13 +1315,10 @@ else:
         # because state `data` can be modified in callback we need to update
         # descriptor events with currently registered callbacks.
         if not custom:
-          var update = false
           var newEvents: set[Event] = {}
-          p.selector.withData(fd, adata) do:
+          if rLength != -1 and wLength != -1:
-            if len(adata.readList) > 0: incl(newEvents, Event.Read)
+            if rLength > 0: incl(newEvents, Event.Read)
-            if len(adata.writeList) > 0: incl(newEvents, Event.Write)
+            if wLength > 0: incl(newEvents, Event.Write)
-            update = true
-          if update:
             p.selector.updateHandle(SocketHandle(fd), newEvents)
         inc(i)
 

lib/windows/winlean.nim

@@ -419,9 +419,6 @@ const
 
   ws2dll = "Ws2_32.dll"
 
-  WSAEWOULDBLOCK* = 10035
-  WSAEINPROGRESS* = 10036
-
 proc wsaGetLastError*(): cint {.importc: "WSAGetLastError", dynlib: ws2dll.}
 
 type
@@ -760,6 +757,11 @@ const
   WSAEDISCON* = 10101
   WSAENETRESET* = 10052
   WSAETIMEDOUT* = 10060
+  WSANOTINITIALISED* = 10093
+  WSAENOTSOCK* = 10038
+  WSAEINPROGRESS* = 10036
+  WSAEINTR* = 10004
+  WSAEWOULDBLOCK* = 10035
   ERROR_NETNAME_DELETED* = 64
   STATUS_PENDING* = 0x103
 

tests/async/tacceptcloserace.nim

@@ -0,0 +1,36 @@
+discard """
+  exitcode: 0
+  output: ""
+"""
+
+import asyncdispatch, net, os, nativesockets
+
+# bug: https://github.com/nim-lang/Nim/issues/5279
+
+proc setupServerSocket(hostname: string, port: Port): AsyncFD =
+  let fd = newNativeSocket()
+  if fd == osInvalidSocket:
+    raiseOSError(osLastError())
+  setSockOptInt(fd, SOL_SOCKET, SO_REUSEADDR, 1)
+  var aiList = getAddrInfo(hostname, port)
+  if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32:
+    freeAddrInfo(aiList)
+    raiseOSError(osLastError())
+  freeAddrInfo(aiList)
+  if listen(fd) != 0:
+    raiseOSError(osLastError())
+  setBlocking(fd, false)
+  result = fd.AsyncFD
+  register(result)
+
+const port = Port(5614)
+for i in 0..100:
+  let serverFd = setupServerSocket("localhost", port)
+  serverFd.accept().callback = proc(fut: Future[AsyncFD]) =
+    if not fut.failed:
+      fut.read().closeSocket()
+
+  var fd = newAsyncNativeSocket()
+  waitFor fd.connect("localhost", port)
+  serverFd.closeSocket()
+  fd.closeSocket()

tests/async/tasync_in_seq_constr.nim

@@ -0,0 +1,17 @@
+discard """
+  output: '''@[1, 2, 3, 4]'''
+"""
+
+# bug #5314
+
+import asyncdispatch
+
+proc bar(): Future[int] {.async.} =
+    await sleepAsync(500)
+    result = 3
+
+proc foo(): Future[seq[int]] {.async.} =
+    await sleepAsync(500)
+    result = @[1, 2, await bar(), 4] # <--- The bug is here
+
+echo waitFor foo()

tests/async/tupcoming_async.nim

@@ -1,9 +1,6 @@
 discard """
   output: '''
 OK
-OK
-OK
-OK
 '''
 """
 
@@ -31,11 +28,50 @@ when defined(upcoming):
     var fut = waitEvent(event)
     asyncCheck(delayedSet(event, 500))
     waitFor(fut or sleepAsync(1000))
-    if fut.finished:
+    if not fut.finished:
-      echo "OK"
-    else:
       echo "eventTest: Timeout expired before event received!"
 
+  proc eventTest5304() =
+    # Event should not be signaled if it was uregistered,
+    # even in case, when poll() was not called yet.
+    # Issue #5304.
+    var unregistered = false
+    let e = newAsyncEvent()
+    addEvent(e) do (fd: AsyncFD) -> bool:
+      assert(not unregistered)
+    e.setEvent()
+    e.unregister()
+    unregistered = true
+    poll()
+
+  proc eventTest5298() =
+    # Event must raise `AssertionError` if event was unregistered twice.
+    # Issue #5298.
+    let e = newAsyncEvent()
+    var eventReceived = false
+    addEvent(e) do (fd: AsyncFD) -> bool:
+      eventReceived = true
+      return true
+    e.setEvent()
+    while not eventReceived:
+      poll()
+    try:
+      e.unregister()
+    except AssertionError:
+      discard
+    e.close()
+
+  proc eventTest5331() =
+    # Event must not raise any exceptions while was unregistered inside of
+    # own callback.
+    # Issue #5331.
+    let e = newAsyncEvent()
+    addEvent(e) do (fd: AsyncFD) -> bool:
+      e.unregister()
+      e.close()
+    e.setEvent()
+    poll()
+
   when ioselSupportedPlatform or defined(windows):
 
     import osproc
@@ -56,7 +92,6 @@ when defined(upcoming):
 
     proc timerTest() =
       waitFor(waitTimer(200))
-      echo "OK"
 
     proc processTest() =
       when defined(windows):
@@ -70,7 +105,7 @@ when defined(upcoming):
       var fut = waitProcess(process)
       waitFor(fut or waitTimer(2000))
       if fut.finished and process.peekExitCode() == 0:
-        echo "OK"
+        discard
       else:
         echo "processTest: Timeout expired before process exited!"
 
@@ -92,23 +127,31 @@ when defined(upcoming):
       var fut = waitSignal(posix.SIGINT)
       asyncCheck(delayedSignal(posix.SIGINT, 500))
       waitFor(fut or waitTimer(1000))
-      if fut.finished:
+      if not fut.finished:
-        echo "OK"
-      else:
         echo "signalTest: Timeout expired before signal received!"
 
   when ioselSupportedPlatform:
     timerTest()
     eventTest()
+    eventTest5304()
+    eventTest5298()
+    eventTest5331()
     processTest()
     signalTest()
+    echo "OK"
   elif defined(windows):
     timerTest()
     eventTest()
+    eventTest5304()
+    eventTest5298()
+    eventTest5331()
     processTest()
     echo "OK"
   else:
     eventTest()
-    echo "OK\nOK\nOK"
+    eventTest5304()
+    eventTest5298()
+    eventTest5331()
+    echo "OK"
 else:
-  echo "OK\nOK\nOK\nOK"
+  echo "OK"

tests/ccgbugs/tmissingvolatile.nim

@@ -1,7 +1,7 @@
 discard """
   output: "1"
   cmd: r"nim c --hints:on $options -d:release $file"
-  ccodecheck: "'NI volatile state0;'"
+  ccodecheck: "'NI volatile state;'"
 """
 
 # bug #1539

tests/collections/ttables.nim

@@ -190,28 +190,26 @@ block zeroHashKeysTest:
   doZeroHashValueTest(toOrderedTable[string,string]({"egg": "sausage"}),
       "", "spam")
 
-# Until #4448 is fixed, these tests will fail
+block clearTableTest:
-when false:
+  var t = data.toTable
-  block clearTableTest:
+  assert t.len() != 0
-    var t = data.toTable
+  t.clear()
-    assert t.len() != 0
+  assert t.len() == 0
-    t.clear()
-    assert t.len() == 0
 
-  block clearOrderedTableTest:
+block clearOrderedTableTest:
-    var t = data.toOrderedTable
+  var t = data.toOrderedTable
-    assert t.len() != 0
+  assert t.len() != 0
-    t.clear()
+  t.clear()
-    assert t.len() == 0
+  assert t.len() == 0
 
-  block clearCountTableTest:
+block clearCountTableTest:
-    var t = initCountTable[string]()
+  var t = initCountTable[string]()
-    t.inc("90", 3)
+  t.inc("90", 3)
-    t.inc("12", 2)
+  t.inc("12", 2)
-    t.inc("34", 1)
+  t.inc("34", 1)
-    assert t.len() != 0
+  assert t.len() != 0
-    t.clear()
+  t.clear()
-    assert t.len() == 0
+  assert t.len() == 0
 
 proc orderedTableSortTest() =
   var t = initOrderedTable[string, int](2)

tests/misc/tfsmonitor.nim

@@ -1,6 +1,6 @@
-#
+discard """
-# fsmonitor test
+  disabled: windows
-#
+"""
 
 import unittest
 import fsmonitor
@@ -9,4 +9,3 @@ suite "fsmonitor":
   test "should not raise OSError, bug# 3611":
     let m = newMonitor()
     m.add("foo", {MonitorCloseWrite, MonitorCloseNoWrite})
-

tests/seq/tsequtils.nim

@@ -7,7 +7,8 @@ Filter Iterator: 7
 Filter: [3, 5, 7]
 FilterIt: [1, 3, 7]
 Concat: [1, 3, 5, 7, 2, 4, 6]
-Deduplicate: [1, 2, 3, 4, 5, 7]'''
+Deduplicate: [1, 2, 3, 4, 5, 7]
+@[()]'''
 
 """
 
@@ -52,4 +53,12 @@ echo "Concat: ", $$(concatseq)
 var seq3 = @[1,2,3,4,5,5,5,7]
 var dedupseq = deduplicate(seq3)
 echo "Deduplicate: ", $$(dedupseq)
+# bug #4973
+type
+  SomeObj = object
+  OtherObj = object
+    field: SomeObj
 
+let aSeq = @[OtherObj(field: SomeObj())]
+let someObjSeq = aSeq.mapIt(it.field)
+echo someObjSeq

tests/stdlib/tstring.nim

@@ -0,0 +1,55 @@
+discard """
+  file: "tstring.nim"
+  output: "OK"
+"""
+const characters = "abcdefghijklmnopqrstuvwxyz"
+const numbers = "1234567890"
+
+var s: string
+
+proc test_string_slice() =
+  # test "slice of length == len(characters)":
+  # replace characters completely by numbers
+  s = characters
+  s[0..^1] = numbers
+  doAssert s == numbers
+
+  # test "slice of length > len(numbers)":
+  # replace characters by slice of same length
+  s = characters
+  s[1..16] = numbers
+  doAssert s == "a1234567890rstuvwxyz"
+
+  # test "slice of length == len(numbers)":
+  # replace characters by slice of same length
+  s = characters
+  s[1..10] = numbers
+  doAssert s == "a1234567890lmnopqrstuvwxyz"
+
+  # test "slice of length < len(numbers)":
+  # replace slice of length. and insert remaining chars
+  s = characters
+  s[1..4] = numbers
+  doAssert s == "a1234567890fghijklmnopqrstuvwxyz"
+
+  # test "slice of length == 1":
+  # replace first character. and insert remaining 9 chars
+  s = characters
+  s[1..1] = numbers
+  doAssert s == "a1234567890cdefghijklmnopqrstuvwxyz"
+
+  # test "slice of length == 0":
+  # insert chars at slice start index
+  s = characters
+  s[2..1] = numbers
+  doAssert s == "ab1234567890cdefghijklmnopqrstuvwxyz"
+
+  # test "slice of negative length":
+  # same as slice of zero length
+  s = characters
+  s[2..0] = numbers
+  doAssert s == "ab1234567890cdefghijklmnopqrstuvwxyz"
+
+  echo("OK")
+
+test_string_slice()

tests/template/tgensymregression.nim

@@ -0,0 +1,21 @@
+
+template mathPerComponent(op: untyped): untyped =
+  proc op*[N,T](v,u: array[N,T]): array[N,T] {.inline.} =
+    for i in 0 ..< len(result):
+      result[i] = `*`(v[i], u[i])
+
+mathPerComponent(`***`)
+# bug #5285
+when true:
+  if isMainModule:
+    var v1: array[3, float64]
+    var v2: array[3, float64]
+    echo repr(v1 *** v2)
+
+
+proc foo(): void =
+  var v1: array[4, float64]
+  var v2: array[4, float64]
+  echo repr(v1 *** v2)
+
+foo()

tests/template/typedescids.nim

@@ -6,7 +6,7 @@ discard """
 
 var i {.compileTime.} = 2
 
-template defineId*(t: typedesc): stmt =
+template defineId*(t: typedesc) =
   const id {.genSym.} = i
   static: inc(i)
   proc idFor*(T: typedesc[t]): int {.inline, raises: [].} = id

tests/testament/categories.nim

@@ -284,9 +284,9 @@ proc compileExample(r: var TResults, pattern, options: string, cat: Category) =
     testNoSpec r, makeTest(test, options, cat)
 
 proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
-  var disabledSet = disabledFiles.toSet()
   for test in os.walkFiles(pattern):
-    if test notin disabledSet:
+    let name = extractFilename(test)
+    if name notin disabledFiles:
       let contents = readFile(test).string
       if contents.contains("when isMainModule"):
         testSpec r, makeTest(test, options, cat, actionRunNoSpec)

tests/testament/tester.nim

@@ -108,12 +108,6 @@ proc callCompiler(cmdTemplate, filename, options: string,
   elif suc =~ pegSuccess:
     result.err = reSuccess
 
-  if result.err == reNimcCrash and
-     ("Your platform is not supported" in result.msg or
-      "cannot open 'sdl'" in result.msg or
-      "cannot open 'opengl'" in result.msg):
-    result.err = reIgnored
-
 proc callCCompiler(cmdTemplate, filename, options: string,
                   target: TTarget): TSpec =
   let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
@@ -393,9 +387,14 @@ proc makeTest(test, options: string, cat: Category, action = actionCompile,
   result = TTest(cat: cat, name: test, options: options,
                  target: target, action: action, startTime: epochTime())
 
-const
+when defined(windows):
-  # array of modules disabled from compilation test of stdlib.
+  const
-  disabledFiles = ["-"]
+    # array of modules disabled from compilation test of stdlib.
+    disabledFiles = ["coro.nim", "fsmonitor.nim"]
+else:
+  const
+    # array of modules disabled from compilation test of stdlib.
+    disabledFiles = ["-"]
 
 include categories
 
@@ -460,7 +459,9 @@ proc main() =
   backend.close()
   if optPedantic:
     var failed = r.total - r.passed - r.skipped
-    if failed > 0 : quit(QuitFailure)
+    if failed > 0:
+      echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ", r.skipped
+      quit(QuitFailure)
 
 if paramCount() == 0:
   quit Usage

tests/vm/tableinstatic.nim

@@ -0,0 +1,38 @@
+discard """
+  nimout: '''0
+0
+0
+{hallo: 123, welt: 456}'''
+"""
+
+import tables
+
+# bug #5327
+
+type
+  MyType* = object
+    counter: int
+
+proc foo(t: var MyType) =
+  echo t.counter
+
+proc bar(t: MyType) =
+  echo t.counter
+
+static:
+  var myValue: MyType
+  myValue.foo # works nicely
+
+  var refValue: ref MyType
+  refValue.new
+
+  refValue[].foo # fails to compile
+  refValue[].bar # works again nicely
+
+static:
+  var otherTable = newTable[string, string]()
+
+  otherTable["hallo"] = "123"
+  otherTable["welt"]  = "456"
+
+  echo otherTable

tests/vm/tcompiletimetable.nim

@@ -1,5 +1,5 @@
 discard """
-  msg: '''2
+  nimout: '''2
 3
 4:2
 Got Hi
@@ -13,7 +13,7 @@ import macros, tables, strtabs
 var ZOOT{.compileTime.} = initTable[int, int](2)
 var iii {.compiletime.} = 1
 
-macro zoo:stmt=
+macro zoo: untyped =
   ZOOT[iii] = iii*2
   inc iii
   echo iii
@@ -22,7 +22,7 @@ zoo
 zoo
 
 
-macro tupleUnpack: stmt =
+macro tupleUnpack: untyped =
   var (y,z) = (4, 2)
   echo y, ":", z
 
@@ -32,14 +32,14 @@ tupleUnpack
 
 var x {.compileTime.}: StringTableRef
 
-macro addStuff(stuff, body: expr): stmt {.immediate.} =
+macro addStuff(stuff, body: untyped): untyped =
   result = newNimNode(nnkStmtList)
 
   if x.isNil:
     x = newStringTable(modeStyleInsensitive)
   x[$stuff] = ""
 
-macro dump(): stmt =
+macro dump(): untyped =
   result = newNimNode(nnkStmtList)
   for y in x.keys: echo "Got ", y
 

tests/vm/tcopy_global_var.nim

@@ -0,0 +1,30 @@
+discard """
+  nimout: "static done"
+"""
+
+# bug #5269
+
+proc assertEq[T](arg0, arg1: T): void =
+  assert arg0 == arg1, $arg0 & " == " & $arg1
+
+type
+  MyType = object
+    str: string
+    a: int
+
+block:
+  var localValue = MyType(str: "Original strning, (OK)", a: 0)
+  var valueCopy = localValue
+  valueCopy.a = 123
+  valueCopy.str = "Modified strning, (not OK when in localValue)"
+  assertEq(localValue.str, "Original strning, (OK)")
+  assertEq(localValue.a,   0)
+
+static:
+  var localValue = MyType(str: "Original strning, (OK)", a: 0)
+  var valueCopy = localValue
+  valueCopy.a = 123
+  valueCopy.str = "Modified strning, (not OK when in localValue)"
+  assertEq(localValue.str, "Original strning, (OK)")
+  assertEq(localValue.a,   0)
+  echo "static done"

tools/downloader.nim

@@ -22,7 +22,7 @@ proc download(pkg: string; c: Controls) {.async.} =
 
   client.onProgressChanged = onProgressChanged
   # XXX give a destination filename instead
-  let contents = await client.getContent("http://nim-lang.org/download/" & pkg & ".zip")
+  let contents = await client.getContent("https://nim-lang.org/download/" & pkg & ".zip")
   let z = "dist" / pkg & ".zip"
   # XXX make this async somehow:
   writeFile(z, contents)

tools/finish.nim

@@ -1,12 +1,12 @@
 
 # -------------- post unzip steps ---------------------------------------------
 
-import strutils, os, osproc, browsers
+import strutils, os, osproc, streams, browsers
 
 const arch = $(sizeof(int)*8)
 
 proc downloadMingw() =
-  openDefaultBrowser("http://nim-lang.org/download/mingw$1.zip" % arch)
+  openDefaultBrowser("https://nim-lang.org/download/mingw$1.zip" % arch)
 
 when defined(windows):
   import registry
@@ -86,18 +86,21 @@ when defined(windows):
   proc checkGccArch(mingw: string): bool =
     let gccExe = mingw / r"gcc.exe"
     if fileExists(gccExe):
+      const nimCompat = "nim_compat.c"
+      writeFile(nimCompat, """typedef int
+        Nim_and_C_compiler_disagree_on_target_architecture[
+          $# == sizeof(void*) ? 1 : -1];
+      """ % $sizeof(int))
       try:
-        let arch = execProcess(gccExe, ["-dumpmachine"], nil, {poStdErrToStdOut,
+        let p = startProcess(gccExe, "", ["-c", nimCompat], nil,
-                                                               poUsePath}).strip
+                            {poStdErrToStdOut, poUsePath})
-        when hostCPU == "i386":
+        #echo p.outputStream.readAll()
-          result = (arch.contains("i686-") and not arch.contains("w64")) or
+        result = p.waitForExit() == 0
-                    arch == "mingw32"
-        elif hostCPU == "amd64":
-          result = arch.contains("x86_64-") or arch.contains("i686-w64-mingw32")
-        else:
-          {.error: "Unknown CPU for Windows.".}
       except OSError, IOError:
         result = false
+      finally:
+        removeFile(nimCompat)
+        removeFile(nimCompat.changeFileExt("o"))
 
   proc defaultMingwLocations(): seq[string] =
     proc probeDir(dir: string; result: var seq[string]) =

tools/niminst/niminst.nim

@@ -507,7 +507,7 @@ proc srcdist(c: var ConfigData) =
   if not existsDir(getOutputDir(c) / "c_code"):
     createDir(getOutputDir(c) / "c_code")
   for x in walkFiles(c.libpath / "lib/*.h"):
-    echo(getOutputDir(c) / "c_code" / extractFilename(x))
+    when false: echo(getOutputDir(c) / "c_code" / extractFilename(x))
     copyFile(dest=getOutputDir(c) / "c_code" / extractFilename(x), source=x)
   var winIndex = -1
   var intel32Index = -1
@@ -624,7 +624,7 @@ proc xzDist(c: var ConfigData; windowsZip=false) =
 
   proc processFile(destFile, src: string) =
     let dest = tmpDir / destFile
-    echo "Copying ", src, " to ", dest
+    when false: echo "Copying ", src, " to ", dest
     if not existsFile(src):
       echo "[Warning] Source file doesn't exist: ", src
     let destDir = dest.splitFile.dir

tools/nimsuggest/crashtester.nim

@@ -0,0 +1,52 @@
+
+
+import strutils, os, osproc, streams
+
+const
+  DummyEof = "!EOF!"
+
+proc getPosition(s: string): (int, int) =
+  result = (1, 1)
+  var col = 0
+  for i in 0..<s.len:
+    if s[i] == '\L':
+      inc result[0]
+      col = 0
+    else:
+      inc col
+  result[1] = col+1
+
+proc callNimsuggest() =
+  let cl = parseCmdLine("nimsuggest --tester temp000.nim")
+  var p = startProcess(command=cl[0], args=cl[1 .. ^1],
+                       options={poStdErrToStdOut, poUsePath,
+                       poInteractive, poDemon})
+  let outp = p.outputStream
+  let inp = p.inputStream
+  var report = ""
+  var a = newStringOfCap(120)
+  let contents = readFile("tools/nimsuggest/crashtester.nim")
+  try:
+    # read and ignore anything nimsuggest says at startup:
+    while outp.readLine(a):
+      if a == DummyEof: break
+
+    var line = 0
+    for i in 0..< contents.len:
+      let slic = contents[0..i]
+      writeFile("temp000.nim", slic)
+      let (line, col) = getPosition(slic)
+      inp.writeLine("sug temp000.nim:$#:$#" % [$line, $col])
+      inp.flush()
+      var answer = ""
+      while outp.readLine(a):
+        if a == DummyEof: break
+        answer.add a
+        answer.add '\L'
+      echo answer
+  finally:
+    inp.writeLine("quit")
+    inp.flush()
+    close(p)
+
+callNimsuggest()

tools/nimsuggest/nimsuggest.nim

@@ -418,7 +418,7 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
       options.gProjectName = unixToNativePath(p.key)
       # if processArgument(pass, p, argsCount): break
 
-proc handleCmdLine(cache: IdentCache) =
+proc handleCmdLine(cache: IdentCache; config: ConfigRef) =
   if paramCount() == 0:
     stdout.writeline(Usage)
   else:
@@ -444,23 +444,23 @@ proc handleCmdLine(cache: IdentCache) =
     gPrefixDir = binaryPath.splitPath().head.parentDir()
     #msgs.writelnHook = proc (line: string) = logStr(line)
 
-    loadConfigs(DefaultConfig, cache) # load all config files
+    loadConfigs(DefaultConfig, cache, config) # load all config files
     # now process command line arguments again, because some options in the
     # command line can overwite the config file's settings
     options.command = "nimsuggest"
     let scriptFile = gProjectFull.changeFileExt("nims")
     if fileExists(scriptFile):
-      runNimScript(cache, scriptFile, freshDefines=false)
+      runNimScript(cache, scriptFile, freshDefines=false, config)
       # 'nim foo.nims' means to just run the NimScript file and do nothing more:
       if scriptFile == gProjectFull: return
     elif fileExists(gProjectPath / "config.nims"):
       # directory wide NimScript file
-      runNimScript(cache, gProjectPath / "config.nims", freshDefines=false)
+      runNimScript(cache, gProjectPath / "config.nims", freshDefines=false, config)
 
     extccomp.initVars()
     processCmdLine(passCmd2, "")
 
-    let graph = newModuleGraph()
+    let graph = newModuleGraph(config)
     graph.suggestMode = true
     mainCommand(graph, cache)
 
@@ -472,4 +472,4 @@ when false:
 
 condsyms.initDefines()
 defineSymbol "nimsuggest"
-handleCmdline(newIdentCache())
+handleCmdline(newIdentCache(), newConfigRef())

tools/nimsuggest/tester.nim

@@ -161,8 +161,16 @@ proc runTest(filename: string): int =
           answer.add '\L'
         if resp != answer and not smartCompare(resp, answer):
           report.add "\nTest failed: " & filename
-          report.add "\n  Expected:  " & resp
+          var hasDiff = false
-          report.add "\n  But got:   " & answer
+          for i in 0..min(resp.len-1, answer.len-1):
+            if resp[i] != answer[i]:
+              report.add "\n  Expected:  " & resp.substr(i)
+              report.add "\n  But got:   " & answer.substr(i)
+              hasDiff = true
+              break
+          if not hasDiff:
+            report.add "\n  Expected:  " & resp
+            report.add "\n  But got:   " & answer
   finally:
     inp.writeLine("quit")
     inp.flush()

tools/nimsuggest/tests/tdef1.nim

@@ -1,12 +1,12 @@
 discard """
 $nimsuggest --tester $file
 >def $1
-def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100
+def;;skProc;;tdef1.hello;;proc (): string{.noSideEffect, gcsafe, locks: 0.};;$file;;9;;5;;"Return hello";;100
 >def $1
-def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100
+def;;skProc;;tdef1.hello;;proc (): string{.noSideEffect, gcsafe, locks: 0.};;$file;;9;;5;;"Return hello";;100
 """
 
-proc hello() string =
+proc hello(): string =
   ## Return hello
   "Hello"
 

tools/nimsuggest/tests/tstrutils.nim

@@ -1,7 +1,7 @@
 discard """
 $nimsuggest --tester lib/pure/strutils.nim
 >def lib/pure/strutils.nim:2300:6
-def;;skTemplate;;system.doAssert;;proc (cond: bool, msg: string): typed;;*/lib/system.nim;;*;;9;;"";;100
+def;;skTemplate;;system.doAssert;;proc (cond: bool, msg: string): typed;;*/lib/system.nim;;*;;9;;"same as `assert` but is always turned on and not affected by the\x0A``--assertions`` command line switch.";;100
 """
 
 # Line 2300 in strutils.nim is doAssert and this is unlikely to change

tools/nimsuggest/tests/twithin_macro.nim

@@ -0,0 +1,213 @@
+
+import macros
+
+macro class*(head, body: untyped): untyped =
+  # The macro is immediate, since all its parameters are untyped.
+  # This means, it doesn't resolve identifiers passed to it.
+
+  var typeName, baseName: NimNode
+
+  # flag if object should be exported
+  var exported: bool
+
+  if head.kind == nnkInfix and head[0].ident == !"of":
+    # `head` is expression `typeName of baseClass`
+    # echo head.treeRepr
+    # --------------------
+    # Infix
+    #   Ident !"of"
+    #   Ident !"Animal"
+    #   Ident !"RootObj"
+    typeName = head[1]
+    baseName = head[2]
+
+  elif head.kind == nnkInfix and head[0].ident == !"*" and
+       head[2].kind == nnkPrefix and head[2][0].ident == !"of":
+    # `head` is expression `typeName* of baseClass`
+    # echo head.treeRepr
+    # --------------------
+    # Infix
+    #   Ident !"*"
+    #   Ident !"Animal"
+    #   Prefix
+    #     Ident !"of"
+    #     Ident !"RootObj"
+    typeName = head[1]
+    baseName = head[2][1]
+    exported = true
+
+  else:
+    quit "Invalid node: " & head.lispRepr
+
+  # The following prints out the AST structure:
+  #
+  # import macros
+  # dumptree:
+  #   type X = ref object of Y
+  #     z: int
+  # --------------------
+  # StmtList
+  #   TypeSection
+  #     TypeDef
+  #       Ident !"X"
+  #       Empty
+  #       RefTy
+  #         ObjectTy
+  #           Empty
+  #           OfInherit
+  #             Ident !"Y"
+  #           RecList
+  #             IdentDefs
+  #               Ident !"z"
+  #               Ident !"int"
+  #               Empty
+
+  # create a type section in the result
+  result =
+    if exported:
+      # mark `typeName` with an asterisk
+      quote do:
+        type `typeName`* = ref object of `baseName`
+    else:
+      quote do:
+        type `typeName` = ref object of `baseName`
+
+  # echo treeRepr(body)
+  # --------------------
+  # StmtList
+  #   VarSection
+  #     IdentDefs
+  #       Ident !"name"
+  #       Ident !"string"
+  #       Empty
+  #     IdentDefs
+  #       Ident !"age"
+  #       Ident !"int"
+  #       Empty
+  #   MethodDef
+  #     Ident !"vocalize"
+  #     Empty
+  #     Empty
+  #     FormalParams
+  #       Ident !"string"
+  #     Empty
+  #     Empty
+  #     StmtList
+  #       StrLit ...
+  #   MethodDef
+  #     Ident !"age_human_yrs"
+  #     Empty
+  #     Empty
+  #     FormalParams
+  #       Ident !"int"
+  #     Empty
+  #     Empty
+  #     StmtList
+  #       DotExpr
+  #         Ident !"this"
+  #         Ident !"age"
+
+  # var declarations will be turned into object fields
+  var recList = newNimNode(nnkRecList)
+
+  # expected name of constructor
+  let ctorName = newIdentNode("new" & $typeName)
+
+  # Iterate over the statements, adding `this: T`
+  # to the parameters of functions, unless the
+  # function is a constructor
+  for node in body.children:
+    case node.kind:
+
+      of nnkMethodDef, nnkProcDef:
+        # check if it is the ctor proc
+        if node.name.kind != nnkAccQuoted and node.name.basename == ctorName:
+          # specify the return type of the ctor proc
+          node.params[0] = typeName
+        else:
+          # inject `self: T` into the arguments
+          node.params.insert(1, newIdentDefs(ident("self"), typeName))
+        result.add(node)
+
+      of nnkVarSection:
+        # variables get turned into fields of the type.
+        for n in node.children:
+          recList.add(n)
+
+      else:
+        result.add(node)
+
+  # Inspect the tree structure:
+  #
+  # echo result.treeRepr
+  # --------------------
+  # StmtList
+  #   TypeSection
+  #     TypeDef
+  #       Ident !"Animal"
+  #       Empty
+  #       RefTy
+  #         ObjectTy
+  #           Empty
+  #           OfInherit
+  #             Ident !"RootObj"
+  #           Empty   <= We want to replace this
+  # MethodDef
+  # ...
+
+  result[0][0][2][0][2] = recList
+
+  # Lets inspect the human-readable version of the output
+  #echo repr(result)
+
+# ---
+
+class Animal of RootObj:
+  var name: string
+  var age: int
+  method vocalize: string {.base.} = "..." # use `base` pragma to annonate base methods
+  method age_human_yrs: int {.base.} = self.age # `this` is injected
+  proc `$`: string = "animal:" & self.name & ":" & $self.age
+
+class Dog of Animal:
+  method vocalize: string = "woof"
+  method age_human_yrs: int = self.age * 7
+  proc `$`: string = "dog:" & self.name & ":" & $self.age
+
+class Cat of Animal:
+  method vocalize: string = "meow"
+  proc `$`: string = "cat:" & self.name & ":" & $self.age
+
+class Rabbit of Animal:
+  proc newRabbit(name: string, age: int) = # the constructor doesn't need a return type
+    result = Rabbit(name: name, age: age)
+  method vocalize: string = "meep"
+  proc `$`: string =
+    self.#[!]#
+    result = "rabbit:" & self.name & ":" & $self.age
+
+# ---
+
+var animals: seq[Animal] = @[]
+animals.add(Dog(name: "Sparky", age: 10))
+animals.add(Cat(name: "Mitten", age: 10))
+
+for a in animals:
+  echo a.vocalize()
+  echo a.age_human_yrs()
+
+let r = newRabbit("Fluffy", 3)
+echo r.vocalize()
+echo r.age_human_yrs()
+echo r
+
+discard """
+$nimsuggest --tester $file
+>sug $1
+sug;;skField;;name;;string;;$file;;166;;6;;"";;100
+sug;;skField;;age;;int;;$file;;167;;6;;"";;100
+sug;;skMethod;;twithin_macro.age_human_yrs;;proc (self: Animal): int{.noSideEffect, gcsafe, locks: 0.};;$file;;169;;9;;"";;100
+sug;;skMacro;;twithin_macro.class;;proc (head: untyped, body: untyped): untyped{.gcsafe, locks: <unknown>.};;$file;;4;;6;;"Iterates over the children of the NimNode ``n``.";;100
+sug;;skMethod;;twithin_macro.vocalize;;proc (self: Animal): string{.noSideEffect, gcsafe, locks: 0.};;$file;;168;;9;;"";;100
+sug;;skMethod;;twithin_macro.vocalize;;proc (self: Rabbit): string{.noSideEffect, gcsafe, locks: 0.};;$file;;184;;9;;"";;100*
+"""

tools/nimweb.nim

@@ -263,8 +263,8 @@ proc findNim(): string =
 
 proc exec(cmd: string) =
   echo(cmd)
-  let (_, exitCode) = osproc.execCmdEx(cmd)
+  let (outp, exitCode) = osproc.execCmdEx(cmd)
-  if exitCode != 0: quit("external program failed")
+  if exitCode != 0: quit outp
 
 proc sexec(cmds: openarray[string]) =
   ## Serial queue wrapper around exec.
@@ -272,10 +272,13 @@ proc sexec(cmds: openarray[string]) =
 
 proc mexec(cmds: openarray[string], processors: int) =
   ## Multiprocessor version of exec
-  if processors < 2:
+  doAssert processors > 0, "nimweb needs at least one processor"
+  if processors == 1:
     sexec(cmds)
     return
-  if execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd}) != 0:
+  let r = execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd},
+    n = processors)
+  if r != 0:
     echo "external program failed, retrying serial work queue for logs!"
     sexec(cmds)
 

tools/website.tmpl

@@ -187,15 +187,15 @@ runForever()
 					</div>
 					<div>
 						<h4>Community</h4>
-						<a href="http://forum.nim-lang.org">User Forum</a>
+						<a href="https://forum.nim-lang.org">User Forum</a>
             <a href="http://webchat.freenode.net/?channels=nim">Online IRC</a>
-            <a href="http://irclogs.nim-lang.org/">IRC Logs</a>
+            <a href="https://irclogs.nim-lang.org/">IRC Logs</a>
 					</div>
 				</div>
 				<div id="foot-legal">
 					<h4>Written in Nim - Powered by <a href="https://github.com/dom96/jester">Jester</a></h4>
 					Web Design by <a href="http://reign-studios.net/philipwitte/">Philip Witte</a> &amp; <a href="http://picheta.me/">Dominik Picheta</a><br>
-					Copyright © 2015 - <a href="http://nim-lang.org/blog/">Andreas Rumpf</a> &amp; <a href="https://github.com/nim-lang/nim/graphs/contributors">Contributors</a>
+					Copyright © 2017 - <a href="https://nim-lang.org/blog/">Andreas Rumpf</a> &amp; <a href="https://github.com/nim-lang/nim/graphs/contributors">Contributors</a>
 				</div>
 			</div>
 		</footer>

web/community.rst

@@ -6,7 +6,7 @@ Nim's Community
   Forum
   -----
 
-  The `Nim forum <http://forum.nim-lang.org/>`_ is the place where most
+  The `Nim forum <https://forum.nim-lang.org/>`_ is the place where most
   discussions related to the language happen. It not only includes discussions
   relating to the design of Nim but also allows for beginners to ask questions
   relating to Nim.
@@ -35,7 +35,7 @@ Nim's Community
   welcome any questions that you may have!
 
   You may also be interested in reading the
-  `IRC logs <http://irclogs.nim-lang.org/>`_ which are an archive of all
+  `IRC logs <https://irclogs.nim-lang.org/>`_ which are an archive of all
   of the previous discussions that took place in the IRC channel.
 
 

web/download.rst

@@ -16,8 +16,14 @@ We now encourage you to install via the provided zipfiles:
 * | 64 bit: `nim-0.16.0_x64.zip <download/nim-0.16.0_x64.zip>`_
   | SHA-256  e667cdad1ae8e9429147aea5031fa8a80c4ccef6d274cec0e9480252d9c3168c
 
-Unzip these where you want and optionally run ``finish.exe`` to
+Unzip these where you want and **optionally** run ``finish.exe`` to
-detect your MingW environment.
+detect your MingW environment. (Though that's not reliable yet.)
+
+You can find the required DLLs here, if you lack them for some reason:
+
+* | 32 and 64 bit: `DLLs.zip <download/dlls.zip>`_
+  | SHA-256  198112d3d6dc74d7964ba452158d44bfa57adef4dc47be8c39903f2a24e4a555
+
 
 Exes
 %%%%

web/news/e031_version_0_16_2.rst

@@ -41,3 +41,69 @@ these procedures.
 
 In the near future we will be converting all exception types to refs to
 remove the need for the ``newException`` template.
+
+Bugfixes
+--------
+
+The list below has been generated based on the commits in Nim's git
+repository. As such it lists only the issues which have been closed
+via a commit, for a full list see
+`this link on Github <https://github.com/nim-lang/Nim/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222017-01-07+..+2017-02-06%22+>`_.
+
+- Fixed "Weird compilation bug"
+  (`#4884 <https://github.com/nim-lang/Nim/issues/4884>`_)
+- Fixed "Return by arg optimization does not set result to default value"
+  (`#5098 <https://github.com/nim-lang/Nim/issues/5098>`_)
+- Fixed "upcoming asyncdispatch doesn't remove recv callback if remote side closed socket"
+  (`#5128 <https://github.com/nim-lang/Nim/issues/5128>`_)
+- Fixed "compiler bug, executable writes into wrong memory"
+  (`#5218 <https://github.com/nim-lang/Nim/issues/5218>`_)
+- Fixed "Module aliasing fails when multiple modules have the same original name"
+  (`#5112 <https://github.com/nim-lang/Nim/issues/5112>`_)
+- Fixed "JS: var argument + case expr with arg = bad codegen"
+  (`#5244 <https://github.com/nim-lang/Nim/issues/5244>`_)
+- Fixed "compiler reject proc's param shadowing inside template"
+  (`#5225 <https://github.com/nim-lang/Nim/issues/5225>`_)
+- Fixed "const value not accessible in proc"
+  (`#3434 <https://github.com/nim-lang/Nim/issues/3434>`_)
+- Fixed "Compilation regression 0.13.0 vs 0.16.0 in compile-time evaluation"
+  (`#5237 <https://github.com/nim-lang/Nim/issues/5237>`_)
+- Fixed "Regression: JS: wrong field-access codegen"
+  (`#5234 <https://github.com/nim-lang/Nim/issues/5234>`_)
+- Fixed "fixes #5234"
+  (`#5240 <https://github.com/nim-lang/Nim/issues/5240>`_)
+- Fixed "JS Codegen: duplicated fields in object constructor"
+  (`#5271 <https://github.com/nim-lang/Nim/issues/5271>`_)
+- Fixed "RFC: improving JavaScript FFI"
+  (`#4873 <https://github.com/nim-lang/Nim/issues/4873>`_)
+- Fixed "Wrong result type when using bitwise and"
+  (`#5216 <https://github.com/nim-lang/Nim/issues/5216>`_)
+- Fixed "upcoming.asyncdispatch is prone to memory leaks"
+  (`#5290 <https://github.com/nim-lang/Nim/issues/5290>`_)
+- Fixed "Using threadvars leads to crash on Windows when threads are created/destroyed"
+  (`#5301 <https://github.com/nim-lang/Nim/issues/5301>`_)
+- Fixed "Type inferring templates do not work with non-ref types."
+  (`#4973 <https://github.com/nim-lang/Nim/issues/4973>`_)
+- Fixed "Nimble package list no longer works on lib.html"
+  (`#5318 <https://github.com/nim-lang/Nim/issues/5318>`_)
+- Fixed "Missing file name and line number in error message"
+  (`#4992 <https://github.com/nim-lang/Nim/issues/4992>`_)
+- Fixed "ref type can't be converted to var parameter in VM"
+  (`#5327 <https://github.com/nim-lang/Nim/issues/5327>`_)
+- Fixed "nimweb ignores the value of --parallelBuild"
+  (`#5328 <https://github.com/nim-lang/Nim/issues/5328>`_)
+- Fixed "Cannot unregister/close AsyncEvent from within its handler"
+  (`#5331 <https://github.com/nim-lang/Nim/issues/5331>`_)
+- Fixed "name collision with template instanciated generic inline function with inlined iterator specialization used from different modules"
+  (`#5285 <https://github.com/nim-lang/Nim/issues/5285>`_)
+- Fixed "object in VM does not have value semantic"
+  (`#5269 <https://github.com/nim-lang/Nim/issues/5269>`_)
+- Fixed "Unstable tuple destructuring behavior in Nim VM"
+  (`#5221 <https://github.com/nim-lang/Nim/issues/5221>`_)
+- Fixed "nre module breaks os templates"
+  (`#4996 <https://github.com/nim-lang/Nim/issues/4996>`_)
+- Fixed "Cannot implement distinct seq with setLen"
+  (`#5090 <https://github.com/nim-lang/Nim/issues/5090>`_)
+- Fixed "await inside array/dict literal produces invalid code"
+  (`#5314 <https://github.com/nim-lang/Nim/issues/5314>`_)
+