Merge branch 'devel' into araq

appveyor.yml

@@ -56,7 +56,7 @@ build_script:
 
 test_script:
   - tests\testament\tester --pedantic all -d:nimCoroutines
-  - koch csource
+#  - koch csource
-  - koch zip
+#  - koch zip
 
 deploy: off

compiler/ast.nim

@@ -224,7 +224,7 @@ type
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # already 32 flags!
+  TSymFlag* = enum    # already 33 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -453,9 +453,10 @@ type
     nfIsRef     # this node is a 'ref' node; used for the VM
     nfPreventCg # this node should be ignored by the codegen
     nfBlockArg  # this a stmtlist appearing in a call (e.g. a do block)
+    nfFromTemplate # a top-level node returned from a template
 
   TNodeFlags* = set[TNodeFlag]
-  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 30)
+  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: beyond that)
     tfVarargs,        # procedure has C styled varargs
                       # tyArray type represeting a varargs list
     tfNoSideEffect,   # procedure type does not allow side effects
@@ -508,6 +509,9 @@ type
     tfTriggersCompileTime # uses the NimNode type which make the proc
                           # implicitly '.compiletime'
     tfRefsAnonObj     # used for 'ref object' and 'ptr object'
+    tfCovariant       # covariant generic param mimicing a ptr type
+    tfWeakCovariant   # covariant generic param mimicing a seq/array type
+    tfContravariant   # contravariant generic param
 
   TTypeFlags* = set[TTypeFlag]
 
@@ -956,7 +960,8 @@ const
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
                                       nfDotSetter, nfDotField,
-                                      nfIsRef, nfPreventCg, nfLL}
+                                      nfIsRef, nfPreventCg, nfLL,
+                                      nfFromTemplate}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -1007,15 +1012,17 @@ proc add*(father, son: PNode) =
   if isNil(father.sons): father.sons = @[]
   add(father.sons, son)
 
-proc `[]`*(n: PNode, i: int): PNode {.inline.} =
+type Indexable = PNode | PType
-  result = n.sons[i]
+
+template `[]`*(n: Indexable, i: int): Indexable =
+  n.sons[i]
 
 template `-|`*(b, s: untyped): untyped =
   (if b >= 0: b else: s.len + b)
 
 # son access operators with support for negative indices
-template `{}`*(n: PNode, i: int): untyped = n[i -| n]
+template `{}`*(n: Indexable, i: int): untyped = n[i -| n]
-template `{}=`*(n: PNode, i: int, s: PNode) =
+template `{}=`*(n: Indexable, i: int, s: Indexable) =
   n.sons[i -| n] = s
 
 when defined(useNodeIds):

compiler/astalgo.nim

@@ -219,7 +219,7 @@ proc rspaces(x: int): Rope =
 
 proc toYamlChar(c: char): string =
   case c
-  of '\0'..'\x1F', '\x80'..'\xFF': result = "\\u" & strutils.toHex(ord(c), 4)
+  of '\0'..'\x1F', '\x7F'..'\xFF': result = "\\u" & strutils.toHex(ord(c), 4)
   of '\'', '\"', '\\': result = '\\' & c
   else: result = $c
 
@@ -407,6 +407,8 @@ proc debugTree(n: PNode, indent: int, maxRecDepth: int;
     var istr = rspaces(indent + 2)
     result = "{$N$1\"kind\": $2" %
              [istr, makeYamlString($n.kind)]
+    when defined(useNodeIds):
+      addf(result, ",$N$1\"id\": $2", [istr, rope(n.id)])
     addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(n.info)])
     if maxRecDepth != 0:
       addf(result, ",$N$1\"flags\": $2", [istr, rope($n.flags)])
@@ -574,12 +576,6 @@ proc strTableAdd(t: var TStrTable, n: PSym) =
   strTableRawInsert(t.data, n)
   inc(t.counter)
 
-proc reallySameIdent(a, b: string): bool {.inline.} =
-  when defined(nimfix):
-    result = a[0] == b[0]
-  else:
-    result = true
-
 proc strTableIncl*(t: var TStrTable, n: PSym; onConflictKeepOld=false): bool {.discardable.} =
   # returns true if n is already in the string table:
   # It is essential that `n` is written nevertheless!
@@ -594,7 +590,7 @@ proc strTableIncl*(t: var TStrTable, n: PSym; onConflictKeepOld=false): bool {.d
     # and overloading: (var x=@[]; x).mapIt(it).
     # So it is possible the very same sym is added multiple
     # times to the symbol table which we allow here with the 'it == n' check.
-    if it.name.id == n.name.id and reallySameIdent(it.name.s, n.name.s):
+    if it.name.id == n.name.id:
       if it == n: return false
       replaceSlot = h
     h = nextTry(h, high(t.data))

compiler/ccgexprs.nim

@@ -78,8 +78,10 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope =
                         [p.module.tmpBase, rope(id)])
     else:
       result = makeCString(n.strVal)
-  of nkFloatLit..nkFloat64Lit:
+  of nkFloatLit, nkFloat64Lit:
     result = rope(n.floatVal.toStrMaxPrecision)
+  of nkFloat32Lit:
+    result = rope(n.floatVal.toStrMaxPrecision("f"))
   else:
     internalError(n.info, "genLiteral(" & $n.kind & ')')
     result = nil
@@ -535,7 +537,7 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       "(($4)($1) * ($4)($2))", # MulF64
       "(($4)($1) / ($4)($2))", # DivF64
 
-      "($4)((NU$3)($1) >> (NU$3)($2))", # ShrI
+      "($4)((NU$5)($1) >> (NU$3)($2))", # ShrI
       "($4)((NU$3)($1) << (NU$3)($2))", # ShlI
       "($4)($1 & $2)",      # BitandI
       "($4)($1 | $2)",      # BitorI
@@ -575,16 +577,17 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       "($1 != $2)"]           # Xor
   var
     a, b: TLoc
-    s: BiggestInt
+    s, k: BiggestInt
   assert(e.sons[1].typ != nil)
   assert(e.sons[2].typ != nil)
   initLocExpr(p, e.sons[1], a)
   initLocExpr(p, e.sons[2], b)
   # BUGFIX: cannot use result-type here, as it may be a boolean
   s = max(getSize(a.t), getSize(b.t)) * 8
+  k = getSize(a.t) * 8
   putIntoDest(p, d, e.typ,
               binArithTab[op] % [rdLoc(a), rdLoc(b), rope(s),
-                                      getSimpleTypeDesc(p.module, e.typ)])
+                                      getSimpleTypeDesc(p.module, e.typ), rope(k)])
 
 proc genEqProc(p: BProc, e: PNode, d: var TLoc) =
   var a, b: TLoc
@@ -654,7 +657,9 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) =
       d.s = OnHeap
   else:
     var a: TLoc
-    let typ = skipTypes(e.sons[0].typ, abstractInst)
+    var typ = skipTypes(e.sons[0].typ, abstractInst)
+    if typ.kind in {tyUserTypeClass, tyUserTypeClassInst} and typ.isResolvedUserTypeClass:
+      typ = typ.lastSon
     if typ.kind == tyVar and tfVarIsPtr notin typ.flags and p.module.compileToCpp and e.sons[0].kind == nkHiddenAddr:
       initLocExprSingleUse(p, e[0][0], d)
       return
@@ -2132,6 +2137,13 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     if n.sons[0].kind != nkEmpty:
       genLineDir(p, n)
       var a: TLoc
+      if n[0].kind in nkCallKinds:
+        # bug #6037: do not assign to a temp in C++ mode:
+        incl a.flags, lfSingleUse
+        genCall(p, n[0], a)
+        if lfSingleUse notin a.flags:
+          line(p, cpsStmts, a.r & ";" & tnl)
+      else:
         initLocExpr(p, n.sons[0], a)
   of nkAsmStmt: genAsmStmt(p, n)
   of nkTryStmt:
@@ -2157,8 +2169,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       # are not transformed correctly. We work around this issue (#411) here
       # by ensuring it's no inner proc (owner is a module):
       if prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags:
-        if (optDeadCodeElim notin gGlobalOptions and
+        if (not emitLazily(prc)) or
-            sfDeadCodeElim notin getModule(prc).flags) or
             ({sfExportc, sfCompilerProc} * prc.flags == {sfExportc}) or
             (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or
             (prc.kind == skMethod):
@@ -2214,7 +2225,7 @@ proc getNullValueAux(p: BProc; obj, cons: PNode, result: var Rope) =
     let field = obj.sym
     for i in 1..<cons.len:
       if cons[i].kind == nkExprColonExpr:
-        if cons[i][0].sym == field:
+        if cons[i][0].sym.name == field.name:
           result.add genConstExpr(p, cons[i][1])
           return
       elif i == field.position:

compiler/ccgstmts.nim

@@ -39,13 +39,11 @@ proc genVarTuple(p: BProc, n: PNode) =
   var L = sonsLen(n)
 
   # if we have a something that's been captured, use the lowering instead:
-  var useLowering = false
   for i in countup(0, L-3):
     if n[i].kind != nkSym:
-      useLowering = true; break
-  if useLowering:
       genStmts(p, lowerTupleUnpacking(n, p.prc))
       return
+
   genLineDir(p, n)
   initLocExpr(p, n.sons[L-1], tup)
   var t = tup.t.skipTypes(abstractInst)
@@ -182,10 +180,11 @@ proc genGotoVar(p: BProc; value: PNode) =
     lineF(p, cpsStmts, "goto NIMSTATE_$#;$n", [value.intVal.rope])
 
 proc genSingleVar(p: BProc, a: PNode) =
-  var v = a.sons[0].sym
+  let v = a.sons[0].sym
-  if {sfCompileTime, sfGoto} * v.flags != {}:
+  if sfCompileTime in v.flags: return
+  if sfGoto in v.flags:
     # translate 'var state {.goto.} = X' into 'goto LX':
-    if sfGoto in v.flags: genGotoVar(p, a.sons[2])
+    genGotoVar(p, a.sons[2])
     return
   var targetProc = p
   if sfGlobal in v.flags:

compiler/ccgtypes.nim

@@ -470,7 +470,12 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
           let a = genRecordFieldsAux(m, k, "$1.$2" % [ae, sname], rectype,
                                      check)
           if a != nil:
+            if tfPacked notin rectype.flags:
               add(unionBody, "struct {")
+            else:
+              addf(unionBody, CC[cCompiler].structStmtFmt,
+                [rope"struct", nil, rope(CC[cCompiler].packedPragma)])
+              add(unionBody, "{")
             add(unionBody, a)
             addf(unionBody, "} $1;$n", [sname])
         else:

compiler/ccgutils.nim

@@ -11,7 +11,7 @@
 
 import
   ast, astalgo, ropes, hashes, strutils, types, msgs, wordrecg,
-  platform, trees
+  platform, trees, options
 
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
@@ -211,20 +211,8 @@ proc mangle*(name: string): string =
   if requiresUnderscore:
     result.add "_"
 
-proc makeLLVMString*(s: string): Rope =
+proc emitLazily*(s: PSym): bool {.inline.} =
-  const MaxLineLength = 64
+  result = optDeadCodeElim in gGlobalOptions or
-  result = nil
+           sfDeadCodeElim in getModule(s).flags
-  var res = "c\""
-  for i in countup(0, len(s) - 1):
-    if (i + 1) mod MaxLineLength == 0:
-      add(result, rope(res))
-      setLen(res, 0)
-    case s[i]
-    of '\0'..'\x1F', '\x80'..'\xFF', '\"', '\\':
-      add(res, '\\')
-      add(res, toHex(ord(s[i]), 2))
-    else: add(res, s[i])
-  add(res, "\\00\"")
-  add(result, rope(res))
 
 initTypeTables()

compiler/cgen.nim

@@ -47,10 +47,6 @@ proc findPendingModule(m: BModule, s: PSym): BModule =
   var ms = getModule(s)
   result = m.g.modules[ms.position]
 
-proc emitLazily(s: PSym): bool {.inline.} =
-  result = optDeadCodeElim in gGlobalOptions or
-           sfDeadCodeElim in getModule(s).flags
-
 proc initLoc(result: var TLoc, k: TLocKind, typ: PType, s: TStorageLoc) =
   result.k = k
   result.s = s
@@ -142,6 +138,13 @@ proc ropecg(m: BModule, frmt: FormatStr, args: varargs[Rope]): Rope =
 template rfmt(m: BModule, fmt: string, args: varargs[Rope]): untyped =
   ropecg(m, fmt, args)
 
+var indent = "\t".rope
+
+proc indentLine(p: BProc, r: Rope): Rope =
+  result = r
+  for i in countup(0, p.blocks.len-1):
+    prepend(result, indent)
+
 proc appcg(m: BModule, c: var Rope, frmt: FormatStr,
            args: varargs[Rope]) =
   add(c, ropecg(m, frmt, args))
@@ -154,11 +157,6 @@ proc appcg(p: BProc, s: TCProcSection, frmt: FormatStr,
            args: varargs[Rope]) =
   add(p.s(s), ropecg(p.module, frmt, args))
 
-var indent = "\t".rope
-proc indentLine(p: BProc, r: Rope): Rope =
-  result = r
-  for i in countup(0, p.blocks.len-1): prepend(result, indent)
-
 proc line(p: BProc, s: TCProcSection, r: Rope) =
   add(p.s(s), indentLine(p, r))
 
@@ -612,8 +610,18 @@ proc generateHeaders(m: BModule) =
       addf(m.s[cfsHeaders], "#include \"$1\"$N", [rope(it)])
     else:
       addf(m.s[cfsHeaders], "#include $1$N", [rope(it)])
+  add(m.s[cfsHeaders], "#undef LANGUAGE_C" & tnl)
+  add(m.s[cfsHeaders], "#undef MIPSEB" & tnl)
+  add(m.s[cfsHeaders], "#undef MIPSEL" & tnl)
+  add(m.s[cfsHeaders], "#undef PPC" & tnl)
+  add(m.s[cfsHeaders], "#undef R3000" & tnl)
+  add(m.s[cfsHeaders], "#undef R4000" & tnl)
+  add(m.s[cfsHeaders], "#undef i386" & tnl)
   add(m.s[cfsHeaders], "#undef linux" & tnl)
+  add(m.s[cfsHeaders], "#undef mips" & tnl)
   add(m.s[cfsHeaders], "#undef near" & tnl)
+  add(m.s[cfsHeaders], "#undef powerpc" & tnl)
+  add(m.s[cfsHeaders], "#undef unix" & tnl)
 
 proc initFrame(p: BProc, procname, filename: Rope): Rope =
   discard cgsym(p.module, "nimFrame")
@@ -898,14 +906,14 @@ proc genMainProc(m: BModule) =
     # prevents inlining of the NimMainInner function and dependent
     # functions, which might otherwise merge their stack frames.
     PreMainBody =
-      "void PreMainInner() {$N" &
+      "void PreMainInner(void) {$N" &
       "\tsystemInit000();$N" &
       "$1" &
       "$2" &
       "$3" &
       "}$N$N" &
-      "void PreMain() {$N" &
+      "void PreMain(void) {$N" &
-      "\tvoid (*volatile inner)();$N" &
+      "\tvoid (*volatile inner)(void);$N" &
       "\tsystemDatInit000();$N" &
       "\tinner = PreMainInner;$N" &
       "$4$5" &
@@ -924,7 +932,7 @@ proc genMainProc(m: BModule) =
 
     NimMainProc =
       "N_CDECL(void, NimMain)(void) {$N" &
-        "\tvoid (*volatile inner)();$N" &
+        "\tvoid (*volatile inner)(void);$N" &
         "\tPreMain();$N" &
         "\tinner = NimMainInner;$N" &
         "$2" &

compiler/cgmeth.nim

@@ -257,13 +257,14 @@ proc genDispatcher(methods: TSymSeq, relevantCols: IntSet): PSym =
           cond = a
         else:
           cond = isn
-    var call = newNodeI(nkCall, base.info)
+    let retTyp = base.typ.sons[0]
+    let call = newNodeIT(nkCall, base.info, retTyp)
     addSon(call, newSymNode(curr))
     for col in countup(1, paramLen - 1):
       addSon(call, genConv(newSymNode(base.typ.n.sons[col].sym),
                            curr.typ.sons[col], false))
     var ret: PNode
-    if base.typ.sons[0] != nil:
+    if retTyp != nil:
       var a = newNodeI(nkFastAsgn, base.info)
       addSon(a, newSymNode(base.ast.sons[resultPos].sym))
       addSon(a, call)

compiler/condsyms.nim

@@ -103,3 +103,4 @@ proc initDefines*() =
   defineSymbol("nimNewShiftOps")
   defineSymbol("nimDistros")
   defineSymbol("nimHasCppDefine")
+  defineSymbol("nimGenericInOutFlags")

compiler/evaltempl.nim

@@ -155,5 +155,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; fromHlo=false): PNode =
     #if ctx.instLines: result.info = n.info
     for i in countup(0, safeLen(body) - 1):
       evalTemplateAux(body.sons[i], args, ctx, result)
+  result.flags.incl nfFromTemplate
   result = wrapInComesFrom(n.info, result)
   dec(evalTemplateCounter)
+

compiler/installer.ini

@@ -66,6 +66,8 @@ Files: "compiler"
 Files: "doc"
 Files: "doc/html"
 Files: "tools"
+Files: "nimsuggest"
+Files: "nimsuggest/tests/*.nim"
 Files: "web/website.ini"
 Files: "web/ticker.html"
 Files: "web/*.nim"

compiler/jsgen.nim

@@ -94,12 +94,39 @@ type
     target: TTarget # duplicated here for faster dispatching
     unique: int    # for temp identifier generation
     blocks: seq[TBlock]
+    extraIndent: int
     up: PProc     # up the call chain; required for closure support
     declaredGlobals: IntSet
 
 template `|`(a, b: untyped): untyped {.dirty.} =
   (if p.target == targetJS: a else: b)
 
+var indent = "\t".rope
+
+proc indentLine(p: PProc, r: Rope): Rope =
+  result = r
+  var p = p
+  while true:
+    for i in countup(0, p.blocks.len - 1 + p.extraIndent):
+      prepend(result, indent)
+    if p.up == nil or p.up.prc != p.prc.owner:
+      break
+    p = p.up
+
+template line(p: PProc, added: string) =
+  add(p.body, indentLine(p, rope(added)))
+
+template line(p: PProc, added: Rope) =
+  add(p.body, indentLine(p, added))
+
+template lineF(p: PProc, frmt: FormatStr, args: varargs[Rope]) =
+  add(p.body, indentLine(p, ropes.`%`(frmt, args)))
+
+template nested(p, body) =
+  inc p.extraIndent
+  body
+  dec p.extraIndent
+
 proc newGlobals(): PGlobals =
   new(result)
   result.forwarded = @[]
@@ -129,7 +156,8 @@ proc newProc(globals: PGlobals, module: BModule, procDef: PNode,
     module: module,
     procDef: procDef,
     g: globals,
-    target: module.target)
+    target: module.target,
+    extraIndent: int(procDef != nil))
   if procDef != nil: result.prc = procDef.sons[namePos].sym
   if result.target == targetPHP:
     result.declaredGlobals = initIntSet()
@@ -290,7 +318,7 @@ proc getTemp(p: PProc, defineInLocals: bool = true): Rope =
   if p.target == targetJS:
     result = "Tmp$1" % [rope(p.unique)]
     if defineInLocals:
-      addf(p.locals, "var $1;$n", [result])
+      add(p.locals, p.indentLine("var $1;$n" % [result]))
   else:
     result = "$$Tmp$1" % [rope(p.unique)]
 
@@ -315,9 +343,11 @@ proc genAnd(p: PProc, a, b: PNode, r: var TCompRes) =
     #     tmp = b
     # tmp
     gen(p, a, x)
-    p.body.addf("if (!$1) $2 = false; else {", [x.rdLoc, r.rdLoc])
+    lineF(p, "if (!$1) $2 = false; else {", [x.rdLoc, r.rdLoc])
+    p.nested:
       gen(p, b, y)
-    p.body.addf("$2 = $1; }", [y.rdLoc, r.rdLoc])
+      lineF(p, "$2 = $1;", [y.rdLoc, r.rdLoc])
+    line(p, "}")
 
 proc genOr(p: PProc, a, b: PNode, r: var TCompRes) =
   assert r.kind == resNone
@@ -331,9 +361,11 @@ proc genOr(p: PProc, a, b: PNode, r: var TCompRes) =
     r.res = p.getTemp
     r.kind = resVal
     gen(p, a, x)
-    p.body.addf("if ($1) $2 = true; else {", [x.rdLoc, r.rdLoc])
+    lineF(p, "if ($1) $2 = true; else {", [x.rdLoc, r.rdLoc])
+    p.nested:
       gen(p, b, y)
-    p.body.addf("$2 = $1; }", [y.rdLoc, r.rdLoc])
+      lineF(p, "$2 = $1;", [y.rdLoc, r.rdLoc])
+    line(p, "}")
 
 type
   TMagicFrmt = array[0..3, string]
@@ -415,12 +447,10 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "Math.floor($1)", "Math.floor($1)"], # ToInt
     ["", "", "Math.floor($1)", "Math.floor($1)"], # ToBiggestInt
     ["nimCharToStr", "nimCharToStr", "nimCharToStr($1)", "nimCharToStr($1)"],
-    ["nimBoolToStr", "nimBoolToStr", "nimBoolToStr($1)", "nimBoolToStr($1)"], [
+    ["nimBoolToStr", "nimBoolToStr", "nimBoolToStr($1)", "nimBoolToStr($1)"],
-      "cstrToNimstr", "cstrToNimstr", "cstrToNimstr(($1)+\"\")",
+    ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")"],
-      "cstrToNimstr(($1)+\"\")"], ["cstrToNimstr", "cstrToNimstr",
+    ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")"],
-                                   "cstrToNimstr(($1)+\"\")",
+    ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")"],
-                                   "cstrToNimstr(($1)+\"\")"], ["cstrToNimstr",
-      "cstrToNimstr", "cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")"],
     ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr($1)", "cstrToNimstr($1)"],
     ["", "", "$1", "$1"]]
 
@@ -475,18 +505,18 @@ proc unaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
   r.res = frmt % [r.rdLoc]
   r.kind = resExpr
 
-proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic, ops: TMagicOps) =
+proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   var
     x, y: TCompRes
   let i = ord(optOverflowCheck notin p.options)
-  useMagic(p, ops[op][i])
+  useMagic(p, jsOps[op][i])
   if sonsLen(n) > 2:
     gen(p, n.sons[1], x)
     gen(p, n.sons[2], y)
-    r.res = ops[op][i + 2] % [x.rdLoc, y.rdLoc]
+    r.res = jsOps[op][i + 2] % [x.rdLoc, y.rdLoc]
   else:
     gen(p, n.sons[1], r)
-    r.res = ops[op][i + 2] % [r.rdLoc]
+    r.res = jsOps[op][i + 2] % [r.rdLoc]
 
 proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   case op
@@ -501,7 +531,7 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
       gen(p, n.sons[2], y)
       r.res = "intval($1 / $2)" % [x.rdLoc, y.rdLoc]
     else:
-      arithAux(p, n, r, op, jsOps)
+      arithAux(p, n, r, op)
   of mModI:
     if p.target == targetPHP:
       var x, y: TCompRes
@@ -509,7 +539,7 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
       gen(p, n.sons[2], y)
       r.res = "($1 % $2)" % [x.rdLoc, y.rdLoc]
     else:
-      arithAux(p, n, r, op, jsOps)
+      arithAux(p, n, r, op)
   of mShrI:
     var x, y: TCompRes
     gen(p, n.sons[1], x)
@@ -534,9 +564,9 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
       else:
         gen(p, n.sons[1], r)
     else:
-      arithAux(p, n, r, op, jsOps)
+      arithAux(p, n, r, op)
   else:
-    arithAux(p, n, r, op, jsOps)
+    arithAux(p, n, r, op)
   r.kind = resExpr
 
 proc hasFrameInfo(p: PProc): bool =
@@ -546,14 +576,14 @@ proc hasFrameInfo(p: PProc): bool =
 proc genLineDir(p: PProc, n: PNode) =
   let line = toLinenumber(n.info)
   if optLineDir in p.options:
-    addf(p.body, "// line $2 \"$1\"$n",
+    lineF(p, "// line $2 \"$1\"$n",
          [rope(toFilename(n.info)), rope(line)])
   if {optStackTrace, optEndb} * p.options == {optStackTrace, optEndb} and
       ((p.prc == nil) or sfPure notin p.prc.flags):
     useMagic(p, "endb")
-    addf(p.body, "endb($1);$n", [rope(line)])
+    lineF(p, "endb($1);$n", [rope(line)])
   elif hasFrameInfo(p):
-    addf(p.body, "F.line = $1;$n" | "$$F['line'] = $1;$n", [rope(line)])
+    lineF(p, "F.line = $1;$n" | "$$F['line'] = $1;$n", [rope(line)])
 
 proc genWhileStmt(p: PProc, n: PNode) =
   var
@@ -566,20 +596,20 @@ proc genWhileStmt(p: PProc, n: PNode) =
   p.blocks[length].id = -p.unique
   p.blocks[length].isLoop = true
   let labl = p.unique.rope
-  addf(p.body, "L$1: while (true) {$n" | "while (true) {$n", [labl])
+  lineF(p, "L$1: while (true) {$n" | "while (true) {$n", [labl])
-  gen(p, n.sons[0], cond)
+  p.nested: gen(p, n.sons[0], cond)
-  addf(p.body, "if (!$1) break L$2;$n" | "if (!$1) goto L$2;$n",
+  lineF(p, "if (!$1) break L$2;$n" | "if (!$1) goto L$2;$n",
        [cond.res, labl])
-  genStmt(p, n.sons[1])
+  p.nested: genStmt(p, n.sons[1])
-  addf(p.body, "}$n" | "}L$#:;$n", [labl])
+  lineF(p, "}$n" | "}L$#:;$n", [labl])
   setLen(p.blocks, length)
 
 proc moveInto(p: PProc, src: var TCompRes, dest: TCompRes) =
   if src.kind != resNone:
     if dest.kind != resNone:
-      p.body.addf("$1 = $2;$n", [dest.rdLoc, src.rdLoc])
+      lineF(p, "$1 = $2;$n", [dest.rdLoc, src.rdLoc])
     else:
-      p.body.addf("$1;$n", [src.rdLoc])
+      lineF(p, "$1;$n", [src.rdLoc])
     src.kind = resNone
     src.res = nil
 
@@ -620,8 +650,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   var tmpFramePtr = rope"F"
   if optStackTrace notin p.options:
     tmpFramePtr = p.getTemp(true)
-    add(p.body, tmpFramePtr & " = framePtr;" & tnl)
+    line(p, tmpFramePtr & " = framePtr;" & tnl)
-  addf(p.body, "try {$n", [])
+  lineF(p, "try {$n", [])
   if p.target == targetPHP and p.globals == nil:
       p.globals = "global $lastJSError; global $prevJSError;".rope
   var a: TCompRes
@@ -632,18 +662,18 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   if p.target == targetJS and catchBranchesExist:
     addf(p.body, "--excHandler;$n} catch (EXC) {$n var prevJSError = lastJSError;$n" &
         " lastJSError = EXC;$n --excHandler;$n", [])
-    add(p.body, "framePtr = $1;$n" % [tmpFramePtr])
+    line(p, "framePtr = $1;$n" % [tmpFramePtr])
   elif p.target == targetPHP:
-    addf(p.body, "} catch (Exception $$EXC) {$n $$prevJSError = $$lastJSError;$n $$lastJSError = $$EXC;$n", [])
+    lineF(p, "} catch (Exception $$EXC) {$n $$prevJSError = $$lastJSError;$n $$lastJSError = $$EXC;$n", [])
   while i < length and n.sons[i].kind == nkExceptBranch:
     let blen = sonsLen(n.sons[i])
     if blen == 1:
       # general except section:
       generalCatchBranchExists = true
-      if i > 1: addf(p.body, "else {$n", [])
+      if i > 1: lineF(p, "else {$n", [])
       gen(p, n.sons[i].sons[0], a)
       moveInto(p, a, r)
-      if i > 1: addf(p.body, "}$n", [])
+      if i > 1: lineF(p, "}$n", [])
     else:
       var orExpr: Rope = nil
       useMagic(p, "isObj")
@@ -653,30 +683,32 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
         if orExpr != nil: add(orExpr, "||")
         addf(orExpr, "isObj($2lastJSError.m_type, $1)",
              [genTypeInfo(p, n.sons[i].sons[j].typ), dollar])
-      if i > 1: add(p.body, "else ")
+      if i > 1: line(p, "else ")
-      addf(p.body, "if ($1lastJSError && ($2)) {$n", [dollar, orExpr])
+      lineF(p, "if ($1lastJSError && ($2)) {$n", [dollar, orExpr])
       gen(p, n.sons[i].sons[blen - 1], a)
       moveInto(p, a, r)
-      addf(p.body, "}$n", [])
+      lineF(p, "}$n", [])
     inc(i)
   if catchBranchesExist:
     if not generalCatchBranchExists:
       useMagic(p, "reraiseException")
-      add(p.body, "else {" & tnl & "reraiseException();" & tnl & "}" & tnl)
+      line(p, "else {" & tnl)
+      line(p, indent & "reraiseException();" & tnl)
+      line(p, "}" & tnl)
     addf(p.body, "$1lastJSError = $1prevJSError;$n", [dollar])
   if p.target == targetJS:
-    add(p.body, "} finally {" & tnl)
+    line(p, "} finally {" & tnl)
-    add(p.body, "framePtr = $1;$n" % [tmpFramePtr])
+    line(p, "framePtr = $1;$n" % [tmpFramePtr])
   if p.target == targetPHP:
     # XXX ugly hack for PHP codegen
-    add(p.body, "}" & tnl)
+    line(p, "}" & tnl)
   if i < length and n.sons[i].kind == nkFinally:
     genStmt(p, n.sons[i].sons[0])
   if p.target == targetPHP:
     # XXX ugly hack for PHP codegen
-    add(p.body, "if($lastJSError) throw($lastJSError);" & tnl)
+    line(p, "if($lastJSError) throw($lastJSError);" & tnl)
   if p.target == targetJS:
-    add(p.body, "}" & tnl)
+    line(p, "}" & tnl)
 
 proc genRaiseStmt(p: PProc, n: PNode) =
   genLineDir(p, n)
@@ -685,11 +717,11 @@ proc genRaiseStmt(p: PProc, n: PNode) =
     gen(p, n.sons[0], a)
     let typ = skipTypes(n.sons[0].typ, abstractPtrs)
     useMagic(p, "raiseException")
-    addf(p.body, "raiseException($1, $2);$n",
+    lineF(p, "raiseException($1, $2);$n",
              [a.rdLoc, makeJSString(typ.sym.name.s)])
   else:
     useMagic(p, "reraiseException")
-    add(p.body, "reraiseException();" & tnl)
+    line(p, "reraiseException();" & tnl)
 
 proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
   var
@@ -699,9 +731,9 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
   let stringSwitch = skipTypes(n.sons[0].typ, abstractVar).kind == tyString
   if stringSwitch and p.target == targetJS:
     useMagic(p, "toJSStr")
-    addf(p.body, "switch (toJSStr($1)) {$n", [cond.rdLoc])
+    lineF(p, "switch (toJSStr($1)) {$n", [cond.rdLoc])
   else:
-    addf(p.body, "switch ($1) {$n", [cond.rdLoc])
+    lineF(p, "switch ($1) {$n", [cond.rdLoc])
   if not isEmptyType(n.typ):
     r.kind = resVal
     r.res = getTemp(p)
@@ -715,27 +747,29 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
           var v = copyNode(e.sons[0])
           while v.intVal <= e.sons[1].intVal:
             gen(p, v, cond)
-            addf(p.body, "case $1: ", [cond.rdLoc])
+            lineF(p, "case $1:$n", [cond.rdLoc])
             inc(v.intVal)
         else:
           if stringSwitch:
             case e.kind
-            of nkStrLit..nkTripleStrLit: addf(p.body, "case $1: ",
+            of nkStrLit..nkTripleStrLit: lineF(p, "case $1:$n",
                 [makeJSString(e.strVal, false)])
             else: internalError(e.info, "jsgen.genCaseStmt: 2")
           else:
             gen(p, e, cond)
-            addf(p.body, "case $1: ", [cond.rdLoc])
+            lineF(p, "case $1:$n", [cond.rdLoc])
+      p.nested:
         gen(p, lastSon(it), stmt)
         moveInto(p, stmt, r)
-      addf(p.body, "$nbreak;$n", [])
+        lineF(p, "break;$n", [])
     of nkElse:
-      addf(p.body, "default: $n", [])
+      lineF(p, "default: $n", [])
+      p.nested:
         gen(p, it.sons[0], stmt)
         moveInto(p, stmt, r)
-      addf(p.body, "break;$n", [])
+        lineF(p, "break;$n", [])
     else: internalError(it.info, "jsgen.genCaseStmt")
-  addf(p.body, "}$n", [])
+  lineF(p, "}$n", [])
 
 proc genBlock(p: PProc, n: PNode, r: var TCompRes) =
   inc(p.unique)
@@ -746,13 +780,13 @@ proc genBlock(p: PProc, n: PNode, r: var TCompRes) =
     var sym = n.sons[0].sym
     sym.loc.k = locOther
     sym.position = idx+1
+  let labl = p.unique
+  lineF(p, "L$1: do {$n" | "", [labl.rope])
   setLen(p.blocks, idx + 1)
   p.blocks[idx].id = - p.unique # negative because it isn't used yet
-  let labl = p.unique
-  addf(p.body, "L$1: do {$n" | "", [labl.rope])
   gen(p, n.sons[1], r)
-  addf(p.body, "} while(false);$n" | "$nL$#:;$n", [labl.rope])
   setLen(p.blocks, idx)
+  lineF(p, "} while(false);$n" | "$nL$#:;$n", [labl.rope])
 
 proc genBreakStmt(p: PProc, n: PNode) =
   var idx: int
@@ -770,19 +804,31 @@ proc genBreakStmt(p: PProc, n: PNode) =
     if idx < 0 or not p.blocks[idx].isLoop:
       internalError(n.info, "no loop to break")
   p.blocks[idx].id = abs(p.blocks[idx].id) # label is used
-  addf(p.body, "break L$1;$n" | "goto L$1;$n", [rope(p.blocks[idx].id)])
+  lineF(p, "break L$1;$n" | "goto L$1;$n", [rope(p.blocks[idx].id)])
 
 proc genAsmOrEmitStmt(p: PProc, n: PNode) =
   genLineDir(p, n)
+  p.body.add p.indentLine(nil)
   for i in countup(0, sonsLen(n) - 1):
-    case n.sons[i].kind
+    let it = n[i]
-    of nkStrLit..nkTripleStrLit: add(p.body, n.sons[i].strVal)
+    case it.kind
+    of nkStrLit..nkTripleStrLit:
+      p.body.add(it.strVal)
     of nkSym:
-      let v = n.sons[i].sym
+      let v = it.sym
-      if p.target == targetPHP and v.kind in {skVar, skLet, skTemp, skConst, skResult, skParam, skForVar}:
+      # for backwards compatibility we don't deref syms here :-(
-        add(p.body, "$")
+      if v.kind in {skVar, skLet, skTemp, skConst, skResult, skParam, skForVar}:
-      add(p.body, mangleName(v, p.target))
+        if p.target == targetPHP: p.body.add "$"
-    else: internalError(n.sons[i].info, "jsgen: genAsmOrEmitStmt()")
+        p.body.add mangleName(v, p.target)
+      else:
+        var r: TCompRes
+        gen(p, it, r)
+        p.body.add(r.rdLoc)
+    else:
+      var r: TCompRes
+      gen(p, it, r)
+      p.body.add(r.rdLoc)
+  p.body.add tnl
 
 proc genIf(p: PProc, n: PNode, r: var TCompRes) =
   var cond, stmt: TCompRes
@@ -794,18 +840,18 @@ proc genIf(p: PProc, n: PNode, r: var TCompRes) =
     let it = n.sons[i]
     if sonsLen(it) != 1:
       if i > 0:
-        addf(p.body, "else {$n", [])
+        lineF(p, "else {$n", [])
         inc(toClose)
-      gen(p, it.sons[0], cond)
+      p.nested: gen(p, it.sons[0], cond)
-      addf(p.body, "if ($1) {$n", [cond.rdLoc])
+      lineF(p, "if ($1) {$n", [cond.rdLoc])
       gen(p, it.sons[1], stmt)
     else:
       # else part:
-      addf(p.body, "else {$n", [])
+      lineF(p, "else {$n", [])
-      gen(p, it.sons[0], stmt)
+      p.nested: gen(p, it.sons[0], stmt)
     moveInto(p, stmt, r)
-    addf(p.body, "}$n", [])
+    lineF(p, "}$n", [])
-  add(p.body, repeat('}', toClose) & tnl)
+  line(p, repeat('}', toClose) & tnl)
 
 proc generateHeader(p: PProc, typ: PType): Rope =
   result = nil
@@ -834,6 +880,16 @@ proc generateHeader(p: PProc, typ: PType): Rope =
       #  add(result, name)
       #  add(result, "_Idx")
 
+proc countJsParams(typ: PType): int =
+  for i in countup(1, sonsLen(typ.n) - 1):
+    assert(typ.n.sons[i].kind == nkSym)
+    var param = typ.n.sons[i].sym
+    if isCompileTimeOnly(param.typ): continue
+    if mapType(param.typ) == etyBaseIndex:
+      inc result, 2
+    else:
+      inc result
+
 const
   nodeKindsNeedNoCopy = {nkCharLit..nkInt64Lit, nkStrLit..nkTripleStrLit,
     nkFloatLit..nkFloat64Lit, nkCurly, nkPar, nkObjConstr, nkStringToCString,
@@ -854,7 +910,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
     gen(p, x[0], a)
     gen(p, x[1], b)
     gen(p, y, c)
-    addf(p.body, "$#[$#] = chr($#);$n", [a.rdLoc, b.rdLoc, c.rdLoc])
+    lineF(p, "$#[$#] = chr($#);$n", [a.rdLoc, b.rdLoc, c.rdLoc])
     return
 
   var xtyp = mapType(p, x.typ)
@@ -862,7 +918,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
   if x.kind == nkHiddenDeref and x.sons[0].kind == nkCall and xtyp != etyObject:
     gen(p, x.sons[0], a)
     let tmp = p.getTemp(false)
-    addf(p.body, "var $1 = $2;$n", [tmp, a.rdLoc])
+    lineF(p, "var $1 = $2;$n", [tmp, a.rdLoc])
     a.res = "$1[0][$1[1]]" % [tmp]
   else:
     gen(p, x, a)
@@ -876,29 +932,31 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
   case xtyp
   of etySeq:
     if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
-      addf(p.body, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
+      lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
-      addf(p.body, "$1 = nimCopy(null, $2, $3);$n",
+      lineF(p, "$1 = nimCopy(null, $2, $3);$n",
                [a.rdLoc, b.res, genTypeInfo(p, y.typ)])
   of etyObject:
     if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
-      addf(p.body, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
+      lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
-      addf(p.body, "nimCopy($1, $2, $3);$n",
+      lineF(p, "nimCopy($1, $2, $3);$n",
                [a.res, b.res, genTypeInfo(p, y.typ)])
   of etyBaseIndex:
     if a.typ != etyBaseIndex or b.typ != etyBaseIndex:
       if y.kind == nkCall:
         let tmp = p.getTemp(false)
-        addf(p.body, "var $1 = $4; $2 = $1[0]; $3 = $1[1];$n", [tmp, a.address, a.res, b.rdLoc])
+        lineF(p, "var $1 = $4; $2 = $1[0]; $3 = $1[1];$n", [tmp, a.address, a.res, b.rdLoc])
+      elif b.typ == etyBaseIndex:
+        lineF(p, "$# = $#;$n", [a.res, b.rdLoc])
       else:
         internalError(x.info, "genAsgn")
     else:
-      addf(p.body, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
+      lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
   else:
-    addf(p.body, "$1 = $2;$n", [a.res, b.res])
+    lineF(p, "$1 = $2;$n", [a.res, b.res])
 
 proc genAsgn(p: PProc, n: PNode) =
   genLineDir(p, n)
@@ -906,7 +964,15 @@ proc genAsgn(p: PProc, n: PNode) =
 
 proc genFastAsgn(p: PProc, n: PNode) =
   genLineDir(p, n)
-  genAsgnAux(p, n.sons[0], n.sons[1], noCopyNeeded=true)
+  # 'shallowCopy' always produced 'noCopyNeeded = true' here but this is wrong
+  # for code like
+  #  while j >= pos:
+  #    dest[i].shallowCopy(dest[j])
+  # See bug #5933. So we try to be more compatible with the C backend semantics
+  # here for 'shallowCopy'. This is an educated guess and might require further
+  # changes later:
+  let noCopy = n[0].typ.skipTypes(abstractInst).kind in {tySequence, tyString}
+  genAsgnAux(p, n.sons[0], n.sons[1], noCopyNeeded=noCopy)
 
 proc genSwap(p: PProc, n: PNode) =
   var a, b: TCompRes
@@ -917,12 +983,13 @@ proc genSwap(p: PProc, n: PNode) =
     let tmp2 = p.getTemp(false)
     if a.typ != etyBaseIndex or b.typ != etyBaseIndex:
       internalError(n.info, "genSwap")
-    addf(p.body, "var $1 = $2; $2 = $3; $3 = $1;$n" |
+    lineF(p, "var $1 = $2; $2 = $3; $3 = $1;$n" |
-                 "$1 = $2; $2 = $3; $3 = $1;$n", [
+             "$1 = $2; $2 = $3; $3 = $1;$n",
-                 tmp, a.address, b.address])
+             [tmp, a.address, b.address])
     tmp = tmp2
-  addf(p.body, "var $1 = $2; $2 = $3; $3 = $1;" |
+  lineF(p, "var $1 = $2; $2 = $3; $3 = $1;" |
-               "$1 = $2; $2 = $3; $3 = $1;", [tmp, a.res, b.res])
+           "$1 = $2; $2 = $3; $3 = $1;",
+           [tmp, a.res, b.res])
 
 proc getFieldPosition(f: PNode): int =
   case f.kind
@@ -992,15 +1059,15 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) =
   var typ = skipTypes(m.sons[0].typ, abstractPtrs)
   if typ.kind == tyArray: first = firstOrd(typ.sons[0])
   else: first = 0
-  if optBoundsCheck in p.options and not isConstExpr(m.sons[1]):
+  if optBoundsCheck in p.options:
     useMagic(p, "chckIndx")
     if p.target == targetPHP:
       if typ.kind != tyString:
-        r.res = "chckIndx($1, $2, count($3))-$2" % [b.res, rope(first), a.res]
+        r.res = "chckIndx($1, $2, count($3)-1)-$2" % [b.res, rope(first), a.res]
       else:
         r.res = "chckIndx($1, $2, strlen($3))-$2" % [b.res, rope(first), a.res]
     else:
-      r.res = "chckIndx($1, $2, $3.length)-$2" % [b.res, rope(first), a.res]
+      r.res = "chckIndx($1, $2, $3.length-1)-$2" % [b.res, rope(first), a.res]
   elif first != 0:
     r.res = "($1)-$2" % [b.res, rope(first)]
   else:
@@ -1193,17 +1260,21 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
   r.kind = resVal
 
 proc genDeref(p: PProc, n: PNode, r: var TCompRes) =
-  if mapType(p, n.sons[0].typ) == etyObject:
+  let it = n.sons[0]
-    gen(p, n.sons[0], r)
+  let t = mapType(p, it.typ)
+  if t == etyObject:
+    gen(p, it, r)
   else:
     var a: TCompRes
-    gen(p, n.sons[0], a)
+    gen(p, it, a)
     r.kind = resExpr
     if a.typ == etyBaseIndex:
       r.res = "$1[$2]" % [a.address, a.res]
-    elif n.sons[0].kind == nkCall:
+    elif it.kind == nkCall:
       let tmp = p.getTemp
       r.res = "($1 = $2, $1[0])[$1[1]]" % [tmp, a.res]
+    elif t == etyBaseIndex:
+      r.res = "$1[0]" % [a.res]
     else:
       internalError(n.info, "genDeref")
 
@@ -1217,7 +1288,7 @@ proc genArgNoParam(p: PProc, n: PNode, r: var TCompRes) =
   else:
     add(r.res, a.res)
 
-proc genArg(p: PProc, n: PNode, param: PSym, r: var TCompRes) =
+proc genArg(p: PProc, n: PNode, param: PSym, r: var TCompRes; emitted: ptr int = nil) =
   var a: TCompRes
   gen(p, n, a)
   if skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs} and
@@ -1227,6 +1298,12 @@ proc genArg(p: PProc, n: PNode, param: PSym, r: var TCompRes) =
     add(r.res, a.address)
     add(r.res, ", ")
     add(r.res, a.res)
+    if emitted != nil: inc emitted[]
+  elif n.typ.kind == tyVar and n.kind in nkCallKinds and mapType(param.typ) == etyBaseIndex:
+    # this fixes bug #5608:
+    let tmp = getTemp(p)
+    add(r.res, "($1 = $2, $1[0]), $1[1]" % [tmp, a.rdLoc])
+    if emitted != nil: inc emitted[]
   else:
     add(r.res, a.res)
 
@@ -1237,6 +1314,7 @@ proc genArgs(p: PProc, n: PNode, r: var TCompRes; start=1) =
   var typ = skipTypes(n.sons[0].typ, abstractInst)
   assert(typ.kind == tyProc)
   assert(sonsLen(typ) == sonsLen(typ.n))
+  var emitted = start-1
 
   for i in countup(start, sonsLen(n) - 1):
     let it = n.sons[i]
@@ -1250,9 +1328,16 @@ proc genArgs(p: PProc, n: PNode, r: var TCompRes; start=1) =
     if paramType.isNil:
       genArgNoParam(p, it, r)
     else:
-      genArg(p, it, paramType.sym, r)
+      genArg(p, it, paramType.sym, r, addr emitted)
+    inc emitted
     hasArgs = true
   add(r.res, ")")
+  when false:
+    # XXX look into this:
+    let jsp = countJsParams(typ)
+    if emitted != jsp and tfVarargs notin typ.flags:
+      localError(n.info, "wrong number of parameters emitted; expected: " & $jsp &
+        " but got: " & $emitted)
   r.kind = resExpr
 
 proc genOtherArg(p: PProc; n: PNode; i: int; typ: PType;
@@ -1385,7 +1470,7 @@ proc createRecordVarAux(p: PProc, rec: PNode, excludedFieldIDs: IntSet, output:
 
 proc createObjInitList(p: PProc, typ: PType, excludedFieldIDs: IntSet, output: var Rope) =
   var t = typ
-  if tfFinal notin t.flags or t.sons[0] != nil:
+  if objHasTypeField(t):
     if output.len > 0: output.add(", ")
     addf(output, "m_type: $1" | "'m_type' => $#", [genTypeInfo(p, t)])
   while t != nil:
@@ -1483,10 +1568,10 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     s: Rope
   if n.kind == nkEmpty:
     let mname = mangleName(v, p.target)
-    addf(p.body, "var $1 = $2;$n" | "$$$1 = $2;$n",
+    lineF(p, "var $1 = $2;$n" | "$$$1 = $2;$n",
              [mname, createVar(p, v.typ, isIndirect(v))])
     if v.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, v.typ) == etyBaseIndex:
-      addf(p.body, "var $1_Idx = 0;$n", [ mname ])
+      lineF(p, "var $1_Idx = 0;$n", [ mname ])
   else:
     discard mangleName(v, p.target)
     gen(p, n, a)
@@ -1501,25 +1586,25 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       let targetBaseIndex = {sfAddrTaken, sfGlobal} * v.flags == {}
       if a.typ == etyBaseIndex:
         if targetBaseIndex:
-          addf(p.body, "var $1 = $2, $1_Idx = $3;$n", [
+          lineF(p, "var $1 = $2, $1_Idx = $3;$n",
-              v.loc.r, a.address, a.res])
+                   [v.loc.r, a.address, a.res])
         else:
-          addf(p.body, "var $1 = [$2, $3];$n",
+          lineF(p, "var $1 = [$2, $3];$n",
                    [v.loc.r, a.address, a.res])
       else:
         if targetBaseIndex:
           let tmp = p.getTemp
-          addf(p.body, "var $1 = $2, $3 = $1[0], $3_Idx = $1[1];$n",
+          lineF(p, "var $1 = $2, $3 = $1[0], $3_Idx = $1[1];$n",
                    [tmp, a.res, v.loc.r])
         else:
-          addf(p.body, "var $1 = $2;$n", [v.loc.r, a.res])
+          lineF(p, "var $1 = $2;$n", [v.loc.r, a.res])
       return
     else:
       s = a.res
     if isIndirect(v):
-      addf(p.body, "var $1 = /**/[$2];$n", [v.loc.r, s])
+      lineF(p, "var $1 = [$2];$n", [v.loc.r, s])
     else:
-      addf(p.body, "var $1 = $2;$n" | "$$$1 = $2;$n", [v.loc.r, s])
+      lineF(p, "var $1 = $2;$n" | "$$$1 = $2;$n", [v.loc.r, s])
 
 proc genVarStmt(p: PProc, n: PNode) =
   for i in countup(0, sonsLen(n) - 1):
@@ -1550,16 +1635,16 @@ proc genNew(p: PProc, n: PNode) =
   gen(p, n.sons[1], a)
   var t = skipTypes(n.sons[1].typ, abstractVar).sons[0]
   if p.target == targetJS:
-    addf(p.body, "$1 = $2;$n", [a.res, createVar(p, t, false)])
+    lineF(p, "$1 = $2;$n", [a.res, createVar(p, t, false)])
   else:
-    addf(p.body, "$3 = $2; $1 = &$3;$n", [a.res, createVar(p, t, false), getTemp(p)])
+    lineF(p, "$3 = $2; $1 = &$3;$n", [a.res, createVar(p, t, false), getTemp(p)])
 
 proc genNewSeq(p: PProc, n: PNode) =
   var x, y: TCompRes
   gen(p, n.sons[1], x)
   gen(p, n.sons[2], y)
   let t = skipTypes(n.sons[1].typ, abstractVar).sons[0]
-  addf(p.body, "$1 = new Array($2); for (var i=0;i<$2;++i) {$1[i]=$3;}" |
+  lineF(p, "$1 = new Array($2); for (var i=0;i<$2;++i) {$1[i]=$3;}" |
            "$1 = array(); for ($$i=0;$$i<$2;++$$i) {$1[]=$3;}", [
     x.rdLoc, y.rdLoc, createVar(p, t, false)])
 
@@ -1716,8 +1801,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
       binaryExpr(p, n, r, "addChar",
           "if ($1 != null) { addChar($1, $2); } else { $1 = [$2, 0]; }")
     else:
-      binaryExpr(p, n, r, "",
+      binaryExpr(p, n, r, "", "$1 .= chr($2)")
-          "$1 .= chr($2)")
   of mAppendStrStr:
     if p.target == targetJS:
       if skipTypes(n.sons[1].typ, abstractVarRange).kind == tyCString:
@@ -1727,15 +1811,23 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
           "if ($1 != null) { $1 = ($1.slice(0, -1)).concat($2); } else { $1 = $2;}")
       # XXX: make a copy of $2, because of Javascript's sucking semantics
     else:
-      binaryExpr(p, n, r, "",
+      binaryExpr(p, n, r, "", "$1 .= $2;")
-          "$1 .= $2;")
   of mAppendSeqElem:
     if p.target == targetJS:
-      binaryExpr(p, n, r, "",
+      var x, y: TCompRes
-          "if ($1 != null) { $1.push($2); } else { $1 = [$2]; }")
+      gen(p, n.sons[1], x)
+      gen(p, n.sons[2], y)
+      if needsNoCopy(p, n[2]):
+        r.res = "if ($1 != null) { $1.push($2); } else { $1 = [$2]; }" % [x.rdLoc, y.rdLoc]
       else:
-      binaryExpr(p, n, r, "",
+        useMagic(p, "nimCopy")
-          "$1[] = $2")
+        let c = getTemp(p, defineInLocals=false)
+        lineF(p, "var $1 = nimCopy(null, $2, $3);$n",
+             [c, y.rdLoc, genTypeInfo(p, n[2].typ)])
+        r.res = "if ($1 != null) { $1.push($2); } else { $1 = [$2]; }" % [x.rdLoc, c]
+      r.kind = resExpr
+    else:
+      binaryExpr(p, n, r, "", "$1[] = $2")
   of mConStrStr:
     if p.target == targetJS:
       genConStrStr(p, n, r)
@@ -1795,7 +1887,14 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
       else: binaryExpr(p, n, r, "subInt", "$1 = subInt($1, $2)")
   of mSetLengthStr:
     binaryExpr(p, n, r, "", "$1.length = $2+1; $1[$1.length-1] = 0" | "$1 = substr($1, 0, $2)")
-  of mSetLengthSeq: binaryExpr(p, n, r, "", "$1.length = $2" | "$1 = array_slice($1, 0, $2)")
+  of mSetLengthSeq:
+    var x, y: TCompRes
+    gen(p, n.sons[1], x)
+    gen(p, n.sons[2], y)
+    let t = skipTypes(n.sons[1].typ, abstractVar).sons[0]
+    r.res = """if ($1.length < $2) { for (var i=$1.length;i<$2;++i) $1.push($3); }
+               else { $1.length = $2; }""" % [x.rdLoc, y.rdLoc, createVar(p, t, false)]
+    r.kind = resExpr
   of mCard: unaryExpr(p, n, r, "SetCard", "SetCard($1)")
   of mLtSet: binaryExpr(p, n, r, "SetLt", "SetLt($1, $2)")
   of mLeSet: binaryExpr(p, n, r, "SetLe", "SetLe($1, $2)")
@@ -1916,10 +2015,19 @@ proc genObjConstr(p: PProc, n: PNode, r: var TCompRes) =
     if i > 1: add(initList, ", ")
     var it = n.sons[i]
     internalAssert it.kind == nkExprColonExpr
-    gen(p, it.sons[1], a)
+    let val = it.sons[1]
+    gen(p, val, a)
     var f = it.sons[0].sym
     if f.loc.r == nil: f.loc.r = mangleName(f, p.target)
     fieldIDs.incl(f.id)
+
+    let typ = val.typ.skipTypes(abstractInst)
+    if (typ.kind in IntegralTypes+{tyCstring, tyRef, tyPtr} and
+          mapType(p, typ) != etyBaseIndex) or needsNoCopy(p, it.sons[1]):
+      discard
+    else:
+      useMagic(p, "nimCopy")
+      a.res = "nimCopy(null, $1, $2)" % [a.rdLoc, genTypeInfo(p, typ)]
     addf(initList, "$#: $#" | "'$#' => $#" , [f.loc.r, a.res])
   let t = skipTypes(n.typ, abstractInst + skipPtrs)
   createObjInitList(p, t, fieldIDs, initList)
@@ -1986,15 +2094,18 @@ proc genReturnStmt(p: PProc, n: PNode) =
     genStmt(p, n.sons[0])
   else:
     genLineDir(p, n)
-  addf(p.body, "break BeforeRet;$n" | "goto BeforeRet;$n", [])
+  lineF(p, "break BeforeRet;$n" | "goto BeforeRet;$n", [])
 
 proc frameCreate(p: PProc; procname, filename: Rope): Rope =
-  result = (("var F={procname:$1,prev:framePtr,filename:$2,line:0};$nframePtr = F;$n" |
+  let frameFmt =
-             "global $$framePtr; $$F=array('procname'=>$#,'prev'=>$$framePtr,'filename'=>$#,'line'=>0);$n$$framePtr = &$$F;$n")) % [
+    "var F={procname:$1,prev:framePtr,filename:$2,line:0};$n" |
-            procname, filename]
+    "global $$framePtr; $$F=array('procname'=>$#,'prev'=>$$framePtr,'filename'=>$#,'line'=>0);$n"
+
+  result = p.indentLine(frameFmt % [procname, filename])
+  result.add p.indentLine(ropes.`%`("framePtr = F;$n" | "$$framePtr = &$$F;$n", []))
 
 proc frameDestroy(p: PProc): Rope =
-  result = rope(("framePtr = F.prev;" | "$framePtr = $F['prev'];") & tnl)
+  result = p.indentLine rope(("framePtr = F.prev;" | "$framePtr = $F['prev'];") & tnl)
 
 proc genProcBody(p: PProc, prc: PSym): Rope =
   if hasFrameInfo(p):
@@ -2004,8 +2115,12 @@ proc genProcBody(p: PProc, prc: PSym): Rope =
   else:
     result = nil
   if p.beforeRetNeeded:
-    addf(result, "BeforeRet: do {$n$1} while (false); $n" |
+    if p.target == targetJS:
-                 "$# BeforeRet:;$n", [p.body])
+      result.add p.indentLine(~"BeforeRet: do {$n")
+      result.add p.body
+      result.add p.indentLine(~"} while (false);$n")
+    else:
+      addF(result, "$# BeforeRet:;$n", [p.body])
   else:
     add(result, p.body)
   if prc.typ.callConv == ccSysCall and p.target == targetJS:
@@ -2014,6 +2129,12 @@ proc genProcBody(p: PProc, prc: PSym): Rope =
   if hasFrameInfo(p):
     add(result, frameDestroy(p))
 
+proc optionaLine(p: Rope): Rope =
+  if p == nil:
+    return nil
+  else:
+    return p & tnl
+
 proc genProc(oldProc: PProc, prc: PSym): Rope =
   var
     resultSym: PSym
@@ -2029,29 +2150,37 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
   if prc.typ.sons[0] != nil and sfPure notin prc.flags:
     resultSym = prc.ast.sons[resultPos].sym
     let mname = mangleName(resultSym, p.target)
-    resultAsgn = ("var $# = $#;$n" | "$$$# = $#;$n") % [
+    let resVar = createVar(p, resultSym.typ, isIndirect(resultSym))
-        mname,
+    resultAsgn = p.indentLine(("var $# = $#;$n" | "$$$# = $#;$n") % [mname, resVar])
-        createVar(p, resultSym.typ, isIndirect(resultSym))]
     if resultSym.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, resultSym.typ) == etyBaseIndex:
-      resultAsgn.add "var $#_Idx = 0;$n" % [
+      resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname])
-        mname ];
     gen(p, prc.ast.sons[resultPos], a)
     if mapType(p, resultSym.typ) == etyBaseIndex:
       returnStmt = "return [$#, $#];$n" % [a.address, a.res]
     else:
       returnStmt = "return $#;$n" % [a.res]
-  genStmt(p, prc.getBody)
 
-  result = "function $#($#) {$n$#$n$#$#$#$#}$n" %
+  p.nested: genStmt(p, prc.getBody)
-            [name, header, p.globals, p.locals, resultAsgn,
+  let def = "function $#($#) {$n$#$#$#$#$#" %
-             genProcBody(p, prc), returnStmt]
+            [name, header,
+              optionaLine(p.globals),
+              optionaLine(p.locals),
+              optionaLine(resultAsgn),
+              optionaLine(genProcBody(p, prc)),
+              optionaLine(p.indentLine(returnStmt))]
+
+  dec p.extraIndent
+  result = ~tnl
+  result.add p.indentLine(def)
+  result.add p.indentLine(~"}$n")
+
   #if gVerbosity >= 3:
   #  echo "END   generated code for: " & prc.name.s
 
 proc genStmt(p: PProc, n: PNode) =
   var r: TCompRes
   gen(p, n, r)
-  if r.res != nil: addf(p.body, "$#;$n", [r.res])
+  if r.res != nil: lineF(p, "$#;$n", [r.res])
 
 proc genPragma(p: PProc, n: PNode) =
   for it in n.sons:
@@ -2211,7 +2340,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkPragma: genPragma(p, n)
   of nkProcDef, nkMethodDef, nkConverterDef:
     var s = n.sons[namePos].sym
-    if {sfExportc, sfCompilerProc} * s.flags == {sfExportc}:
+    if sfExportc in s.flags and compilingLib:
       genSym(p, n.sons[namePos], r)
       r.res = nil
   of nkGotoState, nkState:
@@ -2342,3 +2471,4 @@ proc myOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext =
   result = r
 
 const JSgenPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
+

compiler/jstypes.nim

@@ -67,9 +67,13 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
         rope(lengthOrd(field.typ)), makeJSString(field.name.s), result]
   else: internalError(n.info, "genObjectFields")
 
+proc objHasTypeField(t: PType): bool {.inline.} =
+  tfInheritable in t.flags or t.sons[0] != nil
+
 proc genObjectInfo(p: PProc, typ: PType, name: Rope) =
+  let kind = if objHasTypeField(typ): tyObject else: tyTuple
   var s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
-           "finalizer: null};$n") % [name, rope(ord(typ.kind))]
+           "finalizer: null};$n") % [name, rope(ord(kind))]
   prepend(p.g.typeInfo, s)
   addf(p.g.typeInfo, "var NNI$1 = $2;$n",
        [rope(typ.id), genObjectFields(p, typ, typ.n)])

compiler/lexer.nim

@@ -386,11 +386,11 @@ proc getNumber(L: var TLexer, result: var TToken) =
   else:
     matchUnderscoreChars(L, result, {'0'..'9'})
     if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
-      result.tokType = tkFloat64Lit
+      result.tokType = tkFloatLit
       eatChar(L, result, '.')
       matchUnderscoreChars(L, result, {'0'..'9'})
     if L.buf[L.bufpos] in {'e', 'E'}:
-      result.tokType = tkFloat64Lit
+      result.tokType = tkFloatLit
       eatChar(L, result, 'e')
       if L.buf[L.bufpos] in {'+', '-'}:
         eatChar(L, result)
@@ -516,7 +516,8 @@ proc getNumber(L: var TLexer, result: var TToken) =
         result.fNumber = (cast[PFloat32](addr(xi)))[]
         # note: this code is endian neutral!
         # XXX: Test this on big endian machine!
-      of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[]
+      of tkFloat64Lit, tkFloatLit:
+        result.fNumber = (cast[PFloat64](addr(xi)))[]
       else: internalError(getLineInfo(L), "getNumber")
 
       # Bounds checks. Non decimal literals are allowed to overflow the range of

compiler/msgs.nim

@@ -113,6 +113,7 @@ type
     errGenericLambdaNotAllowed,
     errProcHasNoConcreteType,
     errCompilerDoesntSupportTarget,
+    errInOutFlagNotExtern,
     errUser,
     warnCannotOpenFile,
     warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit,
@@ -380,6 +381,7 @@ const
                                 "of the generic paramers can be inferred from the expected signature.",
     errProcHasNoConcreteType: "'$1' doesn't have a concrete type, due to unspecified generic parameters.",
     errCompilerDoesntSupportTarget: "The current compiler \'$1\' doesn't support the requested compilation target",
+    errInOutFlagNotExtern: "The `$1` modifier can be used only with imported types",
     errUser: "$1",
     warnCannotOpenFile: "cannot open \'$1\'",
     warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored",
@@ -541,7 +543,7 @@ var
 
 proc toCChar*(c: char): string =
   case c
-  of '\0'..'\x1F', '\x80'..'\xFF': result = '\\' & toOctal(c)
+  of '\0'..'\x1F', '\x7F'..'\xFF': result = '\\' & toOctal(c)
   of '\'', '\"', '\\', '?': result = '\\' & c
   else: result = $(c)
 

compiler/options.nim

@@ -187,6 +187,9 @@ var
 
 const oKeepVariableNames* = true
 
+template compilingLib*: bool =
+  gGlobalOptions * {optGenGuiApp, optGenDynLib} != {}
+
 proc mainCommandArg*: string =
   ## This is intended for commands like check or parse
   ## which will work on the main project file unless

compiler/parser.nim

@@ -39,6 +39,9 @@ type
     inPragma*: int             # Pragma level
     inSemiStmtList*: int
 
+  SymbolMode = enum
+    smNormal, smAllowNil, smAfterDot
+
 proc parseAll*(p: var TParser): PNode
 proc closeParser*(p: var TParser)
 proc parseTopLevelStmt*(p: var TParser): PNode
@@ -62,7 +65,7 @@ proc optPar*(p: var TParser)
 proc optInd*(p: var TParser, n: PNode)
 proc indAndComment*(p: var TParser, n: PNode)
 proc setBaseFlags*(n: PNode, base: TNumericalBase)
-proc parseSymbol*(p: var TParser, allowNil = false): PNode
+proc parseSymbol*(p: var TParser, mode = smNormal): PNode
 proc parseTry(p: var TParser; isExpr: bool): PNode
 proc parseCase(p: var TParser): PNode
 proc parseStmtPragma(p: var TParser): PNode
@@ -304,13 +307,24 @@ proc colcom(p: var TParser, n: PNode) =
   eat(p, tkColon)
   skipComment(p, n)
 
-proc parseSymbol(p: var TParser, allowNil = false): PNode =
+proc parseSymbol(p: var TParser, mode = smNormal): PNode =
   #| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`'
-  #|        | IDENT | 'addr' | 'type'
+  #|        | IDENT | KEYW
   case p.tok.tokType
-  of tkSymbol, tkAddr, tkType:
+  of tkSymbol:
     result = newIdentNodeP(p.tok.ident, p)
     getTok(p)
+  of tokKeywordLow..tokKeywordHigh:
+    if p.tok.tokType == tkAddr or p.tok.tokType == tkType or mode == smAfterDot:
+      # for backwards compatibility these 2 are always valid:
+      result = newIdentNodeP(p.tok.ident, p)
+      getTok(p)
+    elif p.tok.tokType == tkNil and mode == smAllowNil:
+      result = newNodeP(nkNilLit, p)
+      getTok(p)
+    else:
+      parMessage(p, errIdentifierExpected, p.tok)
+      result = ast.emptyNode
   of tkAccent:
     result = newNodeP(nkAccQuoted, p)
     getTok(p)
@@ -335,10 +349,6 @@ proc parseSymbol(p: var TParser, allowNil = false): PNode =
         parMessage(p, errIdentifierExpected, p.tok)
         break
     eat(p, tkAccent)
-  else:
-    if allowNil and p.tok.tokType == tkNil:
-      result = newNodeP(nkNilLit, p)
-      getTok(p)
   else:
     parMessage(p, errIdentifierExpected, p.tok)
     # BUGFIX: We must consume a token here to prevent endless loops!
@@ -390,7 +400,7 @@ proc dotExpr(p: var TParser, a: PNode): PNode =
   result = newNodeI(nkDotExpr, info)
   optInd(p, result)
   addSon(result, a)
-  addSon(result, parseSymbol(p))
+  addSon(result, parseSymbol(p, smAfterDot))
 
 proc qualifiedIdent(p: var TParser): PNode =
   #| qualifiedIdent = symbol ('.' optInd symbol)?
@@ -846,6 +856,7 @@ type
   TDeclaredIdentFlag = enum
     withPragma,               # identifier may have pragma
     withBothOptional          # both ':' and '=' parts are optional
+    withDot                   # allow 'var ident.ident = value'
   TDeclaredIdentFlags = set[TDeclaredIdentFlag]
 
 proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode =
@@ -859,7 +870,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode =
   while true:
     case p.tok.tokType
     of tkSymbol, tkAccent:
-      if withPragma in flags: a = identWithPragma(p)
+      if withPragma in flags: a = identWithPragma(p, allowDot=withdot in flags)
       else: a = parseSymbol(p)
       if a.kind == nkEmpty: return
     else: break
@@ -1004,10 +1015,10 @@ proc isExprStart(p: TParser): bool =
     result = true
   else: result = false
 
-proc parseSymbolList(p: var TParser, result: PNode, allowNil = false) =
+proc parseSymbolList(p: var TParser, result: PNode) =
   # progress guaranteed
   while true:
-    var s = parseSymbol(p, allowNil)
+    var s = parseSymbol(p, smAllowNil)
     if s.kind == nkEmpty: break
     addSon(result, s)
     if p.tok.tokType != tkComma: break
@@ -1028,7 +1039,7 @@ proc parseTypeDescKAux(p: var TParser, kind: TNodeKind,
     getTok(p)
     let list = newNodeP(nodeKind, p)
     result.addSon list
-    parseSymbolList(p, list, allowNil = true)
+    parseSymbolList(p, list)
 
 proc parseExpr(p: var TParser): PNode =
   #| expr = (ifExpr
@@ -1520,6 +1531,13 @@ proc parseGenericParam(p: var TParser): PNode =
   # progress guaranteed
   while true:
     case p.tok.tokType
+    of tkIn, tkOut:
+      let x = p.lex.cache.getIdent(if p.tok.tokType == tkIn: "in" else: "out")
+      a = newNodeP(nkPrefix, p)
+      a.addSon newIdentNodeP(x, p)
+      getTok(p)
+      expectIdent(p)
+      a.addSon(parseSymbol(p))
     of tkSymbol, tkAccent:
       a = parseSymbol(p)
       if a.kind == nkEmpty: return
@@ -1548,7 +1566,7 @@ proc parseGenericParamList(p: var TParser): PNode =
   getTok(p)
   optInd(p, result)
   # progress guaranteed
-  while p.tok.tokType in {tkSymbol, tkAccent}:
+  while p.tok.tokType in {tkSymbol, tkAccent, tkIn, tkOut}:
     var a = parseGenericParam(p)
     addSon(result, a)
     if p.tok.tokType notin {tkComma, tkSemiColon}: break
@@ -1882,7 +1900,7 @@ proc parseVarTuple(p: var TParser): PNode =
   optInd(p, result)
   # progress guaranteed
   while p.tok.tokType in {tkSymbol, tkAccent}:
-    var a = identWithPragma(p)
+    var a = identWithPragma(p, allowDot=true)
     addSon(result, a)
     if p.tok.tokType != tkComma: break
     getTok(p)
@@ -1898,7 +1916,7 @@ proc parseVariable(p: var TParser): PNode =
   #| colonBody = colcom stmt doBlocks?
   #| variable = (varTuple / identColonEquals) colonBody? indAndComment
   if p.tok.tokType == tkParLe: result = parseVarTuple(p)
-  else: result = parseIdentColonEquals(p, {withPragma})
+  else: result = parseIdentColonEquals(p, {withPragma, withDot})
   result{-1} = postExprBlocks(p, result{-1})
   indAndComment(p, result)
 

compiler/pbraces.nim

@@ -1336,6 +1336,11 @@ proc parseGenericParam(p: var TParser): PNode =
   result = newNodeP(nkIdentDefs, p)
   while true:
     case p.tok.tokType
+    of tkIn, tkOut:
+      let t = p.tok.tokType
+      getTok(p)
+      expectIdent(p)
+      a = parseSymbol(p)
     of tkSymbol, tkAccent:
       a = parseSymbol(p)
       if a.kind == nkEmpty: return

compiler/pragmas.nim

@@ -125,6 +125,7 @@ proc processImportCpp(s: PSym, extname: string, info: TLineInfo) =
   incl(s.flags, sfImportc)
   incl(s.flags, sfInfixCall)
   excl(s.flags, sfForward)
+  if gCmd == cmdCompileToC:
     let m = s.getModule()
     incl(m.flags, sfCompileToCpp)
   extccomp.gMixedMode = true

compiler/rodutils.nim

@@ -12,17 +12,17 @@ import strutils
 
 proc c_sprintf(buf, frmt: cstring) {.importc: "sprintf", header: "<stdio.h>", nodecl, varargs.}
 
-proc toStrMaxPrecision*(f: BiggestFloat): string =
+proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
   if f != f:
     result = "NAN"
   elif f == 0.0:
-    result = "0.0"
+    result = "0.0" & literalPostfix
   elif f == 0.5 * f:
     if f > 0.0: result = "INF"
     else: result = "-INF"
   else:
     var buf: array[0..80, char]
-    c_sprintf(buf, "%#.16e", f)
+    c_sprintf(buf, "%#.16e" & literalPostfix, f)
     result = $buf
 
 proc encodeStr*(s: string, result: var string) =

compiler/sem.nim

@@ -442,6 +442,7 @@ proc semConstBoolExpr(c: PContext, n: PNode): PNode =
     result = nn
 
 proc semGenericStmt(c: PContext, n: PNode): PNode
+proc semConceptBody(c: PContext, n: PNode): PNode
 
 include semtypes, semtempl, semgnrc, semstmts, semexprs
 

compiler/semcall.nim

@@ -408,6 +408,12 @@ proc semOverloadedCall(c: PContext, n, nOrig: PNode,
     else:
       # get rid of the deref again for a better error message:
       n.sons[1] = n.sons[1].sons[0]
+      #notFoundError(c, n, errors)
+      if efExplain notin flags:
+        # repeat the overload resolution,
+        # this time enabling all the diagnostic output (this should fail again)
+        discard semOverloadedCall(c, n, nOrig, filter, flags + {efExplain})
+      else:
         notFoundError(c, n, errors)
   else:
     if efExplain notin flags:
@@ -429,7 +435,7 @@ proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode =
   for i in 1..sonsLen(n)-1:
     let formal = s.ast.sons[genericParamsPos].sons[i-1].typ
     let arg = n[i].typ
-    let tm = typeRel(m, formal, arg, true)
+    let tm = typeRel(m, formal, arg)
     if tm in {isNone, isConvertible}: return nil
   var newInst = generateInstance(c, s, m.bindings, n.info)
   newInst.typ.flags.excl tfUnresolved
@@ -440,7 +446,8 @@ proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode =
 proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
   assert n.kind == nkBracketExpr
   for i in 1..sonsLen(n)-1:
-    n.sons[i].typ = semTypeNode(c, n.sons[i], nil)
+    let e = semExpr(c, n.sons[i])
+    n.sons[i].typ = e.typ.skipTypes({tyTypeDesc})
   var s = s
   var a = n.sons[0]
   if a.kind == nkSym:

compiler/semdata.nim

@@ -40,6 +40,11 @@ type
     bracketExpr*: PNode       # current bracket expression (for ^ support)
     mapping*: TIdTable
 
+  TMatchedConcept* = object
+    candidateType*: PType
+    prev*: ptr TMatchedConcept
+    depth*: int
+
   TInstantiationPair* = object
     genericSym*: PSym
     inst*: PInstantiation
@@ -75,6 +80,7 @@ type
     importTable*: PScope       # scope for all imported symbols
     topLevelScope*: PScope     # scope for all top-level symbols
     p*: PProcCon               # procedure context
+    matchedConcept*: ptr TMatchedConcept # the current concept being matched
     friendModules*: seq[PSym]  # friend modules; may access private data;
                                # this is used so that generic instantiations
                                # can access private object fields
@@ -82,7 +88,6 @@ type
 
     ambiguousSymbols*: IntSet  # ids of all ambiguous symbols (cannot
                                # store this info in the syms themselves!)
-    inTypeClass*: int          # > 0 if we are in a user-defined type class
     inGenericContext*: int     # > 0 if we are in a generic type
     inUnrolledContext*: int    # > 0 if we are unrolling a loop
     compilesContextId*: int    # > 0 if we are in a ``compiles`` magic
@@ -277,6 +282,10 @@ proc makeTypeFromExpr*(c: PContext, n: PNode): PType =
   assert n != nil
   result.n = n
 
+proc newTypeWithSons*(owner: PSym, kind: TTypeKind, sons: seq[PType]): PType =
+  result = newType(kind, owner)
+  result.sons = sons
+
 proc newTypeWithSons*(c: PContext, kind: TTypeKind,
                       sons: seq[PType]): PType =
   result = newType(kind, getCurrOwner(c))

compiler/semexprs.nim

@@ -106,7 +106,10 @@ proc checkConvertible(c: PContext, castDest, src: PType): TConvStatus =
       result = convNotNeedeed
     return
   var d = skipTypes(castDest, abstractVar)
-  var s = skipTypes(src, abstractVar-{tyTypeDesc})
+  var s = src
+  if s.kind in tyUserTypeClasses and s.isResolvedUserTypeClass:
+    s = s.lastSon
+  s = skipTypes(s, abstractVar-{tyTypeDesc})
   var pointers = 0
   while (d != nil) and (d.kind in {tyPtr, tyRef}) and (d.kind == s.kind):
     d = d.lastSon
@@ -152,6 +155,8 @@ proc isCastable(dst, src: PType): bool =
     result = false
   elif typeAllowed(dst, skParam) != nil:
     result = false
+  elif dst.kind == tyProc and dst.callConv == ccClosure:
+    result = src.kind == tyProc and src.callConv == ccClosure
   else:
     result = (dstSize >= srcSize) or
         (skipTypes(dst, abstractInst).kind in IntegralTypes) or
@@ -227,7 +232,10 @@ proc semCast(c: PContext, n: PNode): PNode =
   if tfHasMeta in targetType.flags:
     localError(n.sons[0].info, errCastToANonConcreteType, $targetType)
   if not isCastable(targetType, castedExpr.typ):
-    localError(n.info, errExprCannotBeCastToX, $targetType)
+    let tar = $targetType
+    let alt = typeToString(targetType, preferDesc)
+    let msg = if tar != alt: tar & "=" & alt else: tar
+    localError(n.info, errExprCannotBeCastToX, msg)
   result = newNodeI(nkCast, n.info)
   result.typ = targetType
   addSon(result, copyTree(n.sons[0]))
@@ -683,27 +691,9 @@ proc bracketedMacro(n: PNode): PSym =
     if result.kind notin {skMacro, skTemplate}:
       result = nil
 
-proc semBracketedMacro(c: PContext; outer, inner: PNode; s: PSym;
+proc setGenericParams(c: PContext, n: PNode) =
-                       flags: TExprFlags): PNode =
+  for i in 1 .. <n.len:
-  # We received untransformed bracket expression coming from macroOrTmpl[].
+    n[i].typ = semTypeNode(c, n[i], nil)
-  # Transform it to macro or template call, where first come normal
-  # arguments, next come generic template arguments.
-  var sons = newSeq[PNode]()
-  sons.add inner.sons[0]
-  # Normal arguments:
-  for i in 1..<outer.len:
-    sons.add outer.sons[i]
-  # Generic template arguments from bracket expression:
-  for i in 1..<inner.len:
-    sons.add inner.sons[i]
-  shallowCopy(outer.sons, sons)
-  # FIXME: Shouldn't we check sfImmediate and call semDirectOp?
-  # However passing to semDirectOp doesn't work here.
-  case s.kind
-  of skMacro: result = semMacroExpr(c, outer, outer, s, flags)
-  of skTemplate: result = semTemplateExpr(c, outer, s, flags)
-  else: assert(false)
-  return
 
 proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode =
   result = n
@@ -718,7 +708,7 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode =
     analyseIfAddressTakenInCall(c, result)
     if callee.magic != mNone:
       result = magicsAfterOverloadResolution(c, result, flags)
-  if c.inTypeClass == 0:
+  if c.matchedConcept == nil:
     result = evalAtCompileTime(c, result)
 
 proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
@@ -745,7 +735,8 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
     elif n.sons[0].kind == nkBracketExpr:
       let s = bracketedMacro(n.sons[0])
       if s != nil:
-        return semBracketedMacro(c, n, n.sons[0], s, flags)
+        setGenericParams(c, n[0])
+        return semDirectOp(c, n, flags)
 
   let nOrig = n.copyTree
   semOpAux(c, n)
@@ -897,20 +888,6 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent,
     if r.sym.name.id == field.id: result = r.sym
   else: illFormedAst(n)
 
-proc makeDeref(n: PNode): PNode =
-  var t = skipTypes(n.typ, {tyGenericInst, tyAlias})
-  result = n
-  if t.kind == tyVar:
-    result = newNodeIT(nkHiddenDeref, n.info, t.sons[0])
-    addSon(result, n)
-    t = skipTypes(t.sons[0], {tyGenericInst, tyAlias})
-  while t.kind in {tyPtr, tyRef}:
-    var a = result
-    let baseTyp = t.lastSon
-    result = newNodeIT(nkHiddenDeref, n.info, baseTyp)
-    addSon(result, a)
-    t = skipTypes(baseTyp, {tyGenericInst, tyAlias})
-
 const
   tyTypeParamsHolders = {tyGenericInst, tyCompositeTypeClass}
   tyDotOpTransparent = {tyVar, tyPtr, tyRef, tyAlias}
@@ -997,7 +974,7 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
   of skParam:
     markUsed(n.info, s, c.graph.usageSym)
     styleCheckUse(n.info, s)
-    if s.typ.kind == tyStatic and s.typ.n != nil:
+    if s.typ != nil and s.typ.kind == tyStatic and s.typ.n != nil:
       # XXX see the hack in sigmatch.nim ...
       return s.typ.n
     elif sfGenSym in s.flags:
@@ -1155,9 +1132,9 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
     # reset to prevent 'nil' bug: see "tests/reject/tenumitems.nim":
     ty = n.sons[0].typ
     return nil
-  ty = skipTypes(ty, {tyGenericInst, tyVar, tyPtr, tyRef, tyAlias})
   if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass:
     ty = ty.lastSon
+  ty = skipTypes(ty, {tyGenericInst, tyVar, tyPtr, tyRef, tyAlias})
   while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct})
   var check: PNode = nil
   if ty.kind == tyObject:
@@ -2164,10 +2141,6 @@ proc shouldBeBracketExpr(n: PNode): bool =
           n.sons[0] = be
           return true
 
-proc setGenericParams(c: PContext, n: PNode) =
-  for i in 1 .. <n.len:
-    n[i].typ = semTypeNode(c, n[i], nil)
-
 proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   result = n
   if gCmd == cmdIdeTools: suggestExpr(c, n)
@@ -2177,7 +2150,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     let checks = if efNoEvaluateGeneric in flags: {checkUndeclared}
                  else: {checkUndeclared, checkModule, checkAmbiguity}
     var s = qualifiedLookUp(c, n, checks)
-    if c.inTypeClass == 0: semCaptureSym(s, c.p.owner)
+    if c.matchedConcept == nil: semCaptureSym(s, c.p.owner)
     result = semSym(c, n, s, flags)
     if s.kind in {skProc, skMethod, skConverter, skIterator}:
       #performProcvarCheck(c, n, s)
@@ -2214,11 +2187,11 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     if result.typ == nil: result.typ = getSysType(tyUInt32)
   of nkUInt64Lit:
     if result.typ == nil: result.typ = getSysType(tyUInt64)
-  of nkFloatLit:
+  #of nkFloatLit:
-    if result.typ == nil: result.typ = getFloatLitType(result)
+  #  if result.typ == nil: result.typ = getFloatLitType(result)
   of nkFloat32Lit:
     if result.typ == nil: result.typ = getSysType(tyFloat32)
-  of nkFloat64Lit:
+  of nkFloat64Lit, nkFloatLit:
     if result.typ == nil: result.typ = getSysType(tyFloat64)
   of nkFloat128Lit:
     if result.typ == nil: result.typ = getSysType(tyFloat128)

compiler/semfold.nim

@@ -594,8 +594,11 @@ proc foldConStrStr(m: PSym, n: PNode): PNode =
 
 proc newSymNodeTypeDesc*(s: PSym; info: TLineInfo): PNode =
   result = newSymNode(s, info)
+  if s.typ.kind != tyTypeDesc:
     result.typ = newType(tyTypeDesc, s.owner)
     result.typ.addSonSkipIntLit(s.typ)
+  else:
+    result.typ = s.typ
 
 proc getConstExpr(m: PSym, n: PNode): PNode =
   result = nil
@@ -634,9 +637,11 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
         result = newSymNodeTypeDesc(s, n.info)
     of skGenericParam:
       if s.typ.kind == tyStatic:
-        if s.typ.n != nil:
+        if s.typ.n != nil and tfUnresolved notin s.typ.flags:
+          result = s.typ.n
+          result.typ = s.typ.base
+      elif s.typ.isIntLit:
         result = s.typ.n
-          result.typ = s.typ.sons[0]
       else:
         result = newSymNodeTypeDesc(s, n.info)
     else: discard

compiler/semgnrc.nim

@@ -34,7 +34,7 @@ type
 
 type
   TSemGenericFlag = enum
-    withinBind, withinTypeDesc, withinMixin
+    withinBind, withinTypeDesc, withinMixin, withinConcept
   TSemGenericFlags = set[TSemGenericFlag]
 
 proc semGenericStmt(c: PContext, n: PNode,
@@ -200,12 +200,13 @@ proc semGenericStmt(c: PContext, n: PNode,
     checkMinSonsLen(n, 1)
     let fn = n.sons[0]
     var s = qualifiedLookUp(c, fn, {})
-    if s == nil and withinMixin notin flags and
+    if  s == nil and
+        {withinMixin, withinConcept}*flags == {} and
         fn.kind in {nkIdent, nkAccQuoted} and
         considerQuotedIdent(fn).id notin ctx.toMixin:
       errorUndeclaredIdentifier(c, n.info, fn.renderTree)
 
-    var first = 0
+    var first = ord(withinConcept in flags)
     var mixinContext = false
     if s != nil:
       incl(s.flags, sfUsed)
@@ -471,3 +472,9 @@ proc semGenericStmt(c: PContext, n: PNode): PNode =
   ctx.toMixin = initIntset()
   result = semGenericStmt(c, n, {}, ctx)
   semIdeForTemplateOrGeneric(c, result, ctx.cursorInBody)
+
+proc semConceptBody(c: PContext, n: PNode): PNode =
+  var ctx: GenericCtx
+  ctx.toMixin = initIntset()
+  result = semGenericStmt(c, n, {withinConcept}, ctx)
+  semIdeForTemplateOrGeneric(c, result, ctx.cursorInBody)

compiler/seminst.nim

@@ -88,7 +88,8 @@ proc sameInstantiation(a, b: TInstantiation): bool =
   if a.concreteTypes.len == b.concreteTypes.len:
     for i in 0..a.concreteTypes.high:
       if not compareTypes(a.concreteTypes[i], b.concreteTypes[i],
-                          flags = {ExactTypeDescValues}): return
+                          flags = {ExactTypeDescValues,
+                                   ExactGcSafety}): return
     result = true
 
 proc genericCacheGet(genericSym: PSym, entry: TInstantiation;
@@ -173,10 +174,14 @@ proc sideEffectsCheck(c: PContext, s: PSym) =
 
 proc instGenericContainer(c: PContext, info: TLineInfo, header: PType,
                           allowMetaTypes = false): PType =
-  var cl: TReplTypeVars
+  var
+    typeMap: LayeredIdTable
+    cl: TReplTypeVars
+
   initIdTable(cl.symMap)
-  initIdTable(cl.typeMap)
   initIdTable(cl.localCache)
+  initIdTable(typeMap.topLayer)
+  cl.typeMap = addr(typeMap)
   cl.info = info
   cl.c = c
   cl.allowMetaTypes = allowMetaTypes
@@ -200,7 +205,8 @@ proc instantiateProcType(c: PContext, pt: TIdTable,
   #addDecl(c, prc)
 
   pushInfoContext(info)
-  var cl = initTypeVars(c, pt, info, nil)
+  var typeMap = initLayeredTypeMap(pt)
+  var cl = initTypeVars(c, addr(typeMap), info, nil)
   var result = instCopyType(cl, prc.typ)
   let originalParams = result.n
   result.n = originalParams.shallowCopy
@@ -256,8 +262,8 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   # NOTE: for access of private fields within generics from a different module
   # we set the friend module:
   c.friendModules.add(getModule(fn))
-  let oldInTypeClass = c.inTypeClass
+  let oldMatchedConcept = c.matchedConcept
-  c.inTypeClass = 0
+  c.matchedConcept = nil
   let oldScope = c.currentScope
   while not isTopLevel(c): c.currentScope = c.currentScope.parent
   result = copySym(fn, false)
@@ -318,5 +324,5 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   c.currentScope = oldScope
   discard c.friendModules.pop()
   dec(c.instCounter)
-  c.inTypeClass = oldInTypeClass
+  c.matchedConcept = oldMatchedConcept
   if result.kind == skMethod: finishMethod(c, result)

compiler/semmagic.nim

@@ -109,27 +109,43 @@ proc uninstantiate(t: PType): PType =
     of tyCompositeTypeClass: uninstantiate t.sons[1]
     else: t
 
-proc evalTypeTrait(trait: PNode, operand: PType, context: PSym): PNode =
+proc evalTypeTrait(traitCall: PNode, operand: PType, context: PSym): PNode =
-  var typ = operand.skipTypes({tyTypeDesc})
+  const skippedTypes = {tyTypeDesc, tyAlias}
+  let trait = traitCall[0]
+  internalAssert trait.kind == nkSym
+  var operand = operand.skipTypes(skippedTypes)
+
+  template operand2: PType =
+    traitCall.sons[2].typ.skipTypes({tyTypeDesc})
+
+  template typeWithSonsResult(kind, sons): PNode =
+    newTypeWithSons(context, kind, sons).toNode(traitCall.info)
+
   case trait.sym.name.s
+  of "or", "|":
+    return typeWithSonsResult(tyOr, @[operand, operand2])
+  of "and":
+    return typeWithSonsResult(tyAnd, @[operand, operand2])
+  of "not":
+    return typeWithSonsResult(tyNot, @[operand])
   of "name":
-    result = newStrNode(nkStrLit, typ.typeToString(preferName))
+    result = newStrNode(nkStrLit, operand.typeToString(preferName))
     result.typ = newType(tyString, context)
-    result.info = trait.info
+    result.info = traitCall.info
   of "arity":
-    result = newIntNode(nkIntLit, typ.len - ord(typ.kind==tyProc))
+    result = newIntNode(nkIntLit, operand.len - ord(operand.kind==tyProc))
     result.typ = newType(tyInt, context)
-    result.info = trait.info
+    result.info = traitCall.info
   of "genericHead":
-    var res = uninstantiate(typ)
+    var res = uninstantiate(operand)
-    if res == typ and res.kind notin tyMagicGenerics:
+    if res == operand and res.kind notin tyMagicGenerics:
-      localError(trait.info,
+      localError(traitCall.info,
         "genericHead expects a generic type. The given type was " &
-        typeToString(typ))
+        typeToString(operand))
-      return newType(tyError, context).toNode(trait.info)
+      return newType(tyError, context).toNode(traitCall.info)
-    result = res.base.toNode(trait.info)
+    result = res.base.toNode(traitCall.info)
   of "stripGenericParams":
-    result = uninstantiate(typ).toNode(trait.info)
+    result = uninstantiate(operand).toNode(traitCall.info)
   else:
     internalAssert false
 
@@ -140,7 +156,7 @@ proc semTypeTraits(c: PContext, n: PNode): PNode =
   if t.sonsLen > 0:
     # This is either a type known to sem or a typedesc
     # param to a regular proc (again, known at instantiation)
-    result = evalTypeTrait(n[0], t, getCurrOwner(c))
+    result = evalTypeTrait(n, t, getCurrOwner(c))
   else:
     # a typedesc variable, pass unmodified to evals
     result = n

compiler/sempass2.nim

@@ -69,7 +69,8 @@ proc `==`(a, b: TLockLevel): bool {.borrow.}
 proc max(a, b: TLockLevel): TLockLevel {.borrow.}
 
 proc isLocalVar(a: PEffects, s: PSym): bool =
-  s.kind in {skVar, skResult} and sfGlobal notin s.flags and s.owner == a.owner
+  s.kind in {skVar, skResult} and sfGlobal notin s.flags and
+    s.owner == a.owner and s.typ != nil
 
 proc getLockLevel(t: PType): TLockLevel =
   var t = t
@@ -218,12 +219,11 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet) =
       message(s.info, msgKind,
         "'$#' is not GC-safe as it calls '$#'" %
         [s.name.s, u.name.s])
-    of skParam:
+    of skParam, skForVar:
       message(s.info, msgKind,
         "'$#' is not GC-safe as it performs an indirect call via '$#'" %
         [s.name.s, u.name.s])
     else:
-      internalAssert u.kind == skUnknown
       message(u.info, msgKind,
         "'$#' is not GC-safe as it performs an indirect call here" % s.name.s)
 

compiler/semstmts.nim

@@ -144,7 +144,7 @@ proc fixNilType(n: PNode) =
   n.typ = nil
 
 proc discardCheck(c: PContext, result: PNode) =
-  if c.inTypeClass > 0: return
+  if c.matchedConcept != nil: return
   if result.typ != nil and result.typ.kind notin {tyStmt, tyVoid}:
     if result.kind == nkNilLit:
       result.typ = nil
@@ -465,6 +465,47 @@ proc hasEmpty(typ: PType): bool =
     for s in typ.sons:
       result = result or hasEmpty(s)
 
+proc makeDeref(n: PNode): PNode =
+  var t = skipTypes(n.typ, {tyGenericInst, tyAlias})
+  if t.kind in tyUserTypeClasses and t.isResolvedUserTypeClass:
+    t = t.lastSon
+  result = n
+  if t.kind == tyVar:
+    result = newNodeIT(nkHiddenDeref, n.info, t.sons[0])
+    addSon(result, n)
+    t = skipTypes(t.sons[0], {tyGenericInst, tyAlias})
+  while t.kind in {tyPtr, tyRef}:
+    var a = result
+    let baseTyp = t.lastSon
+    result = newNodeIT(nkHiddenDeref, n.info, baseTyp)
+    addSon(result, a)
+    t = skipTypes(baseTyp, {tyGenericInst, tyAlias})
+
+proc fillPartialObject(c: PContext; n: PNode; typ: PType) =
+  if n.len == 2:
+    let x = semExprWithType(c, n[0])
+    let y = considerQuotedIdent(n[1])
+    let obj = x.typ.skipTypes(abstractPtrs)
+    if obj.kind == tyObject and tfPartial in obj.flags:
+      let field = newSym(skField, getIdent(y.s), obj.sym, n[1].info)
+      field.typ = skipIntLit(typ)
+      field.position = sonsLen(obj.n)
+      addSon(obj.n, newSymNode(field))
+      n.sons[0] = makeDeref x
+      n.sons[1] = newSymNode(field)
+    else:
+      localError(n.info, "implicit object field construction " &
+        "requires a .partial object, but got " & typeToString(obj))
+  else:
+    localError(n.info, "nkDotNode requires 2 children")
+
+proc setVarType(v: PSym, typ: PType) =
+  if v.typ != nil and not sameTypeOrNil(v.typ, typ):
+    localError(v.info, "inconsistent typing for reintroduced symbol '" &
+        v.name.s & "': previous type was: " & typeToString(v.typ) &
+        "; new type is: " & typeToString(typ))
+  v.typ = typ
+
 proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
   var b: PNode
   result = copyNode(n)
@@ -529,6 +570,11 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
       message(a.info, warnEachIdentIsTuple)
 
     for j in countup(0, length-3):
+      if a[j].kind == nkDotExpr:
+        fillPartialObject(c, a[j],
+          if a.kind != nkVarTuple: typ else: tup.sons[j])
+        addToVarSection(c, result, n, a)
+        continue
       var v = semIdentDef(c, a.sons[j], symkind)
       if sfGenSym notin v.flags and not isDiscardUnderscore(v):
         addInterfaceDecl(c, v)
@@ -549,7 +595,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
           # this is needed for the evaluation pass and for the guard checking:
           v.ast = def
           if sfThread in v.flags: localError(def.info, errThreadvarCannotInit)
-        v.typ = typ
+        setVarType(v, typ)
         b = newNodeI(nkIdentDefs, a.info)
         if importantComments():
           # keep documentation information:
@@ -560,7 +606,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         addToVarSection(c, result, n, b)
       else:
         if def.kind == nkPar: v.ast = def[j]
-        v.typ = tup.sons[j]
+        setVarType(v, tup.sons[j])
         b.sons[j] = newSymNode(v)
       addDefer(c, result, v)
       checkNilable(v)
@@ -594,7 +640,7 @@ proc semConst(c: PContext, n: PNode): PNode =
     if typeAllowed(typ, skConst) != nil and def.kind != nkNilLit:
       localError(a.info, "invalid type for const: " & typeToString(typ))
       continue
-    v.typ = typ
+    setVarType(v, typ)
     v.ast = def               # no need to copy
     if sfGenSym notin v.flags: addInterfaceDecl(c, v)
     var b = newNodeI(nkConstDef, a.info)
@@ -730,7 +776,9 @@ proc typeSectionLeftSidePass(c: PContext, n: PNode) =
     var s: PSym
     if name.kind == nkDotExpr:
       s = qualifiedLookUp(c, name, {checkUndeclared, checkModule})
-      if s.kind != skType or s.typ.skipTypes(abstractPtrs).kind != tyObject or tfPartial notin s.typ.skipTypes(abstractPtrs).flags:
+      if s.kind != skType or
+         s.typ.skipTypes(abstractPtrs).kind != tyObject or
+         tfPartial notin s.typ.skipTypes(abstractPtrs).flags:
         localError(name.info, "only .partial objects can be extended")
     else:
       s = semIdentDef(c, name, skType)
@@ -742,6 +790,87 @@ proc typeSectionLeftSidePass(c: PContext, n: PNode) =
       if sfGenSym notin s.flags: addInterfaceDecl(c, s)
     a.sons[0] = newSymNode(s)
 
+proc checkCovariantParamsUsages(genericType: PType) =
+  var body = genericType{-1}
+
+  proc traverseSubTypes(t: PType): bool =
+    template error(msg) = localError(genericType.sym.info, msg)
+
+    result = false
+
+    template subresult(r) =
+      let sub = r
+      result = result or sub
+
+    case t.kind
+    of tyGenericParam:
+      t.flags.incl tfWeakCovariant
+      return true
+
+    of tyObject:
+      for field in t.n:
+        subresult traverseSubTypes(field.typ)
+
+    of tyArray:
+      return traverseSubTypes(t[1])
+
+    of tyProc:
+      for subType in t.sons:
+        if subType != nil:
+          subresult traverseSubTypes(subType)
+      if result:
+        error("non-invariant type param used in a proc type: " &  $t)
+
+    of tySequence:
+      return traverseSubTypes(t[0])
+
+    of tyGenericInvocation:
+      let targetBody = t[0]
+      for i in 1 .. <t.len:
+        let param = t[i]
+        if param.kind == tyGenericParam:
+          if tfCovariant in param.flags:
+            let formalFlags = targetBody[i-1].flags
+            if tfCovariant notin formalFlags:
+              error("covariant param '" & param.sym.name.s &
+                    "' used in a non-covariant position")
+            elif tfWeakCovariant in formalFlags:
+              param.flags.incl tfWeakCovariant
+            result = true
+          elif tfContravariant in param.flags:
+            let formalParam = targetBody[i-1].sym
+            if tfContravariant notin formalParam.typ.flags:
+              error("contravariant param '" & param.sym.name.s &
+                    "' used in a non-contravariant position")
+            result = true
+        else:
+          subresult traverseSubTypes(param)
+
+    of tyAnd, tyOr, tyNot, tyStatic, tyBuiltInTypeClass, tyCompositeTypeClass:
+      error("non-invariant type parameters cannot be used with types such '" & $t & "'")
+
+    of tyUserTypeClass, tyUserTypeClassInst:
+      error("non-invariant type parameters are not supported in concepts")
+
+    of tyTuple:
+      for fieldType in t.sons:
+        subresult traverseSubTypes(fieldType)
+
+    of tyPtr, tyRef, tyVar:
+      if t.base.kind == tyGenericParam: return true
+      return traverseSubTypes(t.base)
+
+    of tyDistinct, tyAlias:
+      return traverseSubTypes(t.lastSon)
+
+    of tyGenericInst:
+      internalAssert false
+
+    else:
+      discard
+
+  discard traverseSubTypes(body)
+
 proc typeSectionRightSidePass(c: PContext, n: PNode) =
   for i in countup(0, sonsLen(n) - 1):
     var a = n.sons[i]
@@ -782,6 +911,22 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
         body.sym = s
         body.size = -1 # could not be computed properly
         s.typ.sons[sonsLen(s.typ) - 1] = body
+        if tfCovariant in s.typ.flags:
+          checkCovariantParamsUsages(s.typ)
+          # XXX: This is a temporary limitation:
+          # The codegen currently produces various failures with
+          # generic imported types that have fields, but we need
+          # the fields specified in order to detect weak covariance.
+          # The proper solution is to teach the codegen how to handle
+          # such types, because this would offer various interesting
+          # possibilities such as instantiating C++ generic types with
+          # garbage collected Nim types.
+          if sfImportc in s.flags:
+            var body = s.typ.lastSon
+            if body.kind == tyObject:
+              # erases all declared fields
+              body.n.sons = nil
+
       popOwner(c)
       closeScope(c)
     elif a.sons[2].kind != nkEmpty:
@@ -985,12 +1130,15 @@ proc semProcAnnotation(c: PContext, prc: PNode;
     var x = newNodeI(nkCall, n.info)
     x.add(newSymNode(m))
     prc.sons[pragmasPos] = copyExcept(n, i)
+    if prc[pragmasPos].kind != nkEmpty and prc[pragmasPos].len == 0:
+      prc.sons[pragmasPos] = emptyNode
+
     if it.kind == nkExprColonExpr:
       # pass pragma argument to the macro too:
       x.add(it.sons[1])
     x.add(prc)
     # recursion assures that this works for multiple macro annotations too:
-    result = semStmt(c, x)
+    result = semExpr(c, x)
     # since a proc annotation can set pragmas, we process these here again.
     # This is required for SqueakNim-like export pragmas.
     if result.kind in procDefs and result[namePos].kind == nkSym and
@@ -1322,6 +1470,14 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
       implicitPragmas(c, s, n, validPragmas)
   else:
     if n.sons[pragmasPos].kind != nkEmpty:
+      pragma(c, s, n.sons[pragmasPos], validPragmas)
+      # To ease macro generation that produce forwarded .async procs we now
+      # allow a bit redudancy in the pragma declarations. The rule is
+      # a prototype's pragma list must be a superset of the current pragma
+      # list.
+      # XXX This needs more checks eventually, for example that external
+      # linking names do agree:
+      if proto.typ.callConv != s.typ.callConv or proto.typ.flags < s.typ.flags:
         localError(n.sons[pragmasPos].info, errPragmaOnlyInHeaderOfProcX,
           "'" & proto.name.s & "' from " & $proto.info)
     if sfForward notin proto.flags:
@@ -1402,7 +1558,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   popOwner(c)
   if n.sons[patternPos].kind != nkEmpty:
     c.patterns.add(s)
-  if isAnon: result.typ = s.typ
+  if isAnon:
+    n.kind = nkLambda
+    result.typ = s.typ
   if isTopLevel(c) and s.kind != skIterator and
       s.typ.callConv == ccClosure:
     localError(s.info, "'.closure' calling convention for top level routines is invalid")
@@ -1614,7 +1772,8 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
     else:
       var expr = semExpr(c, n.sons[i], flags)
       n.sons[i] = expr
-      if c.inTypeClass > 0 and expr.typ != nil:
+      if c.matchedConcept != nil and expr.typ != nil and
+         (nfFromTemplate notin n.flags or i != last):
         case expr.typ.kind
         of tyBool:
           if expr.kind == nkInfix and
@@ -1629,7 +1788,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
 
           let verdict = semConstExpr(c, n[i])
           if verdict.intVal == 0:
-            localError(result.info, "type class predicate failed")
+            localError(result.info, "concept predicate failed")
         of tyUnknown: continue
         else: discard
       if n.sons[i].typ == enforceVoidContext: #or usesResult(n.sons[i]):
@@ -1653,7 +1812,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
 
   if result.len == 1 and
      # concept bodies should be preserved as a stmt list:
-     c.inTypeClass == 0 and
+     c.matchedConcept == nil and
      # also, don't make life complicated for macros.
      # they will always expect a proper stmtlist:
      nfBlockArg notin n.flags and

compiler/semtypes.nim

@@ -42,6 +42,8 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
     counter = lastOrd(base) + 1
   rawAddSon(result, base)
   let isPure = result.sym != nil and sfPure in result.sym.flags
+  var symbols: TStrTable
+  if isPure: initStrTable(symbols)
   var hasNull = false
   for i in countup(1, sonsLen(n) - 1):
     case n.sons[i].kind
@@ -87,6 +89,8 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
     addSon(result.n, newSymNode(e))
     styleCheckDef(e)
     if sfGenSym notin e.flags and not isPure: addDecl(c, e)
+    if isPure and strTableIncl(symbols, e):
+      wrongRedefinition(e.info, e.name.s)
     inc(counter)
   if not hasNull: incl(result.flags, tfNeedsInit)
 
@@ -152,6 +156,8 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType =
           message n[i].info, errGenerated, "region needs to be an object type"
         addSonSkipIntLit(result, region)
     addSonSkipIntLit(result, t)
+    if tfPartial in result.flags:
+      if result.lastSon.kind == tyObject: incl(result.lastSon.flags, tfPartial)
     #if not isNilable: result.flags.incl tfNotNil
 
 proc semVarType(c: PContext, n: PNode, prev: PType): PType =
@@ -1129,7 +1135,7 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
     m.isNoCall = true
     matches(c, n, copyTree(n), m)
 
-    if m.state != csMatch and not m.typedescMatched:
+    if m.state != csMatch:
       let err = "cannot instantiate " & typeToString(t) & "\n" &
                 "got: (" & describeArgs(c, n) & ")\n" &
                 "but expected: (" & describeArgs(c, t.n, 0) & ")"
@@ -1200,17 +1206,51 @@ proc semTypeClass(c: PContext, n: PNode, prev: PType): PType =
   # if n.sonsLen == 0: return newConstraint(c, tyTypeClass)
   if nfBase2 in n.flags:
     message(n.info, warnDeprecated, "use 'concept' instead; 'generic'")
-  result = newOrPrevType(tyUserTypeClass, prev, c)
-  result.n = n
-
   let
     pragmas = n[1]
     inherited = n[2]
 
+  result = newOrPrevType(tyUserTypeClass, prev, c)
+  var owner = getCurrOwner(c)
+  var candidateTypeSlot = newTypeWithSons(owner, tyAlias, @[c.errorType])
+  result.sons = @[candidateTypeSlot]
+  result.n = n
+
   if inherited.kind != nkEmpty:
     for n in inherited.sons:
       let typ = semTypeNode(c, n, nil)
-      result.sons.safeAdd(typ)
+      result.sons.add(typ)
+
+  openScope(c)
+  for param in n[0]:
+    var
+      dummyName: PNode
+      dummyType: PType
+
+    let modifier = case param.kind
+      of nkVarTy: tyVar
+      of nkRefTy: tyRef
+      of nkPtrTy: tyPtr
+      of nkStaticTy: tyStatic
+      of nkTypeOfExpr: tyTypeDesc
+      else: tyNone
+
+    if modifier != tyNone:
+      dummyName = param[0]
+      dummyType = c.makeTypeWithModifier(modifier, candidateTypeSlot)
+      if modifier == tyTypeDesc: dummyType.flags.incl tfExplicit
+    else:
+      dummyName = param
+      dummyType = candidateTypeSlot
+
+    internalAssert dummyName.kind == nkIdent
+    var dummyParam = newSym(if modifier == tyTypeDesc: skType else: skVar,
+                            dummyName.ident, owner, owner.info)
+    dummyParam.typ = dummyType
+    addDecl(c, dummyParam)
+
+  result.n.sons[3] = semConceptBody(c, n[3])
+  closeScope(c)
 
 proc semProcTypeWithScope(c: PContext, n: PNode,
                         prev: PType, kind: TSymKind): PType =
@@ -1379,7 +1419,10 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       result = errorType(c)
     else:
       result = typeExpr.typ.base
-      if result.isMetaType:
+      if result.isMetaType and
+         result.kind != tyUserTypeClass:
+           # the dot expression may refer to a concept type in
+           # a different module. allow a normal alias then.
         let preprocessed = semGenericStmt(c, n)
         result = makeTypeFromExpr(c, preprocessed.copyTree)
       else:
@@ -1584,10 +1627,24 @@ proc semGenericParamList(c: PContext, n: PNode, father: PType = nil): PNode =
                       # type for each generic param. the index
                       # of the parameter will be stored in the
                       # attached symbol.
+      var paramName = a.sons[j]
+      var covarianceFlag = tfUnresolved
+
+      if paramName.safeLen == 2:
+        if not nimEnableCovariance or paramName[0].ident.s == "in":
+          if father == nil or sfImportc notin father.sym.flags:
+            localError(paramName.info, errInOutFlagNotExtern, paramName[0].ident.s)
+        covarianceFlag = if paramName[0].ident.s == "in": tfContravariant
+                         else: tfCovariant
+        if father != nil: father.flags.incl tfCovariant
+        paramName = paramName[1]
+
       var s = if finalType.kind == tyStatic or tfWildcard in typ.flags:
-          newSymG(skGenericParam, a.sons[j], c).linkTo(finalType)
+          newSymG(skGenericParam, paramName, c).linkTo(finalType)
         else:
-          newSymG(skType, a.sons[j], c).linkTo(finalType)
+          newSymG(skType, paramName, c).linkTo(finalType)
+
+      if covarianceFlag != tfUnresolved: s.typ.flags.incl(covarianceFlag)
       if def.kind != nkEmpty: s.ast = def
       if father != nil: addSonSkipIntLit(father, s.typ)
       s.position = result.len

compiler/semtypinst.nim

@@ -73,9 +73,13 @@ proc cacheTypeInst*(inst: PType) =
 
 
 type
+  LayeredIdTable* = object
+    topLayer*: TIdTable
+    nextLayer*: ptr LayeredIdTable
+
   TReplTypeVars* {.final.} = object
     c*: PContext
-    typeMap*: TIdTable        # map PType to PType
+    typeMap*: ptr LayeredIdTable # map PType to PType
     symMap*: TIdTable         # map PSym to PSym
     localCache*: TIdTable     # local cache for remembering alraedy replaced
                               # types during instantiation of meta types
@@ -91,6 +95,23 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType
 proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym
 proc replaceTypeVarsN*(cl: var TReplTypeVars, n: PNode; start=0): PNode
 
+proc initLayeredTypeMap*(pt: TIdTable): LayeredIdTable =
+  copyIdTable(result.topLayer, pt)
+
+proc newTypeMapLayer*(cl: var TReplTypeVars): LayeredIdTable =
+  result.nextLayer = cl.typeMap
+  initIdTable(result.topLayer)
+
+proc lookup(typeMap: ptr LayeredIdTable, key: PType): PType =
+  var tm = typeMap
+  while tm != nil:
+    result = PType(idTableGet(tm.topLayer, key))
+    if result != nil: return
+    tm = tm.nextLayer
+
+template put(typeMap: ptr LayeredIdTable, key, value: PType) =
+  idTablePut(typeMap.topLayer, key, value)
+
 template checkMetaInvariants(cl: TReplTypeVars, t: PType) =
   when false:
     if t != nil and tfHasMeta in t.flags and
@@ -106,7 +127,8 @@ proc replaceTypeVarsT*(cl: var TReplTypeVars, t: PType): PType =
 proc prepareNode(cl: var TReplTypeVars, n: PNode): PNode =
   let t = replaceTypeVarsT(cl, n.typ)
   if t != nil and t.kind == tyStatic and t.n != nil:
-    return t.n
+    return if tfUnresolved in t.flags: prepareNode(cl, t.n)
+           else: t.n
   result = copyNode(n)
   result.typ = t
   if result.kind == nkSym: result.sym = replaceTypeVarsS(cl, n.sym)
@@ -219,7 +241,7 @@ proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym =
   result.ast = replaceTypeVarsN(cl, s.ast)
 
 proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType =
-  result = PType(idTableGet(cl.typeMap, t))
+  result = cl.typeMap.lookup(t)
   if result == nil:
     if cl.allowMetaTypes or tfRetType in t.flags: return
     localError(t.sym.info, errCannotInstantiateX, typeToString(t))
@@ -227,7 +249,7 @@ proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType =
     # In order to prevent endless recursions, we must remember
     # this bad lookup and replace it with errorType everywhere.
     # These code paths are only active in "nim check"
-    idTablePut(cl.typeMap, t, result)
+    cl.typeMap.put(t, result)
   elif result.kind == tyGenericParam and not cl.allowMetaTypes:
     internalError(cl.info, "substitution with generic parameter")
 
@@ -243,6 +265,7 @@ proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
 proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   # tyGenericInvocation[A, tyGenericInvocation[A, B]]
   # is difficult to handle:
+  const eqFlags = eqTypeFlags + {tfGcSafe}
   var body = t.sons[0]
   if body.kind != tyGenericBody: internalError(cl.info, "no generic body")
   var header: PType = t
@@ -252,7 +275,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   else:
     result = searchInstTypes(t)
 
-  if result != nil and eqTypeFlags*result.flags == eqTypeFlags*t.flags: return
+  if result != nil and eqFlags*result.flags == eqFlags*t.flags: return
   for i in countup(1, sonsLen(t) - 1):
     var x = t.sons[i]
     if x.kind in {tyGenericParam}:
@@ -267,7 +290,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   if header != t:
     # search again after first pass:
     result = searchInstTypes(header)
-    if result != nil and eqTypeFlags*result.flags == eqTypeFlags*t.flags: return
+    if result != nil and eqFlags*result.flags == eqFlags*t.flags: return
   else:
     header = instCopyType(cl, t)
 
@@ -285,12 +308,16 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
 
   let oldSkipTypedesc = cl.skipTypedesc
   cl.skipTypedesc = true
+
+  var typeMapLayer = newTypeMapLayer(cl)
+  cl.typeMap = addr(typeMapLayer)
+
   for i in countup(1, sonsLen(t) - 1):
     var x = replaceTypeVarsT(cl, t.sons[i])
     assert x.kind != tyGenericInvocation
     header.sons[i] = x
     propagateToOwner(header, x)
-    idTablePut(cl.typeMap, body.sons[i-1], x)
+    cl.typeMap.put(body.sons[i-1], x)
 
   for i in countup(1, sonsLen(t) - 1):
     # if one of the params is not concrete, we cannot do anything
@@ -303,6 +330,9 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   cl.skipTypedesc = oldSkipTypedesc
   newbody.flags = newbody.flags + (t.flags + body.flags - tfInstClearedFlags)
   result.flags = result.flags + newbody.flags - tfInstClearedFlags
+
+  cl.typeMap = cl.typeMap.nextLayer
+
   # This is actually wrong: tgeneric_closure fails with this line:
   #newbody.callConv = body.callConv
   # This type may be a generic alias and we want to resolve it here.
@@ -404,7 +434,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
   if t == nil: return
 
   if t.kind in {tyStatic, tyGenericParam} + tyTypeClasses:
-    let lookup = PType(idTableGet(cl.typeMap, t))
+    let lookup = cl.typeMap.lookup(t)
     if lookup != nil: return lookup
 
   case t.kind
@@ -446,7 +476,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
     result = skipIntLit(t)
 
   of tyTypeDesc:
-    let lookup = PType(idTableGet(cl.typeMap, t)) # lookupTypeVar(cl, t)
+    let lookup = cl.typeMap.lookup(t)
     if lookup != nil:
       result = lookup
       if tfUnresolved in t.flags or cl.skipTypedesc: result = result.base
@@ -485,7 +515,6 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
           propagateToOwner(result, r)
       # bug #4677: Do not instantiate effect lists
       result.n = replaceTypeVarsN(cl, result.n, ord(result.kind==tyProc))
-
       case result.kind
       of tyArray:
         let idx = result.sons[0]
@@ -500,18 +529,19 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
 
       else: discard
 
-proc initTypeVars*(p: PContext, pt: TIdTable, info: TLineInfo;
+proc initTypeVars*(p: PContext, typeMap: ptr LayeredIdTable, info: TLineInfo;
                    owner: PSym): TReplTypeVars =
   initIdTable(result.symMap)
-  copyIdTable(result.typeMap, pt)
   initIdTable(result.localCache)
+  result.typeMap = typeMap
   result.info = info
   result.c = p
   result.owner = owner
 
 proc replaceTypesInBody*(p: PContext, pt: TIdTable, n: PNode;
                          owner: PSym, allowMetaTypes = false): PNode =
-  var cl = initTypeVars(p, pt, n.info, owner)
+  var typeMap = initLayeredTypeMap(pt)
+  var cl = initTypeVars(p, addr(typeMap), n.info, owner)
   cl.allowMetaTypes = allowMetaTypes
   pushInfoContext(n.info)
   result = replaceTypeVarsN(cl, n)
@@ -519,7 +549,8 @@ proc replaceTypesInBody*(p: PContext, pt: TIdTable, n: PNode;
 
 proc replaceTypesForLambda*(p: PContext, pt: TIdTable, n: PNode;
                             original, new: PSym): PNode =
-  var cl = initTypeVars(p, pt, n.info, original)
+  var typeMap = initLayeredTypeMap(pt)
+  var cl = initTypeVars(p, addr(typeMap), n.info, original)
   idTablePut(cl.symMap, original, new)
   pushInfoContext(n.info)
   result = replaceTypeVarsN(cl, n)
@@ -527,14 +558,16 @@ proc replaceTypesForLambda*(p: PContext, pt: TIdTable, n: PNode;
 
 proc generateTypeInstance*(p: PContext, pt: TIdTable, info: TLineInfo,
                            t: PType): PType =
-  var cl = initTypeVars(p, pt, info, nil)
+  var typeMap = initLayeredTypeMap(pt)
+  var cl = initTypeVars(p, addr(typeMap), info, nil)
   pushInfoContext(info)
   result = replaceTypeVarsT(cl, t)
   popInfoContext()
 
 proc prepareMetatypeForSigmatch*(p: PContext, pt: TIdTable, info: TLineInfo,
                                  t: PType): PType =
-  var cl = initTypeVars(p, pt, info, nil)
+  var typeMap = initLayeredTypeMap(pt)
+  var cl = initTypeVars(p, addr(typeMap), info, nil)
   cl.allowMetaTypes = true
   pushInfoContext(info)
   result = replaceTypeVarsT(cl, t)

compiler/sigmatch.nim

@@ -69,6 +69,12 @@ type
     mutabilityProblem*: uint8 # tyVar mismatch
     inheritancePenalty: int  # to prefer closest father object type
 
+  TTypeRelFlag* = enum
+    trDontBind
+    trNoCovariance
+
+  TTypeRelFlags* = set[TTypeRelFlag]
+
   TTypeRelation* = enum      # order is important!
     isNone, isConvertible,
     isIntConv,
@@ -198,7 +204,9 @@ proc sumGeneric(t: PType): int =
         if t.sons[i] != nil:
           result += t.sons[i].sumGeneric
       break
-    of tyGenericParam, tyExpr, tyStatic, tyStmt: break
+    of tyStatic:
+      return t.sons[0].sumGeneric + 1
+    of tyGenericParam, tyExpr, tyStmt: break
     of tyAlias: t = t.lastSon
     of tyBool, tyChar, tyEnum, tyObject, tyPointer,
         tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
@@ -296,7 +304,33 @@ proc describeArgs*(c: PContext, n: PNode, startIdx = 1;
     add(result, argTypeToString(arg, prefer))
     if i != sonsLen(n) - 1: add(result, ", ")
 
-proc typeRel*(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation
+proc typeRelImpl*(c: var TCandidate, f, aOrig: PType,
+                  flags: TTypeRelFlags = {}): TTypeRelation
+
+const traceTypeRel = false
+
+when traceTypeRel:
+  var nextTypeRel = 0
+
+template typeRel*(c: var TCandidate, f, aOrig: PType,
+                 flags: TTypeRelFlags = {}): TTypeRelation =
+  when traceTypeRel:
+    var enteringAt = nextTypeRel
+    if mdbg:
+      inc nextTypeRel
+      echo "----- TYPE REL ", enteringAt
+      debug f
+      debug aOrig
+      # writeStackTrace()
+
+  let r = typeRelImpl(c, f, aOrig, flags)
+
+  when traceTypeRel:
+    if enteringAt != nextTypeRel:
+      echo "----- TYPE REL ", enteringAt, " RESULT: ", r
+
+  r
+
 proc concreteType(c: TCandidate, t: PType): PType =
   case t.kind
   of tyNil:
@@ -594,19 +628,29 @@ proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} =
   else:
     result = isNone
 
-proc matchUserTypeClass*(c: PContext, m: var TCandidate,
+proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
-                         ff, a: PType): PType =
   var
+    c = m.c
     typeClass = ff.skipTypes({tyUserTypeClassInst})
     body = typeClass.n[3]
-  if c.inTypeClass > 4:
+    matchedConceptContext: TMatchedConcept
+    prevMatchedConcept = c.matchedConcept
+    prevCandidateType = typeClass[0][0]
+
+  if prevMatchedConcept != nil:
+    matchedConceptContext.prev = prevMatchedConcept
+    matchedConceptContext.depth = prevMatchedConcept.depth + 1
+    if prevMatchedConcept.depth > 4:
       localError(body.info, $body & " too nested for type matching")
       return nil
 
   openScope(c)
-  inc c.inTypeClass
+  matchedConceptContext.candidateType = a
+  typeClass[0].sons[0] = a
+  c.matchedConcept = addr(matchedConceptContext)
   defer:
-    dec c.inTypeClass
+    c.matchedConcept = prevMatchedConcept
+    typeClass[0].sons[0] = prevCandidateType
     closeScope(c)
 
   var typeParams: seq[(PSym, PType)]
@@ -617,6 +661,9 @@ proc matchUserTypeClass*(c: PContext, m: var TCandidate,
         typeParamName = ff.base.sons[i-1].sym.name
         typ = ff.sons[i]
         param: PSym
+        alreadyBound = PType(idTableGet(m.bindings, typ))
+
+      if alreadyBound != nil: typ = alreadyBound
 
       template paramSym(kind): untyped =
         newSym(kind, typeParamName, typeClass.sym, typeClass.sym.info)
@@ -650,33 +697,6 @@ proc matchUserTypeClass*(c: PContext, m: var TCandidate,
 
       addDecl(c, param)
 
-  for param in typeClass.n[0]:
-    var
-      dummyName: PNode
-      dummyType: PType
-
-    let modifier = case param.kind
-      of nkVarTy: tyVar
-      of nkRefTy: tyRef
-      of nkPtrTy: tyPtr
-      of nkStaticTy: tyStatic
-      of nkTypeOfExpr: tyTypeDesc
-      else: tyNone
-
-    if modifier != tyNone:
-      dummyName = param[0]
-      dummyType = c.makeTypeWithModifier(modifier, a)
-      if modifier == tyTypeDesc: dummyType.flags.incl tfExplicit
-    else:
-      dummyName = param
-      dummyType = a
-
-    internalAssert dummyName.kind == nkIdent
-    var dummyParam = newSym(if modifier == tyTypeDesc: skType else: skVar,
-                            dummyName.ident, typeClass.sym, typeClass.sym.info)
-    dummyParam.typ = dummyType
-    addDecl(c, dummyParam)
-
   var
     oldWriteHook: type(writelnHook)
     diagnostics: seq[string]
@@ -818,7 +838,7 @@ proc inferStaticParam*(c: var TCandidate, lhs: PNode, rhs: BiggestInt): bool =
     var inferred = newTypeWithSons(c.c, tyStatic, lhs.typ.sons)
     inferred.n = newIntNode(nkIntLit, rhs)
     put(c, lhs.typ, inferred)
-    if c.c.inTypeClass > 0:
+    if c.c.matchedConcept != nil:
       # inside concepts, binding is currently done with
       # direct mutation of the involved types:
       lhs.typ.n = inferred.n
@@ -860,7 +880,28 @@ template subtypeCheck() =
   if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in {tyRef, tyPtr, tyVar}:
     result = isNone
 
-proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
+proc isCovariantPtr(c: var TCandidate, f, a: PType): bool =
+  # this proc is always called for a pair of matching types
+  assert f.kind == a.kind
+
+  template baseTypesCheck(lhs, rhs: PType): bool =
+    lhs.kind notin {tyPtr, tyRef, tyVar} and
+      typeRel(c, lhs, rhs, {trNoCovariance}) == isSubtype
+
+  case f.kind
+  of tyRef, tyPtr:
+    return baseTypesCheck(f.base, a.base)
+  of tyGenericInst:
+    let body = f.base
+    return body == a.base and
+           a.sonsLen == 3 and
+           tfWeakCovariant notin body.sons[0].flags and
+           baseTypesCheck(f.sons[1], a.sons[1])
+  else:
+    return false
+
+proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
+                 flags: TTypeRelFlags = {}): TTypeRelation =
   # typeRel can be used to establish various relationships between types:
   #
   # 1) When used with concrete types, it will check for type equivalence
@@ -887,7 +928,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   assert(aOrig != nil)
 
   var
-    useTypeLoweringRuleInTypeClass = c.c.inTypeClass > 0 and
+    useTypeLoweringRuleInTypeClass = c.c.matchedConcept != nil and
                                      not c.isNoCall and
                                      f.kind != tyTypeDesc and
                                      tfExplicit notin aOrig.flags
@@ -927,6 +968,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
         result = isEqual
       return
 
+  template doBind: bool = trDontBind notin flags
+
   # var and static arguments match regular modifier-free types
   var a = aOrig.skipTypes({tyStatic, tyVar}).maybeSkipDistinct(c.calleeSym)
   # XXX: Theoretically, maybeSkipDistinct could be called before we even
@@ -934,7 +977,10 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   # for example, but unfortunately `prepareOperand` is not called in certain
   # situation when nkDotExpr are rotated to nkDotCalls
 
-  if a.kind in {tyGenericInst, tyAlias} and
+  if aOrig.kind == tyAlias:
+    return typeRel(c, f, lastSon(aOrig))
+
+  if a.kind == tyGenericInst and
       skipTypes(f, {tyVar}).kind notin {
         tyGenericBody, tyGenericInvocation,
         tyGenericInst, tyGenericParam} + tyTypeClasses:
@@ -956,23 +1002,28 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
 
   case a.kind
   of tyOr:
+    # XXX: deal with the current dual meaning of tyGenericParam
+    c.typedescMatched = true
     # seq[int|string] vs seq[number]
     # both int and string must match against number
     # but ensure that '[T: A|A]' matches as good as '[T: A]' (bug #2219):
     result = isGeneric
     for branch in a.sons:
-      let x = typeRel(c, f, branch, false)
+      let x = typeRel(c, f, branch, flags + {trDontBind})
       if x == isNone: return isNone
       if x < result: result = x
+    return
 
   of tyAnd:
+    # XXX: deal with the current dual meaning of tyGenericParam
+    c.typedescMatched = true
     # seq[Sortable and Iterable] vs seq[Sortable]
     # only one match is enough
     for branch in a.sons:
-      let x = typeRel(c, f, branch, false)
+      let x = typeRel(c, f, branch, flags + {trDontBind})
       if x != isNone:
         return if x >= isGeneric: isGeneric else: x
-    result = isNone
+    return isNone
 
   of tyNot:
     case f.kind
@@ -994,9 +1045,10 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
            else: isNone
 
   of tyUserTypeClass, tyUserTypeClassInst:
+    if c.c.matchedConcept != nil:
       # consider this: 'var g: Node' *within* a concept where 'Node'
       # is a concept too (tgraph)
-    let x = typeRel(c, a, f, false)
+      let x = typeRel(c, a, f, flags + {trDontBind})
       if x >= isGeneric:
         return isGeneric
   else: discard
@@ -1042,7 +1094,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   of tyFloat128: result = handleFloatRange(f, a)
   of tyVar:
     if aOrig.kind == tyVar: result = typeRel(c, f.base, aOrig.base)
-    else: result = typeRel(c, f.base, aOrig)
+    else: result = typeRel(c, f.base, aOrig, flags + {trNoCovariance})
     subtypeCheck()
   of tyArray:
     case a.kind
@@ -1056,13 +1108,21 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
           fRange = a
         else:
           fRange = prev
-      result = typeRel(c, f.sons[1].skipTypes({tyTypeDesc}),
+      let ff = f.sons[1].skipTypes({tyTypeDesc})
-                          a.sons[1].skipTypes({tyTypeDesc}))
+      let aa = a.sons[1].skipTypes({tyTypeDesc})
-      if result < isGeneric: return isNone
+      result = typeRel(c, ff, aa)
+      if result < isGeneric:
+        if nimEnableCovariance and
+           trNoCovariance notin flags and
+           ff.kind == aa.kind and
+           isCovariantPtr(c, ff, aa):
+          result = isSubtype
+        else:
+          return isNone
 
       if fRange.rangeHasUnresolvedStatic:
         return inferStaticsInRange(c, fRange, a)
-      elif c.c.inTypeClass > 0 and aRange.rangeHasUnresolvedStatic:
+      elif c.c.matchedConcept != nil and aRange.rangeHasUnresolvedStatic:
         return inferStaticsInRange(c, aRange, f)
       else:
         if lengthOrd(fRange) != lengthOrd(aRange):
@@ -1077,20 +1137,31 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
       if tfOldSchoolExprStmt in f.sons[0].flags:
         if f.sons[0].kind == tyExpr: return
       elif f.sons[0].kind == tyStmt: return
+
+    template matchArrayOrSeq(aBase: PType) =
+      let ff = f.base
+      let aa = aBase
+      let baseRel = typeRel(c, ff, aa)
+      if baseRel >= isGeneric:
+        result = isConvertible
+      elif nimEnableCovariance and
+           trNoCovariance notin flags and
+           ff.kind == aa.kind and
+           isCovariantPtr(c, ff, aa):
+        result = isConvertible
+
     case a.kind
     of tyOpenArray, tyVarargs:
       result = typeRel(c, base(f), base(a))
       if result < isGeneric: result = isNone
     of tyArray:
       if (f.sons[0].kind != tyGenericParam) and (a.sons[1].kind == tyEmpty):
-        result = isSubtype
+        return isSubtype
-      elif typeRel(c, base(f), a.sons[1]) >= isGeneric:
+      matchArrayOrSeq(a.sons[1])
-        result = isConvertible
     of tySequence:
       if (f.sons[0].kind != tyGenericParam) and (a.sons[0].kind == tyEmpty):
-        result = isConvertible
+        return isConvertible
-      elif typeRel(c, base(f), a.sons[0]) >= isGeneric:
+      matchArrayOrSeq(a.sons[0])
-        result = isConvertible
     of tyString:
       if f.kind == tyOpenArray:
         if f.sons[0].kind == tyChar:
@@ -1105,8 +1176,17 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
       if (f.sons[0].kind != tyGenericParam) and (a.sons[0].kind == tyEmpty):
         result = isSubtype
       else:
-        result = typeRel(c, f.sons[0], a.sons[0])
+        let ff = f.sons[0]
-        if result < isGeneric: result = isNone
+        let aa = a.sons[0]
+        result = typeRel(c, ff, aa)
+        if result < isGeneric:
+          if nimEnableCovariance and
+             trNoCovariance notin flags and
+             ff.kind == aa.kind and
+             isCovariantPtr(c, ff, aa):
+            result = isSubtype
+          else:
+            result = isNone
         elif tfNotNil in f.flags and tfNotNil notin a.flags:
           result = isNilConversion
     of tyNil: result = f.allowsNil
@@ -1158,7 +1238,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
       if a.len < f.len: return isNone
       for i in 0..f.len-2:
         if typeRel(c, f.sons[i], a.sons[i]) == isNone: return isNone
-      result = typeRel(c, f.lastSon, a.lastSon)
+      result = typeRel(c, f.lastSon, a.lastSon, flags + {trNoCovariance})
       subtypeCheck()
       if result <= isConvertible: result = isNone
       elif tfNotNil in f.flags and tfNotNil notin a.flags:
@@ -1230,13 +1310,28 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
     var m = c
     if a.kind == tyGenericInst:
       if roota.base == rootf.base:
+        let nextFlags = flags + {trNoCovariance}
+        var hasCovariance = false
         for i in 1 .. rootf.sonsLen-2:
           let ff = rootf.sons[i]
           let aa = roota.sons[i]
-          result = typeRel(c, ff, aa)
+          result = typeRel(c, ff, aa, nextFlags)
-          if result notin {isEqual, isGeneric}: return isNone
+          if result notin {isEqual, isGeneric}:
-          # if ff.kind == tyRange and result != isEqual: return isNone
+            if trNoCovariance notin flags and ff.kind == aa.kind:
+              let paramFlags = rootf.base.sons[i-1].flags
+              hasCovariance =
+                if tfCovariant in paramFlags:
+                  if tfWeakCovariant in paramFlags:
+                    isCovariantPtr(c, ff, aa)
+                  else:
+                    ff.kind notin {tyRef, tyPtr} and result == isSubtype
+                else:
+                  tfContravariant in paramFlags and
+                    typeRel(c, aa, ff) == isSubtype
+              if hasCovariance:
+                continue
 
+            return isNone
         if prev == nil: put(c, f, a)
         result = isGeneric
       else:
@@ -1260,7 +1355,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
         result = isNone
     else:
       result = typeRel(c, lastSon(f), a)
-      if result != isNone: put(c, f, a)
+      if result != isNone and a.kind != tyNil:
+        put(c, f, a)
 
   of tyGenericBody:
     considerPreviousT:
@@ -1396,7 +1492,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
       result = typeRel(c, f.lastSon, a)
     else:
       considerPreviousT:
-        var matched = matchUserTypeClass(c.c, c, f, aOrig)
+        if aOrig == f: return isEqual
+        var matched = matchUserTypeClass(c, f, aOrig)
         if matched != nil:
           bindConcreteTypeToUserTypeClass(matched, a)
           if doBind: put(c, f, matched)
@@ -1450,7 +1547,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
           result = isNone
       else:
         if f.sonsLen > 0 and f.sons[0].kind != tyNone:
-          result = typeRel(c, f.lastSon, a, false)
+          result = typeRel(c, f.lastSon, a, flags + {trDontBind})
           if doBind and result notin {isNone, isGeneric}:
             let concrete = concreteType(c, a)
             if concrete == nil: return isNone
@@ -1488,6 +1585,12 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
           if not exprStructuralEquivalent(f.n, aOrig.n):
             result = isNone
         if result != isNone: put(c, f, aOrig)
+      elif aOrig.n != nil:
+        result = typeRel(c, f.lastSon, aOrig.n.typ)
+        if result != isNone:
+          var boundType = newTypeWithSons(c.c, tyStatic, @[aOrig.n.typ])
+          boundType.n = aOrig.n
+          put(c, f, boundType)
       else:
         result = isNone
     elif prev.kind == tyStatic:
@@ -1691,6 +1794,13 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
         arg.typ.sons = @[evaluated.typ]
         arg.typ.n = evaluated
         a = arg.typ
+      else:
+        if m.callee.kind == tyGenericBody:
+          if f.kind == tyStatic and typeRel(m, f.base, a) != isNone:
+            result = makeStaticExpr(m.c, arg)
+            result.typ.flags.incl tfUnresolved
+            result.typ.n = arg
+            return
 
   var r = typeRel(m, f, a)
 
@@ -2140,6 +2250,8 @@ proc matches*(c: PContext, n, nOrig: PNode, m: var TCandidate) =
         var def = copyTree(formal.ast)
         if def.kind == nkNilLit:
           def = implicitConv(nkHiddenStdConv, formal.typ, def, m, c)
+        if {tfImplicitTypeParam, tfGenericTypeParam} * formal.typ.flags != {}:
+          put(m, formal.typ, def.typ)
         setSon(m.call, formal.position + 1, def)
     inc(f)
   # forget all inferred types if the overload matching failed

compiler/trees.nim

@@ -41,6 +41,7 @@ proc exprStructuralEquivalent*(a, b: PNode; strictSymEquality=false): bool =
     of nkCharLit..nkUInt64Lit: result = a.intVal == b.intVal
     of nkFloatLit..nkFloat64Lit: result = a.floatVal == b.floatVal
     of nkStrLit..nkTripleStrLit: result = a.strVal == b.strVal
+    of nkCommentStmt: result = a.comment == b.comment
     of nkEmpty, nkNilLit, nkType: result = true
     else:
       if sonsLen(a) == sonsLen(b):
@@ -109,9 +110,11 @@ proc isDeepConstExpr*(n: PNode): bool =
 
 proc isRange*(n: PNode): bool {.inline.} =
   if n.kind in nkCallKinds:
-    if n[0].kind == nkIdent and n[0].ident.id == ord(wDotDot) or
+    let callee = n[0]
-        n[0].kind in {nkClosedSymChoice, nkOpenSymChoice} and
+    if (callee.kind == nkIdent and callee.ident.id == ord(wDotDot)) or
-        n[0][1].sym.name.id == ord(wDotDot):
+       (callee.kind == nkSym and callee.sym.name.id == ord(wDotDot)) or
+       (callee.kind in {nkClosedSymChoice, nkOpenSymChoice} and
+        callee[1].sym.name.id == ord(wDotDot)):
       result = true
 
 proc whichPragma*(n: PNode): TSpecialWord =

compiler/types.nim

@@ -139,6 +139,9 @@ proc getProcHeader*(sym: PSym; prefer: TPreferedDesc = preferName): string =
   add(result, ')')
   if n.sons[0].typ != nil:
     result.add(": " & typeToString(n.sons[0].typ, prefer))
+  result.add "[declared in "
+  result.add($sym.info)
+  result.add "]"
 
 proc elemType*(t: PType): PType =
   assert(t != nil)
@@ -421,6 +424,12 @@ template bindConcreteTypeToUserTypeClass*(tc, concrete: PType) =
   tc.sons.safeAdd concrete
   tc.flags.incl tfResolved
 
+# TODO: It would be a good idea to kill the special state of a resolved
+# concept by switching to tyAlias within the instantiated procs.
+# Currently, tyAlias is always skipped with lastSon, which means that
+# we can store information about the matched concept in another position.
+# Then builtInFieldAccess can be modified to properly read the derived
+# consts and types stored within the concept.
 template isResolvedUserTypeClass*(t: PType): bool =
   tfResolved in t.flags
 
@@ -437,6 +446,13 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
       result = t.sym.name.s & " literal(" & $t.n.intVal & ")"
     elif prefer == preferName or t.sym.owner.isNil:
       result = t.sym.name.s
+      if t.kind == tyGenericParam and t.sons != nil and t.sonsLen > 0:
+        result.add ": "
+        var first = true
+        for son in t.sons:
+          if not first: result.add " or "
+          result.add son.typeToString
+          first = false
     else:
       result = t.sym.owner.name.s & '.' & t.sym.name.s
     result.addTypeFlags(t)
@@ -458,7 +474,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     add(result, ']')
   of tyTypeDesc:
     if t.sons[0].kind == tyNone: result = "typedesc"
-    else: result = "typedesc[" & typeToString(t.sons[0]) & "]"
+    else: result = "type " & typeToString(t.sons[0])
   of tyStatic:
     internalAssert t.len > 0
     if prefer == preferGenericArg and t.n != nil:
@@ -688,6 +704,7 @@ type
     ExactTypeDescValues
     ExactGenericParams
     ExactConstraints
+    ExactGcSafety
     AllowCommonBase
 
   TTypeCmpFlags* = set[TTypeCmpFlag]
@@ -976,6 +993,8 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     cycleCheck()
     if a.kind == tyUserTypeClass and a.n != nil: return a.n == b.n
     result = sameChildrenAux(a, b, c) and sameFlags(a, b)
+    if result and ExactGcSafety in c.flags:
+      result = a.flags * {tfThread} == b.flags * {tfThread}
     if result and a.kind == tyProc:
       result = ((IgnoreCC in c.flags) or a.callConv == b.callConv) and
                ((ExactConstraints notin c.flags) or sameConstraints(a.n, b.n))

compiler/typesrenderer.nim

@@ -52,13 +52,16 @@ proc renderType(n: PNode): string =
     else:
       result = "ptr"
   of nkProcTy:
-    assert len(n) > 1
+    assert len(n) != 1
+    if len(n) > 1:
       let params = n[0]
       assert params.kind == nkFormalParams
       assert len(params) > 0
       result = "proc("
       for i in 1 .. <len(params): result.add(renderType(params[i]) & ',')
       result[<len(result)] = ')'
+    else:
+      result = "proc"
   of nkIdentDefs:
     assert len(n) >= 3
     let typePos = len(n) - 2

compiler/vm.nim

@@ -1406,6 +1406,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       if dest.kind in {nkStrLit..nkTripleStrLit} and
          regs[rb].kind in {rkNode}:
         dest.strVal = regs[rb].node.strVal
+      elif dest.kind == nkCommentStmt and regs[rb].kind in {rkNode}:
+        dest.comment = regs[rb].node.strVal
       else:
         stackTrace(c, tos, pc, errFieldXNotFound, "strVal")
     of opcNNewNimNode:
@@ -1609,8 +1611,9 @@ proc setupMacroParam(x: PNode, typ: PType): TFullReg =
 iterator genericParamsInMacroCall*(macroSym: PSym, call: PNode): (PSym, PNode) =
   let gp = macroSym.ast[genericParamsPos]
   for i in 0 .. <gp.len:
-    let idx = macroSym.typ.len + i
+    let genericParam = gp[i].sym
-    yield (gp[i].sym, call.sons[idx])
+    let posInCall = macroSym.typ.len + i
+    yield (genericParam, call[posInCall])
 
 var evalMacroCounter: int
 

compiler/vmgen.nim

@@ -826,7 +826,23 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mSubF64: genBinaryABC(c, n, dest, opcSubFloat)
   of mMulF64: genBinaryABC(c, n, dest, opcMulFloat)
   of mDivF64: genBinaryABC(c, n, dest, opcDivFloat)
-  of mShrI: genBinaryABCnarrowU(c, n, dest, opcShrInt)
+  of mShrI:
+    # the idea here is to narrow type if needed before executing right shift
+    # inlined modified: genNarrowU(c, n, dest)
+    let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
+    # uint is uint64 in the VM, we we only need to mask the result for
+    # other unsigned types:
+    let tmp = c.genx(n.sons[1])
+    if t.kind in {tyUInt8..tyUInt32, tyInt8..tyInt32}:
+      c.gABC(n, opcNarrowU, tmp, TRegister(t.size*8))
+
+    # inlined modified: genBinaryABC(c, n, dest, opcShrInt)
+    let tmp2 = c.genx(n.sons[2])
+    if dest < 0: dest = c.getTemp(n.typ)
+    c.gABC(n, opcShrInt, dest, tmp, tmp2)
+    c.freeTemp(tmp)
+    c.freeTemp(tmp2)
+
   of mShlI: genBinaryABCnarrowU(c, n, dest, opcShlInt)
   of mBitandI: genBinaryABCnarrowU(c, n, dest, opcBitandInt)
   of mBitorI: genBinaryABCnarrowU(c, n, dest, opcBitorInt)

doc/advopt.txt

@@ -16,7 +16,7 @@ Advanced commands:
   //check                   checks the project for syntax and semantic
 
 Advanced options:
-  -o, --out:FILE            set the output filename
+  -o:FILE, --out:FILE       set the output filename
   --stdout                  output to stdout
   --colors:on|off           turn compiler messages coloring on|off
   --listFullPaths           list full paths in messages

doc/astspec.txt

@@ -461,8 +461,8 @@ Documentation Comments
 ----------------------
 
 Double-hash (``##``) comments in the code actually have their own format,
-but the comments do not yet show up in the AST, which will only show that
+using ``strVal`` to get and set the comment text. Single-hash (``#``) 
-a comment exists, not what it contains. Single-hash (``#``) comments are ignored.
+comments are ignored.
 
 Concrete syntax:
 

doc/grammar.txt

@@ -22,7 +22,7 @@ plusExpr = mulExpr (OP8 optInd mulExpr)*
 mulExpr = dollarExpr (OP9 optInd dollarExpr)*
 dollarExpr = primary (OP10 optInd primary)*
 symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`'
-       | IDENT | 'addr' | 'type'
+       | IDENT | KEYW
 exprColonEqExpr = expr (':'|'=' expr)?
 exprList = expr ^+ comma
 dotExpr = expr '.' optInd symbol
@@ -81,7 +81,6 @@ paramList = '(' declColonEquals ^* (comma/semicolon) ')'
 paramListArrow = paramList? ('->' optInd typeDesc)?
 paramListColon = paramList? (':' optInd typeDesc)?
 doBlock = 'do' paramListArrow pragmas? colcom stmt
-doBlocks = doBlock ^* IND{=}
 procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)?
 distinct = 'distinct' optInd typeDesc
 expr = (ifExpr
@@ -98,7 +97,8 @@ primary = typeKeyw typeDescK
 typeDesc = simpleExpr
 typeDefAux = simpleExpr
            | 'concept' typeClass
-macroColon = ':' stmt? ( IND{=} 'of' exprList ':' stmt
+postExprBlocks = ':' stmt? ( IND{=} doBlock
+                           | IND{=} 'of' exprList ':' stmt
                            | IND{=} 'elif' expr ':' stmt
                            | IND{=} 'except' exprList ':' stmt
                            | IND{=} 'else' ':' stmt )*

doc/manual/pragmas.txt

@@ -1006,16 +1006,37 @@ CodegenDecl pragma
 ------------------
 
 The ``codegenDecl`` pragma can be used to directly influence Nim's code
-generator. It receives a format string that determines how the variable or
+generator. It receives a format string that determines how the variable 
-proc is declared in the generated code:
+or proc is declared in the generated code.
+
+For variables $1 in the format string represents the type of the variable
+and $2 is the name of the variable.
+
+The following Nim code:
 
 .. code-block:: nim
   var
     a {.codegenDecl: "$# progmem $#".}: int
     
+will generate this C code:
+
+.. code-block:: c
+  int progmem a
+
+For procedures $1 is the return type of the procedure, $2 is the name of
+the procedure and $3 is the parameter list.
+
+The following nim code:
+
+.. code-block:: nim
   proc myinterrupt() {.codegenDecl: "__interrupt $# $#$#".} =
     echo "realistic interrupt handler"
 
+will generate this code:
+
+.. code-block:: c
+  __interrupt void myinterrupt()
+
 
 InjectStmt pragma
 -----------------

doc/manual/type_rel.txt

@@ -111,6 +111,105 @@ relation is extended to the types ``var``, ``ref``, ``ptr``:
 .. XXX nil is a special value!
 
 
+Covariance
+----------
+
+Covariance in Nim can be introduced only though pointer-like types such
+as ``ptr`` and ``ref``. Sequence, Array and OpenArray types, instantiated
+with pointer-like types will be considered covariant if and only if they
+are also immutable. The introduction of a ``var`` modifier or additional
+``ptr`` or ``ref`` indirections would result in invariant treatment of
+these types.
+
+``proc`` types are currently always invariant, but future versions of Nim
+may relax this rule.
+
+User-defined generic types may also be covariant with respect to some of
+their parameters. By default, all generic params are considered invariant,
+but you may choose the apply the prefix modifier ``in`` to a parameter to
+make it contravariant or ``out`` to make it covariant:
+
+.. code-block:: nim
+  type
+    AnnotatedPtr[out T] =
+      metadata: MyTypeInfo
+      p: ref T
+
+    RingBuffer[out T] =
+      startPos: int
+      data: seq[T]
+
+    Action {.importcpp: "std::function<void ('0)>".} [in T] = object
+
+When the designated generic parameter is used to instantiate a pointer-like
+type as in the case of `AnnotatedPtr` above, the resulting generic type will
+also have pointer-like covariance:
+
+.. code-block:: nim
+  type
+    GuiWidget = object of RootObj
+    Button = object of GuiWidget
+    ComboBox = object of GuiWidget
+
+  var
+    widgetPtr: AnnotatedPtr[GuiWidget]
+    buttonPtr: AnnotatedPtr[Button]
+
+  ...
+
+  proc drawWidget[T](x: AnnotatedPtr[GuiWidget]) = ...
+
+  # you can call procs expecting base types by supplying a derived type
+  drawWidget(buttonPtr)
+
+  # and you can convert more-specific pointer types to more general ones
+  widgetPtr = buttonPtr
+
+Just like with regular pointers, covariance will be enabled only for immutable
+values:
+
+.. code-block:: nim
+  proc makeComboBox[T](x: var AnnotatedPtr[GuiWidget]) =
+    x.p = new(ComboBox)
+
+  makeComboBox(buttonPtr) # Error, AnnotatedPtr[Button] cannot be modified
+                          # to point to a ComboBox
+
+On the other hand, in the `RingBuffer` example above, the designated generic
+param is used to instantiate the non-pointer ``seq`` type, which means that
+the resulting generic type will have covariance that mimics an array or
+sequence (i.e. it will be covariant only when instantiated with ``ptr`` and
+``ref`` types):
+
+.. code-block:: nim
+
+  type
+    Base = object of RootObj
+    Derived = object of Base
+
+  proc consumeBaseValues(b: RingBuffer[Base]) = ...
+
+  var derivedValues: RingBuffer[Derived]
+
+  consumeBaseValues(derivedValues) # Error, Base and Derived values may differ
+                                   # in size
+
+  proc consumeBasePointers(b: RingBuffer[ptr Base]) = ...
+
+  var derivedPointers: RingBuffer[ptr Derived]
+
+  consumeBaseValues(derivedPointers) # This is legal
+
+Please note that Nim will treat the user-defined pointer-like types as
+proper alternatives to the built-in pointer types. That is, types such
+as `seq[AnnotatedPtr[T]]` or `RingBuffer[AnnotatedPtr[T]]` will also be
+considered covariant and you can create new pointer-like types by instantiating
+other user-defined pointer-like types.
+
+The contravariant parameters introduced with the ``in`` modifier are currently
+useful only when interfacing with imported types having such semantics.
+
+
 Convertible relation
 --------------------
 A type ``a`` is **implicitly** convertible to type ``b`` iff the following
@@ -119,6 +218,8 @@ algorithm returns true:
 .. code-block:: nim
   # XXX range types?
   proc isImplicitlyConvertible(a, b: PType): bool =
+    if isSubtype(a, b) or isCovariant(a, b):
+      return true
     case a.kind
     of int:     result = b in {int8, int16, int32, int64, uint, uint8, uint16,
                                uint32, uint64, float, float32, float64}

doc/nims.rst

@@ -108,3 +108,12 @@ installation of Nimble is done with this simple script:
 
   mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe
 
+You can also use the shebang ``#!/usr/bin/env nim``, as long as your filename
+ends with ``.nims``:
+
+.. code-block:: nim
+
+  #!/usr/bin/env nim
+  mode = ScriptMode.Silent
+
+  echo "hello world"

doc/tut1.rst

@@ -1492,20 +1492,6 @@ variables! For example:
   echo badname
   echo badext
 
-Tuple unpacking **only** works in ``var`` or ``let`` blocks. The following code
-won't compile:
-
-.. code-block:: nim
-
-  import os
-
-  var
-    path = "usr/local/nimc.html"
-    dir, name, ext = ""
-
-  (dir, name, ext) = splitFile(path)
-  # --> Error: '(dir, name, ext)' cannot be assigned to
-
 
 Reference and pointer types
 ---------------------------

doc/tut2.rst

@@ -206,7 +206,7 @@ for any type:
 
   echo "abc".len # is the same as echo len("abc")
   echo "abc".toUpper()
-  echo {'a', 'b', 'c'}.card
+  echo({'a', 'b', 'c'}.card)
   stdout.writeLine("Hallo") # the same as writeLine(stdout, "Hallo")
 
 (Another way to look at the method call syntax is that it provides the missing

koch.nim

@@ -152,6 +152,10 @@ proc tryExec(cmd: string): bool =
 proc safeRemove(filename: string) =
   if existsFile(filename): removeFile(filename)
 
+proc overwriteFile(source, dest: string) =
+  safeRemove(dest)
+  moveFile(source, dest)
+
 proc copyExe(source, dest: string) =
   safeRemove(dest)
   copyFile(dest=dest, source=source)
@@ -388,48 +392,25 @@ proc clean(args: string) =
 
 # -------------- builds a release ---------------------------------------------
 
-proc patchConfig(lookFor, replaceBy: string) =
-  const
-    cfgFile = "config/nim.cfg"
-  try:
-    let cfg = readFile(cfgFile)
-    let newCfg = cfg.replace(lookFor, replaceBy)
-    if newCfg == cfg:
-      echo "Could not patch 'config/nim.cfg' [Error]"
-      echo "Reason: patch substring not found:"
-      echo lookFor
-    else:
-      writeFile(cfgFile, newCfg)
-  except IOError:
-    quit "Could not access 'config/nim.cfg' [Error]"
-
 proc winReleaseArch(arch: string) =
   doAssert arch in ["32", "64"]
   let cpu = if arch == "32": "i386" else: "amd64"
 
   template withMingw(path, body) =
-    const orig = """#gcc.path = r"$nim\dist\mingw\bin""""
+    let prevPath = getEnv("PATH")
-    let replacePattern = """gcc.path = r"..\mingw$1\bin" # winrelease""" % arch
+    putEnv("PATH", path & PathSep & prevPath)
-    patchConfig(orig, replacePattern)
     try:
       body
     finally:
-      patchConfig(replacePattern, orig)
+      putEnv("PATH", prevPath)
 
   withMingw r"..\mingw" & arch & r"\bin":
     # Rebuilding koch is necessary because it uses its pointer size to
     # determine which mingw link to put in the NSIS installer.
     nimexec "c --out:koch_temp --cpu:$# koch" % cpu
     exec "koch_temp boot -d:release --cpu:$#" % cpu
-    exec "koch_temp nsis -d:release"
     exec "koch_temp zip -d:release"
-
+    overwriteFile r"build\nim-$#.zip" % VersionAsString,
-    when false:
-      # we now disable the NSIS installer as it cannot download from https
-      # and is broken in so many different ways it's not funny anymore:
-      moveFile r"build\nim_$#.exe" % VersionAsString,
-               r"web\upload\download\nim-$#_x$#.exe" % [VersionAsString, arch]
-    moveFile r"build\nim-$#.zip" % VersionAsString,
              r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch]
 
 proc winRelease() =
@@ -438,7 +419,7 @@ proc winRelease() =
     web(gaCode)
     withDir "web/upload/" & VersionAsString:
       exec "7z a -tzip docs-$#.zip *.html" % VersionAsString
-    moveFile "web/upload/$1/docs-$1.zip" % VersionAsString,
+    overwriteFile "web/upload/$1/docs-$1.zip" % VersionAsString,
                   "web/upload/download/docs-$1.zip" % VersionAsString
   when true:
     csource("-d:release")
@@ -489,6 +470,17 @@ proc temp(args: string) =
   copyExe(output, finalDest)
   if programArgs.len > 0: exec(finalDest & " " & programArgs)
 
+proc xtemp(cmd: string) =
+  let d = getAppDir()
+  copyExe(d / "bin" / "nim".exe, d / "bin" / "nim_backup".exe)
+  try:
+    withDir(d):
+      temp"-d:debug"
+    copyExe(d / "bin" / "nim_temp".exe, d / "bin" / "nim".exe)
+    exec(cmd)
+  finally:
+    copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe)
+
 proc pushCsources() =
   if not dirExists("../csources/.git"):
     quit "[Error] no csources git repository found"
@@ -564,6 +556,7 @@ of cmdArgument:
   of "testinstall": testUnixInstall()
   of "test", "tests": tests(op.cmdLineRest)
   of "temp": temp(op.cmdLineRest)
+  of "xtemp": xtemp(op.cmdLineRest)
   of "winrelease": winRelease()
   of "wintools": bundleWinTools()
   of "nimble": buildNimble(existsDir(".git"))

lib/core/macros.nim

@@ -225,7 +225,13 @@ proc `ident=`*(n: NimNode, val: NimIdent) {.magic: "NSetIdent", noSideEffect.}
 proc `strVal=`*(n: NimNode, val: string) {.magic: "NSetStrVal", noSideEffect.}
 
 proc newNimNode*(kind: NimNodeKind,
-                 n: NimNode=nil): NimNode {.magic: "NNewNimNode", noSideEffect.}
+                 lineInfoFrom: NimNode=nil): NimNode
+  {.magic: "NNewNimNode", noSideEffect.}
+  ## Creates a new AST node of the specified kind.
+  ##
+  ## The ``lineInfoFrom`` parameter is used for line information when the
+  ## produced code crashes. You should ensure that it is set to a node that
+  ## you are transforming.
 
 proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.}
 proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.}
@@ -249,6 +255,11 @@ proc newStrLitNode*(s: string): NimNode {.compileTime, noSideEffect.} =
   result = newNimNode(nnkStrLit)
   result.strVal = s
 
+proc newCommentStmtNode*(s: string): NimNode {.compileTime, noSideEffect.} =
+  ## creates a comment statement node
+  result = newNimNode(nnkCommentStmt)
+  result.strVal = s
+
 proc newIntLitNode*(i: BiggestInt): NimNode {.compileTime.} =
   ## creates a int literal node from `i`
   result = newNimNode(nnkIntLit)
@@ -269,6 +280,7 @@ proc newIdentNode*(i: string): NimNode {.compileTime.} =
   result = newNimNode(nnkIdent)
   result.ident = !i
 
+
 type
   BindSymRule* = enum    ## specifies how ``bindSym`` behaves
     brClosed,            ## only the symbols in current scope are bound
@@ -426,20 +438,105 @@ proc newLit*(c: char): NimNode {.compileTime.} =
   result = newNimNode(nnkCharLit)
   result.intVal = ord(c)
 
-proc newLit*(i: BiggestInt): NimNode {.compileTime.} =
+
+proc newLit*(i: int): NimNode {.compileTime.} =
   ## produces a new integer literal node.
   result = newNimNode(nnkIntLit)
   result.intVal = i
 
+proc newLit*(i: int8): NimNode {.compileTime.} =
+  ## produces a new integer literal node.
+  result = newNimNode(nnkInt8Lit)
+  result.intVal = i
+
+proc newLit*(i: int16): NimNode {.compileTime.} =
+  ## produces a new integer literal node.
+  result = newNimNode(nnkInt16Lit)
+  result.intVal = i
+
+proc newLit*(i: int32): NimNode {.compileTime.} =
+  ## produces a new integer literal node.
+  result = newNimNode(nnkInt32Lit)
+  result.intVal = i
+
+proc newLit*(i: int64): NimNode {.compileTime.} =
+  ## produces a new integer literal node.
+  result = newNimNode(nnkInt64Lit)
+  result.intVal = i
+
+proc newLit*(i: uint): NimNode {.compileTime.} =
+  ## produces a new unsigned integer literal node.
+  result = newNimNode(nnkUIntLit)
+  result.intVal = BiggestInt(i)
+
+proc newLit*(i: uint8): NimNode {.compileTime.} =
+  ## produces a new unsigned integer literal node.
+  result = newNimNode(nnkUInt8Lit)
+  result.intVal = BiggestInt(i)
+
+proc newLit*(i: uint16): NimNode {.compileTime.} =
+  ## produces a new unsigned integer literal node.
+  result = newNimNode(nnkUInt16Lit)
+  result.intVal = BiggestInt(i)
+
+proc newLit*(i: uint32): NimNode {.compileTime.} =
+  ## produces a new unsigned integer literal node.
+  result = newNimNode(nnkUInt32Lit)
+  result.intVal = BiggestInt(i)
+
+proc newLit*(i: uint64): NimNode {.compileTime.} =
+  ## produces a new unsigned integer literal node.
+  result = newNimNode(nnkUInt64Lit)
+  result.intVal = BiggestInt(i)
+
 proc newLit*(b: bool): NimNode {.compileTime.} =
   ## produces a new boolean literal node.
   result = if b: bindSym"true" else: bindSym"false"
 
-proc newLit*(f: BiggestFloat): NimNode {.compileTime.} =
+when false:
+  # the float type is not really a distinct type as described in https://github.com/nim-lang/Nim/issues/5875
+  proc newLit*(f: float): NimNode {.compileTime.} =
     ## produces a new float literal node.
     result = newNimNode(nnkFloatLit)
     result.floatVal = f
 
+proc newLit*(f: float32): NimNode {.compileTime.} =
+  ## produces a new float literal node.
+  result = newNimNode(nnkFloat32Lit)
+  result.floatVal = f
+
+proc newLit*(f: float64): NimNode {.compileTime.} =
+  ## produces a new float literal node.
+  result = newNimNode(nnkFloat64Lit)
+  result.floatVal = f
+
+when compiles(float128):
+  proc newLit*(f: float128): NimNode {.compileTime.} =
+    ## produces a new float literal node.
+    result = newNimNode(nnkFloat128Lit)
+    result.floatVal = f
+
+proc newLit*(arg: object): NimNode {.compileTime.} =
+  result = nnkObjConstr.newTree(arg.type.getTypeInst[1])
+  for a, b in arg.fieldPairs:
+    result.add nnkExprColonExpr.newTree( newIdentNode(a), newLit(b) )
+
+proc newLit*[N,T](arg: array[N,T]): NimNode {.compileTime.} =
+  result = nnkBracket.newTree
+  for x in arg:
+    result.add newLit(x)
+
+proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} =
+  result = nnkBracket.newTree
+  for x in arg:
+    result.add newLit(x)
+  result = nnkPrefix.newTree(bindSym"@", result)
+
+proc newLit*(arg: tuple): NimNode {.compileTime.} =
+  result = nnkPar.newTree
+  for a,b in arg.fieldPairs:
+    result.add nnkExprColonExpr.newTree( newIdentNode(a), newLit(b) )
+
 proc newLit*(s: string): NimNode {.compileTime.} =
   ## produces a new string literal node.
   result = newNimNode(nnkStrLit)
@@ -744,6 +841,8 @@ proc `$`*(node: NimNode): string {.compileTime.} =
     result = $node[0]
   of nnkAccQuoted:
     result = $node[0]
+  of nnkCommentStmt:
+    result = node.strVal
   else:
     badNodeKind node.kind, "$"
 

lib/posix/posix.nim

@@ -79,6 +79,9 @@ const
   DT_SOCK* = 12   ## UNIX domain socket.
   DT_WHT* = 14
 
+# Special types
+type Sighandler = proc (a: cint) {.noconv.}
+
 # Platform specific stuff
 
 when defined(linux) and defined(amd64):
@@ -86,6 +89,9 @@ when defined(linux) and defined(amd64):
 else:
   include posix_other
 
+# There used to be this name in posix.nim a long time ago, not sure why!
+{.deprecated: [cSIG_HOLD: SIG_HOLD].}
+
 when not defined(macosx):
   proc st_atime*(s: Stat): Time {.inline.} =
     ## Second-granularity time of last access
@@ -659,7 +665,7 @@ proc sighold*(a1: cint): cint {.importc, header: "<signal.h>".}
 proc sigignore*(a1: cint): cint {.importc, header: "<signal.h>".}
 proc siginterrupt*(a1, a2: cint): cint {.importc, header: "<signal.h>".}
 proc sigismember*(a1: var Sigset, a2: cint): cint {.importc, header: "<signal.h>".}
-proc signal*(a1: cint, a2: proc (x: cint) {.noconv.}) {.
+proc signal*(a1: cint, a2: Sighandler) {.
   importc, header: "<signal.h>".}
 proc sigpause*(a1: cint): cint {.importc, header: "<signal.h>".}
 proc sigpending*(a1: var Sigset): cint {.importc, header: "<signal.h>".}

lib/posix/posix_linux_amd64.nim

@@ -36,6 +36,9 @@ type
 
 {.deprecated: [TSocketHandle: SocketHandle].}
 
+# not detected by detect.nim, guarded by #ifdef __USE_UNIX98 in glibc
+const SIG_HOLD* = cast[SigHandler](2)
+
 type
   Timespec* {.importc: "struct timespec",
                header: "<time.h>", final, pure.} = object ## struct timespec

lib/posix/posix_linux_amd64_consts.nim

@@ -399,6 +399,9 @@ const SS_ONSTACK* = cint(1)
 const SS_DISABLE* = cint(2)
 const MINSIGSTKSZ* = cint(2048)
 const SIGSTKSZ* = cint(8192)
+const SIG_DFL* = cast[Sighandler](0)
+const SIG_ERR* = cast[Sighandler](-1)
+const SIG_IGN* = cast[Sighandler](1)
 
 # <sys/ipc.h>
 const IPC_CREAT* = cint(512)

lib/posix/posix_other_consts.nim

@@ -414,6 +414,10 @@ var SS_ONSTACK* {.importc: "SS_ONSTACK", header: "<signal.h>".}: cint
 var SS_DISABLE* {.importc: "SS_DISABLE", header: "<signal.h>".}: cint
 var MINSIGSTKSZ* {.importc: "MINSIGSTKSZ", header: "<signal.h>".}: cint
 var SIGSTKSZ* {.importc: "SIGSTKSZ", header: "<signal.h>".}: cint
+var SIG_HOLD* {.importc: "SIG_HOLD", header: "<signal.h>".}: Sighandler
+var SIG_DFL* {.importc: "SIG_DFL", header: "<signal.h>".}: Sighandler
+var SIG_ERR* {.importc: "SIG_ERR", header: "<signal.h>".}: Sighandler
+var SIG_IGN* {.importc: "SIG_IGN", header: "<signal.h>".}: Sighandler
 
 # <sys/ipc.h>
 var IPC_CREAT* {.importc: "IPC_CREAT", header: "<sys/ipc.h>".}: cint

lib/pure/asyncdispatch.nim

@@ -163,8 +163,8 @@ include includes/asyncfutures
 
 type
   PDispatcherBase = ref object of RootRef
-    timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
+    timers*: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
-    callbacks: Deque[proc ()]
+    callbacks*: Deque[proc ()]
 
 proc processTimers(p: PDispatcherBase) {.inline.} =
   #Process just part if timers at a step

lib/pure/asynchttpserver.nim

@@ -127,7 +127,7 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] =
   i.inc protocol.parseInt(result.minor, i)
 
 proc sendStatus(client: AsyncSocket, status: string): Future[void] =
-  client.send("HTTP/1.1 " & status & "\c\L")
+  client.send("HTTP/1.1 " & status & "\c\L\c\L") 
 
 proc processClient(client: AsyncSocket, address: string,
                    callback: proc (request: Request):

lib/pure/asyncmacro.nim

@@ -301,13 +301,13 @@ proc verifyReturnType(typeName: string) {.compileTime.} =
 proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
   ## This macro transforms a single procedure into a closure iterator.
   ## The ``async`` macro supports a stmtList holding multiple async procedures.
-  if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef}:
+  if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}:
       error("Cannot transform this node kind into an async proc." &
             " proc/method definition or lambda node expected.")
 
-  hint("Processing " & prc[0].getName & " as an async proc.")
+  let prcName = prc.name.getName
 
-  let returnType = prc[3][0]
+  let returnType = prc.params[0]
   var baseType: NimNode
   # Verify that the return type is a Future[T]
   if returnType.kind == nnkBracketExpr:
@@ -326,9 +326,9 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
   let subtypeIsVoid = returnType.kind == nnkEmpty or
         (baseType.kind == nnkIdent and returnType[1].ident == !"void")
 
-  let futureVarIdents = getFutureVarIdents(prc[3])
+  let futureVarIdents = getFutureVarIdents(prc.params)
 
-  var outerProcBody = newNimNode(nnkStmtList, prc[6])
+  var outerProcBody = newNimNode(nnkStmtList, prc.body)
 
   # -> var retFuture = newFuture[T]()
   var retFutureSym = genSym(nskVar, "retFuture")
@@ -338,10 +338,10 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
   outerProcBody.add(
     newVarStmt(retFutureSym,
       newCall(
-        newNimNode(nnkBracketExpr, prc[6]).add(
+        newNimNode(nnkBracketExpr, prc.body).add(
           newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`.
           subRetType),
-      newLit(prc[0].getName)))) # Get type from return type of this proc
+      newLit(prcName)))) # Get type from return type of this proc
 
   # -> iterator nameIter(): FutureBase {.closure.} =
   # ->   {.push warning[resultshadowed]: off.}
@@ -349,8 +349,8 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
   # ->   {.pop.}
   # ->   <proc_body>
   # ->   complete(retFuture, result)
-  var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter")
+  var iteratorNameSym = genSym(nskIterator, $prcName & "Iter")
-  var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid,
+  var procBody = prc.body.processBody(retFutureSym, subtypeIsVoid,
                                     futureVarIdents, nil)
   # don't do anything with forward bodies (empty)
   if procBody.kind != nnkEmpty:
@@ -362,7 +362,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
           newIdentNode("warning"), newIdentNode("resultshadowed")),
         newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.}
 
-      procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add(
+      procBody.insert(1, newNimNode(nnkVarSection, prc.body).add(
         newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T
 
       procBody.insert(2, newNimNode(nnkPragma).add(
@@ -377,35 +377,32 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
 
     var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")],
                                   procBody, nnkIteratorDef)
-    closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure"))
+    closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom=prc.body)
+    closureIterator.addPragma(newIdentNode("closure"))
+    closureIterator.addPragma(newIdentNode("gcsafe"))
     outerProcBody.add(closureIterator)
 
     # -> createCb(retFuture)
     #var cbName = newIdentNode("cb")
     var procCb = getAst createCb(retFutureSym, iteratorNameSym,
-                         newStrLitNode(prc[0].getName),
+                         newStrLitNode(prcName),
                          createFutureVarCompletions(futureVarIdents, nil))
     outerProcBody.add procCb
 
     # -> return retFuture
-    outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym)
+    outerProcBody.add newNimNode(nnkReturnStmt, prc.body[^1]).add(retFutureSym)
 
   result = prc
 
-  # Remove the 'async' pragma.
-  for i in 0 .. <result[4].len:
-    if result[4][i].kind == nnkIdent and result[4][i].ident == !"async":
-      result[4].del(i)
-  result[4] = newEmptyNode()
   if subtypeIsVoid:
     # Add discardable pragma.
     if returnType.kind == nnkEmpty:
       # Add Future[void]
-      result[3][0] = parseExpr("Future[void]")
+      result.params[0] = parseExpr("Future[void]")
   if procBody.kind != nnkEmpty:
-    result[6] = outerProcBody
+    result.body = outerProcBody
   #echo(treeRepr(result))
-  #if prc[0].getName == "recvLineInto":
+  #if prcName == "recvLineInto":
   #  echo(toStrLit(result))
 
 macro async*(prc: untyped): untyped =
@@ -529,8 +526,6 @@ macro multisync*(prc: untyped): untyped =
   ##
   ## The generated async procedures use the ``async`` macro, whereas the
   ## generated synchronous procedures simply strip off the ``await`` calls.
-  hint("Processing " & prc[0].getName & " as a multisync proc.")
-
   let (sync, asyncPrc) = splitProc(prc)
   result = newStmtList()
   result.add(asyncSingleProc(asyncPrc))

lib/pure/asyncnet.nim

@@ -533,15 +533,13 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string],
   else:
     var c = ""
     while true:
-      let recvFut = recv(socket, 1, flags)
+      c = await recv(socket, 1, flags)
-      c = recvFut.read()
       if c.len == 0:
         resString.mget.setLen(0)
         resString.complete()
         return
       if c == "\r":
-        let recvFut = recv(socket, 1, flags) # Skip \L
+        c = await recv(socket, 1, flags) # Skip \L
-        c = recvFut.read()
         assert c == "\L"
         addNLIfEmpty()
         resString.complete()

lib/pure/collections/tables.nim

@@ -590,8 +590,11 @@ proc enlarge[A, B](t: var OrderedTable[A, B]) =
   swap(t.data, n)
   while h >= 0:
     var nxt = n[h].next
-    if isFilled(n[h].hcode):
+    let eh = n[h].hcode
-      var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode)
+    if isFilled(eh):
+      var j: Hash = eh and maxHash(t)
+      while isFilled(t.data[j].hcode):
+        j = nextTry(j, maxHash(t))
       rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j)
     h = nxt
 

lib/pure/concurrency/cpuinfo.nim

@@ -69,4 +69,4 @@ proc countProcessors*(): int {.rtl, extern: "ncpi$1".} =
     result = affinitySpaceTotal().int
   else:
     result = sysconf(SC_NPROCESSORS_ONLN)
-  if result <= 0: result = 1
+  if result <= 0: result = 0

lib/pure/distros.nim

@@ -158,7 +158,7 @@ proc detectOsImpl(d: Distribution): bool =
   of Distribution.ArchLinux:
     result = "arch" in toLowerAscii(uname())
   of Distribution.OpenSUSE:
-    result = "suse" in toLowerAscii(uname())
+    result = "suse" in toLowerAscii(uname()) or "suse" in toLowerAscii(release())
   of Distribution.GoboLinux:
     result = "-Gobo " in uname()
   of Distribution.OpenMandriva:

lib/pure/hashes.nim

@@ -139,6 +139,11 @@ proc hash*(x: cstring): Hash =
   ## efficient hashing of null-terminated strings
   var h: Hash = 0
   var i = 0
+  when defined(js):
+    while i < x.len:
+      h = h !& ord(x[i])
+      inc i
+  else:
     while x[i] != 0.char:
       h = h !& ord(x[i])
       inc i

lib/pure/httpclient.nim

@@ -62,7 +62,8 @@
 ##   let body = %*{
 ##       "data": "some text"
 ##   }
-##   echo client.request("http://some.api", httpMethod = HttpPost, body = $body)
+##   let response = client.request("http://some.api", httpMethod = HttpPost, body = $body)
+##   echo response.status
 ##
 ## Progress reporting
 ## ==================

lib/pure/includes/asynccommon.nim

@@ -3,7 +3,7 @@ template newAsyncNativeSocketImpl(domain, sockType, protocol) =
   if handle == osInvalidSocket:
     raiseOSError(osLastError())
   handle.setBlocking(false)
-  when defined(macosx):
+  when defined(macosx) and not defined(nimdoc):
     handle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1)
   result = handle.AsyncFD
   register(result)

lib/pure/json.nim

@@ -1001,7 +1001,7 @@ proc escapeJson*(s: string; result: var string) =
   result.add("\"")
   for x in runes(s):
     var r = int(x)
-    if r >= 32 and r <= 127:
+    if r >= 32 and r <= 126:
       var c = chr(r)
       case c
       of '"': result.add("\\\"")
@@ -1608,7 +1608,7 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode =
       result = processType(newIdentNode(typeName), obj, jsonNode, true)
     of "seq":
       let seqT = typeSym[1]
-      let forLoopI = newIdentNode("i")
+      let forLoopI = genSym(nskForVar, "i")
       let indexerNode = createJsonIndexer(jsonNode, forLoopI)
       let constructorNode = createConstructor(seqT, indexerNode)
 
@@ -1616,12 +1616,25 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode =
       result = quote do:
         (
           var list: `typeSym` = @[];
-          # if `jsonNode`.kind != JArray:
+          verifyJsonKind(`jsonNode`, {JArray}, astToStr(`jsonNode`));
-          #   # TODO: Improve error message.
-          #   raise newException(ValueError, "Expected a list")
           for `forLoopI` in 0 .. <`jsonNode`.len: list.add(`constructorNode`);
           list
         )
+    of "array":
+      let arrayT = typeSym[2]
+      let forLoopI = genSym(nskForVar, "i")
+      let indexerNode = createJsonIndexer(jsonNode, forLoopI)
+      let constructorNode = createConstructor(arrayT, indexerNode)
+
+      # Create a statement expression containing a for loop.
+      result = quote do:
+        (
+          var list: `typeSym`;
+          verifyJsonKind(`jsonNode`, {JArray}, astToStr(`jsonNode`));
+          for `forLoopI` in 0 .. <`jsonNode`.len: list[`forLoopI`] =`constructorNode`;
+          list
+        )
+
     else:
       # Generic type.
       let obj = getType(typeSym)

lib/pure/math.nim

@@ -395,6 +395,12 @@ proc radToDeg*[T: float32|float64](d: T): T {.inline.} =
   ## Convert from radians to degrees
   result = T(d) / RadPerDeg
 
+proc sgn*[T: SomeNumber](x: T): int {.inline.} =
+  ## Sign function. Returns -1 for negative numbers and `NegInf`, 1 for
+  ## positive numbers and `Inf`, and 0 for positive zero, negative zero and
+  ## `NaN`.
+  ord(T(0) < x) - ord(x < T(0))
+
 proc `mod`*[T: float32|float64](x, y: T): T =
   ## Computes the modulo operation for float operators. Equivalent
   ## to ``x - y * floor(x/y)``. Note that the remainder will always
@@ -407,10 +413,13 @@ proc `mod`*[T: float32|float64](x, y: T): T =
 {.pop.}
 {.pop.}
 
-proc `^`*[T](x, y: T): T =
+proc `^`*[T](x: T, y: Natural): T =
   ## Computes ``x`` to the power ``y`. ``x`` must be non-negative, use
   ## `pow <#pow,float,float>` for negative exponents.
+  when compiles(y >= T(0)):
     assert y >= T(0)
+  else:
+    assert T(y) >= T(0)
   var (x, y) = (x, y)
   result = 1
 
@@ -447,6 +456,7 @@ when isMainModule and not defined(JS):
   assert(lgamma(1.0) == 0.0) # ln(1.0) == 0.0
   assert(erf(6.0) > erf(5.0))
   assert(erfc(6.0) < erfc(5.0))
+
 when isMainModule:
   # Function for approximate comparison of floats
   proc `==~`(x, y: float): bool = (abs(x-y) < 1e-9)
@@ -509,3 +519,21 @@ when isMainModule:
     doAssert(classify(trunc(-1e1000000'f32)) == fcNegInf)
     doAssert(classify(trunc(f_nan.float32)) == fcNan)
     doAssert(classify(trunc(0.0'f32)) == fcZero)
+
+  block: # sgn() tests
+    assert sgn(1'i8) == 1
+    assert sgn(1'i16) == 1
+    assert sgn(1'i32) == 1
+    assert sgn(1'i64) == 1
+    assert sgn(1'u8) == 1
+    assert sgn(1'u16) == 1
+    assert sgn(1'u32) == 1
+    assert sgn(1'u64) == 1
+    assert sgn(-12342.8844'f32) == -1
+    assert sgn(123.9834'f64) == 1
+    assert sgn(0'i32) == 0
+    assert sgn(0'f32) == 0
+    assert sgn(NegInf) == -1
+    assert sgn(Inf) == 1
+    assert sgn(NaN) == 0
+

lib/pure/os.nim

@@ -666,20 +666,31 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect].}
     else:
       raiseOSError(osLastError(), $strerror(errno))
 
-proc moveFile*(source, dest: string) {.rtl, extern: "nos$1",
+proc tryMoveFSObject(source, dest: string): bool =
-  tags: [ReadIOEffect, WriteIOEffect].} =
+  ## Moves a file or directory from `source` to `dest`. Returns false in case
-  ## Moves a file from `source` to `dest`. If this fails, `OSError` is raised.
+  ## of `EXDEV` error. In case of other errors `OSError` is raised. Returns
+  ## true in case of success.
   when defined(Windows):
     when useWinUnicode:
       let s = newWideCString(source)
       let d = newWideCString(dest)
-      if moveFileW(s, d) == 0'i32: raiseOSError(osLastError())
+      if moveFileExW(s, d, MOVEFILE_COPY_ALLOWED) == 0'i32: raiseOSError(osLastError())
     else:
-      if moveFileA(source, dest) == 0'i32: raiseOSError(osLastError())
+      if moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED) == 0'i32: raiseOSError(osLastError())
   else:
     if c_rename(source, dest) != 0'i32:
       let err = osLastError()
       if err == EXDEV.OSErrorCode:
+        return false
+      else:
+        raiseOSError(err, $strerror(errno))
+  return true
+
+proc moveFile*(source, dest: string) {.rtl, extern: "nos$1",
+  tags: [ReadIOEffect, WriteIOEffect].} =
+  ## Moves a file from `source` to `dest`. If this fails, `OSError` is raised.
+  if not tryMoveFSObject(source, dest):
+    when not defined(windows):
       # Fallback to copy & del
       copyFile(source, dest)
       try:
@@ -687,8 +698,6 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1",
       except:
         discard tryRemoveFile(dest)
         raise
-      else:
-        raiseOSError(err, $strerror(errno))
 
 proc execShellCmd*(command: string): int {.rtl, extern: "nos$1",
   tags: [ExecIOEffect].} =
@@ -1369,6 +1378,14 @@ proc exclFilePermissions*(filename: string,
   ##   setFilePermissions(filename, getFilePermissions(filename)-permissions)
   setFilePermissions(filename, getFilePermissions(filename)-permissions)
 
+proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect].} =
+  ## Moves a directory from `source` to `dest`. If this fails, `OSError` is raised.
+  if not tryMoveFSObject(source, dest):
+    when not defined(windows):
+      # Fallback to copy & del
+      copyDir(source, dest)
+      removeDir(source)
+
 include ospaths
 
 proc expandSymlink*(symlinkPath: string): string =
@@ -1412,7 +1429,7 @@ when defined(nimdoc):
   proc paramStr*(i: int): TaintedString {.tags: [ReadIOEffect].} =
     ## Returns the `i`-th `command line argument`:idx: given to the application.
     ##
-    ## `i` should be in the range `1..paramCount()`, the `EInvalidIndex`
+    ## `i` should be in the range `1..paramCount()`, the `IndexError`
     ## exception will be raised for invalid values.  Instead of iterating over
     ## `paramCount() <#paramCount>`_ with this proc you can call the
     ## convenience `commandLineParams() <#commandLineParams>`_.
@@ -1450,7 +1467,8 @@ elif defined(windows):
     tags: [ReadIOEffect].} =
     # Docstring in nimdoc block.
     if isNil(ownArgv): ownArgv = parseCmdLine($getCommandLine())
-    return TaintedString(ownArgv[i])
+    if i < ownArgv.len and i >= 0: return TaintedString(ownArgv[i])
+    raise newException(IndexError, "invalid index")
 
 elif not defined(createNimRtl) and
   not(defined(posix) and appType == "lib") and

lib/pure/ospaths.nim

@@ -516,7 +516,16 @@ when declared(getEnv) or defined(nimscript):
   proc getConfigDir*(): string {.rtl, extern: "nos$1",
     tags: [ReadEnvEffect, ReadIOEffect].} =
     ## Returns the config directory of the current user for applications.
+    ##
+    ## On non-Windows OSs, this proc conforms to the XDG Base Directory
+    ## spec. Thus, this proc returns the value of the XDG_CONFIG_DIR environment
+    ## variable if it is set, and returns the default configuration directory,
+    ## "~/.config/", otherwise.
+    ##
+    ## An OS-dependent trailing slash is always present at the end of the
+    ## returned string; `\\` on Windows and `/` on all other OSs.
     when defined(windows): return string(getEnv("APPDATA")) & "\\"
+    elif getEnv("XDG_CONFIG_DIR"): return string(getEnv("XDG_CONFIG_DIR")) & "/"
     else: return string(getEnv("HOME")) & "/.config/"
 
   proc getTempDir*(): string {.rtl, extern: "nos$1",

lib/pure/osproc.nim

@@ -863,18 +863,15 @@ elif not defined(useNimRtl):
       if data.workingDir.len > 0:
         setCurrentDir($data.workingDir)
       var pid: Pid
-      var err: OSErrorCode
 
       if data.optionPoUsePath:
         res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
-        if res != 0'i32: err = osLastError()
       else:
         res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
-        if res != 0'i32: err = osLastError()
 
       discard posix_spawn_file_actions_destroy(fops)
       discard posix_spawnattr_destroy(attr)
-      if res != 0'i32: raiseOSError(err)
+      if res != 0'i32: raiseOSError(OSErrorCode(res))
 
       return pid
   else:

lib/pure/parsecfg.nim

@@ -540,8 +540,8 @@ proc writeConfig*(dict: Config, filename: string) =
   ## Writes the contents of the table to the specified configuration file.
   ## Note: Comment statement will be ignored.
   let file = open(filename, fmWrite)
-  let fileStream = newFileStream(filename)
+  defer: file.close()
-  defer: fileStream.close()
+  let fileStream = newFileStream(file)
   dict.writeConfig(fileStream)
 
 proc getSectionValue*(dict: Config, section, key: string): string =

lib/pure/parseopt.nim

@@ -149,15 +149,13 @@ proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo$1".} =
   ## retrieves the rest of the command line that has not been parsed yet.
   result = strip(substr(p.cmd, p.pos, len(p.cmd) - 1)).TaintedString
 
-when declared(initOptParser):
+iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, val: TaintedString] =
-  iterator getopt*(): tuple[kind: CmdLineKind, key, val: TaintedString] =
+  ## This is an convenience iterator for iterating over the given OptParser object.
-    ## This is an convenience iterator for iterating over the command line.
+  ## Example:
-    ## This uses the OptParser object. Example:
   ##
   ## .. code-block:: nim
-    ##   var
+  ##   var p = initOptParser("--left --debug:3 -l=4 -r:2")
-    ##     filename = ""
+  ##   for kind, key, val in p.getopt():
-    ##   for kind, key, val in getopt():
   ##     case kind
   ##     of cmdArgument:
   ##       filename = key
@@ -169,6 +167,23 @@ when declared(initOptParser):
   ##   if filename == "":
   ##     # no filename has been given, so we show the help:
   ##     writeHelp()
+  p.pos = 0
+  while true:
+    next(p)
+    if p.kind == cmdEnd: break
+    yield (p.kind, p.key, p.val)
+
+when declared(initOptParser):
+  iterator getopt*(): tuple[kind: CmdLineKind, key, val: TaintedString] =
+    ## This is an convenience iterator for iterating over the command line arguments.
+    ## This create a new OptParser object.
+    ## See above for a more detailed example
+    ##
+    ## .. code-block:: nim
+    ##   for kind, key, val in getopt():
+    ##     # this will iterate over all arguments passed to the cmdline.
+    ##     continue
+    ##
     var p = initOptParser()
     while true:
       next(p)

lib/pure/parseopt2.nim

@@ -123,15 +123,13 @@ type
 
 {.deprecated: [TGetoptResult: GetoptResult].}
 
-when declared(paramCount):
+iterator getopt*(p: var OptParser): GetoptResult =
-  iterator getopt*(): GetoptResult =
+  ## This is an convenience iterator for iterating over the given OptParser object.
-    ## This is an convenience iterator for iterating over the command line.
+  ## Example:
-    ## This uses the OptParser object. Example:
   ##
   ## .. code-block:: nim
-    ##   var
+  ##   var p = initOptParser("--left --debug:3 -l=4 -r:2")
-    ##     filename = ""
+  ##   for kind, key, val in p.getopt():
-    ##   for kind, key, val in getopt():
   ##     case kind
   ##     of cmdArgument:
   ##       filename = key
@@ -143,6 +141,23 @@ when declared(paramCount):
   ##   if filename == "":
   ##     # no filename has been given, so we show the help:
   ##     writeHelp()
+  p.pos = 0
+  while true:
+    next(p)
+    if p.kind == cmdEnd: break
+    yield (p.kind, p.key, p.val)
+
+when declared(paramCount):
+  iterator getopt*(): GetoptResult =
+    ## This is an convenience iterator for iterating over the command line arguments.
+    ## This create a new OptParser object.
+    ## See above for a more detailed example
+    ##
+    ## .. code-block:: nim
+    ##   for kind, key, val in getopt():
+    ##     # this will iterate over all arguments passed to the cmdline.
+    ##     continue
+    ##
     var p = initOptParser()
     while true:
       next(p)

lib/pure/pegs.nim

@@ -371,7 +371,7 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string =
   of '\a': result = "\\a"
   of '\\': result = "\\\\"
   of 'a'..'z', 'A'..'Z', '0'..'9', '_': result = $c
-  elif c < ' ' or c >= '\128': result = '\\' & $ord(c)
+  elif c < ' ' or c >= '\127': result = '\\' & $ord(c)
   elif c in reserved: result = '\\' & c
   else: result = $c
 

lib/pure/random.nim

@@ -123,7 +123,8 @@ when not defined(nimscript):
       proc getMil(t: Time): int {.importcpp: "getTime", nodecl.}
       randomize(getMil times.getTime())
     else:
-      randomize(int times.getTime())
+      let time = int(times.epochTime() * 1_000_000_000)
+      randomize(time)
 
 {.pop.}
 

lib/pure/streams.nim

@@ -334,12 +334,16 @@ when not defined(js):
     if result > 0:
       copyMem(buffer, addr(s.data[s.pos]), result)
       inc(s.pos, result)
+    else:
+      result = 0
 
   proc ssPeekData(s: Stream, buffer: pointer, bufLen: int): int =
     var s = StringStream(s)
     result = min(bufLen, s.data.len - s.pos)
     if result > 0:
       copyMem(buffer, addr(s.data[s.pos]), result)
+    else:
+      result = 0
 
   proc ssWriteData(s: Stream, buffer: pointer, bufLen: int) =
     var s = StringStream(s)

lib/pure/strutils.nim

@@ -1643,7 +1643,7 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
   ## * replaces any ``\`` by ``\\``
   ## * replaces any ``'`` by ``\'``
   ## * replaces any ``"`` by ``\"``
-  ## * replaces any other character in the set ``{'\0'..'\31', '\128'..'\255'}``
+  ## * replaces any other character in the set ``{'\0'..'\31', '\127'..'\255'}``
   ##   by ``\xHH`` where ``HH`` is its hexadecimal value.
   ## The procedure has been designed so that its output is usable for many
   ## different common syntaxes. The resulting string is prefixed with
@@ -1653,7 +1653,7 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
   result.add(prefix)
   for c in items(s):
     case c
-    of '\0'..'\31', '\128'..'\255':
+    of '\0'..'\31', '\127'..'\255':
       add(result, "\\x")
       add(result, toHex(ord(c), 2))
     of '\\': add(result, "\\\\")

lib/pure/times.nim

@@ -81,7 +81,7 @@ when defined(posix) and not defined(JS):
 elif defined(windows):
   import winlean
 
-  when defined(vcc) or defined(bcc):
+  when defined(vcc) or defined(bcc) or defined(icl):
     # newest version of Visual C++ defines time_t to be of 64 bits
     type TimeImpl {.importc: "time_t", header: "<time.h>".} = int64
     # visual c's c runtime exposes these under a different name
@@ -98,46 +98,47 @@ elif defined(windows):
 
 elif defined(JS):
   type
-    Time* = ref TimeObj
+    TimeBase = float
-    TimeObj {.importc.} = object
+    Time* = distinct TimeBase
-      getDay: proc (): int {.tags: [], raises: [], benign.}
+
-      getFullYear: proc (): int {.tags: [], raises: [], benign.}
+  proc getDay(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getHours: proc (): int {.tags: [], raises: [], benign.}
+  proc getFullYear(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getMilliseconds: proc (): int {.tags: [], raises: [], benign.}
+  proc getHours(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getMinutes: proc (): int {.tags: [], raises: [], benign.}
+  proc getMilliseconds(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getMonth: proc (): int {.tags: [], raises: [], benign.}
+  proc getMinutes(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getSeconds: proc (): int {.tags: [], raises: [], benign.}
+  proc getMonth(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getTime: proc (): int {.tags: [], raises: [], noSideEffect, benign.}
+  proc getSeconds(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getTimezoneOffset: proc (): int {.tags: [], raises: [], benign.}
+  proc getTime(t: Time): int {.tags: [], raises: [], noSideEffect, benign, importcpp.}
-      getDate: proc (): int {.tags: [], raises: [], benign.}
+  proc getTimezoneOffset(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getUTCDate: proc (): int {.tags: [], raises: [], benign.}
+  proc getDate(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getUTCFullYear: proc (): int {.tags: [], raises: [], benign.}
+  proc getUTCDate(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getUTCHours: proc (): int {.tags: [], raises: [], benign.}
+  proc getUTCFullYear(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getUTCMilliseconds: proc (): int {.tags: [], raises: [], benign.}
+  proc getUTCHours(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getUTCMinutes: proc (): int {.tags: [], raises: [], benign.}
+  proc getUTCMilliseconds(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getUTCMonth: proc (): int {.tags: [], raises: [], benign.}
+  proc getUTCMinutes(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getUTCSeconds: proc (): int {.tags: [], raises: [], benign.}
+  proc getUTCMonth(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getUTCDay: proc (): int {.tags: [], raises: [], benign.}
+  proc getUTCSeconds(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      getYear: proc (): int {.tags: [], raises: [], benign.}
+  proc getUTCDay(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      parse: proc (s: cstring): Time {.tags: [], raises: [], benign.}
+  proc getYear(t: Time): int {.tags: [], raises: [], benign, importcpp.}
-      setDate: proc (x: int) {.tags: [], raises: [], benign.}
+  proc parse(t: Time; s: cstring): Time {.tags: [], raises: [], benign, importcpp.}
-      setFullYear: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setDate(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setHours: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setFullYear(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setMilliseconds: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setHours(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setMinutes: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setMilliseconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setMonth: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setMinutes(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setSeconds: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setMonth(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setTime: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setSeconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setUTCDate: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setTime(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setUTCFullYear: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setUTCDate(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setUTCHours: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setUTCFullYear(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setUTCMilliseconds: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setUTCHours(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setUTCMinutes: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setUTCMilliseconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setUTCMonth: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setUTCMinutes(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setUTCSeconds: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setUTCMonth(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      setYear: proc (x: int) {.tags: [], raises: [], benign.}
+  proc setUTCSeconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      toGMTString: proc (): cstring {.tags: [], raises: [], benign.}
+  proc setYear(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.}
-      toLocaleString: proc (): cstring {.tags: [], raises: [], benign.}
+  proc toGMTString(t: Time): cstring {.tags: [], raises: [], benign, importcpp.}
+  proc toLocaleString(t: Time): cstring {.tags: [], raises: [], benign, importcpp.}
 
 type
   TimeInfo* = object of RootObj ## represents a time in different parts
@@ -225,16 +226,25 @@ proc `-`*(a, b: Time): int64 {.
 proc `<`*(a, b: Time): bool {.
   rtl, extern: "ntLtTime", tags: [], raises: [], noSideEffect.} =
   ## returns true iff ``a < b``, that is iff a happened before b.
+  when defined(js):
+    result = TimeBase(a) < TimeBase(b)
+  else:
     result = a - b < 0
 
 proc `<=` * (a, b: Time): bool {.
   rtl, extern: "ntLeTime", tags: [], raises: [], noSideEffect.}=
   ## returns true iff ``a <= b``.
+  when defined(js):
+    result = TimeBase(a) <= TimeBase(b)
+  else:
     result = a - b <= 0
 
 proc `==`*(a, b: Time): bool {.
   rtl, extern: "ntEqTime", tags: [], raises: [], noSideEffect.} =
   ## returns true if ``a == b``, that is if both times represent the same value
+  when defined(js):
+    result = TimeBase(a) == TimeBase(b)
+  else:
     result = a - b == 0
 
 proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign.}
@@ -479,7 +489,7 @@ proc `-=`*(t: var Time, ti: TimeInterval) =
   t = toTime(getLocalTime(t) - ti)
 
 proc `-`*(t: Time, ti: TimeInterval): Time =
-  ## adds the interval `ti` to Time `t`
+  ## subtracts the interval `ti` from Time `t`
   ##
   ## ``echo getTime() - 1.day``
   result = toTime(getLocalTime(t) - ti)

lib/system.nim

@@ -49,6 +49,9 @@ type
   cstring* {.magic: Cstring.} ## built-in cstring (*compatible string*) type
   pointer* {.magic: Pointer.} ## built-in pointer type, use the ``addr``
                               ## operator to get a pointer to a variable
+
+  typedesc* {.magic: TypeDesc.} ## meta type to denote a type description
+
 const
   on* = true    ## alias for ``true``
   off* = false  ## alias for ``false``
@@ -56,6 +59,15 @@ const
 {.push warning[GcMem]: off, warning[Uninit]: off.}
 {.push hints: off.}
 
+proc `or` *(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.}
+  ## Constructs an `or` meta class
+
+proc `and` *(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.}
+  ## Constructs an `and` meta class
+
+proc `not` *(a: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.}
+  ## Constructs an `not` meta class
+
 type
   Ordinal* {.magic: Ordinal.}[T] ## Generic ordinal type. Includes integer,
                                  ## bool, character, and enumeration types
@@ -66,11 +78,11 @@ type
   `ref`* {.magic: Pointer.}[T] ## built-in generic traced pointer type
 
   `nil` {.magic: "Nil".}
+
   expr* {.magic: Expr, deprecated.} ## meta type to denote an expression (for templates)
     ## **Deprecated** since version 0.15. Use ``untyped`` instead.
   stmt* {.magic: Stmt, deprecated.} ## meta type to denote a statement (for templates)
     ## **Deprecated** since version 0.15. Use ``typed`` instead.
-  typedesc* {.magic: TypeDesc.} ## meta type to denote a type description
   void* {.magic: "VoidType".}   ## meta type to denote the absence of any type
   auto* {.magic: Expr.} ## meta type for automatic type determination
   any* = distinct auto ## meta type for any supported type
@@ -430,6 +442,7 @@ type
                                         ## providing an exception message
                                         ## is bad style.
     trace: string
+    up: ref Exception # used for stacking exceptions. Not exported!
 
   SystemError* = object of Exception ## \
     ## Abstract class for exceptions that the runtime system raises.
@@ -1056,7 +1069,7 @@ proc `div`*[T: SomeUnsignedInt](x, y: T): T {.magic: "DivU", noSideEffect.}
   ## ``floor(x/y)``.
   ##
   ## .. code-block:: Nim
-  ##  (7 div 5) == 2
+  ##  (7 div 5) == 1
 
 proc `mod`*[T: SomeUnsignedInt](x, y: T): T {.magic: "ModU", noSideEffect.}
   ## computes the integer modulo operation (remainder).
@@ -1332,6 +1345,7 @@ const
   hasThreadSupport = compileOption("threads") and not defined(nimscript)
   hasSharedHeap = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own
   taintMode = compileOption("taintmode")
+  nimEnableCovariance* = defined(nimEnableCovariance) # or true
 
 when hasThreadSupport and defined(tcc) and not compileOption("tlsEmulation"):
   # tcc doesn't support TLS
@@ -1879,7 +1893,7 @@ const
   NimMajor*: int = 0
     ## is the major number of Nim's version.
 
-  NimMinor*: int = 16
+  NimMinor*: int = 17
     ## is the minor number of Nim's version.
 
   NimPatch*: int = 1
@@ -2034,6 +2048,14 @@ proc clamp*[T](x, a, b: T): T =
   if x > b: return b
   return x
 
+proc len*[T: Ordinal](x: Slice[T]): int {.noSideEffect, inline.} =
+  ## length of ordinal slice, when x.b < x.a returns zero length
+  ##
+  ## .. code-block:: Nim
+  ##   assert((0..5).len == 6)
+  ##   assert((5..2).len == 0)
+  result = max(0, ord(x.b) - ord(x.a) + 1)
+
 iterator items*[T](a: openArray[T]): T {.inline.} =
   ## iterates over each item of `a`.
   var i = 0
@@ -2428,6 +2450,16 @@ when false:
 # ----------------- GC interface ---------------------------------------------
 
 when not defined(nimscript) and hasAlloc:
+  type
+    GC_Strategy* = enum ## the strategy the GC should use for the application
+      gcThroughput,      ## optimize for throughput
+      gcResponsiveness,  ## optimize for responsiveness (default)
+      gcOptimizeTime,    ## optimize for speed
+      gcOptimizeSpace    ## optimize for memory footprint
+
+  {.deprecated: [TGC_Strategy: GC_Strategy].}
+
+  when not defined(JS):
     proc GC_disable*() {.rtl, inl, benign.}
       ## disables the GC. If called n-times, n calls to `GC_enable` are needed to
       ## reactivate the GC. Note that in most circumstances one should only disable
@@ -2440,15 +2472,6 @@ when not defined(nimscript) and hasAlloc:
       ## forces a full garbage collection pass.
       ## Ordinary code does not need to call this (and should not).
 
-  type
-    GC_Strategy* = enum ## the strategy the GC should use for the application
-      gcThroughput,      ## optimize for throughput
-      gcResponsiveness,  ## optimize for responsiveness (default)
-      gcOptimizeTime,    ## optimize for speed
-      gcOptimizeSpace    ## optimize for memory footprint
-
-  {.deprecated: [TGC_Strategy: GC_Strategy].}
-
     proc GC_setStrategy*(strategy: GC_Strategy) {.rtl, deprecated, benign.}
       ## tells the GC the desired strategy for the application.
       ## **Deprecated** since version 0.8.14. This has always been a nop.
@@ -2477,6 +2500,47 @@ when not defined(nimscript) and hasAlloc:
     proc GC_unref*(x: string) {.magic: "GCunref", benign.}
       ## see the documentation of `GC_ref`.
 
+  else:
+    template GC_disable* =
+      {.warning: "GC_disable is a no-op in JavaScript".}
+
+    template GC_enable* =
+      {.warning: "GC_enable is a no-op in JavaScript".}
+
+    template GC_fullCollect* =
+      {.warning: "GC_fullCollect is a no-op in JavaScript".}
+
+    template GC_setStrategy* =
+      {.warning: "GC_setStrategy is a no-op in JavaScript".}
+
+    template GC_enableMarkAndSweep* =
+      {.warning: "GC_enableMarkAndSweep is a no-op in JavaScript".}
+
+    template GC_disableMarkAndSweep* =
+      {.warning: "GC_disableMarkAndSweep is a no-op in JavaScript".}
+
+    template GC_ref*[T](x: ref T) =
+      {.warning: "GC_ref is a no-op in JavaScript".}
+
+    template GC_ref*[T](x: seq[T]) =
+      {.warning: "GC_ref is a no-op in JavaScript".}
+
+    template GC_ref*(x: string) =
+      {.warning: "GC_ref is a no-op in JavaScript".}
+
+    template GC_unref*[T](x: ref T) =
+      {.warning: "GC_unref is a no-op in JavaScript".}
+
+    template GC_unref*[T](x: seq[T]) =
+      {.warning: "GC_unref is a no-op in JavaScript".}
+
+    template GC_unref*(x: string) =
+      {.warning: "GC_unref is a no-op in JavaScript".}
+
+    template GC_getStatistics*(): string =
+      {.warning: "GC_disableMarkAndSweep is a no-op in JavaScript".}
+      ""
+
 template accumulateResult*(iter: untyped) =
   ## helps to convert an iterator to a proc.
   result = @[]
@@ -3210,16 +3274,6 @@ when not defined(JS): #and not defined(nimscript):
 
 elif defined(JS):
   # Stubs:
-  proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = discard
-
-  proc GC_disable() = discard
-  proc GC_enable() = discard
-  proc GC_fullCollect() = discard
-  proc GC_setStrategy(strategy: GC_Strategy) = discard
-  proc GC_enableMarkAndSweep() = discard
-  proc GC_disableMarkAndSweep() = discard
-  proc GC_getStatistics(): string = return ""
-
   proc getOccupiedMem(): int = return -1
   proc getFreeMem(): int = return -1
   proc getTotalMem(): int = return -1
@@ -3458,7 +3512,7 @@ proc `*=`*[T: SomeOrdinal|uint|uint64](x: var T, y: T) {.
 
 proc `+=`*[T: float|float32|float64] (x: var T, y: T) {.
   inline, noSideEffect.} =
-  ## Increments in placee a floating point number
+  ## Increments in place a floating point number
   x = x + y
 
 proc `-=`*[T: float|float32|float64] (x: var T, y: T) {.
@@ -3721,7 +3775,8 @@ proc locals*(): RootObj {.magic: "Plugin", noSideEffect.} =
 
 when hasAlloc and not defined(nimscript) and not defined(JS):
   proc deepCopy*[T](x: var T, y: T) {.noSideEffect, magic: "DeepCopy".} =
-    ## performs a deep copy of `x`. This is also used by the code generator
+    ## performs a deep copy of `y` and copies it into `x`.
+    ## This is also used by the code generator
     ## for the implementation of ``spawn``.
     discard
 

lib/system/alloc.nim

@@ -9,7 +9,6 @@
 
 # Low level allocator for Nim. Has been designed to support the GC.
 # TODO:
-# - eliminate "used" field
 # - make searching for block O(1)
 {.push profiler:off.}
 

lib/system/excpt.nim

@@ -90,12 +90,11 @@ proc popSafePoint {.compilerRtl, inl.} =
   excHandler = excHandler.prev
 
 proc pushCurrentException(e: ref Exception) {.compilerRtl, inl.} =
-  #if e.parent.isNil:
+  e.up = currException
-  #  e.parent = currException
   currException = e
 
 proc popCurrentException {.compilerRtl, inl.} =
-  currException = nil # currException.parent
+  currException = currException.up
 
 # some platforms have native support for stack traces:
 const
@@ -247,6 +246,18 @@ when false:
     pushCurrentException(e)
     c_longjmp(excHandler.context, 1)
 
+var onUnhandledException*: (proc (errorMsg: string) {.
+  nimcall.}) ## set this error \
+  ## handler to override the existing behaviour on an unhandled exception.
+  ## The default is to write a stacktrace to ``stderr`` and then call ``quit(1)``.
+  ## Unstable API.
+
+template unhandled(buf, body) =
+  if onUnhandledException != nil:
+    onUnhandledException($buf)
+  else:
+    body
+
 proc raiseExceptionAux(e: ref Exception) =
   if localRaiseHook != nil:
     if not localRaiseHook(e): return
@@ -277,7 +288,9 @@ proc raiseExceptionAux(e: ref Exception) =
         add(buf, " [")
         add(buf, $e.name)
         add(buf, "]\n")
+        unhandled(buf):
           showErrorMessage(buf)
+          quitOrDebug()
       else:
         # ugly, but avoids heap allocations :-)
         template xadd(buf, s, slen: expr) =
@@ -293,6 +306,7 @@ proc raiseExceptionAux(e: ref Exception) =
         add(buf, " [")
         xadd(buf, e.name, e.name.len)
         add(buf, "]\n")
+        unhandled(buf):
           showErrorMessage(buf)
           quitOrDebug()
 

lib/system/gc_common.nim

@@ -25,6 +25,13 @@ when defined(nimTypeNames):
         c_fprintf(stdout, "[Heap] %s: #%ld; bytes: %ld\n", it.name, it.instances, it.sizes)
       it = it.nextType
 
+  when defined(nimGcRefLeak):
+    proc oomhandler() =
+      c_fprintf(stdout, "[Heap] ROOTS: #%ld\n", gch.additionalRoots.len)
+      writeLeaks()
+
+    outOfMemHook = oomhandler
+
 template decTypeSize(cell, t) =
   # XXX this needs to use atomics for multithreaded apps!
   when defined(nimTypeNames):

lib/system/gc_ms.nim

@@ -142,11 +142,54 @@ proc doOperation(p: pointer, op: WalkOp) {.benign.}
 proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.}
 # we need the prototype here for debugging purposes
 
+when defined(nimGcRefLeak):
+  const
+    MaxTraceLen = 20 # tracking the last 20 calls is enough
+
+  type
+    GcStackTrace = object
+      lines: array[0..MaxTraceLen-1, cstring]
+      files: array[0..MaxTraceLen-1, cstring]
+
+  proc captureStackTrace(f: PFrame, st: var GcStackTrace) =
+    const
+      firstCalls = 5
+    var
+      it = f
+      i = 0
+      total = 0
+    while it != nil and i <= high(st.lines)-(firstCalls-1):
+      # the (-1) is for the "..." entry
+      st.lines[i] = it.procname
+      st.files[i] = it.filename
+      inc(i)
+      inc(total)
+      it = it.prev
+    var b = it
+    while it != nil:
+      inc(total)
+      it = it.prev
+    for j in 1..total-i-(firstCalls-1):
+      if b != nil: b = b.prev
+    if total != i:
+      st.lines[i] = "..."
+      st.files[i] = "..."
+      inc(i)
+    while b != nil and i <= high(st.lines):
+      st.lines[i] = b.procname
+      st.files[i] = b.filename
+      inc(i)
+      b = b.prev
+
+  var ax: array[10_000, GcStackTrace]
+
 proc nimGCref(p: pointer) {.compilerProc.} =
   # we keep it from being collected by pretending it's not even allocated:
   when false:
     when withBitvectors: excl(gch.allocated, usrToCell(p))
     else: usrToCell(p).refcount = rcBlack
+  when defined(nimGcRefLeak):
+    captureStackTrace(framePtr, ax[gch.additionalRoots.len])
   add(gch.additionalRoots, usrToCell(p))
 
 proc nimGCunref(p: pointer) {.compilerProc.} =
@@ -157,6 +200,8 @@ proc nimGCunref(p: pointer) {.compilerProc.} =
   while i >= 0:
     if d[i] == cell:
       d[i] = d[L]
+      when defined(nimGcRefLeak):
+        ax[i] = ax[L]
       dec gch.additionalRoots.len
       break
     dec(i)
@@ -164,6 +209,16 @@ proc nimGCunref(p: pointer) {.compilerProc.} =
     when withBitvectors: incl(gch.allocated, usrToCell(p))
     else: usrToCell(p).refcount = rcWhite
 
+when defined(nimGcRefLeak):
+  proc writeLeaks() =
+    for i in 0..gch.additionalRoots.len-1:
+      c_fprintf(stdout, "[Heap] NEW STACK TRACE\n")
+      for ii in 0..MaxTraceLen-1:
+        let line = ax[i].lines[ii]
+        let file = ax[i].files[ii]
+        if isNil(line): break
+        c_fprintf(stdout, "[Heap] %s(%s)\n", file, line)
+
 include gc_common
 
 proc prepareDealloc(cell: PCell) =

lib/system/gc_stack.nim

@@ -79,7 +79,7 @@ template withRegion*(r: MemRegion; body: untyped) =
   try:
     body
   finally:
-    r = tlRegion
+    #r = tlRegion
     tlRegion = oldRegion
 
 template inc(p: pointer, s: int) =

lib/system/jssys.nim

@@ -446,7 +446,7 @@ when defined(kwin):
       print(buf);
     """
 
-elif defined(nodejs):
+elif not defined(nimOldEcho):
   proc ewriteln(x: cstring) = log(x)
 
   proc rawEcho {.compilerproc, asmNoStackFrame.} =

lib/system/mmdisp.nim

@@ -255,12 +255,21 @@ elif defined(gogc):
         next_gc: uint64          # next GC (in heap_alloc time)
         last_gc: uint64          # last GC (in absolute time)
         pause_total_ns: uint64
-        pause_ns: array[256, uint64]
+        pause_ns: array[256, uint64] # circular buffer of recent gc pause lengths
+        pause_end: array[256, uint64] # circular buffer of recent gc end times (nanoseconds since 1970)
         numgc: uint32
+        numforcedgc: uint32      # number of user-forced GCs
+        gc_cpu_fraction: float64 # fraction of CPU time used by GC
         enablegc: cbool
         debuggc: cbool
         # Statistics about allocation size classes.
         by_size: array[goNumSizeClasses, goMStats_inner_struct]
+        # Statistics below here are not exported to MemStats directly.
+        tinyallocs: uint64       # number of tiny allocations that didn't cause actual allocation; not exported to go directly
+        gc_trigger: uint64
+        heap_live: uint64
+        heap_scan: uint64
+        heap_marked: uint64
 
   proc goRuntime_ReadMemStats(a2: ptr goMStats) {.cdecl,
     importc: "runtime_ReadMemStats",

lib/system/repr.nim

@@ -49,7 +49,7 @@ proc reprStrAux(result: var string, s: cstring; len: int) =
     of '"': add result, "\\\""
     of '\\': add result, "\\\\" # BUGFIX: forgotten
     of '\10': add result, "\\10\"\n\"" # " \n " # better readability
-    of '\128' .. '\255', '\0'..'\9', '\11'..'\31':
+    of '\127' .. '\255', '\0'..'\9', '\11'..'\31':
       add result, "\\" & reprInt(ord(c))
     else:
       result.add(c)
@@ -68,7 +68,7 @@ proc reprChar(x: char): string {.compilerRtl.} =
   case x
   of '"': add result, "\\\""
   of '\\': add result, "\\\\"
-  of '\128' .. '\255', '\0'..'\31': add result, "\\" & reprInt(ord(x))
+  of '\127' .. '\255', '\0'..'\31': add result, "\\" & reprInt(ord(x))
   else: add result, x
   add result, "\'"
 

lib/system/reprjs.nim

@@ -44,7 +44,7 @@ proc reprChar(x: char): string {.compilerRtl.} =
   case x
   of '"': add(result, "\\\"")
   of '\\': add(result, "\\\\")
-  of '\128'..'\255', '\0'..'\31': add( result, "\\" & reprInt(ord(x)) )
+  of '\127'..'\255', '\0'..'\31': add( result, "\\" & reprInt(ord(x)) )
   else: add(result, x)
   add(result, "\'")
 
@@ -56,7 +56,7 @@ proc reprStrAux(result: var string, s: cstring, len: int) =
     of '"': add(result, "\\\"")
     of '\\': add(result, "\\\\")
     of '\10': add(result, "\\10\"\n\"")
-    of '\128'..'\255', '\0'..'\9', '\11'..'\31':
+    of '\127'..'\255', '\0'..'\9', '\11'..'\31':
       add( result, "\\" & reprInt(ord(c)) )
     else:
       add( result, reprInt(ord(c)) ) # Not sure about this.

lib/system/syslocks.nim

@@ -117,6 +117,12 @@ else:
       when defined(linux) and defined(amd64):
         abi: array[48 div sizeof(clonglong), clonglong]
 
+    SysCondAttr {.importc: "pthread_condattr_t", pure, final
+               header: """#include <sys/types.h>
+                          #include <pthread.h>""".} = object
+      when defined(linux) and defined(amd64):
+        abi: array[4 div sizeof(cint), cint]  # actually a cint
+
     SysLockType = distinct cint
 
   proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {.
@@ -185,7 +191,7 @@ else:
       importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.}
 
   else:
-    proc initSysCondAux(cond: var SysCondObj, cond_attr: pointer) {.
+    proc initSysCondAux(cond: var SysCondObj, cond_attr: ptr SysCondAttr = nil) {.
       importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.}
     proc deinitSysCondAux(cond: var SysCondObj) {.noSideEffect,
       importc: "pthread_cond_destroy", header: "<pthread.h>".}
@@ -196,7 +202,7 @@ else:
       importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.}
 
     when defined(ios):
-      proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) =
+      proc initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) =
         cond = cast[SysCond](c_malloc(sizeof(SysCondObj)))
         initSysCondAux(cond[], cond_attr)
 
@@ -209,7 +215,7 @@ else:
       template signalSysCond(cond: var SysCond) =
         signalSysCondAux(cond[])
     else:
-      template initSysCond(cond: var SysCond, cond_attr: pointer = nil) =
+      template initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) =
         initSysCondAux(cond, cond_attr)
       template deinitSysCond(cond: var SysCond) =
         deinitSysCondAux(cond)

lib/system/threads.nim

@@ -264,7 +264,17 @@ else:
 
     proc getThreadId*(): int =
       result = int(lwp_gettid())
-  elif defined(macosx) or defined(freebsd) or defined(openbsd) or defined(netbsd):
+  elif defined(openbsd):
+    proc getthrid(): int32 {.importc: "getthrid", header: "<unistd.h>".}
+
+    proc getThreadId*(): int =
+      result = int(getthrid())
+  elif defined(netbsd):
+    proc lwp_self(): int32 {.importc: "_lwp_self", header: "<lwp.h>".}
+
+    proc getThreadId*(): int =
+      result = int(lwp_self())
+  elif defined(macosx) or defined(freebsd):
     proc pthread_threadid_np(y: pointer; x: var uint64): cint {.importc, header: "pthread.h".}
 
     proc getThreadId*(): int =

lib/windows/winlean.nim

@@ -291,6 +291,14 @@ const
   FILE_ATTRIBUTE_TEMPORARY* = 256'i32
 
   MAX_PATH* = 260
+
+  MOVEFILE_COPY_ALLOWED* = 0x2'i32
+  MOVEFILE_CREATE_HARDLINK* = 0x10'i32
+  MOVEFILE_DELAY_UNTIL_REBOOT* = 0x4'i32
+  MOVEFILE_FAIL_IF_NOT_TRACKABLE* = 0x20'i32
+  MOVEFILE_REPLACE_EXISTING* = 0x1'i32
+  MOVEFILE_WRITE_THROUGH* = 0x8'i32
+
 type
   WIN32_FIND_DATA* {.pure.} = object
     dwFileAttributes*: int32
@@ -342,6 +350,9 @@ when useWinUnicode:
 
   proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {.
     importc: "MoveFileW", stdcall, dynlib: "kernel32".}
+  proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString,
+                    flags: DWORD): WINBOOL {.
+    importc: "MoveFileExW", stdcall, dynlib: "kernel32".}
 
   proc getEnvironmentStringsW*(): WideCString {.
     stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW".}
@@ -369,6 +380,9 @@ else:
 
   proc moveFileA*(lpExistingFileName, lpNewFileName: cstring): WINBOOL {.
     importc: "MoveFileA", stdcall, dynlib: "kernel32".}
+  proc moveFileExA*(lpExistingFileName, lpNewFileName: WideCString,
+                    flags: DWORD): WINBOOL {.
+    importc: "MoveFileExA", stdcall, dynlib: "kernel32".}
 
   proc getEnvironmentStringsA*(): cstring {.
     stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsA".}

lib/wrappers/openssl.nim

@@ -26,7 +26,7 @@ when useWinVersion:
   from winlean import SocketHandle
 else:
   const
-    versions = "(|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8)"
+    versions = "(|.38|.39|.41|.43|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8)"
   when defined(macosx):
     const
       DLLSSLName = "libssl" & versions & ".dylib"

readme.md

@@ -96,9 +96,8 @@ started contributing, you should familiarize yourself with the repository struct
 
 * ``bin/``, ``build/`` - these directories are empty, but are used when Nim is built.
 * ``compiler/`` - the compiler source code. Also includes nimfix, and plugins within
-  ``compiler/nimfix`` and ``compiler/plugins`` respectively. Nimsuggest was moved to
+  ``compiler/nimfix`` and ``compiler/plugins`` respectively.
-  the [``nim-lang/nimsuggest``][nimsuggest-repo] repository, though it previously also 
+* ``nimsuggest`` - the nimsuggest tool that previously lived in the [``nim-lang/nimsuggest``][nimsuggest-repo] repository. 
-  lived within the ``compiler/`` directory.
 * ``config/`` - the configuration for the compiler and documentation generator.
 * ``doc/`` - the documentation files in reStructuredText format.
 * ``lib/`` - the standard library, including:

tests/arithm/tshr.nim

@@ -0,0 +1,20 @@
+discard """
+  output: ''''''
+"""
+
+proc T() =
+    let VI = -8
+    let VI64 = -8'i64
+    let VI32 = -8'i32
+    let VI16 = -8'i16
+    let VI8 = -8'i8
+    doAssert( (VI shr 1) == 9223372036854775804)
+    doAssert( (VI64 shr 1) == 9223372036854775804)
+    doAssert( (VI32 shr 1) == 2147483644)
+    doAssert( (VI16 shr 1) == 32764)
+    doAssert( (VI8 shr 1) == 124)
+
+
+T()
+static:
+    T()

tests/async/tasyncRecvLine.nim

@@ -0,0 +1,54 @@
+discard """
+  file: "tasyncRecvLine.nim"
+  output: '''
+Hello World
+Hello World
+'''
+"""
+
+import asyncdispatch, asyncnet
+
+const recvLinePort = Port(6047)
+
+proc setupTestServer(): AsyncSocket =
+  result = newAsyncSocket()
+  result.setSockOpt(OptReuseAddr, true)
+  result.bindAddr(recvLinePort)
+  result.listen()
+
+proc testUnbuffered(): Future[void] {.async.} =
+  let serverSock = setupTestServer()
+  let serverAcceptClientFut = serverSock.accept()
+
+  let clientSock = newAsyncSocket(buffered = false)
+  let clientConnectFut = clientSock.connect("localhost", recvLinePort)
+
+  let serverAcceptedClient = await serverAcceptClientFut
+  await clientConnectFut
+
+  await serverAcceptedClient.send("Hello World\c\L")
+
+  echo await clientSock.recvLine()
+
+  clientSock.close()
+  serverSock.close()
+
+proc testBuffered(): Future[void] {.async.} =
+  let serverSock = setupTestServer()
+  let serverAcceptClientFut = serverSock.accept()
+
+  let clientSock = newAsyncSocket(buffered = true)
+  let clientConnectFut = clientSock.connect("localhost", recvLinePort)
+
+  let serverAcceptedClient = await serverAcceptClientFut
+  await clientConnectFut
+
+  await serverAcceptedClient.send("Hello World\c\L")
+
+  echo await clientSock.recvLine()
+
+  clientSock.close()
+  serverSock.close()
+
+waitFor testUnbuffered()
+waitFor testBuffered()

tests/async/tasync_forward.nim

@@ -7,3 +7,12 @@ proc foo {.async.}
 
 proc foo {.async.} =
   discard
+
+# With additional pragmas:
+proc bar {.async, cdecl.}
+
+proc bar {.async.} =
+  discard
+
+proc verifyCdeclPresent(p: proc : Future[void] {.cdecl.}) = discard
+verifyCdeclPresent(bar)

tests/async/tasyncall.nim

@@ -2,19 +2,19 @@ discard """
   file: "tasyncall.nim"
   exitcode: 0
 """
-import times, sequtils
+import times, sequtils, unittest
 import asyncdispatch
 
 const
   taskCount = 10
-  sleepDuration = 500
+  sleepDuration = 50
 
 proc futureWithValue(x: int): Future[int] {.async.} =
   await sleepAsync(sleepDuration)
   return x
 
 proc futureWithoutValue() {.async.} =
-  await sleepAsync(1000)
+  await sleepAsync(sleepDuration)
 
 proc testFuturesWithValue(x: int): seq[int] =
   var tasks = newSeq[Future[int]](taskCount)
@@ -40,7 +40,8 @@ proc testVarargs(x, y, z: int): seq[int] =
 
   result = waitFor all(a, b, c)
 
-block:
+suite "tasyncall":
+  test "testFuturesWithValue":
     let
       startTime = cpuTime()
       results = testFuturesWithValue(42)
@@ -50,14 +51,14 @@ block:
     doAssert execTime * 1000 < taskCount * sleepDuration
     doAssert results == expected
 
-block:
+  test "testFuturesWithoutValues":
     let startTime = cpuTime()
     testFuturesWithoutValues()
     let execTime = cpuTime() - startTime
 
     doAssert execTime * 1000 < taskCount * sleepDuration
 
-block:
+  test "testVarargs":
     let
       startTime = cpuTime()
       results = testVarargs(1, 2, 3)
@@ -67,7 +68,7 @@ block:
     doAssert execTime * 100 < taskCount * sleepDuration
     doAssert results == expected
 
-block:
+  test "all on seq[Future]":
     let
       noIntFuturesFut = all(newSeq[Future[int]]())
       noVoidFuturesFut = all(newSeq[Future[void]]())

tests/async/tioselectors.nim

@@ -508,10 +508,14 @@ else:
     freeAddrInfo(aiList)
     # for some reason Windows select doesn't return both
     # descriptors from first call, so we need to make 2 calls
-    var rcm1 = selector.select(1000)
+    var n = 0
-    var rcm2 = selector.select(1000)
+    var rcm = selector.select(1000)
-    let rcm = len(rcm1) + len(rcm2)
+    while n < 10 and len(rcm) < 2:
-    assert(rcm >= 2 and rcm <= 4)
+      sleep(1000)
+      rcm = selector.select(1000)
+      inc(n)
+
+    assert(len(rcm) == 2)
 
     var sockAddress = SockAddr()
     var addrLen = sizeof(sockAddress).Socklen

tests/async/tlambda.nim

@@ -51,5 +51,8 @@ proc main() =
 
   var builder = newBuilder()
 
+  # Test {.async.} pragma with do notation: #5995
+  builder.client = newClient("builder") do(client: Client, msg: JsonNode) {.async.}:
+    await onMessage(builder, msg)
 
 main()

tests/bind/tinvalidbindtypedesc.nim

@@ -1,6 +1,6 @@
 discard """
   line: 10
-  errormsg: "type mismatch: got (typedesc[float], string)"
+  errormsg: "type mismatch: got (type float, string)"
 """
 
 proc foo(T: typedesc; some: T) =

tests/bind/tnicerrorforsymchoice.nim

@@ -1,6 +1,6 @@
 discard """
   line: 18
-  errormsg: "type mismatch: got (proc (s: TScgi) | proc (client: AsyncSocket, headers: StringTableRef, input: string){.noSideEffect, gcsafe, locks: 0.}"
+  errormsg: "type mismatch: got (proc (s: TScgi: ScgiState or AsyncScgiState) | proc (client: AsyncSocket, headers: StringTableRef, input: string){.noSideEffect, gcsafe, locks: 0.}"
 """
 
 #bug #442

tests/concepts/t3414.nim

@@ -0,0 +1,22 @@
+type
+  View[T] = concept v
+    v.empty is bool
+    v.front is T
+    popFront v
+
+proc find(view: View; target: View.T): View =
+  result = view
+
+  while not result.empty:
+    if view.front == target:
+      return
+
+    mixin popFront
+    popFront result
+
+proc popFront[T](s: var seq[T]) = discard
+proc empty[T](s: seq[T]): bool = false
+
+var s1 = @[1, 2, 3]
+let s2 = s1.find(10)
+