Merge branch 'devel' into araq

.gitignore

@@ -9,6 +9,10 @@ dnimcache/
 
 *.o
 !/icons/*.o
+*.obj
+*.ilk
+*.pdb
+*.dll
 *.exe
 *.so
 *.dylib

.travis.yml

@@ -42,4 +42,5 @@ script:
   - nimble install niminst
   - nim c --taintMode:on -d:nimCoroutines tests/testament/tester
   - tests/testament/tester --pedantic all -d:nimCoroutines
+  - ./koch web
   - ./koch csource

appveyor.yml

@@ -43,7 +43,6 @@ install:
 
 build_script:
   - bin\nim c koch
-  - koch boot
   - koch boot -d:release
   - koch nimble
   - nim e tests/test_nimscript.nims

compiler/ast.nim

@@ -292,6 +292,8 @@ const
 
   sfNoForward* = sfRegister
     # forward declarations are not required (per module)
+  sfReorder* = sfForward
+    # reordering pass is enabled
 
   sfCompileToCpp* = sfInfixCall       # compile the module as C++ code
   sfCompileToObjc* = sfNamedParamCall # compile the module as Objective-C code

compiler/ccgexprs.nim

@@ -744,14 +744,17 @@ proc genTupleElem(p: BProc, e: PNode, d: var TLoc) =
   addf(r, ".Field$1", [rope(i)])
   putIntoDest(p, d, tupType.sons[i], r, a.s)
 
-proc lookupFieldAgain(p: BProc, ty: PType; field: PSym; r: var Rope): PSym =
+proc lookupFieldAgain(p: BProc, ty: PType; field: PSym; r: var Rope;
+                      resTyp: ptr PType = nil): PSym =
   var ty = ty
   assert r != nil
   while ty != nil:
     ty = ty.skipTypes(skipPtrs)
     assert(ty.kind in {tyTuple, tyObject})
     result = lookupInRecord(ty.n, field.name)
-    if result != nil: break
+    if result != nil:
+      if resTyp != nil: resTyp[] = ty
+      break
     if not p.module.compileToCpp: add(r, ".Sup")
     ty = ty.sons[0]
   if result == nil: internalError(field.info, "genCheckedRecordField")
@@ -768,8 +771,9 @@ proc genRecordField(p: BProc, e: PNode, d: var TLoc) =
     addf(r, ".Field$1", [rope(f.position)])
     putIntoDest(p, d, f.typ, r, a.s)
   else:
-    let field = lookupFieldAgain(p, ty, f, r)
+    var rtyp: PType
-    if field.loc.r == nil: fillObjectFields(p.module, ty)
+    let field = lookupFieldAgain(p, ty, f, r, addr rtyp)
+    if field.loc.r == nil and rtyp != nil: fillObjectFields(p.module, rtyp)
     if field.loc.r == nil: internalError(e.info, "genRecordField 3 " & typeToString(ty))
     addf(r, ".$1", [field.loc.r])
     putIntoDest(p, d, field.typ, r, a.s)
@@ -1057,7 +1061,7 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) =
                            "$1 = ($2) #incrSeqV2(&($1)->Sup, sizeof($3));$n"
                          else:
                            "$1 = ($2) #incrSeqV2($1, sizeof($3));$n"
-  var a, b, dest: TLoc
+  var a, b, dest, tmpL: TLoc
   initLocExpr(p, e.sons[1], a)
   initLocExpr(p, e.sons[2], b)
   let bt = skipTypes(e.sons[2].typ, {tyVar})
@@ -1068,9 +1072,10 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) =
   #if bt != b.t:
   #  echo "YES ", e.info, " new: ", typeToString(bt), " old: ", typeToString(b.t)
   initLoc(dest, locExpr, bt, OnHeap)
-  dest.r = rfmt(nil, "$1->data[$1->$2]", rdLoc(a), lenField(p))
+  getIntTemp(p, tmpL)
+  lineCg(p, cpsStmts, "$1 = $2->$3++;$n", tmpL.r, rdLoc(a), lenField(p))
+  dest.r = rfmt(nil, "$1->data[$2]", rdLoc(a), tmpL.r)
   genAssignment(p, dest, b, {needToCopy, afDestIsNil})
-  lineCg(p, cpsStmts, "++$1->$2;$n", rdLoc(a), lenField(p))
   gcUsage(e)
 
 proc genReset(p: BProc, n: PNode) =
@@ -1096,9 +1101,9 @@ proc rawGenNew(p: BProc, a: TLoc, sizeExpr: Rope) =
   if a.s == OnHeap and usesNativeGC():
     # use newObjRC1 as an optimization
     if canFormAcycle(a.t):
-      linefmt(p, cpsStmts, "if ($1) #nimGCunref($1);$n", a.rdLoc)
+      linefmt(p, cpsStmts, "if ($1) { #nimGCunrefRC1($1); $1 = NIM_NIL; }$n", a.rdLoc)
     else:
-      linefmt(p, cpsStmts, "if ($1) #nimGCunrefNoCycle($1);$n", a.rdLoc)
+      linefmt(p, cpsStmts, "if ($1) { #nimGCunrefNoCycle($1); $1 = NIM_NIL; }$n", a.rdLoc)
     b.r = ropecg(p.module, "($1) #newObjRC1($2, $3)", args)
     linefmt(p, cpsStmts, "$1 = $2;$n", a.rdLoc, b.rdLoc)
   else:
@@ -1126,9 +1131,9 @@ proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope) =
   initLoc(call, locExpr, dest.t, OnHeap)
   if dest.s == OnHeap and usesNativeGC():
     if canFormAcycle(dest.t):
-      linefmt(p, cpsStmts, "if ($1) #nimGCunref($1);$n", dest.rdLoc)
+      linefmt(p, cpsStmts, "if ($1) { #nimGCunrefRC1($1); $1 = NIM_NIL; }$n", dest.rdLoc)
     else:
-      linefmt(p, cpsStmts, "if ($1) #nimGCunrefNoCycle($1);$n", dest.rdLoc)
+      linefmt(p, cpsStmts, "if ($1) { #nimGCunrefNoCycle($1); $1 = NIM_NIL; }$n", dest.rdLoc)
     call.r = ropecg(p.module, "($1) #newSeqRC1($2, $3)", args)
     linefmt(p, cpsStmts, "$1 = $2;$n", dest.rdLoc, call.rdLoc)
   else:
@@ -1170,6 +1175,9 @@ proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool =
 
 proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
   #echo rendertree e, " ", e.isDeepConstExpr
+  # inheritance in C++ does not allow struct initialization so
+  # we skip this step here:
+  if not p.module.compileToCpp:
     if handleConstExpr(p, e, d): return
   var tmp: TLoc
   var t = e.typ.skipTypes(abstractInst)
@@ -1378,13 +1386,30 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     useStringh(p.module)
     if op == mHigh: unaryExpr(p, e, d, "($1 ? (strlen($1)-1) : -1)")
     else: unaryExpr(p, e, d, "($1 ? strlen($1) : 0)")
-  of tyString, tySequence:
+  of tyString:
     if not p.module.compileToCpp:
       if op == mHigh: unaryExpr(p, e, d, "($1 ? ($1->Sup.len-1) : -1)")
       else: unaryExpr(p, e, d, "($1 ? $1->Sup.len : 0)")
     else:
       if op == mHigh: unaryExpr(p, e, d, "($1 ? ($1->len-1) : -1)")
       else: unaryExpr(p, e, d, "($1 ? $1->len : 0)")
+  of tySequence:
+    var a, tmp: TLoc
+    initLocExpr(p, e[1], a)
+    getIntTemp(p, tmp)
+    var frmt: FormatStr
+    if not p.module.compileToCpp:
+      if op == mHigh:
+        frmt = "$1 = ($2 ? ($2->Sup.len-1) : -1);$n"
+      else:
+        frmt = "$1 = ($2 ? $2->Sup.len : 0);$n"
+    else:
+      if op == mHigh:
+        frmt = "$1 = ($2 ? ($2->len-1) : -1);$n"
+      else:
+        frmt = "$1 = ($2 ? $2->len : 0);$n"
+    lineCg(p, cpsStmts, frmt, tmp.r, rdLoc(a))
+    putIntoDest(p, d, e.typ, tmp.r)
   of tyArray:
     # YYY: length(sideeffect) is optimized away incorrectly?
     if op == mHigh: putIntoDest(p, d, e.typ, rope(lastOrd(typ)))
@@ -1742,11 +1767,23 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mOrd: genOrd(p, e, d)
   of mLengthArray, mHigh, mLengthStr, mLengthSeq, mLengthOpenArray:
     genArrayLen(p, e, d, op)
-  of mXLenStr, mXLenSeq:
+  of mXLenStr:
     if not p.module.compileToCpp:
       unaryExpr(p, e, d, "($1->Sup.len)")
     else:
       unaryExpr(p, e, d, "$1->len")
+  of mXLenSeq:
+    # see 'taddhigh.nim' for why we need to use a temporary here:
+    var a, tmp: TLoc
+    initLocExpr(p, e[1], a)
+    getIntTemp(p, tmp)
+    var frmt: FormatStr
+    if not p.module.compileToCpp:
+      frmt = "$1 = $2->Sup.len;$n"
+    else:
+      frmt = "$1 = $2->len;$n"
+    lineCg(p, cpsStmts, frmt, tmp.r, rdLoc(a))
+    putIntoDest(p, d, e.typ, tmp.r)
   of mGCref: unaryStmt(p, e, d, "#nimGCref($1);$n")
   of mGCunref: unaryStmt(p, e, d, "#nimGCunref($1);$n")
   of mSetLengthStr: genSetLengthStr(p, e, d)
@@ -2123,17 +2160,19 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       # See tests/run/tcnstseq3 for an example that would fail otherwise.
       genAsgn(p, n, fastAsgn=p.prc != nil)
   of nkDiscardStmt:
-    if n.sons[0].kind != nkEmpty:
+    let ex = n[0]
+    if ex.kind != nkEmpty:
       genLineDir(p, n)
       var a: TLoc
-      if n[0].kind in nkCallKinds:
+      if ex.kind in nkCallKinds and (ex[0].kind != nkSym or
+                                     ex[0].sym.magic == mNone):
         # bug #6037: do not assign to a temp in C++ mode:
         incl a.flags, lfSingleUse
-        genCall(p, n[0], a)
+        genCall(p, ex, a)
         if lfSingleUse notin a.flags:
           line(p, cpsStmts, a.r & ";" & tnl)
       else:
-        initLocExpr(p, n.sons[0], a)
+        initLocExpr(p, ex, a)
   of nkAsmStmt: genAsmStmt(p, n)
   of nkTryStmt:
     if p.module.compileToCpp and optNoCppExceptions notin gGlobalOptions:
@@ -2201,20 +2240,22 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope =
   else:
     globalError(info, "cannot create null element for: " & $t.kind)
 
-proc getNullValueAux(p: BProc; obj, cons: PNode, result: var Rope) =
+proc getNullValueAux(p: BProc; t: PType; obj, cons: PNode, result: var Rope; count: var int) =
   case obj.kind
   of nkRecList:
-    for i in countup(0, sonsLen(obj) - 1): getNullValueAux(p, obj.sons[i], cons, result)
+    for i in countup(0, sonsLen(obj) - 1):
+      getNullValueAux(p, t, obj.sons[i], cons, result, count)
   of nkRecCase:
-    getNullValueAux(p, obj.sons[0], cons, result)
+    getNullValueAux(p, t, obj.sons[0], cons, result, count)
     for i in countup(1, sonsLen(obj) - 1):
-      getNullValueAux(p, lastSon(obj.sons[i]), cons, result)
+      getNullValueAux(p, t, lastSon(obj.sons[i]), cons, result, count)
   of nkSym:
-    if not result.isNil: result.add ", "
+    if count > 0: result.add ", "
+    inc count
     let field = obj.sym
     for i in 1..<cons.len:
       if cons[i].kind == nkExprColonExpr:
-        if cons[i][0].sym.name == field.name:
+        if cons[i][0].sym.name.id == field.name.id:
           result.add genConstExpr(p, cons[i][1])
           return
       elif i == field.position:
@@ -2225,13 +2266,31 @@ proc getNullValueAux(p: BProc; obj, cons: PNode, result: var Rope) =
   else:
     localError(cons.info, "cannot create null element for: " & $obj)
 
+proc getNullValueAuxT(p: BProc; orig, t: PType; obj, cons: PNode, result: var Rope; count: var int) =
+  var base = t.sons[0]
+  let oldRes = result
+  if not p.module.compileToCpp: result.add "{"
+  let oldcount = count
+  if base != nil:
+    base = skipTypes(base, skipPtrs)
+    getNullValueAuxT(p, orig, base, base.n, cons, result, count)
+  elif not isObjLackingTypeField(t) and not p.module.compileToCpp:
+    addf(result, "$1", [genTypeInfo(p.module, orig)])
+    inc count
+  getNullValueAux(p, t, obj, cons, result, count)
+  # do not emit '{}' as that is not valid C:
+  if oldcount == count: result = oldres
+  elif not p.module.compileToCpp: result.add "}"
+
 proc genConstObjConstr(p: BProc; n: PNode): Rope =
-  var length = sonsLen(n)
   result = nil
   let t = n.typ.skipTypes(abstractInst)
-  if not isObjLackingTypeField(t) and not p.module.compileToCpp:
+  var count = 0
-    addf(result, "{$1}", [genTypeInfo(p.module, t)])
+  #if not isObjLackingTypeField(t) and not p.module.compileToCpp:
-  getNullValueAux(p, t.n, n, result)
+  #  addf(result, "{$1}", [genTypeInfo(p.module, t)])
+  #  inc count
+  getNullValueAuxT(p, t, t, t.n, n, result, count)
+  if p.module.compileToCpp:
     result = "{$1}$n" % [result]
 
 proc genConstSimpleList(p: BProc, n: PNode): Rope =
@@ -2279,6 +2338,15 @@ proc genConstExpr(p: BProc, n: PNode): Rope =
     var t = skipTypes(n.typ, abstractInst)
     if t.kind == tySequence:
       result = genConstSeq(p, n, n.typ)
+    elif t.kind == tyProc and t.callConv == ccClosure and not n.sons.isNil and
+         n.sons[0].kind == nkNilLit and n.sons[1].kind == nkNilLit:
+      # this hack fixes issue that nkNilLit is expanded to {NIM_NIL,NIM_NIL}
+      # this behaviour is needed since closure_var = nil must be
+      # expanded to {NIM_NIL,NIM_NIL}
+      # in VM closures are initialized with nkPar(nkNilLit, nkNilLit)
+      # leading to duplicate code like this:
+      # "{NIM_NIL,NIM_NIL}, {NIM_NIL,NIM_NIL}"
+      result = ~"{NIM_NIL,NIM_NIL}"
     else:
       result = genConstSimpleList(p, n)
   of nkObjConstr:

compiler/ccgstmts.nim

@@ -164,11 +164,11 @@ proc genBreakState(p: BProc, n: PNode) =
   if n.sons[0].kind == nkClosure:
     # XXX this produces quite inefficient code!
     initLocExpr(p, n.sons[0].sons[1], a)
-    lineF(p, cpsStmts, "if (((NI*) $1)[0] < 0) break;$n", [rdLoc(a)])
+    lineF(p, cpsStmts, "if (((NI*) $1)[1] < 0) break;$n", [rdLoc(a)])
   else:
     initLocExpr(p, n.sons[0], a)
-    # the environment is guaranteed to contain the 'state' field at offset 0:
+    # the environment is guaranteed to contain the 'state' field at offset 1:
-    lineF(p, cpsStmts, "if ((((NI*) $1.ClE_0)[0]) < 0) break;$n", [rdLoc(a)])
+    lineF(p, cpsStmts, "if ((((NI*) $1.ClE_0)[1]) < 0) break;$n", [rdLoc(a)])
   #  lineF(p, cpsStmts, "if (($1) < 0) break;$n", [rdLoc(a)])
 
 proc genVarPrototypeAux(m: BModule, sym: PSym)

compiler/ccgtrav.nim

@@ -72,9 +72,15 @@ proc genTraverseProc(c: var TTraversalClosure, accessor: Rope, typ: PType) =
     let arraySize = lengthOrd(typ.sons[0])
     var i: TLoc
     getTemp(p, getSysType(tyInt), i)
+    let oldCode = p.s(cpsStmts)
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
             i.r, arraySize.rope)
+    let oldLen = p.s(cpsStmts).len
     genTraverseProc(c, rfmt(nil, "$1[$2]", accessor, i.r), typ.sons[1])
+    if p.s(cpsStmts).len == oldLen:
+      # do not emit dummy long loops for faster debug builds:
+      p.s(cpsStmts) = oldCode
+    else:
       lineF(p, cpsStmts, "}$n", [])
   of tyObject:
     for i in countup(0, sonsLen(typ) - 1):
@@ -99,9 +105,15 @@ proc genTraverseProcSeq(c: var TTraversalClosure, accessor: Rope, typ: PType) =
   assert typ.kind == tySequence
   var i: TLoc
   getTemp(p, getSysType(tyInt), i)
+  let oldCode = p.s(cpsStmts)
   lineF(p, cpsStmts, "for ($1 = 0; $1 < $2->$3; $1++) {$n",
       [i.r, accessor, rope(if c.p.module.compileToCpp: "len" else: "Sup.len")])
+  let oldLen = p.s(cpsStmts).len
   genTraverseProc(c, "$1->data[$2]" % [accessor, i.r], typ.sons[0])
+  if p.s(cpsStmts).len == oldLen:
+    # do not emit dummy long loops for faster debug builds:
+    p.s(cpsStmts) = oldCode
+  else:
     lineF(p, cpsStmts, "}$n", [])
 
 proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash;

compiler/ccgtypes.nim

@@ -12,6 +12,7 @@
 # ------------------------- Name Mangling --------------------------------
 
 import sighashes
+from lowerings import createObj
 
 proc isKeyword(w: PIdent): bool =
   # Nim and C++ share some keywords
@@ -122,10 +123,11 @@ const
 
 proc typeName(typ: PType): Rope =
   let typ = typ.skipTypes(irrelevantForBackend)
-  result = if typ.sym != nil and typ.kind in {tyObject, tyEnum}:
+  result =
-             typ.sym.name.s.mangle.rope
+    if typ.sym != nil and typ.kind in {tyObject, tyEnum}:
+      rope($typ.kind & '_' & typ.sym.name.s.mangle)
     else:
-             ~"TY"
+      rope($typ.kind)
 
 proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
   var t = typ
@@ -335,6 +337,7 @@ proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope =
     if result == nil: result = cacheGetType(m.typeCache, sig)
 
 proc structOrUnion(t: PType): Rope =
+  let t = t.skipTypes({tyAlias})
   (if tfUnion in t.flags: rope("union") else: rope("struct"))
 
 proc getForwardStructFormat(m: BModule): string =
@@ -473,11 +476,14 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
             if tfPacked notin rectype.flags:
               add(unionBody, "struct {")
             else:
-              addf(unionBody, CC[cCompiler].structStmtFmt,
+              if hasAttribute in CC[cCompiler].props:
-                [rope"struct", nil, rope(CC[cCompiler].packedPragma)])
+                add(unionBody, "struct __attribute__((__packed__)){" )
-              add(unionBody, "{")
+              else:
+                addf(unionBody, "#pragma pack(1)$nstruct{", [])
             add(unionBody, a)
             addf(unionBody, "} $1;$n", [sname])
+            if tfPacked in rectype.flags and hasAttribute notin CC[cCompiler].props:
+              addf(unionBody, "#pragma pack(pop)$n", [])
         else:
           add(unionBody, genRecordFieldsAux(m, k, ae, rectype, check))
       else: internalError("genRecordFieldsAux(record case branch)")
@@ -524,12 +530,16 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
   # declare the record:
   var hasField = false
 
-  var attribute: Rope =
+  if tfPacked in typ.flags:
-    if tfPacked in typ.flags: rope(CC[cCompiler].packedPragma)
+    if hasAttribute in CC[cCompiler].props:
-    else: nil
+      result = structOrUnion(typ) & " __attribute__((__packed__))"
+    else:
+      result = "#pragma pack(1)" & tnl & structOrUnion(typ)
+  else:
+    result = structOrUnion(typ)
 
-  result = ropecg(m, CC[cCompiler].structStmtFmt,
+  result.add " "
-    [structOrUnion(typ), name, attribute])
+  result.add name
 
   if typ.kind == tyObject:
 
@@ -537,7 +547,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
       if (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags:
         appcg(m, result, " {$n", [])
       else:
-        appcg(m, result, " {$n#TNimType* m_type;$n", [name, attribute])
+        appcg(m, result, " {$n#TNimType* m_type;$n", [])
         hasField = true
     elif m.compileToCpp:
       appcg(m, result, " : public $1 {$n",
@@ -556,6 +566,8 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
   else:
     add(result, desc)
   add(result, "};" & tnl)
+  if tfPacked in typ.flags and hasAttribute notin CC[cCompiler].props:
+    result.add "#pragma pack(pop)" & tnl
 
 proc getTupleDesc(m: BModule, typ: PType, name: Rope,
                   check: var IntSet): Rope =
@@ -787,7 +799,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
         add(m.s[cfsTypes], recdesc)
       elif tfIncompleteStruct notin t.flags: addAbiCheck(m, t, result)
   of tySet:
-    result = getTypeName(m, t.lastSon, hashType t.lastSon) & "_Set"
+    result = $t.kind & '_' & getTypeName(m, t.lastSon, hashType t.lastSon)
     m.typeCache[sig] = result
     if not isImportedType(t):
       let s = int(getSize(t))
@@ -1083,7 +1095,8 @@ proc fakeClosureType(owner: PSym): PType =
   result = newType(tyTuple, owner)
   result.rawAddSon(newType(tyPointer, owner))
   var r = newType(tyRef, owner)
-  r.rawAddSon(newType(tyTuple, owner))
+  let obj = createObj(owner, owner.info, final=false)
+  r.rawAddSon(obj)
   result.rawAddSon(r)
 
 type

compiler/cgen.nim

@@ -209,7 +209,7 @@ proc genLineDir(p: BProc, t: PNode) =
   if ({optStackTrace, optEndb} * p.options == {optStackTrace, optEndb}) and
       (p.prc == nil or sfPure notin p.prc.flags):
     if freshLineInfo(p, tt.info):
-      linefmt(p, cpsStmts, "#endb($1, $2);$n",
+      linefmt(p, cpsStmts, "#endb($1, $2);$N",
               line.rope, makeCString(toFilename(tt.info)))
   elif ({optLineTrace, optStackTrace} * p.options ==
       {optLineTrace, optStackTrace}) and
@@ -345,6 +345,15 @@ proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
   result.flags = {}
   constructLoc(p, result, not needsInit)
 
+proc getIntTemp(p: BProc, result: var TLoc) =
+  inc(p.labels)
+  result.r = "T" & rope(p.labels) & "_"
+  linefmt(p, cpsLocals, "NI $1;$n", result.r)
+  result.k = locTemp
+  result.s = OnStack
+  result.t = getSysType(tyInt)
+  result.flags = {}
+
 proc initGCFrame(p: BProc): Rope =
   if p.gcFrameId > 0: result = "struct {$1} GCFRAME_;$n" % [p.gcFrameType]
 
@@ -385,7 +394,8 @@ proc assignLocalVar(p: BProc, s: PSym) =
   #assert(s.loc.k == locNone) # not yet assigned
   # this need not be fulfilled for inline procs; they are regenerated
   # for each module that uses them!
-  let decl = localVarDecl(p, s) & ";" & tnl
+  let nl = if optLineDir in gOptions: "" else: tnl
+  let decl = localVarDecl(p, s) & ";" & nl
   line(p, cpsLocals, decl)
   localDebugInfo(p, s)
 
@@ -618,11 +628,11 @@ proc initFrame(p: BProc, procname, filename: Rope): Rope =
   discard cgsym(p.module, "nimFrame")
   if p.maxFrameLen > 0:
     discard cgsym(p.module, "VarSlot")
-    result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4)$N",
+    result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4);$n",
                   procname, filename, p.maxFrameLen.rope,
                   p.blocks[0].frameLen.rope)
   else:
-    result = rfmt(nil, "\tnimfr_($1, $2)$N", procname, filename)
+    result = rfmt(nil, "\tnimfr_($1, $2);$n", procname, filename)
 
 proc deinitFrame(p: BProc): Rope =
   result = rfmt(p.module, "\t#popFrame();$n")
@@ -1302,6 +1312,7 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
   if b == nil or passes.skipCodegen(n): return
   var m = BModule(b)
   m.initProc.options = initProcOptions(m)
+  softRnl = if optLineDir in gOptions: noRnl else: rnl
   genStmts(m.initProc, n)
 
 proc finishModule(m: BModule) =

compiler/commands.nim

@@ -207,7 +207,7 @@ proc testCompileOptionArg*(switch, arg: string, info: TLineInfo): bool =
     of "generational": result = gSelectedGC == gcGenerational
     of "go":           result = gSelectedGC == gcGo
     of "none":         result = gSelectedGC == gcNone
-    of "stack":        result = gSelectedGC == gcStack
+    of "stack", "regions": result = gSelectedGC == gcRegions
     else: localError(info, errNoneBoehmRefcExpectedButXFound, arg)
   of "opt":
     case arg.normalize
@@ -429,9 +429,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "none":
       gSelectedGC = gcNone
       defineSymbol("nogc")
-    of "stack":
+    of "stack", "regions":
-      gSelectedGC= gcStack
+      gSelectedGC= gcRegions
-      defineSymbol("gcstack")
+      defineSymbol("gcregions")
     else: localError(info, errNoneBoehmRefcExpectedButXFound, arg)
   of "warnings", "w":
     if processOnOffSwitchOrList({optWarns}, arg, pass, info): listWarnings()
@@ -454,6 +454,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "native", "gdb":
       incl(gGlobalOptions, optCDebug)
       gOptions = gOptions + {optLineDir} - {optEndb}
+      defineSymbol("nimTypeNames", nil) # type names are used in gdb pretty printing
       undefSymbol("endb")
     else:
       localError(info, "expected endb|gdb but found " & arg)

compiler/condsyms.nim

@@ -41,7 +41,10 @@ proc isDefined*(symbol: string): bool =
       result = targetOS in {osLinux, osMorphos, osSkyos, osIrix, osPalmos,
                             osQnx, osAtari, osAix,
                             osHaiku, osVxWorks, osSolaris, osNetbsd,
-                            osFreebsd, osOpenbsd, osDragonfly, osMacosx}
+                            osFreebsd, osOpenbsd, osDragonfly, osMacosx,
+                            osAndroid}
+    of "linux":
+      result = targetOS in {osLinux, osAndroid}
     of "bsd":
       result = targetOS in {osNetbsd, osFreebsd, osOpenbsd, osDragonfly}
     of "emulatedthreadvars":

compiler/docgen.nim

@@ -462,11 +462,13 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
     var path = n.info.toFullPath
     if path.startsWith(cwd):
       path = path[cwd.len+1 .. ^1].replace('\\', '/')
+    let gitUrl = getConfigVar("git.url")
+    if gitUrl.len > 0:
       var commit = getConfigVar("git.commit")
       if commit.len == 0: commit = "master"
       dispA(seeSrcRope, "$1", "", [ropeFormatNamedVars(docItemSeeSrc,
           ["path", "line", "url", "commit"], [rope path,
-        rope($n.info.line), rope getConfigVar("git.url"),
+          rope($n.info.line), rope gitUrl,
           rope commit])])
 
   add(d.section[k], ropeFormatNamedVars(getConfigVar("doc.item"),

compiler/evalffi.nim

@@ -104,9 +104,9 @@ proc mapCallConv(cc: TCallingConvention, info: TLineInfo): TABI =
   else:
     globalError(info, "cannot map calling convention to FFI")
 
-template rd(T, p: expr): expr {.immediate.} = (cast[ptr T](p))[]
+template rd(T, p: untyped): untyped = (cast[ptr T](p))[]
-template wr(T, p, v: expr) {.immediate.} = (cast[ptr T](p))[] = v
+template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v
-template `+!`(x, y: expr): expr {.immediate.} =
+template `+!`(x, y: untyped): untyped =
   cast[pointer](cast[ByteAddress](x) + y)
 
 proc packSize(v: PNode, typ: PType): int =
@@ -171,7 +171,7 @@ const maxPackDepth = 20
 var packRecCheck = 0
 
 proc pack(v: PNode, typ: PType, res: pointer) =
-  template awr(T, v: expr) {.immediate, dirty.} =
+  template awr(T, v: untyped): untyped =
     wr(T, res, v)
 
   case typ.kind
@@ -302,7 +302,7 @@ proc canonNodeKind(k: TNodeKind): TNodeKind =
   else: result = k
 
 proc unpack(x: pointer, typ: PType, n: PNode): PNode =
-  template aw(k, v, field: expr) {.immediate, dirty.} =
+  template aw(k, v, field: untyped): untyped =
     if n.isNil:
       result = newNode(k)
       result.typ = typ
@@ -326,9 +326,9 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
       result.kind = nkNilLit
       result.typ = typ
 
-  template awi(kind, v: expr) {.immediate, dirty.} = aw(kind, v, intVal)
+  template awi(kind, v: untyped): untyped = aw(kind, v, intVal)
-  template awf(kind, v: expr) {.immediate, dirty.} = aw(kind, v, floatVal)
+  template awf(kind, v: untyped): untyped = aw(kind, v, floatVal)
-  template aws(kind, v: expr) {.immediate, dirty.} = aw(kind, v, strVal)
+  template aws(kind, v: untyped): untyped = aw(kind, v, strVal)
 
   case typ.kind
   of tyBool: awi(nkIntLit, rd(bool, x).ord)

compiler/extccomp.nim

@@ -53,7 +53,6 @@ type
                          # used on some platforms
     asmStmtFrmt: string, # format of ASM statement
     structStmtFmt: string, # Format for struct statement
-    packedPragma: string,  # Attribute/pragma to make struct packed (1-byte aligned)
     props: TInfoCCProps] # properties of the C compiler
 
 
@@ -86,7 +85,6 @@ compiler gcc:
     pic: "-fPIC",
     asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
-    packedPragma: "__attribute__((__packed__))",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasAttribute})
 
@@ -129,7 +127,6 @@ compiler vcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$3$n$1 $2",
-    packedPragma: "#pragma pack(1)",
     props: {hasCpp, hasAssume, hasDeclspec})
 
 # Intel C/C++ Compiler
@@ -166,7 +163,6 @@ compiler lcc:
     pic: "",
     asmStmtFrmt: "_asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
-    packedPragma: "", # XXX: not supported yet
     props: {})
 
 # Borland C Compiler
@@ -191,7 +187,6 @@ compiler bcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
-    packedPragma: "", # XXX: not supported yet
     props: {hasCpp})
 
 # Digital Mars C Compiler
@@ -216,7 +211,6 @@ compiler dmc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$3$n$1 $2",
-    packedPragma: "#pragma pack(1)",
     props: {hasCpp})
 
 # Watcom C Compiler
@@ -241,7 +235,6 @@ compiler wcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
-    packedPragma: "", # XXX: not supported yet
     props: {hasCpp})
 
 # Tiny C Compiler
@@ -266,7 +259,6 @@ compiler tcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
-    packedPragma: "", # XXX: not supported yet
     props: {hasSwitchRange, hasComputedGoto})
 
 # Pelles C Compiler
@@ -292,7 +284,6 @@ compiler pcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
-    packedPragma: "", # XXX: not supported yet
     props: {})
 
 # Your C Compiler
@@ -317,7 +308,6 @@ compiler ucc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
-    packedPragma: "", # XXX: not supported yet
     props: {})
 
 const
@@ -687,11 +677,14 @@ proc getLinkCmd(projectfile, objfiles: string): string =
     exefile = quoteShell(exefile)
     let linkOptions = getLinkOptions() & " " &
                       getConfigVar(cCompiler, ".options.linker")
+    var linkTmpl = getConfigVar(cCompiler, ".linkTmpl")
+    if linkTmpl.len == 0:
+      linkTmpl = CC[cCompiler].linkTmpl
     result = quoteShell(result % ["builddll", builddll,
         "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles,
         "exefile", exefile, "nim", getPrefixDir(), "lib", libpath])
     result.add ' '
-    addf(result, CC[cCompiler].linkTmpl, ["builddll", builddll,
+    addf(result, linkTmpl, ["builddll", builddll,
         "buildgui", buildgui, "options", linkOptions,
         "objfiles", objfiles, "exefile", exefile,
         "nim", quoteShell(getPrefixDir()),

compiler/installer.ini

@@ -6,7 +6,7 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;amd64;powerpc64;arm;sparc;mips;mipsel;powerpc;powerpc64el;arm64
+  linux: i386;amd64;powerpc64;arm;sparc;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64
   macosx: i386;amd64;powerpc64
   solaris: i386;amd64;sparc;sparc64
   freebsd: i386;amd64
@@ -14,6 +14,7 @@ Platforms: """
   openbsd: i386;amd64
   dragonfly: i386;amd64
   haiku: i386;amd64
+  android: i386;arm;arm64
 """
 
 Authors: "Andreas Rumpf"
@@ -47,7 +48,7 @@ Start: "doc/html/overview.html"
 
 
 [Other]
-Files: "readme.txt;copying.txt"
+Files: "readme.txt;copying.txt;install.txt"
 Files: "makefile"
 Files: "koch.nim"
 Files: "install_nimble.nims"

compiler/jsgen.nim

@@ -2340,7 +2340,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkPragma: genPragma(p, n)
   of nkProcDef, nkMethodDef, nkConverterDef:
     var s = n.sons[namePos].sym
-    if sfExportc in s.flags and compilingLib:
+    if {sfExportc, sfCompilerProc} * s.flags == {sfExportc}:
       genSym(p, n.sons[namePos], r)
       r.res = nil
   of nkGotoState, nkState:

compiler/lambdalifting.nim

@@ -142,7 +142,7 @@ proc createStateField(iter: PSym): PSym =
 proc createEnvObj(owner: PSym; info: TLineInfo): PType =
   # YYY meh, just add the state field for every closure for now, it's too
   # hard to figure out if it comes from a closure iterator:
-  result = createObj(owner, info)
+  result = createObj(owner, info, final=false)
   rawAddField(result, createStateField(owner))
 
 proc getIterResult(iter: PSym): PSym =

compiler/lowerings.nim

@@ -109,12 +109,14 @@ proc lowerSwap*(n: PNode; owner: PSym): PNode =
   result.add newFastAsgnStmt(n[1], n[2])
   result.add newFastAsgnStmt(n[2], tempAsNode)
 
-proc createObj*(owner: PSym, info: TLineInfo): PType =
+proc createObj*(owner: PSym, info: TLineInfo; final=true): PType =
   result = newType(tyObject, owner)
+  if final:
     rawAddSon(result, nil)
     incl result.flags, tfFinal
+  else:
+    rawAddSon(result, getCompilerProc("RootObj").typ)
   result.n = newNodeI(nkRecList, info)
-  when true:
   let s = newSym(skType, getIdent("Env_" & info.toFilename),
                   owner, info)
   incl s.flags, sfAnon

compiler/msgs.nim

@@ -746,7 +746,7 @@ proc toFileLine*(info: TLineInfo): string {.inline.} =
   result = info.toFilename & ":" & $info.line
 
 proc toFileLineCol*(info: TLineInfo): string {.inline.} =
-  result = info.toFilename & "(" & $info.line & "," & $info.col & ")"
+  result = info.toFilename & "(" & $info.line & ", " & $info.col & ")"
 
 proc `$`*(info: TLineInfo): string = toFileLineCol(info)
 

compiler/nimblecmd.nim

@@ -9,7 +9,7 @@
 
 ## Implements some helper procs for Nimble (Nim's package manager) support.
 
-import parseutils, strutils, strtabs, os, options, msgs
+import parseutils, strutils, strtabs, os, options, msgs, sequtils
 
 proc addPath*(path: string, info: TLineInfo) =
   if not options.searchPaths.contains(path):
@@ -21,51 +21,76 @@ proc versionSplitPos(s: string): int =
   while result > 1 and s[result] != '-': dec result
   if s[result] != '-': result = s.len
 
-const
+type
-  latest = ""
+  Version = distinct string
 
-proc `<.`(a, b: string): bool =
+proc `$`(ver: Version): string {.borrow.}
-  # wether a has a smaller version than b:
+
-  if a == latest: return true
+proc newVersion(ver: string): Version =
-  elif b == latest: return false
+  doAssert(ver.len == 0 or ver[0] in {'#', '\0'} + Digits,
-  var i = 0
+           "Wrong version: " & ver)
-  var j = 0
+  return Version(ver)
-  var verA = 0
+
-  var verB = 0
+proc isSpecial(ver: Version): bool =
-  while true:
+  return ($ver).len > 0 and ($ver)[0] == '#'
-    let ii = parseInt(a, verA, i)
+
-    let jj = parseInt(b, verB, j)
+proc `<`(ver: Version, ver2: Version): bool =
-    if ii <= 0 or jj <= 0:
+  ## This is synced from Nimble's version module.
-      # if A has no number and B has but A has no number whatsoever ("#head"),
+
-      # A is preferred:
+  # Handling for special versions such as "#head" or "#branch".
-      if ii > 0 and jj <= 0 and j == 0: return true
+  if ver.isSpecial or ver2.isSpecial:
-      if ii <= 0 and jj > 0 and i == 0: return false
+    if ver2.isSpecial and ($ver2).normalize == "#head":
-      # if A has no number left, but B has, B is preferred:  0.8 vs 0.8.3
+      return ($ver).normalize != "#head"
-      return jj > 0
+
-    if verA < verB: return true
+    if not ver2.isSpecial:
-    elif verA > verB: return false
+      # `#aa111 < 1.1`
-    # else: same version number; continue:
+      return ($ver).normalize != "#head"
-    inc i, ii
+
-    inc j, jj
+  # Handling for normal versions such as "0.1.0" or "1.0".
-    if a[i] == '.': inc i
+  var sVer = string(ver).split('.')
-    if b[j] == '.': inc j
+  var sVer2 = string(ver2).split('.')
+  for i in 0..max(sVer.len, sVer2.len)-1:
+    var sVerI = 0
+    if i < sVer.len:
+      discard parseInt(sVer[i], sVerI)
+    var sVerI2 = 0
+    if i < sVer2.len:
+      discard parseInt(sVer2[i], sVerI2)
+    if sVerI < sVerI2:
+      return true
+    elif sVerI == sVerI2:
+      discard
+    else:
+      return false
 
 proc addPackage(packages: StringTableRef, p: string) =
   let x = versionSplitPos(p)
   let name = p.substr(0, x-1)
-  let version = if x < p.len: p.substr(x+1) else: ""
+  let version = newVersion(if x < p.len: p.substr(x+1) else: "")
-  if packages.getOrDefault(name) <. version:
+  if packages.getOrDefault(name).newVersion < version or
-    packages[name] = version
+     (not packages.hasKey(name)):
+    packages[name] = $version
 
 iterator chosen(packages: StringTableRef): string =
   for key, val in pairs(packages):
-    let res = if val == latest: key else: key & '-' & val
+    let res = if val.len == 0: key else: key & '-' & val
     yield res
 
 proc addNimblePath(p: string, info: TLineInfo) =
-  if not contains(options.searchPaths, p):
+  var path = p
-    message(info, hintPath, p)
+  let nimbleLinks = toSeq(walkPattern(p / "*.nimble-link"))
-    options.lazyPaths.insert(p, 0)
+  if nimbleLinks.len > 0:
+    # If the user has more than one .nimble-link file then... we just ignore it.
+    # Spec for these files is available in Nimble's readme:
+    # https://github.com/nim-lang/nimble#nimble-link
+    let nimbleLinkLines = readFile(nimbleLinks[0]).splitLines()
+    path = nimbleLinkLines[1]
+    if not path.isAbsolute():
+      path = p / path
+
+  if not contains(options.searchPaths, path):
+    message(info, hintPath, path)
+    options.lazyPaths.insert(path, 0)
 
 proc addPathRec(dir: string, info: TLineInfo) =
   var packages = newStringTable(modeStyleInsensitive)
@@ -82,7 +107,17 @@ proc nimblePath*(path: string, info: TLineInfo) =
   addNimblePath(path, info)
 
 when isMainModule:
+  proc v(s: string): Version = s.newVersion
+  # #head is special in the sense that it's assumed to always be newest.
+  doAssert v"1.0" < v"#head"
+  doAssert v"1.0" < v"1.1"
+  doAssert v"1.0.1" < v"1.1"
+  doAssert v"1" < v"1.1"
+  doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer.
+  doAssert v"#a111" < v"#head"
+
   var rr = newStringTable()
+  addPackage rr, "irc-#a111"
   addPackage rr, "irc-#head"
   addPackage rr, "irc-0.1.0"
   addPackage rr, "irc"
@@ -93,5 +128,6 @@ when isMainModule:
   addPackage rr, "ab-0.1"
   addPackage rr, "justone"
 
-  for p in rr.chosen:
+  doAssert toSeq(rr.chosen) ==
-    echo p
+    @["irc-#head", "another-0.1", "ab-0.1.3", "justone"]
+

compiler/nimconf.nim

@@ -152,7 +152,7 @@ proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef) =
     parseDirective(L, tok, config)    # else: give the token to the parser
 
 proc checkSymbol(L: TLexer, tok: TToken) =
-  if tok.tokType notin {tkSymbol..pred(tkIntLit), tkStrLit..tkTripleStrLit}:
+  if tok.tokType notin {tkSymbol..tkInt64Lit, tkStrLit..tkTripleStrLit}:
     lexMessage(L, errIdentifierExpected, tokToStr(tok))
 
 proc parseAssignment(L: var TLexer, tok: var TToken; config: ConfigRef) =

compiler/options.nim

@@ -94,7 +94,7 @@ type
     cmdRun                    # run the project via TCC backend
   TStringSeq* = seq[string]
   TGCMode* = enum             # the selected GC
-    gcNone, gcBoehm, gcGo, gcStack, gcMarkAndSweep, gcRefc,
+    gcNone, gcBoehm, gcGo, gcRegions, gcMarkAndSweep, gcRefc,
     gcV2, gcGenerational
 
   IdeCmd* = enum
@@ -291,8 +291,8 @@ proc pathSubs*(p, config: string): string =
     "projectpath", options.gProjectPath,
     "projectdir", options.gProjectPath,
     "nimcache", getNimcacheDir()])
-  if '~' in result:
+  if "~/" in result:
-    result = result.replace("~", home)
+    result = result.replace("~/", home & '/')
 
 proc toGeneratedFile*(path, ext: string): string =
   ## converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod"

compiler/parser.nim

@@ -685,6 +685,11 @@ proc namedParams(p: var TParser, callee: PNode,
   # progress guaranteed
   exprColonEqExprListAux(p, endTok, result)
 
+proc commandParam(p: var TParser): PNode =
+  result = parseExpr(p)
+  if p.tok.tokType == tkDo:
+    result = postExprBlocks(p, result)
+
 proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
   #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks?
   #|       | doBlocks
@@ -733,7 +738,7 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
         when true:
           # progress NOT guaranteed
           p.hasProgress = false
-          addSon result, parseExpr(p)
+          addSon result, commandParam(p)
           if not p.hasProgress: break
         else:
           while p.tok.tokType != tkEof:
@@ -1253,14 +1258,12 @@ proc parseExprStmt(p: var TParser): PNode =
       while true:
         getTok(p)
         optInd(p, result)
-        var e = parseExpr(p)
+        addSon(result, commandParam(p))
-        addSon(result, e)
         if p.tok.tokType != tkComma: break
     elif p.tok.indent < 0 and isExprStart(p):
       result = newNode(nkCommand, a.info, @[a])
       while true:
-        var e = parseExpr(p)
+        addSon(result, commandParam(p))
-        addSon(result, e)
         if p.tok.tokType != tkComma: break
         getTok(p)
         optInd(p, result)

compiler/passes.nim

@@ -13,7 +13,7 @@
 import
   strutils, options, ast, astalgo, llstream, msgs, platform, os,
   condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
-  nimsets, syntaxes, times, rodread, idgen, modulegraphs
+  nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder
 
 type
   TPassContext* = object of RootObj # the pass's context
@@ -202,7 +202,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
         if graph.stopCompile(): break
         var n = parseTopLevelStmt(p)
         if n.kind == nkEmpty: break
-        if sfNoForward in module.flags:
+        if {sfNoForward, sfReorder} * module.flags != {}:
           # read everything, no streaming possible
           var sl = newNodeI(nkStmtList, n.info)
           sl.add n
@@ -210,6 +210,8 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
             var n = parseTopLevelStmt(p)
             if n.kind == nkEmpty: break
             sl.add n
+          if sfReorder in module.flags:
+            sl = reorder sl
           discard processTopLevelStmt(sl, a)
           break
         elif not processTopLevelStmt(n, a): break

compiler/platform.nim

@@ -21,8 +21,8 @@ type
                     # conditionals to condsyms (end of module).
     osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris,
     osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osAix, osPalmos, osQnx,
-    osAmiga, osAtari, osNetware, osMacos, osMacosx, osHaiku, osVxworks, osGenode
+    osAmiga, osAtari, osNetware, osMacos, osMacosx, osHaiku, osAndroid, osVxworks
-    osJS, osNimrodVM, osStandalone
+    osGenode, osJS, osNimrodVM, osStandalone
 
 type
   TInfoOSProp* = enum
@@ -143,6 +143,10 @@ const
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
       props: {ospNeedsPIC, ospPosix, ospLacksThreadVars}),
+     (name: "Android", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
+      objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
+      scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
+      props: {ospNeedsPIC, ospPosix}),
      (name: "VxWorks", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
       objExt: ".o", newLine: "\x0A", pathSep: ";", dirSep: "\\",
       scriptExt: ".sh", curDir: ".", exeExt: ".vxe", extSep: ".",
@@ -171,7 +175,8 @@ type
                      # alias conditionals to condsyms (end of module).
     cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64,
     cpuPowerpc64el, cpuSparc, cpuVm, cpuIa64, cpuAmd64, cpuMips, cpuMipsel,
-    cpuArm, cpuArm64, cpuJS, cpuNimrodVM, cpuAVR, cpuMSP430, cpuSparc64
+    cpuArm, cpuArm64, cpuJS, cpuNimrodVM, cpuAVR, cpuMSP430, cpuSparc64,
+    cpuMips64, cpuMips64el
 
 type
   TEndian* = enum
@@ -200,7 +205,9 @@ const
     (name: "nimrodvm", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
     (name: "avr", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16),
     (name: "msp430", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16),
-    (name: "sparc64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64)]
+    (name: "sparc64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64),
+    (name: "mips64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64),
+    (name: "mips64el", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)]
 
 var
   targetCPU*, hostCPU*: TSystemCPU

compiler/pragmas.nim

@@ -45,7 +45,7 @@ const
     wFatal, wDefine, wUndef, wCompile, wLink, wLinksys, wPure, wPush, wPop,
     wBreakpoint, wWatchPoint, wPassl, wPassc, wDeadCodeElim, wDeprecated,
     wFloatchecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll,
-    wLinearScanEnd, wPatterns, wEffects, wNoForward, wComputedGoto,
+    wLinearScanEnd, wPatterns, wEffects, wNoForward, wReorder, wComputedGoto,
     wInjectStmt, wDeprecated, wExperimental, wThis}
   lambdaPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl,
     wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader,
@@ -210,9 +210,9 @@ proc pragmaDeadCodeElim(c: PContext, n: PNode) =
   if isTurnedOn(c, n): incl(c.module.flags, sfDeadCodeElim)
   else: excl(c.module.flags, sfDeadCodeElim)
 
-proc pragmaNoForward(c: PContext, n: PNode) =
+proc pragmaNoForward(c: PContext, n: PNode; flag=sfNoForward) =
-  if isTurnedOn(c, n): incl(c.module.flags, sfNoForward)
+  if isTurnedOn(c, n): incl(c.module.flags, flag)
-  else: excl(c.module.flags, sfNoForward)
+  else: excl(c.module.flags, flag)
 
 proc processCallConv(c: PContext, n: PNode) =
   if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent):
@@ -726,6 +726,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         incl(sym.flags, sfThread)
       of wDeadCodeElim: pragmaDeadCodeElim(c, it)
       of wNoForward: pragmaNoForward(c, it)
+      of wReorder: pragmaNoForward(c, it, sfReorder)
       of wMagic: processMagic(c, it, sym)
       of wCompileTime:
         noVal(it)

compiler/renderer.nim

@@ -1327,7 +1327,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     
     if n.hasExplicitParams:
       put(g, tkBracketLe, "[")
-      gcomma(g, n)
+      gsemicolon(g, n)
       put(g, tkBracketRi, "]")
   of nkFormalParams:
     put(g, tkParLe, "(")

compiler/reorder.nim

@@ -0,0 +1,102 @@
+
+import intsets, tables, ast, idents, renderer
+
+const
+  nfTempMark = nfTransf
+  nfPermMark = nfNoRewrite
+
+proc accQuoted(n: PNode): PIdent =
+  var id = ""
+  for i in 0 .. <n.len:
+    let x = n[i]
+    case x.kind
+    of nkIdent: id.add(x.ident.s)
+    of nkSym: id.add(x.sym.name.s)
+    else: discard
+  result = getIdent(id)
+
+proc addDecl(n: PNode; declares: var IntSet) =
+  case n.kind
+  of nkPostfix: addDecl(n[1], declares)
+  of nkPragmaExpr: addDecl(n[0], declares)
+  of nkIdent:
+    declares.incl n.ident.id
+  of nkSym:
+    declares.incl n.sym.name.id
+  of nkAccQuoted:
+    declares.incl accQuoted(n).id
+  else: discard
+
+proc computeDeps(n: PNode, declares, uses: var IntSet; topLevel: bool) =
+  template deps(n) = computeDeps(n, declares, uses, false)
+  template decl(n) =
+    if topLevel: addDecl(n, declares)
+  case n.kind
+  of procDefs:
+    decl(n[0])
+    for i in 1..bodyPos: deps(n[i])
+  of nkLetSection, nkVarSection, nkUsingStmt:
+    for a in n:
+      if a.kind in {nkIdentDefs, nkVarTuple}:
+        for j in countup(0, a.len-3): decl(a[j])
+        for j in a.len-2..a.len-1: deps(a[j])
+  of nkConstSection, nkTypeSection:
+    for a in n:
+      if a.len >= 3:
+        decl(a[0])
+        for i in 1..<a.len: deps(a[i])
+  of nkIdent: uses.incl n.ident.id
+  of nkSym: uses.incl n.sym.name.id
+  of nkAccQuoted: uses.incl accQuoted(n).id
+  of nkOpenSymChoice, nkClosedSymChoice:
+    uses.incl n.sons[0].sym.name.id
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse:
+    for i in 0..<len(n): computeDeps(n[i], declares, uses, topLevel)
+  else:
+    for i in 0..<safeLen(n): deps(n[i])
+
+proc visit(i: int; all, res: PNode; deps: var seq[(IntSet, IntSet)]): bool =
+  let n = all[i]
+  if nfTempMark in n.flags:
+    # not a DAG!
+    return true
+  if nfPermMark notin n.flags:
+    incl n.flags, nfTempMark
+    var uses = deps[i][1]
+    for j in 0..<all.len:
+      if j != i:
+        let declares = deps[j][0]
+        for d in declares:
+          if uses.contains(d):
+            let oldLen = res.len
+            if visit(j, all, res, deps):
+              result = true
+              # rollback what we did, it turned out to be a dependency that caused
+              # trouble:
+              for k in oldLen..<res.len:
+                res.sons[k].flags = res.sons[k].flags - {nfPermMark, nfTempMark}
+              if oldLen != res.len: res.sons.setLen oldLen
+            break
+    n.flags = n.flags + {nfPermMark} - {nfTempMark}
+    res.add n
+
+proc reorder*(n: PNode): PNode =
+  result = newNodeI(nkStmtList, n.info)
+  var deps = newSeq[(IntSet, IntSet)](n.len)
+  for i in 0..<n.len:
+    deps[i][0] = initIntSet()
+    deps[i][1] = initIntSet()
+    computeDeps(n[i], deps[i][0], deps[i][1], true)
+
+  for i in 0 .. n.len-1:
+    discard visit(i, n, result, deps)
+  for i in 0..<result.len:
+    result.sons[i].flags = result.sons[i].flags - {nfTempMark, nfPermMark}
+  when false:
+    # reverse the result:
+    let L = result.len-1
+    for i in 0 .. result.len div 2:
+      result.sons[i].flags = result.sons[i].flags - {nfTempMark, nfPermMark}
+      result.sons[L - i].flags = result.sons[L - i].flags - {nfTempMark, nfPermMark}
+      swap(result.sons[i], result.sons[L - i])
+  #echo result

compiler/ropes.nim

@@ -228,6 +228,7 @@ proc prepend*(a: var Rope, b: string) = a = b & a
 var
   rnl* = tnl.newRope
   softRnl* = tnl.newRope
+  noRnl* = "".newRope
 
 proc `%`*(frmt: FormatStr, args: openArray[Rope]): Rope =
   var i = 0

compiler/semexprs.nim

@@ -47,8 +47,6 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     #raiseRecoverableError("")
     result = errorNode(c, n)
   if result.typ == nil or result.typ == enforceVoidContext:
-    if n.kind != nkStmtList:
-      # we cannot check for 'void' in macros ...
     localError(n.info, errExprXHasNoType,
                 renderTree(result, {renderNoComments}))
     result.typ = errorType(c)
@@ -276,41 +274,6 @@ proc semSizeof(c: PContext, n: PNode): PNode =
   n.typ = getSysType(tyInt)
   result = n
 
-proc semOf(c: PContext, n: PNode): PNode =
-  if sonsLen(n) == 3:
-    n.sons[1] = semExprWithType(c, n.sons[1])
-    n.sons[2] = semExprWithType(c, n.sons[2], {efDetermineType})
-    #restoreOldStyleType(n.sons[1])
-    #restoreOldStyleType(n.sons[2])
-    let a = skipTypes(n.sons[1].typ, abstractPtrs)
-    let b = skipTypes(n.sons[2].typ, abstractPtrs)
-    let x = skipTypes(n.sons[1].typ, abstractPtrs-{tyTypeDesc})
-    let y = skipTypes(n.sons[2].typ, abstractPtrs-{tyTypeDesc})
-
-    if x.kind == tyTypeDesc or y.kind != tyTypeDesc:
-      localError(n.info, errXExpectsObjectTypes, "of")
-    elif b.kind != tyObject or a.kind != tyObject:
-      localError(n.info, errXExpectsObjectTypes, "of")
-    else:
-      let diff = inheritanceDiff(a, b)
-      # | returns: 0 iff `a` == `b`
-      # | returns: -x iff `a` is the x'th direct superclass of `b`
-      # | returns: +x iff `a` is the x'th direct subclass of `b`
-      # | returns: `maxint` iff `a` and `b` are not compatible at all
-      if diff <= 0:
-        # optimize to true:
-        message(n.info, hintConditionAlwaysTrue, renderTree(n))
-        result = newIntNode(nkIntLit, 1)
-        result.info = n.info
-        result.typ = getSysType(tyBool)
-        return result
-      elif diff == high(int):
-        localError(n.info, errXcanNeverBeOfThisSubtype, typeToString(a))
-  else:
-    localError(n.info, errXExpectsTwoArguments, "of")
-  n.typ = getSysType(tyBool)
-  result = n
-
 proc isOpImpl(c: PContext, n: PNode, flags: TExprFlags): PNode =
   internalAssert n.sonsLen == 3 and
     n[1].typ != nil and n[1].typ.kind == tyTypeDesc and
@@ -1121,9 +1084,11 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
       if ty.n != nil and ty.n.kind == nkRecList:
         let field = lookupInRecord(ty.n, i)
         if field != nil:
-          n.typ = newTypeWithSons(c, tyFieldAccessor, @[ty, field.typ])
+          n.typ = makeTypeDesc(c, field.typ)
-          n.typ.n = copyTree(n)
           return n
+          #n.typ = newTypeWithSons(c, tyFieldAccessor, @[ty, field.typ])
+          #n.typ.n = copyTree(n)
+          #return n
     else:
       tryReadingGenericParam(ty)
       return
@@ -1490,6 +1455,9 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
   var t = skipTypes(restype, {tyGenericInst, tyAlias})
   case t.kind
   of tyVar:
+    if n.sons[0].kind in {nkHiddenStdConv, nkHiddenSubConv}:
+      n.sons[0] = n.sons[0].sons[1]
+
     n.sons[0] = takeImplicitAddr(c, n.sons[0])
   of tyTuple:
     for i in 0.. <t.sonsLen:
@@ -1829,11 +1797,11 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
   of mDefined: result = semDefined(c, setMs(n, s), false)
   of mDefinedInScope: result = semDefined(c, setMs(n, s), true)
   of mCompiles: result = semCompiles(c, setMs(n, s), flags)
-  of mLow: result = semLowHigh(c, setMs(n, s), mLow)
+  #of mLow: result = semLowHigh(c, setMs(n, s), mLow)
-  of mHigh: result = semLowHigh(c, setMs(n, s), mHigh)
+  #of mHigh: result = semLowHigh(c, setMs(n, s), mHigh)
   of mSizeOf: result = semSizeof(c, setMs(n, s))
   of mIs: result = semIs(c, setMs(n, s), flags)
-  of mOf: result = semOf(c, setMs(n, s))
+  #of mOf: result = semOf(c, setMs(n, s))
   of mShallowCopy: result = semShallowCopy(c, n, flags)
   of mExpandToAst: result = semExpandToAst(c, n, s, flags)
   of mQuoteAst: result = semQuoteAst(c, n)
@@ -2207,9 +2175,14 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     message(n.info, warnDeprecated, "bind")
     result = semExpr(c, n.sons[0], flags)
   of nkTypeOfExpr, nkTupleTy, nkTupleClassTy, nkRefTy..nkEnumTy, nkStaticTy:
+    if c.matchedConcept != nil and n.len == 1:
+      let modifier = n.modifierTypeKindOfNode
+      if modifier != tyNone:
+        var baseType = semExpr(c, n[0]).typ.skipTypes({tyTypeDesc})
+        result.typ = c.makeTypeDesc(c.newTypeWithSons(modifier, @[baseType]))
+        return
     var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc})
     result.typ = makeTypeDesc(c, typ)
-    #result = symNodeFromType(c, typ, n.info)
   of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit:
     # check if it is an expression macro:
     checkMinSonsLen(n, 1)

compiler/semmagic.nim

@@ -200,6 +200,41 @@ proc isStrangeArray(t: PType): bool =
   let t = t.skipTypes(abstractInst)
   result = t.kind == tyArray and t.firstOrd != 0
 
+proc semOf(c: PContext, n: PNode): PNode =
+  if sonsLen(n) == 3:
+    n.sons[1] = semExprWithType(c, n.sons[1])
+    n.sons[2] = semExprWithType(c, n.sons[2], {efDetermineType})
+    #restoreOldStyleType(n.sons[1])
+    #restoreOldStyleType(n.sons[2])
+    let a = skipTypes(n.sons[1].typ, abstractPtrs)
+    let b = skipTypes(n.sons[2].typ, abstractPtrs)
+    let x = skipTypes(n.sons[1].typ, abstractPtrs-{tyTypeDesc})
+    let y = skipTypes(n.sons[2].typ, abstractPtrs-{tyTypeDesc})
+
+    if x.kind == tyTypeDesc or y.kind != tyTypeDesc:
+      localError(n.info, errXExpectsObjectTypes, "of")
+    elif b.kind != tyObject or a.kind != tyObject:
+      localError(n.info, errXExpectsObjectTypes, "of")
+    else:
+      let diff = inheritanceDiff(a, b)
+      # | returns: 0 iff `a` == `b`
+      # | returns: -x iff `a` is the x'th direct superclass of `b`
+      # | returns: +x iff `a` is the x'th direct subclass of `b`
+      # | returns: `maxint` iff `a` and `b` are not compatible at all
+      if diff <= 0:
+        # optimize to true:
+        message(n.info, hintConditionAlwaysTrue, renderTree(n))
+        result = newIntNode(nkIntLit, 1)
+        result.info = n.info
+        result.typ = getSysType(tyBool)
+        return result
+      elif diff == high(int):
+        localError(n.info, errXcanNeverBeOfThisSubtype, typeToString(a))
+  else:
+    localError(n.info, errXExpectsTwoArguments, "of")
+  n.typ = getSysType(tyBool)
+  result = n
+
 proc magicsAfterOverloadResolution(c: PContext, n: PNode,
                                    flags: TExprFlags): PNode =
   case n[0].sym.magic
@@ -219,6 +254,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     result.typ = getSysType(tyString)
   of mInstantiationInfo: result = semInstantiationInfo(c, n)
   of mOrd: result = semOrd(c, n)
+  of mOf: result = semOf(c, n)
   of mHigh, mLow: result = semLowHigh(c, n, n[0].sym.magic)
   of mShallowCopy: result = semShallowCopy(c, n, flags)
   of mNBindSym: result = semBindSym(c, n)

compiler/semobjconstr.nim

@@ -44,7 +44,9 @@ proc locateFieldInInitExpr(field: PSym, initExpr: PNode): PNode =
   let fieldId = field.name.id
   for i in 1 .. <initExpr.len:
     let assignment = initExpr[i]
-    internalAssert assignment.kind == nkExprColonExpr
+    if assignment.kind != nkExprColonExpr:
+      localError(initExpr.info, "incorrect object construction syntax")
+      continue
 
     if fieldId == considerQuotedIdent(assignment[0]).id:
       return assignment
@@ -278,6 +280,9 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
   for i in 1.. <result.len:
     let field = result[i]
     if nfSem notin field.flags:
+      if field.kind != nkExprColonExpr:
+        localError(n.info, "incorrect object construction syntax")
+        continue
       let id = considerQuotedIdent(field[0])
       # This node was not processed. There are two possible reasons:
       # 1) It was shadowed by a field with the same name on the left

compiler/semtempl.nim

@@ -12,14 +12,14 @@
 discard """
   hygienic templates:
 
-    template `||` (a, b: expr): expr =
+    template `||` (a, b: untyped): untyped =
       let aa = a
       if aa: aa else: b
 
     var
       a, b: T
 
-    a || b || a
+    echo a || b || a
 
   Each evaluation context has to be different and we need to perform
   some form of preliminary symbol lookup in template definitions. Hygiene is

compiler/semtypes.nim

@@ -1202,6 +1202,15 @@ proc freshType(res, prev: PType): PType {.inline.} =
   else:
     result = res
 
+template modifierTypeKindOfNode(n: PNode): TTypeKind =
+  case n.kind
+  of nkVarTy: tyVar
+  of nkRefTy: tyRef
+  of nkPtrTy: tyPtr
+  of nkStaticTy: tyStatic
+  of nkTypeOfExpr: tyTypeDesc
+  else: tyNone
+
 proc semTypeClass(c: PContext, n: PNode, prev: PType): PType =
   # if n.sonsLen == 0: return newConstraint(c, tyTypeClass)
   if nfBase2 in n.flags:
@@ -1227,13 +1236,7 @@ proc semTypeClass(c: PContext, n: PNode, prev: PType): PType =
       dummyName: PNode
       dummyType: PType
 
-    let modifier = case param.kind
+    let modifier = param.modifierTypeKindOfNode
-      of nkVarTy: tyVar
-      of nkRefTy: tyRef
-      of nkPtrTy: tyPtr
-      of nkStaticTy: tyStatic
-      of nkTypeOfExpr: tyTypeDesc
-      else: tyNone
 
     if modifier != tyNone:
       dummyName = param[0]
@@ -1509,9 +1512,26 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     dec c.inTypeContext
 
 proc setMagicType(m: PSym, kind: TTypeKind, size: int) =
+  # source : https://en.wikipedia.org/wiki/Data_structure_alignment#x86
   m.typ.kind = kind
-  m.typ.align = size.int16
   m.typ.size = size
+  # this usually works for most basic types
+  # Assuming that since ARM, ARM64  don't support unaligned access
+  # data is aligned to type size
+  m.typ.align = size.int16
+
+  # FIXME: proper support for clongdouble should be added.
+  # long double size can be 8, 10, 12, 16 bytes depending on platform & compiler
+  if targetCPU == cpuI386 and size == 8:
+    #on Linux/BSD i386, double are aligned to 4bytes (except with -malign-double)
+    if kind in {tyFloat64, tyFloat} and
+       targetOS in {osLinux, osAndroid, osNetbsd, osFreebsd, osOpenbsd, osDragonfly}:
+      m.typ.align = 4
+    # on i386, all known compiler, 64bits ints are aligned to 4bytes (except with -malign-double)
+    elif kind in {tyInt, tyUInt, tyInt64, tyUInt64}:
+      m.typ.align = 4
+  else:
+    discard
 
 proc processMagicType(c: PContext, m: PSym) =
   case m.magic

compiler/sigmatch.nim

@@ -297,7 +297,9 @@ proc describeArgs*(c: PContext, n: PNode, startIdx = 1;
         n.sons[i].typ = arg.typ
         n.sons[i].sons[1] = arg
     else:
-      if arg.typ.isNil and arg.kind notin {nkStmtList, nkDo}:
+      if arg.typ.isNil and arg.kind notin {nkStmtList, nkDo, nkElse,
+                                           nkOfBranch, nkElifBranch,
+                                           nkExceptBranch}:
         arg = c.semOperand(c, n.sons[i])
         n.sons[i] = arg
     if arg.typ != nil and arg.typ.kind == tyError: return

compiler/testability.nim

@@ -1,5 +0,0 @@
-template tests*(body: stmt) {.immediate.} =
-  when defined(selftest):
-    when not declared(unittest): import unittest
-    body
-

compiler/vm.nim

@@ -409,6 +409,28 @@ proc recSetFlagIsRef(arg: PNode) =
   for i in 0 ..< arg.safeLen:
     arg.sons[i].recSetFlagIsRef
 
+proc setLenSeq(c: PCtx; node: PNode; newLen: int; info: TLineInfo) =
+  # FIXME: this doesn't attempt to solve incomplete
+  # support of tyPtr, tyRef in VM.
+  let typ = node.typ.skipTypes(abstractInst+{tyRange}-{tyTypeDesc})
+  let typeEntry = typ.sons[0].skipTypes(abstractInst+{tyRange}-{tyTypeDesc})
+  let typeKind = case typeEntry.kind
+  of tyUInt..tyUInt64: nkUIntLit
+  of tyRange, tyEnum, tyBool, tyChar, tyInt..tyInt64: nkIntLit
+  of tyFloat..tyFloat128: nkFloatLit
+  of tyString: nkStrLit
+  of tyObject: nkObjConstr
+  of tySequence: nkNilLit
+  of tyProc, tyTuple: nkPar
+  else: nkEmpty
+
+  let oldLen = node.len
+  setLen(node.sons, newLen)
+  if oldLen < newLen:
+    # TODO: This is still not correct for tyPtr, tyRef default value
+    for i in oldLen .. <newLen:
+      node.sons[i] = newNodeI(typeKind, info)
+
 proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
   var pc = start
   var tos = tos
@@ -1118,14 +1140,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeB(rkNode)
       let newLen = regs[rb].intVal.int
       if regs[ra].node.isNil: stackTrace(c, tos, pc, errNilAccess)
-      else:
+      else: c.setLenSeq(regs[ra].node, newLen, c.debug[pc])
-        let oldLen = regs[ra].node.len
-        setLen(regs[ra].node.sons, newLen)
-        if oldLen < newLen:
-          # XXX This is still not entirely correct
-          # set to default value:
-          for i in oldLen .. <newLen:
-            regs[ra].node.sons[i] = newNodeI(nkEmpty, c.debug[pc])
     of opcReset:
       internalError(c.debug[pc], "too implement")
     of opcNarrowS:
@@ -1307,12 +1322,24 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       ensureKind(rkNode)
       if c.callsite != nil: regs[ra].node = c.callsite
       else: stackTrace(c, tos, pc, errFieldXNotFound, "callsite")
-    of opcNLineInfo:
+    of opcNGetFile:
       decodeB(rkNode)
       let n = regs[rb].node
-      createStr regs[ra]
+      regs[ra].node = newStrNode(nkStrLit, n.info.toFilename)
-      regs[ra].node.strVal = n.info.toFileLineCol
+      regs[ra].node.info = n.info
-      regs[ra].node.info = c.debug[pc]
+      regs[ra].node.typ = n.typ
+    of opcNGetLine:
+      decodeB(rkNode)
+      let n = regs[rb].node
+      regs[ra].node = newIntNode(nkIntLit, n.info.line)
+      regs[ra].node.info = n.info
+      regs[ra].node.typ = n.typ
+    of opcNGetColumn:
+      decodeB(rkNode)
+      let n = regs[rb].node
+      regs[ra].node = newIntNode(nkIntLit, n.info.col)
+      regs[ra].node.info = n.info
+      regs[ra].node.typ = n.typ
     of opcEqIdent:
       decodeBC(rkInt)
       if regs[rb].node.kind == nkIdent and regs[rc].node.kind == nkIdent:
@@ -1471,6 +1498,17 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       createStrKeepNode(regs[ra])
       if regs[ra].node.strVal.isNil: regs[ra].node.strVal = newStringOfCap(1000)
       storeAny(regs[ra].node.strVal, typ, regs[rb].regToNode)
+    of opcToNarrowInt:
+      decodeBC(rkInt)
+      let mask = (1'i64 shl rc) - 1 # 0xFF
+      let signbit = 1'i64 shl (rc - 1) # 0x80
+      let toggle = mask - signbit # 0x7F
+      # algorithm: -((i8 and 0xFF) xor 0x7F) + 0x7F
+      # mask off higher bits.
+      # uses two's complement to sign-extend integer.
+      # reajust integer into desired range.
+      regs[ra].intVal = -((regs[rb].intVal and mask) xor toggle) + toggle
+
     inc pc
 
 proc execute(c: PCtx, start: int): PNode =

compiler/vmdef.nim

@@ -98,7 +98,7 @@ type
     opcNError,
     opcNWarning,
     opcNHint,
-    opcNLineInfo,
+    opcNGetLine, opcNGetColumn, opcNGetFile,
     opcEqIdent,
     opcStrToIdent,
     opcIdentToStr,
@@ -136,7 +136,8 @@ type
     opcNBindSym,
     opcSetType,   # dest.typ = types[Bx]
     opcTypeTrait,
-    opcMarshalLoad, opcMarshalStore
+    opcMarshalLoad, opcMarshalStore,
+    opcToNarrowInt
 
   TBlock* = object
     label*: PSym

compiler/vmdeps.nim

@@ -224,12 +224,13 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
       result.add copyTree(c)
   of tyTuple:
     if inst:
-      result = newNodeX(nkTupleTy)
       # only named tuples have a node, unnamed tuples don't
       if t.n.isNil:
+        result = newNodeX(nkPar)
         for subType in t.sons:
           result.add mapTypeToAst(subType, info)
       else:
+        result = newNodeX(nkTupleTy)
         for s in t.n.sons:
           result.add newIdentDefs(s)
     else:

compiler/vmgen.nim

@@ -656,16 +656,17 @@ proc genNarrow(c: PCtx; n: PNode; dest: TDest) =
   let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
   # uint is uint64 in the VM, we we only need to mask the result for
   # other unsigned types:
-  if t.kind in {tyUInt8..tyUInt32}:
+  if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8):
     c.gABC(n, opcNarrowU, dest, TRegister(t.size*8))
-  elif t.kind in {tyInt8..tyInt32}:
+  elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and t.size < 8):
     c.gABC(n, opcNarrowS, dest, TRegister(t.size*8))
 
 proc genNarrowU(c: PCtx; n: PNode; dest: TDest) =
   let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
   # uint is uint64 in the VM, we we only need to mask the result for
   # other unsigned types:
-  if t.kind in {tyUInt8..tyUInt32, tyInt8..tyInt32}:
+  if t.kind in {tyUInt8..tyUInt32, tyInt8..tyInt32} or
+    (t.kind in {tyUInt, tyInt} and t.size < 8):
     c.gABC(n, opcNarrowU, dest, TRegister(t.size*8))
 
 proc genBinaryABCnarrow(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
@@ -875,11 +876,25 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mBitnotI:
     genUnaryABC(c, n, dest, opcBitnotInt)
     genNarrowU(c, n, dest)
-  of mZe8ToI, mZe8ToI64, mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64,
+  of mToFloat, mToBiggestFloat, mToInt,
-     mToU8, mToU16, mToU32, mToFloat, mToBiggestFloat, mToInt,
      mToBiggestInt, mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr,
      mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr:
     genConv(c, n, n.sons[1], dest)
+  of mZe8ToI, mZe8ToI64, mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64:
+    #genNarrowU modified
+    let t = skipTypes(n.sons[1].typ, abstractVar-{tyTypeDesc})
+    let tmp = c.genx(n.sons[1])
+    c.gABC(n, opcNarrowU, tmp, TRegister(t.size*8))
+    # assign result to dest register
+    if dest < 0: dest = c.getTemp(n.typ)
+    c.gABC(n, opcAsgnInt, dest, tmp)
+    c.freeTemp(tmp)
+  of mToU8, mToU16, mToU32:
+    let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
+    var tmp = c.genx(n.sons[1])
+    if dest < 0: dest = c.getTemp(n.typ)
+    c.gABC(n, opcToNarrowInt, dest, tmp, TRegister(t.size*8))
+    c.freeTemp(tmp)
   of mEqStr, mEqCString: genBinaryABC(c, n, dest, opcEqStr)
   of mLeStr: genBinaryABC(c, n, dest, opcLeStr)
   of mLtStr: genBinaryABC(c, n, dest, opcLtStr)
@@ -1071,7 +1086,16 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mEqIdent: genBinaryABC(c, n, dest, opcEqIdent)
   of mEqNimrodNode: genBinaryABC(c, n, dest, opcEqNimrodNode)
   of mSameNodeType: genBinaryABC(c, n, dest, opcSameNodeType)
-  of mNLineInfo: genUnaryABC(c, n, dest, opcNLineInfo)
+  of mNLineInfo:
+    case n[0].sym.name.s
+    of "getFile":
+      genUnaryABC(c, n, dest, opcNGetFile)
+    of "getLine":
+      genUnaryABC(c, n, dest, opcNGetLine)
+    of "getColumn":
+      genUnaryABC(c, n, dest, opcNGetColumn)
+    else:
+      internalAssert false
   of mNHint:
     unused(n, dest)
     genUnaryStmt(c, n, opcNHint)

compiler/wordrecg.nim

@@ -55,7 +55,7 @@ type
     wFloatchecks, wNanChecks, wInfChecks,
     wAssertions, wPatterns, wWarnings,
     wHints, wOptimization, wRaises, wWrites, wReads, wSize, wEffects, wTags,
-    wDeadCodeElim, wSafecode, wNoForward, wNoRewrite,
+    wDeadCodeElim, wSafecode, wNoForward, wReorder, wNoRewrite,
     wPragma,
     wCompileTime, wNoInit,
     wPassc, wPassl, wBorrow, wDiscardable,
@@ -143,7 +143,7 @@ const
 
     "assertions", "patterns", "warnings", "hints",
     "optimization", "raises", "writes", "reads", "size", "effects", "tags",
-    "deadcodeelim", "safecode", "noforward", "norewrite",
+    "deadcodeelim", "safecode", "noforward", "reorder", "norewrite",
     "pragma",
     "compiletime", "noinit",
     "passc", "passl", "borrow", "discardable", "fieldchecks",

config/nim.cfg

@@ -90,6 +90,18 @@ path="$lib/pure"
   @end
 @end
 
+@if android:
+  cc = clang
+  @if termux:
+    gcc.options.linker = "-landroid-glob"
+    gcc.cpp.options.linker = "-landroid-glob"
+    clang.options.linker = "-landroid-glob"
+    clang.cpp.options.linker = "-landroid-glob"
+    tcc.options.linker = "-landroid-glob"
+    define:"useShPath:/system/bin/sh"
+  @end
+@end
+
 # Configuration for the Intel C/C++ compiler:
 @if windows:
   icl.options.speed = "/Ox /arch:SSE2"
@@ -109,6 +121,9 @@ path="$lib/pure"
   tlsEmulation:on
   gcc.options.always = "-w"
   gcc.cpp.options.always = "-w -fpermissive"
+@elif windows:
+  gcc.options.always = "-w -mno-ms-bitfields"
+  gcc.cpp.options.always = "-w -fpermissive -mno-ms-bitfields"
 @else:
   gcc.options.always = "-w"
   gcc.cpp.options.always = "-w -fpermissive"
@@ -181,26 +196,38 @@ clang.options.size = "-Os"
 
 # Configuration for the Visual C/C++ compiler:
 vcc.exe =     "vccexe.exe"
+vcc.cpp.exe = "vccexe.exe"
 vcc.linkerexe =     "vccexe.exe"
+vcc.cpp.linkerexe = "vccexe.exe"
 
 # set the options for specific platforms:
-@if i386:
-vcc.options.always = "--platform:x86 /nologo"
-vcc.options.linker = "--platform:x86 /nologo /DEBUG /Zi /F33554432" # set the stack size to 32 MiB
-@elif amd64:
-vcc.options.always = "--platform:amd64 /nologo"
-vcc.options.linker = "--platform:amd64 /nologo /DEBUG /Zi /F33554432" # set the stack size to 32 MiB
-@elif arm:
-vcc.options.always = "--platform:arm /nologo"
-vcc.options.linker = "--platform:arm /nologo /DEBUG /Zi /F33554432" # set the stack size to 32 MiB
-@else:
 vcc.options.always =      "/nologo"
+vcc.cpp.options.always %=  "${vcc.options.always} /EHsc"
 vcc.options.linker =      "/nologo /DEBUG /Zi /F33554432" # set the stack size to 32 MiB
+vcc.cpp.options.linker %=  "${vcc.options.linker}"
+@if i386:
+vcc.options.always %=      "--platform:x86 ${vcc.options.always}"
+vcc.cpp.options.always %=  "--platform:x86 ${vcc.cpp.options.always}"
+vcc.options.linker %=      "--platform:x86 ${vcc.options.linker}"
+vcc.cpp.options.linker %=  "--platform:x86 ${vcc.cpp.options.linker}"
+@elif amd64:
+vcc.options.always %=      "--platform:amd64 ${vcc.options.always}"
+vcc.cpp.options.always %=  "--platform:amd64 ${vcc.cpp.options.always}"
+vcc.options.linker %=      "--platform:amd64 ${vcc.options.linker}"
+vcc.cpp.options.linker %=  "--platform:amd64 ${vcc.cpp.options.linker}"
+@elif arm:
+vcc.options.always %=      "--platform:arm ${vcc.options.always}"
+vcc.cpp.options.always %=  "--platform:arm ${vcc.cpp.options.always}"
+vcc.options.linker %=      "--platform:arm ${vcc.options.linker}"
+vcc.cpp.options.linker %=  "--platform:arm ${vcc.cpp.options.linker}"
 @end
 
 vcc.options.debug =     "/Zi /FS /Od"
+vcc.cpp.options.debug = "/Zi /FS /Od"
 vcc.options.speed =     "/O2"
+vcc.cpp.options.speed = "/O2"
 vcc.options.size =     "/O1"
+vcc.cpp.options.size = "/O1"
 
 # Configuration for the Tiny C Compiler:
 tcc.options.always = "-w"

doc/advopt.txt

@@ -65,7 +65,7 @@ Advanced options:
   --skipUserCfg             do not read the user's configuration file
   --skipParentCfg           do not read the parent dirs' configuration files
   --skipProjCfg             do not read the project's configuration file
-  --gc:refc|v2|markAndSweep|boehm|go|none
+  --gc:refc|v2|markAndSweep|boehm|go|none|regions
                             select the GC to use; default is 'refc'
   --index:on|off            turn index file generation on|off
   --putenv:key=value        set an environment variable

doc/lib.rst

@@ -539,9 +539,6 @@ Network Programming and Internet Protocols
 * `joyent_http_parser <joyent_http_parser.html>`_
   Wrapper for the joyent's high-performance HTTP parser.
 
-* `libcurl <libcurl.html>`_
-  Wrapper for the libcurl library.
-
 * `openssl <openssl.html>`_
   Wrapper for OpenSSL.
 

doc/manual/ffi.txt

@@ -132,7 +132,7 @@ translated into a C array of undetermined size:
 
 .. code-block:: nim
   type
-    ArrayPart{.unchecked.} = array[0..0, int]
+    ArrayPart{.unchecked.} = array[0, int]
     MySeq = object
       len, cap: int
       data: ArrayPart
@@ -146,10 +146,6 @@ Produces roughly this C code:
     NI data[];
   } MySeq;
 
-The bounds checking done at compile time is not disabled for now, so to access
-``s.data[C]`` (where ``C`` is a constant) the array's index needs to
-include ``C``.
-
 The base type of the unchecked array may not contain any GC'ed memory but this
 is currently not checked.
 

doc/manual/lexing.txt

@@ -130,7 +130,7 @@ Two identifiers are considered equal if the following algorithm returns true:
 .. code-block:: nim
   proc sameIdentifier(a, b: string): bool =
     a[0] == b[0] and
-      a.replace(re"_|–", "").toLower == b.replace(re"_|–", "").toLower
+      a.replace("_", "").toLower == b.replace("_", "").toLower
 
 That means only the first letters are compared in a case sensitive manner. Other
 letters are compared case insensitively and underscores are ignored.

doc/manual/procs.txt

@@ -248,12 +248,20 @@ calls can use the ``do`` keyword:
 .. code-block:: nim
   sort(cities) do (x,y: string) -> int:
     cmp(x.len, y.len)
+
   # Less parenthesis using the method plus command syntax:
   cities = cities.map do (x:string) -> string:
     "City of " & x
 
+  # In macros, the do notation is often used for quasi-quoting
+  macroResults.add quote do:
+    if not `ex`:
+      echo `info`, ": Check failed: ", `expString`
+
 ``do`` is written after the parentheses enclosing the regular proc params.
 The proc expression represented by the do block is appended to them.
+In calls using the command syntax, the do block will bind to the immediately
+preceeding expression, transforming it in a call.
 
 ``do`` with parentheses is an anonymous ``proc``; however a ``do`` without
 parentheses is just a block of code. The ``do`` notation can be used to
@@ -275,8 +283,8 @@ Nonoverloadable builtins
 The following builtin procs cannot be overloaded for reasons of implementation
 simplicity (they require specialized semantic checking)::
 
-  declared, defined, definedInScope, compiles, low, high, sizeOf,
+  declared, defined, definedInScope, compiles, sizeOf,
-  is, of, shallowCopy, getAst, astToStr, spawn, procCall
+  is, shallowCopy, getAst, astToStr, spawn, procCall
 
 Thus they act more like keywords than like ordinary identifiers; unlike a
 keyword however, a redefinition may `shadow`:idx: the definition in

doc/nimc.rst

@@ -189,12 +189,28 @@ resides in its own directory so that the generated ``nimcache`` directory
 is not shared between different projects.
 
 
+Compiler Selection
+==================
+
+To change the compiler from the default compiler (at the command line)::
+
+  nim c --cc:llvm_gcc --compile_only myfile.nim
+
+This uses the configuration defined in ``config\nim.cfg`` for ``lvm_gcc``.
+
+If nimcache already contains compiled code from a different compiler for the same project,
+add the ``-f`` flag to force all files to be recompiled.
+
+The default compiler is defined at the top of ``config\nim.cfg``.  Changing this setting
+affects the compiler used by ``koch`` to (re)build Nim.
+
+
 Cross compilation
 =================
 
 To cross compile, use for example::
 
-  nim c --cpu:i386 --os:linux --compile_only --gen_script myproject.nim
+  nim c --cpu:i386 --os:linux --compileOnly --genScript myproject.nim
 
 Then move the C code and the compile script ``compile_myproject.sh`` to your
 Linux i386 machine and run the script.
@@ -262,6 +278,15 @@ Define               Effect
                      what's in the Nim file with what's in the C header
                      (requires a C compiler with _Static_assert support, like
                      any C11 compiler)
+``tempDir``          This symbol takes a string as its value, like 
+                     ``--define:tempDir:/some/temp/path`` to override the
+                     temporary directory returned by ``os.getTempDir()``.
+                     The value **should** end with a directory separator 
+                     character. (Relevant for the Android platform)
+``useShPath``        This symbol takes a string as its value, like 
+                     ``--define:useShPath:/opt/sh/bin/sh`` to override the
+                     path for the ``sh`` binary, in cases where it is not
+                     located in the default location ``/bin/sh``
 ==================   =========================================================
 
 

doc/tut1.rst

@@ -138,7 +138,7 @@ comments can also be nested.
     ]#
   ]#
 
-You can also use the `discard statement`_ together with *long string
+You can also use the `discard statement <#procedures-discard-statement>`_ together with *long string
 literals* to create block comments:
 
 .. code-block:: nim
@@ -364,8 +364,7 @@ iterator:
     echo i
   # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines
 
-The built-in `$ <system.html#$>`_ operator turns an integer (``int``) and many
+The variable ``i`` is implicitly declared by the
-other types into a string. The variable ``i`` is implicitly declared by the
 ``for`` loop and has the type ``int``, because that is what `countup
 <system.html#countup>`_ returns. ``i`` runs through the values 1, 2, .., 10.
 Each value is ``echo``-ed. This code does the same:
@@ -501,10 +500,6 @@ differences:
 The ``when`` statement is useful for writing platform specific code, similar to
 the ``#ifdef`` construct in the C programming language.
 
-**Note**: To comment out a large piece of code, it is often better to use a
-``when false:`` statement than to use real comments. This way nesting is
-possible.
-
 
 Statements and indentation
 ==========================

doc/tut2.rst

@@ -218,7 +218,7 @@ So "pure object oriented" code is easy to write:
   import strutils, sequtils
 
   stdout.writeLine("Give a list of numbers (separated by spaces): ")
-  stdout.write(stdin.readLine.split.map(parseInt).max.`$`)
+  stdout.write(stdin.readLine.splitWhitespace.map(parseInt).max.`$`)
   stdout.writeLine(" is the maximum!")
 
 
@@ -233,15 +233,15 @@ is needed:
 
   type
     Socket* = ref object of RootObj
-      host: int # cannot be accessed from the outside of the module due to missing star
+      h: int # cannot be accessed from the outside of the module due to missing star
 
   proc `host=`*(s: var Socket, value: int) {.inline.} =
     ## setter of host address
-    s.host = value
+    s.h = value
 
   proc host*(s: Socket): int {.inline.} =
     ## getter of host address
-    s.host
+    s.h
 
   var s: Socket
   new s
@@ -723,7 +723,7 @@ regular expressions:
 
 .. code-block:: nim
 
-  macro case_token(n: typed): typed =
+  macro case_token(n: varargs[untyped]): typed =
     # creates a lexical analyzer from regular expressions
     # ... (implementation is an exercise for the reader :-)
     discard

examples/allany.nim

@@ -1,6 +1,6 @@
 # All and any
 
-template all(container, cond: expr): expr {.immediate.} =
+template all(container, cond: untyped): bool =
   block:
     var result = true
     for it in items(container):
@@ -9,7 +9,7 @@ template all(container, cond: expr): expr {.immediate.} =
         break
     result
 
-template any(container, cond: expr): expr {.immediate.} =
+template any(container, cond: untyped): bool =
   block:
     var result = false
     for it in items(container):

koch.nim

@@ -227,14 +227,15 @@ proc bundleWinTools() =
   removeFile("tools/finish".exe)
   buildVccTool()
   nimexec("c -o:bin/nimgrab.exe -d:ssl tools/nimgrab.nim")
+  nimexec("c -o:bin/nimgrep.exe tools/nimgrep.nim")
   when false:
     # not yet a tool worth including
     nimexec(r"c --cc:vcc --app:gui -o:bin\downloader.exe -d:ssl --noNimblePath " &
             r"--path:..\ui tools\downloader.nim")
 
 proc zip(args: string) =
-  bundleNimbleSrc()
+  bundleNimbleExe()
-  bundleNimsuggest(false)
+  bundleNimsuggest(true)
   bundleWinTools()
   nimexec("cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" %
        [VersionAsString, compileNimInst])
@@ -407,13 +408,15 @@ proc winReleaseArch(arch: string) =
   withMingw r"..\mingw" & arch & r"\bin":
     # Rebuilding koch is necessary because it uses its pointer size to
     # determine which mingw link to put in the NSIS installer.
-    nimexec "c --out:koch_temp --cpu:$# koch" % cpu
+    nimexec "c --cpu:$# koch" % cpu
-    exec "koch_temp boot -d:release --cpu:$#" % cpu
+    exec "koch boot -d:release --cpu:$#" % cpu
-    exec "koch_temp zip -d:release"
+    exec "koch zip -d:release"
     overwriteFile r"build\nim-$#.zip" % VersionAsString,
              r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch]
 
-proc winRelease() =
+proc winRelease*() =
+  # Now used from "tools/winrelease" and not directly supported by koch
+  # anymore!
   # Build -docs file:
   when true:
     web(gaCode)
@@ -530,11 +533,12 @@ proc showHelp() =
   quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)),
                    CompileDate, CompileTime], QuitSuccess)
 
-var op = initOptParser()
+when isMainModule:
-op.next()
+  var op = initOptParser()
-case op.kind
+  op.next()
-of cmdLongOption, cmdShortOption: showHelp()
+  case op.kind
-of cmdArgument:
+  of cmdLongOption, cmdShortOption: showHelp()
+  of cmdArgument:
     case normalize(op.key)
     of "boot": boot(op.cmdLineRest)
     of "clean": clean(op.cmdLineRest)
@@ -557,7 +561,7 @@ of cmdArgument:
     of "test", "tests": tests(op.cmdLineRest)
     of "temp": temp(op.cmdLineRest)
     of "xtemp": xtemp(op.cmdLineRest)
-  of "winrelease": winRelease()
+    #of "winrelease": winRelease()
     of "wintools": bundleWinTools()
     of "nimble": buildNimble(existsDir(".git"))
     of "nimsuggest": bundleNimsuggest(buildExe=true)
@@ -565,4 +569,4 @@ of cmdArgument:
     of "pushcsource", "pushcsources": pushCsources()
     of "valgrind": valgrind(op.cmdLineRest)
     else: showHelp()
-of cmdEnd: showHelp()
+  of cmdEnd: showHelp()

lib/core/locks.nim

@@ -19,6 +19,8 @@ type
 
 {.deprecated: [TLock: Lock, TCond: Cond].}
 
+{.push stackTrace: off.}
+
 proc initLock*(lock: var Lock) {.inline.} =
   ## Initializes the given lock.
   initSysLock(lock)
@@ -59,9 +61,12 @@ proc signal*(cond: var Cond) {.inline.} =
 template withLock*(a: Lock, body: untyped) =
   ## Acquires the given lock, executes the statements in body and
   ## releases the lock after the statements finish executing.
+  mixin acquire, release
   a.acquire()
   {.locks: [a].}:
     try:
       body
     finally:
       a.release()
+
+{.pop.}

lib/core/macros.nim

@@ -94,8 +94,9 @@ type
     ntyVarargs,
     ntyUnused,
     ntyError,
-    ntyBuiltinTypeClass, ntyConcept, ntyConceptInst, ntyComposite,
+    ntyBuiltinTypeClass, ntyUserTypeClass, ntyUserTypeClassInst,
-    ntyAnd, ntyOr, ntyNot
+    ntyCompositeTypeClass, ntyInferred, ntyAnd, ntyOr, ntyNot,
+    ntyAnything, ntyStatic, ntyFromExpr, ntyFieldAccessor, ntyVoid
 
   TNimTypeKinds* {.deprecated.} = set[NimTypeKind]
   NimSymKind* = enum
@@ -149,6 +150,9 @@ proc `==`*(a, b: NimIdent): bool {.magic: "EqIdent", noSideEffect.}
 proc `==`*(a, b: NimNode): bool {.magic: "EqNimrodNode", noSideEffect.}
   ## compares two Nim nodes
 
+proc `==`*(a, b: NimSym): bool {.magic: "EqNimrodNode", noSideEffect.}
+  ## compares two Nim symbols
+
 proc sameType*(a, b: NimNode): bool {.magic: "SameNodeType", noSideEffect.} =
   ## compares two Nim nodes' types. Return true if the types are the same,
   ## eg. true when comparing alias with original type.
@@ -317,10 +321,30 @@ proc toStrLit*(n: NimNode): NimNode {.compileTime.} =
   ## in a string literal node
   return newStrLitNode(repr(n))
 
-proc lineinfo*(n: NimNode): string {.magic: "NLineInfo", noSideEffect.}
+type
+  LineInfo* = object
+    filename*: string
+    line*,column*: int
+
+proc `$`*(arg: Lineinfo): string =
+  result = arg.filename & "(" & $arg.line & ", " & $arg.column & ")"
+
+#proc lineinfo*(n: NimNode): LineInfo {.magic: "NLineInfo", noSideEffect.}
   ## returns the position the node appears in the original source file
   ## in the form filename(line, col)
 
+proc getLine(arg: NimNode): int {.magic: "NLineInfo", noSideEffect.}
+proc getColumn(arg: NimNode): int {.magic: "NLineInfo", noSideEffect.}
+proc getFile(arg: NimNode): string {.magic: "NLineInfo", noSideEffect.}
+
+proc lineInfoObj*(n: NimNode): LineInfo {.compileTime.} =
+  result.filename = n.getFile
+  result.line = n.getLine
+  result.column = n.getColumn
+
+proc lineInfo*(arg: NimNode): string {.compileTime.} =
+  $arg.lineInfoObj
+
 proc internalParseExpr(s: string): NimNode {.
   magic: "ParseExprToAst", noSideEffect.}
 
@@ -527,10 +551,17 @@ proc newLit*[N,T](arg: array[N,T]): NimNode {.compileTime.} =
     result.add newLit(x)
 
 proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} =
-  result = nnkBracket.newTree
+  var bracket = nnkBracket.newTree
   for x in arg:
-    result.add newLit(x)
+    bracket.add newLit(x)
-  result = nnkPrefix.newTree(bindSym"@", result)
+
+  result = nnkCall.newTree(
+    nnkBracketExpr.newTree(
+      nnkAccQuoted.newTree( bindSym"@" ),
+      getTypeInst( bindSym"T" )
+    ),
+    bracket
+  )
 
 proc newLit*(arg: tuple): NimNode {.compileTime.} =
   result = nnkPar.newTree
@@ -557,7 +588,8 @@ proc nestList*(theProc: NimIdent,
 proc treeRepr*(n: NimNode): string {.compileTime, benign.} =
   ## Convert the AST `n` to a human-readable tree-like string.
   ##
-  ## See also `repr` and `lispRepr`.
+  ## See also `repr`, `lispRepr`, and `astGenRepr`.
+
   proc traverse(res: var string, level: int, n: NimNode) {.benign.} =
     for i in 0..level-1: res.add "  "
     res.add(($n.kind).substr(3))
@@ -582,7 +614,7 @@ proc treeRepr*(n: NimNode): string {.compileTime, benign.} =
 proc lispRepr*(n: NimNode): string {.compileTime, benign.} =
   ## Convert the AST `n` to a human-readable lisp-like string,
   ##
-  ## See also `repr` and `treeRepr`.
+  ## See also `repr`, `treeRepr`, and `astGenRepr`.
 
   result = ($n.kind).substr(3)
   add(result, "(")
@@ -605,9 +637,96 @@ proc lispRepr*(n: NimNode): string {.compileTime, benign.} =
 
   add(result, ")")
 
+proc astGenRepr*(n: NimNode): string {.compileTime, benign.} =
+  ## Convert the AST `n` to the code required to generate that AST. So for example
+  ##
+  ## .. code-block:: nim
+  ##   astGenRepr:
+  ##     echo "Hello world"
+  ##
+  ## Would output:
+  ##
+  ## .. code-block:: nim
+  ##   nnkStmtList.newTree(
+  ##     nnkCommand.newTree(
+  ##       newIdentNode(!"echo"),
+  ##       newLit("Hello world")
+  ##     )
+  ##   )
+  ##
+  ## See also `repr`, `treeRepr`, and `lispRepr`.
+
+  const
+    NodeKinds = {nnkEmpty, nnkNilLit, nnkIdent, nnkSym, nnkNone}
+    LitKinds = {nnkCharLit..nnkInt64Lit, nnkFloatLit..nnkFloat64Lit, nnkStrLit..nnkTripleStrLit}
+
+  proc escape(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect.} =
+    ## Functions copied from strutils
+    proc toHex(x: BiggestInt, len: Positive): string {.noSideEffect, rtl.} =
+      const
+        HexChars = "0123456789ABCDEF"
+      var
+        t = x
+      result = newString(len)
+      for j in countdown(len-1, 0):
+        result[j] = HexChars[t and 0xF]
+        t = t shr 4
+        # handle negative overflow
+        if t == 0 and x < 0: t = -1
+
+    result = newStringOfCap(s.len + s.len shr 2)
+    result.add(prefix)
+    for c in items(s):
+      case c
+      of '\0'..'\31', '\128'..'\255':
+        add(result, "\\x")
+        add(result, toHex(ord(c), 2))
+      of '\\': add(result, "\\\\")
+      of '\'': add(result, "\\'")
+      of '\"': add(result, "\\\"")
+      else: add(result, c)
+    add(result, suffix)
+
+  proc traverse(res: var string, level: int, n: NimNode) {.benign.} =
+    for i in 0..level-1: res.add "  "
+    if n.kind in NodeKinds:
+      res.add("new" & ($n.kind).substr(3) & "Node(")
+    elif n.kind in LitKinds:
+      res.add("newLit(")
+    else:
+      res.add($n.kind)
+
+    case n.kind
+    of nnkEmpty: discard
+    of nnkNilLit: res.add("nil")
+    of nnkCharLit: res.add("'" & $chr(n.intVal) & "'")
+    of nnkIntLit..nnkInt64Lit: res.add($n.intVal)
+    of nnkFloatLit..nnkFloat64Lit: res.add($n.floatVal)
+    of nnkStrLit..nnkTripleStrLit: res.add($n.strVal.escape())
+    of nnkIdent: res.add("!" & ($n.ident).escape())
+    of nnkSym: res.add(($n.symbol).escape())
+    of nnkNone: assert false
+    else:
+      res.add(".newTree(")
+      for j in 0..<n.len:
+        res.add "\n"
+        traverse(res, level + 1, n[j])
+        if j != n.len-1:
+          res.add(",")
+
+      res.add("\n")
+      for i in 0..level-1: res.add "  "
+      res.add(")")
+
+    if n.kind in NodeKinds+LitKinds:
+      res.add(")")
+
+  result = ""
+  traverse(result, 0, n)
+
 macro dumpTree*(s: untyped): untyped = echo s.treeRepr
   ## Accepts a block of nim code and prints the parsed abstract syntax
-  ## tree using the `toTree` function. Printing is done *at compile time*.
+  ## tree using the `treeRepr` function. Printing is done *at compile time*.
   ##
   ## You can use this as a tool to explore the Nim's abstract syntax
   ## tree and to discover what kind of nodes must be created to represent
@@ -615,7 +734,16 @@ macro dumpTree*(s: untyped): untyped = echo s.treeRepr
 
 macro dumpLisp*(s: untyped): untyped = echo s.lispRepr
   ## Accepts a block of nim code and prints the parsed abstract syntax
-  ## tree using the `toLisp` function. Printing is done *at compile time*.
+  ## tree using the `lispRepr` function. Printing is done *at compile time*.
+  ##
+  ## See `dumpTree`.
+
+macro dumpAstGen*(s: untyped): untyped = echo s.astGenRepr
+  ## Accepts a block of nim code and prints the parsed abstract syntax
+  ## tree using the `astGenRepr` function. Printing is done *at compile time*.
+  ##
+  ## You can use this as a tool to write macros quicker by writing example
+  ## outputs and then copying the snippets into the macro for modification.
   ##
   ## See `dumpTree`.
 
@@ -766,6 +894,8 @@ template expectRoutine(node: NimNode) =
 proc name*(someProc: NimNode): NimNode {.compileTime.} =
   someProc.expectRoutine
   result = someProc[0]
+  if result.kind == nnkPostfix:
+    result = result[1]
 proc `name=`*(someProc: NimNode; val: NimNode) {.compileTime.} =
   someProc.expectRoutine
   someProc[0] = val

lib/core/typeinfo.nim

@@ -97,7 +97,7 @@ proc newObj(typ: PNimType, size: int): pointer {.importCompilerProc.}
 proc newSeq(typ: PNimType, len: int): pointer {.importCompilerProc.}
 proc objectInit(dest: pointer, typ: PNimType) {.importCompilerProc.}
 
-template `+!!`(a, b: expr): expr = cast[pointer](cast[ByteAddress](a) + b)
+template `+!!`(a, b): untyped = cast[pointer](cast[ByteAddress](a) + b)
 
 proc getDiscriminant(aa: pointer, n: ptr TNimNode): int =
   assert(n.kind == nkCase)

lib/deprecated/pure/sockets.nim

@@ -42,6 +42,8 @@ from times import epochTime
 
 when defined(ssl):
   import openssl
+else:
+  type SSLAcceptResult = int
 
 when defined(Windows):
   import winlean
@@ -206,16 +208,16 @@ proc htons*(x: int16): int16 =
   ## order, this is a no-op; otherwise, it performs a 2-byte swap operation.
   result = sockets.ntohs(x)
 
-template ntohl(x: uint32): expr =
+template ntohl(x: uint32): uint32 =
   cast[uint32](sockets.ntohl(cast[int32](x)))
 
-template ntohs(x: uint16): expr =
+template ntohs(x: uint16): uint16 =
   cast[uint16](sockets.ntohs(cast[int16](x)))
 
-template htonl(x: uint32): expr =
+template htonl(x: uint32): uint32 =
   sockets.ntohl(x)
 
-template htons(x: uint16): expr =
+template htons(x: uint16): uint16 =
   sockets.ntohs(x)
 
 when defined(Posix):
@@ -442,8 +444,7 @@ proc parseIp4*(s: string): BiggestInt =
   if s[i] != '\0': invalidIp4(s)
   result = BiggestInt(a shl 24 or b shl 16 or c shl 8 or d)
 
-template gaiNim(a, p, h, list: expr): stmt =
+template gaiNim(a, p, h, list: untyped): untyped =
-  block:
   var gaiResult = getaddrinfo(a, $p, addr(h), list)
   if gaiResult != 0'i32:
     when defined(windows):
@@ -493,8 +494,8 @@ proc getSockName*(socket: Socket): Port =
     raiseOSError(osLastError())
   result = Port(sockets.ntohs(name.sin_port))
 
-template acceptAddrPlain(noClientRet, successRet: expr,
+template acceptAddrPlain(noClientRet, successRet: SSLAcceptResult or int,
-                         sslImplementation: stmt): stmt {.immediate.} =
+                         sslImplementation: untyped): untyped =
   assert(client != nil)
   var sockAddress: Sockaddr_in
   var addrLen = sizeof(sockAddress).SockLen
@@ -550,7 +551,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string) {.
   ##
   ## **Warning:** When using SSL with non-blocking sockets, it is best to use
   ## the acceptAddrSSL procedure as this procedure will most likely block.
-  acceptAddrPlain(-1, -1):
+  acceptAddrPlain(SSLAcceptResult(-1), SSLAcceptResult(-1)):
     when defined(ssl):
       if server.isSSL:
         # We must wrap the client sock in a ssl context.
@@ -594,7 +595,7 @@ when defined(ssl):
     ##
     ## ``AcceptNoClient`` will be returned when no client is currently attempting
     ## to connect.
-    template doHandshake(): stmt =
+    template doHandshake(): untyped =
       when defined(ssl):
         if server.isSSL:
           client.setBlocking(false)
@@ -1278,7 +1279,7 @@ proc recvLine*(socket: Socket, line: var TaintedString, timeout = -1): bool {.
   ## **Deprecated since version 0.9.2**: This function has been deprecated in
   ## favour of readLine.
 
-  template addNLIfEmpty(): stmt =
+  template addNLIfEmpty(): untyped =
     if line.len == 0:
       line.add("\c\L")
 
@@ -1319,7 +1320,7 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1) {.
   ## A timeout can be specified in milliseconds, if data is not received within
   ## the specified time an ETimeout exception will be raised.
 
-  template addNLIfEmpty(): stmt =
+  template addNLIfEmpty(): untyped =
     if line.len == 0:
       line.add("\c\L")
 

lib/impure/nre/private/util.nim

@@ -16,7 +16,7 @@ proc checkNil(arg: string): string =
   else:
     return arg
 
-template formatStr*(howExpr, namegetter, idgetter: expr): expr =
+template formatStr*(howExpr, namegetter, idgetter): untyped =
   let how = howExpr
   var val = newStringOfCap(how.len)
   var i = 0

lib/js/jsffi.nim

@@ -408,20 +408,19 @@ macro `{}`*(typ: typedesc, xs: varargs[untyped]): auto =
         kString = quote do:
           when compiles($`k`): $`k` else: "invalid"
         v = x[1]
-      body.add(quote do:
+      body.add quote do:
         when compiles(`a`.`k`):
           `a`.`k` = `v`
         elif compiles(`a`[`k`]):
           `a`[`k`] = `v`
         else:
           `a`[`kString`] = `v`
-      )
+
     else:
       error("Expression `" & $x.toStrLit & "` not allowed in `{}` macro")
 
-  body.add(quote do:
+  body.add quote do:
     return `a`
-  )
 
   result = quote do:
     proc inner(): `typ` {.gensym.} =

lib/packages/docutils/rstgen.nim

@@ -765,7 +765,7 @@ proc renderTocEntries*(d: var RstGenerator, j: var int, lvl: int,
     result.add(tmp)
 
 proc renderImage(d: PDoc, n: PRstNode, result: var string) =
-  template valid(s): expr =
+  template valid(s): bool =
     s.len > 0 and allCharsInSet(s, {'.','/',':','%','_','\\','\128'..'\xFF'} +
                                    Digits + Letters + WhiteSpace)
   let
@@ -1194,7 +1194,7 @@ proc defaultConfig*(): StringTableRef =
   ## ``rstToHtml`` to generate the bare minimum HTML.
   result = newStringTable(modeStyleInsensitive)
 
-  template setConfigVar(key, val: expr) =
+  template setConfigVar(key, val) =
     result[key] = val
 
   # If you need to modify these values, it might be worth updating the template

lib/posix/posix.nim

@@ -92,7 +92,7 @@ else:
 # There used to be this name in posix.nim a long time ago, not sure why!
 {.deprecated: [cSIG_HOLD: SIG_HOLD].}
 
-when not defined(macosx):
+when not defined(macosx) and not defined(android):
   proc st_atime*(s: Stat): Time {.inline.} =
     ## Second-granularity time of last access
     result = s.st_atim.tv_sec

lib/posix/posix_other.nim

@@ -15,7 +15,7 @@ const
   hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays
   hasAioH = defined(linux)
 
-when defined(linux):
+when defined(linux) and not defined(android):
   # On Linux:
   # timer_{create,delete,settime,gettime},
   # clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt
@@ -48,7 +48,7 @@ type
       # DragonflyBSD doesn't have `d_reclen` field.
       d_type*: uint8
     elif defined(linux) or defined(macosx) or defined(freebsd) or
-         defined(netbsd) or defined(openbsd):
+         defined(netbsd) or defined(openbsd) or defined(genode):
       d_reclen*: cshort ## Length of this record. (not POSIX)
       d_type*: int8 ## Type of file; not supported by all filesystem types.
                     ## (not POSIX)
@@ -215,7 +215,7 @@ type
                            ## For a typed memory object, the length in bytes.
                            ## For other file types, the use of this field is
                            ## unspecified.
-    when defined(macosx):
+    when defined(macosx) or defined(android):
       st_atime*: Time      ## Time of last access.
       st_mtime*: Time      ## Time of last data modification.
       st_ctime*: Time      ## Time of last status change.
@@ -572,7 +572,8 @@ else:
     MAP_POPULATE*: cint = 0
 
 when defined(linux) or defined(nimdoc):
-  when defined(alpha) or defined(mips) or defined(parisc) or
+  when defined(alpha) or defined(mips) or defined(mipsel) or
+      defined(mips64) or defined(mips64el) or defined(parisc) or
       defined(sparc) or defined(nimdoc):
     const SO_REUSEPORT* = cint(0x0200)
       ## Multiple binding: load balancing on incoming TCP connections

lib/posix/termios.nim

@@ -174,7 +174,7 @@ var
 # Compare a character C to a value VAL from the `cc' array in a
 #   `struct termios'.  If VAL is _POSIX_VDISABLE, no character can match it.
 
-template cceq*(val, c: expr): expr =
+template cceq*(val, c): untyped =
   c == val and val != POSIX_VDISABLE
 
 # Return the output baud rate stored in *TERMIOS_P.

lib/pure/asyncdispatch.nim

@@ -9,11 +9,12 @@
 
 include "system/inclrtl"
 
-import os, tables, strutils, times, heapqueue, options
+import os, tables, strutils, times, heapqueue, options, asyncstreams
-
+import asyncfutures except callSoon
 import nativesockets, net, deques
 
 export Port, SocketFlag
+export asyncfutures, asyncstreams
 
 #{.injectStmt: newGcInvariant().}
 
@@ -159,8 +160,6 @@ export Port, SocketFlag
 
 # TODO: Check if yielded future is nil and throw a more meaningful exception
 
-include includes/asyncfutures
-
 type
   PDispatcherBase = ref object of RootRef
     timers*: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
@@ -190,6 +189,12 @@ proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
       result = int((timerTimeout - curTime) * 1000)
       if result < 0: result = 0
 
+proc callSoon(cbproc: proc ()) {.gcsafe.}
+
+proc initCallSoonProc =
+  if asyncfutures.getCallSoonProc().isNil:
+    asyncfutures.setCallSoonProc(callSoon)
+
 when defined(windows) or defined(nimdoc):
   import winlean, sets, hashes
   type
@@ -237,15 +242,17 @@ when defined(windows) or defined(nimdoc):
     result.callbacks = initDeque[proc ()](64)
 
   var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
-  proc getGlobalDispatcher*(): PDispatcher =
-    ## Retrieves the global thread-local dispatcher.
-    if gDisp.isNil: gDisp = newDispatcher()
-    result = gDisp
 
   proc setGlobalDispatcher*(disp: PDispatcher) =
     if not gDisp.isNil:
       assert gDisp.callbacks.len == 0
     gDisp = disp
+    initCallSoonProc()
+
+  proc getGlobalDispatcher*(): PDispatcher =
+    if gDisp.isNil:
+      setGlobalDispatcher(newDispatcher())
+    result = gDisp
 
   proc register*(fd: AsyncFD) =
     ## Registers ``fd`` with the dispatcher.
@@ -932,14 +939,17 @@ else:
     result.callbacks = initDeque[proc ()](64)
 
   var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
-  proc getGlobalDispatcher*(): PDispatcher =
-    if gDisp.isNil: gDisp = newDispatcher()
-    result = gDisp
 
   proc setGlobalDispatcher*(disp: PDispatcher) =
     if not gDisp.isNil:
       assert gDisp.callbacks.len == 0
     gDisp = disp
+    initCallSoonProc()
+
+  proc getGlobalDispatcher*(): PDispatcher =
+    if gDisp.isNil:
+      setGlobalDispatcher(newDispatcher())
+    result = gDisp
 
   proc update(fd: AsyncFD, events: set[Event]) =
     let p = getGlobalDispatcher()
@@ -1307,7 +1317,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} =
   ##
   ## **Deprecated since version 0.15.0**: Use ``asyncnet.recvLine()`` instead.
 
-  template addNLIfEmpty(): stmt =
+  template addNLIfEmpty(): untyped =
     if result.len == 0:
       result.add("\c\L")
 
@@ -1327,7 +1337,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} =
       return
     add(result, c)
 
-proc callSoon*(cbproc: proc ()) =
+proc callSoon(cbproc: proc ()) =
   ## Schedule `cbproc` to be called as soon as possible.
   ## The callback is called when control returns to the event loop.
   getGlobalDispatcher().callbacks.addLast(cbproc)

lib/pure/asyncfile.nim

@@ -81,27 +81,32 @@ proc getFileSize*(f: AsyncFile): int64 =
   else:
     result = lseek(f.fd.cint, 0, SEEK_END)
 
+proc newAsyncFile*(fd: AsyncFd): AsyncFile =
+  ## Creates `AsyncFile` with a previously opened file descriptor `fd`.
+  new result
+  result.fd = fd
+  register(result.fd)
+
 proc openAsync*(filename: string, mode = fmRead): AsyncFile =
   ## Opens a file specified by the path in ``filename`` using
   ## the specified ``mode`` asynchronously.
-  new result
   when defined(windows) or defined(nimdoc):
     let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL
     let desiredAccess = getDesiredAccess(mode)
     let creationDisposition = getCreationDisposition(mode, filename)
     when useWinUnicode:
-      result.fd = createFileW(newWideCString(filename), desiredAccess,
+      let fd = createFileW(newWideCString(filename), desiredAccess,
           FILE_SHARE_READ,
           nil, creationDisposition, flags, 0).AsyncFd
     else:
-      result.fd = createFileA(filename, desiredAccess,
+      let fd = createFileA(filename, desiredAccess,
           FILE_SHARE_READ,
           nil, creationDisposition, flags, 0).AsyncFd
 
-    if result.fd.Handle == INVALID_HANDLE_VALUE:
+    if fd.Handle == INVALID_HANDLE_VALUE:
       raiseOSError(osLastError())
 
-    register(result.fd)
+    result = newAsyncFile(fd)
 
     if mode == fmAppend:
       result.offset = getFileSize(result)
@@ -110,11 +115,11 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile =
     let flags = getPosixFlags(mode)
     # RW (Owner), RW (Group), R (Other)
     let perm = S_IRUSR or S_IWUSR or S_IRGRP or S_IWGRP or S_IROTH
-    result.fd = open(filename, flags, perm).AsyncFD
+    let fd = open(filename, flags, perm).AsyncFD
-    if result.fd.cint == -1:
+    if fd.cint == -1:
       raiseOSError(osLastError())
 
-    register(result.fd)
+    result = newAsyncFile(fd)
 
 proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
   ## Read ``size`` bytes from the specified file asynchronously starting at

lib/pure/asyncfutures.nim

@@ -1,8 +1,16 @@
+import os, tables, strutils, times, heapqueue, options, deques
 
 # TODO: This shouldn't need to be included, but should ideally be exported.
 type
+  CallbackFunc = proc () {.closure, gcsafe.}
+
+  CallbackList = object
+    function: CallbackFunc
+    next: ref CallbackList
+
   FutureBase* = ref object of RootObj ## Untyped future.
-    cb: proc () {.closure,gcsafe.}
+    callbacks: CallbackList
+
     finished: bool
     error*: ref Exception ## Stored exception
     errorStackTrace*: string
@@ -16,12 +24,6 @@ type
 
   FutureVar*[T] = distinct Future[T]
 
-  FutureStream*[T] = ref object of FutureBase   ## Special future that acts as
-                                                ## a queue. Its API is still
-                                                ## experimental and so is
-                                                ## subject to change.
-    queue: Deque[T]
-
   FutureError* = object of Exception
     cause*: FutureBase
 
@@ -30,7 +32,27 @@ type
 when not defined(release):
   var currentID = 0
 
-proc callSoon*(cbproc: proc ()) {.gcsafe.}
+var callSoonProc {.threadvar.}: proc (cbproc: proc ()) {.gcsafe.}
+
+proc getCallSoonProc*(): (proc(cbproc: proc ()) {.gcsafe.}) =
+  ## Get current implementation of ``callSoon``.
+  return callSoonProc
+
+proc setCallSoonProc*(p: (proc(cbproc: proc ()) {.gcsafe.})) =
+  ## Change current implementation of ``callSoon``. This is normally called when dispatcher from ``asyncdispatcher`` is initialized.
+  callSoonProc = p
+
+proc callSoon*(cbproc: proc ()) =
+  ## Call ``cbproc`` "soon".
+  ##
+  ## If async dispatcher is running, ``cbproc`` will be executed during next dispatcher tick.
+  ##
+  ## If async dispatcher is not running, ``cbproc`` will be executed immediately.
+  if callSoonProc.isNil:
+    # Loop not initialized yet. Call the function directly to allow setup code to use futures.
+    cbproc()
+  else:
+    callSoonProc(cbproc)
 
 template setupFutureBase(fromProc: string) =
   new(result)
@@ -56,22 +78,6 @@ proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] =
   ## that this future belongs to, is a good habit as it helps with debugging.
   result = FutureVar[T](newFuture[T](fromProc))
 
-proc newFutureStream*[T](fromProc = "unspecified"): FutureStream[T] =
-  ## Create a new ``FutureStream``. This future's callback is activated when
-  ## two events occur:
-  ##
-  ## * New data is written into the future stream.
-  ## * The future stream is completed (this means that no more data will be
-  ##   written).
-  ##
-  ## Specifying ``fromProc``, which is a string specifying the name of the proc
-  ## that this future belongs to, is a good habit as it helps with debugging.
-  ##
-  ## **Note:** The API of FutureStream is still new and so has a higher
-  ## likelihood of changing in the future.
-  setupFutureBase(fromProc)
-  result.queue = initDeque[T]()
-
 proc clean*[T](future: FutureVar[T]) =
   ## Resets the ``finished`` status of ``future``.
   Future[T](future).finished = false
@@ -98,6 +104,33 @@ proc checkFinished[T](future: Future[T]) =
       err.cause = future
       raise err
 
+proc call(callbacks: var CallbackList) =
+  var current = callbacks
+
+  while true:
+    if not current.function.isNil:
+      callSoon(current.function)
+
+    if current.next.isNil:
+      break
+    else:
+      current = current.next[]
+
+  # callback will be called only once, let GC collect them now
+  callbacks.next = nil
+  callbacks.function = nil
+
+proc add(callbacks: var CallbackList, function: CallbackFunc) =
+  if callbacks.function.isNil:
+    callbacks.function = function
+    assert callbacks.next == nil
+  else:
+    let newNext = new(ref CallbackList)
+    newNext.function = callbacks.function
+    newNext.next = callbacks.next
+    callbacks.next = newNext
+    callbacks.function = function
+
 proc complete*[T](future: Future[T], val: T) =
   ## Completes ``future`` with value ``val``.
   #assert(not future.finished, "Future already finished, cannot finish twice.")
@@ -105,8 +138,7 @@ proc complete*[T](future: Future[T], val: T) =
   assert(future.error == nil)
   future.value = val
   future.finished = true
-  if future.cb != nil:
+  future.callbacks.call()
-    future.cb()
 
 proc complete*(future: Future[void]) =
   ## Completes a void ``future``.
@@ -114,8 +146,7 @@ proc complete*(future: Future[void]) =
   checkFinished(future)
   assert(future.error == nil)
   future.finished = true
-  if future.cb != nil:
+  future.callbacks.call()
-    future.cb()
 
 proc complete*[T](future: FutureVar[T]) =
   ## Completes a ``FutureVar``.
@@ -123,8 +154,7 @@ proc complete*[T](future: FutureVar[T]) =
   checkFinished(fut)
   assert(fut.error == nil)
   fut.finished = true
-  if fut.cb != nil:
+  fut.callbacks.call()
-    fut.cb()
 
 proc complete*[T](future: FutureVar[T], val: T) =
   ## Completes a ``FutureVar`` with value ``val``.
@@ -135,14 +165,7 @@ proc complete*[T](future: FutureVar[T], val: T) =
   assert(fut.error.isNil())
   fut.finished = true
   fut.value = val
-  if not fut.cb.isNil():
+  fut.callbacks.call()
-    fut.cb()
-
-proc complete*[T](future: FutureStream[T]) =
-  ## Completes a ``FutureStream`` signalling the end of data.
-  future.finished = true
-  if not future.cb.isNil():
-    future.cb()
 
 proc fail*[T](future: Future[T], error: ref Exception) =
   ## Completes ``future`` with ``error``.
@@ -152,26 +175,40 @@ proc fail*[T](future: Future[T], error: ref Exception) =
   future.error = error
   future.errorStackTrace =
     if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error)
-  if future.cb != nil:
+  future.callbacks.call()
-    future.cb()
+
+proc clearCallbacks(future: FutureBase) =
+  future.callbacks.function = nil
+  future.callbacks.next = nil
+
+proc addCallback*(future: FutureBase, cb: proc() {.closure,gcsafe.}) =
+  ## Adds the callbacks proc to be called when the future completes.
+  ##
+  ## If future has already completed then ``cb`` will be called immediately.
+  assert cb != nil
+  if future.finished:
+    callSoon(cb)
   else:
-    # This is to prevent exceptions from being silently ignored when a future
+    future.callbacks.add cb
-    # is discarded.
+
-    # TODO: This may turn out to be a bad idea.
+proc addCallback*[T](future: Future[T],
-    # Turns out this is a bad idea.
+                     cb: proc (future: Future[T]) {.closure,gcsafe.}) =
-    #raise error
+  ## Adds the callbacks proc to be called when the future completes.
-    discard
+  ##
+  ## If future has already completed then ``cb`` will be called immediately.
+  future.addCallback(
+    proc() =
+      cb(future)
+  )
 
 proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) =
-  ## Sets the callback proc to be called when the future completes.
+  ## Clears the list of callbacks and sets the callback proc to be called when the future completes.
   ##
   ## If future has already completed then ``cb`` will be called immediately.
   ##
-  ## **Note**: You most likely want the other ``callback`` setter which
+  ## It's recommended to use ``addCallback`` or ``then`` instead.
-  ## passes ``future`` as a param to the callback.
+  future.clearCallbacks
-  future.cb = cb
+  future.addCallback cb
-  if future.finished:
-    callSoon(future.cb)
 
 proc `callback=`*[T](future: Future[T],
     cb: proc (future: Future[T]) {.closure,gcsafe.}) =
@@ -180,20 +217,6 @@ proc `callback=`*[T](future: Future[T],
   ## If future has already completed then ``cb`` will be called immediately.
   future.callback = proc () = cb(future)
 
-proc `callback=`*[T](future: FutureStream[T],
-    cb: proc (future: FutureStream[T]) {.closure,gcsafe.}) =
-  ## Sets the callback proc to be called when data was placed inside the
-  ## future stream.
-  ##
-  ## The callback is also called when the future is completed. So you should
-  ## use ``finished`` to check whether data is available.
-  ##
-  ## If the future stream already has data or is finished then ``cb`` will be
-  ## called immediately.
-  future.cb = proc () = cb(future)
-  if future.queue.len > 0 or future.finished:
-    callSoon(future.cb)
-
 proc injectStacktrace[T](future: Future[T]) =
   # TODO: Come up with something better.
   when not defined(release):
@@ -240,18 +263,12 @@ proc mget*[T](future: FutureVar[T]): var T =
   ## Future has not been finished.
   result = Future[T](future).value
 
-proc finished*[T](future: Future[T] | FutureVar[T] | FutureStream[T]): bool =
+proc finished*[T](future: Future[T] | FutureVar[T]): bool =
   ## Determines whether ``future`` has completed.
   ##
   ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish.
-  ##
-  ## For a ``FutureStream`` a ``true`` value means that no more data will be
-  ## placed inside the stream _and_ that there is no data waiting to be
-  ## retrieved.
   when future is FutureVar[T]:
     result = (Future[T](future)).finished
-  elif future is FutureStream[T]:
-    result = future.finished and future.queue.len == 0
   else:
     result = future.finished
 
@@ -259,57 +276,6 @@ proc failed*(future: FutureBase): bool =
   ## Determines whether ``future`` completed with an error.
   return future.error != nil
 
-proc write*[T](future: FutureStream[T], value: T): Future[void] =
-  ## Writes the specified value inside the specified future stream.
-  ##
-  ## This will raise ``ValueError`` if ``future`` is finished.
-  result = newFuture[void]("FutureStream.put")
-  if future.finished:
-    let msg = "FutureStream is finished and so no longer accepts new data."
-    result.fail(newException(ValueError, msg))
-    return
-  # TODO: Implement limiting of the streams storage to prevent it growing
-  # infinitely when no reads are occuring.
-  future.queue.addLast(value)
-  if not future.cb.isNil: future.cb()
-  result.complete()
-
-proc read*[T](future: FutureStream[T]): Future[(bool, T)] =
-  ## Returns a future that will complete when the ``FutureStream`` has data
-  ## placed into it. The future will be completed with the oldest
-  ## value stored inside the stream. The return value will also determine
-  ## whether data was retrieved, ``false`` means that the future stream was
-  ## completed and no data was retrieved.
-  ##
-  ## This function will remove the data that was returned from the underlying
-  ## ``FutureStream``.
-  var resFut = newFuture[(bool, T)]("FutureStream.take")
-  let savedCb = future.cb
-  future.callback =
-    proc (fs: FutureStream[T]) =
-      # We don't want this callback called again.
-      future.cb = nil
-
-      # The return value depends on whether the FutureStream has finished.
-      var res: (bool, T)
-      if finished(fs):
-        # Remember, this callback is called when the FutureStream is completed.
-        res[0] = false
-      else:
-        res[0] = true
-        res[1] = fs.queue.popFirst()
-
-      if not resFut.finished:
-        resFut.complete(res)
-
-      # If the saved callback isn't nil then let's call it.
-      if not savedCb.isNil: savedCb()
-  return resFut
-
-proc len*[T](future: FutureStream[T]): int =
-  ## Returns the amount of data pieces inside the stream.
-  future.queue.len
-
 proc asyncCheck*[T](future: Future[T]) =
   ## Sets a callback on ``future`` which raises an exception if the future
   ## finished with an error.

lib/pure/asynchttpserver.nim

@@ -233,7 +233,7 @@ proc processClient(client: AsyncSocket, address: string,
           await request.respond(Http400, "Bad Request. Content-Length does not match actual.")
           continue
     elif request.reqMethod == HttpPost:
-      await request.respond(Http400, "Bad Request. No Content-Length.")
+      await request.respond(Http411, "Content-Length required.")
       continue
 
     # Call the user's callback.

lib/pure/asyncmacro.nim

@@ -28,7 +28,7 @@ template createCb(retFutureSym, iteratorNameSym,
                   name, futureVarCompletions: untyped) =
   var nameIterVar = iteratorNameSym
   #{.push stackTrace: off.}
-  proc cb {.closure,gcsafe.} =
+  proc cb0 {.closure.} =
     try:
       if not nameIterVar.finished:
         var next = nameIterVar()
@@ -38,7 +38,10 @@ template createCb(retFutureSym, iteratorNameSym,
                     "`nil` Future?"
             raise newException(AssertionError, msg % name)
         else:
-          next.callback = cb
+          {.gcsafe.}:
+            {.push hint[ConvFromXtoItselfNotNeeded]: off.}
+            next.callback = (proc() {.closure, gcsafe.})(cb0)
+            {.pop.}
     except:
       futureVarCompletions
 
@@ -49,7 +52,7 @@ template createCb(retFutureSym, iteratorNameSym,
       else:
         retFutureSym.fail(getCurrentException())
 
-  cb()
+  cb0()
   #{.pop.}
 proc generateExceptionCheck(futSym,
     tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} =
@@ -379,6 +382,9 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
                                   procBody, nnkIteratorDef)
     closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom=prc.body)
     closureIterator.addPragma(newIdentNode("closure"))
+
+    # If proc has an explicit gcsafe pragma, we add it to iterator as well.
+    if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == "gcsafe") != nil:
       closureIterator.addPragma(newIdentNode("gcsafe"))
     outerProcBody.add(closureIterator)
 

lib/pure/asyncnet.nim

@@ -220,7 +220,7 @@ when defineSsl:
       raiseSSLError("Cannot appease SSL.")
 
   template sslLoop(socket: AsyncSocket, flags: set[SocketFlag],
-                   op: expr) =
+                   op: untyped) =
     var opResult {.inject.} = -1.cint
     while opResult < 0:
       # Call the desired operation.
@@ -490,7 +490,7 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string],
   # them when the result future is completed.
   # Can we replace the result future with the FutureVar?
 
-  template addNLIfEmpty(): stmt =
+  template addNLIfEmpty(): untyped =
     if resString.mget.len == 0:
       resString.mget.add("\c\L")
 

lib/pure/asyncstreams.nim

@@ -0,0 +1,105 @@
+import asyncfutures
+
+import deques
+
+type
+  FutureStream*[T] = ref object   ## Special future that acts as
+                                  ## a queue. Its API is still
+                                  ## experimental and so is
+                                  ## subject to change.
+    queue: Deque[T]
+    finished: bool
+    cb: proc () {.closure, gcsafe.}
+
+proc newFutureStream*[T](fromProc = "unspecified"): FutureStream[T] =
+  ## Create a new ``FutureStream``. This future's callback is activated when
+  ## two events occur:
+  ##
+  ## * New data is written into the future stream.
+  ## * The future stream is completed (this means that no more data will be
+  ##   written).
+  ##
+  ## Specifying ``fromProc``, which is a string specifying the name of the proc
+  ## that this future belongs to, is a good habit as it helps with debugging.
+  ##
+  ## **Note:** The API of FutureStream is still new and so has a higher
+  ## likelihood of changing in the future.
+  result = FutureStream[T](finished: false, cb: nil)
+  result.queue = initDeque[T]()
+
+proc complete*[T](future: FutureStream[T]) =
+  ## Completes a ``FutureStream`` signalling the end of data.
+  future.finished = true
+  if not future.cb.isNil:
+    future.cb()
+
+proc `callback=`*[T](future: FutureStream[T],
+    cb: proc (future: FutureStream[T]) {.closure,gcsafe.}) =
+  ## Sets the callback proc to be called when data was placed inside the
+  ## future stream.
+  ##
+  ## The callback is also called when the future is completed. So you should
+  ## use ``finished`` to check whether data is available.
+  ##
+  ## If the future stream already has data or is finished then ``cb`` will be
+  ## called immediately.
+  future.cb = proc () = cb(future)
+  if future.queue.len > 0 or future.finished:
+    callSoon(future.cb)
+
+proc finished*[T](future: FutureStream[T]): bool =
+  ## Check if a ``FutureStream`` is finished. ``true`` value means that
+  ## no more data will be placed inside the stream _and_ that there is
+  ## no data waiting to be retrieved.
+  result = future.finished and future.queue.len == 0
+
+proc write*[T](future: FutureStream[T], value: T): Future[void] =
+  ## Writes the specified value inside the specified future stream.
+  ##
+  ## This will raise ``ValueError`` if ``future`` is finished.
+  result = newFuture[void]("FutureStream.put")
+  if future.finished:
+    let msg = "FutureStream is finished and so no longer accepts new data."
+    result.fail(newException(ValueError, msg))
+    return
+  # TODO: Implement limiting of the streams storage to prevent it growing
+  # infinitely when no reads are occuring.
+  future.queue.addLast(value)
+  if not future.cb.isNil: future.cb()
+  result.complete()
+
+proc read*[T](future: FutureStream[T]): Future[(bool, T)] =
+  ## Returns a future that will complete when the ``FutureStream`` has data
+  ## placed into it. The future will be completed with the oldest
+  ## value stored inside the stream. The return value will also determine
+  ## whether data was retrieved, ``false`` means that the future stream was
+  ## completed and no data was retrieved.
+  ##
+  ## This function will remove the data that was returned from the underlying
+  ## ``FutureStream``.
+  var resFut = newFuture[(bool, T)]("FutureStream.take")
+  let savedCb = future.cb
+  future.callback =
+    proc (fs: FutureStream[T]) =
+      # We don't want this callback called again.
+      future.cb = nil
+
+      # The return value depends on whether the FutureStream has finished.
+      var res: (bool, T)
+      if finished(fs):
+        # Remember, this callback is called when the FutureStream is completed.
+        res[0] = false
+      else:
+        res[0] = true
+        res[1] = fs.queue.popFirst()
+
+      if not resFut.finished:
+        resFut.complete(res)
+
+      # If the saved callback isn't nil then let's call it.
+      if not savedCb.isNil: savedCb()
+  return resFut
+
+proc len*[T](future: FutureStream[T]): int =
+  ## Returns the amount of data pieces inside the stream.
+  future.queue.len

lib/pure/basic2d.nim

@@ -116,13 +116,13 @@ proc safeArccos(v:float):float=
   return arccos(clamp(v,-1.0,1.0))
 
 
-template makeBinOpVector(s:expr)=
+template makeBinOpVector(s) =
   ## implements binary operators ``+``, ``-``, ``*`` and ``/`` for vectors
   proc s*(a,b:Vector2d):Vector2d {.inline,noInit.} = vector2d(s(a.x,b.x),s(a.y,b.y))
   proc s*(a:Vector2d,b:float):Vector2d {.inline,noInit.}  = vector2d(s(a.x,b),s(a.y,b))
   proc s*(a:float,b:Vector2d):Vector2d {.inline,noInit.}  = vector2d(s(a,b.x),s(a,b.y))
 
-template makeBinOpAssignVector(s:expr)=
+template makeBinOpAssignVector(s)=
   ## implements inplace binary operators ``+=``, ``-=``, ``/=`` and ``*=`` for vectors
   proc s*(a:var Vector2d,b:Vector2d) {.inline.} = s(a.x,b.x) ; s(a.y,b.y)
   proc s*(a:var Vector2d,b:float) {.inline.} = s(a.x,b) ; s(a.y,b)
@@ -853,5 +853,3 @@ proc degToRad*(deg:float):float {.inline.}=
 proc radToDeg*(rad:float):float {.inline.}=
   ## converts `rad` radians to degrees
   rad * RAD2DEGCONST
-
-

lib/pure/basic3d.nim

@@ -116,7 +116,7 @@ proc safeArccos(v:float):float=
   ## due to rounding issues
   return arccos(clamp(v,-1.0,1.0))
 
-template makeBinOpVector(s:expr)=
+template makeBinOpVector(s) =
   proc s*(a,b:Vector3d):Vector3d {.inline,noInit.} =
     vector3d(s(a.x,b.x),s(a.y,b.y),s(a.z,b.z))
   proc s*(a:Vector3d,b:float):Vector3d {.inline,noInit.}  =
@@ -124,7 +124,7 @@ template makeBinOpVector(s:expr)=
   proc s*(a:float,b:Vector3d):Vector3d {.inline,noInit.}  =
     vector3d(s(a,b.x),s(a,b.y),s(a,b.z))
 
-template makeBinOpAssignVector(s:expr)=
+template makeBinOpAssignVector(s) =
   proc s*(a:var Vector3d,b:Vector3d) {.inline.} =
     s(a.x,b.x); s(a.y,b.y); s(a.z,b.z)
   proc s*(a:var Vector3d,b:float) {.inline.} =

lib/pure/collections/critbits.nim

@@ -143,15 +143,16 @@ proc `[]=`*[T](c: var CritBitTree[T], key: string, val: T) =
   var n = rawInsert(c, key)
   n.val = val
 
-template get[T](c: CritBitTree[T], key: string): T {.immediate.} =
+template get[T](c: CritBitTree[T], key: string): T =
   let n = rawGet(c, key)
-  if n != nil: result = n.val
+  if n == nil:
-  else:
     when compiles($key):
       raise newException(KeyError, "key not found: " & $key)
     else:
       raise newException(KeyError, "key not found")
 
+  n.val
+
 proc `[]`*[T](c: CritBitTree[T], key: string): T {.inline, deprecatedGet.} =
   ## retrieves the value at ``c[key]``. If `key` is not in `t`, the
   ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether

lib/pure/collections/deques.nim

@@ -33,7 +33,7 @@
 ##     assert deq.peekLast == a
 ##
 ##     while deq.len > 0:  # checking if the deque is empty
-##       echo deq.removeLast()
+##       echo deq.popLast()
 ##
 ## Note: For inter thread communication use
 ## a `Channel <channels.html>`_ instead.

lib/pure/collections/intsets.nim

@@ -31,16 +31,18 @@ const
 
 type
   PTrunk = ref Trunk
-  Trunk {.final.} = object
+  Trunk = object
     next: PTrunk             # all nodes are connected with this pointer
     key: int                 # start address at bit 0
     bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector
 
   TrunkSeq = seq[PTrunk]
   IntSet* = object ## an efficient set of 'int' implemented as a sparse bit set
+    elems: int # only valid for small numbers
     counter, max: int
     head: PTrunk
     data: TrunkSeq
+    a: array[0..33, int] # profiling shows that 34 elements are enough
 
 {.deprecated: [TIntSet: IntSet, TTrunk: Trunk, TTrunkSeq: TrunkSeq].}
 
@@ -95,6 +97,10 @@ proc intSetPut(t: var IntSet, key: int): PTrunk =
 
 proc contains*(s: IntSet, key: int): bool =
   ## returns true iff `key` is in `s`.
+  if s.elems <= s.a.len:
+    for i in 0..<s.elems:
+      if s.a[i] == key: return true
+  else:
     var t = intSetGet(s, `shr`(key, TrunkShift))
     if t != nil:
       var u = key and TrunkMask
@@ -102,15 +108,38 @@ proc contains*(s: IntSet, key: int): bool =
     else:
       result = false
 
-proc incl*(s: var IntSet, key: int) =
+proc bitincl(s: var IntSet, key: int) {.inline.} =
-  ## includes an element `key` in `s`.
   var t = intSetPut(s, `shr`(key, TrunkShift))
   var u = key and TrunkMask
   t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or
       `shl`(1, u and IntMask)
 
+proc incl*(s: var IntSet, key: int) =
+  ## includes an element `key` in `s`.
+  if s.elems <= s.a.len:
+    for i in 0..<s.elems:
+      if s.a[i] == key: return
+    if s.elems < s.a.len:
+      s.a[s.elems] = key
+      inc s.elems
+      return
+    newSeq(s.data, InitIntSetSize)
+    s.max = InitIntSetSize-1
+    for i in 0..<s.elems:
+      bitincl(s, s.a[i])
+    s.elems = s.a.len + 1
+    # fall through:
+  bitincl(s, key)
+
 proc excl*(s: var IntSet, key: int) =
   ## excludes `key` from the set `s`.
+  if s.elems <= s.a.len:
+    for i in 0..<s.elems:
+      if s.a[i] == key:
+        s.a[i] = s.a[s.elems-1]
+        dec s.elems
+        return
+  else:
     var t = intSetGet(s, `shr`(key, TrunkShift))
     if t != nil:
       var u = key and TrunkMask
@@ -120,6 +149,13 @@ proc excl*(s: var IntSet, key: int) =
 proc containsOrIncl*(s: var IntSet, key: int): bool =
   ## returns true if `s` contains `key`, otherwise `key` is included in `s`
   ## and false is returned.
+  if s.elems <= s.a.len:
+    for i in 0..<s.elems:
+      if s.a[i] == key:
+        return true
+    incl(s, key)
+    result = false
+  else:
     var t = intSetGet(s, `shr`(key, TrunkShift))
     if t != nil:
       var u = key and TrunkMask
@@ -133,23 +169,38 @@ proc containsOrIncl*(s: var IntSet, key: int): bool =
 
 proc initIntSet*: IntSet =
   ## creates a new int set that is empty.
-  newSeq(result.data, InitIntSetSize)
+
-  result.max = InitIntSetSize-1
+  #newSeq(result.data, InitIntSetSize)
+  #result.max = InitIntSetSize-1
+  result.data = nil
+  result.max = 0
   result.counter = 0
   result.head = nil
+  result.elems = 0
 
 proc clear*(result: var IntSet) =
-  setLen(result.data, InitIntSetSize)
+  #setLen(result.data, InitIntSetSize)
-  for i in 0..InitIntSetSize-1: result.data[i] = nil
+  #for i in 0..InitIntSetSize-1: result.data[i] = nil
-  result.max = InitIntSetSize-1
+  #result.max = InitIntSetSize-1
+  result.data = nil
+  result.max = 0
   result.counter = 0
   result.head = nil
+  result.elems = 0
 
-proc isNil*(x: IntSet): bool {.inline.} = x.head.isNil
+proc isNil*(x: IntSet): bool {.inline.} = x.head.isNil and x.elems == 0
 
 proc assign*(dest: var IntSet, src: IntSet) =
   ## copies `src` to `dest`. `dest` does not need to be initialized by
   ## `initIntSet`.
+  if src.elems <= src.a.len:
+    dest.data = nil
+    dest.max = 0
+    dest.counter = src.counter
+    dest.head = nil
+    dest.elems = src.elems
+    dest.a = src.a
+  else:
     dest.counter = src.counter
     dest.max = src.max
     newSeq(dest.data, src.data.len)
@@ -173,6 +224,10 @@ proc assign*(dest: var IntSet, src: IntSet) =
 
 iterator items*(s: IntSet): int {.inline.} =
   ## iterates over any included element of `s`.
+  if s.elems <= s.a.len:
+    for i in 0..<s.elems:
+      yield s.a[i]
+  else:
     var r = s.head
     while r != nil:
       var i = 0
@@ -189,7 +244,7 @@ iterator items*(s: IntSet): int {.inline.} =
         inc(i)
       r = r.next
 
-template dollarImpl(): stmt =
+template dollarImpl(): untyped =
   result = "{"
   for key in items(s):
     if result.len > 1: result.add(", ")
@@ -225,3 +280,9 @@ when isMainModule:
   ys.sort(cmp[int])
   assert ys == @[1, 2, 7, 1056]
 
+  var z: IntSet
+  for i in 0..1000:
+    incl z, i
+  for i in 0..1000:
+    assert i in z
+

lib/pure/collections/rtarrays.nim

@@ -19,7 +19,7 @@ type
     L: Natural
     spart: seq[T]
     apart: array[ArrayPartSize, T]
-  UncheckedArray* {.unchecked.}[T] = array[0..100_000_000, T]
+  UncheckedArray* {.unchecked.}[T] = array[0, T]
 
 template usesSeqPart(x): untyped = x.L > ArrayPartSize
 

lib/pure/collections/sequtils.nim

@@ -455,7 +455,7 @@ template anyIt*(seq1, pred: untyped): bool =
       break
   result
 
-template toSeq*(iter: untyped): untyped {.oldimmediate.} =
+template toSeq*(iter: untyped): untyped =
   ## Transforms any iterator into a sequence.
   ##
   ## Example:

lib/pure/collections/sharedstrings.nim

@@ -9,10 +9,8 @@
 
 ## Shared string support for Nim.
 
-const ArrayDummySize = when defined(cpu16): 10_000 else: 100_000_000
-
 type
-  UncheckedCharArray {.unchecked.} = array[0..ArrayDummySize, char]
+  UncheckedCharArray = UncheckedArray[char]
 
 type
   Buffer = ptr object

lib/pure/collections/sharedtables.nim

@@ -25,7 +25,7 @@ type
     counter, dataLen: int
     lock: Lock
 
-template maxHash(t): expr = t.dataLen-1
+template maxHash(t): untyped = t.dataLen-1
 
 include tableimpl
 

lib/pure/collections/tableimpl.nim

@@ -85,7 +85,7 @@ template addImpl(enlarge) {.dirty.} =
   rawInsert(t, t.data, key, val, hc, j)
   inc(t.counter)
 
-template maybeRehashPutImpl(enlarge) {.oldimmediate, dirty.} =
+template maybeRehashPutImpl(enlarge) {.dirty.} =
   if mustRehash(t.dataLen, t.counter):
     enlarge(t)
     index = rawGetKnownHC(t, key, hc)
@@ -93,7 +93,7 @@ template maybeRehashPutImpl(enlarge) {.oldimmediate, dirty.} =
   rawInsert(t, t.data, key, val, hc, index)
   inc(t.counter)
 
-template putImpl(enlarge) {.oldimmediate, dirty.} =
+template putImpl(enlarge) {.dirty.} =
   var hc: Hash
   var index = rawGet(t, key, hc)
   if index >= 0: t.data[index].val = val

lib/pure/collections/tables.nim

@@ -648,7 +648,7 @@ proc `==`*[A, B](s, t: OrderedTable[A, B]): bool =
     var nxtt = t.data[ht].next
     var nxts = s.data[hs].next
     if isFilled(t.data[ht].hcode) and isFilled(s.data[hs].hcode):
-      if (s.data[hs].key != t.data[ht].key) and (s.data[hs].val != t.data[ht].val):
+      if (s.data[hs].key != t.data[ht].key) or (s.data[hs].val != t.data[ht].val):
         return false
     ht = nxtt
     hs = nxts
@@ -939,7 +939,7 @@ proc enlarge[A](t: var CountTable[A]) =
 
 proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
   ## puts a (key, value)-pair into `t`.
-  assert val > 0
+  assert val >= 0
   var h = rawGet(t, key)
   if h >= 0:
     t.data[h].val = val
@@ -1311,3 +1311,17 @@ when isMainModule:
     assert a == c
 
 
+  block: #6250
+    let
+      a = {3: 1}.toOrderedTable
+      b = {3: 2}.toOrderedTable
+    assert((a == b) == false)
+    assert((b == a) == false)
+
+  block: #6250
+    let
+      a = {3: 2}.toOrderedTable
+      b = {3: 2}.toOrderedTable
+    assert((a == b) == true)
+    assert((b == a) == true)
+

lib/pure/colors.nim

@@ -19,18 +19,18 @@ type
 proc `==` *(a, b: Color): bool {.borrow.}
   ## compares two colors.
 
-template extract(a: Color, r, g, b: expr) {.immediate.}=
+template extract(a: Color, r, g, b: untyped) =
   var r = a.int shr 16 and 0xff
   var g = a.int shr 8 and 0xff
   var b = a.int and 0xff
 
-template rawRGB(r, g, b: int): expr =
+template rawRGB(r, g, b: int): Color =
   Color(r shl 16 or g shl 8 or b)
 
-template colorOp(op: expr) {.immediate.} =
+template colorOp(op): Color =
   extract(a, ar, ag, ab)
   extract(b, br, bg, bb)
-  result = rawRGB(op(ar, br), op(ag, bg), op(ab, bb))
+  rawRGB(op(ar, br), op(ag, bg), op(ab, bb))
 
 proc satPlus(a, b: int): int {.inline.} =
   result = a +% b
@@ -67,12 +67,12 @@ proc intensity*(a: Color, f: float): Color =
   if b >% 255: b = 255
   result = rawRGB(r, g, b)
 
-template mix*(a, b: Color, fn: expr): expr =
+template mix*(a, b: Color, fn: untyped): untyped =
   ## uses `fn` to mix the colors `a` and `b`. `fn` is invoked for each component
   ## R, G, and B. This is a template because `fn` should be inlined and the
   ## compiler cannot inline proc pointers yet. If `fn`'s result is not in the
   ## range[0..255], it will be saturated to be so.
-  template `><` (x: expr): expr =
+  template `><` (x: untyped): untyped =
     # keep it in the range 0..255
     block:
       var y = x # eval only once

lib/pure/concurrency/threadpool.nim

@@ -409,20 +409,20 @@ proc preferSpawn*(): bool =
   ## it is not necessary to call this directly; use 'spawnX' instead.
   result = gSomeReady.counter > 0
 
-proc spawn*(call: expr): expr {.magic: "Spawn".}
+proc spawn*(call: typed): void {.magic: "Spawn".}
   ## always spawns a new task, so that the 'call' is never executed on
   ## the calling thread. 'call' has to be proc call 'p(...)' where 'p'
   ## is gcsafe and has a return type that is either 'void' or compatible
   ## with ``FlowVar[T]``.
 
-proc pinnedSpawn*(id: ThreadId; call: expr): expr {.magic: "Spawn".}
+proc pinnedSpawn*(id: ThreadId; call: typed): void {.magic: "Spawn".}
   ## always spawns a new task on the worker thread with ``id``, so that
   ## the 'call' is **always** executed on
   ## the thread. 'call' has to be proc call 'p(...)' where 'p'
   ## is gcsafe and has a return type that is either 'void' or compatible
   ## with ``FlowVar[T]``.
 
-template spawnX*(call: expr): expr =
+template spawnX*(call): void =
   ## spawns a new task if a CPU core is ready, otherwise executes the
   ## call in the calling thread. Usually it is advised to
   ## use 'spawn' in order to not block the producer for an unknown
@@ -431,7 +431,7 @@ template spawnX*(call: expr): expr =
   ## with ``FlowVar[T]``.
   (if preferSpawn(): spawn call else: call)
 
-proc parallel*(body: stmt) {.magic: "Parallel".}
+proc parallel*(body: untyped) {.magic: "Parallel".}
   ## a parallel section can be used to execute a block in parallel. ``body``
   ## has to be in a DSL that is a particular subset of the language. Please
   ## refer to the manual for further information.
@@ -530,6 +530,7 @@ proc nimSpawn4(fn: WorkerProc; data: pointer; id: ThreadId) {.compilerProc.} =
 proc sync*() =
   ## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate
   ## waiting, you have to use an explicit barrier.
+  var toRelease = 0
   while true:
     var allReady = true
     for i in 0 .. <currentPoolSize:
@@ -537,5 +538,9 @@ proc sync*() =
       allReady = allReady and workersData[i].ready
     if allReady: break
     await(gSomeReady)
+    inc toRelease
+
+  for i in 0 ..< toRelease:
+    signal(gSomeReady)
 
 setup()

lib/pure/httpclient.nim

@@ -1245,6 +1245,8 @@ proc downloadFile*(client: HttpClient | AsyncHttpClient,
                    url: string, filename: string): Future[void] {.multisync.} =
   ## Downloads ``url`` and saves it to ``filename``.
   client.getBody = false
+  defer:
+    client.getBody = true
   let resp = await client.get(url)
 
   when client is HttpClient:

lib/pure/includes/osenv.nim

@@ -0,0 +1,159 @@
+## Include file that implements 'getEnv' and friends. Do not import it!
+
+when not declared(ospaths):
+  {.error: "This is an include file for ospaths.nim!".}
+
+proc c_getenv(env: cstring): cstring {.
+  importc: "getenv", header: "<stdlib.h>".}
+proc c_putenv(env: cstring): cint {.
+  importc: "putenv", header: "<stdlib.h>".}
+
+# Environment handling cannot be put into RTL, because the ``envPairs``
+# iterator depends on ``environment``.
+
+var
+  envComputed {.threadvar.}: bool
+  environment {.threadvar.}: seq[string]
+
+when defined(windows) and not defined(nimscript):
+  # because we support Windows GUI applications, things get really
+  # messy here...
+  when useWinUnicode:
+    when defined(cpp):
+      proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.
+        importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "<string.h>".}
+    else:
+      proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.
+        importc: "wcschr", header: "<string.h>".}
+  else:
+    proc strEnd(cstr: cstring, c = 0'i32): cstring {.
+      importc: "strchr", header: "<string.h>".}
+
+  proc getEnvVarsC() =
+    if not envComputed:
+      environment = @[]
+      when useWinUnicode:
+        var
+          env = getEnvironmentStringsW()
+          e = env
+        if e == nil: return # an error occurred
+        while true:
+          var eend = strEnd(e)
+          add(environment, $e)
+          e = cast[WideCString](cast[ByteAddress](eend)+2)
+          if eend[1].int == 0: break
+        discard freeEnvironmentStringsW(env)
+      else:
+        var
+          env = getEnvironmentStringsA()
+          e = env
+        if e == nil: return # an error occurred
+        while true:
+          var eend = strEnd(e)
+          add(environment, $e)
+          e = cast[cstring](cast[ByteAddress](eend)+1)
+          if eend[1] == '\0': break
+        discard freeEnvironmentStringsA(env)
+      envComputed = true
+
+else:
+  const
+    useNSGetEnviron = (defined(macosx) and not defined(ios)) or defined(nimscript)
+
+  when useNSGetEnviron:
+    # From the manual:
+    # Shared libraries and bundles don't have direct access to environ,
+    # which is only available to the loader ld(1) when a complete program
+    # is being linked.
+    # The environment routines can still be used, but if direct access to
+    # environ is needed, the _NSGetEnviron() routine, defined in
+    # <crt_externs.h>, can be used to retrieve the address of environ
+    # at runtime.
+    proc NSGetEnviron(): ptr cstringArray {.
+      importc: "_NSGetEnviron", header: "<crt_externs.h>".}
+  else:
+    var gEnv {.importc: "environ".}: cstringArray
+
+  proc getEnvVarsC() =
+    # retrieves the variables of char** env of C's main proc
+    if not envComputed:
+      environment = @[]
+      when useNSGetEnviron:
+        var gEnv = NSGetEnviron()[]
+      var i = 0
+      while true:
+        if gEnv[i] == nil: break
+        add environment, $gEnv[i]
+        inc(i)
+      envComputed = true
+
+proc findEnvVar(key: string): int =
+  getEnvVarsC()
+  var temp = key & '='
+  for i in 0..high(environment):
+    if startsWith(environment[i], temp): return i
+  return -1
+
+proc getEnv*(key: string): TaintedString {.tags: [ReadEnvEffect].} =
+  ## Returns the value of the `environment variable`:idx: named `key`.
+  ##
+  ## If the variable does not exist, "" is returned. To distinguish
+  ## whether a variable exists or it's value is just "", call
+  ## `existsEnv(key)`.
+  when nimvm:
+    discard "built into the compiler"
+  else:
+    var i = findEnvVar(key)
+    if i >= 0:
+      return TaintedString(substr(environment[i], find(environment[i], '=')+1))
+    else:
+      var env = c_getenv(key)
+      if env == nil: return TaintedString("")
+      result = TaintedString($env)
+
+proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} =
+  ## Checks whether the environment variable named `key` exists.
+  ## Returns true if it exists, false otherwise.
+  when nimvm:
+    discard "built into the compiler"
+  else:
+    if c_getenv(key) != nil: return true
+    else: return findEnvVar(key) >= 0
+
+proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} =
+  ## Sets the value of the `environment variable`:idx: named `key` to `val`.
+  ## If an error occurs, `EInvalidEnvVar` is raised.
+
+  # Note: by storing the string in the environment sequence,
+  # we guarantee that we don't free the memory before the program
+  # ends (this is needed for POSIX compliance). It is also needed so that
+  # the process itself may access its modified environment variables!
+  when nimvm:
+    discard "built into the compiler"
+  else:
+    var indx = findEnvVar(key)
+    if indx >= 0:
+      environment[indx] = key & '=' & val
+    else:
+      add environment, (key & '=' & val)
+      indx = high(environment)
+    when defined(windows) and not defined(nimscript):
+      when useWinUnicode:
+        var k = newWideCString(key)
+        var v = newWideCString(val)
+        if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError())
+      else:
+        if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError())
+    else:
+      if c_putenv(environment[indx]) != 0'i32:
+        raiseOSError(osLastError())
+
+iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} =
+  ## Iterate over all `environments variables`:idx:. In the first component
+  ## of the tuple is the name of the current variable stored, in the second
+  ## its value.
+  getEnvVarsC()
+  for i in 0..high(environment):
+    var p = find(environment[i], '=')
+    yield (TaintedString(substr(environment[i], 0, p-1)),
+           TaintedString(substr(environment[i], p+1)))

lib/pure/includes/oserr.nim

@@ -0,0 +1,135 @@
+## Include file that implements 'osErrorMsg' and friends. Do not import it!
+
+when not declared(ospaths):
+  {.error: "This is an include file for ospaths.nim!".}
+
+when not defined(nimscript):
+  var errno {.importc, header: "<errno.h>".}: cint
+
+  proc c_strerror(errnum: cint): cstring {.
+    importc: "strerror", header: "<string.h>".}
+
+  when defined(windows):
+    import winlean
+
+proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} =
+  ## Retrieves the operating system's error flag, ``errno``.
+  ## On Windows ``GetLastError`` is checked before ``errno``.
+  ## Returns "" if no error occurred.
+  ##
+  ## **Deprecated since version 0.9.4**: use the other ``osErrorMsg`` proc.
+
+  result = ""
+  when defined(Windows) and not defined(nimscript):
+    var err = getLastError()
+    if err != 0'i32:
+      when useWinUnicode:
+        var msgbuf: WideCString
+        if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF,
+                          nil, err, 0, addr(msgbuf), 0, nil) != 0'i32:
+          result = $msgbuf
+          if msgbuf != nil: localFree(cast[pointer](msgbuf))
+      else:
+        var msgbuf: cstring
+        if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF,
+                          nil, err, 0, addr(msgbuf), 0, nil) != 0'i32:
+          result = $msgbuf
+          if msgbuf != nil: localFree(msgbuf)
+  when not defined(nimscript):
+    if errno != 0'i32:
+      result = $c_strerror(errno)
+
+{.push warning[deprecated]: off.}
+proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1",
+                                       deprecated.} =
+  ## raises an OSError exception with the given message ``msg``.
+  ## If ``msg == ""``, the operating system's error flag
+  ## (``errno``) is converted to a readable error message. On Windows
+  ## ``GetLastError`` is checked before ``errno``.
+  ## If no error flag is set, the message ``unknown OS error`` is used.
+  ##
+  ## **Deprecated since version 0.9.4**: use the other ``raiseOSError`` proc.
+  if len(msg) == 0:
+    var m = osErrorMsg()
+    raise newException(OSError, if m.len > 0: m else: "unknown OS error")
+  else:
+    raise newException(OSError, msg)
+{.pop.}
+
+when not defined(nimfix):
+  {.deprecated: [osError: raiseOSError].}
+
+proc `==`*(err1, err2: OSErrorCode): bool {.borrow.}
+proc `$`*(err: OSErrorCode): string {.borrow.}
+
+proc osErrorMsg*(errorCode: OSErrorCode): string =
+  ## Converts an OS error code into a human readable string.
+  ##
+  ## The error code can be retrieved using the ``osLastError`` proc.
+  ##
+  ## If conversion fails, or ``errorCode`` is ``0`` then ``""`` will be
+  ## returned.
+  ##
+  ## On Windows, the ``-d:useWinAnsi`` compilation flag can be used to
+  ## make this procedure use the non-unicode Win API calls to retrieve the
+  ## message.
+  result = ""
+  when defined(nimscript):
+    discard
+  elif defined(Windows):
+    if errorCode != OSErrorCode(0'i32):
+      when useWinUnicode:
+        var msgbuf: WideCString
+        if formatMessageW(0x00000100 or 0x00001000 or 0x00000200,
+                        nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32:
+          result = $msgbuf
+          if msgbuf != nil: localFree(cast[pointer](msgbuf))
+      else:
+        var msgbuf: cstring
+        if formatMessageA(0x00000100 or 0x00001000 or 0x00000200,
+                        nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32:
+          result = $msgbuf
+          if msgbuf != nil: localFree(msgbuf)
+  else:
+    if errorCode != OSErrorCode(0'i32):
+      result = $c_strerror(errorCode.int32)
+
+proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} =
+  ## Raises an ``OSError`` exception. The ``errorCode`` will determine the
+  ## message, ``osErrorMsg`` will be used to get this message.
+  ##
+  ## The error code can be retrieved using the ``osLastError`` proc.
+  ##
+  ## If the error code is ``0`` or an error message could not be retrieved,
+  ## the message ``unknown OS error`` will be used.
+  var e: ref OSError; new(e)
+  e.errorCode = errorCode.int32
+  if additionalInfo.len == 0:
+    e.msg = osErrorMsg(errorCode)
+  else:
+    e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo
+  if e.msg == "":
+    e.msg = "unknown OS error"
+  raise e
+
+{.push stackTrace:off.}
+proc osLastError*(): OSErrorCode =
+  ## Retrieves the last operating system error code.
+  ##
+  ## This procedure is useful in the event when an OS call fails. In that case
+  ## this procedure will return the error code describing the reason why the
+  ## OS call failed. The ``OSErrorMsg`` procedure can then be used to convert
+  ## this code into a string.
+  ##
+  ## **Warning**:
+  ## The behaviour of this procedure varies between Windows and POSIX systems.
+  ## On Windows some OS calls can reset the error code to ``0`` causing this
+  ## procedure to return ``0``. It is therefore advised to call this procedure
+  ## immediately after an OS call fails. On POSIX systems this is not a problem.
+  when defined(nimscript):
+    discard
+  elif defined(windows):
+    result = OSErrorCode(getLastError())
+  else:
+    result = OSErrorCode(errno)
+{.pop.}

lib/pure/ioselectors.nim

@@ -208,7 +208,7 @@ else:
     import locks
 
     type
-      SharedArray {.unchecked.}[T] = array[0..100, T]
+      SharedArray[T] = UncheckedArray[T]
 
     proc allocSharedArray[T](nsize: int): ptr SharedArray[T] =
       result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize))

lib/pure/memfiles.nim

@@ -123,7 +123,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
     result.size = 0
 
   when defined(windows):
-    template fail(errCode: OSErrorCode, msg: expr) =
+    template fail(errCode: OSErrorCode, msg: untyped) =
       rollback()
       if result.fHandle != 0: discard closeHandle(result.fHandle)
       if result.mapHandle != 0: discard closeHandle(result.mapHandle)
@@ -131,7 +131,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
       # return false
       #raise newException(EIO, msg)
 
-    template callCreateFile(winApiProc, filename: expr): expr =
+    template callCreateFile(winApiProc, filename): untyped =
       winApiProc(
         filename,
         # GENERIC_ALL != (GENERIC_READ or GENERIC_WRITE)
@@ -198,7 +198,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
         result.fHandle = INVALID_HANDLE_VALUE
 
   else:
-    template fail(errCode: OSErrorCode, msg: expr) =
+    template fail(errCode: OSErrorCode, msg: string) =
       rollback()
       if result.handle != -1: discard close(result.handle)
       raiseOSError(errCode)

lib/pure/nativesockets.nim

@@ -233,7 +233,7 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET,
   # OpenBSD doesn't support AI_V4MAPPED and doesn't define the macro AI_V4MAPPED.
   # FreeBSD doesn't support AI_V4MAPPED but defines the macro.
   # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198092
-  when not defined(freebsd) and not defined(openbsd) and not defined(netbsd):
+  when not defined(freebsd) and not defined(openbsd) and not defined(netbsd) and not defined(android):
     if domain == AF_INET6:
       hints.ai_flags = AI_V4MAPPED
   var gaiResult = getaddrinfo(address, $port, addr(hints), result)

lib/pure/net.nim

@@ -797,7 +797,7 @@ when false: #defineSsl:
     ##
     ## ``AcceptNoClient`` will be returned when no client is currently attempting
     ## to connect.
-    template doHandshake(): stmt =
+    template doHandshake(): untyped =
       when defineSsl:
         if server.isSSL:
           client.setBlocking(false)

lib/pure/os.nim

@@ -29,150 +29,17 @@ else:
 import ospaths
 export ospaths
 
-when defined(posix):
-  when NoFakeVars:
-    const pathMax = 5000 # doesn't matter really. The concept of PATH_MAX
-                         # doesn't work anymore on modern OSes.
-  else:
-    var
-      pathMax {.importc: "PATH_MAX", header: "<stdlib.h>".}: cint
-
 proc c_remove(filename: cstring): cint {.
   importc: "remove", header: "<stdio.h>".}
 proc c_rename(oldname, newname: cstring): cint {.
   importc: "rename", header: "<stdio.h>".}
 proc c_system(cmd: cstring): cint {.
   importc: "system", header: "<stdlib.h>".}
-proc c_strerror(errnum: cint): cstring {.
-  importc: "strerror", header: "<string.h>".}
 proc c_strlen(a: cstring): cint {.
   importc: "strlen", header: "<string.h>", noSideEffect.}
-proc c_getenv(env: cstring): cstring {.
-  importc: "getenv", header: "<stdlib.h>".}
-proc c_putenv(env: cstring): cint {.
-  importc: "putenv", header: "<stdlib.h>".}
 proc c_free(p: pointer) {.
   importc: "free", header: "<stdlib.h>".}
 
-var errno {.importc, header: "<errno.h>".}: cint
-
-proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} =
-  ## Retrieves the operating system's error flag, ``errno``.
-  ## On Windows ``GetLastError`` is checked before ``errno``.
-  ## Returns "" if no error occurred.
-  ##
-  ## **Deprecated since version 0.9.4**: use the other ``osErrorMsg`` proc.
-
-  result = ""
-  when defined(Windows):
-    var err = getLastError()
-    if err != 0'i32:
-      when useWinUnicode:
-        var msgbuf: WideCString
-        if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF,
-                          nil, err, 0, addr(msgbuf), 0, nil) != 0'i32:
-          result = $msgbuf
-          if msgbuf != nil: localFree(cast[pointer](msgbuf))
-      else:
-        var msgbuf: cstring
-        if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF,
-                          nil, err, 0, addr(msgbuf), 0, nil) != 0'i32:
-          result = $msgbuf
-          if msgbuf != nil: localFree(msgbuf)
-  if errno != 0'i32:
-    result = $os.c_strerror(errno)
-
-{.push warning[deprecated]: off.}
-proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1",
-                                       deprecated.} =
-  ## raises an OSError exception with the given message ``msg``.
-  ## If ``msg == ""``, the operating system's error flag
-  ## (``errno``) is converted to a readable error message. On Windows
-  ## ``GetLastError`` is checked before ``errno``.
-  ## If no error flag is set, the message ``unknown OS error`` is used.
-  ##
-  ## **Deprecated since version 0.9.4**: use the other ``raiseOSError`` proc.
-  if len(msg) == 0:
-    var m = osErrorMsg()
-    raise newException(OSError, if m.len > 0: m else: "unknown OS error")
-  else:
-    raise newException(OSError, msg)
-{.pop.}
-
-when not defined(nimfix):
-  {.deprecated: [osError: raiseOSError].}
-
-proc `==`*(err1, err2: OSErrorCode): bool {.borrow.}
-proc `$`*(err: OSErrorCode): string {.borrow.}
-
-proc osErrorMsg*(errorCode: OSErrorCode): string =
-  ## Converts an OS error code into a human readable string.
-  ##
-  ## The error code can be retrieved using the ``osLastError`` proc.
-  ##
-  ## If conversion fails, or ``errorCode`` is ``0`` then ``""`` will be
-  ## returned.
-  ##
-  ## On Windows, the ``-d:useWinAnsi`` compilation flag can be used to
-  ## make this procedure use the non-unicode Win API calls to retrieve the
-  ## message.
-  result = ""
-  when defined(Windows):
-    if errorCode != OSErrorCode(0'i32):
-      when useWinUnicode:
-        var msgbuf: WideCString
-        if formatMessageW(0x00000100 or 0x00001000 or 0x00000200,
-                        nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32:
-          result = $msgbuf
-          if msgbuf != nil: localFree(cast[pointer](msgbuf))
-      else:
-        var msgbuf: cstring
-        if formatMessageA(0x00000100 or 0x00001000 or 0x00000200,
-                        nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32:
-          result = $msgbuf
-          if msgbuf != nil: localFree(msgbuf)
-  else:
-    if errorCode != OSErrorCode(0'i32):
-      result = $os.c_strerror(errorCode.int32)
-
-proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} =
-  ## Raises an ``OSError`` exception. The ``errorCode`` will determine the
-  ## message, ``osErrorMsg`` will be used to get this message.
-  ##
-  ## The error code can be retrieved using the ``osLastError`` proc.
-  ##
-  ## If the error code is ``0`` or an error message could not be retrieved,
-  ## the message ``unknown OS error`` will be used.
-  var e: ref OSError; new(e)
-  e.errorCode = errorCode.int32
-  if additionalInfo.len == 0:
-    e.msg = osErrorMsg(errorCode)
-  else:
-    e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo
-  if e.msg == "":
-    e.msg = "unknown OS error"
-  raise e
-
-{.push stackTrace:off.}
-proc osLastError*(): OSErrorCode =
-  ## Retrieves the last operating system error code.
-  ##
-  ## This procedure is useful in the event when an OS call fails. In that case
-  ## this procedure will return the error code describing the reason why the
-  ## OS call failed. The ``OSErrorMsg`` procedure can then be used to convert
-  ## this code into a string.
-  ##
-  ## **Warning**:
-  ## The behaviour of this procedure varies between Windows and POSIX systems.
-  ## On Windows some OS calls can reset the error code to ``0`` causing this
-  ## procedure to return ``0``. It is therefore advised to call this procedure
-  ## immediately after an OS call fails. On POSIX systems this is not a problem.
-
-  when defined(windows):
-    result = OSErrorCode(getLastError())
-  else:
-    result = OSErrorCode(errno)
-{.pop.}
 
 when defined(windows):
   when useWinUnicode:
@@ -252,6 +119,60 @@ proc dirExists*(dir: string): bool {.inline.} =
   ## Synonym for existsDir
   existsDir(dir)
 
+when not defined(windows):
+  proc checkSymlink(path: string): bool =
+    var rawInfo: Stat
+    if lstat(path, rawInfo) < 0'i32: result = false
+    else: result = S_ISLNK(rawInfo.st_mode)
+
+const
+  ExeExts* = when defined(windows): ["exe", "cmd", "bat"] else: [""] ## \
+    ## platform specific file extension for executables. On Windows
+    ## ``["exe", "cmd", "bat"]``, on Posix ``[""]``.
+
+proc findExe*(exe: string, followSymlinks: bool = true;
+              extensions: openarray[string]=ExeExts): string {.
+  tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect].} =
+  ## Searches for `exe` in the current working directory and then
+  ## in directories listed in the ``PATH`` environment variable.
+  ## Returns "" if the `exe` cannot be found. `exe`
+  ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none.
+  ## If the system supports symlinks it also resolves them until it
+  ## meets the actual file. This behavior can be disabled if desired.
+  for ext in extensions:
+    result = addFileExt(exe, ext)
+    if existsFile(result): return
+  var path = string(getEnv("PATH"))
+  for candidate in split(path, PathSep):
+    when defined(windows):
+      var x = (if candidate[0] == '"' and candidate[^1] == '"':
+                substr(candidate, 1, candidate.len-2) else: candidate) /
+              exe
+    else:
+      var x = expandTilde(candidate) / exe
+    for ext in extensions:
+      var x = addFileExt(x, ext)
+      if existsFile(x):
+        when not defined(windows):
+          while followSymlinks: # doubles as if here
+            if x.checkSymlink:
+              var r = newString(256)
+              var len = readlink(x, r, 256)
+              if len < 0:
+                raiseOSError(osLastError())
+              if len > 256:
+                r = newString(len+1)
+                len = readlink(x, r, len)
+              setLen(r, len)
+              if isAbsolute(r):
+                x = r
+              else:
+                x = parentDir(x) / r
+            else:
+              break
+        return x
+  result = ""
+
 proc getLastModificationTime*(file: string): Time {.rtl, extern: "nos$1".} =
   ## Returns the `file`'s last modification time.
   when defined(posix):
@@ -714,147 +635,6 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1",
   else:
     result = c_system(command)
 
-# Environment handling cannot be put into RTL, because the ``envPairs``
-# iterator depends on ``environment``.
-
-var
-  envComputed {.threadvar.}: bool
-  environment {.threadvar.}: seq[string]
-
-when defined(windows):
-  # because we support Windows GUI applications, things get really
-  # messy here...
-  when useWinUnicode:
-    when defined(cpp):
-      proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.
-        importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "<string.h>".}
-    else:
-      proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.
-        importc: "wcschr", header: "<string.h>".}
-  else:
-    proc strEnd(cstr: cstring, c = 0'i32): cstring {.
-      importc: "strchr", header: "<string.h>".}
-
-  proc getEnvVarsC() =
-    if not envComputed:
-      environment = @[]
-      when useWinUnicode:
-        var
-          env = getEnvironmentStringsW()
-          e = env
-        if e == nil: return # an error occurred
-        while true:
-          var eend = strEnd(e)
-          add(environment, $e)
-          e = cast[WideCString](cast[ByteAddress](eend)+2)
-          if eend[1].int == 0: break
-        discard freeEnvironmentStringsW(env)
-      else:
-        var
-          env = getEnvironmentStringsA()
-          e = env
-        if e == nil: return # an error occurred
-        while true:
-          var eend = strEnd(e)
-          add(environment, $e)
-          e = cast[cstring](cast[ByteAddress](eend)+1)
-          if eend[1] == '\0': break
-        discard freeEnvironmentStringsA(env)
-      envComputed = true
-
-else:
-  const
-    useNSGetEnviron = defined(macosx) and not defined(ios)
-
-  when useNSGetEnviron:
-    # From the manual:
-    # Shared libraries and bundles don't have direct access to environ,
-    # which is only available to the loader ld(1) when a complete program
-    # is being linked.
-    # The environment routines can still be used, but if direct access to
-    # environ is needed, the _NSGetEnviron() routine, defined in
-    # <crt_externs.h>, can be used to retrieve the address of environ
-    # at runtime.
-    proc NSGetEnviron(): ptr cstringArray {.
-      importc: "_NSGetEnviron", header: "<crt_externs.h>".}
-  else:
-    var gEnv {.importc: "environ".}: cstringArray
-
-  proc getEnvVarsC() =
-    # retrieves the variables of char** env of C's main proc
-    if not envComputed:
-      environment = @[]
-      when useNSGetEnviron:
-        var gEnv = NSGetEnviron()[]
-      var i = 0
-      while true:
-        if gEnv[i] == nil: break
-        add environment, $gEnv[i]
-        inc(i)
-      envComputed = true
-
-proc findEnvVar(key: string): int =
-  getEnvVarsC()
-  var temp = key & '='
-  for i in 0..high(environment):
-    if startsWith(environment[i], temp): return i
-  return -1
-
-proc getEnv*(key: string): TaintedString {.tags: [ReadEnvEffect].} =
-  ## Returns the value of the `environment variable`:idx: named `key`.
-  ##
-  ## If the variable does not exist, "" is returned. To distinguish
-  ## whether a variable exists or it's value is just "", call
-  ## `existsEnv(key)`.
-  var i = findEnvVar(key)
-  if i >= 0:
-    return TaintedString(substr(environment[i], find(environment[i], '=')+1))
-  else:
-    var env = c_getenv(key)
-    if env == nil: return TaintedString("")
-    result = TaintedString($env)
-
-proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} =
-  ## Checks whether the environment variable named `key` exists.
-  ## Returns true if it exists, false otherwise.
-  if c_getenv(key) != nil: return true
-  else: return findEnvVar(key) >= 0
-
-proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} =
-  ## Sets the value of the `environment variable`:idx: named `key` to `val`.
-  ## If an error occurs, `EInvalidEnvVar` is raised.
-
-  # Note: by storing the string in the environment sequence,
-  # we guarantee that we don't free the memory before the program
-  # ends (this is needed for POSIX compliance). It is also needed so that
-  # the process itself may access its modified environment variables!
-  var indx = findEnvVar(key)
-  if indx >= 0:
-    environment[indx] = key & '=' & val
-  else:
-    add environment, (key & '=' & val)
-    indx = high(environment)
-  when defined(windows):
-    when useWinUnicode:
-      var k = newWideCString(key)
-      var v = newWideCString(val)
-      if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError())
-    else:
-      if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError())
-  else:
-    if c_putenv(environment[indx]) != 0'i32:
-      raiseOSError(osLastError())
-
-iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} =
-  ## Iterate over all `environments variables`:idx:. In the first component
-  ## of the tuple is the name of the current variable stored, in the second
-  ## its value.
-  getEnvVarsC()
-  for i in 0..high(environment):
-    var p = find(environment[i], '=')
-    yield (TaintedString(substr(environment[i], 0, p-1)),
-           TaintedString(substr(environment[i], p+1)))
-
 # Templates for filtering directories and files
 when defined(windows):
   template isDir(f: WIN32_FIND_DATA): bool =
@@ -1017,7 +797,7 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path:
               y = dir / y
             var k = pcFile
 
-            when defined(linux) or defined(macosx) or defined(bsd):
+            when defined(linux) or defined(macosx) or defined(bsd) or defined(genode):
               if x.d_type != DT_UNKNOWN:
                 if x.d_type == DT_DIR: k = pcDir
                 if x.d_type == DT_LNK:
@@ -1184,7 +964,9 @@ proc createSymlink*(src, dest: string) =
   ## Some OS's (such as Microsoft Windows) restrict the creation
   ## of symlinks to root users (administrators).
   when defined(Windows):
-    let flag = dirExists(src).int32
+    # 2 is the SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE. This allows
+    # anyone with developer mode on to create a link
+    let flag = dirExists(src).int32 or 2
     when useWinUnicode:
       var wSrc = newWideCString(src)
       var wDst = newWideCString(dest)
@@ -1386,7 +1168,7 @@ proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect].} =
       copyDir(source, dest)
       removeDir(source)
 
-include ospaths
+#include ospaths
 
 proc expandSymlink*(symlinkPath: string): string =
   ## Returns a string representing the path to which the symbolic link points.

lib/pure/ospaths.nim

@@ -7,19 +7,14 @@
 #    distribution, for details about the copyright.
 #
 
-# Included by the ``os`` module but a module in its own right for NimScript
+# Forwarded by the ``os`` module but a module in its own right for NimScript
 # support.
 
-when not declared(os):
+include "system/inclrtl"
-  {.pragma: rtl.}
-  import strutils
 
-when defined(nimscript) or (defined(nimdoc) and not declared(os)):
+import strutils
-  {.pragma: rtl.}
-  {.push hint[ConvFromXtoItselfNotNeeded]:off.}
 
-when not declared(getEnv) or defined(nimscript):
+type
-  type
   ReadEnvEffect* = object of ReadIOEffect   ## effect that denotes a read
                                             ## from an environment variable
   WriteEnvEffect* = object of WriteIOEffect ## effect that denotes a write
@@ -34,15 +29,15 @@ when not declared(getEnv) or defined(nimscript):
 
   OSErrorCode* = distinct int32 ## Specifies an OS Error Code.
 
-  {.deprecated: [FReadEnv: ReadEnvEffect, FWriteEnv: WriteEnvEffect,
+{.deprecated: [FReadEnv: ReadEnvEffect, FWriteEnv: WriteEnvEffect,
     FReadDir: ReadDirEffect,
     FWriteDir: WriteDirEffect,
     TOSErrorCode: OSErrorCode
-  ].}
+].}
-  const
+const
   doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS)
 
-  when defined(Nimdoc): # only for proper documentation:
+when defined(Nimdoc): # only for proper documentation:
   const
     CurDir* = '.'
       ## The constant string used by the operating system to refer to the
@@ -88,7 +83,7 @@ when not declared(getEnv) or defined(nimscript):
       ## The format string to turn a filename into a `DLL`:idx: file (also
       ## called `shared object`:idx: on some operating systems).
 
-  elif defined(macos):
+elif defined(macos):
   const
     CurDir* = ':'
     ParDir* = "::"
@@ -119,7 +114,7 @@ when not declared(getEnv) or defined(nimscript):
   #  waterproof. In case of equal names the first volume found will do.
   #  Two colons "::" are the relative path to the parent. Three is to the
   #  grandparent etc.
-  elif doslikeFileSystem:
+elif doslikeFileSystem:
   const
     CurDir* = '.'
     ParDir* = ".."
@@ -130,7 +125,7 @@ when not declared(getEnv) or defined(nimscript):
     ExeExt* = "exe"
     ScriptExt* = "bat"
     DynlibFormat* = "$1.dll"
-  elif defined(PalmOS) or defined(MorphOS):
+elif defined(PalmOS) or defined(MorphOS):
   const
     DirSep* = '/'
     AltSep* = Dirsep
@@ -140,7 +135,7 @@ when not declared(getEnv) or defined(nimscript):
     ExeExt* = ""
     ScriptExt* = ""
     DynlibFormat* = "$1.prc"
-  elif defined(RISCOS):
+elif defined(RISCOS):
   const
     DirSep* = '.'
     AltSep* = '.'
@@ -150,7 +145,7 @@ when not declared(getEnv) or defined(nimscript):
     ExeExt* = ""
     ScriptExt* = ""
     DynlibFormat* = "lib$1.so"
-  else: # UNIX-like operating system
+else: # UNIX-like operating system
   const
     CurDir* = '.'
     ParDir* = ".."
@@ -162,13 +157,13 @@ when not declared(getEnv) or defined(nimscript):
     ScriptExt* = ""
     DynlibFormat* = when defined(macosx): "lib$1.dylib" else: "lib$1.so"
 
-  const
+const
   ExtSep* = '.'
     ## The character which separates the base filename from the extension;
     ## for example, the '.' in ``os.nim``.
 
 
-  proc joinPath*(head, tail: string): string {.
+proc joinPath*(head, tail: string): string {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Joins two directory names to one.
   ##
@@ -206,7 +201,7 @@ when not declared(getEnv) or defined(nimscript):
     else:
       result = head & DirSep & tail
 
-  proc joinPath*(parts: varargs[string]): string {.noSideEffect,
+proc joinPath*(parts: varargs[string]): string {.noSideEffect,
   rtl, extern: "nos$1OpenArray".} =
   ## The same as `joinPath(head, tail)`, but works with any number of
   ## directory parts. You need to pass at least one element or the proc
@@ -215,7 +210,7 @@ when not declared(getEnv) or defined(nimscript):
   for i in 1..high(parts):
     result = joinPath(result, parts[i])
 
-  proc `/` * (head, tail: string): string {.noSideEffect.} =
+proc `/` * (head, tail: string): string {.noSideEffect.} =
   ## The same as ``joinPath(head, tail)``
   ##
   ## Here are some examples for Unix:
@@ -227,7 +222,7 @@ when not declared(getEnv) or defined(nimscript):
   ##   assert "usr/" / "/lib" == "usr/lib"
   return joinPath(head, tail)
 
-  proc splitPath*(path: string): tuple[head, tail: string] {.
+proc splitPath*(path: string): tuple[head, tail: string] {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Splits a directory into (head, tail), so that
   ## ``head / tail == path`` (except for edge cases like "/usr").
@@ -252,14 +247,14 @@ when not declared(getEnv) or defined(nimscript):
     result.head = ""
     result.tail = path
 
-  proc parentDirPos(path: string): int =
+proc parentDirPos(path: string): int =
   var q = 1
   if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2
   for i in countdown(len(path)-q, 0):
     if path[i] in {DirSep, AltSep}: return i
   result = -1
 
-  proc parentDir*(path: string): string {.
+proc parentDir*(path: string): string {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Returns the parent directory of `path`.
   ##
@@ -273,7 +268,7 @@ when not declared(getEnv) or defined(nimscript):
   else:
     result = ""
 
-  proc tailDir*(path: string): string {.
+proc tailDir*(path: string): string {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Returns the tail part of `path`..
   ##
@@ -287,12 +282,12 @@ when not declared(getEnv) or defined(nimscript):
       return substr(path, i+1)
   result = ""
 
-  proc isRootDir*(path: string): bool {.
+proc isRootDir*(path: string): bool {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Checks whether a given `path` is a root directory
   result = parentDirPos(path) < 0
 
-  iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string =
+iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string =
   ## Walks over all parent directories of a given `path`
   ##
   ## If `fromRoot` is set, the traversal will start from the file system root
@@ -317,7 +312,7 @@ when not declared(getEnv) or defined(nimscript):
 
     if inclusive: yield path
 
-  proc `/../` * (head, tail: string): string {.noSideEffect.} =
+proc `/../`*(head, tail: string): string {.noSideEffect.} =
   ## The same as ``parentDir(head) / tail`` unless there is no parent
   ## directory. Then ``head / tail`` is performed instead.
   let sepPos = parentDirPos(head)
@@ -326,11 +321,11 @@ when not declared(getEnv) or defined(nimscript):
   else:
     result = head / tail
 
-  proc normExt(ext: string): string =
+proc normExt(ext: string): string =
   if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here
   else: result = ExtSep & ext
 
-  proc searchExtPos*(path: string): int =
+proc searchExtPos*(path: string): int =
   ## Returns index of the '.' char in `path` if it signifies the beginning
   ## of extension. Returns -1 otherwise.
   # BUGFIX: do not search until 0! .DS_Store is no file extension!
@@ -342,7 +337,7 @@ when not declared(getEnv) or defined(nimscript):
     elif path[i] in {DirSep, AltSep}:
       break # do not skip over path
 
-  proc splitFile*(path: string): tuple[dir, name, ext: string] {.
+proc splitFile*(path: string): tuple[dir, name, ext: string] {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Splits a filename into (dir, filename, extension).
   ## `dir` does not end in `DirSep`.
@@ -375,7 +370,7 @@ when not declared(getEnv) or defined(nimscript):
     result.name = substr(path, sepPos+1, dotPos-1)
     result.ext = substr(path, dotPos)
 
-  proc extractFilename*(path: string): string {.
+proc extractFilename*(path: string): string {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Extracts the filename of a given `path`. This is the same as
   ## ``name & ext`` from ``splitFile(path)``.
@@ -385,7 +380,7 @@ when not declared(getEnv) or defined(nimscript):
     result = splitPath(path).tail
 
 
-  proc changeFileExt*(filename, ext: string): string {.
+proc changeFileExt*(filename, ext: string): string {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Changes the file extension to `ext`.
   ##
@@ -398,7 +393,7 @@ when not declared(getEnv) or defined(nimscript):
   if extPos < 0: result = filename & normExt(ext)
   else: result = substr(filename, 0, extPos-1) & normExt(ext)
 
-  proc addFileExt*(filename, ext: string): string {.
+proc addFileExt*(filename, ext: string): string {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Adds the file extension `ext` to `filename`, unless
   ## `filename` already has an extension.
@@ -410,7 +405,7 @@ when not declared(getEnv) or defined(nimscript):
   if extPos < 0: result = filename & normExt(ext)
   else: result = filename
 
-  proc cmpPaths*(pathA, pathB: string): int {.
+proc cmpPaths*(pathA, pathB: string): int {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Compares two paths.
   ##
@@ -429,7 +424,7 @@ when not declared(getEnv) or defined(nimscript):
     else:
       result = cmpIgnoreCase(pathA, pathB)
 
-  proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1".} =
+proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1".} =
   ## Checks whether a given `path` is absolute.
   ##
   ## On Windows, network paths are considered absolute too.
@@ -444,7 +439,7 @@ when not declared(getEnv) or defined(nimscript):
   elif defined(posix):
     result = path[0] == '/'
 
-  proc unixToNativePath*(path: string, drive=""): string {.
+proc unixToNativePath*(path: string, drive=""): string {.
   noSideEffect, rtl, extern: "nos$1".} =
   ## Converts an UNIX-like path to a native one.
   ##
@@ -499,12 +494,11 @@ when not declared(getEnv) or defined(nimscript):
         add result, path[i]
         inc(i)
 
-when defined(nimdoc) and not declared(os):
+include "includes/oserr"
-  proc getEnv(x: string): string = discard
+when not defined(nimscript):
-  proc existsFile(x: string): bool = discard
+  include "includes/osenv"
 
-when declared(getEnv) or defined(nimscript):
+proc getHomeDir*(): string {.rtl, extern: "nos$1",
-  proc getHomeDir*(): string {.rtl, extern: "nos$1",
   tags: [ReadEnvEffect, ReadIOEffect].} =
   ## Returns the home directory of the current user.
   ##
@@ -513,7 +507,7 @@ when declared(getEnv) or defined(nimscript):
   when defined(windows): return string(getEnv("USERPROFILE")) & "\\"
   else: return string(getEnv("HOME")) & "/"
 
-  proc getConfigDir*(): string {.rtl, extern: "nos$1",
+proc getConfigDir*(): string {.rtl, extern: "nos$1",
   tags: [ReadEnvEffect, ReadIOEffect].} =
   ## Returns the config directory of the current user for applications.
   ##
@@ -528,14 +522,23 @@ when declared(getEnv) or defined(nimscript):
   elif getEnv("XDG_CONFIG_DIR"): return string(getEnv("XDG_CONFIG_DIR")) & "/"
   else: return string(getEnv("HOME")) & "/.config/"
 
-  proc getTempDir*(): string {.rtl, extern: "nos$1",
+proc getTempDir*(): string {.rtl, extern: "nos$1",
   tags: [ReadEnvEffect, ReadIOEffect].} =
   ## Returns the temporary directory of the current user for applications to
   ## save temporary files in.
-    when defined(windows): return string(getEnv("TEMP")) & "\\"
+  ##
+  ## **Please do not use this**: On Android, it currently
+  ## returns ``getHomeDir()``, and on other Unix based systems it can cause
+  ## security problems too. That said, you can override this implementation
+  ## by adding ``-d:tempDir=mytempname`` to your compiler invokation.
+  when defined(tempDir):
+    const tempDir {.strdefine.}: string = nil
+    return tempDir
+  elif defined(windows): return string(getEnv("TEMP")) & "\\"
+  elif defined(android): return getHomeDir()
   else: return "/tmp/"
 
-  proc expandTilde*(path: string): string {.
+proc expandTilde*(path: string): string {.
   tags: [ReadEnvEffect, ReadIOEffect].} =
   ## Expands a path starting with ``~/`` to a full path.
   ##
@@ -555,70 +558,3 @@ when declared(getEnv) or defined(nimscript):
     result = getHomeDir() / path.substr(2)
   else:
     result = path
-
-  when not declared(split):
-    iterator split(s: string, sep: char): string =
-      var last = 0
-      if len(s) > 0:
-        while last <= len(s):
-          var first = last
-          while last < len(s) and s[last] != sep: inc(last)
-          yield substr(s, first, last-1)
-          inc(last)
-
-  when not defined(windows) and declared(os):
-    proc checkSymlink(path: string): bool =
-      var rawInfo: Stat
-      if lstat(path, rawInfo) < 0'i32: result = false
-      else: result = S_ISLNK(rawInfo.st_mode)
-
-  const
-    ExeExts* = when defined(windows): ["exe", "cmd", "bat"] else: [""] ## \
-      ## platform specific file extension for executables. On Windows
-      ## ``["exe", "cmd", "bat"]``, on Posix ``[""]``.
-
-  proc findExe*(exe: string, followSymlinks: bool = true;
-                extensions: openarray[string]=ExeExts): string {.
-    tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect].} =
-    ## Searches for `exe` in the current working directory and then
-    ## in directories listed in the ``PATH`` environment variable.
-    ## Returns "" if the `exe` cannot be found. `exe`
-    ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none.
-    ## If the system supports symlinks it also resolves them until it
-    ## meets the actual file. This behavior can be disabled if desired.
-    for ext in extensions:
-      result = addFileExt(exe, ext)
-      if existsFile(result): return
-    var path = string(getEnv("PATH"))
-    for candidate in split(path, PathSep):
-      when defined(windows):
-        var x = (if candidate[0] == '"' and candidate[^1] == '"':
-                  substr(candidate, 1, candidate.len-2) else: candidate) /
-               exe
-      else:
-        var x = expandTilde(candidate) / exe
-      for ext in extensions:
-        var x = addFileExt(x, ext)
-        if existsFile(x):
-          when not defined(windows) and declared(os):
-            while followSymlinks: # doubles as if here
-              if x.checkSymlink:
-                var r = newString(256)
-                var len = readlink(x, r, 256)
-                if len < 0:
-                  raiseOSError(osLastError())
-                if len > 256:
-                  r = newString(len+1)
-                  len = readlink(x, r, len)
-                setLen(r, len)
-                if isAbsolute(r):
-                  x = r
-                else:
-                  x = parentDir(x) / r
-              else:
-                break
-          return x
-    result = ""
-
-when defined(nimscript) or (defined(nimdoc) and not declared(os)):
-  {.pop.} # hint[ConvFromXtoItselfNotNeeded]:off

lib/pure/osproc.nim

@@ -119,7 +119,8 @@ proc execProcess*(command: string,
                                                   poUsePath,
                                                   poEvalCommand}): TaintedString {.
                                                   rtl, extern: "nosp$1",
-                                                  tags: [ExecIOEffect, ReadIOEffect].}
+                                                  tags: [ExecIOEffect, ReadIOEffect,
+                                                  RootEffect].}
   ## A convenience procedure that executes ``command`` with ``startProcess``
   ## and returns its output as a string.
   ## WARNING: this function uses poEvalCommand by default for backward compatibility.
@@ -131,7 +132,8 @@ proc execProcess*(command: string,
   ##  # Note: outp may have an interleave of text from the nim compile
   ##  # and any output from mytestfile when it runs
 
-proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect].}
+proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect,
+  ReadIOEffect, RootEffect].}
   ## Executes ``command`` and returns its error code. Standard input, output,
   ## error streams are inherited from the calling process. This operation
   ## is also often called `system`:idx:.
@@ -145,7 +147,8 @@ proc startProcess*(command: string,
                    args: openArray[string] = [],
                    env: StringTableRef = nil,
                    options: set[ProcessOption] = {poStdErrToStdOut}):
-              Process {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadEnvEffect].}
+              Process {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadEnvEffect,
+              RootEffect].}
   ## Starts a process. `Command` is the executable file, `workingDir` is the
   ## process's working directory. If ``workingDir == ""`` the current directory
   ## is used. `args` are the command line arguments that are passed to the
@@ -170,7 +173,7 @@ proc startProcess*(command: string,
 
 proc startCmd*(command: string, options: set[ProcessOption] = {
                poStdErrToStdOut, poUsePath}): Process {.
-               tags: [ExecIOEffect, ReadEnvEffect], deprecated.} =
+               tags: [ExecIOEffect, ReadEnvEffect, RootEffect], deprecated.} =
   ## Deprecated - use `startProcess` directly.
   result = startProcess(command=command, options=options + {poEvalCommand})
 
@@ -721,7 +724,7 @@ elif not defined(useNimRtl):
       inc(i)
 
   type StartProcessData = object
-    sysCommand: cstring
+    sysCommand: string
     sysArgs: cstringArray
     sysEnv: cstringArray
     workingDir: cstring
@@ -735,13 +738,13 @@ elif not defined(useNimRtl):
                              not defined(useClone) and not defined(linux)
   when useProcessAuxSpawn:
     proc startProcessAuxSpawn(data: StartProcessData): Pid {.
-      tags: [ExecIOEffect, ReadEnvEffect], gcsafe.}
+      tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.}
   else:
     proc startProcessAuxFork(data: StartProcessData): Pid {.
-      tags: [ExecIOEffect, ReadEnvEffect], gcsafe.}
+      tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.}
   {.push stacktrace: off, profiler: off.}
   proc startProcessAfterFork(data: ptr StartProcessData) {.
-    tags: [ExecIOEffect, ReadEnvEffect], cdecl, gcsafe.}
+    tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.}
   {.pop.}
 
   proc startProcess(command: string,
@@ -762,7 +765,8 @@ elif not defined(useNimRtl):
     var sysCommand: string
     var sysArgsRaw: seq[string]
     if poEvalCommand in options:
-      sysCommand = "/bin/sh"
+      const useShPath {.strdefine.} = "/bin/sh"
+      sysCommand = useShPath
       sysArgsRaw = @[sysCommand, "-c", command]
       assert args.len == 0, "`args` has to be empty when using poEvalCommand."
     else:
@@ -784,7 +788,7 @@ elif not defined(useNimRtl):
     defer: deallocCStringArray(sysEnv)
 
     var data: StartProcessData
-    data.sysCommand = sysCommand
+    shallowCopy(data.sysCommand, sysCommand)
     data.sysArgs = sysArgs
     data.sysEnv = sysEnv
     data.pStdin = pStdin
@@ -949,11 +953,10 @@ elif not defined(useNimRtl):
     discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC)
 
     if data.optionPoUsePath:
-      when defined(uClibc):
+      when defined(uClibc) or defined(linux):
         # uClibc environment (OpenWrt included) doesn't have the full execvpe
-        discard execve(data.sysCommand, data.sysArgs, data.sysEnv)
+        let exe = findExe(data.sysCommand)
-      elif defined(linux) and not defined(android):
+        discard execve(exe, data.sysArgs, data.sysEnv)
-        discard execvpe(data.sysCommand, data.sysArgs, data.sysEnv)
       else:
         # MacOSX doesn't have execvpe, so we need workaround.
         # On MacOSX we can arrive here only from fork, so this is safe:
@@ -1264,7 +1267,8 @@ elif not defined(useNimRtl):
 proc execCmdEx*(command: string, options: set[ProcessOption] = {
                 poStdErrToStdOut, poUsePath}): tuple[
                 output: TaintedString,
-                exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect], gcsafe.} =
+                exitCode: int] {.tags:
+                [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
   ## a convenience proc that runs the `command`, grabs all its output and
   ## exit code and returns both.
   ##

lib/pure/parsecfg.nim

@@ -320,9 +320,13 @@ proc rawGetTok(c: var CfgParser, tok: var Token) =
     tok.literal = "="
   of '-':
     inc(c.bufpos)
-    if c.buf[c.bufpos] == '-': inc(c.bufpos)
+    if c.buf[c.bufpos] == '-':
+      inc(c.bufpos)
       tok.kind = tkDashDash
       tok.literal = "--"
+    else:
+      dec(c.bufpos)
+      getSymbol(c, tok)
   of ':':
     tok.kind = tkColon
     inc(c.bufpos)

lib/pure/parsesql.nim

@@ -496,6 +496,7 @@ type
     nkPrimaryKey,
     nkForeignKey,
     nkNotNull,
+    nkNull,
 
     nkStmtList,
     nkDot,
@@ -565,8 +566,13 @@ proc newNode(k: SqlNodeKind, s: string): SqlNode =
   result.strVal = s
 
 proc len*(n: SqlNode): int =
-  if isNil(n.sons): result = 0
+  if n.kind in {nkIdent, nkStringLit, nkBitStringLit, nkHexStringLit,
-  else: result = n.sons.len
+                nkIntegerLit, nkNumericLit}:
+    result = 0
+  else:
+    result = n.sons.len
+
+proc `[]`*(n: SqlNode; i: int): SqlNode = n.sons[i]
 
 proc add*(father, n: SqlNode) =
   if isNil(father.sons): father.sons = @[]
@@ -613,6 +619,9 @@ proc eat(p: var SqlParser, keyw: string) =
   else:
     sqlError(p, keyw.toUpper() & " expected")
 
+proc opt(p: var SqlParser, kind: TokKind) =
+  if p.tok.kind == kind: getTok(p)
+
 proc parseDataType(p: var SqlParser): SqlNode =
   if isKeyw(p, "enum"):
     result = newNode(nkEnumDef)
@@ -705,7 +714,7 @@ proc primary(p: var SqlParser): SqlNode =
       result = newNode(nkCall)
       result.add(a)
       getTok(p)
-      while true:
+      while p.tok.kind != tkParRi:
         result.add(parseExpr(p))
         if p.tok.kind == tkComma: getTok(p)
         else: break
@@ -776,9 +785,19 @@ proc parseConstraint(p: var SqlParser): SqlNode =
   expectIdent(p)
   result.add(newNode(nkIdent, p.tok.literal))
   getTok(p)
-  eat(p, "check")
+  optKeyw(p, "check")
   result.add(parseExpr(p))
 
+proc parseParIdentList(p: var SqlParser, father: SqlNode) =
+  eat(p, tkParLe)
+  while true:
+    expectIdent(p)
+    father.add(newNode(nkIdent, p.tok.literal))
+    getTok(p)
+    if p.tok.kind != tkComma: break
+    getTok(p)
+  eat(p, tkParRi)
+
 proc parseColumnConstraints(p: var SqlParser, result: SqlNode) =
   while true:
     if isKeyw(p, "default"):
@@ -795,6 +814,9 @@ proc parseColumnConstraints(p: var SqlParser, result: SqlNode) =
       getTok(p)
       eat(p, "null")
       result.add(newNode(nkNotNull))
+    elif isKeyw(p, "null"):
+      getTok(p)
+      result.add(newNode(nkNull))
     elif isKeyw(p, "identity"):
       getTok(p)
       result.add(newNode(nkIdentity))
@@ -807,6 +829,7 @@ proc parseColumnConstraints(p: var SqlParser, result: SqlNode) =
     elif isKeyw(p, "constraint"):
       result.add(parseConstraint(p))
     elif isKeyw(p, "unique"):
+      getTok(p)
       result.add(newNode(nkUnique))
     else:
       break
@@ -829,16 +852,6 @@ proc parseIfNotExists(p: var SqlParser, k: SqlNodeKind): SqlNode =
   else:
     result = newNode(k)
 
-proc parseParIdentList(p: var SqlParser, father: SqlNode) =
-  eat(p, tkParLe)
-  while true:
-    expectIdent(p)
-    father.add(newNode(nkIdent, p.tok.literal))
-    getTok(p)
-    if p.tok.kind != tkComma: break
-    getTok(p)
-  eat(p, tkParRi)
-
 proc parseTableConstraint(p: var SqlParser): SqlNode =
   if isKeyw(p, "primary"):
     getTok(p)
@@ -866,20 +879,34 @@ proc parseTableConstraint(p: var SqlParser): SqlNode =
   else:
     sqlError(p, "column definition expected")
 
+proc parseUnique(p: var SqlParser): SqlNode =
+  result = parseExpr(p)
+  if result.kind == nkCall: result.kind = nkUnique
+
 proc parseTableDef(p: var SqlParser): SqlNode =
   result = parseIfNotExists(p, nkCreateTable)
   expectIdent(p)
   result.add(newNode(nkIdent, p.tok.literal))
   getTok(p)
   if p.tok.kind == tkParLe:
-    while true:
     getTok(p)
-      if p.tok.kind == tkIdentifier or p.tok.kind == tkQuotedIdentifier:
+    while p.tok.kind != tkParRi:
+      if isKeyw(p, "constraint"):
+        result.add parseConstraint(p)
+      elif isKeyw(p, "primary") or isKeyw(p, "foreign"):
+        result.add parseTableConstraint(p)
+      elif isKeyw(p, "unique"):
+        result.add parseUnique(p)
+      elif p.tok.kind == tkIdentifier or p.tok.kind == tkQuotedIdentifier:
         result.add(parseColumnDef(p))
       else:
         result.add(parseTableConstraint(p))
       if p.tok.kind != tkComma: break
+      getTok(p)
     eat(p, tkParRi)
+    # skip additional crap after 'create table (...) crap;'
+    while p.tok.kind notin {tkSemicolon, tkEof}:
+      getTok(p)
 
 proc parseTypeDef(p: var SqlParser): SqlNode =
   result = parseIfNotExists(p, nkCreateType)
@@ -1046,7 +1073,7 @@ proc parseSelect(p: var SqlParser): SqlNode =
       getTok(p)
     result.add(n)
 
-proc parseStmt(p: var SqlParser): SqlNode =
+proc parseStmt(p: var SqlParser; parent: SqlNode) =
   if isKeyw(p, "create"):
     getTok(p)
     optKeyw(p, "cached")
@@ -1058,21 +1085,23 @@ proc parseStmt(p: var SqlParser): SqlNode =
     optKeyw(p, "unique")
     optKeyw(p, "hash")
     if isKeyw(p, "table"):
-      result = parseTableDef(p)
+      parent.add parseTableDef(p)
     elif isKeyw(p, "type"):
-      result = parseTypeDef(p)
+      parent.add parseTypeDef(p)
     elif isKeyw(p, "index"):
-      result = parseIndexDef(p)
+      parent.add parseIndexDef(p)
     else:
       sqlError(p, "TABLE expected")
   elif isKeyw(p, "insert"):
-    result = parseInsert(p)
+    parent.add parseInsert(p)
   elif isKeyw(p, "update"):
-    result = parseUpdate(p)
+    parent.add parseUpdate(p)
   elif isKeyw(p, "delete"):
-    result = parseDelete(p)
+    parent.add parseDelete(p)
   elif isKeyw(p, "select"):
-    result = parseSelect(p)
+    parent.add parseSelect(p)
+  elif isKeyw(p, "begin"):
+    getTok(p)
   else:
     sqlError(p, "CREATE expected")
 
@@ -1089,9 +1118,8 @@ proc parse(p: var SqlParser): SqlNode =
   ## Syntax errors raise an `EInvalidSql` exception.
   result = newNode(nkStmtList)
   while p.tok.kind != tkEof:
-    var s = parseStmt(p)
+    parseStmt(p, result)
     eat(p, tkSemicolon)
-    result.add(s)
   if result.len == 1:
     result = result.sons[0]
 
@@ -1147,6 +1175,8 @@ proc ra(n: SqlNode, s: var string, indent: int) =
     rs(n, s, indent)
   of nkNotNull:
     s.add(" not null")
+  of nkNull:
+    s.add(" null")
   of nkDot:
     ra(n.sons[0], s, indent)
     s.add(".")
@@ -1330,6 +1360,10 @@ proc renderSQL*(n: SqlNode): string =
   result = ""
   ra(n, result, 0)
 
+proc `$`*(n: SqlNode): string =
+  ## an alias for `renderSQL`.
+  renderSQL(n)
+
 when not defined(testing) and isMainModule:
   echo(renderSQL(parseSQL(newStringStream("""
       CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic');

lib/pure/pegs.nim

@@ -139,7 +139,7 @@ proc addChoice(dest: var Peg, elem: Peg) =
     else: add(dest, elem)
   else: add(dest, elem)
 
-template multipleOp(k: PegKind, localOpt: expr) =
+template multipleOp(k: PegKind, localOpt: untyped) =
   result.kind = k
   result.sons = @[]
   for x in items(a):
@@ -328,32 +328,32 @@ proc newNonTerminal*(name: string, line, column: int): NonTerminal {.
   result.line = line
   result.col = column
 
-template letters*: expr =
+template letters*: Peg =
   ## expands to ``charset({'A'..'Z', 'a'..'z'})``
   charSet({'A'..'Z', 'a'..'z'})
 
-template digits*: expr =
+template digits*: Peg =
   ## expands to ``charset({'0'..'9'})``
   charSet({'0'..'9'})
 
-template whitespace*: expr =
+template whitespace*: Peg =
   ## expands to ``charset({' ', '\9'..'\13'})``
   charSet({' ', '\9'..'\13'})
 
-template identChars*: expr =
+template identChars*: Peg =
   ## expands to ``charset({'a'..'z', 'A'..'Z', '0'..'9', '_'})``
   charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'})
 
-template identStartChars*: expr =
+template identStartChars*: Peg =
   ## expands to ``charset({'A'..'Z', 'a'..'z', '_'})``
   charSet({'a'..'z', 'A'..'Z', '_'})
 
-template ident*: expr =
+template ident*: Peg =
   ## same as ``[a-zA-Z_][a-zA-z_0-9]*``; standard identifier
   sequence(charSet({'a'..'z', 'A'..'Z', '_'}),
            *charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}))
 
-template natural*: expr =
+template natural*: Peg =
   ## same as ``\d+``
   +digits
 
@@ -514,10 +514,10 @@ proc bounds*(c: Captures,
 when not useUnicode:
   type
     Rune = char
-  template fastRuneAt(s, i, ch: expr) =
+  template fastRuneAt(s, i, ch) =
     ch = s[i]
     inc(i)
-  template runeLenAt(s, i: expr): expr = 1
+  template runeLenAt(s, i): untyped = 1
 
   proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z','A'..'Z'}
   proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'}
@@ -735,7 +735,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {.
     else: result = -1
   of pkRule, pkList: assert false
 
-template fillMatches(s, caps, c: expr) =
+template fillMatches(s, caps, c) =
   for k in 0..c.ml-1:
     let startIdx = c.matches[k][0]
     let endIdx = c.matches[k][1]

lib/pure/random.nim

@@ -101,7 +101,7 @@ proc random*[T](a: openArray[T]): T =
   ## returns a random element from the openarray `a`.
   result = a[random(a.low..a.len)]
 
-proc randomize*(seed: int) {.benign.} =
+proc randomize*(seed: int64) {.benign.} =
   ## Initializes the random number generator with a specific seed.
   state.a0 = ui(seed shr 16)
   state.a1 = ui(seed and 0xffff)
@@ -123,7 +123,7 @@ when not defined(nimscript):
       proc getMil(t: Time): int {.importcpp: "getTime", nodecl.}
       randomize(getMil times.getTime())
     else:
-      let time = int(times.epochTime() * 1_000_000_000)
+      let time = int64(times.epochTime() * 1_000_000_000)
       randomize(time)
 
 {.pop.}

lib/pure/streams.nim

@@ -411,12 +411,12 @@ when not defined(js):
     result.writeDataImpl = fsWriteData
     result.flushImpl = fsFlush
 
-  proc newFileStream*(filename: string, mode: FileMode = fmRead): FileStream =
+  proc newFileStream*(filename: string, mode: FileMode = fmRead, bufSize: int = -1): FileStream =
     ## creates a new stream from the file named `filename` with the mode `mode`.
     ## If the file cannot be opened, nil is returned. See the `system
     ## <system.html>`_ module for a list of available FileMode enums.
     var f: File
-    if open(f, filename, mode): result = newFileStream(f)
+    if open(f, filename, mode, bufSize): result = newFileStream(f)
 
 
 when true:

lib/pure/strscans.nim

@@ -187,7 +187,6 @@ overloaded to handle both single characters and sets of character.
   if scanp(content, idx, +( ~{'\L', '\0'} -> entry.add(peekChar($input))), '\L'):
     result.add entry
 
-
 Calling ordinary Nim procs inside the macro is possible:
 
 .. code-block:: nim
@@ -253,6 +252,30 @@ is performed.
 
   for r in collectLinks(body):
     echo r
+
+In this example both macros are combined seamlessly in order to maximise 
+efficiency and perform different checks.
+
+.. code-block:: nim
+
+  iterator parseIps*(soup: string): string =
+    ## ipv4 only!
+    const digits = {'0'..'9'}
+    var a, b, c, d: int
+    var buf = ""
+    var idx = 0
+    while idx < soup.len:
+      if scanp(soup, idx, (`digits`{1,3}, '.', `digits`{1,3}, '.',
+               `digits`{1,3}, '.', `digits`{1,3}) -> buf.add($_)):
+        discard buf.scanf("$i.$i.$i.$i", a, b, c, d)
+        if (a >= 0 and a <= 254) and
+           (b >= 0 and b <= 254) and
+           (c >= 0 and c <= 254) and
+           (d >= 0 and d <= 254):
+          yield buf
+      buf.setLen(0) # need to clear `buf` each time, cause it might contain garbage
+      idx.inc
+
 ]##
 
 

lib/pure/strutils.nim

@@ -1551,6 +1551,37 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect,
   # copy the rest:
   add result, substr(s, i)
 
+proc multiReplace*(s: string, replacements: varargs[(string, string)]): string {.noSideEffect.} =
+  ## Same as replace, but specialized for doing multiple replacements in a single
+  ## pass through the input string.
+  ##
+  ## Calling replace multiple times after each other is inefficient and result in too many allocations
+  ## follwed by immediate deallocations as portions of the string gets replaced.
+  ## multiReplace performs all replacements in a single pass.
+  ##
+  ## If the resulting string is not longer than the original input string, only a single
+  ## memory allocation is required.
+  ##
+  ## The order of the replacements does matter. Earlier replacements are preferred over later
+  ## replacements in the argument list.
+  result = newStringOfCap(s.len)
+  var i = 0
+  var fastChk: set[char] = {}
+  for tup in replacements: fastChk.incl(tup[0][0]) # Include first character of all replacements
+  while i < s.len:
+    block sIteration:
+      # Assume most chars in s are not candidates for any replacement operation
+      if s[i] in fastChk:
+        for tup in replacements:
+          if s.continuesWith(tup[0], i):
+            add result, tup[1]
+            inc(i, tup[0].len)
+            break sIteration
+      # No matching replacement found
+      # copy current character from s
+      add result, s[i]
+      inc(i)
+
 proc delete*(s: var string, first, last: int) {.noSideEffect,
   rtl, extern: "nsuDelete".} =
   ## Deletes in `s` the characters at position `first` .. `last`.
@@ -2324,6 +2355,10 @@ when isMainModule:
 
   doAssert "  foo\n  bar".indent(4, "Q") == "QQQQ  foo\nQQQQ  bar"
 
+  doAssert "abba".multiReplace(("a", "b"), ("b", "a")) == "baab"
+  doAssert "Hello World.".multiReplace(("ello", "ELLO"), ("World.", "PEOPLE!")) == "HELLO PEOPLE!"
+  doAssert "aaaa".multiReplace(("a", "aa"), ("aa", "bb")) == "aaaaaaaa"
+
   doAssert isAlphaAscii('r')
   doAssert isAlphaAscii('A')
   doAssert(not isAlphaAscii('$'))

lib/pure/subexes.nim

@@ -46,12 +46,12 @@ type
     num, i, lineLen: int
 {.deprecated: [TFormatParser: FormatParser].}
 
-template call(x: stmt) {.immediate.} =
+template call(x: untyped): untyped =
   p.i = i
   x
   i = p.i
 
-template callNoLineLenTracking(x: stmt) {.immediate.} =
+template callNoLineLenTracking(x: untyped): untyped =
   let oldLineLen = p.lineLen
   p.i = i
   x