merged devel into epc

.gitignore

@@ -41,4 +41,3 @@ xcuserdata/
 /testresults.html
 /testresults.json
 testament.db
-/csources/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "csources"]
+	path = csources
+	url = ../../nim-lang/csources.git

build.sh

@@ -2,8 +2,8 @@
 set -e
 set -x
 
-if [ ! -d "csources" ]; then
+if [ ! -e csources/.git ]; then
-	git clone --depth 1 https://github.com/nim-lang/csources.git
+	git submodule update --init --depth 1
 fi
 
 cd "csources"

compiler/ast.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nim Compiler
-#        (c) Copyright 2013 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -296,6 +296,7 @@ const
   sfCompileToCpp* = sfInfixCall       # compile the module as C++ code
   sfCompileToObjc* = sfNamedParamCall # compile the module as Objective-C code
   sfExperimental* = sfOverriden       # module uses the .experimental switch
+  sfGoto* = sfOverriden               # var is used for 'goto' code generation
 
 const
   # getting ready for the future expr/stmt merge
@@ -529,19 +530,20 @@ type
   TMagic* = enum # symbols that require compiler magic:
     mNone,
     mDefined, mDefinedInScope, mCompiles,
-    mLow, mHigh, mSizeOf, mTypeTrait, mIs, mOf, mAddr, mTypeOf, mRoof,
+    mLow, mHigh, mSizeOf, mTypeTrait, mIs, mOf, mAddr, mTypeOf, mRoof, mPlugin,
     mEcho, mShallowCopy, mSlurp, mStaticExec,
     mParseExprToAst, mParseStmtToAst, mExpandToAst, mQuoteAst,
     mUnaryLt, mInc, mDec, mOrd, mNew, mNewFinalize, mNewSeq, mLengthOpenArray,
-    mLengthStr, mLengthArray, mLengthSeq, mIncl, mExcl, mCard, mChr, mGCref,
+    mLengthStr, mLengthArray, mLengthSeq, mXLenStr, mXLenSeq,
-    mGCunref,
+    mIncl, mExcl, mCard, mChr,
+    mGCref, mGCunref,
 
     mAddI, mSubI, mMulI, mDivI, mModI, mAddI64, mSubI64, mMulI64,
     mDivI64, mModI64, mSucc, mPred,
     mAddF64, mSubF64, mMulF64, mDivF64,
 
     mShrI, mShlI, mBitandI, mBitorI, mBitxorI, mMinI, mMaxI,
-    mShrI64, mShlI64, mBitandI64, mBitorI64, mBitxorI64, mMinI64, mMaxI64,
+    mShrI64, mShlI64, mBitandI64, mBitorI64, mBitxorI64,
     mMinF64, mMaxF64, mAddU, mSubU, mMulU,
     mDivU, mModU, mEqI, mLeI,
     mLtI,
@@ -550,7 +552,7 @@ type
     mEqEnum, mLeEnum, mLtEnum, mEqCh, mLeCh, mLtCh, mEqB, mLeB, mLtB, mEqRef,
     mEqUntracedRef, mLePtr, mLtPtr, mEqCString, mXor, mEqProc, mUnaryMinusI,
     mUnaryMinusI64, mAbsI, mAbsI64, mNot,
-    mUnaryPlusI, mBitnotI, mUnaryPlusI64,
+    mUnaryPlusI, mBitnotI,
     mBitnotI64, mUnaryPlusF64, mUnaryMinusF64, mAbsF64, mZe8ToI, mZe8ToI64,
     mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64, mToU8, mToU16, mToU32,
     mToFloat, mToBiggestFloat, mToInt, mToBiggestInt, mCharToStr, mBoolToStr,
@@ -589,11 +591,12 @@ type
 const
   ctfeWhitelist* = {mNone, mUnaryLt, mSucc,
     mPred, mInc, mDec, mOrd, mLengthOpenArray,
-    mLengthStr, mLengthArray, mLengthSeq, mIncl, mExcl, mCard, mChr,
+    mLengthStr, mLengthArray, mLengthSeq, mXLenStr, mXLenSeq,
+    mIncl, mExcl, mCard, mChr,
     mAddI, mSubI, mMulI, mDivI, mModI, mAddI64, mSubI64, mMulI64,
     mDivI64, mModI64, mAddF64, mSubF64, mMulF64, mDivF64,
     mShrI, mShlI, mBitandI, mBitorI, mBitxorI, mMinI, mMaxI,
-    mShrI64, mShlI64, mBitandI64, mBitorI64, mBitxorI64, mMinI64, mMaxI64,
+    mShrI64, mShlI64, mBitandI64, mBitorI64, mBitxorI64,
     mMinF64, mMaxF64, mAddU, mSubU, mMulU,
     mDivU, mModU, mEqI, mLeI,
     mLtI,
@@ -602,7 +605,7 @@ const
     mEqEnum, mLeEnum, mLtEnum, mEqCh, mLeCh, mLtCh, mEqB, mLeB, mLtB, mEqRef,
     mEqProc, mEqUntracedRef, mLePtr, mLtPtr, mEqCString, mXor, mUnaryMinusI,
     mUnaryMinusI64, mAbsI, mAbsI64, mNot,
-    mUnaryPlusI, mBitnotI, mUnaryPlusI64,
+    mUnaryPlusI, mBitnotI,
     mBitnotI64, mUnaryPlusF64, mUnaryMinusF64, mAbsF64, mZe8ToI, mZe8ToI64,
     mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64, mToU8, mToU16, mToU32,
     mToFloat, mToBiggestFloat, mToInt, mToBiggestInt, mCharToStr, mBoolToStr,
@@ -1170,7 +1173,9 @@ proc newType*(kind: TTypeKind, owner: PSym): PType =
   result.lockLevel = UnspecifiedLockLevel
   when debugIds:
     registerId(result)
-  #if result.id < 2000:
+  #if result.id == 92231:
+  #  echo "KNID ", kind
+  #  writeStackTrace()
   #  messageOut(typeKindToStr[kind] & ' has id: ' & toString(result.id))
 
 proc mergeLoc(a: var TLoc, b: TLoc) =

compiler/ccgexprs.nim

@@ -566,8 +566,6 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       "($4)($1 & $2)",            # BitandI64
       "($4)($1 | $2)",            # BitorI64
       "($4)($1 ^ $2)",            # BitxorI64
-      "(($1 <= $2) ? $1 : $2)", # MinI64
-      "(($1 >= $2) ? $1 : $2)", # MaxI64
       "(($1 <= $2) ? $1 : $2)", # MinF64
       "(($1 >= $2) ? $1 : $2)", # MaxF64
       "($4)((NU$3)($1) + (NU$3)($2))", # AddU
@@ -640,7 +638,6 @@ proc unaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     unArithTab: array[mNot..mToBiggestInt, string] = ["!($1)", # Not
       "$1",                   # UnaryPlusI
       "($3)((NU$2) ~($1))",   # BitnotI
-      "$1",                   # UnaryPlusI64
       "($3)((NU$2) ~($1))",   # BitnotI64
       "$1",                   # UnaryPlusF64
       "-($1)",                # UnaryMinusF64
@@ -676,7 +673,7 @@ proc isCppRef(p: BProc; typ: PType): bool {.inline.} =
 
 proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) =
   let mt = mapType(e.sons[0].typ)
-  if (mt in {ctArray, ctPtrToArray} and not enforceDeref):
+  if mt in {ctArray, ctPtrToArray} and not enforceDeref:
     # XXX the amount of hacks for C's arrays is incredible, maybe we should
     # simply wrap them in a struct? --> Losing auto vectorization then?
     #if e[0].kind != nkBracketExpr:
@@ -685,7 +682,11 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) =
   else:
     var a: TLoc
     initLocExprSingleUse(p, e.sons[0], a)
+    if d.k == locNone:
       let typ = skipTypes(a.t, abstractInst)
+      # dest = *a;  <-- We do not know that 'dest' is on the heap!
+      # It is completely wrong to set 'd.s' here, unless it's not yet
+      # been assigned to.
       case typ.kind
       of tyRef:
         d.s = OnHeap
@@ -698,6 +699,12 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) =
       of tyPtr:
         d.s = OnUnknown         # BUGFIX!
       else: internalError(e.info, "genDeref " & $a.t.kind)
+    elif p.module.compileToCpp:
+      let typ = skipTypes(a.t, abstractInst)
+      if typ.kind == tyVar and tfVarIsPtr notin typ.flags and
+           e.kind == nkHiddenDeref:
+        putIntoDest(p, d, e.typ, rdLoc(a))
+        return
     if enforceDeref and mt == ctPtrToArray:
       # we lie about the type for better C interop: 'ptr array[3,T]' is
       # translated to 'ptr T', but for deref'ing this produces wrong code.
@@ -957,6 +964,9 @@ proc genEcho(p: BProc, n: PNode) =
   var args: Rope = nil
   var a: TLoc
   for i in countup(0, n.len-1):
+    if n.sons[i].skipConv.kind == nkNilLit:
+      add(args, ", \"nil\"")
+    else:
       initLocExpr(p, n.sons[i], a)
       addf(args, ", $1? ($1)->data:\"nil\"", [rdLoc(a)])
   linefmt(p, cpsStmts, "printf($1$2);$n",
@@ -1345,15 +1355,15 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     else: unaryExpr(p, e, d, "$1Len0")
   of tyCString:
     useStringh(p.module)
-    if op == mHigh: unaryExpr(p, e, d, "(strlen($1)-1)")
+    if op == mHigh: unaryExpr(p, e, d, "($1 ? (strlen($1)-1) : -1)")
-    else: unaryExpr(p, e, d, "strlen($1)")
+    else: unaryExpr(p, e, d, "($1 ? strlen($1) : 0)")
   of tyString, tySequence:
     if not p.module.compileToCpp:
-      if op == mHigh: unaryExpr(p, e, d, "($1->Sup.len-1)")
+      if op == mHigh: unaryExpr(p, e, d, "($1 ? ($1->Sup.len-1) : -1)")
-      else: unaryExpr(p, e, d, "$1->Sup.len")
+      else: unaryExpr(p, e, d, "($1 ? $1->Sup.len : 0)")
     else:
-      if op == mHigh: unaryExpr(p, e, d, "($1->len-1)")
+      if op == mHigh: unaryExpr(p, e, d, "($1 ? ($1->len-1) : -1)")
-      else: unaryExpr(p, e, d, "$1->len")
+      else: unaryExpr(p, e, d, "($1 ? $1->len : 0)")
   of tyArray, tyArrayConstr:
     # YYY: length(sideeffect) is optimized away incorrectly?
     if op == mHigh: putIntoDest(p, d, e.typ, rope(lastOrd(typ)))
@@ -1714,6 +1724,11 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mOrd: genOrd(p, e, d)
   of mLengthArray, mHigh, mLengthStr, mLengthSeq, mLengthOpenArray:
     genArrayLen(p, e, d, op)
+  of mXLenStr, mXLenSeq:
+    if not p.module.compileToCpp:
+      unaryExpr(p, e, d, "($1->Sup.len-1)")
+    else:
+      unaryExpr(p, e, d, "$1->len")
   of mGCref: unaryStmt(p, e, d, "#nimGCref($1);$n")
   of mGCunref: unaryStmt(p, e, d, "#nimGCunref($1);$n")
   of mSetLengthStr: genSetLengthStr(p, e, d)

compiler/ccgstmts.nim

@@ -175,9 +175,18 @@ proc genBreakState(p: BProc, n: PNode) =
 
 proc genVarPrototypeAux(m: BModule, sym: PSym)
 
+proc genGotoVar(p: BProc; value: PNode) =
+  if value.kind notin {nkCharLit..nkUInt64Lit}:
+    localError(value.info, "'goto' target must be a literal value")
+  else:
+    lineF(p, cpsStmts, "goto NIMSTATE_$#;$n", [value.intVal.rope])
+
 proc genSingleVar(p: BProc, a: PNode) =
   var v = a.sons[0].sym
-  if sfCompileTime in v.flags: return
+  if {sfCompileTime, sfGoto} * v.flags != {}:
+    # translate 'var state {.goto.} = X' into 'goto LX':
+    if sfGoto in v.flags: genGotoVar(p, a.sons[2])
+    return
   var targetProc = p
   if sfGlobal in v.flags:
     if v.flags * {sfImportc, sfExportc} == {sfImportc} and
@@ -365,6 +374,19 @@ proc genReturnStmt(p: BProc, t: PNode) =
     linefmt(p, cpsStmts, "if ($1.status != 0) #popCurrentException();$n", safePoint)
   lineF(p, cpsStmts, "goto BeforeRet;$n", [])
 
+proc genGotoForCase(p: BProc; caseStmt: PNode) =
+  for i in 1 .. <caseStmt.len:
+    startBlock(p)
+    let it = caseStmt.sons[i]
+    for j in 0 .. it.len-2:
+      if it.sons[j].kind == nkRange:
+        localError(it.info, "range notation not available for computed goto")
+        return
+      let val = getOrdValue(it.sons[j])
+      lineF(p, cpsStmts, "NIMSTATE_$#:$n", [val.rope])
+    genStmts(p, it.lastSon)
+    endBlock(p)
+
 proc genComputedGoto(p: BProc; n: PNode) =
   # first pass: Generate array of computed labels:
   var casePos = -1
@@ -529,12 +551,7 @@ proc genBreakStmt(p: BProc, t: PNode) =
   lineF(p, cpsStmts, "goto $1;$n", [label])
 
 proc getRaiseFrmt(p: BProc): string =
-  if p.module.compileToCpp:
-    result = "throw NimException($1, $2);$n"
-  elif getCompilerProc("Exception") != nil:
   result = "#raiseException((#Exception*)$1, $2);$n"
-  else:
-    result = "#raiseException((#E_Base*)$1, $2);$n"
 
 proc genRaiseStmt(p: BProc, t: PNode) =
   if p.inExceptBlock > 0:
@@ -736,6 +753,9 @@ proc genCase(p: BProc, t: PNode, d: var TLoc) =
   of tyFloat..tyFloat128:
     genCaseGeneric(p, t, d, "if ($1 >= $2 && $1 <= $3) goto $4;$n",
                             "if ($1 == $2) goto $3;$n")
+  else:
+    if t.sons[0].kind == nkSym and sfGoto in t.sons[0].sym.flags:
+      genGotoForCase(p, t)
     else:
       genOrdinalCase(p, t, d)
 
@@ -772,11 +792,8 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   #  finallyPart();
   if not isEmptyType(t.typ) and d.k == locNone:
     getTemp(p, t.typ, d)
-  var
-    exc: Rope
-    i, length, blen: int
   genLineDir(p, t)
-  exc = getTempName()
+  let exc = getTempName()
   if getCompilerProc("Exception") != nil:
     discard cgsym(p.module, "Exception")
   else:
@@ -784,20 +801,23 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   add(p.nestedTryStmts, t)
   startBlock(p, "try {$n")
   expr(p, t.sons[0], d)
-  length = sonsLen(t)
+  let length = sonsLen(t)
   endBlock(p, ropecg(p.module, "} catch (NimException& $1) {$n", [exc]))
   if optStackTrace in p.options:
-    linefmt(p, cpsStmts, "#setFrame((TFrame*)&F);$n")
+    linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR);$n")
   inc p.inExceptBlock
-  i = 1
+  var i = 1
   var catchAllPresent = false
   while (i < length) and (t.sons[i].kind == nkExceptBranch):
-    blen = sonsLen(t.sons[i])
+    let blen = sonsLen(t.sons[i])
     if i > 1: addf(p.s(cpsStmts), "else ", [])
     if blen == 1:
       # general except section:
       catchAllPresent = true
-      exprBlock(p, t.sons[i].sons[0], d)
+      startBlock(p)
+      expr(p, t.sons[i].sons[0], d)
+      linefmt(p, cpsStmts, "#popCurrentException();$n")
+      endBlock(p)
     else:
       var orExpr: Rope = nil
       for j in countup(0, blen - 2):
@@ -807,7 +827,10 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
               "#isObj($1.exp->m_type, $2)",
               [exc, genTypeInfo(p.module, t.sons[i].sons[j].typ)])
       lineF(p, cpsStmts, "if ($1) ", [orExpr])
-      exprBlock(p, t.sons[i].sons[blen-1], d)
+      startBlock(p)
+      expr(p, t.sons[i].sons[blen-1], d)
+      linefmt(p, cpsStmts, "#popCurrentException();$n")
+      endBlock(p)
     inc(i)
 
   # reraise the exception if there was no catch all
@@ -887,7 +910,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
   startBlock(p, "else {$n")
   linefmt(p, cpsStmts, "#popSafePoint();$n")
   if optStackTrace in p.options:
-    linefmt(p, cpsStmts, "#setFrame((TFrame*)&F);$n")
+    linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR);$n")
   inc p.inExceptBlock
   var i = 1
   while (i < length) and (t.sons[i].kind == nkExceptBranch):
@@ -976,12 +999,19 @@ proc genAsmStmt(p: BProc, t: PNode) =
   else:
     lineF(p, cpsStmts, CC[cCompiler].asmStmtFrmt, [s])
 
+proc determineSection(n: PNode): TCFileSection =
+  result = cfsProcHeaders
+  if n.len >= 1 and n.sons[0].kind in {nkStrLit..nkTripleStrLit}:
+    if n.sons[0].strVal.startsWith("/*TYPESECTION*/"): result = cfsTypes
+    elif n.sons[0].strVal.startsWith("/*VARSECTION*/"): result = cfsVars
+
 proc genEmit(p: BProc, t: PNode) =
   var s = genAsmOrEmitStmt(p, t.sons[1])
   if p.prc == nil:
     # top level emit pragma?
-    genCLineDir(p.module.s[cfsProcHeaders], t.info)
+    let section = determineSection(t[1])
-    add(p.module.s[cfsProcHeaders], s)
+    genCLineDir(p.module.s[section], t.info)
+    add(p.module.s[section], s)
   else:
     genLineDir(p, t)
     line(p, cpsStmts, s)
@@ -1065,7 +1095,9 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
 
 proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
   genLineDir(p, e)
-  if not fieldDiscriminantCheckNeeded(p, e):
+  if e.sons[0].kind == nkSym and sfGoto in e.sons[0].sym.flags:
+    genGotoVar(p, e.sons[1])
+  elif not fieldDiscriminantCheckNeeded(p, e):
     var a: TLoc
     initLocExpr(p, e.sons[0], a)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)

compiler/ccgtypes.nim

@@ -11,20 +11,21 @@
 
 # ------------------------- Name Mangling --------------------------------
 
-proc mangleField(name: string): string =
-  result = mangle(name)
-  result[0] = result[0].toUpper # Mangling makes everything lowercase,
-                                # but some identifiers are C keywords
-
 proc isKeyword(w: PIdent): bool =
-  # nimrod and C++ share some keywords
+  # Nim and C++ share some keywords
-  # it's more efficient to test the whole nimrod keywords range
+  # it's more efficient to test the whole Nim keywords range
   case w.id
   of ccgKeywordsLow..ccgKeywordsHigh,
      nimKeywordsLow..nimKeywordsHigh,
      ord(wInline): return true
   else: return false
 
+proc mangleField(name: PIdent): string =
+  result = mangle(name.s)
+  if isKeyword(name):
+    result[0] = result[0].toUpper # Mangling makes everything lowercase,
+                                  # but some identifiers are C keywords
+
 proc mangleName(s: PSym): Rope =
   result = s.loc.r
   if result == nil:
@@ -110,7 +111,7 @@ proc mapSetType(typ: PType): TCTypeKind =
   else: result = ctArray
 
 proc mapType(typ: PType): TCTypeKind =
-  ## Maps a nimrod type to a C type
+  ## Maps a Nim type to a C type
   case typ.kind
   of tyNone, tyStmt: result = ctVoid
   of tyBool: result = ctBool
@@ -379,7 +380,7 @@ proc mangleRecFieldName(field: PSym, rectype: PType): Rope =
       ({sfImportc, sfExportc} * rectype.sym.flags != {}):
     result = field.loc.r
   else:
-    result = rope(mangleField(field.name.s))
+    result = rope(mangleField(field.name))
   if result == nil: internalError(field.info, "mangleRecFieldName")
 
 proc genRecordFieldsAux(m: BModule, n: PNode,

compiler/ccgutils.nim

@@ -176,7 +176,7 @@ proc mangle*(name: string): string =
   result = newStringOfCap(name.len)
   case name[0]
   of Letters:
-    result.add(name[0].toLower)
+    result.add(name[0])
   of Digits:
     result.add("N" & name[0])
   else:

compiler/commands.nim

@@ -319,7 +319,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
     undefSymbol(arg)
   of "symbol":
     expectArg(switch, arg, pass, info)
-    declareSymbol(arg)
+    # deprecated, do nothing
   of "compile":
     expectArg(switch, arg, pass, info)
     if pass in {passCmd2, passPP}: processCompile(arg)
@@ -488,7 +488,6 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
       if theOS == osNone: localError(info, errUnknownOS, arg)
       elif theOS != platform.hostOS:
         setTarget(theOS, targetCPU)
-        condsyms.initDefines()
   of "cpu":
     expectArg(switch, arg, pass, info)
     if pass in {passCmd1, passPP}:
@@ -496,7 +495,6 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
       if cpu == cpuNone: localError(info, errUnknownCPU, arg)
       elif cpu != platform.hostCPU:
         setTarget(targetOS, cpu)
-        condsyms.initDefines()
   of "run", "r":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optRun)

compiler/condsyms.nim

@@ -12,66 +12,64 @@
 import
   strtabs, platform, strutils, idents
 
-# We need to use a PStringTable here as defined symbols are always guaranteed
+# We need to use a StringTableRef here as defined symbols are always guaranteed
 # to be style insensitive. Otherwise hell would break lose.
 var gSymbols: StringTableRef
 
+const
+  catNone = "false"
+
 proc defineSymbol*(symbol: string) =
   gSymbols[symbol] = "true"
 
-proc declareSymbol*(symbol: string) = 
-  gSymbols[symbol] = "unknown"
-
 proc undefSymbol*(symbol: string) =
-  gSymbols[symbol] = "false"
+  gSymbols[symbol] = catNone
 
 proc isDefined*(symbol: string): bool =
   if gSymbols.hasKey(symbol):
-    result = gSymbols[symbol] == "true"
+    result = gSymbols[symbol] != catNone
+  elif cmpIgnoreStyle(symbol, CPU[targetCPU].name) == 0:
+    result = true
+  elif cmpIgnoreStyle(symbol, platform.OS[targetOS].name) == 0:
+    result = true
+  else:
+    case symbol.normalize
+    of "x86": result = targetCPU == cpuI386
+    of "itanium": result = targetCPU == cpuIa64
+    of "x8664": result = targetCPU == cpuAmd64
+    of "posix", "unix":
+      result = targetOS in {osLinux, osMorphos, osSkyos, osIrix, osPalmos,
+                            osQnx, osAtari, osAix,
+                            osHaiku, osVxWorks, osSolaris, osNetbsd,
+                            osFreebsd, osOpenbsd, osMacosx}
+    of "bsd":
+      result = targetOS in {osNetbsd, osFreebsd, osOpenbsd}
+    of "emulatedthreadvars":
+      result = platform.OS[targetOS].props.contains(ospLacksThreadVars)
+    of "msdos": result = targetOS == osDos
+    of "mswindows", "win32": result = targetOS == osWindows
+    of "macintosh": result = targetOS in {osMacos, osMacosx}
+    of "sunos": result = targetOS == osSolaris
+    of "littleendian": result = CPU[targetCPU].endian == platform.littleEndian
+    of "bigendian": result = CPU[targetCPU].endian == platform.bigEndian
+    of "cpu8": result = CPU[targetCPU].bit == 8
+    of "cpu16": result = CPU[targetCPU].bit == 16
+    of "cpu32": result = CPU[targetCPU].bit == 32
+    of "cpu64": result = CPU[targetCPU].bit == 64
+    of "nimrawsetjmp":
+      result = targetOS in {osSolaris, osNetbsd, osFreebsd, osOpenbsd, osMacosx}
+    else: discard
 
 proc isDefined*(symbol: PIdent): bool = isDefined(symbol.s)
-proc isDeclared*(symbol: PIdent): bool = gSymbols.hasKey(symbol.s)
 
 iterator definedSymbolNames*: string =
   for key, val in pairs(gSymbols):
-    if val == "true": yield key
+    if val != catNone: yield key
 
 proc countDefinedSymbols*(): int =
   result = 0
   for key, val in pairs(gSymbols):
-    if val == "true": inc(result)
+    if val != catNone: inc(result)
-
-# For ease of bootstrapping, we keep them here and not in the global config
-# file for now:
-const
-  additionalSymbols = """
-    x86 itanium x8664
-    msdos mswindows win32 unix posix sunos bsd macintosh RISCOS hpux
-    mac
-
-    hppa hp9000 hp9000s300 hp9000s700 hp9000s800 hp9000s820 ELATE sparcv9
-
-    ecmascript js nimrodvm nimffi nimdoc cpp objc
-    gcc llvmgcc clang lcc bcc dmc wcc vcc tcc pcc ucc icl
-    boehmgc gcmarkandsweep gcgenerational nogc gcUseBitvectors
-    endb profiler
-    executable guiapp consoleapp library dll staticlib
-
-    quick
-    release debug
-    useWinAnsi useFork useNimRtl useMalloc useRealtimeGC ssl memProfiler
-    nodejs kwin nimfix
-
-    usesysassert usegcassert tinyC useFFI
-    useStdoutAsStdmsg createNimRtl
-    booting fulldebug corruption nimsuperops noSignalHandler useGnuReadline
-    noCaas noDocGen noBusyWaiting nativeStackTrace useNodeIds selftest
-    reportMissedDeadlines avoidTimeMachine useClone ignoreAllocationSize
-    debugExecProcesses pcreDll useLipzipSrc
-    preventDeadlocks UNICODE winUnicode trackGcHeaders posixRealtime
-
-    nimStdSetjmp nimRawSetjmp nimSigSetjmp
-  """.split
 
 proc initDefines*() =
   gSymbols = newStringTable(modeStyleInsensitive)
@@ -90,58 +88,3 @@ proc initDefines*() =
   defineSymbol("nimalias")
   defineSymbol("nimlocks")
   defineSymbol("nimnode")
-  
-  # add platform specific symbols:
-  for c in low(CPU)..high(CPU):
-    declareSymbol("cpu" & $CPU[c].bit)
-    declareSymbol(normalize(EndianToStr[CPU[c].endian]))
-    declareSymbol(CPU[c].name)
-  for o in low(platform.OS)..high(platform.OS):
-    declareSymbol(platform.OS[o].name)
-
-  for a in additionalSymbols:
-    declareSymbol(a)
-
-  # -----------------------------------------------------------
-  case targetCPU
-  of cpuI386: defineSymbol("x86")
-  of cpuIa64: defineSymbol("itanium")
-  of cpuAmd64: defineSymbol("x8664")
-  else: discard
-  case targetOS
-  of osDos: 
-    defineSymbol("msdos")
-  of osWindows: 
-    defineSymbol("mswindows")
-    defineSymbol("win32")
-  of osLinux, osMorphos, osSkyos, osIrix, osPalmos, osQnx, osAtari, osAix, 
-     osHaiku, osVxWorks:
-    # these are all 'unix-like'
-    defineSymbol("unix")
-    defineSymbol("posix")
-  of osSolaris: 
-    defineSymbol("sunos")
-    defineSymbol("unix")
-    defineSymbol("posix")
-  of osNetbsd, osFreebsd, osOpenbsd: 
-    defineSymbol("unix")
-    defineSymbol("bsd")
-    defineSymbol("posix")
-  of osMacos: 
-    defineSymbol("macintosh")
-  of osMacosx: 
-    defineSymbol("macintosh")
-    defineSymbol("unix")
-    defineSymbol("posix")
-  else: discard
-  defineSymbol("cpu" & $CPU[targetCPU].bit)
-  defineSymbol(normalize(EndianToStr[CPU[targetCPU].endian]))
-  defineSymbol(CPU[targetCPU].name)
-  defineSymbol(platform.OS[targetOS].name)
-  declareSymbol("emulatedthreadvars")
-  if platform.OS[targetOS].props.contains(ospLacksThreadVars):
-    defineSymbol("emulatedthreadvars")
-  case targetOS
-  of osSolaris, osNetbsd, osFreebsd, osOpenbsd, osMacosx:
-    defineSymbol("nimRawSetjmp")
-  else: discard

compiler/extccomp.nim

@@ -668,7 +668,8 @@ proc callCCompiler*(projectfile: string) =
       it = PStrEntry(it.next)
 
     if optGenStaticLib in gGlobalOptions:
-      linkCmd = CC[c].buildLib % ["libfile", (libNameTmpl() % gProjectName),
+      let name = splitFile(gProjectName).name
+      linkCmd = CC[c].buildLib % ["libfile", (libNameTmpl() % name),
                                   "objfiles", objfiles]
     else:
       var linkerExe = getConfigVar(c, ".linkerexe")

compiler/guards.nim

@@ -22,7 +22,8 @@ const
   someLt = {mLtI, mLtI64, mLtF64, mLtU, mLtU64, mLtEnum,
             mLtCh, mLtB, mLtPtr, mLtStr}
 
-  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq}
+  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq,
+             mXLenStr, mXLenSeq}
 
   someIn = {mInRange, mInSet}
 
@@ -34,8 +35,8 @@ const
   someMul = {mMulI, mMulI64, mMulF64}
   someDiv = {mDivI, mDivI64, mDivF64}
   someMod = {mModI, mModI64}
-  someMax = {mMaxI, mMaxI64, mMaxF64}
+  someMax = {mMaxI, mMaxF64}
-  someMin = {mMinI, mMinI64, mMinF64}
+  someMin = {mMinI, mMinF64}
 
 proc isValue(n: PNode): bool = n.kind in {nkCharLit..nkNilLit}
 proc isLocation(n: PNode): bool = not n.isValue
@@ -122,7 +123,7 @@ proc neg(n: PNode): PNode =
         let eAsNode = newIntNode(nkIntLit, e.sym.position)
         if not inSet(n.sons[1], eAsNode): s.add eAsNode
       result.sons[1] = s
-    elif lengthOrd(t) < 1000:
+    elif t.kind notin {tyString, tySequence} and lengthOrd(t) < 1000:
       result.sons[1] = complement(n.sons[1])
     else:
       # not ({2, 3, 4}.contains(x))   x != 2 and x != 3 and x != 4
@@ -907,5 +908,5 @@ proc buildProperFieldCheck(access, check: PNode): PNode =
 proc checkFieldAccess*(m: TModel, n: PNode) =
   for i in 1..n.len-1:
     let check = buildProperFieldCheck(n.sons[0], n.sons[i])
-    if m.doesImply(check) != impYes:
+    if check != nil and m.doesImply(check) != impYes:
       message(n.info, warnProveField, renderTree(n.sons[0])); break

compiler/installer.ini

@@ -47,7 +47,7 @@ Start: "doc/overview.html"
 
 [Other]
 Files: "readme.txt;install.txt;contributors.txt;copying.txt"
-Files: "configure;makefile"
+Files: "makefile"
 Files: "*.ini"
 Files: "koch.nim"
 
@@ -70,6 +70,10 @@ Files: "doc/*.nim"
 Files: "doc/*.cfg"
 Files: "compiler/nimfix/*.nim"
 Files: "compiler/nimfix/*.cfg"
+Files: "compiler/nimsuggest/*.nim"
+Files: "compiler/nimsuggest/*.cfg"
+Files: "compiler/plugins/locals/*.nim"
+Files: "compiler/plugins/active.nim"
 Files: "tools/*.nim"
 Files: "tools/*.cfg"
 Files: "tools/*.tmpl"
@@ -97,13 +101,8 @@ Files: "lib/pure/concurrency/*.cfg"
 Files: "lib/impure/*.nim"
 Files: "lib/wrappers/*.nim"
 
-Files: "lib/wrappers/cairo/*.nim"
-Files: "lib/wrappers/gtk/*.nim"
-Files: "lib/wrappers/lua/*.nim"
-Files: "lib/wrappers/opengl/*.nim"
 Files: "lib/wrappers/readline/*.nim"
 Files: "lib/wrappers/sdl/*.nim"
-Files: "lib/wrappers/x11/*.nim"
 Files: "lib/wrappers/zip/*.nim"
 Files: "lib/wrappers/zip/libzip_all.c"
 
@@ -115,8 +114,6 @@ Files: "lib/packages/docutils/*.nim"
 
 [Other]
 Files: "examples/*.nim"
-Files: "examples/gtk/*.nim"
-Files: "examples/0mq/*.nim"
 Files: "examples/c++iface/*.nim"
 Files: "examples/objciface/*.nim"
 Files: "examples/cross_calculator/"
@@ -126,12 +123,109 @@ Files: "examples/*.txt"
 Files: "examples/*.cfg"
 Files: "examples/*.tmpl"
 
+Files: "tests/actiontable/*.nim"
+Files: "tests/alias/*.nim"
+Files: "tests/ambsym/*.nim"
+Files: "tests/array/*.nim"
+Files: "tests/assign/*.nim"
+Files: "tests/astoverload/*.nim"
+Files: "tests/async/*.nim"
+Files: "tests/benchmarks/*.nim"
+Files: "tests/bind/*.nim"
+Files: "tests/borrow/*.nim"
+Files: "tests/casestmt/*.nim"
+Files: "tests/ccgbugs/*.nim"
+Files: "tests/clearmsg/*.nim"
+Files: "tests/closure/*.nim"
+Files: "tests/cnstseq/*.nim"
+Files: "tests/collections/*.nim"
+Files: "tests/compiles/*.nim"
+Files: "tests/concat/*.nim"
+Files: "tests/concepts/*.nim"
+Files: "tests/constr/*.nim"
+Files: "tests/constraints/*.nim"
+Files: "tests/controlflow/*.nim"
+Files: "tests/converter/*.nim"
+Files: "tests/cpp/*.nim"
+Files: "tests/defaultprocparam/*.nim"
+Files: "tests/deprecated/*.nim"
+Files: "tests/destructor/*.nim"
+Files: "tests/dir with space/*.nim"
+Files: "tests/discard/*.nim"
+Files: "tests/distinct/*.nim"
+Files: "tests/dll/*.nim"
+Files: "tests/effects/*.nim"
+Files: "tests/enum/*.nim"
+Files: "tests/exception/*.nim"
+Files: "tests/exprs/*.nim"
+Files: "tests/fields/*.nim"
+Files: "tests/float/*.nim"
+Files: "tests/friends/*.nim"
+Files: "tests/gc/*.nim"
+Files: "tests/generics/*.nim"
+Files: "tests/gensym/*.nim"
+Files: "tests/global/*.nim"
+Files: "tests/implicit/*.nim"
+Files: "tests/init/*.nim"
+Files: "tests/iter/*.nim"
+Files: "tests/js/*.nim"
+Files: "tests/js/*.cfg"
+Files: "tests/let/*.nim"
+Files: "tests/lexer/*.nim"
+Files: "tests/lookups/*.nim"
+Files: "tests/macros/*.nim"
+Files: "tests/magics/*.nim"
+Files: "tests/metatype/*.nim"
+Files: "tests/method/*.nim"
+Files: "tests/misc/*.nim"
+Files: "tests/modules/*.nim"
+Files: "tests/namedparams/*.nim"
+Files: "tests/notnil/*.nim"
+Files: "tests/objects/*.nim"
+Files: "tests/objvariant/*.nim"
+Files: "tests/openarray/*.nim"
+Files: "tests/osproc/*.nim"
+Files: "tests/overflw/*.nim"
+Files: "tests/overload/*.nim"
+Files: "tests/parallel/*.nim"
+Files: "tests/parallel/*.cfg"
+Files: "tests/parser/*.nim"
+Files: "tests/pragmas/*.nim"
+Files: "tests/proc/*.nim"
+Files: "tests/procvar/*.nim"
+Files: "tests/range/*.nim"
+Files: "tests/rodfiles/*.nim"
+Files: "tests/seq/*.nim"
+Files: "tests/sets/*.nim"
+Files: "tests/showoff/*.nim"
+Files: "tests/specialops/*.nim"
+Files: "tests/stdlib/*.nim"
+Files: "tests/system/*.nim"
+Files: "tests/template/*.nim"
+Files: "tests/testament/*.nim"
+Files: "tests/testdata/*.nim"
+Files: "tests/threads/*.nim"
+Files: "tests/threads/*.cfg"
+Files: "tests/trmacros/*.nim"
+Files: "tests/tuples/*.nim"
+Files: "tests/typerel/*.nim"
+Files: "tests/types/*.nim"
+Files: "tests/usingstmt/*.nim"
+Files: "tests/varres/*.nim"
+Files: "tests/varstmt/*.nim"
+Files: "tests/vm/*.nim"
+Files: "tests/readme.txt"
+Files: "tests/testament/css/*.css"
+Files: "tests/testament/*.cfg"
+Files: "lib/pure/unidecode/unidecode.dat"
 
 [Windows]
 Files: "bin/nim.exe"
-Files: "bin/nim_debug.exe"
 Files: "bin/c2nim.exe"
 Files: "bin/nimgrep.exe"
+Files: "bin/nimsuggest.exe"
+Files: "bin/nimble.exe"
+Files: "bin/*.dll"
 
 Files: "dist/*.dll"
 Files: "koch.exe"
@@ -142,7 +236,7 @@ BinPath: r"bin;dist\mingw\bin;dist"
 ;           Section | dir | zipFile | size hint (in KB) | url | exe start menu entry
 Download: r"Documentation|doc|docs.zip|13824|http://nim-lang.org/download/docs-${version}.zip|overview.html"
 Download: r"C Compiler (MingW)|dist|mingw.zip|82944|http://nim-lang.org/download/${mingw}.zip"
-Download: r"Aporia IDE|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.1.3.zip|aporia\bin\aporia.exe"
+Download: r"Aporia IDE|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.3.0.zip|aporia-0.3.0\bin\aporia.exe"
 ; for now only NSIS supports optional downloads
 
 [UnixBin]

compiler/jsgen.nim

@@ -124,7 +124,7 @@ proc newProc(globals: PGlobals, module: BModule, procDef: PNode,
 
 const
   MappedToObject = {tyObject, tyArray, tyArrayConstr, tyTuple, tyOpenArray,
-    tySet, tyVar, tyRef, tyPtr, tyBigNum, tyVarargs}
+    tySet, tyBigNum, tyVarargs}
 
 proc mapType(typ: PType): TJSTypeKind =
   let t = skipTypes(typ, abstractInst)
@@ -163,7 +163,8 @@ proc mangleName(s: PSym): Rope =
     add(result, rope(s.id))
     s.loc.r = result
 
-proc makeJSString(s: string): Rope = strutils.escape(s).rope
+proc makeJSString(s: string): Rope =
+  (if s.isNil: "null".rope else: strutils.escape(s).rope)
 
 include jstypes
 
@@ -280,8 +281,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 & $2)", "($1 & $2)"], # BitandI64
     ["", "", "($1 | $2)", "($1 | $2)"], # BitorI64
     ["", "", "($1 ^ $2)", "($1 ^ $2)"], # BitxorI64
-    ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinI64
-    ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxI64
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinF64
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxF64
     ["addU", "addU", "addU($1, $2)", "addU($1, $2)"], # addU
@@ -325,7 +324,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "!($1)", "!($1)"], # Not
     ["", "", "+($1)", "+($1)"], # UnaryPlusI
     ["", "", "~($1)", "~($1)"], # BitnotI
-    ["", "", "+($1)", "+($1)"], # UnaryPlusI64
     ["", "", "~($1)", "~($1)"], # BitnotI64
     ["", "", "+($1)", "+($1)"], # UnaryPlusF64
     ["", "", "-($1)", "-($1)"], # UnaryMinusF64
@@ -382,8 +380,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 & $2)", "($1 & $2)"], # BitandI64
     ["", "", "($1 | $2)", "($1 | $2)"], # BitorI64
     ["", "", "($1 ^ $2)", "($1 ^ $2)"], # BitxorI64
-    ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinI64
-    ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxI64
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinF64
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxF64
     ["addU", "addU", "addU($1, $2)", "addU($1, $2)"], # addU
@@ -427,7 +423,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "not ($1)", "not ($1)"], # Not
     ["", "", "+($1)", "+($1)"], # UnaryPlusI
     ["", "", "~($1)", "~($1)"], # BitnotI
-    ["", "", "+($1)", "+($1)"], # UnaryPlusI64
     ["", "", "~($1)", "~($1)"], # BitnotI64
     ["", "", "+($1)", "+($1)"], # UnaryPlusF64
     ["", "", "-($1)", "-($1)"], # UnaryMinusF64
@@ -937,6 +932,13 @@ proc genArrayAccess(p: PProc, n: PNode, r: var TCompRes) =
   r.address = nil
   r.kind = resExpr
 
+proc isIndirect(v: PSym): bool =
+  result = {sfAddrTaken, sfGlobal} * v.flags != {} and
+    #(mapType(v.typ) != etyObject) and
+    {sfImportc, sfVolatile, sfExportc} * v.flags == {} and
+    v.kind notin {skProc, skConverter, skMethod, skIterator, skClosureIterator,
+                  skConst, skTemp, skLet}
+
 proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
   case n.sons[0].kind
   of nkSym:
@@ -945,12 +947,16 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
     case s.kind
     of skVar, skLet, skResult:
       r.kind = resExpr
-      if mapType(n.sons[0].typ) == etyObject:
+      let jsType = mapType(n.typ)
+      if jsType == etyObject:
         # make addr() a no-op:
         r.typ = etyNone
+        if isIndirect(s):
+          r.res = s.loc.r & "[0]"
+        else:
           r.res = s.loc.r
         r.address = nil
-      elif {sfGlobal, sfAddrTaken} * s.flags != {}:
+      elif {sfGlobal, sfAddrTaken} * s.flags != {} or jsType == etyBaseIndex:
         # for ease of code generation, we do not distinguish between
         # sfAddrTaken and sfGlobal.
         r.typ = etyBaseIndex
@@ -992,7 +998,7 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
       else:
         r.address = s.loc.r
         r.res = s.loc.r & "_Idx"
-    elif k != etyObject and {sfAddrTaken, sfGlobal} * s.flags != {}:
+    elif isIndirect(s):
       r.res = "$1[0]" % [s.loc.r]
     else:
       r.res = s.loc.r
@@ -1124,7 +1130,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   of tyRange, tyGenericInst:
     result = createVar(p, lastSon(typ), indirect)
   of tySet:
-    result = rope("{}")
+    result = putToSeq("{}", indirect)
   of tyBool:
     result = putToSeq("false", indirect)
   of tyArray, tyArrayConstr:
@@ -1144,6 +1150,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
         add(result, createVar(p, e, false))
         inc(i)
       add(result, "]")
+    if indirect: result = "[$1]" % [result]
   of tyTuple:
     result = rope("{")
     for i in 0.. <t.sonsLen:
@@ -1151,6 +1158,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
       addf(result, "Field$1: $2" | "Field$# = $#", [i.rope,
            createVar(p, t.sons[i], false)])
     add(result, "}")
+    if indirect: result = "[$1]" % [result]
   of tyObject:
     result = rope("{")
     var c = 0
@@ -1161,6 +1169,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
       add(result, createRecordVarAux(p, t.n, c))
       t = t.sons[0]
     add(result, "}")
+    if indirect: result = "[$1]" % [result]
   of tyVar, tyPtr, tyRef:
     if mapType(t) == etyBaseIndex:
       result = putToSeq("[null, 0]" | "{nil, 0}", indirect)
@@ -1172,11 +1181,6 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     internalError("createVar: " & $t.kind)
     result = nil
 
-proc isIndirect(v: PSym): bool =
-  result = {sfAddrTaken, sfGlobal} * v.flags != {} and
-    (mapType(v.typ) != etyObject) and
-    v.kind notin {skProc, skConverter, skMethod, skIterator, skClosureIterator}
-
 proc genVarInit(p: PProc, v: PSym, n: PNode) =
   var
     a: TCompRes
@@ -1207,7 +1211,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     else:
       s = a.res
     if isIndirect(v):
-      addf(p.body, "var $1 = [$2];$n" | "local $1 = {$2};$n", [v.loc.r, s])
+      addf(p.body, "var $1 = /**/[$2];$n" | "local $1 = {$2};$n", [v.loc.r, s])
     else:
       addf(p.body, "var $1 = $2;$n" | "local $1 = $2;$n", [v.loc.r, s])
 
@@ -1239,7 +1243,7 @@ proc genNew(p: PProc, n: PNode) =
   var a: TCompRes
   gen(p, n.sons[1], a)
   var t = skipTypes(n.sons[1].typ, abstractVar).sons[0]
-  addf(p.body, "$1 = $2;$n", [a.res, createVar(p, t, true)])
+  addf(p.body, "$1 = $2;$n", [a.res, createVar(p, t, false)])
 
 proc genNewSeq(p: PProc, n: PNode) =
   var x, y: TCompRes
@@ -1325,14 +1329,17 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
     # XXX: range checking?
     if not (optOverflowCheck in p.options): unaryExpr(p, n, r, "", "$1 - 1")
     else: unaryExpr(p, n, r, "subInt", "subInt($1, 1)")
-  of mAppendStrCh: binaryExpr(p, n, r, "addChar", "addChar($1, $2)")
+  of mAppendStrCh: binaryExpr(p, n, r, "addChar",
+        "if ($1 != null) { addChar($1, $2); } else { $1 = [$2, 0]; }")
   of mAppendStrStr:
     if skipTypes(n.sons[1].typ, abstractVarRange).kind == tyCString:
-        binaryExpr(p, n, r, "", "$1 += $2")
+        binaryExpr(p, n, r, "", "if ($1 != null) { $1 += $2; } else { $1 = $2; }")
     else:
-      binaryExpr(p, n, r, "", "$1 = ($1.slice(0, -1)).concat($2)")
+      binaryExpr(p, n, r, "",
+        "if ($1 != null) { $1 = ($1.slice(0, -1)).concat($2); } else { $1 = $2;}")
     # XXX: make a copy of $2, because of Javascript's sucking semantics
-  of mAppendSeqElem: binaryExpr(p, n, r, "", "$1.push($2)")
+  of mAppendSeqElem: binaryExpr(p, n, r, "",
+        "if ($1 != null) { $1.push($2); } else { $1 = [$2]; }")
   of mConStrStr: genConStrStr(p, n, r)
   of mEqStr: binaryExpr(p, n, r, "eqStrings", "eqStrings($1, $2)")
   of mLeStr: binaryExpr(p, n, r, "cmpStrings", "(cmpStrings($1, $2) <= 0)")
@@ -1343,14 +1350,17 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mSizeOf: r.res = rope(getSize(n.sons[1].typ))
   of mChr, mArrToSeq: gen(p, n.sons[1], r)      # nothing to do
   of mOrd: genOrd(p, n, r)
-  of mLengthStr: unaryExpr(p, n, r, "", "($1.length-1)")
+  of mLengthStr: unaryExpr(p, n, r, "", "($1 != null ? $1.length-1 : 0)")
+  of mXLenStr: unaryExpr(p, n, r, "", "$1.length-1")
   of mLengthSeq, mLengthOpenArray, mLengthArray:
+    unaryExpr(p, n, r, "", "($1 != null ? $1.length : 0)")
+  of mXLenSeq:
     unaryExpr(p, n, r, "", "$1.length")
   of mHigh:
     if skipTypes(n.sons[1].typ, abstractVar).kind == tyString:
-      unaryExpr(p, n, r, "", "($1.length-2)")
+      unaryExpr(p, n, r, "", "($1 != null ? ($1.length-2) : -1)")
     else:
-      unaryExpr(p, n, r, "", "($1.length-1)")
+      unaryExpr(p, n, r, "", "($1 != null ? ($1.length-1) : -1)")
   of mInc:
     if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 += $2")
     else: binaryExpr(p, n, r, "addInt", "$1 = addInt($1, $2)")

compiler/lexer.nim

@@ -61,7 +61,7 @@ type
     tkComma, tkSemiColon,
     tkColon, tkColonColon, tkEquals, tkDot, tkDotDot,
     tkOpr, tkComment, tkAccent,
-    tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr,
+    tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
 
   TTokTypes* = set[TTokType]
 
@@ -221,6 +221,10 @@ proc dispMessage(L: TLexer; info: TLineInfo; msg: TMsgKind; arg: string) =
 proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "") =
   L.dispMessage(getLineInfo(L), msg, arg)
 
+proc lexMessageTok*(L: TLexer, msg: TMsgKind, tok: TToken, arg = "") =
+  var info = newLineInfo(L.fileIdx, tok.line, tok.col)
+  L.dispMessage(info, msg, arg)
+
 proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
   var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart)
   L.dispMessage(info, msg, arg)
@@ -863,6 +867,15 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
     of '`':
       tok.tokType = tkAccent
       inc(L.bufpos)
+    of '_':
+      inc(L.bufpos)
+      if L.buf[L.bufpos] notin SymChars:
+        tok.tokType = tkSymbol
+        tok.ident = getIdent("_")
+      else:
+        tok.literal = $c
+        tok.tokType = tkInvalid
+        lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')')
     of '\"':
       # check for extended raw string literal:
       var rawMode = L.bufpos > 0 and L.buf[L.bufpos-1] in SymChars

compiler/lowerings.nim

@@ -15,6 +15,10 @@ const
 import ast, astalgo, types, idents, magicsys, msgs, options
 from trees import getMagic
 
+proc newDeref*(n: PNode): PNode {.inline.} =
+  result = newNodeIT(nkHiddenDeref, n.info, n.typ.sons[0])
+  addSon(result, n)
+
 proc newTupleAccess*(tup: PNode, i: int): PNode =
   result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(
                      abstractInst).sons[i])

compiler/msgs.nim

@@ -738,7 +738,7 @@ proc writeContext(lastinfo: TLineInfo) =
     if msgContext[i] != lastinfo and msgContext[i] != info:
       msgWriteln(PosContextFormat % [toMsgFilename(msgContext[i]),
                                      coordToStr(msgContext[i].line),
-                                     coordToStr(msgContext[i].col),
+                                     coordToStr(msgContext[i].col+1),
                                      getMessageStr(errInstantiationFrom, "")])
     info = msgContext[i]
 
@@ -781,7 +781,7 @@ proc formatMsg*(info: TLineInfo, msg: TMsgKind, arg: string): string =
              of hintMin..hintMax: PosHintFormat
              else: PosErrorFormat
   result = frmt % [toMsgFilename(info), coordToStr(info.line),
-                   coordToStr(info.col), getMessageStr(msg, arg)]
+                   coordToStr(info.col+1), getMessageStr(msg, arg)]
 
 proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
                eh: TErrorHandling) =
@@ -804,8 +804,11 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
     ignoreMsg = optHints notin gOptions or msg notin gNotes
     frmt = PosHintFormat
     inc(gHintCounter)
+  # NOTE: currently line info line numbers start with 1,
+  # but column numbers start with 0, however most editors expect
+  # first column to be 1, so we need to +1 here
   let s = frmt % [toMsgFilename(info), coordToStr(info.line),
-                  coordToStr(info.col), getMessageStr(msg, arg)]
+                  coordToStr(info.col+1), getMessageStr(msg, arg)]
   if not ignoreMsg and not ignoreMsgBecauseOfIdeTools(msg):
     msgWriteln(s)
     if optPrintSurroundingSrc and msg in errMin..errMax:

compiler/nimfix/nimfix.nim.cfg

@@ -5,7 +5,7 @@ hint[XDeclaredButNotUsed]:off
 path:"$projectPath/.."
 
 path:"$lib/packages/docutils"
-path:"$nim/compiler"
+path:"../../compiler"
 
 define:useStdoutAsStdmsg
 symbol:nimfix

compiler/nimsuggest/nimsuggest.nim.cfg

@@ -6,7 +6,7 @@ hint[XDeclaredButNotUsed]:off
 path:"$projectPath/../.."
 
 path:"$lib/packages/docutils"
-path:"$nim/compiler"
+path:"../../compiler"
 
 define:useStdoutAsStdmsg
 define:nimsuggest

compiler/options.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nim Compiler
-#        (c) Copyright 2013 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.

compiler/parampatterns.nim

@@ -190,6 +190,8 @@ proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
         result = arLocalLValue
       else:
         result = arLValue
+    elif n.sym.kind == skParam and n.sym.typ.kind == tyVar:
+      result = arLValue
     elif n.sym.kind == skType:
       let t = n.sym.typ.skipTypes({tyTypeDesc})
       if t.kind == tyVar: result = arStrange

compiler/parser.nim

@@ -91,7 +91,7 @@ proc closeParser(p: var TParser) =
 
 proc parMessage(p: TParser, msg: TMsgKind, arg = "") =
   ## Produce and emit the parser message `arg` to output.
-  lexMessage(p.lex, msg, arg)
+  lexMessageTok(p.lex, msg, p.tok, arg)
 
 proc parMessage(p: TParser, msg: TMsgKind, tok: TToken) =
   ## Produce and emit a parser message to output about the token `tok`
@@ -154,7 +154,7 @@ proc eat(p: var TParser, tokType: TTokType) =
   if p.tok.tokType == tokType:
     getTok(p)
   else:
-    lexMessage(p.lex, errTokenExpected, TokTypeToStr[tokType])
+    lexMessageTok(p.lex, errTokenExpected, p.tok, TokTypeToStr[tokType])
 
 proc parLineInfo(p: TParser): TLineInfo =
   ## Retrieve the line information associated with the parser's current state.
@@ -212,7 +212,8 @@ proc getPrecedence(tok: TToken, strongSpaces: bool): int =
     let relevantChar = tok.ident.s[0]
 
     # arrow like?
-    if L > 1 and tok.ident.s[L-1] == '>': return considerStrongSpaces(1)
+    if L > 1 and tok.ident.s[L-1] == '>' and
+      tok.ident.s[L-2] in {'-', '~', '='}: return considerStrongSpaces(1)
 
     template considerAsgn(value: expr) =
       result = if tok.ident.s[L-1] == '=': 1 else: value
@@ -388,7 +389,6 @@ proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
     if p.tok.tokType != tkComma: break
     getTok(p)
     optInd(p, a)
-  eat(p, endTok)
 
 proc dotExpr(p: var TParser, a: PNode): PNode =
   #| dotExpr = expr '.' optInd symbol
@@ -944,8 +944,7 @@ proc parseDoBlock(p: var TParser): PNode =
   getTok(p)
   let params = parseParamList(p, retColon=false)
   let pragmas = optPragmas(p)
-  eat(p, tkColon)
+  colcom(p, result)
-  skipComment(p, result)
   result = newProcNode(nkDo, info, parseStmt(p),
                        params = params,
                        pragmas = pragmas)
@@ -1139,9 +1138,11 @@ proc parseMacroColon(p: var TParser, x: PNode): PNode =
     result = makeCall(result)
     getTok(p)
     skipComment(p, result)
+    let stmtList = newNodeP(nkStmtList, p)
     if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept}:
       let body = parseStmt(p)
-      addSon(result, makeStmtList(body))
+      stmtList.add body
+      #addSon(result, makeStmtList(body))
     while sameInd(p):
       var b: PNode
       case p.tok.tokType
@@ -1153,19 +1154,22 @@ proc parseMacroColon(p: var TParser, x: PNode): PNode =
         getTok(p)
         optInd(p, b)
         addSon(b, parseExpr(p))
-        eat(p, tkColon)
       of tkExcept:
         b = newNodeP(nkExceptBranch, p)
         exprList(p, tkColon, b)
-        skipComment(p, b)
       of tkElse:
         b = newNodeP(nkElse, p)
         getTok(p)
-        eat(p, tkColon)
       else: break
+      eat(p, tkColon)
       addSon(b, parseStmt(p))
-      addSon(result, b)
+      addSon(stmtList, b)
       if b.kind == nkElse: break
+    if stmtList.len == 1 and stmtList[0].kind == nkStmtList:
+      # to keep backwards compatibility (see tests/vm/tstringnil)
+      result.add stmtList[0]
+    else:
+      result.add stmtList
 
 proc parseExprStmt(p: var TParser): PNode =
   #| exprStmt = simpleExpr
@@ -1310,8 +1314,7 @@ proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode =
     var branch = newNodeP(nkElifBranch, p)
     optInd(p, branch)
     addSon(branch, parseExpr(p))
-    eat(p, tkColon)
+    colcom(p, branch)
-    skipComment(p, branch)
     addSon(branch, parseStmt(p))
     skipComment(p, branch)
     addSon(result, branch)
@@ -1319,8 +1322,7 @@ proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode =
   if p.tok.tokType == tkElse and sameOrNoInd(p):
     var branch = newNodeP(nkElse, p)
     eat(p, tkElse)
-    eat(p, tkColon)
+    colcom(p, branch)
-    skipComment(p, branch)
     addSon(branch, parseStmt(p))
     addSon(result, branch)
 
@@ -1368,13 +1370,11 @@ proc parseCase(p: var TParser): PNode =
       getTok(p)
       optInd(p, b)
       addSon(b, parseExpr(p))
-      eat(p, tkColon)
     of tkElse:
       b = newNodeP(nkElse, p)
       getTok(p)
-      eat(p, tkColon)
     else: break
-    skipComment(p, b)
+    colcom(p, b)
     addSon(b, parseStmt(p))
     addSon(result, b)
     if b.kind == nkElse: break
@@ -1391,8 +1391,7 @@ proc parseTry(p: var TParser; isExpr: bool): PNode =
   #|            (optInd 'finally' colcom stmt)?
   result = newNodeP(nkTryStmt, p)
   getTok(p)
-  eat(p, tkColon)
+  colcom(p, result)
-  skipComment(p, result)
   addSon(result, parseStmt(p))
   var b: PNode = nil
   while sameOrNoInd(p) or isExpr:
@@ -1402,10 +1401,9 @@ proc parseTry(p: var TParser; isExpr: bool): PNode =
       exprList(p, tkColon, b)
     of tkFinally:
       b = newNodeP(nkFinally, p)
-      getTokNoInd(p)
+      getTok(p)
-      eat(p, tkColon)
     else: break
-    skipComment(p, b)
+    colcom(p, b)
     addSon(b, parseStmt(p))
     addSon(result, b)
     if b.kind == nkFinally: break
@@ -1414,7 +1412,7 @@ proc parseTry(p: var TParser; isExpr: bool): PNode =
 proc parseExceptBlock(p: var TParser, kind: TNodeKind): PNode =
   #| exceptBlock = 'except' colcom stmt
   result = newNodeP(kind, p)
-  getTokNoInd(p)
+  getTok(p)
   colcom(p, result)
   addSon(result, parseStmt(p))
 
@@ -1447,7 +1445,7 @@ proc parseStaticOrDefer(p: var TParser; k: TNodeKind): PNode =
   #| staticStmt = 'static' colcom stmt
   #| deferStmt = 'defer' colcom stmt
   result = newNodeP(k, p)
-  getTokNoInd(p)
+  getTok(p)
   colcom(p, result)
   addSon(result, parseStmt(p))
 
@@ -1628,7 +1626,7 @@ proc parseEnum(p: var TParser): PNode =
         p.tok.tokType == tkEof:
       break
   if result.len <= 1:
-    lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok))
+    lexMessageTok(p.lex, errIdentifierExpected, p.tok, prettyTok(p.tok))
 
 proc parseObjectPart(p: var TParser): PNode
 proc parseObjectWhen(p: var TParser): PNode =
@@ -1686,9 +1684,8 @@ proc parseObjectCase(p: var TParser): PNode =
     of tkElse:
       b = newNodeP(nkElse, p)
       getTok(p)
-      eat(p, tkColon)
     else: break
-    skipComment(p, b)
+    colcom(p, b)
     var fields = parseObjectPart(p)
     if fields.kind == nkEmpty:
       parMessage(p, errIdentifierExpected, p.tok)
@@ -1884,7 +1881,7 @@ proc simpleStmt(p: var TParser): PNode =
 
 proc complexOrSimpleStmt(p: var TParser): PNode =
   #| complexOrSimpleStmt = (ifStmt | whenStmt | whileStmt
-  #|                     | tryStmt | finallyStmt | exceptStmt | forStmt
+  #|                     | tryStmt | forStmt
   #|                     | blockStmt | staticStmt | deferStmt | asmStmt
   #|                     | 'proc' routine
   #|                     | 'method' routine

compiler/plugins.nim

@@ -0,0 +1,43 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Plugin support for the Nim compiler. Right now there are no plugins and they
+## need to be build with the compiler, no DLL support.
+
+import ast, semdata, idents
+
+type
+  Transformation* = proc (c: PContext; n: PNode): PNode {.nimcall.}
+  Plugin = ref object
+    fn, module, package: PIdent
+    t: Transformation
+    next: Plugin
+
+proc pluginMatches(p: Plugin; s: PSym): bool =
+  if s.name.id != p.fn.id: return false
+  let module = s.owner
+  if module == nil or module.kind != skModule or
+      module.name.id != p.module.id: return false
+  let package = module.owner
+  if package == nil or package.kind != skPackage or
+      package.name.id != p.package.id: return false
+  return true
+
+var head: Plugin
+
+proc getPlugin*(fn: PSym): Transformation =
+  var it = head
+  while it != nil:
+    if pluginMatches(it, fn): return it.t
+    it = it.next
+
+proc registerPlugin*(package, module, fn: string; t: Transformation) =
+  let oldHead = head
+  head = Plugin(fn: getIdent(fn), module: getIdent(module),
+                 package: getIdent(package), t: t, next: oldHead)

compiler/plugins/active.nim

@@ -0,0 +1,13 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Include file that imports all plugins that are active.
+
+import
+  locals.locals

compiler/plugins/locals/locals.nim

@@ -0,0 +1,42 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## The builtin 'system.locals' implemented as a plugin.
+
+import plugins, ast, astalgo, magicsys, lookups, semdata, lowerings
+
+proc semLocals(c: PContext, n: PNode): PNode =
+  var counter = 0
+  var tupleType = newTypeS(tyTuple, c)
+  result = newNodeIT(nkPar, n.info, tupleType)
+  tupleType.n = newNodeI(nkRecList, n.info)
+  # for now we skip openarrays ...
+  for scope in walkScopes(c.currentScope):
+    if scope == c.topLevelScope: break
+    for it in items(scope.symbols):
+      # XXX parameters' owners are wrong for generics; this caused some pain
+      # for closures too; we should finally fix it.
+      #if it.owner != c.p.owner: return result
+      if it.kind in skLocalVars and
+          it.typ.skipTypes({tyGenericInst, tyVar}).kind notin
+            {tyVarargs, tyOpenArray, tyTypeDesc, tyStatic, tyExpr, tyStmt, tyEmpty}:
+
+        var field = newSym(skField, it.name, getCurrOwner(), n.info)
+        field.typ = it.typ.skipTypes({tyGenericInst, tyVar})
+        field.position = counter
+        inc(counter)
+
+        addSon(tupleType.n, newSymNode(field))
+        addSonSkipIntLit(tupleType, field.typ)
+
+        var a = newSymNode(it, result.info)
+        if it.typ.skipTypes({tyGenericInst}).kind == tyVar: a = newDeref(a)
+        result.add(a)
+
+registerPlugin("stdlib", "system", "locals", semLocals)

compiler/pragmas.nim

@@ -60,7 +60,7 @@ const
   varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl,
     wMagic, wHeader, wDeprecated, wCompilerproc, wDynlib, wExtern,
     wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal,
-    wGensym, wInject, wCodegenDecl, wGuard}
+    wGensym, wInject, wCodegenDecl, wGuard, wGoto}
   constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl,
     wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject}
   letPragmas* = varPragmas
@@ -843,6 +843,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
             invalidPragma(it)
           else:
             sym.guard = pragmaGuard(c, it, sym.kind)
+        of wGoto:
+          if sym == nil or sym.kind notin {skVar, skLet}:
+            invalidPragma(it)
+          else:
+            sym.flags.incl sfGoto
         of wInjectStmt:
           if it.kind != nkExprColonExpr:
             localError(it.info, errExprExpected)

compiler/sem.nim

@@ -16,7 +16,7 @@ import
   procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
   intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity,
-  semparallel, lowerings
+  semparallel, lowerings, plugins, plugins.active
 
 when defined(nimfix):
   import nimfix.prettybase
@@ -89,6 +89,10 @@ proc fitNode(c: PContext, formal: PType, arg: PNode): PNode =
       let x = result.skipConv
       if x.kind == nkPar and formal.kind != tyExpr:
         changeType(x, formal, check=true)
+      else:
+        result = skipHiddenSubConv(result)
+        #result.typ = takeType(formal, arg.typ)
+        #echo arg.info, " picked ", result.typ.typeToString
 
 proc inferWithMetatype(c: PContext, formal: PType,
                        arg: PNode, coerceDistincts = false): PNode

compiler/semdata.nim

@@ -118,7 +118,6 @@ proc newOptionEntry*(): POptionEntry
 proc newLib*(kind: TLibKind): PLib
 proc addToLib*(lib: PLib, sym: PSym)
 proc makePtrType*(c: PContext, baseType: PType): PType
-proc makeVarType*(c: PContext, baseType: PType): PType
 proc newTypeS*(kind: TTypeKind, c: PContext): PType
 proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext)
 
@@ -213,7 +212,10 @@ proc makePtrType(c: PContext, baseType: PType): PType =
   result = newTypeS(tyPtr, c)
   addSonSkipIntLit(result, baseType.assertNotNil)
 
-proc makeVarType(c: PContext, baseType: PType): PType =
+proc makeVarType*(c: PContext, baseType: PType): PType =
+  if baseType.kind == tyVar:
+    result = baseType
+  else:
     result = newTypeS(tyVar, c)
     addSonSkipIntLit(result, baseType.assertNotNil)
 
@@ -247,6 +249,7 @@ proc makeAndType*(c: PContext, t1, t2: PType): PType =
   propagateToOwner(result, t1)
   propagateToOwner(result, t2)
   result.flags.incl((t1.flags + t2.flags) * {tfHasStatic})
+  result.flags.incl tfHasMeta
 
 proc makeOrType*(c: PContext, t1, t2: PType): PType =
   result = newTypeS(tyOr, c)
@@ -254,12 +257,14 @@ proc makeOrType*(c: PContext, t1, t2: PType): PType =
   propagateToOwner(result, t1)
   propagateToOwner(result, t2)
   result.flags.incl((t1.flags + t2.flags) * {tfHasStatic})
+  result.flags.incl tfHasMeta
 
 proc makeNotType*(c: PContext, t1: PType): PType =
   result = newTypeS(tyNot, c)
   result.sons = @[t1]
   propagateToOwner(result, t1)
   result.flags.incl(t1.flags * {tfHasStatic})
+  result.flags.incl tfHasMeta
 
 proc nMinusOne*(n: PNode): PNode =
   result = newNode(nkCall, n.info, @[

compiler/semexprs.nim

@@ -24,7 +24,7 @@ proc semFieldAccess(c: PContext, n: PNode, flags: TExprFlags = {}): PNode
 proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   # same as 'semExprWithType' but doesn't check for proc vars
   result = semExpr(c, n, flags + {efOperand})
-  if result.kind == nkEmpty:
+  if result.kind == nkEmpty and result.typ.isNil:
     # do not produce another redundant error message:
     #raiseRecoverableError("")
     result = errorNode(c, n)
@@ -389,7 +389,7 @@ proc isOpImpl(c: PContext, n: PNode): PNode =
     maybeLiftType(t2, c, n.info)
     var m: TCandidate
     initCandidate(c, m, t2)
-    let match = typeRel(m, t2, t1) != isNone
+    let match = typeRel(m, t2, t1) >= isSubtype # isNone
     result = newIntNode(nkIntLit, ord(match))
 
   result.typ = n.typ
@@ -447,6 +447,7 @@ proc changeType(n: PNode, newType: PType, check: bool) =
   of nkPar:
     let tup = newType.skipTypes({tyGenericInst})
     if tup.kind != tyTuple:
+      if tup.kind == tyObject: return
       internalError(n.info, "changeType: no tuple type for constructor")
     elif sonsLen(n) > 0 and n.sons[0].kind == nkExprColonExpr:
       # named tuple?
@@ -535,6 +536,18 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
   result.typ.sons[0] = makeRangeType(c, 0, sonsLen(result) - 1, n.info)
 
 proc fixAbstractType(c: PContext, n: PNode) =
+  for i in 1 .. < n.len:
+    let it = n.sons[i]
+    # do not get rid of nkHiddenSubConv for OpenArrays, the codegen needs it:
+    if it.kind == nkHiddenSubConv and
+        skipTypes(it.typ, abstractVar).kind notin {tyOpenArray, tyVarargs}:
+      if skipTypes(it.sons[1].typ, abstractVar).kind in
+            {tyNil, tyArrayConstr, tyTuple, tySet}:
+        var s = skipTypes(it.typ, abstractVar)
+        if s.kind != tyExpr:
+          changeType(it.sons[1], s, check=true)
+        n.sons[i] = it.sons[1]
+  when false:
     # XXX finally rewrite that crap!
     for i in countup(1, sonsLen(n) - 1):
       var it = n.sons[i]
@@ -2040,7 +2053,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   of nkEmpty, nkNone, nkCommentStmt:
     discard
   of nkNilLit:
-    result.typ = getSysType(tyNil)
+    if result.typ == nil: result.typ = getSysType(tyNil)
   of nkIntLit:
     if result.typ == nil: setIntLitType(result)
   of nkInt8Lit:

compiler/semfold.nim

@@ -118,6 +118,9 @@ proc makeRange(typ: PType, first, last: BiggestInt): PType =
   let lowerNode = newIntNode(nkIntLit, minA)
   if typ.kind == tyInt and minA == maxA:
     result = getIntLitType(lowerNode)
+  elif typ.kind in {tyUint, tyUInt64}:
+    # these are not ordinal types, so you get no subrange type for these:
+    result = typ
   else:
     var n = newNode(nkRange)
     addSon(n, lowerNode)
@@ -135,8 +138,9 @@ proc makeRangeF(typ: PType, first, last: BiggestFloat): PType =
   addSonSkipIntLit(result, skipTypes(typ, {tyRange}))
 
 proc getIntervalType*(m: TMagic, n: PNode): PType =
-  # Nimrod requires interval arithmetic for ``range`` types. Lots of tedious
+  # Nim requires interval arithmetic for ``range`` types. Lots of tedious
   # work but the feature is very nice for reducing explicit conversions.
+  const ordIntLit = {nkIntLit..nkUInt64Lit}
   result = n.typ
 
   template commutativeOp(opr: expr) {.immediate.} =
@@ -208,15 +212,15 @@ proc getIntervalType*(m: TMagic, n: PNode): PType =
     var a = n.sons[1]
     var b = n.sons[2]
     # symmetrical:
-    if b.kind notin {nkIntLit..nkUInt32Lit}: swap(a, b)
+    if b.kind notin ordIntLit: swap(a, b)
-    if b.kind in {nkIntLit..nkUInt32Lit}:
+    if b.kind in ordIntLit:
       let x = b.intVal|+|1
       if (x and -x) == x and x >= 0:
         result = makeRange(a.typ, 0, b.intVal)
   of mModU:
     let a = n.sons[1]
     let b = n.sons[2]
-    if b.kind in {nkIntLit..nkUInt32Lit}:
+    if a.kind in ordIntLit:
       if b.intVal >= 0:
         result = makeRange(a.typ, 0, b.intVal-1)
       else:
@@ -232,9 +236,9 @@ proc getIntervalType*(m: TMagic, n: PNode): PType =
         result = makeRange(a.typ, b.intVal+1, -(b.intVal+1))
   of mDivI, mDivI64, mDivU:
     binaryOp(`|div|`)
-  of mMinI, mMinI64:
+  of mMinI:
     commutativeOp(min)
-  of mMaxI, mMaxI64:
+  of mMaxI:
     commutativeOp(max)
   else: discard
 
@@ -282,10 +286,14 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mNot: result = newIntNodeT(1 - getInt(a), n)
   of mCard: result = newIntNodeT(nimsets.cardSet(a), n)
   of mBitnotI, mBitnotI64: result = newIntNodeT(not getInt(a), n)
-  of mLengthStr: result = newIntNodeT(len(getStr(a)), n)
+  of mLengthStr, mXLenStr:
+    if a.kind == nkNilLit: result = newIntNodeT(0, n)
+    else: result = newIntNodeT(len(getStr(a)), n)
   of mLengthArray: result = newIntNodeT(lengthOrd(a.typ), n)
-  of mLengthSeq, mLengthOpenArray: result = newIntNodeT(sonsLen(a), n) # BUGFIX
+  of mLengthSeq, mLengthOpenArray, mXLenSeq:
-  of mUnaryPlusI, mUnaryPlusI64, mUnaryPlusF64: result = a # throw `+` away
+    if a.kind == nkNilLit: result = newIntNodeT(0, n)
+    else: result = newIntNodeT(sonsLen(a), n) # BUGFIX
+  of mUnaryPlusI, mUnaryPlusF64: result = a # throw `+` away
   of mToFloat, mToBiggestFloat:
     result = newFloatNodeT(toFloat(int(getInt(a))), n)
   of mToInt, mToBiggestInt: result = newIntNodeT(system.toInt(getFloat(a)), n)
@@ -305,10 +313,10 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mAddI, mAddI64: result = newIntNodeT(getInt(a) + getInt(b), n)
   of mSubI, mSubI64: result = newIntNodeT(getInt(a) - getInt(b), n)
   of mMulI, mMulI64: result = newIntNodeT(getInt(a) * getInt(b), n)
-  of mMinI, mMinI64:
+  of mMinI:
     if getInt(a) > getInt(b): result = newIntNodeT(getInt(b), n)
     else: result = newIntNodeT(getInt(a), n)
-  of mMaxI, mMaxI64:
+  of mMaxI:
     if getInt(a) > getInt(b): result = newIntNodeT(getInt(a), n)
     else: result = newIntNodeT(getInt(b), n)
   of mShlI, mShlI64:
@@ -426,7 +434,8 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
      mExit, mInc, ast.mDec, mEcho, mSwap, mAppendStrCh,
      mAppendStrStr, mAppendSeqElem, mSetLengthStr, mSetLengthSeq,
      mParseExprToAst, mParseStmtToAst, mExpandToAst, mTypeTrait, mDotDot,
-     mNLen..mNError, mEqRef, mSlurp, mStaticExec, mNGenSym, mSpawn, mParallel:
+     mNLen..mNError, mEqRef, mSlurp, mStaticExec, mNGenSym, mSpawn,
+     mParallel, mPlugin:
     discard
   else: internalError(a.info, "evalOp(" & $m & ')')
 
@@ -519,7 +528,7 @@ proc rangeCheck(n: PNode, value: BiggestInt) =
 proc foldConv*(n, a: PNode; check = false): PNode =
   # XXX range checks?
   case skipTypes(n.typ, abstractRange).kind
-  of tyInt..tyInt64:
+  of tyInt..tyInt64, tyUInt..tyUInt64:
     case skipTypes(a.typ, abstractRange).kind
     of tyFloat..tyFloat64:
       result = newIntNodeT(int(getFloat(a)), n)
@@ -646,7 +655,7 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
     result = copyNode(n)
   of nkIfExpr:
     result = getConstIfExpr(m, n)
-  of nkCall, nkCommand, nkCallStrLit, nkPrefix, nkInfix:
+  of nkCallKinds:
     if n.sons[0].kind != nkSym: return
     var s = n.sons[0].sym
     if s.kind != skProc: return

compiler/seminst.nim

@@ -187,7 +187,9 @@ proc instantiateProcType(c: PContext, pt: TIdTable,
       let param = copySym(oldParam)
       param.owner = prc
       param.typ = result.sons[i]
-      param.ast = oldParam.ast.copyTree
+      if oldParam.ast != nil:
+        param.ast = fitNode(c, param.typ, oldParam.ast)
+
       # don't be lazy here and call replaceTypeVarsN(cl, originalParams[i])!
       result.n.sons[i] = newSymNode(param)
       addDecl(c, param)

compiler/semmacrosanity.nim

@@ -34,7 +34,7 @@ proc ithField(n: PNode, field: int): PSym =
   else: discard
 
 proc annotateType*(n: PNode, t: PType) =
-  let x = t.skipTypes(abstractInst)
+  let x = t.skipTypes(abstractInst+{tyRange})
   # Note: x can be unequal to t and we need to be careful to use 't'
   # to not to skip tyGenericInst
   case n.kind

compiler/semmagic.nim

@@ -54,7 +54,7 @@ proc evalTypeTrait(trait: PNode, operand: PType, context: PSym): PNode =
     result.typ = newType(tyString, context)
     result.info = trait.info
   of "arity":
-    result = newIntNode(nkIntLit, typ.n.len-1)
+    result = newIntNode(nkIntLit, typ.len - ord(typ.kind==tyProc))
     result.typ = newType(tyInt, context)
     result.info = trait.info
   else:
@@ -101,34 +101,6 @@ proc semBindSym(c: PContext, n: PNode): PNode =
   else:
     localError(n.sons[1].info, errUndeclaredIdentifier, sl.strVal)
 
-proc semLocals(c: PContext, n: PNode): PNode =
-  var counter = 0
-  var tupleType = newTypeS(tyTuple, c)
-  result = newNodeIT(nkPar, n.info, tupleType)
-  tupleType.n = newNodeI(nkRecList, n.info)
-  # for now we skip openarrays ...
-  for scope in walkScopes(c.currentScope):
-    if scope == c.topLevelScope: break
-    for it in items(scope.symbols):
-      # XXX parameters' owners are wrong for generics; this caused some pain
-      # for closures too; we should finally fix it.
-      #if it.owner != c.p.owner: return result
-      if it.kind in skLocalVars and
-          it.typ.skipTypes({tyGenericInst, tyVar}).kind notin
-            {tyVarargs, tyOpenArray, tyTypeDesc, tyStatic, tyExpr, tyStmt, tyEmpty}:
-
-        var field = newSym(skField, it.name, getCurrOwner(), n.info)
-        field.typ = it.typ.skipTypes({tyGenericInst, tyVar})
-        field.position = counter
-        inc(counter)
-
-        addSon(tupleType.n, newSymNode(field))
-        addSonSkipIntLit(tupleType, field.typ)
-
-        var a = newSymNode(it, result.info)
-        if it.typ.skipTypes({tyGenericInst}).kind == tyVar: a = newDeref(a)
-        result.add(a)
-
 proc semShallowCopy(c: PContext, n: PNode, flags: TExprFlags): PNode
 
 proc isStrangeArray(t: PType): bool =
@@ -161,7 +133,6 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
   of mHigh, mLow: result = semLowHigh(c, n, n[0].sym.magic)
   of mShallowCopy: result = semShallowCopy(c, n, flags)
   of mNBindSym: result = semBindSym(c, n)
-  of mLocals: result = semLocals(c, n)
   of mProcCall:
     result = n
     result.typ = n[1].typ
@@ -196,4 +167,11 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
         result.add newSymNode(createMagic("-", mSubI), n.info)
         result.add lenExprB
         result.add n.sons[1]
+  of mPlugin:
+    let plugin = getPlugin(n[0].sym)
+    if plugin.isNil:
+      localError(n.info, "cannot find plugin " & n[0].sym.name.s)
+      result = n
+    else:
+      result = plugin(c, n)
   else: result = n

compiler/sempass2.nim

@@ -560,7 +560,10 @@ proc trackCase(tracked: PEffects, n: PNode) =
   track(tracked, n.sons[0])
   let oldState = tracked.init.len
   let oldFacts = tracked.guards.len
-  let interesting = interestingCaseExpr(n.sons[0]) and warnProveField in gNotes
+  let stringCase = skipTypes(n.sons[0].typ,
+        abstractVarRange-{tyTypeDesc}).kind in {tyFloat..tyFloat128, tyString}
+  let interesting = not stringCase and interestingCaseExpr(n.sons[0]) and
+        warnProveField in gNotes
   var inter: TIntersection = @[]
   var toCover = 0
   for i in 1.. <n.len:
@@ -575,13 +578,8 @@ proc trackCase(tracked: PEffects, n: PNode) =
     for i in oldState.. <tracked.init.len:
       addToIntersection(inter, tracked.init[i])
 
-  let exh = case skipTypes(n.sons[0].typ, abstractVarRange-{tyTypeDesc}).kind
-            of tyFloat..tyFloat128, tyString:
-              lastSon(n).kind == nkElse
-            else:
-              true
   setLen(tracked.init, oldState)
-  if exh:
+  if not stringCase or lastSon(n).kind == nkElse:
     for id, count in items(inter):
       if count >= toCover: tracked.init.add id
     # else we can't merge
@@ -714,8 +712,8 @@ proc track(tracked: PEffects, n: PNode) =
   of nkVarSection, nkLetSection:
     for child in n:
       let last = lastSon(child)
+      if last.kind != nkEmpty: track(tracked, last)
       if child.kind == nkIdentDefs and last.kind != nkEmpty:
-        track(tracked, last)
         for i in 0 .. child.len-3:
           initVar(tracked, child.sons[i], volatileCheck=false)
           addAsgnFact(tracked.guards, child.sons[i], last)

compiler/semstmts.nim

@@ -102,10 +102,6 @@ proc semDestructorCheck(c: PContext, n: PNode, flags: TExprFlags) {.inline.} =
         c.p.owner.kind notin {skTemplate, skMacro}:
       localError(n.info, errGenerated, "value expected, but got a type")
 
-proc newDeref(n: PNode): PNode {.inline.} =
-  result = newNodeIT(nkHiddenDeref, n.info, n.typ.sons[0])
-  addSon(result, n)
-
 proc semExprBranch(c: PContext, n: PNode): PNode =
   result = semExpr(c, n)
   if result.typ != nil:
@@ -373,6 +369,11 @@ proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) =
   else:
     result.add identDefs
 
+proc isDiscardUnderscore(v: PSym): bool =
+  if v.name.s == "_":
+    v.flags.incl(sfGenSym)
+    result = true
+
 proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
   var b: PNode
   result = copyNode(n)
@@ -436,7 +437,8 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
 
     for j in countup(0, length-3):
       var v = semIdentDef(c, a.sons[j], symkind)
-      if sfGenSym notin v.flags: addInterfaceDecl(c, v)
+      if sfGenSym notin v.flags and not isDiscardUnderscore(v):
+        addInterfaceDecl(c, v)
       when oKeepVariableNames:
         if c.inUnrolledContext > 0: v.flags.incl(sfShadowed)
         else:
@@ -551,7 +553,8 @@ proc semForVars(c: PContext, n: PNode): PNode =
       if getCurrOwner().kind == skModule: incl(v.flags, sfGlobal)
       v.typ = iter.sons[i]
       n.sons[i] = newSymNode(v)
-      if sfGenSym notin v.flags: addForVarDecl(c, v)
+      if sfGenSym notin v.flags and not isDiscardUnderscore(v):
+        addForVarDecl(c, v)
   inc(c.p.nestedLoopCounter)
   n.sons[length-1] = semStmt(c, n.sons[length-1])
   dec(c.p.nestedLoopCounter)

compiler/semtypes.nim

@@ -261,7 +261,8 @@ proc semArray(c: PContext, n: PNode, prev: PType): PType =
       if not isOrdinalType(indx):
         localError(n.sons[1].info, errOrdinalTypeExpected)
       elif enumHasHoles(indx):
-        localError(n.sons[1].info, errEnumXHasHoles, indx.sym.name.s)
+        localError(n.sons[1].info, errEnumXHasHoles,
+                   typeToString(indx.skipTypes({tyRange})))
     base = semTypeNode(c, n.sons[2], nil)
     addSonSkipIntLit(result, base)
   else:
@@ -628,7 +629,7 @@ proc skipGenericInvocation(t: PType): PType {.inline.} =
   result = t
   if result.kind == tyGenericInvocation:
     result = result.sons[0]
-  if result.kind == tyGenericBody:
+  while result.kind in {tyGenericInst, tyGenericBody}:
     result = lastSon(result)
 
 proc addInheritedFields(c: PContext, check: var IntSet, pos: var int,
@@ -834,7 +835,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
       cp.kind = tyUserTypeClassInst
       return addImplicitGeneric(cp)
 
-    for i in 1 .. (paramType.sons.len - 2):
+    for i in 1 .. paramType.len-2:
       var lifted = liftingWalk(paramType.sons[i])
       if lifted != nil:
         paramType.sons[i] = lifted
@@ -847,7 +848,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
       result.shouldHaveMeta
 
   of tyGenericInvocation:
-    for i in 1 .. <paramType.sonsLen:
+    for i in 1 .. <paramType.len:
       let lifted = liftingWalk(paramType.sons[i])
       if lifted != nil: paramType.sons[i] = lifted
     when false:

compiler/semtypinst.nim

@@ -233,6 +233,8 @@ proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
   # XXX: relying on allowMetaTypes is a kludge
   result = copyType(t, t.owner, cl.allowMetaTypes)
   result.flags.incl tfFromGeneric
+  if not (t.kind in tyMetaTypes or
+         (t.kind == tyStatic and t.n == nil)):
     result.flags.excl tfInstClearedFlags
 
 proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =

compiler/sigmatch.nim

@@ -147,6 +147,7 @@ proc copyCandidate(a: var TCandidate, b: TCandidate) =
 
 proc sumGeneric(t: PType): int =
   var t = t
+  var isvar = 1
   while true:
     case t.kind
     of tyGenericInst, tyArray, tyRef, tyPtr, tyDistinct, tyArrayConstr,
@@ -154,18 +155,20 @@ proc sumGeneric(t: PType): int =
       t = t.lastSon
       inc result
     of tyVar:
-      # but do not make 'var T' more specific than 'T'!
       t = t.sons[0]
+      inc result
+      inc isvar
     of tyGenericInvocation, tyTuple:
-      result = ord(t.kind == tyGenericInvocation)
+      result += ord(t.kind == tyGenericInvocation)
       for i in 0 .. <t.len: result += t.sons[i].sumGeneric
       break
     of tyGenericParam, tyExpr, tyStatic, tyStmt, tyTypeDesc: break
     of tyBool, tyChar, tyEnum, tyObject, tyProc, tyPointer,
         tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
         tyUInt..tyUInt64:
-      return 1
+      return isvar
-    else: return 0
+    else:
+      return 0
 
 #var ggDebug: bool
 
@@ -919,6 +922,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
 
   of tyAnd:
     considerPreviousT:
+      result = isEqual
       for branch in f.sons:
         let x = typeRel(c, branch, aOrig)
         if x < isSubtype: return isNone
@@ -1108,8 +1112,10 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
       localError(f.n.info, errTypeExpected)
       result = isNone
 
+  of tyNone:
+    if a.kind == tyNone: result = isEqual
   else:
-    internalAssert false
+    internalError " unknown type kind " & $f.kind
 
 proc cmpTypes*(c: PContext, f, a: PType): TTypeRelation =
   var m: TCandidate
@@ -1194,15 +1200,6 @@ proc isInlineIterator*(t: PType): bool =
   result = t.kind == tyIter or
           (t.kind == tyBuiltInTypeClass and t.base.kind == tyIter)
 
-proc isEmptyContainer*(t: PType): bool =
-  case t.kind
-  of tyExpr, tyNil: result = true
-  of tyArray, tyArrayConstr: result = t.sons[1].kind == tyEmpty
-  of tySet, tySequence, tyOpenArray, tyVarargs:
-    result = t.sons[0].kind == tyEmpty
-  of tyGenericInst: result = isEmptyContainer(t.lastSon)
-  else: result = false
-
 proc incMatches(m: var TCandidate; r: TTypeRelation; convMatch = 1) =
   case r
   of isConvertible, isIntConv: inc(m.convMatches, convMatch)
@@ -1307,7 +1304,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
     if arg.typ == nil:
       result = arg
     elif skipTypes(arg.typ, abstractVar-{tyTypeDesc}).kind == tyTuple:
-      result = implicitConv(nkHiddenStdConv, f, copyTree(arg), m, c)
+      result = implicitConv(nkHiddenSubConv, f, arg, m, c)
     elif arg.typ.isEmptyContainer:
       result = arg.copyTree
       result.typ = getInstantiatedType(c, arg, m, f)
@@ -1322,7 +1319,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
     inc(m.exactMatches)
     result = arg
     if skipTypes(f, abstractVar-{tyTypeDesc}).kind in {tyTuple}:
-      result = implicitConv(nkHiddenStdConv, f, arg, m, c)
+      result = implicitConv(nkHiddenSubConv, f, arg, m, c)
   of isNone:
     # do not do this in ``typeRel`` as it then can't infere T in ``ref T``:
     if a.kind in {tyProxy, tyUnknown}:
@@ -1468,9 +1465,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
         m.state = csNoMatch
         return
     if formal.typ.kind == tyVar:
-      if n.isLValue:
+      if not n.isLValue:
-        inc(m.genericMatches, 100)
-      else:
         m.state = csNoMatch
         return
 
@@ -1576,6 +1571,8 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
           #assert(container == nil)
           if container.isNil:
             container = newNodeIT(nkBracket, n.sons[a].info, arrayConstr(c, arg))
+          else:
+            incrIndexType(container.typ)
           addSon(container, arg)
           setSon(m.call, formal.position + 1,
                  implicitConv(nkHiddenStdConv, formal.typ, container, m, c))

compiler/transf.nim

@@ -379,6 +379,9 @@ proc transformConv(c: PTransf, n: PNode): PTransNode =
       result = transformSons(c, n)
   of tyOpenArray, tyVarargs:
     result = transform(c, n.sons[1])
+    PNode(result).typ = takeType(n.typ, n.sons[1].typ)
+    #echo n.info, " came here and produced ", typeToString(PNode(result).typ),
+    #   " from ", typeToString(n.typ), " and ", typeToString(n.sons[1].typ)
   of tyCString:
     if source.kind == tyString:
       result = newTransNode(nkStringToCString, n, 1)
@@ -713,8 +716,7 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     add(result, PTransNode(newSymNode(labl)))
   of nkBreakStmt: result = transformBreak(c, n)
   of nkWhileStmt: result = transformWhile(c, n)
-  of nkCall, nkHiddenCallConv, nkCommand, nkInfix, nkPrefix, nkPostfix,
+  of nkCallKinds:
-     nkCallStrLit:
     result = transformCall(c, n)
   of nkAddr, nkHiddenAddr:
     result = transformAddrDeref(c, n, nkDerefExpr, nkHiddenDeref)

compiler/trees.nim

@@ -142,7 +142,7 @@ proc swapOperands*(op: PNode) =
   op.sons[2] = tmp
 
 proc isRange*(n: PNode): bool {.inline.} =
-  if n.kind == nkInfix:
+  if n.kind in nkCallKinds:
     if n[0].kind == nkIdent and n[0].ident.id == ord(wDotDot) or
         n[0].kind in {nkClosedSymChoice, nkOpenSymChoice} and
         n[0][1].sym.name.id == ord(wDotDot):

compiler/types.nim

@@ -541,6 +541,9 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
   of tyProc:
     result = if tfIterator in t.flags: "iterator (" else: "proc ("
     for i in countup(1, sonsLen(t) - 1):
+      if t.n != nil and i < t.n.len and t.n[i].kind == nkSym:
+        add(result, t.n[i].sym.name.s)
+        add(result, ": ")
       add(result, typeToString(t.sons[i]))
       if i < sonsLen(t) - 1: add(result, ", ")
     add(result, ')')
@@ -1436,3 +1439,45 @@ proc skipConv*(n: PNode): PNode =
 proc skipConvTakeType*(n: PNode): PNode =
   result = n.skipConv
   result.typ = n.typ
+
+proc isEmptyContainer*(t: PType): bool =
+  case t.kind
+  of tyExpr, tyNil: result = true
+  of tyArray, tyArrayConstr: result = t.sons[1].kind == tyEmpty
+  of tySet, tySequence, tyOpenArray, tyVarargs:
+    result = t.sons[0].kind == tyEmpty
+  of tyGenericInst: result = isEmptyContainer(t.lastSon)
+  else: result = false
+
+proc takeType*(formal, arg: PType): PType =
+  # param: openArray[string] = []
+  # [] is an array constructor of length 0 of type string!
+  if arg.kind == tyNil:
+    # and not (formal.kind == tyProc and formal.callConv == ccClosure):
+    result = formal
+  elif formal.kind in {tyOpenArray, tyVarargs, tySequence} and
+      arg.isEmptyContainer:
+    let a = copyType(arg.skipTypes({tyGenericInst}), arg.owner, keepId=false)
+    a.sons[ord(arg.kind in {tyArray, tyArrayConstr})] = formal.sons[0]
+    result = a
+  elif formal.kind in {tyTuple, tySet} and arg.kind == formal.kind:
+    result = formal
+  else:
+    result = arg
+
+proc skipHiddenSubConv*(n: PNode): PNode =
+  if n.kind == nkHiddenSubConv:
+    # param: openArray[string] = []
+    # [] is an array constructor of length 0 of type string!
+    let formal = n.typ
+    result = n.sons[1]
+    let arg = result.typ
+    let dest = takeType(formal, arg)
+    if dest == arg and formal.kind != tyExpr:
+      #echo n.info, " came here for ", formal.typeToString
+      result = n
+    else:
+      result = copyTree(result)
+      result.typ = dest
+  else:
+    result = n

compiler/vm.nim

@@ -16,7 +16,8 @@ import ast except getstr
 
 import
   strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes, unsigned,
-  parser, vmdeps, idents, trees, renderer, options, transf, parseutils
+  parser, vmdeps, idents, trees, renderer, options, transf, parseutils,
+  vmmarshal
 
 from semfold import leValueConv, ordinalValToString
 from evaltempl import evalTemplate
@@ -371,11 +372,6 @@ template handleJmpBack() {.dirty.} =
       globalError(c.debug[pc], errTooManyIterations)
   dec(c.loopIterations)
 
-proc skipColon(n: PNode): PNode =
-  result = n
-  if n.kind == nkExprColonExpr:
-    result = n.sons[1]
-
 proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
   var pc = start
   var tos = tos
@@ -1043,7 +1039,14 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeB(rkNode)
       let newLen = regs[rb].intVal.int
       if regs[ra].node.isNil: stackTrace(c, tos, pc, errNilAccess)
-      else: setLen(regs[ra].node.sons, newLen)
+      else:
+        let oldLen = regs[ra].node.len
+        setLen(regs[ra].node.sons, newLen)
+        if oldLen < newLen:
+          # XXX This is still not entirely correct
+          # set to default value:
+          for i in oldLen .. <newLen:
+            regs[ra].node.sons[i] = newNodeI(nkEmpty, c.debug[pc])
     of opcSwap:
       let rb = instr.regB
       if regs[ra].kind == regs[rb].kind:
@@ -1362,6 +1365,19 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       while typ.kind == tyTypeDesc and typ.len > 0: typ = typ.sons[0]
       createStr regs[ra]
       regs[ra].node.strVal = typ.typeToString(preferExported)
+    of opcMarshalLoad:
+      let ra = instr.regA
+      let rb = instr.regB
+      inc pc
+      let typ = c.types[c.code[pc].regBx - wordExcess]
+      putIntoReg(regs[ra], loadAny(regs[rb].node.strVal, typ))
+    of opcMarshalStore:
+      decodeB(rkNode)
+      inc pc
+      let typ = c.types[c.code[pc].regBx - wordExcess]
+      createStrKeepNode(regs[ra])
+      if regs[ra].node.strVal.isNil: regs[ra].node.strVal = newStringOfCap(1000)
+      storeAny(regs[ra].node.strVal, typ, regs[rb].regToNode)
     inc pc
 
 proc execute(c: PCtx, start: int): PNode =

compiler/vmdef.nim

@@ -16,7 +16,7 @@ const
   byteExcess* = 128 # we use excess-K for immediates
   wordExcess* = 32768
 
-  MaxLoopIterations* = 500_000 # max iterations of all loops
+  MaxLoopIterations* = 1500_000 # max iterations of all loops
 
 
 type
@@ -66,7 +66,8 @@ type
     opcMulSet, opcPlusSet, opcMinusSet, opcSymdiffSet, opcConcatStr,
     opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq,
     opcSwap, opcIsNil, opcOf, opcIs,
-    opcSubStr, opcParseFloat, opcConv, opcCast, opcQuit, opcReset,
+    opcSubStr, opcParseFloat, opcConv, opcCast,
+    opcQuit, opcReset,
     opcNarrowS, opcNarrowU,
 
     opcAddStrCh,
@@ -132,7 +133,8 @@ type
     opcLdImmInt,  # dest = immediate value
     opcNBindSym,
     opcSetType,   # dest.typ = types[Bx]
-    opcTypeTrait
+    opcTypeTrait,
+    opcMarshalLoad, opcMarshalStore
 
   TBlock* = object
     label*: PSym
@@ -221,7 +223,8 @@ proc registerCallback*(c: PCtx; name: string; callback: VmCallback) =
 const
   firstABxInstr* = opcTJmp
   largeInstrs* = { # instructions which use 2 int32s instead of 1:
-    opcSubStr, opcConv, opcCast, opcNewSeq, opcOf}
+    opcSubStr, opcConv, opcCast, opcNewSeq, opcOf,
+    opcMarshalLoad, opcMarshalStore}
   slotSomeTemp* = slotTempUnknown
   relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack}
 

compiler/vmdeps.nim

@@ -144,7 +144,9 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
   of tyIter: result = mapTypeToBracket("iter", t, info)
   of tyProxy: result = atomicType"error"
   of tyBuiltInTypeClass: result = mapTypeToBracket("builtinTypeClass", t, info)
-  of tyUserTypeClass: result = mapTypeToBracket("userTypeClass", t, info)
+  of tyUserTypeClass:
+    result = mapTypeToBracket("concept", t, info)
+    result.add t.n.copyTree
   of tyCompositeTypeClass: result = mapTypeToBracket("compositeTypeClass", t, info)
   of tyAnd: result = mapTypeToBracket("and", t, info)
   of tyOr: result = mapTypeToBracket("or", t, info)

compiler/vmgen.nim

@@ -76,6 +76,11 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) =
     elif opc in {opcLdConst, opcAsgnConst}:
       result.addf("\t$#\tr$#, $#", ($opc).substr(3), x.regA,
         c.constants[x.regBx-wordExcess].renderTree)
+    elif opc in {opcMarshalLoad, opcMarshalStore}:
+      let y = c.code[i+1]
+      result.addf("\t$#\tr$#, r$#, $#", ($opc).substr(3), x.regA, x.regB,
+        c.types[y.regBx-wordExcess].typeToString)
+      inc i
     else:
       result.addf("\t$#\tr$#, $#", ($opc).substr(3), x.regA, x.regBx-wordExcess)
     result.add("\t#")
@@ -696,8 +701,7 @@ proc genCard(c: PCtx; n: PNode; dest: var TDest) =
   c.gABC(n, opcCard, dest, tmp)
   c.freeTemp(tmp)
 
-proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
+proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
-  let m = n.sons[0].sym.magic
   case m
   of mAnd: c.genAndOr(n, opcFJmp, dest)
   of mOr:  c.genAndOr(n, opcTJmp, dest)
@@ -742,9 +746,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     c.gABC(n, opcNewStr, dest, tmp)
     c.freeTemp(tmp)
     # XXX buggy
-  of mLengthOpenArray, mLengthArray, mLengthSeq:
+  of mLengthOpenArray, mLengthArray, mLengthSeq, mXLenSeq:
     genUnaryABI(c, n, dest, opcLenSeq)
-  of mLengthStr:
+  of mLengthStr, mXLenStr:
     genUnaryABI(c, n, dest, opcLenStr)
   of mIncl, mExcl:
     unused(n, dest)
@@ -791,7 +795,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     genUnaryABC(c, n, dest, opcUnaryMinusInt)
     genNarrow(c, n, dest)
   of mUnaryMinusF64: genUnaryABC(c, n, dest, opcUnaryMinusFloat)
-  of mUnaryPlusI, mUnaryPlusI64, mUnaryPlusF64: gen(c, n.sons[1], dest)
+  of mUnaryPlusI, mUnaryPlusF64: gen(c, n.sons[1], dest)
   of mBitnotI, mBitnotI64:
     genUnaryABC(c, n, dest, opcBitnotInt)
     genNarrowU(c, n, dest)
@@ -1008,7 +1012,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     if dest < 0: dest = c.getTemp(n.typ)
     c.gABC(n, opcCallSite, dest)
   of mNGenSym: genBinaryABC(c, n, dest, opcGenSym)
-  of mMinI, mMaxI, mMinI64, mMaxI64, mAbsF64, mMinF64, mMaxF64, mAbsI,
+  of mMinI, mMaxI, mAbsF64, mMinF64, mMaxF64, mAbsI,
      mAbsI64, mDotDot:
     c.genCall(n, dest)
   of mExpandToAst:
@@ -1028,6 +1032,22 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     # mGCref, mGCunref,
     internalError(n.info, "cannot generate code for: " & $m)
 
+proc genMarshalLoad(c: PCtx, n: PNode, dest: var TDest) =
+  ## Signature: proc to*[T](data: string): T
+  if dest < 0: dest = c.getTemp(n.typ)
+  var tmp = c.genx(n.sons[1])
+  c.gABC(n, opcMarshalLoad, dest, tmp)
+  c.gABx(n, opcMarshalLoad, 0, c.genType(n.typ))
+  c.freeTemp(tmp)
+
+proc genMarshalStore(c: PCtx, n: PNode, dest: var TDest) =
+  ## Signature: proc `$$`*[T](x: T): string
+  if dest < 0: dest = c.getTemp(n.typ)
+  var tmp = c.genx(n.sons[1])
+  c.gABC(n, opcMarshalStore, dest, tmp)
+  c.gABx(n, opcMarshalStore, 0, c.genType(n.sons[1].typ))
+  c.freeTemp(tmp)
+
 const
   atomicTypes = {tyBool, tyChar,
     tyExpr, tyStmt, tyTypeDesc, tyStatic,
@@ -1364,7 +1384,7 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
   of tyCString, tyString:
     result = newNodeIT(nkStrLit, info, t)
   of tyVar, tyPointer, tyPtr, tySequence, tyExpr,
-     tyStmt, tyTypeDesc, tyStatic, tyRef:
+     tyStmt, tyTypeDesc, tyStatic, tyRef, tyNil:
     result = newNodeIT(nkNilLit, info, t)
   of tyProc:
     if t.callConv != ccClosure:
@@ -1391,7 +1411,7 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
       addSon(result, getNullValue(t.sons[i], info))
   of tySet:
     result = newNodeIT(nkCurly, info, t)
-  else: internalError("getNullValue: " & $t.kind)
+  else: internalError(info, "getNullValue: " & $t.kind)
 
 proc ldNullOpcode(t: PType): TOpcode =
   if fitsRegister(t): opcLdNullReg else: opcLdNull
@@ -1533,6 +1553,15 @@ proc matches(s: PSym; x: string): bool =
     dec L
   result = true
 
+proc matches(s: PSym; y: varargs[string]): bool =
+  var s = s
+  var L = y.len-1
+  while L >= 0:
+    if s == nil or y[L].cmpIgnoreStyle(s.name.s) != 0: return false
+    s = if sfFromGeneric in s.flags: s.owner.owner else: s.owner
+    dec L
+  result = true
+
 proc procIsCallback(c: PCtx; s: PSym): bool =
   if s.offset < -1: return true
   var i = -2
@@ -1570,8 +1599,17 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     else:
       internalError(n.info, "cannot generate code for: " & s.name.s)
   of nkCallKinds:
-    if n.sons[0].kind == nkSym and n.sons[0].sym.magic != mNone:
+    if n.sons[0].kind == nkSym:
-      genMagic(c, n, dest)
+      let s = n.sons[0].sym
+      if s.magic != mNone:
+        genMagic(c, n, dest, s.magic)
+      elif matches(s, "stdlib", "marshal", "to"):
+        genMarshalLoad(c, n, dest)
+      elif matches(s, "stdlib", "marshal", "$$"):
+        genMarshalStore(c, n, dest)
+      else:
+        genCall(c, n, dest)
+        clearDest(c, n, dest)
     else:
       genCall(c, n, dest)
       clearDest(c, n, dest)
@@ -1610,7 +1648,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     genBreak(c, n)
   of nkTryStmt: genTry(c, n, dest)
   of nkStmtList:
-    unused(n, dest)
+    #unused(n, dest)
+    # XXX Fix this bug properly, lexim triggers it
     for x in n: gen(c, x)
   of nkStmtListExpr:
     let L = n.len-1

compiler/vmmarshal.nim

@@ -0,0 +1,283 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implements marshaling for the VM.
+
+import streams, json, intsets, tables, ast, astalgo, idents, types, msgs
+
+proc ptrToInt(x: PNode): int {.inline.} =
+  result = cast[int](x) # don't skip alignment
+
+proc getField(n: PNode; position: int): PSym =
+  case n.kind
+  of nkRecList:
+    for i in countup(0, sonsLen(n) - 1):
+      result = getField(n.sons[i], position)
+      if result != nil: return
+  of nkRecCase:
+    result = getField(n.sons[0], position)
+    if result != nil: return
+    for i in countup(1, sonsLen(n) - 1):
+      case n.sons[i].kind
+      of nkOfBranch, nkElse:
+        result = getField(lastSon(n.sons[i]), position)
+        if result != nil: return
+      else: internalError(n.info, "getField(record case branch)")
+  of nkSym:
+    if n.sym.position == position: result = n.sym
+  else: discard
+
+proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet)
+
+proc storeObj(s: var string; typ: PType; x: PNode; stored: var IntSet) =
+  internalAssert x.kind in {nkObjConstr, nkPar}
+  let start = ord(x.kind == nkObjConstr)
+  for i in countup(start, sonsLen(x) - 1):
+    if i > start: s.add(", ")
+    var it = x.sons[i]
+    if it.kind == nkExprColonExpr:
+      internalAssert it.sons[0].kind == nkSym
+      let field = it.sons[0].sym
+      s.add(escapeJson(field.name.s))
+      s.add(": ")
+      storeAny(s, field.typ, it.sons[1], stored)
+    elif typ.n != nil:
+      let field = getField(typ.n, i)
+      s.add(escapeJson(field.name.s))
+      s.add(": ")
+      storeAny(s, field.typ, it, stored)
+
+proc skipColon*(n: PNode): PNode =
+  result = n
+  if n.kind == nkExprColonExpr:
+    result = n.sons[1]
+
+proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet) =
+  case t.kind
+  of tyNone: assert false
+  of tyBool: s.add($(a.intVal != 0))
+  of tyChar:
+    let ch = char(a.intVal)
+    if ch < '\128':
+      s.add(escapeJson($ch))
+    else:
+      s.add($int(ch))
+  of tyArray, tySequence:
+    if t.kind == tySequence and a.kind == nkNilLit: s.add("null")
+    else:
+      s.add("[")
+      for i in 0 .. a.len-1:
+        if i > 0: s.add(", ")
+        storeAny(s, t.elemType, a[i], stored)
+      s.add("]")
+  of tyTuple:
+    s.add("{")
+    for i in 0.. <t.len:
+      if i > 0: s.add(", ")
+      s.add("\"Field" & $i)
+      s.add("\": ")
+      storeAny(s, t.sons[i], a[i].skipColon, stored)
+    s.add("}")
+  of tyObject:
+    s.add("{")
+    storeObj(s, t, a, stored)
+    s.add("}")
+  of tySet:
+    s.add("[")
+    for i in 0.. <a.len:
+      if i > 0: s.add(", ")
+      if a[i].kind == nkRange:
+        var x = copyNode(a[i][0])
+        storeAny(s, t.lastSon, x, stored)
+        while x.intVal+1 <= a[i][1].intVal:
+          s.add(", ")
+          storeAny(s, t.lastSon, x, stored)
+          inc x.intVal
+      else:
+        storeAny(s, t.lastSon, a[i], stored)
+    s.add("]")
+  of tyRange, tyGenericInst: storeAny(s, t.lastSon, a, stored)
+  of tyEnum:
+    # we need a slow linear search because of enums with holes:
+    for e in items(t.n):
+      if e.sym.position == a.intVal:
+        s.add e.sym.name.s.escapeJson
+        break
+  of tyPtr, tyRef:
+    var x = a
+    if isNil(x) or x.kind == nkNilLit: s.add("null")
+    elif stored.containsOrIncl(x.ptrToInt):
+      # already stored, so we simply write out the pointer as an int:
+      s.add($x.ptrToInt)
+    else:
+      # else as a [value, key] pair:
+      # (reversed order for convenient x[0] access!)
+      s.add("[")
+      s.add($x.ptrToInt)
+      s.add(", ")
+      storeAny(s, t.lastSon, a, stored)
+      s.add("]")
+  of tyString, tyCString:
+    if a.kind == nkNilLit or a.strVal.isNil: s.add("null")
+    else: s.add(escapeJson(a.strVal))
+  of tyInt..tyInt64, tyUInt..tyUInt64: s.add($a.intVal)
+  of tyFloat..tyFloat128: s.add($a.floatVal)
+  else:
+    internalError a.info, "cannot marshal at compile-time " & t.typeToString
+
+proc storeAny*(s: var string; t: PType; a: PNode) =
+  var stored = initIntSet()
+  storeAny(s, t, a, stored)
+
+proc loadAny(p: var JsonParser, t: PType,
+             tab: var Table[BiggestInt, PNode]): PNode =
+  case t.kind
+  of tyNone: assert false
+  of tyBool:
+    case p.kind
+    of jsonFalse: result = newIntNode(nkIntLit, 0)
+    of jsonTrue: result = newIntNode(nkIntLit, 1)
+    else: raiseParseErr(p, "'true' or 'false' expected for a bool")
+    next(p)
+  of tyChar:
+    if p.kind == jsonString:
+      var x = p.str
+      if x.len == 1:
+        result = newIntNode(nkIntLit, ord(x[0]))
+        next(p)
+        return
+    elif p.kind == jsonInt:
+      result = newIntNode(nkIntLit, getInt(p))
+      next(p)
+      return
+    raiseParseErr(p, "string of length 1 expected for a char")
+  of tyEnum:
+    if p.kind == jsonString:
+      for e in items(t.n):
+        if e.sym.name.s == p.str:
+          result = newIntNode(nkIntLit, e.sym.position)
+          next(p)
+          return
+    raiseParseErr(p, "string expected for an enum")
+  of tyArray:
+    if p.kind != jsonArrayStart: raiseParseErr(p, "'[' expected for an array")
+    next(p)
+    result = newNode(nkBracket)
+    while p.kind != jsonArrayEnd and p.kind != jsonEof:
+      result.add loadAny(p, t.elemType, tab)
+    if p.kind == jsonArrayEnd: next(p)
+    else: raiseParseErr(p, "']' end of array expected")
+  of tySequence:
+    case p.kind
+    of jsonNull:
+      result = newNode(nkNilLit)
+      next(p)
+    of jsonArrayStart:
+      next(p)
+      result = newNode(nkBracket)
+      while p.kind != jsonArrayEnd and p.kind != jsonEof:
+        result.add loadAny(p, t.elemType, tab)
+      if p.kind == jsonArrayEnd: next(p)
+      else: raiseParseErr(p, "")
+    else:
+      raiseParseErr(p, "'[' expected for a seq")
+  of tyTuple:
+    if p.kind != jsonObjectStart: raiseParseErr(p, "'{' expected for an object")
+    next(p)
+    result = newNode(nkPar)
+    var i = 0
+    while p.kind != jsonObjectEnd and p.kind != jsonEof:
+      if p.kind != jsonString:
+        raiseParseErr(p, "string expected for a field name")
+      next(p)
+      if i >= t.len:
+        raiseParseErr(p, "too many fields to tuple type " & typeToString(t))
+      result.add loadAny(p, t.sons[i], tab)
+      inc i
+    if p.kind == jsonObjectEnd: next(p)
+    else: raiseParseErr(p, "'}' end of object expected")
+  of tyObject:
+    if p.kind != jsonObjectStart: raiseParseErr(p, "'{' expected for an object")
+    next(p)
+    result = newNode(nkPar)
+    result.sons = @[]
+    while p.kind != jsonObjectEnd and p.kind != jsonEof:
+      if p.kind != jsonString:
+        raiseParseErr(p, "string expected for a field name")
+      let field = lookupInRecord(t.n, getIdent(p.str))
+      if field.isNil:
+        raiseParseErr(p, "unknown field for object of type " & typeToString(t))
+      next(p)
+      if field.position >= result.sons.len:
+        setLen(result.sons, field.position+1)
+      result.sons[field.position] = loadAny(p, field.typ, tab)
+    if p.kind == jsonObjectEnd: next(p)
+    else: raiseParseErr(p, "'}' end of object expected")
+  of tySet:
+    if p.kind != jsonArrayStart: raiseParseErr(p, "'[' expected for a set")
+    next(p)
+    result = newNode(nkCurly)
+    while p.kind != jsonArrayEnd and p.kind != jsonEof:
+      result.add loadAny(p, t.lastSon, tab)
+      next(p)
+    if p.kind == jsonArrayEnd: next(p)
+    else: raiseParseErr(p, "']' end of array expected")
+  of tyPtr, tyRef:
+    case p.kind
+    of jsonNull:
+      result = newNode(nkNilLit)
+      next(p)
+    of jsonInt:
+      result = tab[p.getInt]
+      if result.isNil:
+        raiseParseErr(p, "cannot load object with address " & $p.getInt)
+      next(p)
+    of jsonArrayStart:
+      next(p)
+      if p.kind == jsonInt:
+        let idx = p.getInt
+        next(p)
+        result = loadAny(p, t.lastSon, tab)
+        tab[idx] = result
+      else: raiseParseErr(p, "index for ref type expected")
+      if p.kind == jsonArrayEnd: next(p)
+      else: raiseParseErr(p, "']' end of ref-address pair expected")
+    else: raiseParseErr(p, "int for pointer type expected")
+  of tyString, tyCString:
+    case p.kind
+    of jsonNull:
+      result = newNode(nkNilLit)
+      next(p)
+    of jsonString:
+      result = newStrNode(nkStrLit, p.str)
+      next(p)
+    else: raiseParseErr(p, "string expected")
+  of tyInt..tyInt64, tyUInt..tyUInt64:
+    if p.kind == jsonInt:
+      result = newIntNode(nkIntLit, getInt(p))
+      next(p)
+      return
+    raiseParseErr(p, "int expected")
+  of tyFloat..tyFloat128:
+    if p.kind == jsonFloat:
+      result = newFloatNode(nkFloatLit, getFloat(p))
+      next(p)
+      return
+    raiseParseErr(p, "float expected")
+  of tyRange, tyGenericInst: result = loadAny(p, t.lastSon, tab)
+  else:
+    internalError "cannot marshal at compile-time " & t.typeToString
+
+proc loadAny*(s: string; t: PType): PNode =
+  var tab = initTable[BiggestInt, PNode]()
+  var p: JsonParser
+  open(p, newStringStream(s), "unknown file")
+  next(p)
+  result = loadAny(p, t, tab)
+  close(p)

config/nim.cfg

@@ -96,7 +96,7 @@ path="$lib/pure/unidecode"
 
 # Configuration for the GNU C/C++ compiler:
 @if windows:
-  #gcc.path = r"$nimrod\dist\mingw\bin"
+  #gcc.path = r"$nim\dist\mingw\bin"
   @if gcc:
     tlsEmulation:on
   @end

csources

@@ -0,0 +1 @@
+Subproject commit 15724e2e1f3e7749d508dfcd995e84fea2850802

doc/grammar.txt

@@ -179,7 +179,7 @@ simpleStmt = ((returnStmt | raiseStmt | yieldStmt | discardStmt | breakStmt
            | continueStmt | pragmaStmt | importStmt | exportStmt | fromStmt
            | includeStmt | commentStmt) / exprStmt) COMMENT?
 complexOrSimpleStmt = (ifStmt | whenStmt | whileStmt
-                    | tryStmt | finallyStmt | exceptStmt | forStmt
+                    | tryStmt | forStmt
                     | blockStmt | staticStmt | deferStmt | asmStmt
                     | 'proc' routine
                     | 'method' routine

doc/lib.txt

@@ -453,10 +453,6 @@ Other
 * `zipfiles <zipfiles.html>`_
   This module implements a zip archive creator/reader/modifier.
 
-* `web <web.html>`_
-  This module contains simple high-level procedures for dealing with the
-  Web like loading the contents of a Web page from an URL.
-
 * `ssl <ssl.html>`_
   This module provides an easy to use sockets-style
   Nim interface to the OpenSSL library.
@@ -513,25 +509,6 @@ Regular expressions
   Wrapper for the TRE library.
 
 
-Graphics libraries
-------------------
-
-* `sdl <sdl.html>`_
-  Part of the wrapper for SDL.
-* `sdl_gfx <sdl_gfx.html>`_
-  Part of the wrapper for SDL.
-* `sdl_image <sdl_image.html>`_
-  Part of the wrapper for SDL.
-* `sdl_mixer <sdl_mixer.html>`_
-  Part of the wrapper for SDL.
-* `sdl_net <sdl_net.html>`_
-  Part of the wrapper for SDL.
-* `sdl_ttf <sdl_ttf.html>`_
-  Part of the wrapper for SDL.
-* `smpeg <smpeg.html>`_
-  Part of the wrapper for SDL.
-
-
 GUI libraries
 -------------
 

doc/manual/stmts.txt

@@ -129,6 +129,7 @@ initialized and does not rely on syntactic properties:
       x = a()
     use x
 
+
 let statement
 -------------
 
@@ -141,6 +142,19 @@ nor can their address be taken. They cannot be assigned new values.
 For let variables the same pragmas are available as for ordinary variables.
 
 
+Tuple unpacking
+---------------
+
+In a ``var`` or ``let`` statement tuple unpacking can be performed. The special
+identifier ``_`` can be used to ignore some parts of the tuple:
+
+.. code-block:: nim
+    proc returnsTuple(): (int, int, int) = (4, 2, 3)
+
+    let (x, _, z) = returnsTuple()
+
+
+
 Const section
 -------------
 

doc/manual/syntax.txt

@@ -15,8 +15,6 @@ Associativity
 Binary operators whose first character is ``^`` are right-associative, all
 other binary operators are left-associative.
 
-Operators ending in ``>`` but longer than a single character are 
-called `arrow like`:idx:.
 
 
 Precedence
@@ -33,9 +31,12 @@ as ``(@x).abc`` whereas ``$x.abc`` is parsed as ``$(x.abc)``.
 For binary operators that are not keywords the precedence is determined by the
 following rules:
 
+Operators ending in either ``->``, ``~>`` or ``=>`` are called
+`arrow like`:idx:, and have the lowest precedence of all operators.
+
 If the operator ends with ``=`` and its first character is none of 
 ``<``, ``>``, ``!``, ``=``, ``~``, ``?``, it is an *assignment operator* which
-has the lowest precedence.
+has the second lowest precedence.
 
 Otherwise precedence is determined by the first character.
 
@@ -44,7 +45,7 @@ Precedence level    Operators                                      First charact
 ================  ===============================================  ==================  ===============
  10 (highest)                                                      ``$  ^``            OP10
   9               ``*    /    div   mod   shl  shr  %``            ``*  %  \  /``      OP9
-  8               ``+    -``                                       ``+  ~  |``         OP8
+  8               ``+    -``                                       ``+  -  ~  |``      OP8
   7               ``&``                                            ``&``               OP7
   6               ``..``                                           ``.``               OP6
   5               ``==  <= < >= > !=  in notin is isnot not of``   ``=  <  >  !``      OP5

doc/manual/type_bound_ops.txt

@@ -127,8 +127,8 @@ The signature has to be:
 .. code-block:: nim
   proc `=deepCopy`(x: T): T
 
-This mechanism is used by most data structures that support shared memory like
+This mechanism will be used by most data structures that support shared memory
-channels to implement thread safe automatic memory management.
+like channels to implement thread safe automatic memory management.
 
 The builtin ``deepCopy`` can even clone closures and their environments. See
 the documentation of `spawn`_ for details.

doc/nimc.txt

@@ -386,6 +386,25 @@ Example:
 As can be seen from the example, to Nim symbols can be referred via backticks.
 Use two backticks to produce a single verbatim backtick.
 
+For a toplevel emit statement the section where in the generated C/C++ file
+the code should be emitted can be influenced via the
+prefixes ``/*TYPESECTION*/`` or ``/*VARSECTION*/``:
+
+.. code-block:: Nim
+  {.emit: """/*TYPESECTION*/
+  struct Vector3 {
+  public:
+    Vector3(): x(5) {}
+    Vector3(float x_): x(x_) {}
+    float x;
+  };
+  """.}
+
+  type Vector3 {.importcpp: "Vector3", nodecl} = object
+    x: cfloat
+
+  proc constructVector3(a: cfloat): Vector3 {.importcpp: "Vector3(@)", nodecl}
+
 
 ImportCpp pragma
 ----------------

doc/nimsuggest.txt

@@ -0,0 +1,174 @@
+================================
+  Nim IDE Integration Guide
+================================
+
+:Author: Unknown
+:Version: |nimversion|
+
+.. contents::
+
+
+Nim differs from many other compilers in that it is really fast,
+and being so fast makes it suited to provide external queries for
+text editors about the source code being written. Through the
+``nimsuggest`` tool, any IDE
+can query a ``.nim`` source file and obtain useful information like
+definition of symbols or suggestions for completion.
+
+This document will guide you through the available options. If you
+want to look at practical examples of nimsuggest support you can look
+at the
+`various editor integrations <https://github.com/Araq/Nim/wiki/Editor-Support>`_
+already available.
+
+
+Installation
+============
+
+Nimsuggest is available as a Nimble package but currently does not install
+properly via Nimble. As nimsuggest is part of the compiler it also doesn't make
+too much sense as a Nimble package. Instead we will do the building manually::
+
+  cd compiler/nimsuggest
+  nim c -d:release nimsuggest
+  cp nimsuggest ../../bin
+  # OR: copy the nimsuggest binary to where your 'nim' binary is
+  cd ../..
+
+
+
+Nimsuggest invocation
+=====================
+
+Run it via ``nimsuggest --stdin myproject.nim``. Nimsuggest is a server that
+takes queries that are related to ``myproject``. There is some support so that
+you can throw random ``.nim`` files which are not part of ``myproject`` at
+Nimsuggest too, but usually the query refer to modules/files that are part of
+``myproject``.
+
+``--stdin`` means that Nimsuggest reads the query from ``stdin``. This is great
+for testing things out and playing with it but for an editor communication
+via sockets is more reasonable so that is the default. It listens to port 6000
+by default.
+
+
+Specifying the location of the query
+------------------------------------
+
+Nimsuggest than waits for queries to process. A query consists of a
+cryptic 3 letter "command" ``def`` or ``con`` or ``sug`` or ``use`` followed by
+a location. A query location consists of:
+
+
+``file.nim``
+    This is the name of the module or include file the query refers to.
+
+``dirtyfile.nim``
+    This is optional.
+
+    The ``file`` paramater is enough for static analysis, but IDEs
+    tend to have *unsaved buffers* where the user may still be in
+    the middle of typing a line. In such situations the IDE can
+    save the current contents to a temporary file and then use the
+    ``dirtyfile.nim`` option to tell Nimsuggest that ``foobar.nim`` should
+    be taken from ``temporary/foobar.nim``.
+
+
+``line``
+    An integer with the line you are going to query. For the compiler
+    lines start at **1**.
+
+``col``
+    An integer with the column you are going to query. For the
+    compiler columns start at **1**.
+
+
+Definitions
+-----------
+
+The ``def`` Nimsuggest command performs a query about the definition
+of a specific symbol. If available, Nimsuggest will answer with the
+type, source file, line/column information and other accessory data
+if available like a docstring. With this information an IDE can
+provide the typical *Jump to definition* where a user puts the
+cursor on a symbol or uses the mouse to select it and is redirected
+to the place where the symbol is located.
+
+Since Nim is implemented in Nim, one of the nice things of
+this feature is that any user with an IDE supporting it can quickly
+jump around the standard library implementation and see exactly
+what a proc does, learning about the language and seeing real life
+examples of how to write/implement specific features.
+
+Nimsuggest will always answer with a single definition or none if it
+can't find any valid symbol matching the position of the query.
+
+
+Suggestions
+-----------
+
+The ``sug`` Nimsuggest command performs a query about possible
+completion symbols at some point in the file.
+
+The typical usage scenario for this option is to call it after the
+user has typed the dot character for `the object oriented call
+syntax <tut2.html#method-call-syntax>`_. Nimsuggest will try to return
+the suggestions sorted first by scope (from innermost to outermost)
+and then by item name.
+
+
+Invocation context
+------------------
+
+The ``con`` Nimsuggest command is very similar to the suggestions
+command, but instead of being used after the user has typed a dot
+character, this one is meant to be used after the user has typed
+an opening brace to start typing parameters.
+
+
+Symbol usages
+-------------
+
+The ``use`` Nimsuggest command lists all usages of the symbol at
+a position. IDEs can use this to find all the places in the file
+where the symbol is used and offer the user to rename it in all
+places at the same time.
+
+For this kind of query the IDE will most likely ignore all the
+type/signature info provided by Nimsuggest and concentrate on the
+filename, line and column position of the multiple returned answers.
+
+
+
+Parsing nimsuggest output
+=========================
+
+Nimsuggest output is always returned on single lines separated by
+tab characters (``\t``). The values of each column are:
+
+1. Three characters indicating the type of returned answer (e.g.
+   ``def`` for definition, ``sug`` for suggestion, etc).
+2. Type of the symbol. This can be ``skProc``, ``skLet``, and just
+   about any of the enums defined in the module ``compiler/ast.nim``.
+3. Full qualitifed path of the symbol. If you are querying a symbol
+   defined in the ``proj.nim`` file, this would have the form
+   ``proj.symbolName``.
+4. Type/signature. For variables and enums this will contain the
+   type of the symbol, for procs, methods and templates this will
+   contain the full unique signature (e.g. ``proc (File)``).
+5. Full path to the file containing the symbol.
+6. Line where the symbol is located in the file. Lines start to
+   count at **1**.
+7. Column where the symbol is located in the file. Columns start
+   to count at **1**.
+8. Docstring for the symbol if available or the empty string. To
+   differentiate the docstring from end of answer,
+   the docstring is always provided enclosed in double quotes, and
+   if the docstring spans multiple lines, all following lines of the
+   docstring will start with a blank space to align visually with
+   the starting quote.
+
+   Also, you won't find raw ``\n`` characters breaking the one
+   answer per line format. Instead you will need to parse sequences
+   in the form ``\xHH``, where *HH* is a hexadecimal value (e.g.
+   newlines generate the sequence ``\x0A``).

doc/tools.txt

@@ -4,6 +4,11 @@ Tools available with Nim
 
 The standard distribution ships with the following tools:
 
+- | `Nimsuggest for IDE support <nimsuggest.html>`_
+  | Through the ``nimsuggest`` tool, any IDE can query a ``.nim`` source file
+    and obtain useful information like definition of symbols or suggestions for
+    completion.
+
 - | `Nim Installation Generator <niminst.html>`_
   | How to generate a nice installer for your Nim program.
 

doc/tut1.txt

@@ -1451,7 +1451,7 @@ operators perform implicit dereferencing operations for reference types:
   type
     Node = ref NodeObj
     NodeObj = object
-      le, ri: PNode
+      le, ri: Node
       data: int
   var
     n: Node

koch.nim

@@ -106,6 +106,12 @@ proc zip(args: string) =
   exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" %
        ["tools/niminst/niminst".exe, VersionAsString])
 
+proc targz(args: string) =
+  exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" %
+       [VersionAsString, compileNimInst, findNim()])
+  exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim targz compiler/installer.ini" %
+       ["tools" / "niminst" / "niminst".exe, VersionAsString])
+
 proc buildTool(toolname, args: string) =
   exec("$# cc $# $#" % [findNim(), args, toolname])
   copyFile(dest="bin"/ splitFile(toolname).name.exe, source=toolname.exe)
@@ -113,12 +119,12 @@ proc buildTool(toolname, args: string) =
 proc nsis(args: string) =
   # make sure we have generated the niminst executables:
   buildTool("tools/niminst/niminst", args)
-  buildTool("tools/nimgrep", args)
+  #buildTool("tools/nimgrep", args)
-  # produce 'nimrod_debug.exe':
+  # produce 'nim_debug.exe':
-  exec "nim c compiler" / "nim.nim"
+  #exec "nim c compiler" / "nim.nim"
-  copyExe("compiler/nim".exe, "bin/nim_debug".exe)
+  #copyExe("compiler/nim".exe, "bin/nim_debug".exe)
   exec(("tools" / "niminst" / "niminst --var:version=$# --var:mingw=mingw$#" &
-        " nsis compiler/nim") % [VersionAsString, $(sizeof(pointer)*8)])
+        " nsis compiler/installer.ini") % [VersionAsString, $(sizeof(pointer)*8)])
 
 proc install(args: string) =
   exec("$# cc -r $# --var:version=$# --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" %
@@ -143,8 +149,6 @@ proc findStartNim: string =
   # we try several things before giving up:
   # * bin/nim
   # * $PATH/nim
-  # * bin/nimrod
-  # * $PATH/nimrod
   # If these fail, we try to build nim with the "build.(sh|bat)" script.
   var nim = "nim".exe
   result = "bin" / nim
@@ -152,13 +156,6 @@ proc findStartNim: string =
   for dir in split(getEnv("PATH"), PathSep):
     if existsFile(dir / nim): return dir / nim
 
-  # try the old "nimrod.exe":
-  var nimrod = "nimrod".exe
-  result = "bin" / nimrod
-  if existsFile(result): return
-  for dir in split(getEnv("PATH"), PathSep):
-    if existsFile(dir / nim): return dir / nimrod
-
   when defined(Posix):
     const buildScript = "build.sh"
     if existsFile(buildScript):
@@ -317,12 +314,14 @@ proc winRelease() =
   #buildTool("tools/niminst/niminst", " -d:release")
   buildTool("tools/nimgrep", " -d:release")
   buildTool("compiler/nimfix/nimfix", " -d:release")
+  buildTool("compiler/nimsuggest/nimsuggest", " -d:release")
+
+  #run7z("win32", "bin/nim.exe", "bin/c2nim.exe", "bin/nimgrep.exe",
+  #      "bin/nimfix.exe",
+  #      "bin/nimble.exe", "bin/*.dll",
+  #      "config", "dist/*.dll", "examples", "lib",
+  #      "readme.txt", "contributors.txt", "copying.txt")
 
-  run7z("win32", "bin/nim.exe", "bin/c2nim.exe", "bin/nimgrep.exe",
-        "bin/nimfix.exe",
-        "bin/nimble.exe", "bin/*.dll",
-        "config", "dist/*.dll", "examples", "lib",
-        "readme.txt", "contributors.txt", "copying.txt")
   # second step: XXX build 64 bit version
 
 # -------------- tests --------------------------------------------------------
@@ -366,6 +365,7 @@ of cmdArgument:
   of "pdf": pdf()
   of "csource", "csources": csource(op.cmdLineRest)
   of "zip": zip(op.cmdLineRest)
+  of "targz": targz(op.cmdLineRest)
   of "nsis": nsis(op.cmdLineRest)
   of "install": install(op.cmdLineRest)
   of "test", "tests": tests(op.cmdLineRest)

lib/core/macros.nim

@@ -88,7 +88,9 @@ type
     ntyBigNum,
     ntyConst, ntyMutable, ntyVarargs,
     ntyIter,
-    ntyError
+    ntyError,
+    ntyBuiltinTypeClass, ntyConcept, ntyConceptInst, ntyComposite,
+    ntyAnd, ntyOr, ntyNot
 
   TNimTypeKinds* {.deprecated.} = set[NimTypeKind]
   NimSymKind* = enum
@@ -162,6 +164,7 @@ proc kind*(n: NimNode): NimNodeKind {.magic: "NKind", noSideEffect.}
   ## returns the `kind` of the node `n`.
 
 proc intVal*(n: NimNode): BiggestInt {.magic: "NIntVal", noSideEffect.}
+proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} = n.intVal != 0
 proc floatVal*(n: NimNode): BiggestFloat {.magic: "NFloatVal", noSideEffect.}
 proc symbol*(n: NimNode): NimSym {.magic: "NSymbol", noSideEffect.}
 proc ident*(n: NimNode): NimIdent {.magic: "NIdent", noSideEffect.}
@@ -355,6 +358,12 @@ proc expectLen*(n: NimNode, len: int) {.compileTime.} =
   ## macros that check its number of arguments.
   if n.len != len: error("macro expects a node with " & $len & " children")
 
+proc newTree*(kind: NimNodeKind,
+              children: varargs[NimNode]): NimNode {.compileTime.} =
+  ## produces a new node with children.
+  result = newNimNode(kind)
+  result.add(children)
+
 proc newCall*(theProc: NimNode,
               args: varargs[NimNode]): NimNode {.compileTime.} =
   ## produces a new call node. `theProc` is the proc that is called with
@@ -389,6 +398,11 @@ proc newLit*(i: BiggestInt): NimNode {.compileTime.} =
   result = newNimNode(nnkIntLit)
   result.intVal = i
 
+proc newLit*(b: bool): NimNode {.compileTime.} =
+  ## produces a new boolean literal node.
+  result = newNimNode(nnkIntLit)
+  result.intVal = ord(b)
+
 proc newLit*(f: BiggestFloat): NimNode {.compileTime.} =
   ## produces a new float literal node.
   result = newNimNode(nnkFloatLit)

lib/impure/db_sqlite.nim

@@ -205,7 +205,7 @@ proc setEncoding*(connection: TDbConn, encoding: string): bool {.
   exec(connection, sql"PRAGMA encoding = ?", [encoding])
   result = connection.getValue(sql"PRAGMA encoding") == encoding
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   var db = open("db.sql", "", "", "")
   exec(db, sql"create table tbl1(one varchar(10), two smallint)", [])
   exec(db, sql"insert into tbl1 values('hello!',10)", [])

lib/impure/graphics.nim

@@ -499,7 +499,7 @@ template withEvents*(surf: PSurface, event: expr, actions: stmt): stmt {.
 if sdl.init(sdl.INIT_VIDEO) < 0: raiseEGraphics()
 if sdl_ttf.init() < 0: raiseEGraphics()
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   var surf = newScreenSurface(800, 600)
 
   surf.fillSurface(colWhite)

lib/impure/rdstdin.nim

@@ -135,7 +135,7 @@ else:
     var cur, old: Termios
     discard fd.tcgetattr(cur.addr)
     old = cur
-    cur.lflag = cur.lflag and not Tcflag(ECHO)
+    cur.c_lflag = cur.c_lflag and not Tcflag(ECHO)
     discard fd.tcsetattr(TCSADRAIN, cur.addr)
     stdout.write prompt
     result = stdin.readLine(password)

lib/impure/re.nim

@@ -7,8 +7,11 @@
 #    distribution, for details about the copyright.
 #
 
-## Regular expression support for Nim. Consider using the pegs module
+## Regular expression support for Nim. Consider using the pegs module instead.
-## instead.
+##
+## There is an alternative regular expressions library with a more unified API:
+## `nre <https://github.com/flaviut/nre>`_. It may be added to the standard
+## library in the future, instead of `re`.
 ##
 ## **Note:** The 're' proc defaults to the **extended regular expression
 ## syntax** which lets you use whitespace freely to make your regexes readable.
@@ -43,8 +46,8 @@ type
                          ## expression will be used only once)
   
   RegexDesc = object 
-    h: PPcre
+    h: ptr Pcre
-    e: ptr TExtra
+    e: ptr ExtraData
   
   Regex* = ref RegexDesc ## a compiled regular expression
 
@@ -60,7 +63,7 @@ proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} =
   e.msg = msg
   raise e
 
-proc rawCompile(pattern: string, flags: cint): PPcre =
+proc rawCompile(pattern: string, flags: cint): ptr Pcre =
   var
     msg: cstring
     offset: cint
@@ -84,7 +87,7 @@ proc re*(s: string, flags = {reExtended, reStudy}): Regex =
   result.h = rawCompile(s, cast[cint](flags - {reStudy}))
   if reStudy in flags:
     var msg: cstring
-    result.e = pcre.study(result.h, 0, msg)
+    result.e = pcre.study(result.h, 0, addr msg)
     if not isNil(msg): raiseInvalidRegex($msg)
 
 proc matchOrFind(s: string, pattern: Regex, matches: var openArray[string],
@@ -143,8 +146,8 @@ proc findBounds*(s: string, pattern: Regex,
 
 proc findBounds*(s: string, pattern: Regex,
                  start = 0): tuple[first, last: int] =
-  ## returns the starting position of `pattern` in `s`. If it does not
+  ## returns the starting position and end position of ``pattern`` in ``s``.
-  ## match, ``(-1,0)`` is returned.
+  ## If it does not match, ``(-1,0)`` is returned.
   var
     rtarray = initRtArray[cint](3)
     rawMatches = rtarray.getRawData
@@ -413,22 +416,28 @@ proc escapeRe*(s: string): string =
       result.add(toHex(ord(c), 2))
 
 const ## common regular expressions
-  reIdentifier* = r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b"  ## describes an identifier
+  reIdentifier* {.deprecated.} = r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b"
-  reNatural* = r"\b\d+\b" ## describes a natural number
+    ## describes an identifier
-  reInteger* = r"\b[-+]?\d+\b" ## describes an integer
+  reNatural* {.deprecated.} = r"\b\d+\b"
-  reHex* = r"\b0[xX][0-9a-fA-F]+\b" ## describes a hexadecimal number
+    ## describes a natural number
-  reBinary* = r"\b0[bB][01]+\b" ## describes a binary number (example: 0b11101)
+  reInteger* {.deprecated.} = r"\b[-+]?\d+\b"
-  reOctal* = r"\b0[oO][0-7]+\b" ## describes an octal number (example: 0o777)
+    ## describes an integer
-  reFloat* = r"\b[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b"
+  reHex* {.deprecated.} = r"\b0[xX][0-9a-fA-F]+\b"
+    ## describes a hexadecimal number
+  reBinary* {.deprecated.} = r"\b0[bB][01]+\b"
+    ## describes a binary number (example: 0b11101)
+  reOctal* {.deprecated.} = r"\b0[oO][0-7]+\b"
+    ## describes an octal number (example: 0o777)
+  reFloat* {.deprecated.} = r"\b[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b"
     ## describes a floating point number
-  reEmail* = r"\b[a-zA-Z0-9!#$%&'*+/=?^_`{|}~\-]+(?:\. &" &
+  reEmail* {.deprecated.} = r"\b[a-zA-Z0-9!#$%&'*+/=?^_`{|}~\-]+(?:\. &" &
-             r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)" &
+                            r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@" &
-             r"*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+" &
+                            r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+" &
-             r"(?:[a-zA-Z]{2}|com|org|" &
+                            r"(?:[a-zA-Z]{2}|com|org|net|gov|mil|biz|" &
-             r"net|gov|mil|biz|info|mobi|name|aero|jobs|museum)\b"
+                            r"info|mobi|name|aero|jobs|museum)\b"
     ## describes a common email address
-  reURL* = r"\b(http(s)?|ftp|gopher|telnet|file|notes|ms\-help):" &
+  reURL* {.deprecated.} = r"\b(http(s)?|ftp|gopher|telnet|file|notes|ms-help)" &
-           r"((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-\=\\\.\&]*\b"
+                          r":((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-\=\\\.\&]*\b"
     ## describes an URL
 
 when isMainModule:

lib/impure/ssl.nim

@@ -82,7 +82,7 @@ proc close*(sock: TSecureSocket) =
     ERR_print_errors_fp(stderr)
     raiseOSError(osLastError())
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   var s: TSecureSocket
   echo connect(s, "smtp.gmail.com", 465)
   

lib/impure/zipfiles.nim

@@ -10,7 +10,7 @@
 ## This module implements a zip archive creator/reader/modifier.
 
 import
-  streams, libzip, times, os
+  streams, libzip, times, os, strutils
 
 type
   TZipArchive* = object of RootObj ## represents a zip archive
@@ -111,19 +111,25 @@ proc addFile*(z: var TZipArchive, dest: string, src: Stream) =
 type
   TZipFileStream = object of StreamObj
     f: PZipFile
+    atEnd: bool
 
   PZipFileStream* =
     ref TZipFileStream ## a reader stream of a file within a zip archive
 
 proc fsClose(s: Stream) = zip_fclose(PZipFileStream(s).f)
+proc fsAtEnd(s: Stream): bool = PZipFileStream(s).atEnd
 proc fsReadData(s: Stream, buffer: pointer, bufLen: int): int =
   result = zip_fread(PZipFileStream(s).f, buffer, bufLen)
+  if result == 0:
+    PZipFileStream(s).atEnd = true
 
 proc newZipFileStream(f: PZipFile): PZipFileStream =
   new(result)
   result.f = f
+  result.atEnd = false
   result.closeImpl = fsClose
   result.readDataImpl = fsReadData
+  result.atEndImpl = fsAtEnd
   # other methods are nil!
 
 # ----------------------------------------------------------------------------
@@ -158,12 +164,20 @@ proc extractFile*(z: var TZipArchive, srcFile: string, dest: Stream) =
 
 proc extractFile*(z: var TZipArchive, srcFile: string, dest: string) =
   ## extracts a file from the zip archive `z` to the destination filename.
-  var file = newFileStream(dest, fmReadWrite)
+  var file = newFileStream(dest, fmWrite)
   extractFile(z, srcFile, file)
   file.close()
 
 proc extractAll*(z: var TZipArchive, dest: string) =
   ## extracts all files from archive `z` to the destination directory.
   for file in walkFiles(z):
-    extractFile(z, file, dest / extractFilename(file))
+    if file.endsWith("/"):
+      createDir(dest / file)
+    else:
+      extractFile(z, file, dest / file)
 
+when not defined(testing) and isMainModule:
+  var zip: TZipArchive
+  if not zip.open("nim-0.11.0.zip"):
+    raise newException(IOError, "opening zip failed")
+  zip.extractAll("test")

lib/js/dom.nim

@@ -152,10 +152,12 @@ type
   DocumentObj {.importc.} = object of NodeObj
     alinkColor*: cstring
     bgColor*: cstring
+    body*: Element
     charset*: cstring
     cookie*: cstring
     defaultCharset*: cstring
     fgColor*: cstring
+    head*: Element
     lastModified*: cstring
     linkColor*: cstring
     referrer*: cstring

lib/nimbase.h

@@ -343,15 +343,15 @@ struct TFrame {
 };
 
 #define nimfr(proc, file) \
-  TFrame F; \
+  TFrame FR; \
-  F.procname = proc; F.filename = file; F.line = 0; F.len = 0; nimFrame(&F);
+  FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = 0; nimFrame(&FR);
 
 #define nimfrs(proc, file, slots, length) \
-  struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; TVarSlot s[slots];} F; \
+  struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; TVarSlot s[slots];} FR; \
-  F.procname = proc; F.filename = file; F.line = 0; F.len = length; nimFrame((TFrame*)&F);
+  FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = length; nimFrame((TFrame*)&FR);
 
 #define nimln(n, file) \
-  F.line = n; F.filename = file;
+  FR.line = n; FR.filename = file;
 
 #define NIM_POSIX_INIT  __attribute__((constructor))
 

lib/packages/docutils/rst.nim

@@ -564,7 +564,7 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) =
 
 proc parsePostfix(p: var TRstParser, n: PRstNode): PRstNode =
   result = n
-  if isInlineMarkupEnd(p, "_"):
+  if isInlineMarkupEnd(p, "_") or isInlineMarkupEnd(p, "__"):
     inc(p.idx)
     if p.tok[p.idx-2].symbol == "`" and p.tok[p.idx-3].symbol == ">":
       var a = newRstNode(rnInner)

lib/packages/docutils/rstgen.nim

@@ -1251,5 +1251,6 @@ proc rstToHtml*(s: string, options: TRstParseOptions,
 
 
 when isMainModule:
-  echo rstToHtml("*Hello* **world**!", {},
+  assert rstToHtml("*Hello* **world**!", {},
-    newStringTable(modeStyleInsensitive))
+    newStringTable(modeStyleInsensitive)) ==
+    "<em>Hello</em> <strong>world</strong>!"

lib/posix/termios.nim

@@ -18,16 +18,13 @@ const
   NCCS* = 32
 
 type
-  Termios* = object {.importc: "struct termios", header: "<termios.h>", final, pure.}
+  Termios* {.importc: "struct termios", header: "<termios.h>".} = object
-    iflag*: Tcflag        # input mode flags 
+    c_iflag*: Tcflag        # input mode flags
-    oflag*: Tcflag        # output mode flags 
+    c_oflag*: Tcflag        # output mode flags
-    cflag*: Tcflag        # control mode flags 
+    c_cflag*: Tcflag        # control mode flags
-    lflag*: Tcflag        # local mode flags 
+    c_lflag*: Tcflag        # local mode flags
-    line*: cuchar             # line discipline 
+    c_line*: cuchar         # line discipline
-    cc*: array[NCCS, cuchar]  # control characters 
+    c_cc*: array[NCCS, cuchar]  # control characters
-    ispeed*: Speed        # input speed 
-    ospeed*: Speed        # output speed 
-  
 
 # cc characters
 

lib/pure/actors.nim

@@ -221,7 +221,7 @@ proc spawn*[TIn](p: var TActorPool[TIn, void], input: TIn,
   setupTask()
   schedule()
   
-when isMainModule:
+when not defined(testing) and isMainModule:
   var
     a: TActorPool[int, void]
   createActorPool(a)

lib/pure/algorithm.nim

@@ -40,8 +40,8 @@ proc reverse*[T](a: var openArray[T]) =
 proc reversed*[T](a: openArray[T], first, last: Natural): seq[T] =
   ## returns the reverse of the array `a[first..last]`.
   result = newSeq[T](last - first + 1)
-  var x = first
+  var x = first.int
-  var y = last
+  var y = last.int
   while x <= last:
     result[x] = a[y]
     dec(y)

lib/pure/asyncdispatch.nim

@@ -634,6 +634,93 @@ when defined(windows) or defined(nimdoc):
       # free ``ol``.
     return retFuture
 
+  proc recvInto*(socket: TAsyncFD, buf: cstring, size: int,
+                flags = {SocketFlag.SafeDisconn}): Future[int] =
+    ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must
+    ## at least be of that size. Returned future will complete once all the
+    ## data requested is read, a part of the data has been read, or the socket
+    ## has disconnected in which case the future will complete with a value of
+    ## ``0``.
+    ##
+    ## **Warning**: The ``Peek`` socket flag is not supported on Windows.
+
+
+    # Things to note:
+    #   * When WSARecv completes immediately then ``bytesReceived`` is very
+    #     unreliable.
+    #   * Still need to implement message-oriented socket disconnection,
+    #     '\0' in the message currently signifies a socket disconnect. Who
+    #     knows what will happen when someone sends that to our socket.
+    verifyPresence(socket)
+    assert SocketFlag.Peek notin flags, "Peek not supported on Windows."
+
+    var retFuture = newFuture[int]("recvInto")
+
+    #buf[] = '\0'
+    var dataBuf: TWSABuf
+    dataBuf.buf = buf
+    dataBuf.len = size
+
+    var bytesReceived: Dword
+    var flagsio = flags.toOSFlags().Dword
+    var ol = PCustomOverlapped()
+    GC_ref(ol)
+    ol.data = TCompletionData(fd: socket, cb:
+      proc (fd: TAsyncFD, bytesCount: Dword, errcode: OSErrorCode) =
+        if not retFuture.finished:
+          if errcode == OSErrorCode(-1):
+            if bytesCount == 0 and dataBuf.buf[0] == '\0':
+              retFuture.complete(0)
+            else:
+              retFuture.complete(bytesCount)
+          else:
+            if flags.isDisconnectionError(errcode):
+              retFuture.complete(0)
+            else:
+              retFuture.fail(newException(OSError, osErrorMsg(errcode)))
+        if dataBuf.buf != nil:
+          dataBuf.buf = nil
+    )
+
+    let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived,
+                      addr flagsio, cast[POVERLAPPED](ol), nil)
+    if ret == -1:
+      let err = osLastError()
+      if err.int32 != ERROR_IO_PENDING:
+        if dataBuf.buf != nil:
+          dataBuf.buf = nil
+        GC_unref(ol)
+        if flags.isDisconnectionError(err):
+          retFuture.complete(0)
+        else:
+          retFuture.fail(newException(OSError, osErrorMsg(err)))
+    elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0':
+      # We have to ensure that the buffer is empty because WSARecv will tell
+      # us immediately when it was disconnected, even when there is still
+      # data in the buffer.
+      # We want to give the user as much data as we can. So we only return
+      # the empty string (which signals a disconnection) when there is
+      # nothing left to read.
+      retFuture.complete(0)
+      # TODO: "For message-oriented sockets, where a zero byte message is often
+      # allowable, a failure with an error code of WSAEDISCON is used to
+      # indicate graceful closure."
+      # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx
+    else:
+      # Request to read completed immediately.
+      # From my tests bytesReceived isn't reliable.
+      let realSize =
+        if bytesReceived == 0:
+          size
+        else:
+          bytesReceived
+      assert realSize <= size
+      retFuture.complete(realSize)
+      # We don't deallocate ``ol`` here because even though this completed
+      # immediately poll will still be notified about its completion and it will
+      # free ``ol``.
+    return retFuture
+
   proc send*(socket: TAsyncFD, data: string,
              flags = {SocketFlag.SafeDisconn}): Future[void] =
     ## Sends ``data`` to ``socket``. The returned future will complete once all
@@ -983,6 +1070,30 @@ else:
     addRead(socket, cb)
     return retFuture
 
+  proc recvInto*(socket: TAsyncFD, buf: cstring, size: int,
+                  flags = {SocketFlag.SafeDisconn}): Future[int] =
+    var retFuture = newFuture[int]("recvInto")
+
+    proc cb(sock: TAsyncFD): bool =
+      result = true
+      let res = recv(sock.SocketHandle, buf, size.cint,
+                     flags.toOSFlags())
+      if res < 0:
+        let lastError = osLastError()
+        if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}:
+          if flags.isDisconnectionError(lastError):
+            retFuture.complete(0)
+          else:
+            retFuture.fail(newException(OSError, osErrorMsg(lastError)))
+        else:
+          result = false # We still want this callback to be called.
+      else:
+        retFuture.complete(res)
+    # TODO: The following causes a massive slowdown.
+    #if not cb(socket):
+    addRead(socket, cb)
+    return retFuture
+
   proc send*(socket: TAsyncFD, data: string,
              flags = {SocketFlag.SafeDisconn}): Future[void] =
     var retFuture = newFuture[void]("send")

lib/pure/asyncftpclient.nim

@@ -300,7 +300,7 @@ proc newAsyncFtpClient*(address: string, port = Port(21),
   result.dsockConnected = false
   result.csock = newAsyncSocket()
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test")
   proc main(ftp: AsyncFtpClient) {.async.} =
     await ftp.connect()

lib/pure/asynchttpserver.nim

@@ -23,8 +23,7 @@
 ##    proc cb(req: Request) {.async.} =
 ##      await req.respond(Http200, "Hello World")
 ##
-##    asyncCheck server.serve(Port(8080), cb)
+##    waitFor server.serve(Port(8080), cb)
-##    runForever()
 
 import strtabs, asyncnet, asyncdispatch, parseutils, uri, strutils
 type
@@ -109,22 +108,19 @@ proc sendHeaders*(req: Request, headers: StringTableRef): Future[void] =
   addHeaders(msg, headers)
   return req.client.send(msg)
 
-proc respond*(req: Request, code: HttpCode,
+proc respond*(req: Request, code: HttpCode, content: string,
-        content: string, headers = newStringTable()) {.async.} =
+              headers: StringTableRef = nil): Future[void] =
   ## Responds to the request with the specified ``HttpCode``, headers and
   ## content.
   ##
   ## This procedure will **not** close the client socket.
-  var customHeaders = headers
-  customHeaders["Content-Length"] = $content.len
   var msg = "HTTP/1.1 " & $code & "\c\L"
-  msg.addHeaders(customHeaders)
-  await req.client.send(msg & "\c\L" & content)
 
-proc newRequest(): Request =
+  if headers != nil:
-  result.headers = newStringTable(modeCaseInsensitive)
+    msg.addHeaders(headers)
-  result.hostname = ""
+  msg.add("Content-Length: " & $content.len & "\c\L\c\L")
-  result.body = ""
+  msg.add(content)
+  result = req.client.send(msg)
 
 proc parseHeader(line: string): tuple[key, value: string] =
   var i = 0
@@ -149,52 +145,58 @@ proc sendStatus(client: AsyncSocket, status: string): Future[void] =
 proc processClient(client: AsyncSocket, address: string,
                    callback: proc (request: Request):
                       Future[void] {.closure, gcsafe.}) {.async.} =
+  var request: Request
+  request.url = initUri()
+  request.headers = newStringTable(modeCaseInsensitive)
+  var line = newStringOfCap(80)
+  var key, value = ""
+
   while not client.isClosed:
     # GET /path HTTP/1.1
     # Header: val
     # \n
-    var request = newRequest()
+    request.headers.clear(modeCaseInsensitive)
-    request.hostname = address
+    request.hostname.shallowCopy(address)
     assert client != nil
     request.client = client
 
     # First line - GET /path HTTP/1.1
-    let line = await client.recvLine() # TODO: Timeouts.
+    line.setLen(0)
+    await client.recvLineInto(addr line) # TODO: Timeouts.
     if line == "":
       client.close()
       return
-    let lineParts = line.split(' ')
+
-    if lineParts.len != 3:
+    var i = 0
+    for linePart in line.split(' '):
+      case i
+      of 0: request.reqMethod.shallowCopy(linePart.normalize)
+      of 1: parseUri(linePart, request.url)
+      of 2:
+        try:
+          request.protocol = parseProtocol(linePart)
+        except ValueError:
+          asyncCheck request.respond(Http400,
+            "Invalid request protocol. Got: " & linePart)
+          continue
+      else:
         await request.respond(Http400, "Invalid request. Got: " & line)
         continue
-
+      inc i
-    let reqMethod = lineParts[0]
-    let path = lineParts[1]
-    let protocol = lineParts[2]
 
     # Headers
-    var i = 0
     while true:
       i = 0
-      let headerLine = await client.recvLine()
+      line.setLen(0)
-      if headerLine == "":
+      await client.recvLineInto(addr line)
+
+      if line == "":
         client.close(); return
-      if headerLine == "\c\L": break
+      if line == "\c\L": break
-      # TODO: Compiler crash
+      let (key, value) = parseHeader(line)
-      #let (key, value) = parseHeader(headerLine)
+      request.headers[key] = value
-      let kv = parseHeader(headerLine)
-      request.headers[kv.key] = kv.value
 
-    request.reqMethod = reqMethod
+    if request.reqMethod == "post":
-    request.url = parseUri(path)
-    try:
-      request.protocol = protocol.parseProtocol()
-    except ValueError:
-      asyncCheck request.respond(Http400, "Invalid request protocol. Got: " &
-          protocol)
-      continue
-
-    if reqMethod.normalize == "post":
       # Check for Expect header
       if request.headers.hasKey("Expect"):
         if request.headers["Expect"].toLower == "100-continue":
@@ -215,11 +217,11 @@ proc processClient(client: AsyncSocket, address: string,
         await request.respond(Http400, "Bad Request. No Content-Length.")
         continue
 
-    case reqMethod.normalize
+    case request.reqMethod
     of "get", "post", "head", "put", "delete", "trace", "options", "connect", "patch":
       await callback(request)
     else:
-      await request.respond(Http400, "Invalid request method. Got: " & reqMethod)
+      await request.respond(Http400, "Invalid request method. Got: " & request.reqMethod)
 
     # Persistent connections
     if (request.protocol == HttpVer11 and
@@ -260,7 +262,7 @@ proc close*(server: AsyncHttpServer) =
   ## Terminates the async http server instance.
   server.socket.close()
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   proc main =
     var server = newAsyncHttpServer()
     proc cb(req: Request) {.async.} =

lib/pure/asyncio.nim

@@ -660,7 +660,7 @@ proc len*(disp: Dispatcher): int =
   ## Retrieves the amount of delegates in ``disp``.
   return disp.delegates.len
 
-when isMainModule:
+when not defined(testing) and isMainModule:
 
   proc testConnect(s: AsyncSocket, no: int) =
     echo("Connected! " & $no)

lib/pure/asyncnet.nim

@@ -182,26 +182,30 @@ proc connect*(socket: AsyncSocket, address: string, port: Port,
       sslSetConnectState(socket.sslHandle)
       sslLoop(socket, flags, sslDoHandshake(socket.sslHandle))
 
-proc readInto(buf: cstring, size: int, socket: AsyncSocket,
+template readInto(buf: cstring, size: int, socket: AsyncSocket,
-              flags: set[SocketFlag]): Future[int] {.async.} =
+                  flags: set[SocketFlag]): int =
+  ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. Note that
+  ## this is a template and not a proc.
+  var res = 0
   if socket.isSsl:
     when defined(ssl):
       # SSL mode.
       sslLoop(socket, flags,
         sslRead(socket.sslHandle, buf, size.cint))
-      result = opResult
+      res = opResult
   else:
-    var data = await recv(socket.fd.TAsyncFD, size, flags)
+    var recvIntoFut = recvInto(socket.fd.TAsyncFD, buf, size, flags)
-    if data.len != 0:
+    yield recvIntoFut
-      copyMem(buf, addr data[0], data.len)
     # Not in SSL mode.
-    result = data.len
+    res = recvIntoFut.read()
+  res
 
-proc readIntoBuf(socket: AsyncSocket,
+template readIntoBuf(socket: AsyncSocket,
-    flags: set[SocketFlag]): Future[int] {.async.} =
+    flags: set[SocketFlag]): int =
-  result = await readInto(addr socket.buffer[0], BufferSize, socket, flags)
+  var size = readInto(addr socket.buffer[0], BufferSize, socket, flags)
   socket.currPos = 0
-  socket.bufLen = result
+  socket.bufLen = size
+  size
 
 proc recv*(socket: AsyncSocket, size: int,
            flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} =
@@ -222,10 +226,11 @@ proc recv*(socket: AsyncSocket, size: int,
   ## to be read then the future will complete with a value of ``""``.
   if socket.isBuffered:
     result = newString(size)
+    shallow(result)
     let originalBufPos = socket.currPos
 
     if socket.bufLen == 0:
-      let res = await socket.readIntoBuf(flags - {SocketFlag.Peek})
+      let res = socket.readIntoBuf(flags - {SocketFlag.Peek})
       if res == 0:
         result.setLen(0)
         return
@@ -236,7 +241,7 @@ proc recv*(socket: AsyncSocket, size: int,
         if SocketFlag.Peek in flags:
           # We don't want to get another buffer if we're peeking.
           break
-        let res = await socket.readIntoBuf(flags - {SocketFlag.Peek})
+        let res = socket.readIntoBuf(flags - {SocketFlag.Peek})
         if res == 0:
           break
 
@@ -251,7 +256,7 @@ proc recv*(socket: AsyncSocket, size: int,
     result.setLen(read)
   else:
     result = newString(size)
-    let read = await readInto(addr result[0], size, socket, flags)
+    let read = readInto(addr result[0], size, socket, flags)
     result.setLen(read)
 
 proc send*(socket: AsyncSocket, data: string,
@@ -302,6 +307,82 @@ proc accept*(socket: AsyncSocket,
         retFut.complete(future.read.client)
   return retFut
 
+proc recvLineInto*(socket: AsyncSocket, resString: ptr string,
+    flags = {SocketFlag.SafeDisconn}) {.async.} =
+  ## Reads a line of data from ``socket`` into ``resString``.
+  ##
+  ## If a full line is read ``\r\L`` is not
+  ## added to ``line``, however if solely ``\r\L`` is read then ``line``
+  ## will be set to it.
+  ##
+  ## If the socket is disconnected, ``line`` will be set to ``""``.
+  ##
+  ## If the socket is disconnected in the middle of a line (before ``\r\L``
+  ## is read) then line will be set to ``""``.
+  ## The partial line **will be lost**.
+  ##
+  ## **Warning**: The ``Peek`` flag is not yet implemented.
+  ##
+  ## **Warning**: ``recvLineInto`` on unbuffered sockets assumes that the
+  ## protocol uses ``\r\L`` to delimit a new line.
+  ##
+  ## **Warning**: ``recvLineInto`` currently uses a raw pointer to a string for
+  ## performance reasons. This will likely change soon to use FutureVars.
+  assert SocketFlag.Peek notin flags ## TODO:
+  result = newFuture[void]("asyncnet.recvLineInto")
+
+  template addNLIfEmpty(): stmt =
+    if resString[].len == 0:
+      resString[].add("\c\L")
+
+  if socket.isBuffered:
+    if socket.bufLen == 0:
+      let res = socket.readIntoBuf(flags)
+      if res == 0:
+        return
+
+    var lastR = false
+    while true:
+      if socket.currPos >= socket.bufLen:
+        let res = socket.readIntoBuf(flags)
+        if res == 0:
+          resString[].setLen(0)
+          return
+
+      case socket.buffer[socket.currPos]
+      of '\r':
+        lastR = true
+        addNLIfEmpty()
+      of '\L':
+        addNLIfEmpty()
+        socket.currPos.inc()
+        return
+      else:
+        if lastR:
+          socket.currPos.inc()
+          return
+        else:
+          resString[].add socket.buffer[socket.currPos]
+      socket.currPos.inc()
+  else:
+    var c = ""
+    while true:
+      let recvFut = recv(socket, 1, flags)
+      c = recvFut.read()
+      if c.len == 0:
+        resString[].setLen(0)
+        return
+      if c == "\r":
+        let recvFut = recv(socket, 1, flags) # Skip \L
+        c = recvFut.read()
+        assert c == "\L"
+        addNLIfEmpty()
+        return
+      elif c == "\L":
+        addNLIfEmpty()
+        return
+      resString[].add c
+
 proc recvLine*(socket: AsyncSocket,
     flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} =
   ## Reads a line of data from ``socket``. Returned future will complete once
@@ -325,52 +406,9 @@ proc recvLine*(socket: AsyncSocket,
     if result.len == 0:
       result.add("\c\L")
   assert SocketFlag.Peek notin flags ## TODO:
-  if socket.isBuffered:
-    result = ""
-    if socket.bufLen == 0:
-      let res = await socket.readIntoBuf(flags)
-      if res == 0:
-        return
 
-    var lastR = false
-    while true:
-      if socket.currPos >= socket.bufLen:
-        let res = await socket.readIntoBuf(flags)
-        if res == 0:
   result = ""
-          break
+  await socket.recvLineInto(addr result, flags)
-
-      case socket.buffer[socket.currPos]
-      of '\r':
-        lastR = true
-        addNLIfEmpty()
-      of '\L':
-        addNLIfEmpty()
-        socket.currPos.inc()
-        return
-      else:
-        if lastR:
-          socket.currPos.inc()
-          return
-        else:
-          result.add socket.buffer[socket.currPos]
-      socket.currPos.inc()
-  else:
-    result = ""
-    var c = ""
-    while true:
-      c = await recv(socket, 1, flags)
-      if c.len == 0:
-        return ""
-      if c == "\r":
-        c = await recv(socket, 1, flags) # Skip \L
-        assert c == "\L"
-        addNLIfEmpty()
-        return
-      elif c == "\L":
-        addNLIfEmpty()
-        return
-      add(result.string, c)
 
 proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} =
   ## Marks ``socket`` as accepting connections.
@@ -458,7 +496,7 @@ proc isClosed*(socket: AsyncSocket): bool =
   ## Determines whether the socket has been closed.
   return socket.closed
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   type
     TestCases = enum
       HighClient, LowClient, LowServer

lib/pure/collections/LockFreeHash.nim

@@ -525,7 +525,7 @@ proc get*[K,V](table: var PConcTable[K,V], key: var K): V =
 
 
 #Tests ----------------------------
-when isMainModule:
+when not defined(testing) and isMainModule:
   import locks, times, mersenne
   
   const 

lib/pure/collections/critbits.nim

@@ -286,18 +286,19 @@ proc `$`*[T](c: CritBitTree[T]): string =
     result.add("}")
 
 when isMainModule:
+  import sequtils
+
   var r: CritBitTree[void]
   r.incl "abc"
   r.incl "xyz"
   r.incl "def"
   r.incl "definition"
   r.incl "prefix"
+
   doAssert r.contains"def"
-  #r.del "def"
 
-  for w in r.items:
+  r.excl "def"
-    echo w
 
-  for w in r.itemsWithPrefix("de"):
+  assert toSeq(r.items) == @["abc", "definition", "prefix", "xyz"]
-    echo w
 
+  assert toSeq(r.itemsWithPrefix("de")) == @["definition"]

lib/pure/collections/intsets.nim

@@ -198,14 +198,21 @@ proc empty*(s: IntSet): bool {.inline, deprecated.} =
   result = s.counter == 0
 
 when isMainModule:
+  import sequtils, algorithm
+
   var x = initIntSet()
   x.incl(1)
   x.incl(2)
   x.incl(7)
   x.incl(1056)
-  for e in items(x): echo e
 
-  var y: TIntSet
+  var xs = toSeq(items(x))
+  xs.sort(cmp[int])
+  assert xs == @[1, 2, 7, 1056]
+
+  var y: IntSet
   assign(y, x)
-  for e in items(y): echo e
+  var ys = toSeq(items(y))
+  ys.sort(cmp[int])
+  assert ys == @[1, 2, 7, 1056]
 

lib/pure/collections/sequtils.nim

@@ -492,9 +492,8 @@ when isMainModule:
 
   block: # filter iterator test
     let numbers = @[1, 4, 5, 8, 9, 7, 4]
-    for n in filter(numbers, proc (x: int): bool = x mod 2 == 0):
+    assert toSeq(filter(numbers, proc (x: int): bool = x mod 2 == 0)) ==
-      echo($n)
+      @[4, 8, 4]
-    # echoes 4, 8, 4 in separate lines
 
   block: # keepIf test
     var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1]
@@ -616,4 +615,5 @@ when isMainModule:
     #doAssert a.repeat(-1) == @[] # will not compile!
     doAssert b.repeat(3) == @[]
 
+  when not defined(testing):
     echo "Finished doc tests"

lib/pure/collections/sets.nim

@@ -970,6 +970,7 @@ when isMainModule and not defined(release):
       if s <= i or mustRehash(s, i):
         echo "performance issue: rightSize() will not elide enlarge() at ", i
 
+    when not defined(testing):
       echo "Micro tests run successfully."
 
   testModule()

lib/pure/collections/tables.nim

@@ -819,15 +819,18 @@ proc enlarge[A](t: var CountTable[A]) =
   swap(t.data, n)
 
 proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
-  ## puts a (key, value)-pair into `t`. `val` has to be positive.
+  ## puts a (key, value)-pair into `t`.
   assert val > 0
   var h = rawGet(t, key)
   if h >= 0:
     t.data[h].val = val
   else:
-    h = -1 - h
+    if mustRehash(len(t.data), t.counter): enlarge(t)
-    t.data[h].key = key
+    rawInsert(t, t.data, key, val)
-    t.data[h].val = val
+    inc(t.counter)
+    #h = -1 - h
+    #t.data[h].key = key
+    #t.data[h].val = val
 
 proc initCountTable*[A](initialSize=64): CountTable[A] =
   ## creates a new count table that is empty.
@@ -984,6 +987,22 @@ proc sort*[A](t: CountTableRef[A]) =
   ## `t` in the sorted order.
   t[].sort
 
+proc merge*[A](s: var CountTable[A], t: CountTable[A]) =
+  ## merges the second table into the first one
+  for key, value in t:
+    s.inc(key, value)
+
+proc merge*[A](s, t: CountTable[A]): CountTable[A] =
+  ## merges the two tables into a new one
+  result = initCountTable[A](nextPowerOfTwo(max(s.len, t.len)))
+  for table in @[s, t]:
+    for key, value in table:
+      result.inc(key, value)
+
+proc merge*[A](s, t: CountTableRef[A]) =
+  ## merges the second table into the first one
+  s[].merge(t[])
+
 when isMainModule:
   type
     Person = object
@@ -1012,3 +1031,48 @@ when isMainModule:
   s2[p2] = 45_000
   s3[p1] = 30_000
   s3[p2] = 45_000
+
+  var
+    t1 = initCountTable[string]()
+    t2 = initCountTable[string]()
+  t1.inc("foo")
+  t1.inc("bar", 2)
+  t1.inc("baz", 3)
+  t2.inc("foo", 4)
+  t2.inc("bar")
+  t2.inc("baz", 11)
+  merge(t1, t2)
+  assert(t1["foo"] == 5)
+  assert(t1["bar"] == 3)
+  assert(t1["baz"] == 14)
+
+  let
+    t1r = newCountTable[string]()
+    t2r = newCountTable[string]()
+  t1r.inc("foo")
+  t1r.inc("bar", 2)
+  t1r.inc("baz", 3)
+  t2r.inc("foo", 4)
+  t2r.inc("bar")
+  t2r.inc("baz", 11)
+  merge(t1r, t2r)
+  assert(t1r["foo"] == 5)
+  assert(t1r["bar"] == 3)
+  assert(t1r["baz"] == 14)
+
+  var
+    t1l = initCountTable[string]()
+    t2l = initCountTable[string]()
+  t1l.inc("foo")
+  t1l.inc("bar", 2)
+  t1l.inc("baz", 3)
+  t2l.inc("foo", 4)
+  t2l.inc("bar")
+  t2l.inc("baz", 11)
+  let
+    t1merging = t1l
+    t2merging = t2l
+  let merged = merge(t1merging, t2merging)
+  assert(merged["foo"] == 5)
+  assert(merged["bar"] == 3)
+  assert(merged["baz"] == 14)

lib/pure/concurrency/cpuload.nim

@@ -78,7 +78,7 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice =
     result = doNothing
   inc s.calls
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   proc busyLoop() =
     while true:
       discard random(80)

lib/pure/concurrency/threadpool.nim

@@ -300,7 +300,7 @@ proc setMinPoolSize*(size: range[1..MaxThreadPoolSize]) =
   minPoolSize = size
 
 proc setMaxPoolSize*(size: range[1..MaxThreadPoolSize]) =
-  ## sets the minimal thread pool size. The default value of this
+  ## sets the maximal thread pool size. The default value of this
   ## is ``MaxThreadPoolSize``.
   maxPoolSize = size
   if currentPoolSize > maxPoolSize:

lib/pure/cookies.nim

@@ -56,6 +56,12 @@ proc setCookie*(key, value: string, expires: TimeInfo,
 when isMainModule:
   var tim = Time(int(getTime()) + 76 * (60 * 60 * 24))
 
-  echo(setCookie("test", "value", tim.getGMTime()))
+  let cookie = setCookie("test", "value", tim.getGMTime())
+  when not defined(testing):
+    echo cookie
+  let start = "Set-Cookie: test=value; Expires="
+  assert cookie[0..start.high] == start
   
-  echo parseCookies("uid=1; kp=2")
+  let table = parseCookies("uid=1; kp=2")
+  assert table["uid"] == "1"
+  assert table["kp"] == "2"

lib/pure/encodings.nim

@@ -451,7 +451,7 @@ proc convert*(s: string, destEncoding = "UTF-8",
   finally:
     close(c)
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   let
     orig = "öäüß"
     cp1252 = convert(orig, "CP1252", "UTF-8")

lib/pure/fsmonitor.nim

@@ -198,7 +198,7 @@ proc register*(d: Dispatcher, monitor: FSMonitor,
   var deleg = toDelegate(monitor)
   d.register(deleg)
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   proc main =
     var disp = newDispatcher()
     var monitor = newMonitor()

lib/pure/ftpclient.nim

@@ -593,7 +593,7 @@ proc register*(d: Dispatcher, ftp: AsyncFTPClient): Delegate {.discardable.} =
   ftp.disp = d
   return ftp.disp.register(ftp.csock)
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   proc main =
     var d = newDispatcher()
     let hev =
@@ -629,7 +629,7 @@ when isMainModule:
       if not d.poll(): break
   main()
 
-when isMainModule and false:
+when not defined(testing) and isMainModule:
   var ftp = ftpClient("example.com", user = "foo", pass = "bar")
   ftp.connect()
   echo ftp.pwd()

lib/pure/gentabs.nim

@@ -186,8 +186,21 @@ when isMainModule:
   assert( z["first"]["one"] == 1)   # retrieve from first inner table
   assert( z["second"]["red"] == 10) # retrieve from second inner table
 
+  when false:
+    # disabled: depends on hash order:
+    var output = ""
     for k, v in pairs(z):
-    echo( "$# ($#) ->" % [k,$len(v)] )
+      output.add( "$# ($#) ->\L" % [k,$len(v)] )
-    #for k2,v2 in pairs(v):
+      for k2,v2 in pairs(v):
-    #  echo( "   $# <-> $#" % [k2,$v2] )
+        output.add( "  $# <-> $#\L" % [k2,$v2] )
-  echo()
+
+    let expected = unindent """
+      first (3) ->
+        two <-> 2
+        three <-> 3
+        one <-> 1
+      second (2) ->
+        red <-> 10
+        blue <-> 20
+    """
+    assert output == expected

lib/pure/hashes.nim

@@ -143,16 +143,23 @@ proc hashIgnoreCase*(x: string): THash =
     h = h !& ord(c)
   result = !$h
 
+proc hash*(x: float): THash {.inline.} =
+  var y = x + 1.0
+  result = cast[ptr THash](addr(y))[]
+
+
+# Forward declarations before methods that hash containers. This allows
+# containers to contain other containers
+proc hash*[A](x: openArray[A]): THash
+proc hash*[A](x: set[A]): THash
+
+
 proc hash*[T: tuple](x: T): THash =
   ## efficient hashing of tuples.
   for f in fields(x):
     result = result !& hash(f)
   result = !$result
 
-proc hash*(x: float): THash {.inline.} =
-  var y = x + 1.0
-  result = cast[ptr THash](addr(y))[]
-
 proc hash*[A](x: openArray[A]): THash =
   for it in items(x): result = result !& hash(it)
   result = !$result
@@ -160,3 +167,4 @@ proc hash*[A](x: openArray[A]): THash =
 proc hash*[A](x: set[A]): THash =
   for it in items(x): result = result !& hash(it)
   result = !$result
+

lib/pure/htmlgen.nim

@@ -483,7 +483,8 @@ macro `var`*(e: expr): expr {.immediate.} =
   result = xmlCheckedTag(e, "var", commonAttr)
 
 when isMainModule:
-  var nim = "Nim"
+  let nim = "Nim"
-  echo h1(a(href="http://nim-lang.org", nim))
+  assert h1(a(href="http://nim-lang.org", nim)) ==
-  echo form(action="test", `accept-charset` = "Content-Type")
+    """<h1><a href="http://nim-lang.org">Nim</a></h1>"""
-
+  assert form(action="test", `accept-charset` = "Content-Type") ==
+    """<form action="test" accept-charset="Content-Type"></form>"""

lib/pure/htmlparser.nim

@@ -593,7 +593,7 @@ proc loadHtml*(path: string): XmlNode =
   var errors: seq[string] = @[]
   result = loadHtml(path, errors)
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   import os
 
   var errors: seq[string] = @[]  

lib/pure/httpclient.nim

@@ -819,7 +819,7 @@ proc get*(client: AsyncHttpClient, url: string): Future[Response] {.async.} =
       result = await client.request(redirectTo, httpGET)
       lastUrl = redirectTo
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   when true:
     # Async
     proc main() {.async.} =

lib/pure/httpserver.nim

@@ -514,7 +514,7 @@ proc close*(h: PAsyncHTTPServer) =
   ## Closes the ``PAsyncHTTPServer``.
   h.asyncSocket.close()
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   var counter = 0
 
   var s: TServer

lib/pure/json.nim

@@ -808,22 +808,25 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) =
       return
   obj.fields.add((key, val))
 
-proc `{}`*(node: JsonNode, key: string): JsonNode =
+proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode =
-  ## Transverses the node and gets the given value. If any of the
+  ## Traverses the node and gets the given value. If any of the
-  ## names does not exist, returns nil
+  ## keys do not exist, returns nil. Also returns nil if one of the
+  ## intermediate data structures is not an object
   result = node
-  if isNil(node): return nil
+  for key in keys:
+    if isNil(result) or result.kind!=JObject:
+      return nil    
     result=result[key]
 
-proc `{}=`*(node: JsonNode, names: varargs[string], value: JsonNode) =
+proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) =
-  ## Transverses the node and tries to set the value at the given location
+  ## Traverses the node and tries to set the value at the given location
-  ## to `value` If any of the names are missing, they are added
+  ## to `value` If any of the keys are missing, they are added
   var node = node
-  for i in 0..(names.len-2):
+  for i in 0..(keys.len-2):
-    if isNil(node[names[i]]):
+    if isNil(node[keys[i]]):
-      node[names[i]] = newJObject()
+      node[keys[i]] = newJObject()
-    node = node[names[i]]
+    node = node[keys[i]]
-  node[names[names.len-1]] = value
+  node[keys[keys.len-1]] = value
 
 proc delete*(obj: JsonNode, key: string) =
   ## Deletes ``obj[key]`` preserving the order of the other (key, value)-pairs.
@@ -1150,8 +1153,11 @@ when false:
 when isMainModule:
   #var node = parse("{ \"test\": null }")
   #echo(node.existsKey("test56"))
+  
   var parsed = parseFile("tests/testdata/jsontest.json")
   var parsed2 = parseFile("tests/testdata/jsontest2.json")
+
+  when not defined(testing):
     echo(parsed)
     echo()
     echo(pretty(parsed, 2))
@@ -1186,6 +1192,14 @@ when isMainModule:
   except:
     assert(false, "EInvalidIndex thrown for valid index")
 
+  assert(testJson{"b"}.str=="asd", "Couldn't fetch a singly nested key with {}") 
+  assert(isNil(testJson{"nonexistent"}), "Non-existent keys should return nil") 
+  assert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}")
+  assert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil")
+  assert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil")
+  assert(testJson{"a"}==parseJson"[1, 2, 3, 4]", "Didn't return a non-JObject when there was one to be found")
+  assert(isNil(parseJson("[1, 2, 3]"){"foo"}), "Indexing directly into a list should return nil")
+ 
   # Generator:
   var j = %* [{"name": "John", "age": 30}, {"name": "Susan", "age": 31}]
   assert j == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] 
@@ -1217,6 +1231,7 @@ when isMainModule:
     ]
   assert j3 == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}]
   
+  when not defined(testing):
     discard """
     while true:
       var json = stdin.readLine()

lib/pure/logging.nim

@@ -278,7 +278,7 @@ proc getLogFilter*(): Level =
 
 # --------------
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   var L = newConsoleLogger()
   var fL = newFileLogger("test.log", fmtStr = verboseFmtStr)
   var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr)

lib/pure/marshal.nim

@@ -28,6 +28,8 @@
 ##   new(b)
 ##   a = b
 ##   echo($$a[]) # produces "{}", not "{f: 0}"
+##
+## **Note**: The ``to`` and ``$$`` operations are available at compile-time!
 
 import streams, typeinfo, json, intsets, tables
 
@@ -38,7 +40,12 @@ proc storeAny(s: Stream, a: TAny, stored: var IntSet) =
   case a.kind
   of akNone: assert false
   of akBool: s.write($getBool(a))
-  of akChar: s.write(escapeJson($getChar(a)))
+  of akChar:
+    let ch = getChar(a)
+    if ch < '\128':
+      s.write(escapeJson($ch))
+    else:
+      s.write($int(ch))
   of akArray, akSequence:
     if a.kind == akSequence and isNil(a): s.write("null")
     else:
@@ -105,6 +112,10 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
         setBiggestInt(a, ord(x[0]))
         next(p)
         return
+    elif p.kind == jsonInt:
+      setBiggestInt(a, getInt(p))
+      next(p)
+      return
     raiseParseErr(p, "string of length 1 expected for a char")
   of akEnum:
     if p.kind == jsonString:
@@ -244,7 +255,7 @@ proc to*[T](data: string): T =
   var tab = initTable[BiggestInt, pointer]()
   loadAny(newStringStream(data), toAny(result), tab)
 
-when isMainModule:
+when not defined(testing) and isMainModule:
   template testit(x: expr) = echo($$to[type(x)]($$x))
 
   var x: array[0..4, array[0..4, string]] = [

lib/pure/math.nim

@@ -85,7 +85,7 @@ proc fac*(n: int): int {.noSideEffect.} =
 proc isPowerOfTwo*(x: int): bool {.noSideEffect.} =
   ## returns true, if `x` is a power of two, false otherwise.
   ## Zero and negative numbers are not a power of two.
-  return (x != 0) and ((x and (x - 1)) == 0)
+  return (x > 0) and ((x and (x - 1)) == 0)
 
 proc nextPowerOfTwo*(x: int): int {.noSideEffect.} =
   ## returns `x` rounded up to the nearest power of two.
@@ -114,18 +114,23 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} =
   ## If `x` is empty, 0 is returned.
   for i in items(x): result = result + i
 
-proc mean*(x: openArray[float]): float {.noSideEffect.} = 
+template toFloat(f: float): float = f
-  ## computes the mean of the elements in `x`. 
-  ## If `x` is empty, NaN is returned.
-  result = sum(x) / toFloat(len(x))
 
-proc variance*(x: openArray[float]): float {.noSideEffect.} = 
+proc mean*[T](x: openArray[T]): float {.noSideEffect.} =
+  ## computes the mean of the elements in `x`, which are first converted to floats.
+  ## If `x` is empty, NaN is returned.
+  ## ``toFloat(x: T): float`` must be defined.
+  for i in items(x): result = result + toFloat(i)
+  result = result / toFloat(len(x))
+
+proc variance*[T](x: openArray[T]): float {.noSideEffect.} =
   ## computes the variance of the elements in `x`. 
   ## If `x` is empty, NaN is returned.
+  ## ``toFloat(x: T): float`` must be defined.
   result = 0.0
   var m = mean(x)
-  for i in 0 .. high(x):
+  for i in items(x):
-    var diff = x[i] - m
+    var diff = toFloat(i) - m
     result = result + diff*diff
   result = result / toFloat(len(x))
 
@@ -372,6 +377,8 @@ when isMainModule and not defined(JS):
   randomize(seed)
   for i in 0..SIZE-1:
     assert buf[i] == random(high(int)), "non deterministic random seeding"
+
+  when not defined(testing):
     echo "random values equal after reseeding"
 
   # Check for no side effect annotation

lib/pure/mersenne.nim

@@ -32,7 +32,7 @@ proc getNum*(m: var MersenneTwister): int =
   return int(y)
 
 # Test
-when isMainModule:
+when not defined(testing) and isMainModule:
   var mt = newMersenneTwister(2525)
 
   for i in 0..99: