Merge branch 'master' of https://github.com/Araq/Nim into patch-global-log-level

appveyor.yml

@@ -1,36 +0,0 @@
-clone_depth: 5
-
-artifacts:
-  - path: bin\nim.exe
-
-platform:
-  - x64
-
-before_build:
-  - git log -1
-  - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S zlib-devel"
-  - appveyor DownloadFile http://nim-lang.org/download/dlls.zip
-  - 7z e dlls.zip -odlls
-  - del dlls\libcurl.dll
-  - appveyor DownloadFile http://flatassembler.net/fasmw17139.zip
-  - 7z e fasmw17139.zip -obin fasm.exe
-
-build_script:
-  - SET PATH=C:\msys64\mingw64\bin;dlls;bin;%PATH%
-  - gcc -v
-  - git clone -q --depth 1 https://github.com/nim-lang/csources
-  - cd csources
-  - build64.bat
-  - cd ..
-  - nim c koch
-  - koch boot
-  - koch boot -d:release
-
-before_test:
-  - nim e install_nimble.nims
-  - nimble update
-  - nimble install zip
-
-test_script:
-  - nim c --taintMode:on tests/testament/tester
-  - tests\testament\tester --pedantic all

compiler.nimble

@@ -1,6 +1,6 @@
 [Package]
 name = "compiler"
-version = "0.12.0"
+version = "0.13.0"
 author = "Andreas Rumpf"
 description = "Compiler package providing the compiler sources as a library."
 license = "MIT"

compiler/ast.nim

@@ -500,8 +500,7 @@ type
     skResult,             # special 'result' variable
     skProc,               # a proc
     skMethod,             # a method
-    skIterator,           # an inline iterator
+    skIterator,           # an iterator
-    skClosureIterator,    # a resumable closure iterator
     skConverter,          # a type converter
     skMacro,              # a macro
     skTemplate,           # a template; currently also misused for user-defined
@@ -518,7 +517,7 @@ type
   TSymKinds* = set[TSymKind]
 
 const
-  routineKinds* = {skProc, skMethod, skIterator, skClosureIterator,
+  routineKinds* = {skProc, skMethod, skIterator,
                    skConverter, skMacro, skTemplate}
   tfIncompleteStruct* = tfVarargs
   tfUncheckedArray* = tfVarargs
@@ -905,7 +904,7 @@ type
 # the poor naming choices in the standard library.
 
 const
-  OverloadableSyms* = {skProc, skMethod, skIterator, skClosureIterator,
+  OverloadableSyms* = {skProc, skMethod, skIterator,
     skConverter, skModule, skTemplate, skMacro}
 
   GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody,
@@ -929,11 +928,11 @@ const
   NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tySequence,
     tyProc, tyString, tyError}
   ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType,
-    skIterator, skClosureIterator,
+    skIterator,
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
                                       nfDotSetter, nfDotField,
-                                      nfIsRef, nfIsCursor}
+                                      nfIsRef, nfIsCursor, nfLL}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -958,11 +957,9 @@ const
   nkStrKinds* = {nkStrLit..nkTripleStrLit}
 
   skLocalVars* = {skVar, skLet, skForVar, skParam, skResult}
-  skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skClosureIterator,
+  skProcKinds* = {skProc, skTemplate, skMacro, skIterator,
                   skMethod, skConverter}
 
-  skIterators* = {skIterator, skClosureIterator}
-
 var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things
 
 proc isCallExpr*(n: PNode): bool =
@@ -1013,6 +1010,10 @@ proc newNode*(kind: TNodeKind): PNode =
       writeStackTrace()
     inc gNodeId
 
+proc newTree*(kind: TNodeKind; children: varargs[PNode]): PNode =
+  result = newNode(kind)
+  result.sons = @children
+
 proc newIntNode*(kind: TNodeKind, intVal: BiggestInt): PNode =
   result = newNode(kind)
   result.intVal = intVal
@@ -1554,12 +1555,13 @@ proc isGenericRoutine*(s: PSym): bool =
   else: discard
 
 proc skipGenericOwner*(s: PSym): PSym =
-  internalAssert s.kind in skProcKinds
   ## Generic instantiations are owned by their originating generic
   ## symbol. This proc skips such owners and goes straight to the owner
   ## of the generic itself (the module or the enclosing proc).
-  result = if sfFromGeneric in s.flags: s.owner.owner
+  result = if s.kind in skProcKinds and sfFromGeneric in s.flags:
-           else: s.owner
+             s.owner.owner
+           else:
+             s.owner
 
 proc originatingModule*(s: PSym): PSym =
   result = s.owner

compiler/ccgcalls.nim

@@ -118,6 +118,14 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
         result = "$1->data, $1->$2" % [a.rdLoc, lenField(p)]
     of tyArray, tyArrayConstr:
       result = "$1, $2" % [rdLoc(a), rope(lengthOrd(a.t))]
+    of tyPtr, tyRef:
+      case lastSon(a.t).kind
+      of tyString, tySequence:
+        result = "(*$1)->data, (*$1)->$2" % [a.rdLoc, lenField(p)]
+      of tyArray, tyArrayConstr:
+        result = "$1, $2" % [rdLoc(a), rope(lengthOrd(lastSon(a.t)))]
+      else: 
+        internalError("openArrayLoc: " & typeToString(a.t))
     else: internalError("openArrayLoc: " & typeToString(a.t))
 
 proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} =
@@ -515,7 +523,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     line(p, cpsStmts, pl)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e.sons[0].typ.callConv == ccClosure:
+  if e.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure:
     genClosureCall(p, nil, e, d)
   elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags:
     genInfixCall(p, nil, e, d)
@@ -528,7 +536,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) =
     if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  if ri.sons[0].typ.callConv == ccClosure:
+  if ri.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
   elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags:
     genInfixCall(p, le, ri, d)

compiler/ccgexprs.nim

@@ -959,6 +959,7 @@ proc genEcho(p: BProc, n: PNode) =
       addf(args, ", $1? ($1)->data:\"nil\"", [rdLoc(a)])
   linefmt(p, cpsStmts, "printf($1$2);$n",
           makeCString(repeat("%s", n.len) & tnl), args)
+  linefmt(p, cpsStmts, "fflush(stdout);$n")
 
 proc gcUsage(n: PNode) =
   if gSelectedGC == gcNone: message(n.info, warnGcMem, n.renderTree)
@@ -1415,11 +1416,11 @@ proc binaryExprIn(p: BProc, e: PNode, a, b, d: var TLoc, frmt: string) =
 
 proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) =
   case int(getSize(skipTypes(e.sons[1].typ, abstractVar)))
-  of 1: binaryExprIn(p, e, a, b, d, "(($1 &(1<<(($2)&7)))!=0)")
+  of 1: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&7U)))!=0)")
-  of 2: binaryExprIn(p, e, a, b, d, "(($1 &(1<<(($2)&15)))!=0)")
+  of 2: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&15U)))!=0)")
-  of 4: binaryExprIn(p, e, a, b, d, "(($1 &(1<<(($2)&31)))!=0)")
+  of 4: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&31U)))!=0)")
-  of 8: binaryExprIn(p, e, a, b, d, "(($1 &(IL64(1)<<(($2)&IL64(63))))!=0)")
+  of 8: binaryExprIn(p, e, a, b, d, "(($1 &((NU64)1<<((NU)($2)&63U)))!=0)")
-  else: binaryExprIn(p, e, a, b, d, "(($1[$2/8] &(1<<($2%8)))!=0)")
+  else: binaryExprIn(p, e, a, b, d, "(($1[(NU)($2)>>3] &(1U<<((NU)($2)&7U)))!=0)")
 
 proc binaryStmtInExcl(p: BProc, e: PNode, d: var TLoc, frmt: string) =
   var a, b: TLoc
@@ -1500,8 +1501,8 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     else: internalError(e.info, "genSetOp()")
   else:
     case op
-    of mIncl: binaryStmtInExcl(p, e, d, "$1[$2/8] |=(1<<($2%8));$n")
+    of mIncl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] |=(1U<<($2&7U));$n")
-    of mExcl: binaryStmtInExcl(p, e, d, "$1[$2/8] &= ~(1<<($2%8));$n")
+    of mExcl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] &= ~(1U<<($2&7U));$n")
     of mCard: unaryExprChar(p, e, d, "#cardSet($1, " & $size & ')')
     of mLtSet, mLeSet:
       getTemp(p, getSysType(tyInt), i) # our counter
@@ -1733,8 +1734,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mEcho: genEcho(p, e[1].skipConv)
   of mArrToSeq: genArrToSeq(p, e, d)
   of mNLen..mNError, mSlurp..mQuoteAst:
-    echo "from here ", p.prc.name.s, " ", p.prc.info
-    writestacktrace()
     localError(e.info, errXMustBeCompileTime, e.sons[0].sym.name.s)
   of mSpawn:
     let n = lowerings.wrapProcForSpawn(p.module.module, e, e.typ, nil, nil)
@@ -1788,11 +1787,11 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) =
           initLocExpr(p, e.sons[i].sons[0], a)
           initLocExpr(p, e.sons[i].sons[1], b)
           lineF(p, cpsStmts, "for ($1 = $3; $1 <= $4; $1++) $n" &
-              "$2[$1/8] |=(1<<($1%8));$n", [rdLoc(idx), rdLoc(d),
+              "$2[(NU)($1)>>3] |=(1U<<((NU)($1)&7U));$n", [rdLoc(idx), rdLoc(d),
               rdSetElemLoc(a, e.typ), rdSetElemLoc(b, e.typ)])
         else:
           initLocExpr(p, e.sons[i], a)
-          lineF(p, cpsStmts, "$1[$2/8] |=(1<<($2%8));$n",
+          lineF(p, cpsStmts, "$1[(NU)($2)>>3] |=(1U<<((NU)($2)&7U));$n",
                [rdLoc(d), rdSetElemLoc(a, e.typ)])
     else:
       # small set
@@ -1839,15 +1838,17 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) =
   assert n.kind == nkClosure
 
   if isConstClosure(n):
-    inc(p.labels)
+    inc(p.module.labels)
-    var tmp = "LOC" & rope(p.labels)
+    var tmp = "CNSTCLOSURE" & rope(p.module.labels)
-    addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n",
+    addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = $3;$n",
         [getTypeDesc(p.module, n.typ), tmp, genConstExpr(p, n)])
     putIntoDest(p, d, n.typ, tmp, OnStatic)
   else:
     var tmp, a, b: TLoc
     initLocExpr(p, n.sons[0], a)
     initLocExpr(p, n.sons[1], b)
+    if n.sons[0].skipConv.kind == nkClosure:
+      internalError(n.info, "closure to closure created")
     getTemp(p, n.typ, tmp)
     linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n",
             tmp.rdLoc, a.rdLoc, b.rdLoc)
@@ -1966,7 +1967,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       else:
         genProc(p.module, sym)
       putLocIntoDest(p, d, sym.loc)
-    of skProc, skConverter, skIterators:
+    of skProc, skConverter, skIterator:
+      #if sym.kind == skIterator:
+      #  echo renderTree(sym.getBody, {renderIds})
       if sfCompileTime in sym.flags:
         localError(n.info, "request to generate code for .compileTime proc: " &
            sym.name.s)
@@ -1988,6 +1991,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       if sfGlobal in sym.flags: genVarPrototype(p.module, sym)
       if sym.loc.r == nil or sym.loc.t == nil:
         #echo "FAILED FOR PRCO ", p.prc.name.s
+        #echo renderTree(p.prc.ast, {renderIds})
         internalError n.info, "expr: var not init " & sym.name.s & "_" & $sym.id
       if sfThread in sym.flags:
         accessThreadLocalVar(p, sym)

compiler/ccgstmts.nim

@@ -16,13 +16,13 @@ const
     # above X strings a hash-switch for strings is generated
 
 proc registerGcRoot(p: BProc, v: PSym) =
-  if gSelectedGC in {gcMarkAndSweep, gcGenerational} and
+  if gSelectedGC in {gcMarkAndSweep, gcGenerational, gcV2} and
       containsGarbageCollectedRef(v.loc.t):
     # we register a specialized marked proc here; this has the advantage
     # that it works out of the box for thread local storage then :-)
     let prc = genTraverseProcForGlobal(p.module, v)
-    linefmt(p.module.initProc, cpsStmts,
+    appcg(p.module, p.module.initProc.procSec(cpsStmts),
-      "#nimRegisterGlobalMarker($1);$n", prc)
+      "#nimRegisterGlobalMarker($1);$n", [prc])
 
 proc isAssignedImmediately(n: PNode): bool {.inline.} =
   if n.kind == nkEmpty: return false
@@ -955,7 +955,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
       res.add(t.sons[i].strVal)
     of nkSym:
       var sym = t.sons[i].sym
-      if sym.kind in {skProc, skIterator, skClosureIterator, skMethod}:
+      if sym.kind in {skProc, skIterator, skMethod}:
         var a: TLoc
         initLocExpr(p, t.sons[i], a)
         res.add($rdLoc(a))

compiler/cgen.nim

@@ -594,7 +594,7 @@ proc cgsym(m: BModule, name: string): Rope =
   var sym = magicsys.getCompilerProc(name)
   if sym != nil:
     case sym.kind
-    of skProc, skMethod, skConverter, skIterators: genProc(m, sym)
+    of skProc, skMethod, skConverter, skIterator: genProc(m, sym)
     of skVar, skResult, skLet: genVarPrototype(m, sym)
     of skType: discard getTypeDesc(m, sym.typ)
     else: internalError("cgsym: " & name & ": " & $sym.kind)

compiler/cgmeth.nim

@@ -18,8 +18,10 @@ proc genConv(n: PNode, d: PType, downcast: bool): PNode =
   var source = skipTypes(n.typ, abstractPtrs)
   if (source.kind == tyObject) and (dest.kind == tyObject):
     var diff = inheritanceDiff(dest, source)
-    if diff == high(int): internalError(n.info, "cgmeth.genConv")
+    if diff == high(int):
-    if diff < 0:
+      # no subtype relation, nothing to do
+      result = n
+    elif diff < 0:
       result = newNodeIT(nkObjUpConv, n.info, d)
       addSon(result, n)
       if downcast: internalError(n.info, "cgmeth.genConv: no upcast allowed")

compiler/docgen.nim

@@ -149,10 +149,10 @@ proc ropeFormatNamedVars(frmt: FormatStr, varnames: openArray[string],
 proc genComment(d: PDoc, n: PNode): string =
   result = ""
   var dummyHasToc: bool
-  if n.comment != nil and startsWith(n.comment, "##"):
+  if n.comment != nil:
     renderRstToOut(d[], parseRst(n.comment, toFilename(n.info),
                                toLinenumber(n.info), toColumn(n.info),
-                               dummyHasToc, d.options + {roSkipPounds}), result)
+                               dummyHasToc, d.options), result)
 
 proc genRecComment(d: PDoc, n: PNode): Rope =
   if n == nil: return nil
@@ -537,7 +537,7 @@ proc generateJson(d: PDoc, n: PNode, jArray: JsonNode = nil): JsonNode =
 proc genSection(d: PDoc, kind: TSymKind) =
   const sectionNames: array[skModule..skTemplate, string] = [
     "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Methods",
-    "Iterators", "Iterators", "Converters", "Macros", "Templates"
+    "Iterators", "Converters", "Macros", "Templates"
   ]
   if d.section[kind] == nil: return
   var title = sectionNames[kind].rope

compiler/evaltempl.nim

@@ -38,7 +38,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
     if s.owner.id == c.owner.id:
       if s.kind == skParam and sfGenSym notin s.flags:
         handleParam actual.sons[s.position]
-      elif s.kind == skGenericParam:
+      elif s.kind == skGenericParam or
+           s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam:
         handleParam actual.sons[s.owner.typ.len + s.position - 1]
       else:
         internalAssert sfGenSym in s.flags

compiler/installer.ini

@@ -62,7 +62,7 @@ Files: "icons/koch_icon.o"
 
 Files: "compiler/readme.txt"
 Files: "compiler/installer.ini"
-Files: "compiler/nim.nim.cfg"
+Files: "compiler/*.cfg"
 Files: "compiler/*.nim"
 Files: "doc/*.txt"
 Files: "doc/manual/*.txt"
@@ -84,6 +84,8 @@ Files: "tools/niminst/*.nsh"
 Files: "web/website.ini"
 Files: "web/*.nim"
 Files: "web/*.txt"
+Files: "bin/nimblepkg/*.nim"
+Files: "bin/nimblepkg/*.cfg"
 
 [Lib]
 Files: "lib/nimbase.h"
@@ -107,9 +109,6 @@ Files: "lib/wrappers/readline/*.nim"
 Files: "lib/wrappers/linenoise/*.nim"
 Files: "lib/wrappers/linenoise/*.c"
 Files: "lib/wrappers/linenoise/*.h"
-Files: "lib/wrappers/sdl/*.nim"
-Files: "lib/wrappers/zip/*.nim"
-Files: "lib/wrappers/zip/libzip_all.c"
 
 Files: "lib/windows/*.nim"
 Files: "lib/posix/*.nim"

compiler/jsgen.nim

@@ -931,7 +931,7 @@ proc isIndirect(v: PSym): bool =
   result = {sfAddrTaken, sfGlobal} * v.flags != {} and
     #(mapType(v.typ) != etyObject) and
     {sfImportc, sfVolatile, sfExportc} * v.flags == {} and
-    v.kind notin {skProc, skConverter, skMethod, skIterator, skClosureIterator,
+    v.kind notin {skProc, skConverter, skMethod, skIterator,
                   skConst, skTemp, skLet}
 
 proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
@@ -1636,6 +1636,9 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkSym:
     genSym(p, n, r)
   of nkCharLit..nkInt64Lit:
+    if n.typ.kind == tyBool:
+      r.res = if n.intVal == 0: rope"false" else: rope"true"
+    else:
       r.res = rope(n.intVal)
     r.kind = resExpr
   of nkNilLit:

compiler/jstypes.nim

@@ -116,7 +116,7 @@ proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
          [name, genTypeInfo(p, typ.sons[0])])
 
 proc genTypeInfo(p: PProc, typ: PType): Rope =
-  let t = typ.skipTypes({tyGenericInst})
+  let t = typ.skipTypes({tyGenericInst, tyDistinct})
   result = "NTI$1" % [rope(t.id)]
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
   case t.kind

compiler/lexer.nim

@@ -662,6 +662,7 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
         L.lineNumber = line
         lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart)
         L.lineNumber = line2
+        L.bufpos = pos
         break
       else:
         add(tok.literal, buf[pos])
@@ -768,24 +769,88 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
+proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
+                          isDoc: bool) =
+  var pos = start
+  var buf = L.buf
+  var toStrip = 0
+  # detect the amount of indentation:
+  if isDoc:
+    toStrip = getColNumber(L, pos)
+    while buf[pos] == ' ': inc pos
+    if buf[pos] in {CR, LF}:
+      pos = handleCRLF(L, pos)
+      buf = L.buf
+      toStrip = 0
+      while buf[pos] == ' ':
+        inc pos
+        inc toStrip
+  var nesting = 0
+  while true:
+    case buf[pos]
+    of '#':
+      if isDoc:
+        if buf[pos+1] == '#' and buf[pos+2] == '[':
+          inc nesting
+        tok.literal.add '#'
+      elif buf[pos+1] == '[':
+        inc nesting
+      inc pos
+    of ']':
+      if isDoc:
+        if buf[pos+1] == '#' and buf[pos+2] == '#':
+          if nesting == 0:
+            inc(pos, 3)
+            break
+          dec nesting
+        tok.literal.add ']'
+      elif buf[pos+1] == '#':
+        if nesting == 0:
+          inc(pos, 2)
+          break
+        dec nesting
+      inc pos
+    of '\t':
+      lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
+      inc(pos)
+      if isDoc: tok.literal.add '\t'
+    of CR, LF:
+      pos = handleCRLF(L, pos)
+      buf = L.buf
+      # strip leading whitespace:
+      if isDoc:
+        tok.literal.add "\n"
+        inc tok.iNumber
+        var c = toStrip
+        while buf[pos] == ' ' and c > 0:
+          inc pos
+          dec c
+    of nimlexbase.EndOfFile:
+      lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
+      break
+    else:
+      if isDoc: tok.literal.add buf[pos]
+      inc(pos)
+  L.bufpos = pos
+
 proc scanComment(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
   var buf = L.buf
-  when not defined(nimfix):
-    assert buf[pos+1] == '#'
-    if buf[pos+2] == '[':
-      if buf[pos+3] == ']':
-        #  ##[] is the (rather complex) "cursor token" for idetools
-        tok.tokType = tkComment
-        tok.literal = "[]"
-        inc(L.bufpos, 4)
-        return
-      else:
-        lexMessagePos(L, warnDeprecated, pos, "use '## [' instead; '##['")
-
   tok.tokType = tkComment
   # iNumber contains the number of '\n' in the token
   tok.iNumber = 0
+  when not defined(nimfix):
+    assert buf[pos+1] == '#'
+    if buf[pos+2] == '[':
+      skipMultiLineComment(L, tok, pos+3, true)
+      return
+    inc(pos, 2)
+
+  var toStrip = 0
+  while buf[pos] == ' ':
+    inc pos
+    inc toStrip
+
   when defined(nimfix):
     var col = getColNumber(L, pos)
   while true:
@@ -819,6 +884,12 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     if doContinue():
       tok.literal.add "\n"
       when defined(nimfix): col = indent
+      else:
+        inc(pos, 2)
+        var c = toStrip
+        while buf[pos] == ' ' and c > 0:
+          inc pos
+          dec c
       inc tok.iNumber
     else:
       if buf[pos] > ' ':
@@ -842,9 +913,16 @@ proc skip(L: var TLexer, tok: var TToken) =
       pos = handleCRLF(L, pos)
       buf = L.buf
       var indent = 0
-      while buf[pos] == ' ':
+      while true:
+        if buf[pos] == ' ':
           inc(pos)
           inc(indent)
+        elif buf[pos] == '#' and buf[pos+1] == '[':
+          skipMultiLineComment(L, tok, pos+2, false)
+          pos = L.bufpos
+          buf = L.buf
+        else:
+          break
       tok.strongSpaceA = 0
       when defined(nimfix):
         template doBreak(): expr = buf[pos] > ' '
@@ -862,7 +940,10 @@ proc skip(L: var TLexer, tok: var TToken) =
         # do not skip documentation comment:
         if buf[pos+1] == '#': break
         if buf[pos+1] == '[':
-          lexMessagePos(L, warnDeprecated, pos, "use '# [' instead; '#['")
+          skipMultiLineComment(L, tok, pos+2, false)
+          pos = L.bufpos
+          buf = L.buf
+        else:
           while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos)
     else:
       break                   # EndOfFile also leaves the loop

compiler/lookups.nim

@@ -133,7 +133,7 @@ type
 
 proc getSymRepr*(s: PSym): string =
   case s.kind
-  of skProc, skMethod, skConverter, skIterators: result = getProcHeader(s)
+  of skProc, skMethod, skConverter, skIterator: result = getProcHeader(s)
   else: result = s.name.s
 
 proc ensureNoMissingOrUnusedSymbols(scope: PScope) =

compiler/lowerings.nim

@@ -165,9 +165,10 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo): PNode =
   deref.typ = a.typ.skipTypes(abstractInst).sons[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
+  let bb = getIdent(b)
   while true:
     assert t.kind == tyObject
-    field = getSymFromList(t.n, getIdent(b))
+    field = getSymFromList(t.n, bb)
     if field != nil: break
     t = t.sons[0]
     if t == nil: break
@@ -585,7 +586,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType;
     objType.addField(field)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
     fn = indirectAccess(castExpr, field, n.info)
-  elif fn.kind == nkSym and fn.sym.kind in {skClosureIterator, skIterator}:
+  elif fn.kind == nkSym and fn.sym.kind == skIterator:
     localError(n.info, "iterator in spawn environment is not allowed")
   elif fn.typ.callConv == ccClosure:
     localError(n.info, "closure in spawn environment is not allowed")

compiler/nimsets.nim

@@ -106,13 +106,17 @@ proc toTreeSet(s: TBitSet, settype: PType, info: TLineInfo): PNode =
         inc(b)
         if (b >= len(s) * ElemSize) or not bitSetIn(s, b): break
       dec(b)
+      let aa = newIntTypeNode(nkIntLit, a + first, elemType)
+      aa.info = info
       if a == b:
-        addSon(result, newIntTypeNode(nkIntLit, a + first, elemType))
+        addSon(result, aa)
       else:
         n = newNodeI(nkRange, info)
         n.typ = elemType
-        addSon(n, newIntTypeNode(nkIntLit, a + first, elemType))
+        addSon(n, aa)
-        addSon(n, newIntTypeNode(nkIntLit, b + first, elemType))
+        let bb = newIntTypeNode(nkIntLit, b + first, elemType)
+        bb.info = info
+        addSon(n, bb)
         addSon(result, n)
       e = b
     inc(e)

compiler/parser.nim

@@ -112,11 +112,6 @@ proc rawSkipComment(p: var TParser, node: PNode) =
   if p.tok.tokType == tkComment:
     if node != nil:
       if node.comment == nil: node.comment = ""
-      if p.tok.literal == "[]":
-        node.flags.incl nfIsCursor
-        #echo "parser: "
-        #debug node
-      else:
       add(node.comment, p.tok.literal)
     else:
       parMessage(p, errInternal, "skipComment")
@@ -250,7 +245,9 @@ proc isUnary(p: TParser): bool =
   if p.tok.tokType in {tkOpr, tkDotDot} and
      p.tok.strongSpaceB == 0 and
      p.tok.strongSpaceA > 0:
-    # XXX change this after 0.10.4 is out
+      result = true
+      # versions prior to 0.13.0 used to do this:
+      when false:
         if p.strongSpaces:
           result = true
         else:
@@ -991,7 +988,7 @@ proc isExprStart(p: TParser): bool =
   of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf,
      tkProc, tkIterator, tkBind, tkAddr,
      tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr,
-     tkTuple, tkObject, tkType, tkWhen, tkCase:
+     tkTuple, tkObject, tkType, tkWhen, tkCase, tkOut:
     result = true
   else: result = false
 
@@ -1038,7 +1035,7 @@ proc parseObject(p: var TParser): PNode
 proc parseTypeClass(p: var TParser): PNode
 
 proc primary(p: var TParser, mode: TPrimaryMode): PNode =
-  #| typeKeyw = 'var' | 'ref' | 'ptr' | 'shared' | 'tuple'
+  #| typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple'
   #|          | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum'
   #| primary = typeKeyw typeDescK
   #|         /  prefixOperator* identOrLiteral primarySuffix*
@@ -1112,6 +1109,7 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
     optInd(p, result)
     addSon(result, primary(p, pmNormal))
   of tkVar: result = parseTypeDescKAux(p, nkVarTy, mode)
+  of tkOut: result = parseTypeDescKAux(p, nkVarTy, mode)
   of tkRef: result = parseTypeDescKAux(p, nkRefTy, mode)
   of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, mode)
   of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, mode)
@@ -1763,7 +1761,7 @@ proc parseObject(p: var TParser): PNode =
   addSon(result, parseObjectPart(p))
 
 proc parseTypeClassParam(p: var TParser): PNode =
-  if p.tok.tokType == tkVar:
+  if p.tok.tokType in {tkOut, tkVar}:
     result = newNodeP(nkVarTy, p)
     getTok(p)
     result.addSon(p.parseSymbol)
@@ -1771,7 +1769,7 @@ proc parseTypeClassParam(p: var TParser): PNode =
     result = p.parseSymbol
 
 proc parseTypeClass(p: var TParser): PNode =
-  #| typeClassParam = ('var')? symbol
+  #| typeClassParam = ('var' | 'out')? symbol
   #| typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
   #|               &IND{>} stmt
   result = newNodeP(nkTypeClassTy, p)

compiler/plugins/active.nim

@@ -10,4 +10,4 @@
 ## Include file that imports all plugins that are active.
 
 import
-  locals.locals
+  locals.locals, itersgen

compiler/plugins/itersgen.nim

@@ -0,0 +1,51 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Plugin to transform an inline iterator into a data structure.
+
+import compiler/pluginsupport, compiler/ast, compiler/astalgo,
+  compiler/magicsys, compiler/lookups, compiler/semdata,
+  compiler/lambdalifting, compiler/rodread, compiler/msgs
+
+
+proc iterToProcImpl(c: PContext, n: PNode): PNode =
+  result = newNodeI(nkStmtList, n.info)
+  let iter = n[1]
+  if iter.kind != nkSym or iter.sym.kind != skIterator:
+    localError(iter.info, "first argument needs to be an iterator")
+    return
+  if n[2].typ.isNil:
+    localError(n[2].info, "second argument needs to be a type")
+    return
+  if n[3].kind != nkIdent:
+    localError(n[3].info, "third argument needs to be an identifier")
+    return
+
+  let t = n[2].typ.skipTypes({tyTypeDesc, tyGenericInst})
+  if t.kind notin {tyRef, tyPtr} or t.lastSon.kind != tyObject:
+    localError(n[2].info,
+        "type must be a non-generic ref|ptr to object with state field")
+    return
+  let body = liftIterToProc(iter.sym, iter.sym.getBody, t)
+
+  let prc = newSym(skProc, n[3].ident, iter.sym.owner, iter.sym.info)
+  prc.typ = copyType(iter.sym.typ, prc, false)
+  excl prc.typ.flags, tfCapturesEnv
+  prc.typ.n.add newSymNode(getEnvParam(iter.sym))
+  prc.typ.rawAddSon t
+  let orig = iter.sym.ast
+  prc.ast = newProcNode(nkProcDef, n.info,
+                        name = newSymNode(prc),
+                        params = orig[paramsPos],
+                        pragmas = orig[pragmasPos],
+                        body = body)
+  prc.ast.add iter.sym.ast.sons[resultPos]
+  addInterfaceDecl(c, prc)
+
+registerPlugin("stdlib", "system", "iterToProc", iterToProcImpl)

compiler/plugins/locals/locals.nim

@@ -9,8 +9,8 @@
 
 ## The builtin 'system.locals' implemented as a plugin.
 
-import compiler/plugins, compiler/ast, compiler/astalgo, compiler/magicsys,
+import compiler/pluginsupport, compiler/ast, compiler/astalgo,
-  compiler/lookups, compiler/semdata, compiler/lowerings
+  compiler/magicsys, compiler/lookups, compiler/semdata, compiler/lowerings
 
 proc semLocals(c: PContext, n: PNode): PNode =
   var counter = 0

compiler/pluginsupport.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## Plugin support for the Nim compiler. Right now there are no plugins and they
+## Plugin support for the Nim compiler. Right now they
 ## need to be build with the compiler, no DLL support.
 
 import ast, semdata, idents
@@ -20,13 +20,16 @@ type
     next: Plugin
 
 proc pluginMatches(p: Plugin; s: PSym): bool =
-  if s.name.id != p.fn.id: return false
+  if s.name.id != p.fn.id:
-  let module = s.owner
+    return false
+  let module = s.skipGenericOwner
   if module == nil or module.kind != skModule or
-      module.name.id != p.module.id: return false
+      module.name.id != p.module.id:
+    return false
   let package = module.owner
   if package == nil or package.kind != skPackage or
-      package.name.id != p.package.id: return false
+      package.name.id != p.package.id:
+    return false
   return true
 
 var head: Plugin

compiler/renderer.nim

@@ -167,33 +167,24 @@ proc makeNimString(s: string): string =
 proc putComment(g: var TSrcGen, s: string) =
   if s.isNil: return
   var i = 0
-  var comIndent = 1
   var isCode = (len(s) >= 2) and (s[1] != ' ')
   var ind = g.lineLen
-  var com = ""
+  var com = "## "
   while true:
     case s[i]
     of '\0':
       break
     of '\x0D':
       put(g, tkComment, com)
-      com = ""
+      com = "## "
       inc(i)
       if s[i] == '\x0A': inc(i)
       optNL(g, ind)
     of '\x0A':
       put(g, tkComment, com)
-      com = ""
+      com = "## "
       inc(i)
       optNL(g, ind)
-    of '#':
-      add(com, s[i])
-      inc(i)
-      comIndent = 0
-      while s[i] == ' ':
-        add(com, s[i])
-        inc(i)
-        inc(comIndent)
     of ' ', '\x09':
       add(com, s[i])
       inc(i)
@@ -206,7 +197,7 @@ proc putComment(g: var TSrcGen, s: string) =
       if not isCode and (g.lineLen + (j - i) > MaxLineLen):
         put(g, tkComment, com)
         optNL(g, ind)
-        com = '#' & spaces(comIndent)
+        com = "## "
       while s[i] > ' ':
         add(com, s[i])
         inc(i)
@@ -283,7 +274,7 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool =
   result = false
   if n.comment != nil:
     result = (renderNoComments notin g.flags) or
-        (renderDocComments in g.flags) and startsWith(n.comment, "##")
+        (renderDocComments in g.flags)
 
 proc gcom(g: var TSrcGen, n: PNode) =
   assert(n != nil)
@@ -1330,6 +1321,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     initContext c
     putWithSpace g, tkSymbol, if n.kind == nkState: "state" else: "goto"
     gsons(g, n, c)
+  of nkBreakState:
+    put(g, tkTuple, "breakstate")
   of nkTypeClassTy:
     gTypeClassTy(g, n)
   else:

compiler/sem.nim

@@ -16,7 +16,7 @@ import
   procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
   intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity,
-  semparallel, lowerings, plugins, plugins.active
+  semparallel, lowerings, pluginsupport, plugins.active
 
 when defined(nimfix):
   import nimfix.prettybase
@@ -186,6 +186,8 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
     result.owner = getCurrOwner()
   else:
     result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info)
+  #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule:
+  #  incl(result.flags, sfGlobal)
 
 proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
                  allowed: TSymFlags): PSym
@@ -202,7 +204,7 @@ proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) =
                            "' in this context: '" & typeToString(typ) & "'")
 
 proc paramsTypeCheck(c: PContext, typ: PType) {.inline.} =
-  typeAllowedCheck(typ.n.info, typ, skConst)
+  typeAllowedCheck(typ.n.info, typ, skProc)
 
 proc expectMacroOrTemplateCall(c: PContext, n: PNode): PSym
 proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode
@@ -485,4 +487,3 @@ proc myClose(context: PPassContext, n: PNode): PNode =
   popProcCon(c)
 
 const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
-

compiler/semcall.nim

@@ -75,7 +75,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
           errors.add(err)
     if z.state == csMatch:
       # little hack so that iterators are preferred over everything else:
-      if sym.kind in skIterators: inc(z.exactMatches, 200)
+      if sym.kind == skIterator: inc(z.exactMatches, 200)
       case best.state
       of csEmpty, csNoMatch: best = z
       of csMatch:
@@ -395,7 +395,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     for i in countup(0, len(a)-1):
       var candidate = a.sons[i].sym
       if candidate.kind in {skProc, skMethod, skConverter,
-                            skIterator, skClosureIterator}:
+                            skIterator}:
         # it suffices that the candidate has the proper number of generic
         # type parameters:
         if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1:

compiler/semdata.nim

@@ -315,7 +315,7 @@ proc makeRangeType*(c: PContext; first, last: BiggestInt;
   addSonSkipIntLit(result, intType) # basetype of range
 
 proc markIndirect*(c: PContext, s: PSym) {.inline.} =
-  if s.kind in {skProc, skConverter, skMethod, skIterator, skClosureIterator}:
+  if s.kind in {skProc, skConverter, skMethod, skIterator}:
     incl(s.flags, sfAddrTaken)
     # XXX add to 'c' for global analysis
 

compiler/semexprs.nim

@@ -385,7 +385,8 @@ proc isOpImpl(c: PContext, n: PNode): PNode =
       result = newIntNode(nkIntLit, ord(t.kind == tyProc and
                                         t.callConv == ccClosure and
                                         tfIterator notin t.flags))
-    else: discard
+    else:
+      result = newIntNode(nkIntLit, 0)
   else:
     var t2 = n[2].typ.skipTypes({tyTypeDesc})
     maybeLiftType(t2, c, n.info)
@@ -752,11 +753,11 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
                                      flags: TExprFlags): PNode =
   if flags*{efInTypeof, efWantIterator} != {}:
     # consider: 'for x in pReturningArray()' --> we don't want the restriction
-    # to 'skIterators' anymore; skIterators are preferred in sigmatch already
+    # to 'skIterator' anymore; skIterator is preferred in sigmatch already
     # for typeof support.
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     result = semOverloadedCall(c, n, nOrig,
-      {skProc, skMethod, skConverter, skMacro, skTemplate}+skIterators)
+      {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator})
   else:
     result = semOverloadedCall(c, n, nOrig,
       {skProc, skMethod, skConverter, skMacro, skTemplate})
@@ -769,7 +770,7 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
     case callee.kind
     of skMacro, skTemplate: discard
     else:
-      if callee.kind in skIterators and callee.id == c.p.owner.id:
+      if callee.kind == skIterator and callee.id == c.p.owner.id:
         localError(n.info, errRecursiveDependencyX, callee.name.s)
         # error correction, prevents endless for loop elimination in transf.
         # See bug #2051:
@@ -1200,7 +1201,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
     let s = if n.sons[0].kind == nkSym: n.sons[0].sym
             elif n[0].kind in nkSymChoices: n.sons[0][0].sym
             else: nil
-    if s != nil and s.kind in {skProc, skMethod, skConverter}+skIterators:
+    if s != nil and s.kind in {skProc, skMethod, skConverter, skIterator}:
       # type parameters: partial generic specialization
       n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s)
       result = explicitGenericInstantiation(c, n, s)
@@ -1348,8 +1349,8 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
 proc semReturn(c: PContext, n: PNode): PNode =
   result = n
   checkSonsLen(n, 1)
-  if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro,
+  if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro} or (
-                        skClosureIterator}:
+     c.p.owner.kind == skIterator and c.p.owner.typ.callConv == ccClosure):
     if n.sons[0].kind != nkEmpty:
       # transform ``return expr`` to ``result = expr; return``
       if c.p.resultSym != nil:
@@ -1425,7 +1426,7 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
 proc semYield(c: PContext, n: PNode): PNode =
   result = n
   checkSonsLen(n, 1)
-  if c.p.owner == nil or c.p.owner.kind notin skIterators:
+  if c.p.owner == nil or c.p.owner.kind != skIterator:
     localError(n.info, errYieldNotAllowedHere)
   elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
     localError(n.info, errYieldNotAllowedInTryStmt)
@@ -1434,20 +1435,15 @@ proc semYield(c: PContext, n: PNode): PNode =
     var iterType = c.p.owner.typ
     let restype = iterType.sons[0]
     if restype != nil:
-      let adjustedRes = if restype.kind == tyIter: restype.base
+      if restype.kind != tyExpr:
-                        else: restype
+        n.sons[0] = fitNode(c, restype, n.sons[0])
-      if adjustedRes.kind != tyExpr:
-        n.sons[0] = fitNode(c, adjustedRes, n.sons[0])
       if n.sons[0].typ == nil: internalError(n.info, "semYield")
 
-      if resultTypeIsInferrable(adjustedRes):
+      if resultTypeIsInferrable(restype):
         let inferred = n.sons[0].typ
-        if restype.kind == tyIter:
-          restype.sons[0] = inferred
-        else:
         iterType.sons[0] = inferred
 
-      semYieldVarResult(c, n, adjustedRes)
+      semYieldVarResult(c, n, restype)
     else:
       localError(n.info, errCannotReturnExpr)
   elif c.p.owner.typ.sons[0] != nil:
@@ -1780,7 +1776,24 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     result = setMs(n, s)
     result.sons[1] = semExpr(c, n.sons[1])
     result.typ = n[1].typ
-  else: result = semDirectOp(c, n, flags)
+  of mPlugin:
+    # semDirectOp with conditional 'afterCallActions':
+    let nOrig = n.copyTree
+    #semLazyOpAux(c, n)
+    result = semOverloadedCallAnalyseEffects(c, n, nOrig, flags)
+    if result == nil:
+      result = errorNode(c, n)
+    else:
+      let callee = result.sons[0].sym
+      if callee.magic == mNone:
+        semFinishOperands(c, result)
+      activate(c, result)
+      fixAbstractType(c, result)
+      analyseIfAddressTakenInCall(c, result)
+      if callee.magic != mNone:
+        result = magicsAfterOverloadResolution(c, result, flags)
+  else:
+    result = semDirectOp(c, n, flags)
 
 proc semWhen(c: PContext, n: PNode, semCheck = true): PNode =
   # If semCheck is set to false, ``when`` will return the verbatim AST of
@@ -1804,6 +1817,7 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode =
       whenNimvm = lookUp(c, exprNode).magic == mNimvm
     elif exprNode.kind == nkSym:
       whenNimvm = exprNode.sym.magic == mNimvm
+    if whenNimvm: n.flags.incl nfLL
 
   for i in countup(0, sonsLen(n) - 1):
     var it = n.sons[i]
@@ -2111,7 +2125,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     var s = lookUp(c, n)
     if c.inTypeClass == 0: semCaptureSym(s, c.p.owner)
     result = semSym(c, n, s, flags)
-    if s.kind in {skProc, skMethod, skConverter}+skIterators:
+    if s.kind in {skProc, skMethod, skConverter, skIterator}:
       #performProcvarCheck(c, n, s)
       result = symChoice(c, n, s, scClosed)
       if result.kind == nkSym:
@@ -2167,7 +2181,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     message(n.info, warnDeprecated, "bind")
     result = semExpr(c, n.sons[0], flags)
   of nkTypeOfExpr, nkTupleTy, nkTupleClassTy, nkRefTy..nkEnumTy, nkStaticTy:
-    var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc, tyIter})
+    var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc})
     result.typ = makeTypeDesc(c, typ)
     #result = symNodeFromType(c, typ, n.info)
   of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit:
@@ -2199,7 +2213,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
           localError(n.info, errUseQualifier, s.name.s)
         elif s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
-      of skProc, skMethod, skConverter, skIterators:
+      of skProc, skMethod, skConverter, skIterator:
         if s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
       else:
@@ -2240,7 +2254,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       var tupexp = semTuplePositionsConstr(c, n, flags)
       if isTupleType(tupexp):
         # reinterpret as type
-        var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc, tyIter})
+        var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc})
         result.typ = makeTypeDesc(c, typ)
       else:
         result = tupexp

compiler/semgnrc.nim

@@ -58,7 +58,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of skProc, skMethod, skIterators, skConverter, skModule:
+  of skProc, skMethod, skIterator, skConverter, skModule:
     result = symChoice(c, n, s, scOpen)
   of skTemplate:
     if macroToExpand(s):
@@ -226,7 +226,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       of skUnknown, skParam:
         # Leave it as an identifier.
         discard
-      of skProc, skMethod, skIterators, skConverter, skModule:
+      of skProc, skMethod, skIterator, skConverter, skModule:
         result.sons[0] = symChoice(c, fn, s, scOption)
         # do not check of 's.magic==mRoof' here because it might be some
         # other '^' but after overload resolution the proper one:

compiler/semmagic.nim

@@ -207,7 +207,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
         result = n.sons[1]
       else:
         result = newNodeIT(nkCall, n.info, getSysType(tyInt))
-        result.add newSymNode(createMagic("-", mSubI), n.info)
+        result.add newSymNode(getSysMagic("-", mSubI), n.info)
         result.add lenExprB
         result.add n.sons[1]
   of mPlugin:

compiler/sempass2.nim

@@ -504,7 +504,8 @@ proc notNilCheck(tracked: PEffects, n: PNode, paramType: PType) =
     if n.kind == nkAddr:
       # addr(x[]) can't be proven, but addr(x) can:
       if not containsNode(n, {nkDerefExpr, nkHiddenDeref}): return
-    elif (n.kind == nkSym and n.sym.kind in routineKinds) or n.kind in procDefs:
+    elif (n.kind == nkSym and n.sym.kind in routineKinds) or
+         n.kind in procDefs+{nkObjConstr}:
       # 'p' is not nil obviously:
       return
     case impliesNotNil(tracked.guards, n)
@@ -704,7 +705,15 @@ proc track(tracked: PEffects, n: PNode) =
     for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i))
     if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}:
       # may not look like an assignment, but it is:
-      initVarViaNew(tracked, n.sons[1])
+      let arg = n.sons[1]
+      initVarViaNew(tracked, arg)
+      if {tfNeedsInit} * arg.typ.lastSon.flags != {}:
+        if a.sym.magic == mNewSeq and n[2].kind in {nkCharLit..nkUInt64Lit} and
+            n[2].intVal == 0:
+          # var s: seq[notnil];  newSeq(s, 0)  is a special case!
+          discard
+        else:
+          message(arg.info, warnProveInit, $arg)
     for i in 0 .. <safeLen(n):
       track(tracked, n.sons[i])
   of nkDotExpr:
@@ -875,7 +884,8 @@ proc trackProc*(s: PSym, body: PNode) =
   var t: TEffects
   initEffects(effects, s, t)
   track(t, body)
-  if not isEmptyType(s.typ.sons[0]) and tfNeedsInit in s.typ.sons[0].flags and
+  if not isEmptyType(s.typ.sons[0]) and
+      {tfNeedsInit, tfNotNil} * s.typ.sons[0].flags != {} and
       s.kind in {skProc, skConverter, skMethod}:
     var res = s.ast.sons[resultPos].sym # get result symbol
     if res.id notin t.init:

compiler/semstmts.nim

@@ -84,7 +84,7 @@ proc performProcvarCheck(c: PContext, n: PNode, s: PSym) =
 proc semProcvarCheck(c: PContext, n: PNode) =
   let n = n.skipConv
   if n.kind == nkSym and n.sym.kind in {skProc, skMethod, skConverter,
-                                        skIterator, skClosureIterator}:
+                                        skIterator}:
     performProcvarCheck(c, n, n.sym)
 
 proc semProc(c: PContext, n: PNode): PNode
@@ -326,11 +326,14 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym =
     incl(result.flags, sfGlobal)
   else:
     result = semIdentWithPragma(c, kind, n, {})
+    if result.owner.kind == skModule:
+      incl(result.flags, sfGlobal)
   suggestSym(n.info, result)
   styleCheckDef(result)
 
 proc checkNilable(v: PSym) =
-  if sfGlobal in v.flags and {tfNotNil, tfNeedsInit} * v.typ.flags != {}:
+  if {sfGlobal, sfImportC} * v.flags == {sfGlobal} and
+      {tfNotNil, tfNeedsInit} * v.typ.flags != {}:
     if v.ast.isNil:
       message(v.info, warnProveInit, v.name.s)
     elif tfNotNil in v.typ.flags and tfNotNil notin v.ast.typ.flags:
@@ -539,7 +542,7 @@ proc symForVar(c: PContext, n: PNode): PSym =
 proc semForVars(c: PContext, n: PNode): PNode =
   result = n
   var length = sonsLen(n)
-  let iterBase = n.sons[length-2].typ.skipTypes({tyIter})
+  let iterBase = n.sons[length-2].typ
   var iter = skipTypes(iterBase, {tyGenericInst})
   # length == 3 means that there is one for loop variable
   # and thus no tuple unpacking:
@@ -593,12 +596,11 @@ proc semFor(c: PContext, n: PNode): PNode =
       result.kind = nkParForStmt
     else:
       result = semForFields(c, n, call.sons[0].sym.magic)
-  elif (isCallExpr and call.sons[0].typ.callConv == ccClosure) or
+  elif isCallExpr and call.sons[0].typ.callConv == ccClosure:
-      call.typ.kind == tyIter:
     # first class iterator:
     result = semForVars(c, n)
   elif not isCallExpr or call.sons[0].kind != nkSym or
-      call.sons[0].sym.kind notin skIterators:
+      call.sons[0].sym.kind != skIterator:
     if length == 3:
       n.sons[length-2] = implicitIterator(c, "items", n.sons[length-2])
     elif length == 4:
@@ -958,15 +960,17 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
   var n = n
 
   let original = n.sons[namePos].sym
-  let s = copySym(original, false)
+  let s = original #copySym(original, false)
-  incl(s.flags, sfFromGeneric)
+  #incl(s.flags, sfFromGeneric)
+  #s.owner = original
 
   n = replaceTypesInBody(c, pt, n, original)
   result = n
   s.ast = result
   n.sons[namePos].sym = s
   n.sons[genericParamsPos] = emptyNode
-  let params = n.typ.n
+  # for LL we need to avoid wrong aliasing
+  let params = copyTree n.typ.n
   n.sons[paramsPos] = params
   s.typ = n.typ
   for i in 1..<params.len:
@@ -974,6 +978,7 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
                               tyFromExpr, tyFieldAccessor}+tyTypeClasses:
       localError(params[i].info, "cannot infer type of parameter: " &
                  params[i].sym.name.s)
+    #params[i].sym.owner = s
   openScope(c)
   pushOwner(s)
   addParams(c, params, skProc)
@@ -1006,7 +1011,8 @@ proc activate(c: PContext, n: PNode) =
       discard
 
 proc maybeAddResult(c: PContext, s: PSym, n: PNode) =
-  if s.typ.sons[0] != nil and s.kind != skIterator:
+  if s.typ.sons[0] != nil and not
+      (s.kind == skIterator and s.typ.callConv != ccClosure):
     addResult(c, s.typ.sons[0], n.info, s.kind)
     addResultNode(c, n)
 
@@ -1143,13 +1149,15 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   if tfTriggersCompileTime in s.typ.flags: incl(s.flags, sfCompileTime)
   if n.sons[patternPos].kind != nkEmpty:
     n.sons[patternPos] = semPattern(c, n.sons[patternPos])
-  if s.kind in skIterators:
+  if s.kind == skIterator:
     s.typ.flags.incl(tfIterator)
 
   var proto = searchForProc(c, oldScope, s)
   if proto == nil:
-    if s.kind == skClosureIterator: s.typ.callConv = ccClosure
+    if s.kind == skIterator and s.typ.callConv == ccClosure:
-    else: s.typ.callConv = lastOptionEntry(c).defaultCC
+      discard
+    else:
+      s.typ.callConv = lastOptionEntry(c).defaultCC
     # add it here, so that recursive procs are possible:
     if sfGenSym in s.flags: discard
     elif kind in OverloadableSyms:
@@ -1209,7 +1217,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         n.sons[bodyPos] = transformBody(c.module, semBody, s)
       popProcCon(c)
     else:
-      if s.typ.sons[0] != nil and kind notin skIterators:
+      if s.typ.sons[0] != nil and kind != skIterator:
         addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info))
       openScope(c)
       n.sons[bodyPos] = semGenericStmt(c, n.sons[bodyPos])
@@ -1230,9 +1238,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   if n.sons[patternPos].kind != nkEmpty:
     c.patterns.add(s)
   if isAnon: result.typ = s.typ
-  if isTopLevel(c) and s.kind != skClosureIterator and
+  if isTopLevel(c) and s.kind != skIterator and
       s.typ.callConv == ccClosure:
-    message(s.info, warnDeprecated, "top level '.closure' calling convention")
+    localError(s.info, "'.closure' calling convention for top level routines is invalid")
 
 proc determineType(c: PContext, s: PSym) =
   if s.typ != nil: return
@@ -1240,15 +1248,12 @@ proc determineType(c: PContext, s: PSym) =
   discard semProcAux(c, s.ast, s.kind, {}, stepDetermineType)
 
 proc semIterator(c: PContext, n: PNode): PNode =
-  let kind = if hasPragma(n[pragmasPos], wClosure) or
-                n[namePos].kind == nkEmpty: skClosureIterator
-             else: skIterator
   # gensym'ed iterator?
   if n[namePos].kind == nkSym:
     # gensym'ed iterators might need to become closure iterators:
     n[namePos].sym.owner = getCurrOwner()
-    n[namePos].sym.kind = kind
+    n[namePos].sym.kind = skIterator
-  result = semProcAux(c, n, kind, iteratorPragmas)
+  result = semProcAux(c, n, skIterator, iteratorPragmas)
   var s = result.sons[namePos].sym
   var t = s.typ
   if t.sons[0] == nil and s.typ.callConv != ccClosure:

compiler/semtempl.nim

@@ -228,10 +228,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym): PNode =
   of skParam:
     result = n
   of skType:
-    if (s.typ != nil) and (s.typ.kind != tyGenericParam):
     result = newSymNodeTypeDesc(s, n.info)
-    else:
-      result = n
   else:
     result = newSymNode(s, n.info)
 
@@ -456,9 +453,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   of nkMethodDef:
     result = semRoutineInTemplBody(c, n, skMethod)
   of nkIteratorDef:
-    let kind = if hasPragma(n[pragmasPos], wClosure): skClosureIterator
+    result = semRoutineInTemplBody(c, n, skIterator)
-               else: skIterator
-    result = semRoutineInTemplBody(c, n, kind)
   of nkTemplateDef:
     result = semRoutineInTemplBody(c, n, skTemplate)
   of nkMacroDef:

compiler/semtypes.nim

@@ -135,13 +135,19 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType =
     checkMinSonsLen(n, 1)
     var base = semTypeNode(c, n.lastSon, nil)
     result = newOrPrevType(kind, prev, c)
+    var isNilable = false
     # check every except the last is an object:
     for i in isCall .. n.len-2:
-      let region = semTypeNode(c, n[i], nil)
+      let ni = n[i]
+      if ni.kind == nkNilLit:
+        isNilable = true
+      else:
+        let region = semTypeNode(c, ni, nil)
         if region.skipTypes({tyGenericInst}).kind notin {tyError, tyObject}:
           message n[i].info, errGenerated, "region needs to be an object type"
         addSonSkipIntLit(result, region)
     addSonSkipIntLit(result, base)
+    #if not isNilable: result.flags.incl tfNotNil
 
 proc semVarType(c: PContext, n: PNode, prev: PType): PType =
   if sonsLen(n) == 1:
@@ -826,15 +832,6 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
     result = newTypeWithSons(c, tyCompositeTypeClass, @[paramType, result])
     result = addImplicitGeneric(result)
 
-  of tyIter:
-    if paramType.callConv == ccInline:
-      if procKind notin {skTemplate, skMacro, skIterator}:
-        localError(info, errInlineIteratorsAsProcParams)
-      if paramType.len == 1:
-        let lifted = liftingWalk(paramType.base)
-        if lifted != nil: paramType.sons[0] = lifted
-      result = addImplicitGeneric(paramType)
-
   of tyGenericInst:
     if paramType.lastSon.kind == tyUserTypeClass:
       var cp = copyType(paramType, getCurrOwner(), false)
@@ -865,11 +862,6 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
   of tyUserTypeClass, tyBuiltInTypeClass, tyAnd, tyOr, tyNot:
     result = addImplicitGeneric(copyType(paramType, getCurrOwner(), true))
 
-  of tyExpr:
-    if procKind notin {skMacro, skTemplate}:
-      result = addImplicitGeneric(newTypeS(tyAnything, c))
-      #result = addImplicitGenericImpl(newTypeS(tyGenericParam, c), nil)
-
   of tyGenericParam:
     markUsed(info, paramType.sym)
     styleCheckUse(info, paramType.sym)
@@ -968,10 +960,6 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
   var r: PType
   if n.sons[0].kind != nkEmpty:
     r = semTypeNode(c, n.sons[0], nil)
-  elif kind == skIterator:
-    # XXX This is special magic we should likely get rid of
-    r = newTypeS(tyExpr, c)
-    message(n.info, warnDeprecated, "implicit return type for 'iterator'")
 
   if r != nil:
     # turn explicit 'void' return type into 'nil' because the rest of the
@@ -996,7 +984,8 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
           # see tchainediterators
           # in cases like iterator foo(it: iterator): type(it)
           # we don't need to change the return type to iter[T]
-          if not r.isInlineIterator: r = newTypeWithSons(c, tyIter, @[r])
+          result.flags.incl tfIterator
+          # XXX Would be nice if we could get rid of this
       result.sons[0] = r
       result.n.typ = r
 
@@ -1151,7 +1140,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     checkSonsLen(n, 1)
     let typExpr = semExprWithType(c, n.sons[0], {efInTypeof})
-    result = typExpr.typ.skipTypes({tyIter})
+    result = typExpr.typ
   of nkPar:
     if sonsLen(n) == 1: result = semTypeNode(c, n.sons[0], prev)
     else:
@@ -1169,6 +1158,14 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       result = semTypeNode(c, b, prev)
     elif ident != nil and ident.id == ord(wDotDot):
       result = semRangeAux(c, n, prev)
+    elif n[0].kind == nkNilLit and n.len == 2:
+      result = semTypeNode(c, n.sons[1], prev)
+      if result.skipTypes({tyGenericInst}).kind in NilableTypes+GenericTypes:
+        if tfNotNil in result.flags:
+          result = freshType(result, prev)
+          result.flags.excl(tfNotNil)
+      else:
+        localError(n.info, errGenerated, "invalid type")
     elif n[0].kind notin nkIdentKinds:
       result = semTypeExpr(c, n)
     else:
@@ -1209,7 +1206,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       elif op.id == ord(wType):
         checkSonsLen(n, 2)
         let typExpr = semExprWithType(c, n.sons[1], {efInTypeof})
-        result = typExpr.typ.skipTypes({tyIter})
+        result = typExpr.typ
       else:
         result = semTypeExpr(c, n)
   of nkWhenStmt:
@@ -1290,14 +1287,16 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     result.flags.incl tfHasStatic
   of nkIteratorTy:
     if n.sonsLen == 0:
-      result = newConstraint(c, tyIter)
+      result = newTypeS(tyBuiltInTypeClass, c)
+      let child = newTypeS(tyProc, c)
+      child.flags.incl tfIterator
+      result.addSonSkipIntLit(child)
     else:
-      result = semProcTypeWithScope(c, n, prev, skClosureIterator)
+      result = semProcTypeWithScope(c, n, prev, skIterator)
+      result.flags.incl(tfIterator)
       if n.lastSon.kind == nkPragma and hasPragma(n.lastSon, wInline):
-        result.kind = tyIter
         result.callConv = ccInline
       else:
-        result.flags.incl(tfIterator)
         result.callConv = ccClosure
   of nkProcTy:
     if n.sonsLen == 0:

compiler/semtypinst.nim

@@ -77,7 +77,7 @@ proc cacheTypeInst*(inst: PType) =
   #      update the refcount
   let gt = inst.sons[0]
   let t = if gt.kind == tyGenericBody: gt.lastSon else: gt
-  if t.kind in {tyStatic, tyGenericParam, tyIter} + tyTypeClasses:
+  if t.kind in {tyStatic, tyGenericParam} + tyTypeClasses:
     return
   gt.sym.typeInstCache.safeAdd(inst)
 
@@ -390,7 +390,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
   result = t
   if t == nil: return
 
-  if t.kind in {tyStatic, tyGenericParam, tyIter} + tyTypeClasses:
+  if t.kind in {tyStatic, tyGenericParam} + tyTypeClasses:
     let lookup = PType(idTableGet(cl.typeMap, t))
     if lookup != nil: return lookup
 

compiler/sigmatch.nim

@@ -167,12 +167,12 @@ proc sumGeneric(t: PType): int =
       t = t.lastSon
       if t.kind == tyEmpty: break
       inc result
-    of tyGenericInvocation, tyTuple:
+    of tyGenericInvocation, tyTuple, tyProc:
       result += ord(t.kind == tyGenericInvocation)
       for i in 0 .. <t.len: result += t.sons[i].sumGeneric
       break
     of tyGenericParam, tyExpr, tyStatic, tyStmt: break
-    of tyBool, tyChar, tyEnum, tyObject, tyProc, tyPointer,
+    of tyBool, tyChar, tyEnum, tyObject, tyPointer,
         tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
         tyUInt..tyUInt64:
       return isvar
@@ -1256,10 +1256,6 @@ proc localConvMatch(c: PContext, m: var TCandidate, f, a: PType,
       result.typ = getInstantiatedType(c, arg, m, base(f))
     m.baseTypeMatch = true
 
-proc isInlineIterator*(t: PType): bool =
-  result = t.kind == tyIter or
-          (t.kind == tyBuiltInTypeClass and t.base.kind == tyIter)
-
 proc incMatches(m: var TCandidate; r: TTypeRelation; convMatch = 1) =
   case r
   of isConvertible, isIntConv: inc(m.convMatches, convMatch)
@@ -1323,13 +1319,6 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
     else:
       return argSemantized # argOrig
 
-  if r != isNone and f.isInlineIterator:
-    var inlined = newTypeS(tyStatic, c)
-    inlined.sons = @[argType]
-    inlined.n = argSemantized
-    put(m.bindings, f, inlined)
-    return argSemantized
-
   # If r == isBothMetaConvertible then we rerun typeRel.
   # bothMetaCounter is for safety to avoid any infinite loop,
   #  I don't have any example when it is needed.
@@ -1453,7 +1442,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     z.calleeSym = m.calleeSym
     var best = -1
     for i in countup(0, sonsLen(arg) - 1):
-      if arg.sons[i].sym.kind in {skProc, skMethod, skConverter}+skIterators:
+      if arg.sons[i].sym.kind in {skProc, skMethod, skConverter, skIterator}:
         copyCandidate(z, m)
         z.callee = arg.sons[i].typ
         z.calleeSym = arg.sons[i].sym
@@ -1646,6 +1635,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
           if arg != nil and m.baseTypeMatch and container != nil:
             addSon(container, arg)
             incrIndexType(container.typ)
+            checkConstraint(n.sons[a])
           else:
             m.state = csNoMatch
             return

compiler/transf.nim

@@ -45,7 +45,7 @@ type
     inlining: int            # > 0 if we are in inlining context (copy vars)
     nestedProcs: int         # > 0 if we are in a nested proc
     contSyms, breakSyms: seq[PSym]  # to transform 'continue' and 'break'
-    deferDetected: bool
+    deferDetected, tooEarly: bool
   PTransf = ref TTransfContext
 
 proc newTransNode(a: PNode): PTransNode {.inline.} =
@@ -93,10 +93,15 @@ proc getCurrOwner(c: PTransf): PSym =
   if c.transCon != nil: result = c.transCon.owner
   else: result = c.module
 
-proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PSym =
+proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode =
-  result = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info)
+  let r = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info)
-  result.typ = skipTypes(typ, {tyGenericInst})
+  r.typ = skipTypes(typ, {tyGenericInst})
-  incl(result.flags, sfFromGeneric)
+  incl(r.flags, sfFromGeneric)
+  let owner = getCurrOwner(c)
+  if owner.isIterator and not c.tooEarly:
+    result = freshVarForClosureIter(r, owner)
+  else:
+    result = newSymNode(r)
 
 proc transform(c: PTransf, n: PNode): PTransNode
 
@@ -111,13 +116,22 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode =
   result[1] = ri
 
 proc transformSymAux(c: PTransf, n: PNode): PNode =
-  #if n.sym.kind == skClosureIterator:
+  let s = n.sym
-  #  return liftIterSym(n)
+  if s.typ != nil and s.typ.callConv == ccClosure:
+    if s.kind == skIterator:
+      if c.tooEarly: return n
+      else: return liftIterSym(n, getCurrOwner(c))
+    elif s.kind in {skProc, skConverter, skMethod} and not c.tooEarly:
+      # top level .closure procs are still somewhat supported for 'Nake':
+      return makeClosure(s, nil, n.info)
+  #elif n.sym.kind in {skVar, skLet} and n.sym.typ.callConv == ccClosure:
+  #  echo n.info, " come heer for ", c.tooEarly
+  #  if not c.tooEarly:
   var b: PNode
   var tc = c.transCon
-  if sfBorrow in n.sym.flags and n.sym.kind in routineKinds:
+  if sfBorrow in s.flags and s.kind in routineKinds:
     # simply exchange the symbol:
-    b = n.sym.getBody
+    b = s.getBody
     if b.kind != nkSym: internalError(n.info, "wrong AST for borrowed symbol")
     b = newSymNode(b.sym)
     b.info = n.info
@@ -132,6 +146,16 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
 proc transformSym(c: PTransf, n: PNode): PTransNode =
   result = PTransNode(transformSymAux(c, n))
 
+proc freshVar(c: PTransf; v: PSym): PNode =
+  let owner = getCurrOwner(c)
+  if owner.isIterator and not c.tooEarly:
+    result = freshVarForClosureIter(v, owner)
+  else:
+    var newVar = copySym(v)
+    incl(newVar.flags, sfFromGeneric)
+    newVar.owner = owner
+    result = newSymNode(newVar)
+
 proc transformVarSection(c: PTransf, v: PNode): PTransNode =
   result = newTransNode(v)
   for i in countup(0, sonsLen(v)-1):
@@ -141,35 +165,30 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode =
     elif it.kind == nkIdentDefs:
       if it.sons[0].kind == nkSym:
         internalAssert(it.len == 3)
-        var newVar = copySym(it.sons[0].sym)
+        let x = freshVar(c, it.sons[0].sym)
-        incl(newVar.flags, sfFromGeneric)
+        idNodeTablePut(c.transCon.mapping, it.sons[0].sym, x)
-        # fixes a strange bug for rodgen:
-        #include(it.sons[0].sym.flags, sfFromGeneric);
-        newVar.owner = getCurrOwner(c)
-        idNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
         var defs = newTransNode(nkIdentDefs, it.info, 3)
         if importantComments():
           # keep documentation information:
           PNode(defs).comment = it.comment
-        defs[0] = newSymNode(newVar).PTransNode
+        defs[0] = x.PTransNode
         defs[1] = it.sons[1].PTransNode
         defs[2] = transform(c, it.sons[2])
-        newVar.ast = defs[2].PNode
+        if x.kind == nkSym: x.sym.ast = defs[2].PNode
         result[i] = defs
       else:
-        # has been transformed into 'param.x' for closure iterators, so keep it:
+        # has been transformed into 'param.x' for closure iterators, so just
-        result[i] = PTransNode(it)
+        # transform it:
+        result[i] = transform(c, it)
     else:
       if it.kind != nkVarTuple:
         internalError(it.info, "transformVarSection: not nkVarTuple")
       var L = sonsLen(it)
       var defs = newTransNode(it.kind, it.info, L)
       for j in countup(0, L-3):
-        var newVar = copySym(it.sons[j].sym)
+        let x = freshVar(c, it.sons[j].sym)
-        incl(newVar.flags, sfFromGeneric)
+        idNodeTablePut(c.transCon.mapping, it.sons[j].sym, x)
-        newVar.owner = getCurrOwner(c)
+        defs[j] = x.PTransNode
-        idNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar))
-        defs[j] = newSymNode(newVar).PTransNode
       assert(it.sons[L-2].kind == nkEmpty)
       defs[L-2] = ast.emptyNode.PTransNode
       defs[L-1] = transform(c, it.sons[L-1])
@@ -294,6 +313,14 @@ proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode =
     result = PTransNode(n)
   of nkVarSection, nkLetSection:
     result = transformVarSection(c, n)
+  of nkClosure:
+    # it can happen that for-loop-inlining produced a fresh
+    # set of variables, including some computed environment
+    # (bug #2604). We need to patch this environment here too:
+    let a = n[1]
+    if a.kind == nkSym:
+      n.sons[1] = transformSymAux(c, a)
+    return PTransNode(n)
   else:
     result = newTransNode(n)
     for i in countup(0, sonsLen(n)-1):
@@ -348,6 +375,22 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode =
       # addr ( deref ( x )) --> x
       result = PTransNode(n.sons[0].sons[0])
 
+proc generateThunk(prc: PNode, dest: PType): PNode =
+  ## Converts 'prc' into '(thunk, nil)' so that it's compatible with
+  ## a closure.
+
+  # we cannot generate a proper thunk here for GC-safety reasons
+  # (see internal documentation):
+  if gCmd == cmdCompileToJS: return prc
+  result = newNodeIT(nkClosure, prc.info, dest)
+  var conv = newNodeIT(nkHiddenSubConv, prc.info, dest)
+  conv.add(emptyNode)
+  conv.add(prc)
+  if prc.kind == nkClosure:
+    internalError(prc.info, "closure to closure created")
+  result.add(conv)
+  result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil)))
+
 proc transformConv(c: PTransf, n: PNode): PTransNode =
   # numeric types need range checks:
   var dest = skipTypes(n.typ, abstractVarRange)
@@ -428,6 +471,10 @@ proc transformConv(c: PTransf, n: PNode): PTransNode =
   of tyGenericParam, tyOrdinal:
     result = transform(c, n.sons[1])
     # happens sometimes for generated assignments, etc.
+  of tyProc:
+    result = transformSons(c, n)
+    if dest.callConv == ccClosure and source.callConv == ccDefault:
+      result = generateThunk(result[1].PNode, dest).PTransNode
   else:
     result = transformSons(c, n)
 
@@ -478,11 +525,14 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
     result[1] = newNode(nkEmpty).PTransNode
     return result
   c.breakSyms.add(labl)
-  if call.typ.kind != tyIter and
+  if call.kind notin nkCallKinds or call.sons[0].kind != nkSym or
-    (call.kind notin nkCallKinds or call.sons[0].kind != nkSym or
+      call.sons[0].typ.callConv == ccClosure:
-      call.sons[0].sym.kind != skIterator):
     n.sons[length-1] = transformLoopBody(c, n.sons[length-1]).PNode
-    result[1] = lambdalifting.liftForLoop(n).PTransNode
+    if not c.tooEarly:
+      n.sons[length-2] = transform(c, n.sons[length-2]).PNode
+      result[1] = lambdalifting.liftForLoop(n, getCurrOwner(c)).PTransNode
+    else:
+      result[1] = newNode(nkEmpty).PTransNode
     discard c.breakSyms.pop
     return result
 
@@ -512,16 +562,15 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
   for i in countup(1, sonsLen(call) - 1):
     var arg = transform(c, call.sons[i]).PNode
     var formal = skipTypes(iter.typ, abstractInst).n.sons[i].sym
-    if arg.typ.kind == tyIter: continue
     case putArgInto(arg, formal.typ)
     of paDirectMapping:
       idNodeTablePut(newC.mapping, formal, arg)
     of paFastAsgn:
       # generate a temporary and produce an assignment statement:
       var temp = newTemp(c, formal.typ, formal.info)
-      addVar(v, newSymNode(temp))
+      addVar(v, temp)
-      add(stmtList, newAsgnStmt(c, newSymNode(temp), arg.PTransNode))
+      add(stmtList, newAsgnStmt(c, temp, arg.PTransNode))
-      idNodeTablePut(newC.mapping, formal, newSymNode(temp))
+      idNodeTablePut(newC.mapping, formal, temp)
     of paVarAsgn:
       assert(skipTypes(formal.typ, abstractInst).kind == tyVar)
       idNodeTablePut(newC.mapping, formal, arg)
@@ -702,17 +751,12 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     result = PTransNode(n)
   of nkBracketExpr: result = transformArrayAccess(c, n)
   of procDefs:
-    when false:
-      if n.sons[genericParamsPos].kind == nkEmpty:
     var s = n.sons[namePos].sym
-        n.sons[bodyPos] = PNode(transform(c, s.getBody))
+    if n.typ != nil and s.typ.callConv == ccClosure:
-        if s.ast.sons[bodyPos] != n.sons[bodyPos]:
+      result = transformSym(c, n.sons[namePos])
-          # somehow this can happen ... :-/
+      # use the same node as before if still a symbol:
-          s.ast.sons[bodyPos] = n.sons[bodyPos]
+      if result.PNode.kind == nkSym: result = PTransNode(n)
-        #n.sons[bodyPos] = liftLambdas(s, n)
+    else:
-        #if n.kind == nkMethodDef: methodDef(s, false)
-    #if n.kind == nkIteratorDef and n.typ != nil:
-    #  return liftIterSym(n.sons[namePos]).PTransNode
       result = PTransNode(n)
   of nkMacroDef:
     # XXX no proper closure support yet:
@@ -750,7 +794,7 @@ proc transform(c: PTransf, n: PNode): PTransNode =
         result = newTransNode(nkCommentStmt, n.info, 0)
       tryStmt.addSon(deferPart)
       # disable the original 'defer' statement:
-      n.kind = nkCommentStmt
+      n.kind = nkEmpty
   of nkContinueStmt:
     result = PTransNode(newNodeI(nkBreakStmt, n.info))
     var labl = c.contSyms[c.contSyms.high]
@@ -796,7 +840,14 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     # XXX comment handling really sucks:
     if importantComments():
       PNode(result).comment = n.comment
-  of nkClosure: return PTransNode(n)
+  of nkClosure:
+    # it can happen that for-loop-inlining produced a fresh
+    # set of variables, including some computed environment
+    # (bug #2604). We need to patch this environment here too:
+    let a = n[1]
+    if a.kind == nkSym:
+      n.sons[1] = transformSymAux(c, a)
+    return PTransNode(n)
   else:
     result = transformSons(c, n)
   when false:
@@ -868,11 +919,11 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
     result = n
   else:
     var c = openTransf(module, "")
-    result = processTransf(c, n, prc)
+    result = liftLambdas(prc, n, c.tooEarly)
+    #result = n
+    result = processTransf(c, result, prc)
     liftDefer(c, result)
-    result = liftLambdas(prc, result)
+    #result = liftLambdas(prc, result)
-    #if prc.kind == skClosureIterator:
-    #  result = lambdalifting.liftIterator(prc, result)
     incl(result.flags, nfTransf)
     when useEffectSystem: trackProc(prc, result)
     #if prc.name.s == "testbody":
@@ -885,9 +936,11 @@ proc transformStmt*(module: PSym, n: PNode): PNode =
     var c = openTransf(module, "")
     result = processTransf(c, n, module)
     liftDefer(c, result)
-    result = liftLambdasForTopLevel(module, result)
+    #result = liftLambdasForTopLevel(module, result)
     incl(result.flags, nfTransf)
     when useEffectSystem: trackTopLevelStmt(module, result)
+    #if n.info ?? "temp.nim":
+    #  echo renderTree(result, {renderIds})
 
 proc transformExpr*(module: PSym, n: PNode): PNode =
   if nfTransf in n.flags:

compiler/types.nim

@@ -1061,7 +1061,8 @@ proc typeAllowedNode(marker: var IntSet, n: PNode, kind: TSymKind,
       else:
         for i in countup(0, sonsLen(n) - 1):
           let it = n.sons[i]
-          if it.kind == nkRecCase and kind == skConst: return n.typ
+          if it.kind == nkRecCase and kind in {skProc, skConst}:
+            return n.typ
           result = typeAllowedNode(marker, it, kind, flags)
           if result != nil: break
 
@@ -1076,7 +1077,7 @@ proc matchType*(a: PType, pattern: openArray[tuple[k:TTypeKind, i:int]],
 
 proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
                     flags: TTypeAllowedFlags = {}): PType =
-  assert(kind in {skVar, skLet, skConst, skParam, skResult})
+  assert(kind in {skVar, skLet, skConst, skProc, skParam, skResult})
   # if we have already checked the type, return true, because we stop the
   # evaluation if something is wrong:
   result = nil
@@ -1085,7 +1086,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
   var t = skipTypes(typ, abstractInst-{tyTypeDesc})
   case t.kind
   of tyVar:
-    if kind == skConst: return t
+    if kind in {skProc, skConst}: return t
     var t2 = skipTypes(t.sons[0], abstractInst-{tyTypeDesc})
     case t2.kind
     of tyVar:
@@ -1097,6 +1098,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
       if kind notin {skParam, skResult}: result = t
       else: result = typeAllowedAux(marker, t2, kind, flags)
   of tyProc:
+    if kind == skConst and t.callConv == ccClosure: return t
     for i in countup(1, sonsLen(t) - 1):
       result = typeAllowedAux(marker, t.sons[i], skParam, flags)
       if result != nil: break
@@ -1144,7 +1146,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
       result = typeAllowedAux(marker, t.sons[i], kind, flags)
       if result != nil: break
   of tyObject, tyTuple:
-    if kind == skConst and t.kind == tyObject and t.sons[0] != nil: return t
+    if kind in {skProc, skConst} and
+        t.kind == tyObject and t.sons[0] != nil: return t
     let flags = flags+{taField}
     for i in countup(0, sonsLen(t) - 1):
       result = typeAllowedAux(marker, t.sons[i], kind, flags)

compiler/vm.nim

@@ -255,9 +255,12 @@ proc cleanUpOnException(c: PCtx; tos: PStackFrame):
       nextExceptOrFinally = pc2 + c.code[pc2].regBx - wordExcess
       inc pc2
     while c.code[pc2].opcode == opcExcept:
-      let exceptType = c.types[c.code[pc2].regBx-wordExcess].skipTypes(
+      let excIndex = c.code[pc2].regBx-wordExcess
+      let exceptType = if excIndex > 0: c.types[excIndex].skipTypes(
                           abstractPtrs)
-      if inheritanceDiff(exceptType, raisedType) <= 0:
+                       else: nil
+      #echo typeToString(exceptType), " ", typeToString(raisedType)
+      if exceptType.isNil or inheritanceDiff(exceptType, raisedType) <= 0:
         # mark exception as handled but keep it in B for
         # the getCurrentException() builtin:
         c.currentExceptionB = c.currentExceptionA
@@ -356,7 +359,14 @@ proc opConv*(dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool =
       of tyFloat..tyFloat64:
         dest.intVal = int(src.floatVal)
       else:
-        dest.intVal = src.intVal and ((1 shl (desttyp.size*8))-1)
+        let srcDist = (sizeof(src.intVal) - srctyp.size) * 8
+        let destDist = (sizeof(dest.intVal) - desttyp.size) * 8
+        when system.cpuEndian == bigEndian:
+          dest.intVal = (src.intVal shr srcDist) shl srcDist
+          dest.intVal = (dest.intVal shr destDist) shl destDist
+        else:
+          dest.intVal = (src.intVal shl srcDist) shr srcDist
+          dest.intVal = (dest.intVal shl destDist) shr destDist
     of tyFloat..tyFloat64:
       if dest.kind != rkFloat:
         myreset(dest); dest.kind = rkFloat
@@ -608,7 +618,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       addSon(regs[ra].node, r.copyTree)
     of opcExcl:
       decodeB(rkNode)
-      var b = newNodeIT(nkCurly, regs[rb].node.info, regs[rb].node.typ)
+      var b = newNodeIT(nkCurly, regs[ra].node.info, regs[ra].node.typ)
       addSon(b, regs[rb].regToNode)
       var r = diffSets(regs[ra].node, b)
       discardSons(regs[ra].node)
@@ -1190,6 +1200,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       createStr regs[ra]
       let a = regs[rb].node
       if a.kind in {nkStrLit..nkTripleStrLit}: regs[ra].node.strVal = a.strVal
+      elif a.kind == nkCommentStmt: regs[ra].node.strVal = a.comment
       else: stackTrace(c, tos, pc, errFieldXNotFound, "strVal")
     of opcSlurp:
       decodeB(rkNode)

compiler/vmdeps.nim

@@ -70,7 +70,7 @@ proc atomicTypeX(name: string; t: PType; info: TLineInfo): PNode =
 proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode
 
 proc mapTypeToBracket(name: string; t: PType; info: TLineInfo): PNode =
-  result = newNodeIT(nkBracketExpr, info, t)
+  result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
   result.add atomicTypeX(name, t, info)
   for i in 0 .. < t.len:
     if t.sons[i] == nil:
@@ -92,19 +92,19 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
   of tyStmt: result = atomicType("stmt")
   of tyEmpty: result = atomicType"void"
   of tyArrayConstr, tyArray:
-    result = newNodeIT(nkBracketExpr, info, t)
+    result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     result.add atomicType("array")
     result.add mapTypeToAst(t.sons[0], info)
     result.add mapTypeToAst(t.sons[1], info)
   of tyTypeDesc:
     if t.base != nil:
-      result = newNodeIT(nkBracketExpr, info, t)
+      result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
       result.add atomicType("typeDesc")
       result.add mapTypeToAst(t.base, info)
     else:
       result = atomicType"typeDesc"
   of tyGenericInvocation:
-    result = newNodeIT(nkBracketExpr, info, t)
+    result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     for i in 0 .. < t.len:
       result.add mapTypeToAst(t.sons[i], info)
   of tyGenericInst, tyGenericBody, tyOrdinal, tyUserTypeClassInst:
@@ -117,7 +117,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
   of tyGenericParam, tyForward: result = atomicType(t.sym.name.s)
   of tyObject:
     if allowRecursion:
-      result = newNodeIT(nkObjectTy, info, t)
+      result = newNodeIT(nkObjectTy, if t.n.isNil: info else: t.n.info, t)
       if t.sons[0] == nil:
         result.add ast.emptyNode
       else:
@@ -126,7 +126,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
     else:
       result = atomicType(t.sym.name.s)
   of tyEnum:
-    result = newNodeIT(nkEnumTy, info, t)
+    result = newNodeIT(nkEnumTy, if t.n.isNil: info else: t.n.info, t)
     result.add copyTree(t.n)
   of tyTuple: result = mapTypeToBracket("tuple", t, info)
   of tySet: result = mapTypeToBracket("set", t, info)
@@ -137,7 +137,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
   of tyProc: result = mapTypeToBracket("proc", t, info)
   of tyOpenArray: result = mapTypeToBracket("openArray", t, info)
   of tyRange:
-    result = newNodeIT(nkBracketExpr, info, t)
+    result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     result.add atomicType("range")
     result.add t.n.sons[0].copyTree
     result.add t.n.sons[1].copyTree
@@ -174,7 +174,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
   of tyNot: result = mapTypeToBracket("not", t, info)
   of tyAnything: result = atomicType"anything"
   of tyStatic, tyFromExpr, tyFieldAccessor:
-    result = newNodeIT(nkBracketExpr, info, t)
+    result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     result.add atomicType("static")
     if t.n != nil:
       result.add t.n.copyTree

compiler/vmgen.nim

@@ -1209,7 +1209,7 @@ proc checkCanEval(c: PCtx; n: PNode) =
       not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl:
     cannotEval(n)
   elif s.kind in {skProc, skConverter, skMethod,
-                  skIterator, skClosureIterator} and sfForward in s.flags:
+                  skIterator} and sfForward in s.flags:
     cannotEval(n)
 
 proc isTemp(c: PCtx; dest: TDest): bool =
@@ -1604,7 +1604,8 @@ proc matches(s: PSym; x: string): bool =
   var s = s
   var L = y.len-1
   while L >= 0:
-    if s == nil or y[L].cmpIgnoreStyle(s.name.s) != 0: return false
+    if s == nil or (y[L].cmpIgnoreStyle(s.name.s) != 0 and y[L] != "*"):
+      return false
     s = s.owner
     dec L
   result = true
@@ -1613,7 +1614,8 @@ proc matches(s: PSym; y: varargs[string]): bool =
   var s = s
   var L = y.len-1
   while L >= 0:
-    if s == nil or y[L].cmpIgnoreStyle(s.name.s) != 0: return false
+    if s == nil or (y[L].cmpIgnoreStyle(s.name.s) != 0 and y[L] != "*"):
+      return false
     s = if sfFromGeneric in s.flags: s.owner.owner else: s.owner
     dec L
   result = true
@@ -1636,7 +1638,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     case s.kind
     of skVar, skForVar, skTemp, skLet, skParam, skResult:
       genRdVar(c, n, dest, flags)
-    of skProc, skConverter, skMacro, skTemplate, skMethod, skIterators:
+    of skProc, skConverter, skMacro, skTemplate, skMethod, skIterator:
       # 'skTemplate' is only allowed for 'getAst' support:
       if procIsCallback(c, s): discard
       elif sfImportc in s.flags: c.importcSym(n.info, s)

compiler/vmhooks.nim

@@ -55,9 +55,16 @@ template getX(k, field) {.immediate, dirty.} =
   result = s[i+a.rb+1].field
 
 proc getInt*(a: VmArgs; i: Natural): BiggestInt = getX(rkInt, intVal)
+proc getBool*(a: VmArgs; i: Natural): bool = getInt(a, i) != 0
 proc getFloat*(a: VmArgs; i: Natural): BiggestFloat = getX(rkFloat, floatVal)
 proc getString*(a: VmArgs; i: Natural): string =
   doAssert i < a.rc-1
   let s = cast[seq[TFullReg]](a.slots)
   doAssert s[i+a.rb+1].kind == rkNode
   result = s[i+a.rb+1].node.strVal
+
+proc getNode*(a: VmArgs; i: Natural): PNode =
+  doAssert i < a.rc-1
+  let s = cast[seq[TFullReg]](a.slots)
+  doAssert s[i+a.rb+1].kind == rkNode
+  result = s[i+a.rb+1].node

compiler/vmops.nim

@@ -13,7 +13,7 @@ from math import sqrt, ln, log10, log2, exp, round, arccos, arcsin,
   arctan, arctan2, cos, cosh, hypot, sinh, sin, tan, tanh, pow, trunc,
   floor, ceil, fmod
 
-from os import getEnv, existsEnv, dirExists, fileExists
+from os import getEnv, existsEnv, dirExists, fileExists, walkDir
 
 template mathop(op) {.immediate, dirty.} =
   registerCallback(c, "stdlib.math." & astToStr(op), `op Wrapper`)
@@ -48,6 +48,12 @@ proc getCurrentExceptionMsgWrapper(a: VmArgs) {.nimcall.} =
   setResult(a, if a.currentException.isNil: ""
                else: a.currentException.sons[3].skipColon.strVal)
 
+proc staticWalkDirImpl(path: string, relative: bool): PNode =
+  result = newNode(nkBracket)
+  for k, f in walkDir(path, relative):
+    result.add newTree(nkPar, newIntNode(nkIntLit, k.ord),
+                              newStrNode(nkStrLit, f))
+
 proc registerAdditionalOps*(c: PCtx) =
   wrap1f(sqrt)
   wrap1f(ln)
@@ -78,3 +84,5 @@ proc registerAdditionalOps*(c: PCtx) =
   wrap1s(fileExists)
   wrap2svoid(writeFile)
   systemop getCurrentExceptionMsg
+  registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} =
+    setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1)))

config/nimdoc.cfg

@@ -102,8 +102,8 @@ doc.file = """<?xml version="1.0" encoding="utf-8" ?>
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 
 <!-- Google fonts -->
-<link href='http://fonts.googleapis.com/css?family=Raleway:400,600,900' rel='stylesheet' type='text/css'>
+<link href='http://fonts.googleapis.com/css?family=Raleway:400,600,900' rel='stylesheet' type='text/css'/>
-<link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'>
+<link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 
 <!-- CSS -->
 <title>$title</title>
@@ -1246,7 +1246,7 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator
     <div class="row">
       <div class="twelve-columns footer">
         <span class="nim-sprite"></span>
-        <br>
+        <br/>
         <small>Made with Nim. Generated: $date $time UTC</small>
       </div>
     </div>

doc/astspec.txt

@@ -924,9 +924,11 @@ AST:
 
 .. code-block:: nim
   nnkLetSection(
-    nnkIdentDefs(!"v"),
+    nnkIdentDefs(
-    nnkEmpty(), # for the type
+      nnkIdent(!"a"),
-    nnkIntLit(3)
+      nnkEmpty(), # or nnkIdent(...) for the type
+      nnkIntLit(3),
+    )
   )
 
 Const section

doc/grammar.txt

@@ -35,9 +35,12 @@ castExpr = 'cast' '[' optInd typeDesc optPar ']' '(' optInd expr optPar ')'
 parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try'
         | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let'
         | 'when' | 'var' | 'mixin'
-par = '(' optInd (&parKeyw complexOrSimpleStmt ^+ ';'
+par = '(' optInd
-                 | simpleExpr ('=' expr (';' complexOrSimpleStmt ^+ ';' )? )?
+          ( &parKeyw complexOrSimpleStmt ^+ ';'
-                            | (':' expr)? (',' (exprColonEqExpr comma?)*)?  )?
+          | ';' complexOrSimpleStmt ^+ ';'
+          | pragmaStmt
+          | simpleExpr ( ('=' expr (';' complexOrSimpleStmt ^+ ';' )? )
+                       | (':' expr (',' exprColonEqExpr     ^+ ',' )? ) ) )
           optPar ')'
 literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT
           | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT
@@ -86,7 +89,7 @@ expr = (ifExpr
       | caseExpr
       | tryExpr)
       / simpleExpr
-typeKeyw = 'var' | 'ref' | 'ptr' | 'shared' | 'tuple'
+typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple'
          | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum'
 primary = typeKeyw typeDescK
         /  prefixOperator* identOrLiteral primarySuffix*
@@ -165,7 +168,7 @@ objectCase = 'case' identWithPragma ':' typeDesc ':'? COMMENT?
 objectPart = IND{>} objectPart^+IND{=} DED
            / objectWhen / objectCase / 'nil' / 'discard' / declColonEquals
 object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart
-typeClassParam = ('var')? symbol
+typeClassParam = ('var' | 'out')? symbol
 typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
               &IND{>} stmt
 typeDef = identWithPragma genericParamList? '=' optInd typeDefAux

doc/lib.txt

@@ -165,6 +165,8 @@ Generic Operating System Services
   This module implements the ability to monitor a directory/file for changes
   using Posix's inotify API.
 
+  **Warning:** This module will likely be moved out to a Nimble package soon.
+
 * `asyncfile <asyncfile.html>`_
   This module implements asynchronous file reading and writing using
   ``asyncdispatch``.
@@ -191,6 +193,11 @@ Math libraries
 * `basic3d <basic3d.html>`_
   Basic 3d support with vectors, points, matrices and some basic utilities.
 
+* `mersenne <mersenne.html>`_
+  Mersenne twister random number generator.
+
+* `stats <stats.html>`_
+  Statistical analysis
 
 Internet Protocols and Support
 ------------------------------
@@ -209,7 +216,8 @@ Internet Protocols and Support
   This module implements a simple HTTP server.
 
 * `httpclient <httpclient.html>`_
-  This module implements a simple HTTP client.
+  This module implements a simple HTTP client which supports both synchronous
+  and asynchronous retrieval of web pages.
 
 * `smtp <smtp.html>`_
   This module implement a simple SMTP client.
@@ -226,19 +234,17 @@ Internet Protocols and Support
 * `asyncdispatch <asyncdispatch.html>`_
   This module implements an asynchronous dispatcher for IO operations.
 
-  **Note:** This module is still largely experimental.
-
 * `asyncnet <asyncnet.html>`_
   This module implements asynchronous sockets based on the ``asyncdispatch``
   module.
 
-  **Note:** This module is still largely experimental.
-
 * `asynchttpserver <asynchttpserver.html>`_
   This module implements an asynchronous HTTP server using the ``asyncnet``
   module.
 
-  **Note:** This module is still largely experimental.
+* `asyncftpclient <asyncftpclient.html>`_
+  This module implements an asynchronous FTP client using the ``asyncnet``
+  module.
 
 * `net <net.html>`_
   This module implements a high-level sockets API. It will replace the
@@ -346,6 +352,8 @@ Cryptography and Hashing
 * `base64 <base64.html>`_
   This module implements a base64 encoder and decoder.
 
+* `securehash <securehash.html>`_
+  This module implements a sha1 encoder and decoder.
 
 Multimedia support
 ------------------
@@ -374,10 +382,18 @@ Miscellaneous
 * `logging <logging.html>`_
   This module implements a simple logger.
 
+* `options <options.html>`_
+  Types which encapsulate an optional value.
+
 * `future <future.html>`_
   This module implements new experimental features. Currently the syntax
   sugar for anonymous procedures.
 
+* `coro <coro.html>`_
+  This module implements experimental coroutines in Nim.
+
+* `unittest <unittest.html>`_
+  Implements a Unit testing DSL.
 
 Modules for JS backend
 ---------------------------

doc/manual/generics.txt

@@ -213,7 +213,7 @@ Concepts are written in the following form:
 
     Container[T] = concept c
       c.len is Ordinal
-      items(c) is iterator
+      items(c) is T
       for value in c:
         type(value) is T
 

doc/manual/lexing.txt

@@ -69,6 +69,34 @@ Documentation comments are tokens; they are only allowed at certain places in
 the input file as they belong to the syntax tree!
 
 
+Multiline comments
+------------------
+
+Starting with version 0.13.0 of the language Nim supports multiline comments.
+They look like:
+
+.. code-block:: nim
+  #[Comment here.
+  Multiple lines
+  are not a problem.]#
+
+Multiline comments support nesting:
+
+.. code-block:: nim
+  #[  #[ Multiline comment in already
+     commented out code. ]#
+  proc p[T](x: T) = discard
+  ]#
+
+Multiline documentation comments look like and support nesting too:
+
+.. code-block:: nim
+  proc foo =
+    ##[Long documentation comment
+    here.
+    ]##
+
+
 Identifiers & Keywords
 ----------------------
 

doc/manual/modules.txt

@@ -152,9 +152,11 @@ In module related statements, if any part of the module name /
 path begins with a number, you may have to quote it in double quotes.
 In the following example, it would be seen as a literal number '3.0' of type
 'float64' if not quoted, if uncertain - quote it:
+
 .. code-block:: nim
   import "gfx/3d/somemodule"
 
+
 Scope rules
 -----------
 Identifiers are valid from the point of their declaration until the end of

doc/manual/procs.txt

@@ -236,8 +236,6 @@ executable code.
 Do notation
 -----------
 
-**Note:** The future of the ``do`` notation is uncertain.
-
 As a special more convenient notation, proc expressions involved in procedure
 calls can use the ``do`` keyword:
 
@@ -251,10 +249,12 @@ calls can use the ``do`` keyword:
 ``do`` is written after the parentheses enclosing the regular proc params.
 The proc expression represented by the do block is appended to them.
 
-More than one ``do`` block can appear in a single call:
+``do`` with parentheses is an anonymous ``proc``; however a ``do`` without
+parentheses is just a block of code. The ``do`` notation can be used to
+pass multiple blocks to a macro:
 
 .. code-block:: nim
-  proc performWithUndo(task: proc(), undo: proc()) = ...
+  macro performWithUndo(task, undo: untyped) = ...
 
   performWithUndo do:
     # multiple-line block of code

doc/manual/syntax.txt

@@ -64,6 +64,14 @@ Precedence level    Operators                                      First charact
 ================  ===============================================  ==================  ===============
 
 
+Whether an operator is used a prefix operator is also affected by preceeding whitespace (this parsing change was introduced with version 0.13.0):
+
+.. code-block:: nim
+  echo $foo
+  # is parsed as
+  echo($foo)
+
+
 Strong spaces
 -------------
 

doc/manual/templates.txt

@@ -10,7 +10,7 @@ The syntax to *invoke* a template is the same as calling a procedure.
 Example:
 
 .. code-block:: nim
-  template `!=` (a, b: expr): expr =
+  template `!=` (a, b: untyped): untyped =
     # this definition exists in the System module
     not (a == b)
 
@@ -23,50 +23,56 @@ templates:
 | ``a in b`` is transformed into ``contains(b, a)``.
 | ``notin`` and ``isnot`` have the obvious meanings.
 
-The "types" of templates can be the symbols ``expr`` (stands for *expression*),
+The "types" of templates can be the symbols ``untyped``,
-``stmt`` (stands for *statement*) or ``typedesc`` (stands for *type
+``typed`` or ``typedesc`` (stands for *type
 description*). These are "meta types", they can only be used in certain
-contexts. Real types can be used too; this implies that expressions are
+contexts. Real types can be used too; this implies that ``typed`` expressions
-expected.
+are expected.
 
 
-Ordinary vs immediate templates
+Typed vs untyped parameters
--------------------------------
+---------------------------
 
-There are two different kinds of templates: immediate templates and
+An ``untyped`` parameter means that symbol lookups and type resolution is not
-ordinary templates. Ordinary templates take part in overloading resolution. As
+performed before the expression is passed to the template. This means that for
-such their arguments need to be type checked before the template is invoked.
+example *undeclared* identifiers can be passed to the template:
-So ordinary templates cannot receive undeclared identifiers:
 
 .. code-block:: nim
 
-  template declareInt(x: expr) =
+  template declareInt(x: untyped) =
-    var x: int
-
-  declareInt(x) # error: unknown identifier: 'x'
-
-An ``immediate`` template does not participate in overload resolution and so
-its arguments are not checked for semantics before invocation. So they can
-receive undeclared identifiers:
-
-.. code-block:: nim
-
-  template declareInt(x: expr) {.immediate.} =
     var x: int
 
   declareInt(x) # valid
+  x = 3
+
+
+.. code-block:: nim
+
+  template declareInt(x: typed) =
+    var x: int
+
+  declareInt(x) # invalid, because x has not been declared and so has no type
+
+A template where every parameter is ``untyped`` is called an `immediate`:idx:
+template. For historical reasons templates can be explicitly annotated with
+an ``immediate`` pragma and then these templates do not take part in
+overloading resolution and the parameters' types are *ignored* by the
+compiler. Explicit immediate templates are about to be deprecated in later
+versions of the compiler.
+
+**Note**: For historical reasons ``stmt`` is an alias for ``typed`` and
+``expr`` an alias for ``untyped``, but new code should use the newer,
+clearer names.
 
 
 Passing a code block to a template
 ----------------------------------
 
-If there is a ``stmt`` parameter it should be the last in the template
+You can pass a block of statements as a last parameter to a template via a
-declaration, because statements are passed to a template via a
 special ``:`` syntax:
 
 .. code-block:: nim
-
+  template withFile(f, fn, mode, actions: untyped): untyped =
-  template withFile(f, fn, mode: expr, actions: stmt): stmt {.immediate.} =
     var f: File
     if open(f, fn, mode):
       try:
@@ -84,6 +90,64 @@ In the example the two ``writeLine`` statements are bound to the ``actions``
 parameter.
 
 
+Usually to pass a block of code to a template the parameter that accepts
+the block needs to be of type ``untyped``. Because symbol lookups are then
+delayed until template instantiation time:
+
+.. code-block:: nim
+  template t(body: typed) =
+    block:
+      body
+
+  t:
+    var i = 1
+    echo i
+
+  t:
+    var i = 2  # fails with 'attempt to redeclare i'
+    echo i
+
+The above code fails with the mysterious error message that ``i`` has already
+been declared. The reason for this is that the ``var i = ...`` bodies need to
+be type-checked before they are passed to the ``body`` parameter and type
+checking in Nim implies symbol lookups. For the symbol lookups to succeed
+``i`` needs to be added to the current (i.e. outer) scope. After type checking
+these additions to the symbol table are not rolled back (for better or worse).
+The same code works with ``untyped`` as the passed body is not required to be
+type-checked:
+
+.. code-block:: nim
+  template t(body: untyped) =
+    block:
+      body
+
+  t:
+    var i = 1
+    echo i
+
+  t:
+    var i = 2  # compiles
+    echo i
+
+
+Varargs of untyped
+------------------
+
+In addition to the ``untyped`` meta-type that prevents type checking there is
+also ``varargs[untyped]`` so that not even the number of parameters is fixed:
+
+.. code-block:: nim
+  template hideIdentifiers(x: varargs[untyped]) = discard
+
+  hideIdentifiers(undeclared1, undeclared2)
+
+However, since a template cannot iterate over varargs, this feature is
+generally much more useful for macros.
+
+**Note**: For historical reasons ``varargs[expr]`` is not equivalent
+to ``varargs[untyped]``.
+
+
 Symbol binding in templates
 ---------------------------
 

doc/tools.txt

@@ -4,7 +4,7 @@ Tools available with Nim
 
 The standard distribution ships with the following tools:
 
-- | `Documentation generator <docs/docgen.html>`_
+- | `Documentation generator <docgen.html>`_
   | The builtin document generator ``nim doc2`` generates HTML documentation
     from ``.nim`` source files.
 

doc/tut1.txt

@@ -758,8 +758,7 @@ However, this cannot be done for mutually recursive procedures:
   # forward declaration:
   proc even(n: int): bool
 
-proc even(n: int): bool
+.. code-block:: nim
-
   proc odd(n: int): bool =
     assert(n >= 0) # makes sure we don't run into negative recursion
     if n == 0: false

doc/tut2.txt

@@ -112,7 +112,7 @@ Example:
     Sym = object       # a symbol
       name: string     # the symbol's name
       line: int        # the line the symbol was declared in
-      code: PNode      # the symbol's abstract syntax tree
+      code: Node      # the symbol's abstract syntax tree
 
 
 Type conversions
@@ -162,11 +162,11 @@ An example:
       of nkFloat: floatVal: float
       of nkString: strVal: string
       of nkAdd, nkSub:
-        leftOp, rightOp: PNode
+        leftOp, rightOp: Node
       of nkIf:
-        condition, thenPart, elsePart: PNode
+        condition, thenPart, elsePart: Node
 
-  var n = PNode(kind: nkFloat, floatVal: 1.0)
+  var n = Node(kind: nkFloat, floatVal: 1.0)
   # the following statement raises an `FieldError` exception, because
   # n.kind's value does not fit:
   n.strVal = ""
@@ -990,3 +990,17 @@ generated by `treeRepr <macros.html#treeRepr>`_. If at the end of the this
 example you add ``echo treeRepr(result)`` you should get the same output as
 using the ``dumpTree`` macro, but of course you can call that at any point of
 the macro where you might be having troubles.
+
+
+Compilation to JavaScript
+=========================
+
+Nim code can be compiled to JavaScript. However in order to write
+JavaScript-compatible code you should remember the following:
+- ``addr`` and ``ptr`` have slightly different semantic meaning in JavaScript.
+  It is recommended to avoid those if you're not sure how they are translated
+  to JavaScript.
+- ``cast[T](x)`` in JavaScript is translated to ``(x)``.
+- ``cstring`` in JavaScript means JavaScript string. It is a good practice to
+  use ``cstring`` only when it is semantically appropriate. E.g. don't use
+  ``cstring`` as a binary data buffer.

examples/statcsv.nim

@@ -3,7 +3,7 @@
 # the standard deviation of its columns.
 # The CSV file can have a header which is then used for the output.
 
-import os, streams, parsecsv, strutils, math
+import os, streams, parsecsv, strutils, math, stats
 
 if paramCount() < 1:
   quit("Usage: statcsv filename[.csv]")

install_nimble.nims

@@ -1,4 +1,6 @@
 
+import ospaths
+
 mode = ScriptMode.Verbose
 
 var id = 0
@@ -10,4 +12,8 @@ exec "git clone https://github.com/nim-lang/nimble.git nimble" & $id
 withDir "nimble" & $id & "/src":
   exec "nim c nimble"
 
+mkDir "bin/nimblepkg"
+for file in listFiles("nimble" & $id & "/src/nimblepkg/"):
+  cpFile file, "bin/nimblepkg/" & file.extractFilename
+
 mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe

lib/core/macros.nim

@@ -97,7 +97,7 @@ type
     nskUnknown, nskConditional, nskDynLib, nskParam,
     nskGenericParam, nskTemp, nskModule, nskType, nskVar, nskLet,
     nskConst, nskResult,
-    nskProc, nskMethod, nskIterator, nskClosureIterator,
+    nskProc, nskMethod, nskIterator,
     nskConverter, nskMacro, nskTemplate, nskField,
     nskEnumField, nskForVar, nskLabel,
     nskStub
@@ -416,8 +416,7 @@ proc newLit*(i: BiggestInt): NimNode {.compileTime.} =
 
 proc newLit*(b: bool): NimNode {.compileTime.} =
   ## produces a new boolean literal node.
-  result = newNimNode(nnkIntLit)
+  result = if b: bindSym"true" else: bindSym"false"
-  result.intVal = ord(b)
 
 proc newLit*(f: BiggestFloat): NimNode {.compileTime.} =
   ## produces a new float literal node.

lib/core/typeinfo.nim

@@ -61,7 +61,10 @@ type
                           ## wrapped value and **must not** live longer than
                           ## its wrapped value.
     value: pointer
+    when defined(js):
       rawType: PNimType
+    else:
+      rawTypePtr: pointer
 
   ppointer = ptr pointer
   pbyteArray = ptr array[0.. 0xffff, int8]
@@ -71,6 +74,14 @@ type
     when defined(gogc):
       elemSize: int
   PGenSeq = ptr TGenericSeq
+
+when not defined(js):
+  template rawType(x: Any): PNimType =
+    cast[PNimType](x.rawTypePtr)
+
+  template `rawType=`(x: var Any, p: PNimType) =
+    x.rawTypePtr = cast[pointer](p)
+
 {.deprecated: [TAny: Any, TAnyKind: AnyKind].}
 
 when defined(gogc):
@@ -108,7 +119,7 @@ proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode =
   else:
     result = n.sons[n.len]
 
-proc newAny(value: pointer, rawType: PNimType): Any =
+proc newAny(value: pointer, rawType: PNimType): Any {.inline.} =
   result.value = value
   result.rawType = rawType
 
@@ -126,8 +137,7 @@ proc toAny*[T](x: var T): Any {.inline.} =
   ## constructs a ``Any`` object from `x`. This captures `x`'s address, so
   ## `x` can be modified with its ``Any`` wrapper! The client needs to ensure
   ## that the wrapper **does not** live longer than `x`!
-  result.value = addr(x)
+  newAny(addr(x), cast[PNimType](getTypeInfo(x)))
-  result.rawType = cast[PNimType](getTypeInfo(x))
 
 proc kind*(x: Any): AnyKind {.inline.} =
   ## get the type kind
@@ -345,7 +355,7 @@ proc `[]`*(x: Any, fieldName: string): Any =
     result.value = x.value +!! n.offset
     result.rawType = n.typ
   elif x.rawType.kind == tyObject and x.rawType.base != nil:
-    return `[]`(Any(value: x.value, rawType: x.rawType.base), fieldName)
+    return `[]`(newAny(x.value, x.rawType.base), fieldName)
   else:
     raise newException(ValueError, "invalid field name: " & fieldName)
 

lib/impure/db_mysql.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nim's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -10,7 +10,49 @@
 ## A higher level `mySQL`:idx: database wrapper. The same interface is
 ## implemented for other databases too.
 ##
-## Example:
+## See also: `db_odbc <db_odbc.html>`_, `db_sqlite <db_sqlite.html>`_,
+## `db_postgres <db_postgres.html>`_.
+##
+## Parameter substitution
+## ----------------------
+##
+## All ``db_*`` modules support the same form of parameter substitution.
+## That is, using the ``?`` (question mark) to signify the place where a
+## value should be placed. For example:
+##
+## .. code-block:: Nim
+##     sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)"
+##
+##
+## Examples
+## --------
+##
+## Opening a connection to a database
+## ==================================
+##
+## .. code-block:: Nim
+##     import db_mysql
+##     let db = open("localhost", "user", "password", "dbname")
+##     db.close()
+##
+## Creating a table
+## ================
+##
+## .. code-block:: Nim
+##      db.exec(sql"DROP TABLE IF EXISTS myTable")
+##      db.exec(sql("""CREATE TABLE myTable (
+##                       id integer,
+##                       name varchar(50) not null)"""))
+##
+## Inserting data
+## ==============
+##
+## .. code-block:: Nim
+##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",
+##             "Dominik")
+##
+## Larger example
+## ==============
 ##
 ## .. code-block:: Nim
 ##
@@ -43,45 +85,26 @@
 
 import strutils, mysql
 
+import db_common
+export db_common
+
 type
   DbConn* = PMySQL     ## encapsulates a database connection
   Row* = seq[string]   ## a row of a dataset. NULL database values will be
-                       ## transformed always to the empty string.
+                       ## converted to nil.
-  InstantRow* = tuple[row: cstringArray, len: int]  ## a handle that can be
+  InstantRow* = object ## a handle that can be used to get a row's
-                                                    ## used to get a row's
                        ## column text on demand
-  EDb* = object of IOError ## exception that is raised if a database error occurs
+    row: cstringArray
+    len: int
+{.deprecated: [TRow: Row, TDbConn: DbConn].}
 
-  SqlQuery* = distinct string ## an SQL query string
+proc dbError*(db: DbConn) {.noreturn.} =
-
+  ## raises a DbError exception.
-  FDb* = object of IOEffect ## effect that denotes a database operation
+  var e: ref DbError
-  FReadDb* = object of FDb   ## effect that denotes a read operation
-  FWriteDb* = object of FDb  ## effect that denotes a write operation
-{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].}
-
-proc sql*(query: string): SqlQuery {.noSideEffect, inline.} =
-  ## constructs a SqlQuery from the string `query`. This is supposed to be
-  ## used as a raw-string-literal modifier:
-  ## ``sql"update user set counter = counter + 1"``
-  ##
-  ## If assertions are turned off, it does nothing. If assertions are turned
-  ## on, later versions will check the string for valid syntax.
-  result = SqlQuery(query)
-
-proc dbError(db: DbConn) {.noreturn.} =
-  ## raises an EDb exception.
-  var e: ref EDb
   new(e)
   e.msg = $mysql.error(db)
   raise e
 
-proc dbError*(msg: string) {.noreturn.} =
-  ## raises an EDb exception with message `msg`.
-  var e: ref EDb
-  new(e)
-  e.msg = msg
-  raise e
-
 when false:
   proc dbQueryOpt*(db: DbConn, query: string, args: varargs[string, `$`]) =
     var stmt = mysql_stmt_init(db)
@@ -114,7 +137,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
       add(result, c)
 
 proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {.
-  tags: [FReadDB, FWriteDb].} =
+  tags: [ReadDbEffect, WriteDbEffect].} =
   ## tries to execute the query and returns true if successful, false otherwise.
   var q = dbFormat(query, args)
   return mysql.realQuery(db, q, q.len) == 0'i32
@@ -124,7 +147,7 @@ proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) =
   if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db)
 
 proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
-  tags: [FReadDB, FWriteDb].} =
+  tags: [ReadDbEffect, WriteDbEffect].} =
   ## executes the query and raises EDB if not successful.
   var q = dbFormat(query, args)
   if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db)
@@ -139,7 +162,7 @@ proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) =
   mysql.freeResult(sqlres)
 
 iterator fastRows*(db: DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## executes the query and iterates over the result dataset.
   ##
   ## This is very fast, but potentially dangerous.  Use this iterator only
@@ -167,9 +190,9 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
 
 iterator instantRows*(db: DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
-                      {.tags: [FReadDb].} =
+                      {.tags: [ReadDbEffect].} =
-  ## same as fastRows but returns a handle that can be used to get column text
+  ## Same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within the interator body.
+  ## on demand using []. Returned handle is valid only within the iterator body.
   rawExec(db, query, args)
   var sqlres = mysql.useResult(db)
   if sqlres != nil:
@@ -178,20 +201,102 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
     while true:
       row = mysql.fetchRow(sqlres)
       if row == nil: break
-      yield (row: row, len: L)
+      yield InstantRow(row: row, len: L)
     properFreeResult(sqlres, row)
 
+proc setTypeName(t: var DbType; f: PFIELD) =
+  shallowCopy(t.name, $f.name)
+  t.maxReprLen = Natural(f.max_length)
+  if (NOT_NULL_FLAG and f.flags) != 0: t.notNull = true
+  case f.ftype
+  of TYPE_DECIMAL:
+    t.kind = dbDecimal
+  of TYPE_TINY:
+    t.kind = dbInt
+    t.size = 1
+  of TYPE_SHORT:
+    t.kind = dbInt
+    t.size = 2
+  of TYPE_LONG:
+    t.kind = dbInt
+    t.size = 4
+  of TYPE_FLOAT:
+    t.kind = dbFloat
+    t.size = 4
+  of TYPE_DOUBLE:
+    t.kind = dbFloat
+    t.size = 8
+  of TYPE_NULL:
+    t.kind = dbNull
+  of TYPE_TIMESTAMP:
+    t.kind = dbTimestamp
+  of TYPE_LONGLONG:
+    t.kind = dbInt
+    t.size = 8
+  of TYPE_INT24:
+    t.kind = dbInt
+    t.size = 3
+  of TYPE_DATE:
+    t.kind = dbDate
+  of TYPE_TIME:
+    t.kind = dbTime
+  of TYPE_DATETIME:
+    t.kind = dbDatetime
+  of TYPE_YEAR:
+    t.kind = dbDate
+  of TYPE_NEWDATE:
+    t.kind = dbDate
+  of TYPE_VARCHAR, TYPE_VAR_STRING, TYPE_STRING:
+    t.kind = dbVarchar
+  of TYPE_BIT:
+    t.kind = dbBit
+  of TYPE_NEWDECIMAL:
+    t.kind = dbDecimal
+  of TYPE_ENUM: t.kind = dbEnum
+  of TYPE_SET: t.kind = dbSet
+  of TYPE_TINY_BLOB, TYPE_MEDIUM_BLOB, TYPE_LONG_BLOB,
+     TYPE_BLOB: t.kind = dbBlob
+  of TYPE_GEOMETRY:
+    t.kind = dbGeometry
+
+proc setColumnInfo(columns: var DbColumns; res: PRES; L: int) =
+  setLen(columns, L)
+  for i in 0..<L:
+    let fp = mysql.fetch_field_direct(res, cint(i))
+    setTypeName(columns[i].typ, fp)
+    columns[i].name = $fp.name
+    columns[i].tableName = $fp.table
+    columns[i].primaryKey = (fp.flags and PRI_KEY_FLAG) != 0
+    #columns[i].foreignKey = there is no such thing in mysql
+
+iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery;
+                      args: varargs[string, `$`]): InstantRow =
+  ## Same as fastRows but returns a handle that can be used to get column text
+  ## on demand using []. Returned handle is valid only within the iterator body.
+  rawExec(db, query, args)
+  var sqlres = mysql.useResult(db)
+  if sqlres != nil:
+    let L = int(mysql.numFields(sqlres))
+    setColumnInfo(columns, sqlres, L)
+    var row: cstringArray
+    while true:
+      row = mysql.fetchRow(sqlres)
+      if row == nil: break
+      yield InstantRow(row: row, len: L)
+    properFreeResult(sqlres, row)
+
+
 proc `[]`*(row: InstantRow, col: int): string {.inline.} =
-  ## returns text for given column of the row
+  ## Returns text for given column of the row.
   $row.row[col]
 
 proc len*(row: InstantRow): int {.inline.} =
-  ## returns number of columns in the row
+  ## Returns number of columns in the row.
   row.len
 
 proc getRow*(db: DbConn, query: SqlQuery,
-             args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## retrieves a single row. If the query doesn't return any rows, this proc
+  ## Retrieves a single row. If the query doesn't return any rows, this proc
   ## will return a Row with empty strings for each column.
   rawExec(db, query, args)
   var sqlres = mysql.useResult(db)
@@ -209,7 +314,7 @@ proc getRow*(db: DbConn, query: SqlQuery,
     properFreeResult(sqlres, row)
 
 proc getAllRows*(db: DbConn, query: SqlQuery,
-                 args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} =
+                 args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} =
   ## executes the query and returns the whole result dataset.
   result = @[]
   rawExec(db, query, args)
@@ -232,19 +337,19 @@ proc getAllRows*(db: DbConn, query: SqlQuery,
     mysql.freeResult(sqlres)
 
 iterator rows*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## same as `fastRows`, but slower and safe.
   for r in items(getAllRows(db, query, args)): yield r
 
 proc getValue*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): string {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} =
   ## executes the query and returns the first column of the first row of the
   ## result dataset. Returns "" if the dataset contains no rows or the database
   ## value is NULL.
   result = getRow(db, query, args)[0]
 
 proc tryInsertId*(db: DbConn, query: SqlQuery,
-                  args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} =
+                  args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row or -1 in case of an error.
   var q = dbFormat(query, args)
@@ -254,7 +359,7 @@ proc tryInsertId*(db: DbConn, query: SqlQuery,
     result = mysql.insertId(db)
 
 proc insertId*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} =
+               args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row.
   result = tryInsertID(db, query, args)
@@ -262,18 +367,18 @@ proc insertId*(db: DbConn, query: SqlQuery,
 
 proc execAffectedRows*(db: DbConn, query: SqlQuery,
                        args: varargs[string, `$`]): int64 {.
-                       tags: [FReadDB, FWriteDb].} =
+                       tags: [ReadDbEffect, WriteDbEffect].} =
   ## runs the query (typically "UPDATE") and returns the
   ## number of affected rows
   rawExec(db, query, args)
   result = mysql.affectedRows(db)
 
-proc close*(db: DbConn) {.tags: [FDb].} =
+proc close*(db: DbConn) {.tags: [DbEffect].} =
   ## closes the database connection.
   if db != nil: mysql.close(db)
 
 proc open*(connection, user, password, database: string): DbConn {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## opens a database connection. Raises `EDb` if the connection could not
   ## be established.
   result = mysql.init(nil)
@@ -291,7 +396,7 @@ proc open*(connection, user, password, database: string): DbConn {.
     dbError(errmsg)
 
 proc setEncoding*(connection: DbConn, encoding: string): bool {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## sets the encoding of a database connection, returns true for
   ## success, false for failure.
   result = mysql.set_character_set(connection, encoding) == 0

lib/impure/db_odbc.nim

@@ -0,0 +1,505 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Nim Contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## A higher level `ODBC` database wrapper.
+##
+## This is the same interface that is implemented for other databases.
+##
+## This has NOT yet been (extensively) tested against ODBC drivers for
+## Teradata, Oracle, Sybase, MSSqlvSvr, et. al.  databases.
+##
+## Currently all queries are ANSI calls, not Unicode.
+##
+## See also: `db_postgres <db_postgres.html>`_, `db_sqlite <db_sqlite.html>`_,
+## `db_mysql <db_mysql.html>`_.
+##
+## Parameter substitution
+## ----------------------
+##
+## All ``db_*`` modules support the same form of parameter substitution.
+## That is, using the ``?`` (question mark) to signify the place where a
+## value should be placed. For example:
+##
+## .. code-block:: Nim
+##     sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)"
+##
+##
+## Examples
+## --------
+##
+## Opening a connection to a database
+## ==================================
+##
+## .. code-block:: Nim
+##     import db_odbc
+##     let db = open("localhost", "user", "password", "dbname")
+##     db.close()
+##
+## Creating a table
+## ================
+##
+## .. code-block:: Nim
+##      db.exec(sql"DROP TABLE IF EXISTS myTable")
+##      db.exec(sql("""CREATE TABLE myTable (
+##                       id integer,
+##                       name varchar(50) not null)"""))
+##
+## Inserting data
+## ==============
+##
+## .. code-block:: Nim
+##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",
+##             "Andreas")
+##
+## Large example
+## =============
+##
+## .. code-block:: Nim
+##
+##  import db_odbc, math
+##
+##  let theDb = open("localhost", "nim", "nim", "test")
+##
+##  theDb.exec(sql"Drop table if exists myTestTbl")
+##  theDb.exec(sql("create table myTestTbl (" &
+##      " Id    INT(11)     NOT NULL AUTO_INCREMENT PRIMARY KEY, " &
+##      " Name  VARCHAR(50) NOT NULL, " &
+##      " i     INT(11), " &
+##      " f     DECIMAL(18,10))"))
+##
+##  theDb.exec(sql"START TRANSACTION")
+##  for i in 1..1000:
+##    theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)",
+##          "Item#" & $i, i, sqrt(i.float))
+##  theDb.exec(sql"COMMIT")
+##
+##  for x in theDb.fastRows(sql"select * from myTestTbl"):
+##    echo x
+##
+##  let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)",
+##          "Item#1001", 1001, sqrt(1001.0))
+##  echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id)
+##
+##  theDb.close()
+
+
+import strutils, odbcsql
+
+import db_common
+export db_common
+
+type
+  OdbcConnTyp = tuple[hDb: SqlHDBC, env: SqlHEnv, stmt: SqlHStmt]
+  DbConn* = OdbcConnTyp    ## encapsulates a database connection
+  Row* = seq[string]   ## a row of a dataset. NULL database values will be
+                       ## converted to nil.
+  InstantRow* = tuple[row: seq[string], len: int]  ## a handle that can be
+                                                    ## used to get a row's
+                                                    ## column text on demand
+
+{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].}
+
+var
+  buf: array[0..4096, char]
+
+proc properFreeResult(hType: int, sqlres: var SqlHandle) {.
+          tags: [WriteDbEffect], raises: [].} =
+  try:
+    discard SQLFreeHandle(hType.TSqlSmallInt, sqlres)
+    sqlres = nil
+  except: discard
+
+proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {.
+          tags: [ReadDbEffect], raises: [].} =
+  ## Returns ODBC error information
+  var
+    sqlState: array[0..512, char]
+    nativeErr: array[0..512, char]
+    errMsg: array[0..512, char]
+    retSz: TSqlSmallInt = 0
+    res: TSqlSmallInt = 0
+  try:
+    sqlState[0] = '\0'
+    nativeErr[0] = '\0'
+    errMsg[0] = '\0'
+    res = SQLErr(db.env, db.hDb, db.stmt,
+              cast[PSQLCHAR](sqlState.addr),
+              cast[PSQLCHAR](nativeErr.addr),
+              cast[PSQLCHAR](errMsg.addr),
+              511.TSqlSmallInt, retSz.addr.PSQLSMALLINT)
+  except:
+    discard
+  return (res.int, $sqlState, $nativeErr, $errMsg)
+
+proc dbError*(db: var DbConn) {.
+          tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} =
+  ## Raises an `[DbError]` exception with ODBC error information
+  var
+    e: ref DbError
+    ss, ne, msg: string = ""
+    isAnError = false
+    res: int = 0
+    prevSs = ""
+  while true:
+    prevSs = ss
+    (res, ss, ne, msg) = db.getErrInfo()
+    if prevSs == ss:
+      break
+    # sqlState of 00000 is not an error
+    elif ss == "00000":
+      break
+    elif ss == "01000":
+      echo "\nWarning: ", ss, " ", msg
+      continue
+    else:
+      isAnError = true
+      echo "\nError: ", ss, " ", msg
+  if isAnError:
+    new(e)
+    e.msg = "ODBC Error"
+    if db.stmt != nil:
+      properFreeResult(SQL_HANDLE_STMT, db.stmt)
+    properFreeResult(SQL_HANDLE_DBC, db.hDb)
+    properFreeResult(SQL_HANDLE_ENV, db.env)
+    raise e
+
+proc SqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [DbError]} =
+  ## Wrapper that checks if ``resVal`` is not SQL_SUCCESS and if so, raises [EDb]
+  if resVal != SQL_SUCCESS: dbError(db)
+
+proc SqlGetDBMS(db: var DbConn): string {.
+        tags: [ReadDbEffect, WriteDbEffect], raises: [] .} =
+  ## Returns the ODBC SQL_DBMS_NAME string
+  const
+    SQL_DBMS_NAME = 17.SqlUSmallInt
+  var
+    sz: TSqlSmallInt = 0
+  buf[0] = '\0'
+  try:
+    db.SqlCheck(SQLGetInfo(db.hDb, SQL_DBMS_NAME, cast[SqlPointer](buf.addr),
+                        4095.TSqlSmallInt, sz.addr))
+  except: discard
+  return $buf.cstring
+
+proc dbQuote*(s: string): string {.noSideEffect.} =
+  ## DB quotes the string.
+  result = "'"
+  for c in items(s):
+    if c == '\'': add(result, "''")
+    else: add(result, c)
+  add(result, '\'')
+
+proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {.
+                  noSideEffect.} =
+  ## Replace any ``?`` placeholders with `args`,
+  ## and quotes the arguments
+  result = ""
+  var a = 0
+  for c in items(string(formatstr)):
+    if c == '?':
+      if args[a] == nil:
+        add(result, "NULL")
+      else:
+        add(result, dbQuote(args[a]))
+      inc(a)
+    else:
+      add(result, c)
+
+proc prepareFetch(db: var DbConn, query: SqlQuery,
+                args: varargs[string, `$`]) {.
+                tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  # Prepare a statement, execute it and fetch the data to the driver
+  # ready for retrieval of the data
+  # Used internally by iterators and retrieval procs
+  # requires calling
+  #      properFreeResult(SQL_HANDLE_STMT, db.stmt)
+  # when finished
+  db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt))
+  var q = dbFormat(query, args)
+  db.SqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt))
+  db.SqlCheck(SQLExecute(db.stmt))
+  db.SqlCheck(SQLFetch(db.stmt))
+
+proc prepareFetchDirect(db: var DbConn, query: SqlQuery,
+                args: varargs[string, `$`]) {.
+                tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  # Prepare a statement, execute it and fetch the data to the driver
+  # ready for retrieval of the data
+  # Used internally by iterators and retrieval procs
+  # requires calling
+  #      properFreeResult(SQL_HANDLE_STMT, db.stmt)
+  # when finished
+  db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt))
+  var q = dbFormat(query, args)
+  db.SqlCheck(SQLExecDirect(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt))
+  db.SqlCheck(SQLFetch(db.stmt))
+
+proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {.
+  tags: [ReadDbEffect, WriteDbEffect], raises: [].} =
+  ## Tries to execute the query and returns true if successful, false otherwise.
+  var
+    res:TSqlSmallInt = -1
+  try:
+    db.prepareFetchDirect(query, args)
+    var
+      rCnt = -1
+    res = SQLRowCount(db.stmt, rCnt)
+    if res != SQL_SUCCESS: dbError(db)
+    properFreeResult(SQL_HANDLE_STMT, db.stmt)
+  except: discard
+  return res == SQL_SUCCESS
+
+proc rawExec(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
+            tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  db.prepareFetchDirect(query, args)
+
+proc exec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
+            tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Executes the query and raises EDB if not successful.
+  db.prepareFetchDirect(query, args)
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+proc newRow(L: int): Row {.noSideEFfect.} =
+  newSeq(result, L)
+  for i in 0..L-1: result[i] = ""
+
+iterator fastRows*(db: var DbConn, query: SqlQuery,
+                   args: varargs[string, `$`]): Row {.
+                tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Executes the query and iterates over the result dataset.
+  ##
+  ## This is very fast, but potentially dangerous.  Use this iterator only
+  ## if you require **ALL** the rows.
+  ##
+  ## Breaking the fastRows() iterator during a loop may cause a driver error
+  ## for subsequenct queries
+  ##
+  ## Rows are retrieved from the server at each iteration.
+  var
+    rowRes: Row
+    sz: TSqlSmallInt = 0
+    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    rCnt = -1
+
+  db.prepareFetch(query, args)
+  db.SqlCheck(SQLNumResultCols(db.stmt, cCnt))
+  db.SqlCheck(SQLRowCount(db.stmt, rCnt))
+  rowRes = newRow(cCnt)
+  for rNr in 1..rCnt:
+    for colId in 1..cCnt:
+      buf[0] = '\0'
+      db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
+                               cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+      rowRes[colId-1] = $buf.cstring
+    db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1))
+    yield rowRes
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+iterator instantRows*(db: var DbConn, query: SqlQuery,
+                      args: varargs[string, `$`]): InstantRow
+                {.tags: [ReadDbEffect, WriteDbEffect].} =
+  ## Same as fastRows but returns a handle that can be used to get column text
+  ## on demand using []. Returned handle is valid only within the interator body.
+  var
+    rowRes: Row
+    sz: TSqlSmallInt = 0
+    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    rCnt = -1
+  db.prepareFetch(query, args)
+  db.SqlCheck(SQLNumResultCols(db.stmt, cCnt))
+  db.SqlCheck(SQLRowCount(db.stmt, rCnt))
+  rowRes = newRow(cCnt)
+  for rNr in 1..rCnt:
+    for colId in 1..cCnt:
+      buf[0] = '\0'
+      db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
+                               cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+      rowRes[colId-1] = $buf.cstring
+    db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1))
+    yield (row: rowRes, len: cCnt.int)
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+proc `[]`*(row: InstantRow, col: int): string {.inline.} =
+  ## Returns text for given column of the row
+  row.row[col]
+
+proc len*(row: InstantRow): int {.inline.} =
+  ## Returns number of columns in the row
+  row.len
+
+proc getRow*(db: var DbConn, query: SqlQuery,
+             args: varargs[string, `$`]): Row {.
+          tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Retrieves a single row. If the query doesn't return any rows, this proc
+  ## will return a Row with empty strings for each column.
+  var
+    sz: TSqlSmallInt = 0.TSqlSmallInt
+    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    rCnt = -1
+  result = @[]
+  db.prepareFetch(query, args)
+  db.SqlCheck(SQLNumResultCols(db.stmt, cCnt))
+
+  db.SqlCheck(SQLRowCount(db.stmt, rCnt))
+  for colId in 1..cCnt:
+    db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
+                             cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+    result.add($buf.cstring)
+  db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1))
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+proc getAllRows*(db: var DbConn, query: SqlQuery,
+                 args: varargs[string, `$`]): seq[Row] {.
+           tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Executes the query and returns the whole result dataset.
+  var
+    rowRes: Row
+    sz: TSqlSmallInt = 0
+    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    rCnt = -1
+  db.prepareFetch(query, args)
+  db.SqlCheck(SQLNumResultCols(db.stmt, cCnt))
+  db.SqlCheck(SQLRowCount(db.stmt, rCnt))
+  result = @[]
+  for rNr in 1..rCnt:
+    rowRes = @[]
+    buf[0] = '\0'
+    for colId in 1..cCnt:
+      db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
+                               cast[SqlPointer](buf.addr), 4095.TSqlSmallInt, sz.addr))
+      rowRes.add($buf.cstring)
+    db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1))
+    result.add(rowRes)
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+iterator rows*(db: var DbConn, query: SqlQuery,
+               args: varargs[string, `$`]): Row {.
+         tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Same as `fastRows`, but slower and safe.
+  ##
+  ## This retrieves ALL rows into memory before
+  ## iterating through the rows.
+  ## Large dataset queries will impact on memory usage.
+  for r in items(getAllRows(db, query, args)): yield r
+
+proc getValue*(db: var DbConn, query: SqlQuery,
+               args: varargs[string, `$`]): string {.
+           tags: [ReadDbEffect, WriteDbEffect], raises: [].} =
+  ## Executes the query and returns the first column of the first row of the
+  ## result dataset. Returns "" if the dataset contains no rows or the database
+  ## value is NULL.
+  result = ""
+  try:
+    result = getRow(db, query, args)[0]
+  except: discard
+
+proc tryInsertId*(db: var DbConn, query: SqlQuery,
+                  args: varargs[string, `$`]): int64 {.
+            tags: [ReadDbEffect, WriteDbEffect], raises: [].} =
+  ## Executes the query (typically "INSERT") and returns the
+  ## generated ID for the row or -1 in case of an error.
+  if not tryExec(db, query, args):
+    result = -1'i64
+  else:
+    echo "DBMS: ",SqlGetDBMS(db).toLower()
+    result = -1'i64
+    try:
+      case SqlGetDBMS(db).toLower():
+      of "postgresql":
+        result = getValue(db, sql"SELECT LASTVAL();", []).parseInt
+      of "mysql":
+        result = getValue(db, sql"SELECT LAST_INSERT_ID();", []).parseInt
+      of "sqlite":
+        result = getValue(db, sql"SELECT LAST_INSERT_ROWID();", []).parseInt
+      of "microsoft sql server":
+        result = getValue(db, sql"SELECT SCOPE_IDENTITY();", []).parseInt
+      of "oracle":
+        result = getValue(db, sql"SELECT id.currval FROM DUAL;", []).parseInt
+      else: result = -1'i64
+    except: discard
+
+proc insertId*(db: var DbConn, query: SqlQuery,
+               args: varargs[string, `$`]): int64 {.
+         tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Executes the query (typically "INSERT") and returns the
+  ## generated ID for the row.
+  result = tryInsertID(db, query, args)
+  if result < 0: dbError(db)
+
+proc execAffectedRows*(db: var DbConn, query: SqlQuery,
+                       args: varargs[string, `$`]): int64 {.
+             tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Runs the query (typically "UPDATE") and returns the
+  ## number of affected rows
+  result = -1
+  var res = SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt.SqlHandle)
+  if res != SQL_SUCCESS: dbError(db)
+  var q = dbFormat(query, args)
+  res = SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)
+  if res != SQL_SUCCESS: dbError(db)
+  rawExec(db, query, args)
+  var rCnt = -1
+  result = SQLRowCount(db.hDb, rCnt)
+  if res != SQL_SUCCESS: dbError(db)
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+  result = rCnt
+
+proc close*(db: var DbConn) {.
+      tags: [WriteDbEffect], raises: [].} =
+  ## Closes the database connection.
+  if db.hDb != nil:
+    try:
+      var res = SQLDisconnect(db.hDb)
+      if db.stmt != nil:
+        res = SQLFreeHandle(SQL_HANDLE_STMT, db.stmt)
+      res = SQLFreeHandle(SQL_HANDLE_DBC, db.hDb)
+      res = SQLFreeHandle(SQL_HANDLE_ENV, db.env)
+      db = (hDb: nil, env: nil, stmt: nil)
+    except:
+      discard
+
+proc open*(connection, user, password, database: string): DbConn {.
+  tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Opens a database connection.
+  ##
+  ## Raises `EDb` if the connection could not be established.
+  ##
+  ## Currently the database parameter is ignored,
+  ## but included to match ``open()`` in the other db_xxxxx library modules.
+  var
+    val: TSqlInteger = SQL_OV_ODBC3
+    resLen = 0
+  result = (hDb: nil, env: nil, stmt: nil)
+  # allocate environment handle
+  var res = SQLAllocHandle(SQL_HANDLE_ENV, result.env, result.env)
+  if res != SQL_SUCCESS: dbError("Error: unable to initialise ODBC environment.")
+  res = SQLSetEnvAttr(result.env,
+                      SQL_ATTR_ODBC_VERSION.TSqlInteger,
+                      val, resLen.TSqlInteger)
+  if res != SQL_SUCCESS: dbError("Error: unable to set ODBC driver version.")
+  # allocate hDb handle
+  res = SQLAllocHandle(SQL_HANDLE_DBC, result.env, result.hDb)
+  if res != SQL_SUCCESS: dbError("Error: unable to allocate connection handle.")
+
+  # Connect: connection = dsn str,
+  res = SQLConnect(result.hDb,
+                  connection.PSQLCHAR , connection.len.TSqlSmallInt,
+                  user.PSQLCHAR, user.len.TSqlSmallInt,
+                  password.PSQLCHAR, password.len.TSqlSmallInt)
+  if res != SQL_SUCCESS:
+    result.dbError()
+
+proc setEncoding*(connection: DbConn, encoding: string): bool {.
+  tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Currently not implemented for ODBC.
+  ##
+  ## Sets the encoding of a database connection, returns true for
+  ## success, false for failure.
+  #result = set_character_set(connection, encoding) == 0
+  dbError("setEncoding() is currently not implemented by the db_odbc module")

lib/impure/db_postgres.nim

@@ -10,6 +10,9 @@
 ## A higher level `PostgreSQL`:idx: database wrapper. This interface
 ## is implemented for other databases also.
 ##
+## See also: `db_odbc <db_odbc.html>`_, `db_sqlite <db_sqlite.html>`_,
+## `db_mysql <db_mysql.html>`_.
+##
 ## Parameter substitution
 ## ----------------------
 ##
@@ -62,47 +65,28 @@
 ##             "Dominik")
 import strutils, postgres
 
+import db_common
+export db_common
+
 type
   DbConn* = PPGconn   ## encapsulates a database connection
   Row* = seq[string]  ## a row of a dataset. NULL database values will be
-                       ## transformed always to the empty string.
+                      ## converted to nil.
   InstantRow* = tuple[res: PPGresult, line: int32]  ## a handle that can be
                                                     ## used to get a row's
                                                     ## column text on demand
-  EDb* = object of IOError ## exception that is raised if a database error occurs
-
-  SqlQuery* = distinct string ## an SQL query string
   SqlPrepared* = distinct string ## a identifier for the prepared queries
 
-  FDb* = object of IOEffect ## effect that denotes a database operation
+{.deprecated: [TRow: Row, TDbConn: DbConn,
-  FReadDb* = object of FDb   ## effect that denotes a read operation
-  FWriteDb* = object of FDb  ## effect that denotes a write operation
-{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn,
               TSqlPrepared: SqlPrepared].}
 
-proc sql*(query: string): SqlQuery {.noSideEffect, inline.} =
-  ## constructs a SqlQuery from the string `query`. This is supposed to be
-  ## used as a raw-string-literal modifier:
-  ## ``sql"update user set counter = counter + 1"``
-  ##
-  ## If assertions are turned off, it does nothing. If assertions are turned
-  ## on, later versions will check the string for valid syntax.
-  result = SqlQuery(query)
-
 proc dbError*(db: DbConn) {.noreturn.} =
-  ## raises an EDb exception.
+  ## raises a DbError exception.
-  var e: ref EDb
+  var e: ref DbError
   new(e)
   e.msg = $pqErrorMessage(db)
   raise e
 
-proc dbError*(msg: string) {.noreturn.} =
-  ## raises an EDb exception with message `msg`.
-  var e: ref EDb
-  new(e)
-  e.msg = msg
-  raise e
-
 proc dbQuote*(s: string): string =
   ## DB quotes the string.
   result = "'"
@@ -127,7 +111,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
       add(result, c)
 
 proc tryExec*(db: DbConn, query: SqlQuery,
-              args: varargs[string, `$`]): bool {.tags: [FReadDB, FWriteDb].} =
+              args: varargs[string, `$`]): bool {.tags: [ReadDbEffect, WriteDbEffect].} =
   ## tries to execute the query and returns true if successful, false otherwise.
   var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil,
                         nil, nil, 0)
@@ -135,7 +119,8 @@ proc tryExec*(db: DbConn, query: SqlQuery,
   pqclear(res)
 
 proc tryExec*(db: DbConn, stmtName: SqlPrepared,
-              args: varargs[string, `$`]): bool {.tags: [FReadDB, FWriteDb].} =
+              args: varargs[string, `$`]): bool {.tags: [
+              ReadDbEffect, WriteDbEffect].} =
   ## tries to execute the query and returns true if successful, false otherwise.
   var arr = allocCStringArray(args)
   var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr,
@@ -145,7 +130,7 @@ proc tryExec*(db: DbConn, stmtName: SqlPrepared,
   pqclear(res)
 
 proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
-  tags: [FReadDB, FWriteDb].} =
+  tags: [ReadDbEffect, WriteDbEffect].} =
   ## executes the query and raises EDB if not successful.
   var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil,
                         nil, nil, 0)
@@ -153,7 +138,7 @@ proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
   pqclear(res)
 
 proc exec*(db: DbConn, stmtName: SqlPrepared,
-          args: varargs[string]) {.tags: [FReadDB, FWriteDb].} =
+          args: varargs[string]) {.tags: [ReadDbEffect, WriteDbEffect].} =
   var arr = allocCStringArray(args)
   var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr,
                            nil, nil, 0)
@@ -167,11 +152,7 @@ proc newRow(L: int): Row =
 
 proc setupQuery(db: DbConn, query: SqlQuery,
                 args: varargs[string]): PPGresult =
-  # s is a dummy unique id str for each setupQuery query
+  result = pqexec(db, dbFormat(query, args))
-  let s = "setupQuery_Query_" & string(query)
-  var res = pqprepare(db, s, dbFormat(query, args), 0, nil)
-  result = pqexecPrepared(db, s, 0, nil,
-                        nil, nil, 0)
   if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db)
 
 proc setupQuery(db: DbConn, stmtName: SqlPrepared,
@@ -184,8 +165,10 @@ proc setupQuery(db: DbConn, stmtName: SqlPrepared,
 
 proc prepare*(db: DbConn; stmtName: string, query: SqlQuery;
               nParams: int): SqlPrepared =
+  ## Creates a new ``SqlPrepared`` statement. Parameter substitution is done
+  ## via ``$1``, ``$2``, ``$3``, etc.
   if nParams > 0 and not string(query).contains("$1"):
-    dbError("""parameter substitution expects "$1" """)
+    dbError("parameter substitution expects \"$1\"")
   var res = pqprepare(db, stmtName, query.string, int32(nParams), nil)
   if pqResultStatus(res) != PGRES_COMMAND_OK: dbError(db)
   return SqlPrepared(stmtName)
@@ -200,7 +183,7 @@ proc setRow(res: PPGresult, r: var Row, line, cols: int32) =
       add(r[col], x)
 
 iterator fastRows*(db: DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## executes the query and iterates over the result dataset. This is very
   ## fast, but potenially dangerous: If the for-loop-body executes another
   ## query, the results can be undefined. For Postgres it is safe though.
@@ -213,7 +196,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
   pqclear(res)
 
 iterator fastRows*(db: DbConn, stmtName: SqlPrepared,
-                   args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## executes the prepared query and iterates over the result dataset.
   var res = setupQuery(db, stmtName, args)
   var L = pqNfields(res)
@@ -225,9 +208,9 @@ iterator fastRows*(db: DbConn, stmtName: SqlPrepared,
 
 iterator instantRows*(db: DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
-                      {.tags: [FReadDb].} =
+                      {.tags: [ReadDbEffect].} =
   ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within interator body.
+  ## on demand using []. Returned handle is valid only within iterator body.
   var res = setupQuery(db, query, args)
   for i in 0..pqNtuples(res)-1:
     yield (res: res, line: i)
@@ -235,9 +218,9 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
 
 iterator instantRows*(db: DbConn, stmtName: SqlPrepared,
                       args: varargs[string, `$`]): InstantRow
-                      {.tags: [FReadDb].} =
+                      {.tags: [ReadDbEffect].} =
   ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within interator body.
+  ## on demand using []. Returned handle is valid only within iterator body.
   var res = setupQuery(db, stmtName, args)
   for i in 0..pqNtuples(res)-1:
     yield (res: res, line: i)
@@ -252,7 +235,7 @@ proc len*(row: InstantRow): int32 {.inline.} =
   pqNfields(row.res)
 
 proc getRow*(db: DbConn, query: SqlQuery,
-             args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## retrieves a single row. If the query doesn't return any rows, this proc
   ## will return a Row with empty strings for each column.
   var res = setupQuery(db, query, args)
@@ -262,7 +245,7 @@ proc getRow*(db: DbConn, query: SqlQuery,
   pqclear(res)
 
 proc getRow*(db: DbConn, stmtName: SqlPrepared,
-             args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   var res = setupQuery(db, stmtName, args)
   var L = pqNfields(res)
   result = newRow(L)
@@ -270,39 +253,52 @@ proc getRow*(db: DbConn, stmtName: SqlPrepared,
   pqClear(res)
 
 proc getAllRows*(db: DbConn, query: SqlQuery,
-                 args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} =
+                 args: varargs[string, `$`]): seq[Row] {.
+                 tags: [ReadDbEffect].} =
   ## executes the query and returns the whole result dataset.
   result = @[]
   for r in fastRows(db, query, args):
     result.add(r)
 
 proc getAllRows*(db: DbConn, stmtName: SqlPrepared,
-                 args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} =
+                 args: varargs[string, `$`]): seq[Row] {.tags:
+                 [ReadDbEffect].} =
   ## executes the prepared query and returns the whole result dataset.
   result = @[]
   for r in fastRows(db, stmtName, args):
     result.add(r)
 
 iterator rows*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## same as `fastRows`, but slower and safe.
   for r in items(getAllRows(db, query, args)): yield r
 
 iterator rows*(db: DbConn, stmtName: SqlPrepared,
-               args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## same as `fastRows`, but slower and safe.
   for r in items(getAllRows(db, stmtName, args)): yield r
 
 proc getValue*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): string {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): string {.
+               tags: [ReadDbEffect].} =
   ## executes the query and returns the first column of the first row of the
   ## result dataset. Returns "" if the dataset contains no rows or the database
   ## value is NULL.
   var x = pqgetvalue(setupQuery(db, query, args), 0, 0)
   result = if isNil(x): "" else: $x
 
+proc getValue*(db: DbConn, stmtName: SqlPrepared,
+               args: varargs[string, `$`]): string {.
+               tags: [ReadDbEffect].} =
+  ## executes the query and returns the first column of the first row of the
+  ## result dataset. Returns "" if the dataset contains no rows or the database
+  ## value is NULL.
+  var x = pqgetvalue(setupQuery(db, stmtName, args), 0, 0)
+  result = if isNil(x): "" else: $x
+
 proc tryInsertID*(db: DbConn, query: SqlQuery,
-                  args: varargs[string, `$`]): int64  {.tags: [FWriteDb].}=
+                  args: varargs[string, `$`]): int64 {.
+                  tags: [WriteDbEffect].}=
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row or -1 in case of an error. For Postgre this adds
   ## ``RETURNING id`` to the query, so it only works if your primary key is
@@ -315,7 +311,8 @@ proc tryInsertID*(db: DbConn, query: SqlQuery,
     result = -1
 
 proc insertID*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} =
+               args: varargs[string, `$`]): int64 {.
+               tags: [WriteDbEffect].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row. For Postgre this adds
   ## ``RETURNING id`` to the query, so it only works if your primary key is
@@ -325,7 +322,7 @@ proc insertID*(db: DbConn, query: SqlQuery,
 
 proc execAffectedRows*(db: DbConn, query: SqlQuery,
                        args: varargs[string, `$`]): int64 {.tags: [
-                       FReadDB, FWriteDb].} =
+                       ReadDbEffect, WriteDbEffect].} =
   ## executes the query (typically "UPDATE") and returns the
   ## number of affected rows.
   var q = dbFormat(query, args)
@@ -336,7 +333,7 @@ proc execAffectedRows*(db: DbConn, query: SqlQuery,
 
 proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared,
                        args: varargs[string, `$`]): int64 {.tags: [
-                       FReadDB, FWriteDb].} =
+                       ReadDbEffect, WriteDbEffect].} =
   ## executes the query (typically "UPDATE") and returns the
   ## number of affected rows.
   var arr = allocCStringArray(args)
@@ -347,12 +344,12 @@ proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared,
   result = parseBiggestInt($pqcmdTuples(res))
   pqclear(res)
 
-proc close*(db: DbConn) {.tags: [FDb].} =
+proc close*(db: DbConn) {.tags: [DbEffect].} =
   ## closes the database connection.
   if db != nil: pqfinish(db)
 
 proc open*(connection, user, password, database: string): DbConn {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## opens a database connection. Raises `EDb` if the connection could not
   ## be established.
   ##
@@ -374,7 +371,7 @@ proc open*(connection, user, password, database: string): DbConn {.
   if pqStatus(result) != CONNECTION_OK: dbError(result) # result = nil
 
 proc setEncoding*(connection: DbConn, encoding: string): bool {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## sets the encoding of a database connection, returns true for
   ## success, false for failure.
   return pqsetClientEncoding(connection, encoding) == 0

lib/impure/db_sqlite.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nim's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -10,7 +10,48 @@
 ## A higher level `SQLite`:idx: database wrapper. This interface
 ## is implemented for other databases too.
 ##
-## Example:
+## See also: `db_odbc <db_odbc.html>`_, `db_postgres <db_postgres.html>`_,
+## `db_mysql <db_mysql.html>`_.
+##
+## Parameter substitution
+## ----------------------
+##
+## All ``db_*`` modules support the same form of parameter substitution.
+## That is, using the ``?`` (question mark) to signify the place where a
+## value should be placed. For example:
+##
+## .. code-block:: Nim
+##     sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)"
+##
+## Examples
+## --------
+##
+## Opening a connection to a database
+## ==================================
+##
+## .. code-block:: Nim
+##     import db_sqlite
+##     let db = open("localhost", "user", "password", "dbname")
+##     db.close()
+##
+## Creating a table
+## ================
+##
+## .. code-block:: Nim
+##      db.exec(sql"DROP TABLE IF EXISTS myTable")
+##      db.exec(sql("""CREATE TABLE myTable (
+##                       id integer,
+##                       name varchar(50) not null)"""))
+##
+## Inserting data
+## ==============
+##
+## .. code-block:: Nim
+##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",
+##             "Jack")
+##
+## Larger example
+## ==============
 ##
 ## .. code-block:: nim
 ##
@@ -40,47 +81,30 @@
 ##
 ##  theDb.close()
 
+{.deadCodeElim:on.}
+
 import strutils, sqlite3
 
+import db_common
+export db_common
+
 type
   DbConn* = PSqlite3  ## encapsulates a database connection
   Row* = seq[string]  ## a row of a dataset. NULL database values will be
-                       ## transformed always to the empty string.
+                       ## converted to nil.
   InstantRow* = Pstmt  ## a handle that can be used to get a row's column
                        ## text on demand
-  EDb* = object of IOError ## exception that is raised if a database error occurs
+{.deprecated: [TRow: Row, TDbConn: DbConn].}
 
-  SqlQuery* = distinct string ## an SQL query string
+proc dbError*(db: DbConn) {.noreturn.} =
-
+  ## raises a DbError exception.
-  FDb* = object of IOEffect ## effect that denotes a database operation
+  var e: ref DbError
-  FReadDb* = object of FDb   ## effect that denotes a read operation
-  FWriteDb* = object of FDb  ## effect that denotes a write operation
-{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].}
-
-proc sql*(query: string): SqlQuery {.noSideEffect, inline.} =
-  ## constructs a SqlQuery from the string `query`. This is supposed to be
-  ## used as a raw-string-literal modifier:
-  ## ``sql"update user set counter = counter + 1"``
-  ##
-  ## If assertions are turned off, it does nothing. If assertions are turned
-  ## on, later versions will check the string for valid syntax.
-  result = SqlQuery(query)
-
-proc dbError(db: DbConn) {.noreturn.} =
-  ## raises an EDb exception.
-  var e: ref EDb
   new(e)
   e.msg = $sqlite3.errmsg(db)
   raise e
 
-proc dbError*(msg: string) {.noreturn.} =
+proc dbQuote*(s: string): string =
-  ## raises an EDb exception with message `msg`.
+  ## DB quotes the string.
-  var e: ref EDb
-  new(e)
-  e.msg = msg
-  raise e
-
-proc dbQuote(s: string): string =
   if s.isNil: return "NULL"
   result = "'"
   for c in items(s):
@@ -99,7 +123,8 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
       add(result, c)
 
 proc tryExec*(db: DbConn, query: SqlQuery,
-              args: varargs[string, `$`]): bool {.tags: [FReadDb, FWriteDb].} =
+              args: varargs[string, `$`]): bool {.
+              tags: [ReadDbEffect, WriteDbEffect].} =
   ## tries to execute the query and returns true if successful, false otherwise.
   var q = dbFormat(query, args)
   var stmt: sqlite3.Pstmt
@@ -108,8 +133,8 @@ proc tryExec*(db: DbConn, query: SqlQuery,
       result = finalize(stmt) == SQLITE_OK
 
 proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`])  {.
-  tags: [FReadDb, FWriteDb].} =
+  tags: [ReadDbEffect, WriteDbEffect].} =
-  ## executes the query and raises EDB if not successful.
+  ## executes the query and raises DbError if not successful.
   if not tryExec(db, query, args): dbError(db)
 
 proc newRow(L: int): Row =
@@ -129,14 +154,14 @@ proc setRow(stmt: Pstmt, r: var Row, cols: cint) =
     if not isNil(x): add(r[col], x)
 
 iterator fastRows*(db: DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row  {.tags: [FReadDb].} =
+                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## Executes the query and iterates over the result dataset.
   ##
   ## This is very fast, but potentially dangerous.  Use this iterator only
   ## if you require **ALL** the rows.
   ##
   ## Breaking the fastRows() iterator during a loop will cause the next
-  ## database query to raise an [EDb] exception ``unable to close due to ...``.
+  ## database query to raise a DbError exception ``unable to close due to ...``.
   var stmt = setupQuery(db, query, args)
   var L = (column_count(stmt))
   var result = newRow(L)
@@ -147,14 +172,47 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
 
 iterator instantRows*(db: DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
-                      {.tags: [FReadDb].} =
+                      {.tags: [ReadDbEffect].} =
   ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within the interator body.
+  ## on demand using []. Returned handle is valid only within the iterator body.
   var stmt = setupQuery(db, query, args)
   while step(stmt) == SQLITE_ROW:
     yield stmt
   if finalize(stmt) != SQLITE_OK: dbError(db)
 
+proc toTypeKind(t: var DbType; x: int32) =
+  case x
+  of SQLITE_INTEGER:
+    t.kind = dbInt
+    t.size = 8
+  of SQLITE_FLOAT:
+    t.kind = dbFloat
+    t.size = 8
+  of SQLITE_BLOB: t.kind = dbBlob
+  of SQLITE_NULL: t.kind = dbNull
+  of SQLITE_TEXT: t.kind = dbVarchar
+  else: t.kind = dbUnknown
+
+proc setColumns(columns: var DbColumns; x: PStmt) =
+  let L = column_count(x)
+  setLen(columns, L)
+  for i in 0'i32 ..< L:
+    columns[i].name = $column_name(x, i)
+    columns[i].typ.name = $column_decltype(x, i)
+    toTypeKind(columns[i].typ, column_type(x, i))
+    columns[i].tableName = $column_table_name(x, i)
+
+iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery,
+                      args: varargs[string, `$`]): InstantRow
+                      {.tags: [ReadDbEffect].} =
+  ## same as fastRows but returns a handle that can be used to get column text
+  ## on demand using []. Returned handle is valid only within the iterator body.
+  var stmt = setupQuery(db, query, args)
+  setColumns(columns, stmt)
+  while step(stmt) == SQLITE_ROW:
+    yield stmt
+  if finalize(stmt) != SQLITE_OK: dbError(db)
+
 proc `[]`*(row: InstantRow, col: int32): string {.inline.} =
   ## returns text for given column of the row
   $column_text(row, col)
@@ -164,7 +222,7 @@ proc len*(row: InstantRow): int32 {.inline.} =
   column_count(row)
 
 proc getRow*(db: DbConn, query: SqlQuery,
-             args: varargs[string, `$`]): Row {.tags: [FReadDb].} =
+             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## retrieves a single row. If the query doesn't return any rows, this proc
   ## will return a Row with empty strings for each column.
   var stmt = setupQuery(db, query, args)
@@ -175,19 +233,19 @@ proc getRow*(db: DbConn, query: SqlQuery,
   if finalize(stmt) != SQLITE_OK: dbError(db)
 
 proc getAllRows*(db: DbConn, query: SqlQuery,
-                 args: varargs[string, `$`]): seq[Row] {.tags: [FReadDb].} =
+                 args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} =
   ## executes the query and returns the whole result dataset.
   result = @[]
   for r in fastRows(db, query, args):
     result.add(r)
 
 iterator rows*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): Row {.tags: [FReadDb].} =
+               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## same as `FastRows`, but slower and safe.
   for r in fastRows(db, query, args): yield r
 
 proc getValue*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): string {.tags: [FReadDb].} =
+               args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} =
   ## executes the query and returns the first column of the first row of the
   ## result dataset. Returns "" if the dataset contains no rows or the database
   ## value is NULL.
@@ -205,7 +263,7 @@ proc getValue*(db: DbConn, query: SqlQuery,
 
 proc tryInsertID*(db: DbConn, query: SqlQuery,
                   args: varargs[string, `$`]): int64
-                  {.tags: [FWriteDb], raises: [].} =
+                  {.tags: [WriteDbEffect], raises: [].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row or -1 in case of an error.
   var q = dbFormat(query, args)
@@ -218,7 +276,7 @@ proc tryInsertID*(db: DbConn, query: SqlQuery,
       result = -1
 
 proc insertID*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} =
+               args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row. For Postgre this adds
   ## ``RETURNING id`` to the query, so it only works if your primary key is
@@ -228,18 +286,18 @@ proc insertID*(db: DbConn, query: SqlQuery,
 
 proc execAffectedRows*(db: DbConn, query: SqlQuery,
                        args: varargs[string, `$`]): int64 {.
-                       tags: [FReadDb, FWriteDb].} =
+                       tags: [ReadDbEffect, WriteDbEffect].} =
   ## executes the query (typically "UPDATE") and returns the
   ## number of affected rows.
   exec(db, query, args)
   result = changes(db)
 
-proc close*(db: DbConn) {.tags: [FDb].} =
+proc close*(db: DbConn) {.tags: [DbEffect].} =
   ## closes the database connection.
   if sqlite3.close(db) != SQLITE_OK: dbError(db)
 
 proc open*(connection, user, password, database: string): DbConn {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## opens a database connection. Raises `EDb` if the connection could not
   ## be established. Only the ``connection`` parameter is used for ``sqlite``.
   var db: DbConn
@@ -249,7 +307,7 @@ proc open*(connection, user, password, database: string): DbConn {.
     dbError(db)
 
 proc setEncoding*(connection: DbConn, encoding: string): bool {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## sets the encoding of a database connection, returns true for
   ## success, false for failure.
   ##

lib/js/dom.nim

@@ -14,36 +14,35 @@ when not defined(js) and not defined(Nimdoc):
   {.error: "This module only works on the JavaScript platform".}
 
 type
-  TEventHandlers* {.importc.} = object of RootObj
+  EventTarget* = ref EventTargetObj
-    onabort*: proc (event: ref TEvent) {.nimcall.}
+  EventTargetObj {.importc.} = object of RootObj
-    onblur*: proc (event: ref TEvent) {.nimcall.}
+    onabort*: proc (event: Event) {.nimcall.}
-    onchange*: proc (event: ref TEvent) {.nimcall.}
+    onblur*: proc (event: Event) {.nimcall.}
-    onclick*: proc (event: ref TEvent) {.nimcall.}
+    onchange*: proc (event: Event) {.nimcall.}
-    ondblclick*: proc (event: ref TEvent) {.nimcall.}
+    onclick*: proc (event: Event) {.nimcall.}
-    onerror*: proc (event: ref TEvent) {.nimcall.}
+    ondblclick*: proc (event: Event) {.nimcall.}
-    onfocus*: proc (event: ref TEvent) {.nimcall.}
+    onerror*: proc (event: Event) {.nimcall.}
-    onkeydown*: proc (event: ref TEvent) {.nimcall.}
+    onfocus*: proc (event: Event) {.nimcall.}
-    onkeypress*: proc (event: ref TEvent) {.nimcall.}
+    onkeydown*: proc (event: Event) {.nimcall.}
-    onkeyup*: proc (event: ref TEvent) {.nimcall.}
+    onkeypress*: proc (event: Event) {.nimcall.}
-    onload*: proc (event: ref TEvent) {.nimcall.}
+    onkeyup*: proc (event: Event) {.nimcall.}
-    onmousedown*: proc (event: ref TEvent) {.nimcall.}
+    onload*: proc (event: Event) {.nimcall.}
-    onmousemove*: proc (event: ref TEvent) {.nimcall.}
+    onmousedown*: proc (event: Event) {.nimcall.}
-    onmouseout*: proc (event: ref TEvent) {.nimcall.}
+    onmousemove*: proc (event: Event) {.nimcall.}
-    onmouseover*: proc (event: ref TEvent) {.nimcall.}
+    onmouseout*: proc (event: Event) {.nimcall.}
-    onmouseup*: proc (event: ref TEvent) {.nimcall.}
+    onmouseover*: proc (event: Event) {.nimcall.}
-    onreset*: proc (event: ref TEvent) {.nimcall.}
+    onmouseup*: proc (event: Event) {.nimcall.}
-    onselect*: proc (event: ref TEvent) {.nimcall.}
+    onreset*: proc (event: Event) {.nimcall.}
-    onsubmit*: proc (event: ref TEvent) {.nimcall.}
+    onselect*: proc (event: Event) {.nimcall.}
-    onunload*: proc (event: ref TEvent) {.nimcall.}
+    onsubmit*: proc (event: Event) {.nimcall.}
-
+    onunload*: proc (event: Event) {.nimcall.}
-    addEventListener*: proc(ev: cstring, cb: proc(ev: ref TEvent), useCapture: bool = false) {.nimcall.}
 
   Window* = ref WindowObj
-  WindowObj {.importc.} = object of TEventHandlers
+  WindowObj {.importc.} = object of EventTargetObj
     document*: Document
-    event*: ref TEvent
+    event*: Event
-    history*: ref THistory
+    history*: History
-    location*: ref TLocation
+    location*: Location
     closed*: bool
     defaultStatus*: cstring
     innerHeight*, innerWidth*: int
@@ -57,50 +56,15 @@ type
     statusbar*: ref TStatusBar
     status*: cstring
     toolbar*: ref TToolBar
-
-    alert*: proc (msg: cstring) {.nimcall.}
-    back*: proc () {.nimcall.}
-    blur*: proc () {.nimcall.}
-    captureEvents*: proc (eventMask: int) {.nimcall.}
-    clearInterval*: proc (interval: ref TInterval) {.nimcall.}
-    clearTimeout*: proc (timeout: ref TTimeOut) {.nimcall.}
-    close*: proc () {.nimcall.}
-    confirm*: proc (msg: cstring): bool {.nimcall.}
-    disableExternalCapture*: proc () {.nimcall.}
-    enableExternalCapture*: proc () {.nimcall.}
-    find*: proc (text: cstring, caseSensitive = false,
-                 backwards = false) {.nimcall.}
-    focus*: proc () {.nimcall.}
-    forward*: proc () {.nimcall.}
-    handleEvent*: proc (e: ref TEvent) {.nimcall.}
-    home*: proc () {.nimcall.}
-    moveBy*: proc (x, y: int) {.nimcall.}
-    moveTo*: proc (x, y: int) {.nimcall.}
-    open*: proc (uri, windowname: cstring,
-                 properties: cstring = nil): Window {.nimcall.}
-    print*: proc () {.nimcall.}
-    prompt*: proc (text, default: cstring): cstring {.nimcall.}
-    releaseEvents*: proc (eventMask: int) {.nimcall.}
-    resizeBy*: proc (x, y: int) {.nimcall.}
-    resizeTo*: proc (x, y: int) {.nimcall.}
-    routeEvent*: proc (event: ref TEvent) {.nimcall.}
-    scrollBy*: proc (x, y: int) {.nimcall.}
-    scrollTo*: proc (x, y: int) {.nimcall.}
-    setInterval*: proc (code: cstring, pause: int): ref TInterval {.nimcall.}
-    setTimeout*: proc (code: cstring, pause: int): ref TTimeOut {.nimcall.}
-    stop*: proc () {.nimcall.}
     frames*: seq[TFrame]
 
   Frame* = ref FrameObj
   FrameObj {.importc.} = object of WindowObj
 
-  ClassList* {.importc.} = object of RootObj
+  ClassList* = ref ClassListObj
-    add*: proc (class: cstring) {.nimcall.}
+  ClassListObj {.importc.} = object of RootObj
-    remove*: proc (class: cstring) {.nimcall.}
-    contains*: proc (class: cstring):bool {.nimcall.}
-    toggle*: proc (class: cstring) {.nimcall.}
 
-  TNodeType* = enum
+  NodeType* = enum
     ElementNode = 1,
     AttributeNode,
     TextNode,
@@ -115,7 +79,7 @@ type
     NotationNode
 
   Node* = ref NodeObj
-  NodeObj {.importc.} = object of TEventHandlers
+  NodeObj {.importc.} = object of EventTargetObj
     attributes*: seq[Node]
     childNodes*: seq[Node]
     children*: seq[Node]
@@ -124,29 +88,12 @@ type
     lastChild*: Node
     nextSibling*: Node
     nodeName*: cstring
-    nodeType*: TNodeType
+    nodeType*: NodeType
     nodeValue*: cstring
     parentNode*: Node
     previousSibling*: Node
-    appendChild*: proc (child: Node) {.nimcall.}
-    appendData*: proc (data: cstring) {.nimcall.}
-    cloneNode*: proc (copyContent: bool): Node {.nimcall.}
-    deleteData*: proc (start, len: int) {.nimcall.}
-    getAttribute*: proc (attr: cstring): cstring {.nimcall.}
-    getAttributeNode*: proc (attr: cstring): Node {.nimcall.}
-    hasChildNodes*: proc (): bool {.nimcall.}
     innerHTML*: cstring
-    insertBefore*: proc (newNode, before: Node) {.nimcall.}
+    style*: Style
-    insertData*: proc (position: int, data: cstring) {.nimcall.}
-    removeAttribute*: proc (attr: cstring) {.nimcall.}
-    removeAttributeNode*: proc (attr: Node) {.nimcall.}
-    removeChild*: proc (child: Node) {.nimcall.}
-    replaceChild*: proc (newNode, oldNode: Node) {.nimcall.}
-    replaceData*: proc (start, len: int, text: cstring) {.nimcall.}
-    scrollIntoView*: proc () {.nimcall.}
-    setAttribute*: proc (name, value: cstring) {.nimcall.}
-    setAttributeNode*: proc (attr: Node) {.nimcall.}
-    style*: ref TStyle
 
   Document* = ref DocumentObj
   DocumentObj {.importc.} = object of NodeObj
@@ -164,31 +111,16 @@ type
     title*: cstring
     URL*: cstring
     vlinkColor*: cstring
-    captureEvents*: proc (eventMask: int) {.nimcall.}
-    createAttribute*: proc (identifier: cstring): Node {.nimcall.}
-    createElement*: proc (identifier: cstring): Element {.nimcall.}
-    createTextNode*: proc (identifier: cstring): Node {.nimcall.}
-    getElementById*: proc (id: cstring): Element {.nimcall.}
-    getElementsByName*: proc (name: cstring): seq[Element] {.nimcall.}
-    getElementsByTagName*: proc (name: cstring): seq[Element] {.nimcall.}
-    getElementsByClassName*: proc (name: cstring): seq[Element] {.nimcall.}
-    getSelection*: proc (): cstring {.nimcall.}
-    handleEvent*: proc (event: ref TEvent) {.nimcall.}
-    open*: proc () {.nimcall.}
-    releaseEvents*: proc (eventMask: int) {.nimcall.}
-    routeEvent*: proc (event: ref TEvent) {.nimcall.}
-    write*: proc (text: cstring) {.nimcall.}
-    writeln*: proc (text: cstring) {.nimcall.}
     anchors*: seq[AnchorElement]
     forms*: seq[FormElement]
     images*: seq[ImageElement]
-    applets*: seq[ref TApplet]
+    applets*: seq[Element]
     embeds*: seq[EmbedElement]
     links*: seq[LinkElement]
 
   Element* = ref ElementObj
   ElementObj {.importc.} = object of NodeObj
-    classList*: ref Classlist
+    classList*: Classlist
     checked*: bool
     defaultChecked*: bool
     defaultValue*: cstring
@@ -196,14 +128,7 @@ type
     form*: FormElement
     name*: cstring
     readOnly*: bool
-    blur*: proc () {.nimcall.}
-    click*: proc () {.nimcall.}
-    focus*: proc () {.nimcall.}
-    handleEvent*: proc (event: ref TEvent) {.nimcall.}
-    select*: proc () {.nimcall.}
     options*: seq[OptionElement]
-    getElementsByTagName*: proc (name: cstring): seq[Element] {.nimcall.}
-    getElementsByClassName*: proc (name: cstring): seq[Element] {.nimcall.}
 
   LinkElement* = ref LinkObj
   LinkObj {.importc.} = object of ElementObj
@@ -220,16 +145,12 @@ type
     width*: int
     `type`*: cstring
     vspace*: int
-    play*: proc () {.nimcall.}
-    stop*: proc () {.nimcall.}
 
   AnchorElement* = ref AnchorObj
   AnchorObj {.importc.} = object of ElementObj
     text*: cstring
     x*, y*: int
 
-  TApplet* {.importc.} = object of RootObj
-
   OptionElement* = ref OptionObj
   OptionObj {.importc.} = object of ElementObj
     defaultSelected*: bool
@@ -244,8 +165,6 @@ type
     encoding*: cstring
     `method`*: cstring
     target*: cstring
-    reset*: proc () {.nimcall.}
-    submit*: proc () {.nimcall.}
     elements*: seq[Element]
 
   ImageElement* = ref ImageObj
@@ -259,8 +178,8 @@ type
     vspace*: int
     width*: int
 
-
+  Style = ref StyleObj
-  TStyle* {.importc.} = object of RootObj
+  StyleObj {.importc.} = object of RootObj
     background*: cstring
     backgroundAttachment*: cstring
     backgroundColor*: cstring
@@ -350,11 +269,9 @@ type
     width*: cstring
     wordSpacing*: cstring
     zIndex*: int
-    getAttribute*: proc (attr: cstring, caseSensitive=false): cstring {.nimcall.}
-    removeAttribute*: proc (attr: cstring, caseSensitive=false) {.nimcall.}
-    setAttribute*: proc (attr, value: cstring, caseSensitive=false) {.nimcall.}
 
-  TEvent* {.importc.} = object of RootObj
+  Event* = ref EventObj
+  EventObj {.importc.} = object of RootObj
     target*: Node
     altKey*, ctrlKey*, shiftKey*: bool
     button*: int
@@ -393,7 +310,8 @@ type
     SUBMIT*: int
     UNLOAD*: int
 
-  TLocation* {.importc.} = object of RootObj
+  Location* = ref LocationObj
+  LocationObj {.importc.} = object of RootObj
     hash*: cstring
     host*: cstring
     hostname*: cstring
@@ -402,16 +320,13 @@ type
     port*: cstring
     protocol*: cstring
     search*: cstring
-    reload*: proc () {.nimcall.}
-    replace*: proc (s: cstring) {.nimcall.}
 
-  THistory* {.importc.} = object of RootObj
+  History* = ref HistoryObj
+  HistoryObj {.importc.} = object of RootObj
     length*: int
-    back*: proc () {.nimcall.}
-    forward*: proc () {.nimcall.}
-    go*: proc (pagesToJump: int) {.nimcall.}
 
-  TNavigator* {.importc.} = object of RootObj
+  Navigator* = ref NavigatorObj
+  NavigatorObj {.importc.} = object of RootObj
     appCodeName*: cstring
     appName*: cstring
     appVersion*: cstring
@@ -419,7 +334,6 @@ type
     language*: cstring
     platform*: cstring
     userAgent*: cstring
-    javaEnabled*: proc (): bool {.nimcall.}
     mimeTypes*: seq[ref TMimeType]
 
   TPlugin* {.importc.} = object of RootObj
@@ -441,7 +355,8 @@ type
   TToolBar* = TLocationBar
   TStatusBar* = TLocationBar
 
-  TScreen* {.importc.} = object of RootObj
+  Screen = ref ScreenObj
+  ScreenObj {.importc.} = object of RootObj
     availHeight*: int
     availWidth*: int
     colorDepth*: int
@@ -452,11 +367,127 @@ type
   TTimeOut* {.importc.} = object of RootObj
   TInterval* {.importc.} = object of RootObj
 
+{.push importcpp.}
+
+# EventTarget "methods"
+proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false)
+
+# Window "methods"
+proc alert*(w: Window, msg: cstring)
+proc back*(w: Window)
+proc blur*(w: Window)
+proc captureEvents*(w: Window, eventMask: int) {.deprecated.}
+proc clearInterval*(w: Window, interval: ref TInterval)
+proc clearTimeout*(w: Window, timeout: ref TTimeOut)
+proc close*(w: Window)
+proc confirm*(w: Window, msg: cstring): bool
+proc disableExternalCapture*(w: Window)
+proc enableExternalCapture*(w: Window)
+proc find*(w: Window, text: cstring, caseSensitive = false,
+           backwards = false)
+proc focus*(w: Window)
+proc forward*(w: Window)
+proc handleEvent*(w: Window, e: Event)
+proc home*(w: Window)
+proc moveBy*(w: Window, x, y: int)
+proc moveTo*(w: Window, x, y: int)
+proc open*(w: Window, uri, windowname: cstring,
+           properties: cstring = nil): Window
+proc print*(w: Window)
+proc prompt*(w: Window, text, default: cstring): cstring
+proc releaseEvents*(w: Window, eventMask: int) {.deprecated.}
+proc resizeBy*(w: Window, x, y: int)
+proc resizeTo*(w: Window, x, y: int)
+proc routeEvent*(w: Window, event: Event)
+proc scrollBy*(w: Window, x, y: int)
+proc scrollTo*(w: Window, x, y: int)
+proc setInterval*(w: Window, code: cstring, pause: int): ref TInterval
+proc setTimeout*(w: Window, code: cstring, pause: int): ref TTimeOut
+proc stop*(w: Window)
+
+# Node "methods"
+proc appendChild*(n, child: Node)
+proc appendData*(n: Node, data: cstring)
+proc cloneNode*(n: Node, copyContent: bool): Node
+proc deleteData*(n: Node, start, len: int)
+proc getAttribute*(n: Node, attr: cstring): cstring
+proc getAttributeNode*(n: Node, attr: cstring): Node
+proc hasChildNodes*(n: Node): bool
+proc insertBefore*(n, newNode, before: Node)
+proc insertData*(n: Node, position: int, data: cstring)
+proc removeAttribute*(n: Node, attr: cstring)
+proc removeAttributeNode*(n, attr: Node)
+proc removeChild*(n, child: Node)
+proc replaceChild*(n, newNode, oldNode: Node)
+proc replaceData*(n: Node, start, len: int, text: cstring)
+proc scrollIntoView*(n: Node)
+proc setAttribute*(n: Node, name, value: cstring)
+proc setAttributeNode*(n: Node, attr: Node)
+
+# Document "methods"
+proc captureEvents*(d: Document, eventMask: int) {.deprecated.}
+proc createAttribute*(d: Document, identifier: cstring): Node
+proc createElement*(d: Document, identifier: cstring): Element
+proc createTextNode*(d: Document, identifier: cstring): Node
+proc getElementById*(d: Document, id: cstring): Element
+proc getElementsByName*(d: Document, name: cstring): seq[Element]
+proc getElementsByTagName*(d: Document, name: cstring): seq[Element]
+proc getElementsByClassName*(d: Document, name: cstring): seq[Element]
+proc getSelection*(d: Document): cstring
+proc handleEvent*(d: Document, event: Event)
+proc open*(d: Document)
+proc releaseEvents*(d: Document, eventMask: int) {.deprecated.}
+proc routeEvent*(d: Document, event: Event)
+proc write*(d: Document, text: cstring)
+proc writeln*(d: Document, text: cstring)
+
+# Element "methods"
+proc blur*(e: Element)
+proc click*(e: Element)
+proc focus*(e: Element)
+proc handleEvent*(e: Element, event: Event)
+proc select*(e: Element)
+proc getElementsByTagName*(e: Element, name: cstring): seq[Element]
+proc getElementsByClassName*(e: Element, name: cstring): seq[Element]
+
+# FormElement "methods"
+proc reset*(f: FormElement)
+proc submit*(f: FormElement)
+
+# EmbedElement "methods"
+proc play*(e: EmbedElement)
+proc stop*(e: EmbedElement)
+
+# Location "methods"
+proc reload*(loc: Location)
+proc replace*(loc: Location, s: cstring)
+
+# History "methods"
+proc back*(h: History)
+proc forward*(h: History)
+proc go*(h: History, pagesToJump: int)
+
+# Navigator "methods"
+proc javaEnabled*(h: Navigator): bool
+
+# ClassList "methods"
+proc add*(c: ClassList, class: cstring)
+proc remove*(c: ClassList, class: cstring)
+proc contains*(c: ClassList, class: cstring):bool
+proc toggle*(c: ClassList, class: cstring)
+
+# Style "methods"
+proc getAttribute*(s: Style, attr: cstring, caseSensitive=false): cstring
+proc removeAttribute*(s: Style, attr: cstring, caseSensitive=false)
+proc setAttribute*(s: Style, attr, value: cstring, caseSensitive=false)
+
+{.pop.}
+
 var
   window* {.importc, nodecl.}: Window
   document* {.importc, nodecl.}: Document
-  navigator* {.importc, nodecl.}: ref TNavigator
+  navigator* {.importc, nodecl.}: Navigator
-  screen* {.importc, nodecl.}: ref TScreen
+  screen* {.importc, nodecl.}: Screen
 
 proc decodeURI*(uri: cstring): cstring {.importc, nodecl.}
 proc encodeURI*(uri: cstring): cstring {.importc, nodecl.}
@@ -474,6 +505,7 @@ proc parseInt*(s: cstring, radix: int):int {.importc, nodecl.}
 
 
 type
+  TEventHandlers* {.deprecated.} = EventTargetObj
   TWindow* {.deprecated.} = WindowObj
   TFrame* {.deprecated.} = FrameObj
   TNode* {.deprecated.} = NodeObj
@@ -485,3 +517,11 @@ type
   TOption* {.deprecated.} = OptionObj
   TForm* {.deprecated.} = FormObj
   TImage* {.deprecated.} = ImageObj
+  TNodeType* {.deprecated.} = NodeType
+  TEvent* {.deprecated.} = EventObj
+  TLocation* {.deprecated.} = LocationObj
+  THistory* {.deprecated.} = HistoryObj
+  TNavigator* {.deprecated.} = NavigatorObj
+  TStyle* {.deprecated.} = StyleObj
+  TScreen* {.deprecated.} = ScreenObj
+  TApplet* {.importc, deprecated.} = object of RootObj

lib/nimbase.h

@@ -418,10 +418,6 @@ typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(
 #  define NIM_EXTERNC
 #endif
 
-/* we have to tinker with TNimType as it's both part of system.nim and
-   typeinfo.nim but system.nim doesn't export it cleanly... */
-typedef struct TNimType TNimType;
-
 /* ---------------- platform specific includes ----------------------- */
 
 /* VxWorks related includes */

lib/packages/docutils/highlite.nim

@@ -173,7 +173,41 @@ proc nimNextToken(g: var GeneralTokenizer) =
       while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos)
     of '#':
       g.kind = gtComment
-      while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos)
+      inc(pos)
+      var isDoc = false
+      if g.buf[pos] == '#':
+        inc(pos)
+        isDoc = true
+      if g.buf[pos] == '[':
+        g.kind = gtLongComment
+        var nesting = 0
+        while true:
+          case g.buf[pos]
+          of '\0': break
+          of '#':
+            if isDoc:
+              if g.buf[pos+1] == '#' and g.buf[pos+2] == '[':
+                inc nesting
+            elif g.buf[pos+1] == '[':
+              inc nesting
+            inc pos
+          of ']':
+            if isDoc:
+              if g.buf[pos+1] == '#' and g.buf[pos+2] == '#':
+                if nesting == 0:
+                  inc(pos, 3)
+                  break
+                dec nesting
+            elif g.buf[pos+1] == '#':
+              if nesting == 0:
+                inc(pos, 2)
+                break
+              dec nesting
+            inc pos
+          else:
+            inc pos
+      else:
+        while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos)
     of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF':
       var id = ""
       while g.buf[pos] in SymChars + {'_'}:

lib/packages/docutils/rstgen.nim

@@ -534,7 +534,7 @@ proc generateDocumentationJumps(docs: IndexedDocs): string =
   for title in titles:
     chunks.add("<a href=\"" & title.link & "\">" & title.keyword & "</a>")
 
-  result.add(chunks.join(", ") & ".<br>")
+  result.add(chunks.join(", ") & ".<br/>")
 
 proc generateModuleJumps(modules: seq[string]): string =
   ## Returns a plain list of hyperlinks to the list of modules.
@@ -544,7 +544,7 @@ proc generateModuleJumps(modules: seq[string]): string =
   for name in modules:
     chunks.add("<a href=\"" & name & ".html\">" & name & "</a>")
 
-  result.add(chunks.join(", ") & ".<br>")
+  result.add(chunks.join(", ") & ".<br/>")
 
 proc readIndexDir(dir: string):
     tuple[modules: seq[string], symbols: seq[IndexEntry], docs: IndexedDocs] =

lib/posix/posix.nim

@@ -2165,6 +2165,10 @@ proc pwrite*(a1: cint, a2: pointer, a3: int, a4: Off): int {.
   importc, header: "<unistd.h>".}
 proc read*(a1: cint, a2: pointer, a3: int): int {.importc, header: "<unistd.h>".}
 proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "<unistd.h>".}
+proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl",
+      header: "<sys/ioctl.h>", varargs, tags: [WriteIOEffect].}
+  ## A system call for device-specific input/output operations and other
+  ## operations which cannot be expressed by regular system calls
 
 proc rmdir*(a1: cstring): cint {.importc, header: "<unistd.h>".}
 proc setegid*(a1: Gid): cint {.importc, header: "<unistd.h>".}

lib/pure/asyncftpclient.nim

@@ -288,7 +288,7 @@ proc defaultOnProgressChanged*(total, progress: BiggestInt,
   result.complete()
 
 proc retrFile*(ftp: AsyncFtpClient, file, dest: string,
-               onProgressChanged = defaultOnProgressChanged) {.async.} =
+               onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} =
   ## Downloads ``file`` and saves it to ``dest``.
   ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function
   ## when the download is finished. The event's ``filename`` field will be equal
@@ -339,7 +339,7 @@ proc doUpload(ftp: AsyncFtpClient, file: File,
     await countdownFut or sendFut
 
 proc store*(ftp: AsyncFtpClient, file, dest: string,
-            onProgressChanged = defaultOnProgressChanged) {.async.} =
+            onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} =
   ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this
   ## function asynchronously is recommended to view the progress of
   ## the download.

lib/pure/collections/critbits.nim

@@ -232,7 +232,7 @@ iterator mpairs*[T](c: var CritBitTree[T]): tuple[key: string, val: var T] =
   ## yields all (key, value)-pairs of `c`. The yielded values can be modified.
   for x in leaves(c.root): yield (x.key, x.val)
 
-proc allprefixedAux[T](c: CritBitTree[T], key: string): Node[T] =
+proc allprefixedAux[T](c: CritBitTree[T], key: string; longestMatch: bool): Node[T] =
   var p = c.root
   var top = p
   if p != nil:
@@ -242,43 +242,51 @@ proc allprefixedAux[T](c: CritBitTree[T], key: string): Node[T] =
       let dir = (1 + (ch.ord or p.otherBits.ord)) shr 8
       p = p.child[dir]
       if q.byte < key.len: top = p
+    if not longestMatch:
       for i in 0 .. <key.len:
         if p.key[i] != key[i]: return
     result = top
 
-iterator itemsWithPrefix*[T](c: CritBitTree[T], prefix: string): string =
+iterator itemsWithPrefix*[T](c: CritBitTree[T], prefix: string;
-  ## yields all keys starting with `prefix`.
+                             longestMatch=false): string =
-  let top = allprefixedAux(c, prefix)
+  ## yields all keys starting with `prefix`. If `longestMatch` is true,
+  ## the longest match is returned, it doesn't have to be a complete match then.
+  let top = allprefixedAux(c, prefix, longestMatch)
   for x in leaves(top): yield x.key
 
-iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string): string =
+iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string;
+                            longestMatch=false): string =
   ## yields all keys starting with `prefix`.
-  let top = allprefixedAux(c, prefix)
+  let top = allprefixedAux(c, prefix, longestMatch)
   for x in leaves(top): yield x.key
 
-iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): T =
+iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string;
+                              longestMatch=false): T =
   ## yields all values of `c` starting with `prefix` of the
   ## corresponding keys.
-  let top = allprefixedAux(c, prefix)
+  let top = allprefixedAux(c, prefix, longestMatch)
   for x in leaves(top): yield x.val
 
-iterator mvaluesWithPrefix*[T](c: var CritBitTree[T], prefix: string): var T =
+iterator mvaluesWithPrefix*[T](c: var CritBitTree[T], prefix: string;
+                               longestMatch=false): var T =
   ## yields all values of `c` starting with `prefix` of the
   ## corresponding keys. The values can be modified.
-  let top = allprefixedAux(c, prefix)
+  let top = allprefixedAux(c, prefix, longestMatch)
   for x in leaves(top): yield x.val
 
 iterator pairsWithPrefix*[T](c: CritBitTree[T],
-                             prefix: string): tuple[key: string, val: T] =
+                             prefix: string;
+                             longestMatch=false): tuple[key: string, val: T] =
   ## yields all (key, value)-pairs of `c` starting with `prefix`.
-  let top = allprefixedAux(c, prefix)
+  let top = allprefixedAux(c, prefix, longestMatch)
   for x in leaves(top): yield (x.key, x.val)
 
 iterator mpairsWithPrefix*[T](c: var CritBitTree[T],
-                              prefix: string): tuple[key: string, val: var T] =
+                              prefix: string;
+                             longestMatch=false): tuple[key: string, val: var T] =
   ## yields all (key, value)-pairs of `c` starting with `prefix`.
   ## The yielded values can be modified.
-  let top = allprefixedAux(c, prefix)
+  let top = allprefixedAux(c, prefix, longestMatch)
   for x in leaves(top): yield (x.key, x.val)
 
 proc `$`*[T](c: CritBitTree[T]): string =

lib/pure/collections/sequtils.nim

@@ -10,12 +10,9 @@
 ## :Author: Alexander Mitchell-Robinson (Amrykid)
 ##
 ## This module implements operations for the built-in `seq`:idx: type which
-## were inspired by functional programming languages. If you are looking for
+## were inspired by functional programming languages.
-## the typical `map` function which applies a function to every element in a
-## sequence, it already exists in the `system <system.html>`_ module in both
-## mutable and immutable styles.
 ##
-## Also, for functional style programming you may want to pass `anonymous procs
+## For functional style programming you may want to pass `anonymous procs
 ## <manual.html#anonymous-procs>`_ to procs like ``filter`` to reduce typing.
 ## Anonymous procs can use `the special do notation <manual.html#do-notation>`_
 ## which is more convenient in certain situations.

lib/pure/collections/tables.nim

@@ -887,7 +887,7 @@ proc mget*[A](t: CountTableRef[A], key: A): var int {.deprecated.} =
   result = t[][key]
 
 proc getOrDefault*[A](t: CountTableRef[A], key: A): int =
-  getOrDefaultImpl(t, key)
+  result = t[].getOrDefault(key)
 
 proc hasKey*[A](t: CountTableRef[A], key: A): bool =
   ## returns true iff `key` is in the table `t`.
@@ -1028,3 +1028,15 @@ when isMainModule:
   assert(merged["foo"] == 5)
   assert(merged["bar"] == 3)
   assert(merged["baz"] == 14)
+
+  block:
+    const testKey = "TESTKEY"
+    let t: CountTableRef[string] = newCountTable[string]()
+
+    # Before, does not compile with error message:
+    #test_counttable.nim(7, 43) template/generic instantiation from here
+    #lib/pure/collections/tables.nim(117, 21) template/generic instantiation from here
+    #lib/pure/collections/tableimpl.nim(32, 27) Error: undeclared field: 'hcode
+    doAssert 0 == t.getOrDefault(testKey)
+    t.inc(testKey,3)
+    doAssert 3 == t.getOrDefault(testKey)

lib/pure/db_common.nim

@@ -0,0 +1,103 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Common datatypes and definitions for all ``db_*.nim`` (
+## `db_mysql <db_mysql.html>`_, `db_postgres <db_postgres.html>`_,
+## and `db_sqlite <db_sqlite.html>`_) modules.
+
+type
+  DbError* = object of IOError ## exception that is raised if a database error occurs
+
+  SqlQuery* = distinct string ## an SQL query string
+
+
+  DbEffect* = object of IOEffect ## effect that denotes a database operation
+  ReadDbEffect* = object of DbEffect   ## effect that denotes a read operation
+  WriteDbEffect* = object of DbEffect  ## effect that denotes a write operation
+
+  DbTypeKind* = enum  ## a superset of datatypes that might be supported.
+    dbUnknown,        ## unknown datatype
+    dbSerial,         ## datatype used for primary auto-increment keys
+    dbNull,           ## datatype used for the NULL value
+    dbBit,            ## bit datatype
+    dbBool,           ## boolean datatype
+    dbBlob,           ## blob datatype
+    dbFixedChar,      ## string of fixed length
+    dbVarchar,        ## string datatype
+    dbJson,           ## JSON datatype
+    dbXml,            ## XML datatype
+    dbInt,            ## some integer type
+    dbUInt,           ## some unsigned integer type
+    dbDecimal,        ## decimal numbers (fixed-point number)
+    dbFloat,          ## some floating point type
+    dbDate,           ## a year-month-day description
+    dbTime,           ## HH:MM:SS information
+    dbDatetime,       ## year-month-day and HH:MM:SS information,
+                      ## plus optional time or timezone information
+    dbTimestamp,      ## Timestamp values are stored as the number of seconds
+                      ## since the epoch ('1970-01-01 00:00:00' UTC).
+    dbTimeInterval,   ## an interval [a,b] of times
+    dbEnum,           ## some enum
+    dbSet,            ## set of enum values
+    dbArray,          ## an array of values
+    dbComposite,      ## composite type (record, struct, etc)
+    dbUrl,            ## a URL
+    dbUuid,           ## a UUID
+    dbInet,           ## an IP address
+    dbMacAddress,     ## a MAC address
+    dbGeometry,       ## some geometric type
+    dbPoint,          ## Point on a plane   (x,y)
+    dbLine,           ## Infinite line ((x1,y1),(x2,y2))
+    dbLseg,           ## Finite line segment   ((x1,y1),(x2,y2))
+    dbBox,            ## Rectangular box   ((x1,y1),(x2,y2))
+    dbPath,           ## Closed or open path (similar to polygon) ((x1,y1),...)
+    dbPolygon,        ## Polygon (similar to closed path)   ((x1,y1),...)
+    dbCircle,         ## Circle   <(x,y),r> (center point and radius)
+    dbUser1,          ## user definable datatype 1 (for unknown extensions)
+    dbUser2,          ## user definable datatype 2 (for unknown extensions)
+    dbUser3,          ## user definable datatype 3 (for unknown extensions)
+    dbUser4,          ## user definable datatype 4 (for unknown extensions)
+    dbUser5           ## user definable datatype 5 (for unknown extensions)
+
+  DbType* = object    ## describes a database type
+    kind*: DbTypeKind ## the kind of the described type
+    notNull*: bool    ## does the type contain NULL?
+    name*: string     ## the name of the type
+    size*: Natural    ## the size of the datatype; 0 if of variable size
+    maxReprLen*: Natural ## maximal length required for the representation
+    precision*, scale*: Natural ## precision and scale of the number
+    min*, max*: BiggestInt ## the minimum and maximum of allowed values
+    validValues*: seq[string] ## valid values of an enum or a set
+
+  DbColumn* = object   ## information about a database column
+    name*: string      ## name of the column
+    tableName*: string ## name of the table the column belongs to (optional)
+    typ*: DbType       ## type of the column
+    primaryKey*: bool  ## is this a primary key?
+    foreignKey*: bool  ## is this a foreign key?
+  DbColumns* = seq[DbColumn]
+
+{.deprecated: [EDb: DbError, TSqlQuery: SqlQuery, FDb: DbEffect,
+              FReadDb: ReadDbEffect, FWriteDb: WriteDbEffect].}
+
+template sql*(query: string): SqlQuery =
+  ## constructs a SqlQuery from the string `query`. This is supposed to be
+  ## used as a raw-string-literal modifier:
+  ## ``sql"update user set counter = counter + 1"``
+  ##
+  ## If assertions are turned off, it does nothing. If assertions are turned
+  ## on, later versions will check the string for valid syntax.
+  SqlQuery(query)
+
+proc dbError*(msg: string) {.noreturn, noinline.} =
+  ## raises an DbError exception with message `msg`.
+  var e: ref DbError
+  new(e)
+  e.msg = msg
+  raise e

lib/pure/fsmonitor.nim

@@ -10,6 +10,9 @@
 ## This module allows you to monitor files or directories for changes using
 ## asyncio.
 ##
+## **Warning**: This module will likely disappear soon and be moved into a
+## new Nimble package.
+##
 ## Windows support is not yet implemented.
 ##
 ## **Note:** This module uses ``inotify`` on Linux (Other Unixes are not yet
@@ -34,8 +37,8 @@ type
   MonitorEventType* = enum ## Monitor event type
     MonitorAccess,       ## File was accessed.
     MonitorAttrib,       ## Metadata changed.
-    MonitorCloseWrite,   ## Writtable file was closed.
+    MonitorCloseWrite,   ## Writable file was closed.
-    MonitorCloseNoWrite, ## Unwrittable file closed.
+    MonitorCloseNoWrite, ## Non-writable file closed.
     MonitorCreate,       ## Subfile was created.
     MonitorDelete,       ## Subfile was deleted.
     MonitorDeleteSelf,   ## Watched file/directory was itself deleted.
@@ -78,21 +81,21 @@ proc add*(monitor: FSMonitor, target: string,
   ## watched paths of ``monitor``.
   ## You can specify the events to report using the ``filters`` parameter.
 
-  var INFilter = -1
+  var INFilter = 0
   for f in filters:
     case f
-    of MonitorAccess: INFilter = INFilter and IN_ACCESS
+    of MonitorAccess: INFilter = INFilter or IN_ACCESS
-    of MonitorAttrib: INFilter = INFilter and IN_ATTRIB
+    of MonitorAttrib: INFilter = INFilter or IN_ATTRIB
-    of MonitorCloseWrite: INFilter = INFilter and IN_CLOSE_WRITE
+    of MonitorCloseWrite: INFilter = INFilter or IN_CLOSE_WRITE
-    of MonitorCloseNoWrite: INFilter = INFilter and IN_CLOSE_NO_WRITE
+    of MonitorCloseNoWrite: INFilter = INFilter or IN_CLOSE_NO_WRITE
-    of MonitorCreate: INFilter = INFilter and IN_CREATE
+    of MonitorCreate: INFilter = INFilter or IN_CREATE
-    of MonitorDelete: INFilter = INFilter and IN_DELETE
+    of MonitorDelete: INFilter = INFilter or IN_DELETE
-    of MonitorDeleteSelf: INFilter = INFilter and IN_DELETE_SELF
+    of MonitorDeleteSelf: INFilter = INFilter or IN_DELETE_SELF
-    of MonitorModify: INFilter = INFilter and IN_MODIFY
+    of MonitorModify: INFilter = INFilter or IN_MODIFY
-    of MonitorMoveSelf: INFilter = INFilter and IN_MOVE_SELF
+    of MonitorMoveSelf: INFilter = INFilter or IN_MOVE_SELF
-    of MonitorMoved: INFilter = INFilter and IN_MOVED_FROM and IN_MOVED_TO
+    of MonitorMoved: INFilter = INFilter or IN_MOVED_FROM or IN_MOVED_TO
-    of MonitorOpen: INFilter = INFilter and IN_OPEN
+    of MonitorOpen: INFilter = INFilter or IN_OPEN
-    of MonitorAll: INFilter = INFilter and IN_ALL_EVENTS
+    of MonitorAll: INFilter = INFilter or IN_ALL_EVENTS
 
   result = inotifyAddWatch(monitor.fd, target, INFilter.uint32)
   if result < 0:
@@ -200,9 +203,18 @@ proc register*(d: Dispatcher, monitor: FSMonitor,
 
 when not defined(testing) and isMainModule:
   proc main =
-    var disp = newDispatcher()
+    var
-    var monitor = newMonitor()
+      disp = newDispatcher()
-    echo monitor.add("/home/dom/inotifytests/")
+      monitor = newMonitor()
+      n = 0
+    n = monitor.add("/tmp")
+    assert n == 1
+    n = monitor.add("/tmp", {MonitorAll})
+    assert n == 1
+    n = monitor.add("/tmp", {MonitorCloseWrite, MonitorCloseNoWrite})
+    assert n == 1
+    n = monitor.add("/tmp", {MonitorMoved, MonitorOpen, MonitorAccess})
+    assert n == 1
     disp.register(monitor,
       proc (m: FSMonitor, ev: MonitorEvent) =
         echo("Got event: ", ev.kind)

lib/pure/gentabs.nim

@@ -11,6 +11,8 @@
 ## key-value mapping. The keys are required to be strings, but the values
 ## may be any Nim or user defined type. This module supports matching
 ## of keys in case-sensitive, case-insensitive and style-insensitive modes.
+##
+## **Warning:** This module is deprecated, new code shouldn't use it!
 
 {.deprecated.}
 

lib/pure/httpclient.nim

@@ -110,7 +110,7 @@ type
   EInvalidProtocol: ProtocolError, EHttpRequestErr: HttpRequestError
 ].}
 
-const defUserAgent* = "Nim httpclient/0.1"
+const defUserAgent* = "Nim httpclient/" & NimVersion
 
 proc httpError(msg: string) =
   var e: ref ProtocolError
@@ -389,6 +389,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
   ## | An optional timeout can be specified in milliseconds, if reading from the
   ## server takes longer than specified an ETimeout exception will be raised.
   var r = if proxy == nil: parseUri(url) else: proxy.url
+  var hostUrl = if proxy == nil: r else: parseUri(url)
   var headers = substr(httpMethod, len("http"))
   # TODO: Use generateHeaders further down once it supports proxies.
   if proxy == nil:
@@ -402,10 +403,10 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
 
   headers.add(" HTTP/1.1\c\L")
 
-  if r.port == "":
+  if hostUrl.port == "":
-    add(headers, "Host: " & r.hostname & "\c\L")
+    add(headers, "Host: " & hostUrl.hostname & "\c\L")
   else:
-    add(headers, "Host: " & r.hostname & ":" & r.port & "\c\L")
+    add(headers, "Host: " & hostUrl.hostname & ":" & hostUrl.port & "\c\L")
 
   if userAgent != "":
     add(headers, "User-Agent: " & userAgent & "\c\L")
@@ -414,7 +415,6 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
     add(headers, "Proxy-Authorization: basic " & auth & "\c\L")
   add(headers, extraHeaders)
   add(headers, "\c\L")
-
   var s = newSocket()
   if s == nil: raiseOSError(osLastError())
   var port = net.Port(80)

lib/pure/httpserver.nim

@@ -9,6 +9,9 @@
 
 ## This module implements a simple HTTP-Server.
 ##
+## **Warning**: This module will soon be deprecated in favour of
+## the ``asyncdispatch`` module, you should use it instead.
+##
 ## Example:
 ##
 ## .. code-block:: nim

lib/pure/lexbase.nim

@@ -28,7 +28,10 @@ type
   BaseLexer* = object of RootObj ## the base lexer. Inherit your lexer from
                                  ## this object.
     bufpos*: int              ## the current position within the buffer
-    buf*: cstring             ## the buffer itself
+    when defined(js):         ## the buffer itself
+      buf*: string
+    else:
+      buf*: cstring
     bufLen*: int              ## length of buffer in characters
     input: Stream            ## the input stream
     lineNumber*: int          ## the current line number
@@ -43,6 +46,7 @@ const
 
 proc close*(L: var BaseLexer) =
   ## closes the base lexer. This closes `L`'s associated stream too.
+  when not defined(js):
     dealloc(L.buf)
   close(L.input)
 
@@ -58,8 +62,11 @@ proc fillBuffer(L: var BaseLexer) =
   toCopy = L.bufLen - L.sentinel - 1
   assert(toCopy >= 0)
   if toCopy > 0:
-    moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize)
+    when defined(js):
+      for i in 0 ..< toCopy: L.buf[i] = L.buf[L.sentinel + 1 + i]
+    else:
       # "moveMem" handles overlapping regions
+      moveMem(L.buf, addr L.buf[L.sentinel + 1], toCopy * chrSize)
   charsRead = readData(L.input, addr(L.buf[toCopy]),
                        (L.sentinel + 1) * chrSize) div chrSize
   s = toCopy + charsRead
@@ -81,6 +88,9 @@ proc fillBuffer(L: var BaseLexer) =
         # double the buffer's size and try again:
         oldBufLen = L.bufLen
         L.bufLen = L.bufLen * 2
+        when defined(js):
+          L.buf.setLen(L.bufLen)
+        else:
           L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize))
         assert(L.bufLen - oldBufLen == oldBufLen)
         charsRead = readData(L.input, addr(L.buf[oldBufLen]),
@@ -139,6 +149,9 @@ proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192;
   L.bufpos = 0
   L.bufLen = bufLen
   L.refillChars = refillChars
+  when defined(js):
+    L.buf = newString(bufLen)
+  else:
     L.buf = cast[cstring](alloc(bufLen * chrSize))
   L.sentinel = bufLen - 1
   L.lineStart = 0

lib/pure/math.nim

@@ -118,26 +118,6 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} =
   ## If `x` is empty, 0 is returned.
   for i in items(x): result = result + i
 
-template toFloat(f: float): float = f
-
-proc mean*[T](x: openArray[T]): float {.noSideEffect.} =
-  ## Computes the mean of the elements in `x`, which are first converted to floats.
-  ## If `x` is empty, NaN is returned.
-  ## ``toFloat(x: T): float`` must be defined.
-  for i in items(x): result = result + toFloat(i)
-  result = result / toFloat(len(x))
-
-proc variance*[T](x: openArray[T]): float {.noSideEffect.} =
-  ## Computes the variance of the elements in `x`.
-  ## If `x` is empty, NaN is returned.
-  ## ``toFloat(x: T): float`` must be defined.
-  result = 0.0
-  var m = mean(x)
-  for i in items(x):
-    var diff = toFloat(i) - m
-    result = result + diff*diff
-  result = result / toFloat(len(x))
-
 proc random*(max: int): int {.benign.}
   ## Returns a random number in the range 0..max-1. The sequence of
   ## random number is always the same, unless `randomize` is called
@@ -376,48 +356,6 @@ proc random*[T](a: openArray[T]): T =
   ## returns a random element from the openarray `a`.
   result = a[random(a.low..a.len)]
 
-type
-  RunningStat* = object                 ## an accumulator for statistical data
-    n*: int                             ## number of pushed data
-    sum*, min*, max*, mean*: float      ## self-explaining
-    oldM, oldS, newS: float
-
-{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].}
-
-proc push*(s: var RunningStat, x: float) =
-  ## pushes a value `x` for processing
-  inc(s.n)
-  # See Knuth TAOCP vol 2, 3rd edition, page 232
-  if s.n == 1:
-    s.min = x
-    s.max = x
-    s.oldM = x
-    s.mean = x
-    s.oldS = 0.0
-  else:
-    if s.min > x: s.min = x
-    if s.max < x: s.max = x
-    s.mean = s.oldM + (x - s.oldM)/toFloat(s.n)
-    s.newS = s.oldS + (x - s.oldM)*(x - s.mean)
-
-    # set up for next iteration:
-    s.oldM = s.mean
-    s.oldS = s.newS
-  s.sum = s.sum + x
-
-proc push*(s: var RunningStat, x: int) =
-  ## pushes a value `x` for processing. `x` is simply converted to ``float``
-  ## and the other push operation is called.
-  push(s, toFloat(x))
-
-proc variance*(s: RunningStat): float =
-  ## computes the current variance of `s`
-  if s.n > 1: result = s.newS / (toFloat(s.n - 1))
-
-proc standardDeviation*(s: RunningStat): float =
-  ## computes the current standard deviation of `s`
-  result = sqrt(variance(s))
-
 {.pop.}
 {.pop.}
 

lib/pure/nativesockets.nim

@@ -203,8 +203,11 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET,
   hints.ai_family = toInt(domain)
   hints.ai_socktype = toInt(sockType)
   hints.ai_protocol = toInt(protocol)
+  # OpenBSD doesn't support AI_V4MAPPED and doesn't define the macro AI_V4MAPPED.
+  # FreeBSD doesn't support AI_V4MAPPED but defines the macro.
   # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198092
-  when not defined(freebsd):
+  when not defined(freebsd) and not defined(openbsd) and not defined(netbsd):
+    if domain == AF_INET6:
       hints.ai_flags = AI_V4MAPPED
   var gaiResult = getaddrinfo(address, $port, addr(hints), result)
   if gaiResult != 0'i32:

lib/pure/nimprof.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nim's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -117,24 +117,38 @@ when defined(memProfiler):
   var
     gTicker {.threadvar.}: int
 
-  proc hook(st: StackTrace, size: int) {.nimcall.} =
+  proc requestedHook(): bool {.nimcall.} =
     if gTicker == 0:
-      gTicker = -1
+      gTicker = SamplingInterval
+      result = true
+    dec gTicker
+
+  proc hook(st: StackTrace, size: int) {.nimcall.} =
     when defined(ignoreAllocationSize):
       hookAux(st, 1)
     else:
       hookAux(st, size)
-      gTicker = SamplingInterval
-    dec gTicker
 
 else:
   var
     t0 {.threadvar.}: Ticks
+    gTicker: int # we use an additional counter to
+                 # avoid calling 'getTicks' too frequently
+
+  proc requestedHook(): bool {.nimcall.} =
+    if interval == 0: result = true
+    elif gTicker == 0:
+      gTicker = 500
+      if getTicks() - t0 > interval:
+        result = true
+    else:
+      dec gTicker
 
   proc hook(st: StackTrace) {.nimcall.} =
+    #echo "profiling! ", interval
     if interval == 0:
       hookAux(st, 1)
-    elif int64(t0) == 0 or getTicks() - t0 > interval:
+    else:
       hookAux(st, 1)
       t0 = getTicks()
 
@@ -145,9 +159,10 @@ proc cmpEntries(a, b: ptr ProfileEntry): int =
   result = b.getTotal - a.getTotal
 
 proc `//`(a, b: int): string =
-  result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDefault, 2))
+  result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDecimal, 2))
 
 proc writeProfile() {.noconv.} =
+  system.profilingRequestedHook = nil
   when declared(system.StackTrace):
     system.profilerHook = nil
   const filename = "profile_results.txt"
@@ -193,14 +208,15 @@ var
 proc disableProfiling*() =
   when declared(system.StackTrace):
     atomicDec disabled
-    system.profilerHook = nil
+    system.profilingRequestedHook = nil
 
 proc enableProfiling*() =
   when declared(system.StackTrace):
     if atomicInc(disabled) >= 0:
-      system.profilerHook = hook
+      system.profilingRequestedHook = requestedHook
 
 when declared(system.StackTrace):
+  system.profilingRequestedHook = requestedHook
   system.profilerHook = hook
   addQuitProc(writeProfile)
 

lib/pure/numeric.nim

@@ -7,6 +7,9 @@
 #    distribution, for details about the copyright.
 #
 
+## **Warning:** This module will be moved out of the stdlib and into a
+## Nimble package, don't use it.
+
 type OneVarFunction* = proc (x: float): float
 
 {.deprecated: [TOneVarFunction: OneVarFunction].}

lib/pure/options.nim

@@ -28,7 +28,7 @@
 ##
 ## .. code-block:: nim
 ##
-##   import optionals
+##   import options
 ##
 ##   proc find(haystack: string, needle: char): Option[int] =
 ##     for i, c in haystack:
@@ -156,7 +156,7 @@ proc `$`*[T]( self: Option[T] ): string =
 when isMainModule:
   import unittest, sequtils
 
-  suite "optionals":
+  suite "options":
     # work around a bug in unittest
     let intNone = none(int)
     let stringNone = none(string)

lib/pure/os.nim

@@ -810,6 +810,10 @@ type
 
 {.deprecated: [TPathComponent: PathComponent].}
 
+proc staticWalkDir(dir: string; relative: bool): seq[
+                  tuple[kind: PathComponent, path: string]] =
+  discard
+
 iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {.
   tags: [ReadDirEffect].} =
   ## walks over the directory `dir` and yields for each directory or file in
@@ -833,6 +837,10 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path:
   ##   dirA/dirC
   ##   dirA/fileA1.txt
   ##   dirA/fileA2.txt
+  when nimvm:
+    for k, v in items(staticWalkDir(dir, relative)):
+      yield (k, v)
+  else:
     when defined(windows):
       var f: WIN32_FIND_DATA
       var h = findFirstFile(dir / "*", f)
@@ -1353,7 +1361,7 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} =
   # /proc/<pid>/file
   when defined(windows):
     when useWinUnicode:
-      var buf = cast[WideCString](alloc(256*2))
+      var buf = newWideCString("", 256)
       var len = getModuleFileNameW(0, buf, 256)
       result = buf$len
     else:

lib/pure/osproc.nim

@@ -886,7 +886,7 @@ elif not defined(useNimRtl):
     discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error))
     exitnow(1)
 
-  when defined(macosx) or defined(freebsd):
+  when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(android):
     var environ {.importc.}: cstringArray
 
   proc startProcessAfterFork(data: ptr StartProcessData) =
@@ -916,7 +916,7 @@ elif not defined(useNimRtl):
     discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC)
 
     if data.optionPoUsePath:
-      when defined(macosx) or defined(freebsd):
+      when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(android):
         # MacOSX doesn't have execvpe, so we need workaround.
         # On MacOSX we can arrive here only from fork, so this is safe:
         environ = data.sysEnv
@@ -937,6 +937,7 @@ elif not defined(useNimRtl):
     if p.inStream != nil: close(p.inStream)
     if p.outStream != nil: close(p.outStream)
     if p.errStream != nil: close(p.errStream)
+    if poParentStreams notin p.options:
       discard close(p.inHandle)
       discard close(p.outHandle)
       discard close(p.errHandle)

lib/pure/oswalkdir.nim

@@ -0,0 +1,27 @@
+
+## Compile-time only version for walkDir if you need it at compile-time
+## for JavaScript.
+
+type
+  PathComponent* = enum   ## Enumeration specifying a path component.
+    pcFile,               ## path refers to a file
+    pcLinkToFile,         ## path refers to a symbolic link to a file
+    pcDir,                ## path refers to a directory
+    pcLinkToDir           ## path refers to a symbolic link to a directory
+
+proc staticWalkDir(dir: string; relative: bool): seq[
+                  tuple[kind: PathComponent, path: string]] =
+  discard
+
+iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] =
+  for k, v in items(staticWalkDir(dir, relative)):
+    yield (k, v)
+
+iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string =
+  var stack = @[dir]
+  while stack.len > 0:
+    for k,p in walkDir(stack.pop()):
+      if k in filter:
+        case k
+        of pcFile, pcLinkToFile: yield p
+        of pcDir, pcLinkToDir: stack.add(p)

lib/pure/parseopt2.nim

@@ -70,7 +70,7 @@ when not defined(createNimRtl):
     ## Initializes option parser from current command line arguments.
     return initOptParser(commandLineParams())
 
-proc next*(p: var OptParser) {.rtl, extern: "npo$1".}
+proc next*(p: var OptParser) {.rtl, extern: "npo2$1".}
 
 proc nextOption(p: var OptParser, token: string, allowEmpty: bool) =
   for splitchar in [':', '=']:
@@ -113,7 +113,7 @@ proc next(p: var OptParser) =
     p.key = token
     p.val = ""
 
-proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo$1", deprecated.} =
+proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1", deprecated.} =
   ## Returns part of command line string that has not been parsed yet.
   ## Do not use - does not correctly handle whitespace.
   return p.cmd[p.pos..p.cmd.len-1].join(" ")

lib/pure/parseutils.nim

@@ -25,7 +25,7 @@ const
 proc toLower(c: char): char {.inline.} =
   result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c
 
-proc parseHex*(s: string, number: var int, start = 0): int {.
+proc parseHex*(s: string, number: var int, start = 0; maxLen = 0): int {.
   rtl, extern: "npuParseHex", noSideEffect.}  =
   ## Parses a hexadecimal number and stores its value in ``number``.
   ##
@@ -45,11 +45,14 @@ proc parseHex*(s: string, number: var int, start = 0): int {.
   ##   discard parseHex("0x38", value)
   ##   assert value == -200
   ##
+  ## If 'maxLen==0' the length of the hexadecimal number has no
+  ## upper bound. Not more than ```maxLen`` characters are parsed.
   var i = start
   var foundDigit = false
   if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2)
   elif s[i] == '#': inc(i)
-  while true:
+  let last = if maxLen == 0: s.len else: i+maxLen
+  while i < last:
     case s[i]
     of '_': discard
     of '0'..'9':

lib/pure/poly.nim

@@ -7,6 +7,9 @@
 #    distribution, for details about the copyright.
 #
 
+## **Warning:** This module will be moved out of the stdlib and into a
+## Nimble package, don't use it.
+
 import math
 import strutils
 import numeric

lib/pure/romans.nim

@@ -9,6 +9,9 @@
 
 ## Module for converting an integer to a Roman numeral.
 ## See http://en.wikipedia.org/wiki/Roman_numerals for reference.
+##
+## **Warning:** This module will be moved out of the stdlib and into a
+## Nimble package, don't use it.
 
 const
   RomanNumeralDigits* = {'I', 'i', 'V', 'v', 'X', 'x', 'L', 'l', 'C', 'c',

lib/pure/stats.nim

@@ -0,0 +1,348 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+## Statistical analysis framework for performing
+## basic statistical analysis of data.
+## The data is analysed in a single pass, when a data value
+## is pushed to the ``RunningStat`` or ``RunningRegress`` objects
+##
+## ``RunningStat`` calculates for a single data set
+## - n (data count)
+## - min  (smallest value)
+## - max  (largest value)
+## - sum
+## - mean
+## - variance
+## - varianceS (sample var)
+## - standardDeviation
+## - standardDeviationS  (sample stddev)
+## - skewness (the third statistical moment)
+## - kurtosis (the fourth statistical moment)
+##
+## ``RunningRegress`` calculates for two sets of data
+## - n
+## - slope
+## - intercept
+## - correlation
+##
+## Procs have been provided to calculate statistics on arrays and sequences.
+##
+## However, if more than a single statistical calculation is required, it is more
+## efficient to push the data once to the RunningStat object, and
+## call the numerous statistical procs for the RunningStat object.
+##
+## .. code-block:: Nim
+##
+##  var rs: RunningStat
+##  rs.push(MySeqOfData)
+##  rs.mean()
+##  rs.variance()
+##  rs.skewness()
+##  rs.kurtosis()
+
+from math import FloatClass, sqrt, pow, round
+
+{.push debugger:off .} # the user does not want to trace a part
+                       # of the standard library!
+{.push checks:off, line_dir:off, stack_trace:off.}
+
+type
+  RunningStat* = object             ## an accumulator for statistical data
+    n*: int                         ## number of pushed data
+    min*, max*, sum*: float         ## self-explaining
+    mom1, mom2, mom3, mom4: float   ## statistical moments, mom1 is mean
+
+
+  RunningRegress* = object  ## an accumulator for regression calculations
+    n*: int                 ## number of pushed data
+    x_stats*: RunningStat   ## stats for first set of data
+    y_stats*: RunningStat   ## stats for second set of data
+    s_xy: float             ## accumulated data for combined xy
+
+{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].}
+
+# ----------- RunningStat --------------------------
+proc clear*(s: var RunningStat) =
+  ## reset `s`
+  s.n = 0
+  s.min = toBiggestFloat(int.high)
+  s.max = 0.0
+  s.sum = 0.0
+  s.mom1 = 0.0
+  s.mom2 = 0.0
+  s.mom3 = 0.0
+  s.mom4 = 0.0
+
+proc push*(s: var RunningStat, x: float) =
+  ## pushes a value `x` for processing
+  if s.n == 0: s.min = x
+  inc(s.n)
+  # See Knuth TAOCP vol 2, 3rd edition, page 232
+  if s.min > x: s.min = x
+  if s.max < x: s.max = x
+  s.sum += x
+  let n = toFloat(s.n)
+  let delta = x - s.mom1
+  let delta_n = delta / toFloat(s.n)
+  let delta_n2 = delta_n * delta_n
+  let term1 = delta * delta_n * toFloat(s.n - 1)
+  s.mom4 += term1 * delta_n2 * (n*n - 3*n + 3) +
+              6*delta_n2*s.mom2 - 4*delta_n*s.mom3
+  s.mom3 += term1 * delta_n * (n - 2) - 3*delta_n*s.mom2
+  s.mom2 += term1
+  s.mom1 += delta_n
+
+proc push*(s: var RunningStat, x: int) =
+  ## pushes a value `x` for processing.
+  ##
+  ## `x` is simply converted to ``float``
+  ## and the other push operation is called.
+  s.push(toFloat(x))
+
+proc push*(s: var RunningStat, x: openarray[float|int]) =
+  ## pushes all values of `x` for processing.
+  ##
+  ## Int values of `x` are simply converted to ``float`` and
+  ## the other push operation is called.
+  for val in x:
+    s.push(val)
+
+proc mean*(s: RunningStat): float =
+  ## computes the current mean of `s`
+  result = s.mom1
+
+proc variance*(s: RunningStat): float =
+  ## computes the current population variance of `s`
+  result = s.mom2 / toFloat(s.n)
+
+proc varianceS*(s: RunningStat): float =
+  ## computes the current sample variance of `s`
+  if s.n > 1: result = s.mom2 / toFloat(s.n - 1)
+
+proc standardDeviation*(s: RunningStat): float =
+  ## computes the current population standard deviation of `s`
+  result = sqrt(variance(s))
+
+proc standardDeviationS*(s: RunningStat): float =
+  ## computes the current sample standard deviation of `s`
+  result = sqrt(varianceS(s))
+
+proc skewness*(s: RunningStat): float =
+  ## computes the current population skewness of `s`
+  result = sqrt(toFloat(s.n)) * s.mom3 / pow(s.mom2, 1.5)
+
+proc skewnessS*(s: RunningStat): float =
+  ## computes the current sample skewness of `s`
+  let s2 = skewness(s)
+  result = sqrt(toFloat(s.n*(s.n-1)))*s2 / toFloat(s.n-2)
+
+proc kurtosis*(s: RunningStat): float =
+  ## computes the current population kurtosis of `s`
+  result = toFloat(s.n) * s.mom4 / (s.mom2 * s.mom2) - 3.0
+
+proc kurtosisS*(s: RunningStat): float =
+  ## computes the current sample kurtosis of `s`
+  result = toFloat(s.n-1) / toFloat((s.n-2)*(s.n-3)) *
+              (toFloat(s.n+1)*kurtosis(s) + 6)
+
+proc `+`*(a, b: RunningStat): RunningStat =
+  ## combine two RunningStats.
+  ##
+  ## Useful if performing parallel analysis of data series
+  ## and need to re-combine parallel result sets
+  result.clear()
+  result.n = a.n + b.n
+
+  let delta = b.mom1 - a.mom1
+  let delta2 = delta*delta
+  let delta3 = delta*delta2
+  let delta4 = delta2*delta2
+  let n = toFloat(result.n)
+
+  result.mom1 = (a.n.float*a.mom1 + b.n.float*b.mom1) / n
+  result.mom2 = a.mom2 + b.mom2 + delta2 * a.n.float * b.n.float / n
+  result.mom3 = a.mom3 + b.mom3 +
+                delta3 * a.n.float * b.n.float * (a.n.float - b.n.float)/(n*n);
+  result.mom3 += 3.0*delta * (a.n.float*b.mom2 - b.n.float*a.mom2) / n
+  result.mom4 = a.mom4 + b.mom4 +
+            delta4*a.n.float*b.n.float * toFloat(a.n*a.n - a.n*b.n + b.n*b.n) /
+                (n*n*n)
+  result.mom4 += 6.0*delta2 * (a.n.float*a.n.float*b.mom2 + b.n.float*b.n.float*a.mom2) /
+                (n*n) +
+                4.0*delta*(a.n.float*b.mom3 - b.n.float*a.mom3) / n
+  result.max = max(a.max, b.max)
+  result.min = max(a.min, b.min)
+
+proc `+=`*(a: var RunningStat, b: RunningStat) {.inline.} =
+  ## add a second RunningStats `b` to `a`
+  a = a + b
+# ---------------------- standalone array/seq stats ---------------------
+proc mean*[T](x: openArray[T]): float =
+  ## computes the mean of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.mean()
+
+proc variance*[T](x: openArray[T]): float =
+  ## computes the population variance of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.variance()
+
+proc varianceS*[T](x: openArray[T]): float =
+  ## computes the sample variance of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.varianceS()
+
+proc standardDeviation*[T](x: openArray[T]): float =
+  ## computes the population standardDeviation of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.standardDeviation()
+
+proc standardDeviationS*[T](x: openArray[T]): float =
+  ## computes the sanple standardDeviation of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.standardDeviationS()
+
+proc skewness*[T](x: openArray[T]): float =
+  ## computes the population skewness of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.skewness()
+
+proc skewnessS*[T](x: openArray[T]): float =
+  ## computes the sample skewness of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.skewnessS()
+
+proc kurtosis*[T](x: openArray[T]): float =
+  ## computes the population kurtosis of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.kurtosis()
+
+proc kurtosisS*[T](x: openArray[T]): float =
+  ## computes the sample kurtosis of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.kurtosisS()
+
+# ---------------------- Running Regression -----------------------------
+
+proc clear*(r: var RunningRegress) =
+  ## reset `r`
+  r.x_stats.clear()
+  r.y_stats.clear()
+  r.s_xy = 0.0
+  r.n = 0
+
+proc push*(r: var RunningRegress, x, y: float) =
+  ## pushes two values `x` and `y` for processing
+  r.s_xy += (r.x_stats.mean() - x)*(r.y_stats.mean() - y)*
+                toFloat(r.n) / toFloat(r.n + 1)
+  r.x_stats.push(x)
+  r.y_stats.push(y)
+  inc(r.n)
+
+proc push*(r: var RunningRegress, x, y: int) {.inline.} =
+  ## pushes two values `x` and `y` for processing.
+  ##
+  ## `x` and `y` are converted to ``float``
+  ## and the other push operation is called.
+  r.push(toFloat(x), toFloat(y))
+
+proc push*(r: var RunningRegress, x, y: openarray[float|int]) =
+  ## pushes two sets of values `x` and `y` for processing.
+  assert(x.len == y.len)
+  for i in 0..<x.len:
+    r.push(x[i], y[i])
+
+proc slope*(r: RunningRegress): float =
+  ## computes the current slope of `r`
+  let s_xx = r.x_stats.varianceS()*toFloat(r.n - 1)
+  result = r.s_xy / s_xx
+
+proc intercept*(r: RunningRegress): float =
+  ## computes the current intercept of `r`
+  result = r.y_stats.mean() - r.slope()*r.x_stats.mean()
+
+proc correlation*(r: RunningRegress): float =
+  ## computes the current correlation of the two data
+  ## sets pushed into `r`
+  let t = r.x_stats.standardDeviation() * r.y_stats.standardDeviation()
+  result = r.s_xy / ( toFloat(r.n) * t )
+
+proc `+`*(a, b: RunningRegress):  RunningRegress =
+  ## combine two `RunningRegress` objects.
+  ##
+  ## Useful if performing parallel analysis of data series
+  ## and need to re-combine parallel result sets
+  result.clear()
+  result.x_stats = a.x_stats + b.x_stats
+  result.y_stats = a.y_stats + b.y_stats
+  result.n = a.n + b.n
+
+  let delta_x = b.x_stats.mean() - a.x_stats.mean()
+  let delta_y = b.y_stats.mean() - a.y_stats.mean()
+  result.s_xy = a.s_xy + b.s_xy +
+      toFloat(a.n*b.n)*delta_x*delta_y/toFloat(result.n)
+
+proc `+=`*(a: var RunningRegress, b: RunningRegress) =
+  ## add RunningRegress `b` to `a`
+  a = a + b
+
+{.pop.}
+{.pop.}
+
+when isMainModule:
+  proc clean(x: float): float =
+    result = round(1.0e8*x).float * 1.0e-8
+
+  var rs: RunningStat
+  rs.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0])
+  doAssert(rs.n == 8)
+  doAssert(clean(rs.mean) == 2.0)
+  doAssert(clean(rs.variance()) == 1.5)
+  doAssert(clean(rs.varianceS()) == 1.71428571)
+  doAssert(clean(rs.skewness()) == 0.81649658)
+  doAssert(clean(rs.skewnessS()) == 1.01835015)
+  doAssert(clean(rs.kurtosis()) == -1.0)
+  doAssert(clean(rs.kurtosisS()) == -0.7000000000000001)
+
+  var rs1, rs2: RunningStat
+  rs1.push(@[1.0, 2.0, 1.0, 4.0])
+  rs2.push(@[1.0, 4.0, 1.0, 2.0])
+  let rs3 = rs1 + rs2
+  doAssert(clean(rs3.mom2) == clean(rs.mom2))
+  doAssert(clean(rs3.mom3) == clean(rs.mom3))
+  doAssert(clean(rs3.mom4) == clean(rs.mom4))
+  rs1 += rs2
+  doAssert(clean(rs1.mom2) == clean(rs.mom2))
+  doAssert(clean(rs1.mom3) == clean(rs.mom3))
+  doAssert(clean(rs1.mom4) == clean(rs.mom4))
+  rs1.clear()
+  rs1.push(@[1.0, 2.2, 1.4, 4.9])
+  doAssert(rs1.sum == 9.5)
+  doAssert(rs1.mean() == 2.375)
+
+  var rr: RunningRegress
+  rr.push(@[0.0,1.0,2.8,3.0,4.0], @[0.0,1.0,2.3,3.0,4.0])
+  doAssert(rr.slope() == 0.9695585996955861)
+  doAssert(rr.intercept() == -0.03424657534246611)
+  doAssert(rr.correlation() == 0.9905100362239381)
+  var rr1, rr2: RunningRegress
+  rr1.push(@[0.0,1.0], @[0.0,1.0])
+  rr2.push(@[2.8,3.0,4.0], @[2.3,3.0,4.0])
+  let rr3 = rr1 + rr2
+  doAssert(rr3.correlation() == rr.correlation())
+  doAssert(clean(rr3.slope()) == clean(rr.slope()))
+  doAssert(clean(rr3.intercept()) == clean(rr.intercept()))

lib/pure/strutils.nim

@@ -1210,22 +1210,21 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
   ## If `s` does not begin with ``prefix`` and end with ``suffix`` a
   ## ValueError exception will be raised.
   result = newStringOfCap(s.len)
-  var i = 0
+  var i = prefix.len
   if not s.startsWith(prefix):
     raise newException(ValueError,
                        "String does not start with a prefix of: " & prefix)
-  inc(i)
   while true:
     if i == s.len-suffix.len: break
     case s[i]
     of '\\':
       case s[i+1]:
       of 'x':
-        inc i
+        inc i, 2
         var c: int
-        i += parseutils.parseHex(s, c, i)
+        i += parseutils.parseHex(s, c, i, maxLen=2)
         result.add(chr(c))
-        inc(i, 2)
+        dec i, 2
       of '\\':
         result.add('\\')
       of '\'':
@@ -1281,7 +1280,7 @@ proc editDistance*(a, b: string): int {.noSideEffect,
 
   # another special case:
   if len1 == 1:
-    for j in s..len2-1:
+    for j in s..s+len2-1:
       if a[s] == b[j]: return len2 - 1
     return len2
 
@@ -1344,7 +1343,7 @@ proc editDistance*(a, b: string): int {.noSideEffect,
 
 
 # floating point formating:
-
+when not defined(js):
   proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>",
                                      importc: "sprintf", varargs, noSideEffect.}
 
@@ -1370,6 +1369,21 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault,
   ## after the decimal point for Nim's ``biggestFloat`` type.
   ##
   ## If ``precision == 0``, it tries to format it nicely.
+  when defined(js):
+    var res: cstring
+    case format
+    of ffDefault:
+      {.emit: "`res` = `f`.toString();".}
+    of ffDecimal:
+      {.emit: "`res` = `f`.toFixed(`precision`);".}
+    of ffScientific:
+      {.emit: "`res` = `f`.toExponential(`precision`);".}
+    result = $res
+    for i in 0 ..< result.len:
+      # Depending on the locale either dot or comma is produced,
+      # but nothing else is possible:
+      if result[i] in {'.', ','}: result[i] = decimalsep
+  else:
     const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e']
     var
       frmtstr {.noinit.}: array[0..5, char]
@@ -1706,3 +1720,4 @@ when isMainModule:
   doAssert isUpper("ABC")
   doAssert(not isUpper("AAcc"))
   doAssert(not isUpper("A#$"))
+  doAssert(unescape(r"\x013", "", "") == "\x013")

lib/pure/times.nim

@@ -29,7 +29,7 @@
 ##  echo "epochTime() float value: ", epochTime()
 ##  echo "getTime()   float value: ", toSeconds(getTime())
 ##  echo "cpuTime()   float value: ", cpuTime()
-##  echo "An hour from now      : ", getLocalTime(getTime()) + initInterval(0,0,0,1)
+##  echo "An hour from now      : ", getLocalTime(getTime()) + 1.hours
 ##  echo "An hour from (UTC) now: ", getGmTime(getTime()) + initInterval(0,0,0,1)
 
 {.push debugger:off.} # the user does not want to trace a part
@@ -171,11 +171,6 @@ type
 {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time,
     TTimeInterval: TimeInterval, TTimeInfo: TimeInfo].}
 
-proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds
-
-proc `miliseconds=`*(t:var TimeInterval, milliseconds: int) {.deprecated.} =
-  t.milliseconds = milliseconds
-
 proc getTime*(): Time {.tags: [TimeEffect], benign.}
   ## gets the current calendar time as a UNIX epoch value (number of seconds
   ## elapsed since 1970) with integer precission. Use epochTime for higher
@@ -245,13 +240,59 @@ proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.}
 proc initInterval*(milliseconds, seconds, minutes, hours, days, months,
                    years: int = 0): TimeInterval =
   ## creates a new ``TimeInterval``.
-  result.milliseconds = milliseconds
+  ##
-  result.seconds = seconds
+  ## You can also use the convenience procedures called ``milliseconds``,
-  result.minutes = minutes
+  ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``.
-  result.hours = hours
+  ##
-  result.days = days
+  ## Example:
-  result.months = months
+  ##
-  result.years = years
+  ## .. code-block:: nim
+  ##
+  ##     let day = initInterval(hours=24)
+  ##     let tomorrow = getTime() + day
+  ##     echo(tomorrow)
+  var carryO = 0
+  result.milliseconds = `mod`(milliseconds, 1000)
+  carryO = `div`(milliseconds, 1000)
+  result.seconds = `mod`(carryO + seconds, 60)
+  carryO = `div`(seconds, 60)
+  result.minutes = `mod`(carryO + minutes, 60)
+  carryO = `div`(minutes, 60)
+  result.hours = `mod`(carryO + hours, 24)
+  carryO = `div`(hours, 24)
+  result.days = carryO + days
+  carryO = 0
+  result.months = `mod`(months, 12)
+  carryO = `div`(months, 12)
+  result.years = carryO + years
+
+proc `+`*(ti1, ti2: TimeInterval): TimeInterval =
+  ## Adds two ``TimeInterval`` objects together.
+  var carryO = 0
+  result.milliseconds = `mod`(ti1.milliseconds + ti2.milliseconds, 1000)
+  carryO = `div`(ti1.milliseconds + ti2.milliseconds, 1000)
+  result.seconds = `mod`(carryO + ti1.seconds + ti2.seconds, 60)
+  carryO = `div`(ti1.seconds + ti2.seconds, 60)
+  result.minutes = `mod`(carryO + ti1.minutes + ti2.minutes, 60)
+  carryO = `div`(ti1.minutes + ti2.minutes, 60)
+  result.hours = `mod`(carryO + ti1.hours + ti2.hours, 24)
+  carryO = `div`(ti1.hours + ti2.hours, 24)
+  result.days = carryO + ti1.days + ti2.days
+  carryO = 0
+  result.months = `mod`(ti1.months + ti2.months, 12)
+  carryO = `div`(ti1.months + ti2.months, 12)
+  result.years = carryO + ti1.years + ti2.years
+
+proc `-`*(ti1, ti2: TimeInterval): TimeInterval =
+  ## Subtracts TimeInterval ``ti1`` from ``ti2``.
+  result = ti1
+  result.milliseconds -= ti2.milliseconds
+  result.seconds -= ti2.seconds
+  result.minutes -= ti2.minutes
+  result.hours -= ti2.hours
+  result.days -= ti2.days
+  result.months -= ti2.months
+  result.years -= ti2.years
 
 proc isLeapYear*(year: int): bool =
   ## returns true if ``year`` is a leap year
@@ -288,6 +329,15 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float =
 
   newinterv.months += interval.years * 12
   var curMonth = anew.month
+  if newinterv.months < 0:   # subtracting
+    for mth in countDown(-1 * newinterv.months, 1):
+      result -= float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60)
+      if curMonth == mJan:
+        curMonth = mDec
+        anew.year.dec()
+      else:
+        curMonth.dec()
+  else:  # adding
     for mth in 1 .. newinterv.months:
       result += float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60)
       if curMonth == mDec:
@@ -302,28 +352,39 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float =
   result += newinterv.milliseconds / 1000
 
 proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo =
-  ## adds ``interval`` time.
+  ## adds ``interval`` time from TimeInfo ``a``.
   ##
   ## **Note:** This has been only briefly tested and it may not be
   ## very accurate.
   let t = toSeconds(timeInfoToTime(a))
   let secs = toSeconds(a, interval)
-  #if a.tzname == "UTC":
-  #  result = getGMTime(fromSeconds(t + secs))
-  #else:
   result = getLocalTime(fromSeconds(t + secs))
 
 proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo =
-  ## subtracts ``interval`` time.
+  ## subtracts ``interval`` time from TimeInfo ``a``.
   ##
   ## **Note:** This has been only briefly tested, it is inaccurate especially
   ## when you subtract so much that you reach the Julian calendar.
   let t = toSeconds(timeInfoToTime(a))
-  let secs = toSeconds(a, interval)
+  var intval: TimeInterval
-  #if a.tzname == "UTC":
+  intval.milliseconds = - interval.milliseconds
-  #  result = getGMTime(fromSeconds(t - secs))
+  intval.seconds = - interval.seconds
-  #else:
+  intval.minutes = - interval.minutes
-  result = getLocalTime(fromSeconds(t - secs))
+  intval.hours = - interval.hours
+  intval.days = - interval.days
+  intval.months = - interval.months
+  intval.years = - interval.years
+  let secs = toSeconds(a, intval)
+  result = getLocalTime(fromSeconds(t + secs))
+
+proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds
+
+proc `miliseconds=`*(t: var TimeInterval, milliseconds: int) {.deprecated.} =
+  ## An alias for a misspelled field in ``TimeInterval``.
+  ##
+  ## **Warning:** This should not be used! It will be removed in the next
+  ## version.
+  t.milliseconds = milliseconds
 
 when not defined(JS):
   proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].}
@@ -603,6 +664,69 @@ proc `$`*(m: Month): string =
       "November", "December"]
   return lookup[m]
 
+proc milliseconds*(ms: int): TimeInterval {.inline.} =
+  ## TimeInterval of `ms` milliseconds
+  ##
+  ## Note: not all time functions have millisecond resolution
+  initInterval(`mod`(ms,1000), `div`(ms,1000))
+
+proc seconds*(s: int): TimeInterval {.inline.} =
+  ## TimeInterval of `s` seconds
+  ##
+  ## ``echo getTime() + 5.second``
+  initInterval(0,`mod`(s,60), `div`(s,60))
+
+proc minutes*(m: int): TimeInterval {.inline.} =
+  ## TimeInterval of `m` minutes
+  ##
+  ## ``echo getTime() + 5.minutes``
+  initInterval(0,0,`mod`(m,60), `div`(m,60))
+
+proc hours*(h: int): TimeInterval {.inline.} =
+  ## TimeInterval of `h` hours
+  ##
+  ## ``echo getTime() + 2.hours``
+  initInterval(0,0,0,`mod`(h,24),`div`(h,24))
+
+proc days*(d: int): TimeInterval {.inline.} =
+  ## TimeInterval of `d` days
+  ##
+  ## ``echo getTime() + 2.days``
+  initInterval(0,0,0,0,d)
+
+proc months*(m: int): TimeInterval {.inline.} =
+  ## TimeInterval of `m` months
+  ##
+  ## ``echo getTime() + 2.months``
+  initInterval(0,0,0,0,0,`mod`(m,12),`div`(m,12))
+
+proc years*(y: int): TimeInterval {.inline.} =
+  ## TimeInterval of `y` years
+  ##
+  ## ``echo getTime() + 2.years``
+  initInterval(0,0,0,0,0,0,y)
+
+proc `+=`*(t: var Time, ti: TimeInterval) =
+  ## modifies `t` by adding the interval `ti`
+  t = timeInfoToTime(getLocalTime(t) + ti)
+
+proc `+`*(t: Time, ti: TimeInterval): Time =
+  ## adds the interval `ti` to Time `t`
+  ## by converting to localTime, adding the interval, and converting back
+  ##
+  ## ``echo getTime() + 1.day``
+  result = timeInfoToTime(getLocalTime(t) + ti)
+
+proc `-=`*(t: var Time, ti: TimeInterval) =
+  ## modifies `t` by subtracting the interval `ti`
+  t = timeInfoToTime(getLocalTime(t) - ti)
+
+proc `-`*(t: Time, ti: TimeInterval): Time =
+  ## adds the interval `ti` to Time `t`
+  ##
+  ## ``echo getTime() - 1.day``
+  result = timeInfoToTime(getLocalTime(t) - ti)
+
 proc formatToken(info: TimeInfo, token: string, buf: var string) =
   ## Helper of the format proc to parse individual tokens.
   ##
@@ -1192,112 +1316,10 @@ proc timeToTimeInterval*(t: Time): TimeInterval =
   # Milliseconds not available from Time
 
 when isMainModule:
-  # $ date --date='@2147483647'
+  # this is testing non-exported function
-  # Tue 19 Jan 03:14:07 GMT 2038
+  var
-
+    t4 = getGMTime(fromSeconds(876124714)) # Mon  6 Oct 08:58:34 BST 1997
-  var t = getGMTime(fromSeconds(2147483647))
+    t4L = getLocalTime(fromSeconds(876124714))
-  assert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038"
-  assert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038"
-
-  assert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" &
-    " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") ==
-    "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 0 00 00:00 UTC"
-
-  assert t.format("yyyyMMddhhmmss") == "20380119031407"
-
-  var t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975
-  assert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" &
-    " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") ==
-    "27 27 Mon Monday 4 04 16 16 6 06 1 01 Jan January 29 29 P PM 5 75 975 1975 01975 0 00 00:00 UTC"
-
-  when not defined(JS):
-    when sizeof(Time) == 8:
-      var t3 = getGMTime(fromSeconds(889067643645)) # Fri  7 Jun 19:20:45 BST 30143
-      assert t3.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" &
-        " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") ==
-        "7 07 Fri Friday 6 06 18 18 20 20 6 06 Jun June 45 45 P PM 3 43 143 0143 30143 0 00 00:00 UTC"
-      assert t3.format(":,[]()-/") == ":,[]()-/"
-
-  var t4 = getGMTime(fromSeconds(876124714)) # Mon  6 Oct 08:58:34 BST 1997
-  assert t4.format("M MM MMM MMMM") == "10 10 Oct October"
-
-  # Interval tests
-  assert((t4 - initInterval(years = 2)).format("yyyy") == "1995")
-  assert((t4 - initInterval(years = 7, minutes = 34, seconds = 24)).format("yyyy mm ss") == "1990 24 10")
-
-  var s = "Tuesday at 09:04am on Dec 15, 2015"
-  var f = "dddd at hh:mmtt on MMM d, yyyy"
-  assert($s.parse(f) == "Tue Dec 15 09:04:00 2015")
-  # ANSIC       = "Mon Jan _2 15:04:05 2006"
-  s = "Thu Jan 12 15:04:05 2006"
-  f = "ddd MMM dd HH:mm:ss yyyy"
-  assert($s.parse(f) == "Thu Jan 12 15:04:05 2006")
-  # UnixDate    = "Mon Jan _2 15:04:05 MST 2006"
-  s = "Thu Jan 12 15:04:05 MST 2006"
-  f = "ddd MMM dd HH:mm:ss ZZZ yyyy"
-  assert($s.parse(f) == "Thu Jan 12 15:04:05 2006")
-  # RubyDate    = "Mon Jan 02 15:04:05 -0700 2006"
-  s = "Thu Jan 12 15:04:05 -07:00 2006"
-  f = "ddd MMM dd HH:mm:ss zzz yyyy"
-  assert($s.parse(f) == "Thu Jan 12 15:04:05 2006")
-  # RFC822      = "02 Jan 06 15:04 MST"
-  s = "12 Jan 16 15:04 MST"
-  f = "dd MMM yy HH:mm ZZZ"
-  assert($s.parse(f) == "Tue Jan 12 15:04:00 2016")
-  # RFC822Z     = "02 Jan 06 15:04 -0700" # RFC822 with numeric zone
-  s = "12 Jan 16 15:04 -07:00"
-  f = "dd MMM yy HH:mm zzz"
-  assert($s.parse(f) == "Tue Jan 12 15:04:00 2016")
-  # RFC850      = "Monday, 02-Jan-06 15:04:05 MST"
-  s = "Monday, 12-Jan-06 15:04:05 MST"
-  f = "dddd, dd-MMM-yy HH:mm:ss ZZZ"
-  assert($s.parse(f) == "Thu Jan 12 15:04:05 2006")
-  # RFC1123     = "Mon, 02 Jan 2006 15:04:05 MST"
-  s = "Thu, 12 Jan 2006 15:04:05 MST"
-  f = "ddd, dd MMM yyyy HH:mm:ss ZZZ"
-  assert($s.parse(f) == "Thu Jan 12 15:04:05 2006")
-  # RFC1123Z    = "Mon, 02 Jan 2006 15:04:05 -0700" # RFC1123 with numeric zone
-  s = "Thu, 12 Jan 2006 15:04:05 -07:00"
-  f = "ddd, dd MMM yyyy HH:mm:ss zzz"
-  assert($s.parse(f) == "Thu Jan 12 15:04:05 2006")
-  # RFC3339     = "2006-01-02T15:04:05Z07:00"
-  s = "2006-01-12T15:04:05Z-07:00"
-  f = "yyyy-MM-ddTHH:mm:ssZzzz"
-  assert($s.parse(f) == "Thu Jan 12 15:04:05 2006")
-  f = "yyyy-MM-dd'T'HH:mm:ss'Z'zzz"
-  assert($s.parse(f) == "Thu Jan 12 15:04:05 2006")
-  # RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00"
-  s = "2006-01-12T15:04:05.999999999Z-07:00"
-  f = "yyyy-MM-ddTHH:mm:ss.999999999Zzzz"
-  assert($s.parse(f) == "Thu Jan 12 15:04:05 2006")
-  # Kitchen     = "3:04PM"
-  s = "3:04PM"
-  f = "h:mmtt"
-  assert "15:04:00" in $s.parse(f)
-  when not defined(testing):
-    echo "Kitchen: " & $s.parse(f)
-    var ti = timeToTimeInfo(getTime())
-    echo "Todays date after decoding: ", ti
-    var tint = timeToTimeInterval(getTime())
-    echo "Todays date after decoding to interval: ", tint
-  # checking dayOfWeek matches known days
-  assert getDayOfWeek(21, 9, 1900) == dFri
-  assert getDayOfWeek(1, 1, 1970) == dThu
-  assert getDayOfWeek(21, 9, 1970) == dMon
-  assert getDayOfWeek(1, 1, 2000) == dSat
-  assert getDayOfWeek(1, 1, 2021) == dFri
-  # Julian tests
-  assert getDayOfWeekJulian(21, 9, 1900) == dFri
-  assert getDayOfWeekJulian(21, 9, 1970) == dMon
-  assert getDayOfWeekJulian(1, 1, 2000) == dSat
-  assert getDayOfWeekJulian(1, 1, 2021) == dFri
-
-  # toSeconds tests with GM and Local timezones
-  #var t4 = getGMTime(fromSeconds(876124714)) # Mon  6 Oct 08:58:34 BST 1997
-  var t4L = getLocalTime(fromSeconds(876124714))
-  assert toSeconds(timeInfoToTime(t4L)) == 876124714    # fromSeconds is effectively "localTime"
-  assert toSeconds(timeInfoToTime(t4L)) + t4L.timezone.float == toSeconds(timeInfoToTime(t4))
-
   assert toSeconds(t4, initInterval(seconds=0)) == 0.0
   assert toSeconds(t4L, initInterval(milliseconds=1)) == toSeconds(t4, initInterval(milliseconds=1))
   assert toSeconds(t4L, initInterval(seconds=1)) == toSeconds(t4, initInterval(seconds=1))
@@ -1307,12 +1329,5 @@ when isMainModule:
   assert toSeconds(t4L, initInterval(months=1)) == toSeconds(t4, initInterval(months=1))
   assert toSeconds(t4L, initInterval(years=1)) == toSeconds(t4, initInterval(years=1))
 
-  # adding intervals
+  # Further tests are in tests/stdlib/ttime.nim
-  var
+  # koch test c stdlib
-    a1L = toSeconds(timeInfoToTime(t4L + initInterval(hours = 1))) + t4L.timezone.float
-    a1G = toSeconds(timeInfoToTime(t4)) + 60.0 * 60.0
-  assert a1L == a1G
-  # subtracting intervals
-  a1L = toSeconds(timeInfoToTime(t4L - initInterval(hours = 1))) + t4L.timezone.float
-  a1G = toSeconds(timeInfoToTime(t4)) - (60.0 * 60.0)
-  assert a1L == a1G

lib/pure/unicode.nim

@@ -114,6 +114,7 @@ proc validateUtf8*(s: string): int =
     if ord(s[i]) <=% 127:
       inc(i)
     elif ord(s[i]) shr 5 == 0b110:
+      if ord(s[i]) < 0xc2: return i # Catch overlong ascii representations.
       if i+1 < L and ord(s[i+1]) shr 6 == 0b10: inc(i, 2)
       else: return i
     elif ord(s[i]) shr 4 == 0b1110:

lib/pure/xmlparser.nim

@@ -96,7 +96,7 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode =
     next(x)
   of xmlEntity:
     ## &entity;
-    errors.add(errorMsg(x, "unknown entity: " & x.entityName))
+    result = newEntity(x.entityName)
     next(x)
   of xmlEof: discard
 
@@ -143,7 +143,8 @@ proc loadXml*(path: string): XmlNode =
   result = loadXml(path, errors)
   if errors.len > 0: raiseInvalidXml(errors)
 
-when not defined(testing) and isMainModule:
+when isMainModule:
+  when not defined(testing):
     import os
 
     var errors: seq[string] = @[]
@@ -156,4 +157,10 @@ when not defined(testing) and isMainModule:
       f.close()
     else:
       quit("cannot write test.txt")
+  else:
+    block: # correctly parse ../../tests/testdata/doc1.xml
+      let filePath = "tests/testdata/doc1.xml"
+      var errors: seq[string] = @[]
+      var xml = loadXml(filePath, errors)
+      assert(errors.len == 0, "The file tests/testdata/doc1.xml should be parsed without errors.")
 

lib/stdlib.nimble

@@ -1,6 +1,6 @@
 [Package]
 name          = "stdlib"
-version       = "0.9.0"
+version       = "0.13.0"
 author        = "Dominik Picheta"
 description   = "Nim's standard library."
 license       = "MIT"

lib/system.nim

@@ -232,8 +232,8 @@ proc low*[T](x: T): T {.magic: "Low", noSideEffect.}
   ##
   ## .. code-block:: nim
   ##  var arr = [1,2,3,4,5,6,7]
-  ##  high(arr) #=> 0
+  ##  low(arr) #=> 0
-  ##  high(2) #=> -9223372036854775808
+  ##  low(2) #=> -9223372036854775808
 
 type
   range*{.magic: "Range".}[T] ## Generic type to construct range types.
@@ -840,7 +840,7 @@ proc `div` *(x, y: int32): int32 {.magic: "DivI", noSideEffect.}
   ##   1 div 2 == 0
   ##   2 div 2 == 1
   ##   3 div 2 == 1
-  ##   7 div 5 == 2
+  ##   7 div 5 == 1
 
 when defined(nimnomagic64):
   proc `div` *(x, y: int64): int64 {.magic: "DivI", noSideEffect.}
@@ -1808,10 +1808,10 @@ const
   NimMajor*: int = 0
     ## is the major number of Nim's version.
 
-  NimMinor*: int = 12
+  NimMinor*: int = 13
     ## is the minor number of Nim's version.
 
-  NimPatch*: int = 1
+  NimPatch*: int = 0
     ## is the patch number of Nim's version.
 
   NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch
@@ -2584,11 +2584,7 @@ when not defined(JS): #and not defined(nimscript):
 
   when hasAlloc:
     var
-      strDesc: TNimType
+      strDesc = TNimType(size: sizeof(string), kind: tyString, flags: {ntfAcyclic})
-
-    strDesc.size = sizeof(string)
-    strDesc.kind = tyString
-    strDesc.flags = {ntfAcyclic}
 
   when not defined(nimscript):
     include "system/ansi_c"

lib/system/alloc.nim

@@ -399,6 +399,9 @@ iterator allObjects(m: MemRegion): pointer {.inline.} =
           let c = cast[PBigChunk](c)
           yield addr(c.data)
 
+proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {.
+                      magic: "Plugin", compileTime.}
+
 proc isCell(p: pointer): bool {.inline.} =
   result = cast[ptr FreeCell](p).zeroField >% 1