merged upstream master

.gitignore

@@ -6,6 +6,7 @@
 nimcache
 lib/nimcache
 tools/nimcache
+tools/nimweb
 tests/nimcache
 tests/accept/run/nimcache
 tests/accept/compile/nimcache

compiler/ast.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -172,6 +172,9 @@ type
     nkDiscardStmt,        # a discard statement
     nkStmtList,           # a list of statements
     nkImportStmt,         # an import statement
+    nkImportExceptStmt,   # an import x except a statement
+    nkExportStmt,         # an export statement
+    nkExportExceptStmt,   # an 'export except' statement
     nkFromStmt,           # a from * import statement
     nkIncludeStmt,        # an include statement
     nkBindStmt,           # a bind statement
@@ -647,6 +650,7 @@ type
     loc*: TLoc
     annex*: PLib              # additional fields (seldom used, so we use a
                               # reference to another object to safe space)
+    constraint*: PNode        # additional constraints like 'lit|result'
   
   TTypeSeq* = seq[PType]
   TType* = object of TIdObj   # types are identical iff they have the
@@ -673,7 +677,6 @@ type
     align*: int               # the type's alignment requirements
     containerID*: int         # used for type checking of generics
     loc*: TLoc
-    constraint*: PNode        # additional constraints like 'lit|result'
 
   TPair*{.final.} = object 
     key*, val*: PObject

compiler/astalgo.nim

@@ -429,7 +429,7 @@ proc debugTree(n: PNode, indent: int, maxRecDepth: int): PRope =
                    [istr, makeYamlString($n.kind)])
     if maxRecDepth != 0: 
       case n.kind
-      of nkCharLit..nkInt64Lit: 
+      of nkCharLit..nkUInt64Lit:
         appf(result, ",$N$1\"intVal\": $2", [istr, toRope(n.intVal)])
       of nkFloatLit, nkFloat32Lit, nkFloat64Lit: 
         appf(result, ",$N$1\"floatVal\": $2", 
@@ -586,7 +586,8 @@ proc StrTableRawInsert(data: var TSymSeq, n: PSym) =
   var h: THash = n.name.h and high(data)
   while data[h] != nil: 
     if data[h] == n:
-      InternalError(n.info, "StrTableRawInsert: " & n.name.s)
+      # allowed for 'export' feature:
+      #InternalError(n.info, "StrTableRawInsert: " & n.name.s)
       return
     h = nextTry(h, high(data))
   assert(data[h] == nil)
@@ -617,7 +618,7 @@ proc StrTableAdd(t: var TStrTable, n: PSym) =
   StrTableRawInsert(t.data, n)
   inc(t.counter)
 
-proc StrTableIncl*(t: var TStrTable, n: PSym): bool = 
+proc StrTableIncl*(t: var TStrTable, n: PSym): bool {.discardable.} =
   # returns true if n is already in the string table:
   # It is essential that `n` is written nevertheless!
   # This way the newest redefinition is picked by the semantic analyses!

compiler/babelcmd.nim

@@ -0,0 +1,90 @@
+#
+#
+#           The Nimrod Compiler
+#        (c) Copyright 2012 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implements some helper procs for Babel (Nimrod's package manager) support.
+
+import parseutils, strutils, strtabs, os, options, msgs, lists
+
+proc addPath*(path: string, info: TLineInfo) = 
+  if not contains(options.searchPaths, path): 
+    lists.PrependStr(options.searchPaths, path)
+
+proc versionSplitPos(s: string): int =
+  result = s.len-2
+  while result > 1 and s[result] in {'0'..'9', '.'}: dec result
+  if s[result] != '-': result = s.len
+
+const
+  latest = "head"
+
+proc `<.`(a, b: string): bool = 
+  # wether a has a smaller version than b:
+  if a == latest: return false
+  var i = 0
+  var j = 0
+  var verA = 0
+  var verB = 0
+  while true:
+    let ii = parseInt(a, verA, i)
+    let jj = parseInt(b, verB, j)
+    # if A has no number left, but B has, B is prefered:  0.8 vs 0.8.3
+    if ii <= 0 or jj <= 0: return jj > 0
+    if verA < verB: return true
+    elif verA > verB: return false
+    # else: same version number; continue:
+    inc i, ii
+    inc j, jj
+    if a[i] == '.': inc i
+    if b[j] == '.': inc j
+
+proc addPackage(packages: PStringTable, p: string) =
+  let x = versionSplitPos(p)
+  let name = p.subStr(0, x-1)
+  if x < p.len:
+    let version = p.subStr(x+1)
+    if packages[name] <. version:
+      packages[name] = version
+  else:
+    packages[name] = latest
+
+iterator chosen(packages: PStringTable): string =
+  for key, val in pairs(packages):
+    let res = if val == latest: key else: key & '-' & val
+    yield res
+
+proc addBabelPath(p: string, info: TLineInfo) =
+  if not contains(options.searchPaths, p):
+    Message(info, hintPath, p)
+    lists.PrependStr(options.lazyPaths, p)
+
+proc addPathWithNimFiles(p: string, info: TLineInfo) =
+  proc hasNimFile(dir: string): bool =
+    for kind, path in walkDir(dir):
+      if kind == pcFile and path.endsWith(".nim"):
+        result = true
+        break
+  if hasNimFile(p):
+    addBabelPath(p, info)
+  else:
+    for kind, p2 in walkDir(p):
+      if hasNimFile(p2): addBabelPath(p2, info)
+
+proc addPathRec(dir: string, info: TLineInfo) =
+  var packages = newStringTable(modeStyleInsensitive)
+  var pos = dir.len-1
+  if dir[pos] in {DirSep, AltSep}: inc(pos)
+  for k,p in os.walkDir(dir):
+    if k == pcDir and p[pos] != '.':
+      addPackage(packages, p)
+  for p in packages.chosen:
+    addPathWithNimFiles(p, info)
+
+proc babelPath*(path: string, info: TLineInfo) =
+  addPathRec(path, info)
+  addBabelPath(path, info)

compiler/c2nim/cpp.nim

@@ -42,6 +42,7 @@ proc parseDefine(p: var TParser): PNode =
     result = newNodeP(nkTemplateDef, p)
     getTok(p)
     addSon(result, skipIdentExport(p))
+    addSon(result, ast.emptyNode)
     eat(p, pxParLe)
     var params = newNodeP(nkFormalParams, p)
     # return type; not known yet:
@@ -60,6 +61,7 @@ proc parseDefine(p: var TParser): PNode =
     addSon(result, ast.emptyNode) # no generic parameters
     addSon(result, params)
     addSon(result, ast.emptyNode) # no pragmas
+    addSon(result, ast.emptyNode)
     var kind = parseDefineBody(p, result)
     params.sons[0] = newIdentNodeP(kind, p)
     eatNewLine(p, result)

compiler/c2nim/tests/systest.c

@@ -17,6 +17,8 @@ int   aw_instance_callback_set (AW_CALLBACK c, callback_t callback);
 
 unsigned long int wawa;
 
+#define MAX(x, y) ((x) < (y)? (y) : (x))
+
 #define AW_BUILD 85 // AW 5.0
 // Limits
 #define AW_MAX_AVCHANGE_PER_SECOND 10

compiler/ccgcalls.nim

@@ -146,7 +146,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   proc addComma(r: PRope): PRope =
     result = if r == nil: r else: con(r, ~", ")
 
-  const CallPattern = "$1.ClEnv? $1.ClPrc($3$1.ClEnv) : (($4)($1.ClPrc))($2);$n"
+  const CallPattern = "$1.ClEnv? $1.ClPrc($3$1.ClEnv) : (($4)($1.ClPrc))($2)"
   var op: TLoc
   initLocExpr(p, ri.sons[0], op)
   var pl: PRope
@@ -164,7 +164,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     if i < length - 1: app(pl, ~", ")
   
   template genCallPattern =
-    lineF(p, cpsStmts, CallPattern, op.r, pl, pl.addComma, rawProc)
+    lineF(p, cpsStmts, CallPattern & ";$n", op.r, pl, pl.addComma, rawProc)
 
   let rawProc = getRawProcType(p, typ)
   if typ.sons[0] != nil:

compiler/ccgexprs.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -939,22 +939,31 @@ proc genNew(p: BProc, e: PNode) =
   var
     a, b: TLoc
     reftype, bt: PType
+    sizeExpr: PRope
   refType = skipTypes(e.sons[1].typ, abstractVarRange)
   InitLocExpr(p, e.sons[1], a)
   initLoc(b, locExpr, a.t, OnHeap)
+  # 'genNew' also handles 'unsafeNew':
+  if e.len == 3:
+    var se: TLoc
+    InitLocExpr(p, e.sons[2], se)
+    sizeExpr = se.rdLoc
+  else:
+    sizeExpr = ropef("sizeof($1)",
+        getTypeDesc(p.module, skipTypes(reftype.sons[0], abstractRange)))
   let args = [getTypeDesc(p.module, reftype),
               genTypeInfo(p.module, refType),
-              getTypeDesc(p.module, skipTypes(reftype.sons[0], abstractRange))]
+              sizeExpr]
   if a.s == OnHeap and optRefcGc in gGlobalOptions:
     # use newObjRC1 as an optimization; and we don't need 'keepAlive' either
     if canFormAcycle(a.t):
       linefmt(p, cpsStmts, "if ($1) #nimGCunref($1);$n", a.rdLoc)
     else:
       linefmt(p, cpsStmts, "if ($1) #nimGCunrefNoCycle($1);$n", a.rdLoc)
-    b.r = ropecg(p.module, "($1) #newObjRC1($2, sizeof($3))", args)
+    b.r = ropecg(p.module, "($1) #newObjRC1($2, $3)", args)
     linefmt(p, cpsStmts, "$1 = $2;$n", a.rdLoc, b.rdLoc)
   else:
-    b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", args)
+    b.r = ropecg(p.module, "($1) #newObj($2, $3)", args)
     genAssignment(p, a, b, {needToKeepAlive})  # set the object type:
   bt = skipTypes(refType.sons[0], abstractRange)
   genObjectInit(p, cpsStmts, bt, a, false)
@@ -1645,7 +1654,7 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) =
     initLocExpr(p, n.sons[0], a)
     var r = rdLoc(a)
     if skipTypes(n.sons[0].typ, abstractInst).kind in {tyRef, tyPtr, tyVar} and
-        n.sons[0].kind notin {nkHiddenAddr, nkAddr}:
+        n.sons[0].kind notin {nkHiddenAddr, nkAddr, nkObjDownConv}:
       app(r, "->Sup")
       for i in countup(2, abs(inheritanceDiff(dest, src))): app(r, ".Sup")
       r = con("&", r)

compiler/ccgmerge.nim

@@ -148,20 +148,47 @@ proc atEndMark(buf: cstring, pos: int): bool =
   while s < NimMergeEndMark.len and buf[pos+s] == NimMergeEndMark[s]: inc s
   result = s == NimMergeEndMark.len
 
+when false:
+  proc readVerbatimSection(L: var TBaseLexer): PRope = 
+    var pos = L.bufpos
+    var buf = L.buf
+    result = newMutableRope(30_000)
+    while true:
+      case buf[pos]
+      of CR:
+        pos = lexbase.HandleCR(L, pos)
+        buf = L.buf
+        result.data.add(tnl)
+      of LF:
+        pos = lexbase.HandleLF(L, pos)
+        buf = L.buf
+        result.data.add(tnl)
+      of '\0':
+        InternalError("ccgmerge: expected: " & NimMergeEndMark)
+        break
+      else: 
+        if atEndMark(buf, pos):
+          inc pos, NimMergeEndMark.len
+          break
+        result.data.add(buf[pos])
+        inc pos
+    L.bufpos = pos
+    freezeMutableRope(result)
+
 proc readVerbatimSection(L: var TBaseLexer): PRope = 
   var pos = L.bufpos
   var buf = L.buf
-  result = newMutableRope(30_000)
+  var r = newStringOfCap(30_000)
   while true:
     case buf[pos]
     of CR:
       pos = lexbase.HandleCR(L, pos)
       buf = L.buf
-      result.data.add(tnl)
+      r.add(tnl)
     of LF:
       pos = lexbase.HandleLF(L, pos)
       buf = L.buf
-      result.data.add(tnl)
+      r.add(tnl)
     of '\0':
       InternalError("ccgmerge: expected: " & NimMergeEndMark)
       break
@@ -169,10 +196,10 @@ proc readVerbatimSection(L: var TBaseLexer): PRope =
       if atEndMark(buf, pos):
         inc pos, NimMergeEndMark.len
         break
-      result.data.add(buf[pos])
+      r.add(buf[pos])
       inc pos
   L.bufpos = pos
-  freezeMutableRope(result)
+  result = r.toRope
 
 proc readKey(L: var TBaseLexer, result: var string) =
   var pos = L.bufpos

compiler/ccgstmts.nim

@@ -122,6 +122,7 @@ proc genBreakState(p: BProc, n: PNode) =
     lineF(p, cpsStmts, "if ((((NI*) $1.ClEnv)[0]) < 0) break;$n", [rdLoc(a)])
   #  lineF(p, cpsStmts, "if (($1) < 0) break;$n", [rdLoc(a)])
 
+proc genVarPrototypeAux(m: BModule, sym: PSym)
 proc genSingleVar(p: BProc, a: PNode) =
   var v = a.sons[0].sym
   if sfCompileTime in v.flags: return
@@ -140,6 +141,9 @@ proc genSingleVar(p: BProc, a: PNode) =
     genObjectInit(p.module.preInitProc, cpsInit, v.typ, v.loc, true)
     # Alternative construction using default constructor (which may zeromem):
     # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc)
+    if sfExportc in v.flags and generatedHeader != nil:
+      genVarPrototypeAux(generatedHeader, v)
+
   else:
     assignLocalVar(p, v)
     initLocalVar(p, v, immediateAsgn)
@@ -878,7 +882,8 @@ proc genStmts(p: BProc, t: PNode) =
     # we have to emit the type information for object types here to support
     # separate compilation:
     genTypeSection(p.module, t)
-  of nkCommentStmt, nkNilLit, nkIteratorDef, nkIncludeStmt, nkImportStmt, 
+  of nkCommentStmt, nkNilLit, nkIteratorDef, nkIncludeStmt, 
+     nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, 
      nkFromStmt, nkTemplateDef, nkMacroDef: 
     nil
   of nkPragma: genPragma(p, t)

compiler/cgen.nim

@@ -558,17 +558,6 @@ include "ccgexprs.nim", "ccgstmts.nim"
 # ----------------------------- dynamic library handling -----------------
 # We don't finalize dynamic libs as this does the OS for us.
 
-proc libCandidates(s: string, dest: var TStringSeq) = 
-  var le = strutils.find(s, '(')
-  var ri = strutils.find(s, ')', le+1)
-  if le >= 0 and ri > le:
-    var prefix = substr(s, 0, le - 1)
-    var suffix = substr(s, ri + 1)
-    for middle in split(substr(s, le + 1, ri - 1), '|'):
-      libCandidates(prefix & middle & suffix, dest)
-  else: 
-    add(dest, s)
-
 proc isGetProcAddr(lib: PLib): bool =
   let n = lib.path
   result = n.kind in nkCallKinds and n.typ != nil and 
@@ -870,8 +859,6 @@ proc genVarPrototypeAux(m: BModule, sym: PSym) =
 
 proc genVarPrototype(m: BModule, sym: PSym) =
   genVarPrototypeAux(m, sym)
-  if sfExportc in sym.flags and generatedHeader != nil:
-    genVarPrototypeAux(generatedHeader, sym)
 
 proc addIntTypes(result: var PRope) {.inline.} =
   appf(result, "#define NIM_INTBITS $1", [

compiler/cgmeth.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -109,14 +109,13 @@ proc methodDef*(s: PSym, fromCache: bool) =
 proc relevantCol(methods: TSymSeq, col: int): bool =
   # returns true iff the position is relevant
   var t = methods[0].typ.sons[col]
-  result = false
   if skipTypes(t, skipPtrs).kind == tyObject:
     for i in countup(1, high(methods)):
-      if not SameType(methods[i].typ.sons[col], t): 
+      let t2 = skipTypes(methods[i].typ.sons[col], skipPtrs)
+      if not SameType(t2, t):
         return true
   
 proc cmpSignatures(a, b: PSym, relevantCols: TIntSet): int = 
-  result = 0
   for col in countup(1, sonsLen(a.typ) - 1): 
     if Contains(relevantCols, col): 
       var aa = skipTypes(a.typ.sons[col], skipPtrs)

compiler/commands.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -11,7 +11,7 @@
 
 import 
   os, msgs, options, nversion, condsyms, strutils, extccomp, platform, lists, 
-  wordrecg, parseutils
+  wordrecg, parseutils, babelcmd
 
 proc writeCommandLineUsage*()
 
@@ -28,7 +28,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdlinePass, info: TLineInfo)
 
 const
   HelpMessage = "Nimrod Compiler Version $1 (" & compileDate & ") [$2: $3]\n" &
-      "Copyright (c) 2004-2012 by Andreas Rumpf\n"
+      "Copyright (c) 2004-2013 by Andreas Rumpf\n"
 
 const 
   Usage = slurp"doc/basicopt.txt".replace("//", "")
@@ -194,20 +194,6 @@ proc processPath(path: string): string =
     "projectname", options.gProjectName,
     "projectpath", options.gProjectPath])
 
-proc addPath(path: string, info: TLineInfo) = 
-  if not contains(options.searchPaths, path): 
-    lists.PrependStr(options.searchPaths, path)
-
-proc addPathRec(dir: string, info: TLineInfo) =
-  var pos = dir.len-1
-  if dir[pos] in {DirSep, AltSep}: inc(pos)
-  for k,p in os.walkDir(dir):
-    if k == pcDir and p[pos] != '.':
-      addPathRec(p, info)
-      if not contains(options.searchPaths, p): 
-        Message(info, hintPath, p)
-        lists.PrependStr(options.searchPaths, p)
-
 proc track(arg: string, info: TLineInfo) = 
   var a = arg.split(',')
   if a.len != 3: LocalError(info, errTokenExpected, "FILE,LINE,COLUMN")
@@ -227,11 +213,16 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "path", "p": 
     expectArg(switch, arg, pass, info)
     addPath(processPath(arg), info)
-  of "recursivepath":
+  of "babelpath":
+    if pass in {passCmd2, passPP}:
+      expectArg(switch, arg, pass, info)
+      let path = processPath(arg)
+      babelpath(path, info)
+  of "excludepath":
     expectArg(switch, arg, pass, info)
-    var path = processPath(arg)
+    let path = processPath(arg)
-    addPathRec(path, info)
+    lists.ExcludeStr(options.searchPaths, path)
-    addPath(path, info)
+    lists.ExcludeStr(options.lazyPaths, path)
   of "nimcache":
     expectArg(switch, arg, pass, info)
     options.nimcacheDir = processPath(arg)
@@ -474,6 +465,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "stdout":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optStdout)
+  of "listfullpaths":
+    expectNoArg(switch, arg, pass, info)
+    gListFullPaths = true
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(switch, arg)
     else: InvalidCmdLineOption(pass, switch, info)

compiler/condsyms.nim

@@ -60,6 +60,7 @@ proc InitDefines*() =
   DefineSymbol("niminheritable")
   DefineSymbol("nimmixin")
   DefineSymbol("nimeffects")
+  DefineSymbol("nimbabel")
   
   # add platform specific symbols:
   case targetCPU

compiler/depends.nim

@@ -33,7 +33,7 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
     for i in countup(0, sonsLen(n) - 1): 
       var imported = getModuleName(n.sons[i])
       addDependencyAux(g.module.name.s, imported)
-  of nkFromStmt: 
+  of nkFromStmt, nkImportExceptStmt: 
     var imported = getModuleName(n.sons[0])
     addDependencyAux(g.module.name.s, imported)
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr: 

compiler/docgen.nim

@@ -138,6 +138,26 @@ proc genRecComment(d: PDoc, n: PNode): PRope =
   else:
     n.comment = nil
 
+proc findDocComment(n: PNode): PNode =
+  if n == nil: return nil
+  if not isNil(n.comment) and startsWith(n.comment, "##"): return n
+  for i in countup(0, safeLen(n)-1):
+    result = findDocComment(n.sons[i])
+    if result != nil: return
+
+proc extractDocComment*(s: PSym, d: PDoc = nil): string =
+  let n = findDocComment(s.ast)
+  result = ""
+  if not n.isNil:
+    if not d.isNil:
+      var dummyHasToc: bool
+      renderRstToOut(d[], parseRst(n.comment, toFilename(n.info),
+                                   toLineNumber(n.info), toColumn(n.info),
+                                   dummyHasToc, d.options + {roSkipPounds}),
+                     result)
+    else:
+      result = n.comment.substr(2).replace("\n##", "\n").strip
+
 proc isVisible(n: PNode): bool = 
   result = false
   if n.kind == nkPostfix: 
@@ -145,6 +165,9 @@ proc isVisible(n: PNode): bool =
       var v = n.sons[0].ident
       result = v.id == ord(wStar) or v.id == ord(wMinus)
   elif n.kind == nkSym:
+    # we cannot generate code for forwarded symbols here as we have no
+    # exception tracking information here. Instead we copy over the comment
+    # from the proc header.
     result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported}
   elif n.kind == nkPragmaExpr:
     result = isVisible(n.sons[0])
@@ -273,7 +296,7 @@ proc generateDoc*(d: PDoc, n: PNode) =
       generateDoc(d, lastSon(n.sons[0]))
   of nkImportStmt:
     for i in 0 .. sonsLen(n)-1: traceDeps(d, n.sons[i]) 
-  of nkFromStmt: traceDeps(d, n.sons[0])
+  of nkFromStmt, nkImportExceptStmt: traceDeps(d, n.sons[0])
   else: nil
 
 proc genSection(d: PDoc, kind: TSymKind) = 
@@ -350,9 +373,11 @@ proc CommandRstAux(filename, outExt: string) =
   var d = newDocumentor(filen, options.gConfigVars)
   var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc,
                      {roSupportRawDirective})
-  d.modDesc = newMutableRope(30_000)
+  var modDesc = newStringOfCap(30_000)
-  renderRstToOut(d[], rst, d.modDesc.data)
+  #d.modDesc = newMutableRope(30_000)
-  freezeMutableRope(d.modDesc)
+  renderRstToOut(d[], rst, modDesc)
+  #freezeMutableRope(d.modDesc)
+  d.modDesc = toRope(modDesc)
   writeOutput(d, filename, outExt)
   generateIndex(d)
 

compiler/ecmasgen.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -207,20 +207,35 @@ proc genObjectInfo(p: var TProc, typ: PType, name: PRope) =
     appf(p.g.typeInfo, "$1.base = $2;$n", 
          [name, genTypeInfo(p, typ.sons[0])])
 
+proc genTupleFields(p: var TProc, typ: PType): PRope =
+  var s: PRope = nil
+  for i in 0 .. <typ.len:
+    if i > 0: app(s, ", " & tnl)
+    s.appf("{kind: 1, offset: \"Field$1\", len: 0, " &
+           "typ: $2, name: \"Field$1\", sons: null}",
+           [i.toRope, genTypeInfo(p, typ.sons[i])])
+  result = ropef("{kind: 2, len: $1, offset: 0, " &
+                 "typ: null, name: null, sons: [$2]}", [toRope(typ.len), s])
+
+proc genTupleInfo(p: var TProc, typ: PType, name: PRope) = 
+  var s = ropef("var $1 = {size: 0, kind: $2, base: null, node: null, " &
+                "finalizer: null};$n", [name, toRope(ord(typ.kind))])
+  prepend(p.g.typeInfo, s)
+  appf(p.g.typeInfo, "var NNI$1 = $2;$n", 
+       [toRope(typ.id), genTupleFields(p, typ)])
+  appf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, toRope(typ.id)])
+
 proc genEnumInfo(p: var TProc, typ: PType, name: PRope) =
-  var 
+  let length = sonsLen(typ.n)
-    s, n: PRope
+  var s: PRope = nil
-    length: int
-    field: PSym
-  length = sonsLen(typ.n)
-  s = nil
   for i in countup(0, length - 1): 
     if (typ.n.sons[i].kind != nkSym): InternalError(typ.n.info, "genEnumInfo")
-    field = typ.n.sons[i].sym
+    let field = typ.n.sons[i].sym
     if i > 0: app(s, ", " & tnl)
+    let extName = if field.ast == nil: field.name.s else: field.ast.strVal
     appf(s, "{kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}", 
-         [toRope(field.position), name, makeCString(field.name.s)])
+         [toRope(field.position), name, makeCString(extName)])
-  n = ropef("var NNI$1 = {kind: 2, offset: 0, typ: null, " &
+  var n = ropef("var NNI$1 = {kind: 2, offset: 0, typ: null, " &
       "name: null, len: $2, sons: [$3]};$n", [toRope(typ.id), toRope(length), s])
   s = ropef("var $1 = {size: 0, kind: $2, base: null, node: null, " &
       "finalizer: null};$n", [name, toRope(ord(typ.kind))])
@@ -259,7 +274,8 @@ proc genTypeInfo(p: var TProc, typ: PType): PRope =
     appf(p.g.typeInfo, "$1.base = $2;$n", 
          [result, genTypeInfo(p, typ.sons[1])])
   of tyEnum: genEnumInfo(p, t, result)
-  of tyObject, tyTuple: genObjectInfo(p, t, result)
+  of tyObject: genObjectInfo(p, t, result)
+  of tyTuple: genTupleInfo(p, t, result)
   else: InternalError("genTypeInfo(" & $t.kind & ')')
   
 proc gen(p: var TProc, n: PNode, r: var TCompRes)
@@ -938,7 +954,9 @@ proc genSym(p: var TProc, n: PNode, r: var TCompRes) =
   of skProc, skConverter, skMethod:
     discard mangleName(s)
     r.res = s.loc.r
-    if lfNoDecl in s.loc.flags or s.magic != mNone or isGenericRoutine(s): nil
+    if lfNoDecl in s.loc.flags or s.magic != mNone or isGenericRoutine(s) or
+       {sfImportc, sfInfixCall} * s.flags != {}:
+      nil
     elif s.kind == skMethod and s.getBody.kind == nkEmpty:
       # we cannot produce code for the dispatcher yet:
       nil
@@ -962,24 +980,45 @@ proc genDeref(p: var TProc, n: PNode, r: var TCompRes) =
     if a.kind != etyBaseIndex: InternalError(n.info, "genDeref")
     r.res = ropef("$1[$2]", [a.com, a.res])
 
+proc genArg(p: var TProc, n: PNode, r: var TCompRes) =
+  var a: TCompRes
+  gen(p, n, a)
+  if a.kind == etyBaseIndex: 
+    app(r.res, a.com)
+    app(r.res, ", ")
+    app(r.res, a.res)
+  else:
+    app(r.res, mergeExpr(a))
+
 proc genArgs(p: var TProc, n: PNode, r: var TCompRes) =
   app(r.res, "(")
   for i in countup(1, sonsLen(n) - 1): 
     if i > 1: app(r.res, ", ")
-    var a: TCompRes
+    genArg(p, n.sons[i], r)
-    gen(p, n.sons[i], a)
-    if a.kind == etyBaseIndex: 
-      app(r.res, a.com)
-      app(r.res, ", ")
-      app(r.res, a.res)
-    else: 
-      app(r.res, mergeExpr(a))
   app(r.res, ")")
 
 proc genCall(p: var TProc, n: PNode, r: var TCompRes) = 
   gen(p, n.sons[0], r)
   genArgs(p, n, r)
 
+proc genInfixCall(p: var TProc, n: PNode, r: var TCompRes) =
+  gen(p, n.sons[1], r)
+  if r.kind == etyBaseIndex:
+    if r.com == nil:
+      GlobalError(n.info, "cannot invoke with infix syntax")
+    r.res = ropef("$1[0]", [r.res, r.com])
+    r.com = nil
+  app(r.res, ".")
+  var op: TCompRes
+  gen(p, n.sons[0], op)
+  app(r.res, mergeExpr(op))
+  
+  app(r.res, "(")
+  for i in countup(2, sonsLen(n) - 1):
+    if i > 2: app(r.res, ", ")
+    genArg(p, n.sons[i], r)
+  app(r.res, ")")
+
 proc genEcho(p: var TProc, n: PNode, r: var TCompRes) =
   useMagic(p, "rawEcho")
   app(r.res, "rawEcho")
@@ -1176,10 +1215,13 @@ proc genConStrStr(p: var TProc, n: PNode, r: var TCompRes) =
 proc genRepr(p: var TProc, n: PNode, r: var TCompRes) =
   var t = skipTypes(n.sons[1].typ, abstractVarRange)
   case t.kind
-  of tyInt..tyInt64:
+  of tyInt..tyUInt64:
-    unaryExpr(p, n, r, "", "reprInt($1)")
+    unaryExpr(p, n, r, "", "(\"\"+ ($1))")
   of tyEnum, tyOrdinal:
-    binaryExpr(p, n, r, "", "reprEnum($1, $2)")
+    gen(p, n.sons[1], r)
+    useMagic(p, "cstrToNimstr")
+    r.res = ropef("cstrToNimstr($1.node.sons[$2].name)", 
+                 [genTypeInfo(p, t), r.res])
   else:
     # XXX:
     internalError(n.info, "genRepr: Not implemented")
@@ -1463,7 +1505,8 @@ proc genStmt(p: var TProc, n: PNode, r: var TCompRes) =
   of nkAsmStmt: genAsmStmt(p, n, r)
   of nkTryStmt: genTryStmt(p, n, r)
   of nkRaiseStmt: genRaiseStmt(p, n, r)
-  of nkTypeSection, nkCommentStmt, nkIteratorDef, nkIncludeStmt, nkImportStmt, 
+  of nkTypeSection, nkCommentStmt, nkIteratorDef, nkIncludeStmt, 
+     nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, 
      nkFromStmt, nkTemplateDef, nkMacroDef, nkPragma: nil
   of nkProcDef, nkMethodDef, nkConverterDef:
     var s = n.sons[namePos].sym
@@ -1513,6 +1556,9 @@ proc gen(p: var TProc, n: PNode, r: var TCompRes) =
   of nkCallKinds:
     if (n.sons[0].kind == nkSym) and (n.sons[0].sym.magic != mNone): 
       genMagic(p, n, r)
+    elif n.sons[0].kind == nkSym and sfInfixCall in n.sons[0].sym.flags and
+      n.len >= 2:
+      genInfixCall(p, n, r)
     else: 
       genCall(p, n, r)
   of nkCurly: genSetConstr(p, n, r)
@@ -1526,6 +1572,7 @@ proc gen(p: var TProc, n: PNode, r: var TCompRes) =
   of nkCheckedFieldExpr: genCheckedFieldAccess(p, n, r)
   of nkObjDownConv: gen(p, n.sons[0], r)
   of nkObjUpConv: upConv(p, n, r)
+  of nkCast: gen(p, n.sons[1], r)
   of nkChckRangeF: genRangeChck(p, n, r, "chckRangeF")
   of nkChckRange64: genRangeChck(p, n, r, "chckRange64")
   of nkChckRange: genRangeChck(p, n, r, "chckRange")
@@ -1555,7 +1602,7 @@ proc newModule(module: PSym): BModule =
   
 proc genHeader(): PRope = 
   result = ropef("/* Generated by the Nimrod Compiler v$1 */$n" &
-      "/*   (c) 2012 Andreas Rumpf */$n$n" & "$nvar Globals = this;$n" &
+      "/*   (c) 2013 Andreas Rumpf */$n$n" & "$nvar Globals = this;$n" &
       "var framePtr = null;$n" & "var excHandler = null;$n", 
                  [toRope(versionAsString)])
 

compiler/evalffi.nim

@@ -0,0 +1,443 @@
+#
+#
+#           The Nimrod Compiler
+#        (c) Copyright 2012 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This file implements the FFI part of the evaluator for Nimrod code.
+
+import ast, astalgo, ropes, types, options, tables, dynlib, libffi, msgs
+
+when defined(windows):
+  const libcDll = "msvcrt.dll"
+else:
+  const libcDll = "libc.so(.6|.5|)"
+
+type
+  TDllCache = tables.TTable[string, TLibHandle]
+var
+  gDllCache = initTable[string, TLibHandle]()
+  gExeHandle = LoadLib()
+
+proc getDll(cache: var TDllCache; dll: string; info: TLineInfo): pointer =
+  result = cache[dll]
+  if result.isNil:
+    var libs: seq[string] = @[]
+    libCandidates(dll, libs)
+    for c in libs:
+      result = LoadLib(c)
+      if not result.isNil: break
+    if result.isNil:
+      GlobalError(info, "cannot load: " & dll)
+    cache[dll] = result
+
+const
+  nkPtrLit = nkIntLit # hopefully we can get rid of this hack soon
+
+proc importcSymbol*(sym: PSym): PNode =
+  let name = ropeToStr(sym.loc.r)
+  
+  # the AST does not support untyped pointers directly, so we use an nkIntLit
+  # that contains the address instead:
+  result = newNodeIT(nkPtrLit, sym.info, sym.typ)
+  case name
+  of "stdin":  result.intVal = cast[TAddress](system.stdin)
+  of "stdout": result.intVal = cast[TAddress](system.stdout)
+  of "stderr": result.intVal = cast[TAddress](system.stderr)
+  else:
+    let lib = sym.annex
+    if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}:
+      GlobalError(sym.info, "dynlib needs to be a string lit for the REPL")
+    var theAddr: pointer
+    if lib.isNil and not gExehandle.isNil:
+      # first try this exe itself:
+      theAddr = gExehandle.symAddr(name)
+      # then try libc:
+      if theAddr.isNil:
+        let dllhandle = gDllCache.getDll(libcDll, sym.info)
+        theAddr = dllhandle.checkedSymAddr(name)
+    else:
+      let dllhandle = gDllCache.getDll(lib.path.strVal, sym.info)
+      theAddr = dllhandle.checkedSymAddr(name)
+    result.intVal = cast[TAddress](theAddr)
+
+proc mapType(t: ast.PType): ptr libffi.TType =
+  if t == nil: return addr libffi.type_void
+  
+  case t.kind
+  of tyBool, tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64, tySet:
+    case t.getSize
+    of 1: result = addr libffi.type_uint8
+    of 2: result = addr libffi.type_sint16
+    of 4: result = addr libffi.type_sint32
+    of 8: result = addr libffi.type_sint64
+    else: result = nil
+  of tyFloat, tyFloat64: result = addr libffi.type_double
+  of tyFloat32: result = addr libffi.type_float
+  of tyVar, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr,
+     tyStmt, tyTypeDesc, tyProc, tyArray, tyArrayConstr, tyNil:
+    result = addr libffi.type_pointer
+  of tyDistinct:
+    result = mapType(t.sons[0])
+  else:
+    result = nil
+  # too risky:
+  #of tyFloat128: result = addr libffi.type_longdouble
+
+proc mapCallConv(cc: TCallingConvention, info: TLineInfo): TABI =
+  case cc
+  of ccDefault: result = DEFAULT_ABI
+  of ccStdCall: result = when defined(windows): STDCALL else: DEFAULT_ABI
+  of ccCDecl: result = DEFAULT_ABI
+  else:
+    GlobalError(info, "cannot map calling convention to FFI")
+
+template rd(T, p: expr): expr {.immediate.} = (cast[ptr T](p))[]
+template wr(T, p, v: expr) {.immediate.} = (cast[ptr T](p))[] = v
+template `+!`(x, y: expr): expr {.immediate.} =
+  cast[pointer](cast[TAddress](x) + y)
+
+proc packSize(v: PNode, typ: PType): int =
+  ## computes the size of the blob
+  case typ.kind
+  of tyPtr, tyRef, tyVar:
+    if v.kind in {nkNilLit, nkPtrLit}:
+      result = sizeof(pointer)
+    else:
+      result = sizeof(pointer) + packSize(v.sons[0], typ.sons[0])
+  of tyDistinct, tyGenericInst:
+    result = packSize(v, typ.sons[0])
+  of tyArray, tyArrayConstr:
+    # consider: ptr array[0..1000_000, int] which is common for interfacing;
+    # we use the real length here instead
+    if v.kind in {nkNilLit, nkPtrLit}:
+      result = sizeof(pointer)
+    elif v.len != 0:
+      result = v.len * packSize(v.sons[0], typ.sons[1])
+  else:
+    result = typ.getSize.int
+
+proc pack(v: PNode, typ: PType, res: pointer)
+
+proc getField(n: PNode; position: int): PSym =
+  case n.kind
+  of nkRecList:
+    for i in countup(0, sonsLen(n) - 1):
+      result = getField(n.sons[i], position)
+      if result != nil: return 
+  of nkRecCase:
+    result = getField(n.sons[0], position)
+    if result != nil: return
+    for i in countup(1, sonsLen(n) - 1):
+      case n.sons[i].kind
+      of nkOfBranch, nkElse:
+        result = getField(lastSon(n.sons[i]), position)
+        if result != nil: return
+      else: internalError(n.info, "getField(record case branch)")
+  of nkSym:
+    if n.sym.position == position: result = n.sym
+  else: nil
+
+proc packObject(x: PNode, typ: PType, res: pointer) =
+  InternalAssert x.kind == nkPar
+  # compute the field's offsets:
+  discard typ.getSize
+  for i in countup(0, sonsLen(x) - 1):
+    var it = x.sons[i]
+    if it.kind == nkExprColonExpr:
+      internalAssert it.sons[0].kind == nkSym
+      let field = it.sons[0].sym
+      pack(it.sons[1], field.typ, res +! field.offset)
+    elif typ.n != nil:
+      let field = getField(typ.n, i)
+      pack(it, field.typ, res +! field.offset)
+    else:
+      GlobalError(x.info, "cannot pack unnamed tuple")
+
+const maxPackDepth = 20
+var packRecCheck = 0
+
+proc pack(v: PNode, typ: PType, res: pointer) =
+  template awr(T, v: expr) {.immediate, dirty.} =
+    wr(T, res, v)
+
+  case typ.kind
+  of tyBool: awr(bool, v.intVal != 0)
+  of tyChar: awr(char, v.intVal.chr)
+  of tyInt:  awr(int, v.intVal.int)
+  of tyInt8: awr(int8, v.intVal.int8)
+  of tyInt16: awr(int16, v.intVal.int16)
+  of tyInt32: awr(int32, v.intVal.int32)
+  of tyInt64: awr(int64, v.intVal.int64)
+  of tyUInt: awr(uint, v.intVal.uint)
+  of tyUInt8: awr(uint8, v.intVal.uint8)
+  of tyUInt16: awr(uint16, v.intVal.uint16)
+  of tyUInt32: awr(uint32, v.intVal.uint32)
+  of tyUInt64: awr(uint64, v.intVal.uint64)
+  of tyEnum, tySet:
+    case v.typ.getSize
+    of 1: awr(uint8, v.intVal.uint8)
+    of 2: awr(uint16, v.intVal.uint16)
+    of 4: awr(int32, v.intVal.int32)
+    of 8: awr(int64, v.intVal.int64)
+    else:
+      GlobalError(v.info, "cannot map value to FFI (tyEnum, tySet)")
+  of tyFloat: awr(float, v.floatVal)
+  of tyFloat32: awr(float32, v.floatVal)
+  of tyFloat64: awr(float64, v.floatVal)
+  
+  of tyPointer, tyProc,  tyCString, tyString:
+    if v.kind == nkNilLit:
+      # nothing to do since the memory is 0 initialized anyway
+      nil
+    elif v.kind == nkPtrLit:
+      awr(pointer, cast[pointer](v.intVal))
+    elif v.kind in {nkStrLit..nkTripleStrLit}:
+      awr(cstring, cstring(v.strVal))
+    else:
+      GlobalError(v.info, "cannot map pointer/proc value to FFI")
+  of tyPtr, tyRef, tyVar:
+    if v.kind == nkNilLit:
+      # nothing to do since the memory is 0 initialized anyway
+      nil
+    elif v.kind == nkPtrLit:
+      awr(pointer, cast[pointer](v.intVal))
+    else:
+      if packRecCheck > maxPackDepth:
+        packRecCheck = 0
+        GlobalError(v.info, "cannot map value to FFI " & typeToString(v.typ))
+      inc packRecCheck
+      pack(v.sons[0], typ.sons[0], res +! sizeof(pointer))
+      dec packRecCheck
+      awr(pointer, res +! sizeof(pointer))
+  of tyArray, tyArrayConstr:
+    let baseSize = typ.sons[1].getSize
+    for i in 0 .. <v.len:
+      pack(v.sons[i], typ.sons[1], res +! i * baseSize)
+  of tyObject, tyTuple:
+    packObject(v, typ, res)
+  of tyNil:
+    nil
+  of tyDistinct, tyGenericInst:
+    pack(v, typ.sons[0], res)
+  else:
+    GlobalError(v.info, "cannot map value to FFI " & typeToString(v.typ))
+
+proc unpack(x: pointer, typ: PType, n: PNode): PNode
+
+proc unpackObjectAdd(x: pointer, n, result: PNode) =
+  case n.kind
+  of nkRecList:
+    for i in countup(0, sonsLen(n) - 1):
+      unpackObjectAdd(x, n.sons[i], result)
+  of nkRecCase:
+    GlobalError(result.info, "case objects cannot be unpacked")
+  of nkSym:
+    var pair = newNodeI(nkExprColonExpr, result.info, 2)
+    pair.sons[0] = n
+    pair.sons[1] = unpack(x +! n.sym.offset, n.sym.typ, nil)
+    #echo "offset: ", n.sym.name.s, " ", n.sym.offset
+    result.add pair
+  else: nil
+
+proc unpackObject(x: pointer, typ: PType, n: PNode): PNode =
+  # compute the field's offsets:
+  discard typ.getSize
+  
+  # iterate over any actual field of 'n' ... if n is nil we need to create
+  # the nkPar node:
+  if n.isNil:
+    result = newNode(nkPar)
+    result.typ = typ
+    if typ.n.isNil:
+      InternalError("cannot unpack unnamed tuple")
+    unpackObjectAdd(x, typ.n, result)
+  else:
+    result = n
+    if result.kind != nkPar:
+      GlobalError(n.info, "cannot map value from FFI")
+    if typ.n.isNil:
+      GlobalError(n.info, "cannot unpack unnamed tuple")
+    for i in countup(0, sonsLen(n) - 1):
+      var it = n.sons[i]
+      if it.kind == nkExprColonExpr:
+        internalAssert it.sons[0].kind == nkSym
+        let field = it.sons[0].sym
+        it.sons[1] = unpack(x +! field.offset, field.typ, it.sons[1])
+      else:
+        let field = getField(typ.n, i)
+        n.sons[i] = unpack(x +! field.offset, field.typ, it)
+
+proc unpackArray(x: pointer, typ: PType, n: PNode): PNode =
+  if n.isNil:
+    result = newNode(nkBracket)
+    result.typ = typ
+    newSeq(result.sons, lengthOrd(typ).int)
+  else:
+    result = n
+    if result.kind != nkBracket:
+      GlobalError(n.info, "cannot map value from FFI")
+  let baseSize = typ.sons[1].getSize
+  for i in 0 .. < result.len:
+    result.sons[i] = unpack(x +! i * baseSize, typ.sons[1], result.sons[i])
+
+proc canonNodeKind(k: TNodeKind): TNodeKind =
+  case k
+  of nkCharLit..nkUInt64Lit: result = nkIntLit
+  of nkFloatLit..nkFloat128Lit: result = nkFloatLit
+  of nkStrLit..nkTripleStrLit: result = nkStrLit
+  else: result = k
+
+proc unpack(x: pointer, typ: PType, n: PNode): PNode =
+  template aw(k, v, field: expr) {.immediate, dirty.} =
+    if n.isNil:
+      result = newNode(k)
+      result.typ = typ
+    else:
+      # check we have the right field:
+      result = n
+      if result.kind.canonNodeKind != k.canonNodeKind:
+        #echo "expected ", k, " but got ", result.kind
+        #debug result
+        return newNodeI(nkExceptBranch, n.info)
+        #GlobalError(n.info, "cannot map value from FFI")
+    result.field = v
+
+  template setNil() =
+    if n.isNil:
+      result = newNode(nkNilLit)
+      result.typ = typ
+    else:
+      reset n[]
+      result = n
+      result.kind = nkNilLit
+      result.typ = typ
+
+  template awi(kind, v: expr) {.immediate, dirty.} = aw(kind, v, intVal)
+  template awf(kind, v: expr) {.immediate, dirty.} = aw(kind, v, floatVal)
+  template aws(kind, v: expr) {.immediate, dirty.} = aw(kind, v, strVal)
+  
+  case typ.kind
+  of tyBool: awi(nkIntLit, rd(bool, x).ord)
+  of tyChar: awi(nkCharLit, rd(char, x).ord)
+  of tyInt:  awi(nkIntLit, rd(int, x))
+  of tyInt8: awi(nkInt8Lit, rd(int8, x))
+  of tyInt16: awi(nkInt16Lit, rd(int16, x))
+  of tyInt32: awi(nkInt32Lit, rd(int32, x))
+  of tyInt64: awi(nkInt64Lit, rd(int64, x))
+  of tyUInt: awi(nkUIntLit, rd(uint, x).biggestInt)
+  of tyUInt8: awi(nkUInt8Lit, rd(uint8, x).biggestInt)
+  of tyUInt16: awi(nkUInt16Lit, rd(uint16, x).biggestInt)
+  of tyUInt32: awi(nkUInt32Lit, rd(uint32, x).biggestInt)
+  of tyUInt64: awi(nkUInt64Lit, rd(uint64, x).biggestInt)
+  of tyEnum:
+    case typ.getSize
+    of 1: awi(nkIntLit, rd(uint8, x).biggestInt)
+    of 2: awi(nkIntLit, rd(uint16, x).biggestInt)
+    of 4: awi(nkIntLit, rd(int32, x).biggestInt)
+    of 8: awi(nkIntLit, rd(int64, x).biggestInt)
+    else:
+      GlobalError(n.info, "cannot map value from FFI (tyEnum, tySet)")
+  of tyFloat: awf(nkFloatLit, rd(float, x))
+  of tyFloat32: awf(nkFloat32Lit, rd(float32, x))
+  of tyFloat64: awf(nkFloat64Lit, rd(float64, x))
+  of tyPointer, tyProc:
+    let p = rd(pointer, x)
+    if p.isNil:
+      setNil()
+    elif n != nil and n.kind == nkStrLit:
+      # we passed a string literal as a pointer; however strings are already
+      # in their unboxed representation so nothing it to be unpacked:
+      result = n
+    else:
+      awi(nkPtrLit, cast[TAddress](p))
+  of tyPtr, tyRef, tyVar:
+    let p = rd(pointer, x)
+    if p.isNil:
+      setNil()
+    elif n == nil or n.kind == nkPtrLit:
+      awi(nkPtrLit, cast[TAddress](p))
+    elif n != nil and n.len == 1:
+      internalAssert n.kind == nkRefTy
+      n.sons[0] = unpack(p, typ.sons[0], n.sons[0])
+      result = n
+    else:
+      GlobalError(n.info, "cannot map value from FFI " & typeToString(typ))
+  of tyObject, tyTuple:
+    result = unpackObject(x, typ, n)
+  of tyArray, tyArrayConstr:
+    result = unpackArray(x, typ, n)
+  of tyCString, tyString:
+    let p = rd(cstring, x)
+    if p.isNil:
+      setNil()
+    else:
+      aws(nkStrLit, $p)
+  of tyNil:
+    setNil()
+  of tyDistinct, tyGenericInst:
+    result = unpack(x, typ.sons[0], n)
+  else:
+    # XXX what to do with 'array' here?
+    GlobalError(n.info, "cannot map value from FFI " & typeToString(typ))
+
+proc fficast*(x: PNode, destTyp: PType): PNode =
+  if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyPointer, 
+                                           tyProc, tyCString, tyString, 
+                                           tySequence}:
+    result = newNodeIT(x.kind, x.info, destTyp)
+    result.intVal = x.intVal
+  elif x.kind == nkNilLit:
+    result = newNodeIT(x.kind, x.info, destTyp)
+  else:
+    # we play safe here and allocate the max possible size:
+    let size = max(packSize(x, x.typ), packSize(x, destTyp))
+    var a = alloc0(size)
+    pack(x, x.typ, a)
+    # cast through a pointer needs a new inner object:
+    let y = if x.kind == nkRefTy: newNodeI(nkRefTy, x.info, 1)
+            else: x.copyTree
+    y.typ = x.typ
+    result = unpack(a, destTyp, y)
+    dealloc a
+
+proc callForeignFunction*(call: PNode): PNode =
+  InternalAssert call.sons[0].kind == nkPtrLit
+  
+  var cif: TCif
+  var sig: TParamList
+  # use the arguments' types for varargs support:
+  for i in 1..call.len-1:
+    sig[i-1] = mapType(call.sons[i].typ)
+    if sig[i-1].isNil:
+      GlobalError(call.info, "cannot map FFI type")
+  
+  let typ = call.sons[0].typ
+  if prep_cif(cif, mapCallConv(typ.callConv, call.info), cuint(call.len-1),
+              mapType(typ.sons[0]), sig) != OK:
+    GlobalError(call.info, "error in FFI call")
+  
+  var args: TArgList
+  let fn = cast[pointer](call.sons[0].intVal)
+  for i in 1 .. call.len-1:
+    var t = call.sons[i].typ
+    args[i-1] = alloc0(packSize(call.sons[i], t))
+    pack(call.sons[i], t, args[i-1])
+  let retVal = if isEmptyType(typ.sons[0]): pointer(nil)
+               else: alloc(typ.sons[0].getSize.int)
+
+  libffi.call(cif, fn, retVal, args)
+  
+  if retVal.isNil: 
+    result = emptyNode
+  else:
+    result = unpack(retVal, typ.sons[0], nil)
+    result.info = call.info
+
+  if retVal != nil: dealloc retVal
+  for i in 1 .. call.len-1:
+    call.sons[i] = unpack(args[i-1], typ.sons[i], call[i])
+    dealloc args[i-1]

compiler/evals.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -18,6 +18,9 @@ import
   msgs, os, condsyms, idents, renderer, types, passes, semfold, transf, 
   parser, ropes, rodread, idgen, osproc, streams, evaltempl
 
+when hasFFI:
+  import evalffi
+
 type 
   PStackFrame* = ref TStackFrame
   TStackFrame*{.final.} = object 
@@ -34,12 +37,20 @@ type
                               ## emConst?)
     emStatic                  ## evaluate for enforced compile time eval
                               ## ('static' context)
+
+  TSandboxFlag* = enum        ## what the evaluation engine should allow
+    allowCast,                ## allow unsafe language feature: 'cast'
+    allowFFI,                 ## allow the FFI
+    allowInfiniteLoops        ## allow endless loops
+  TSandboxFlags* = set[TSandboxFlag]
+
   TEvalContext* = object of passes.TPassContext
     module*: PSym
     tos*: PStackFrame         # top of stack
     lastException*: PNode
     callsite: PNode           # for 'callsite' magic
     mode*: TEvalMode
+    features: TSandboxFlags
     globals*: TIdNodeTable    # state of global vars
     getType*: proc(n: PNode): PNode {.closure.}
   
@@ -65,6 +76,7 @@ proc newEvalContext*(module: PSym, mode: TEvalMode): PEvalContext =
   new(result)
   result.module = module
   result.mode = mode
+  result.features = {allowFFI}
   initIdNodeTable(result.globals)
 
 proc pushStackFrame*(c: PEvalContext, t: PStackFrame) {.inline.} = 
@@ -163,8 +175,11 @@ proc evalWhile(c: PEvalContext, n: PNode): PNode =
     else: nil
     dec(gWhileCounter)
     if gWhileCounter <= 0:
-      stackTrace(c, n, errTooManyIterations)
+      if allowInfiniteLoops in c.features:
-      break 
+        gWhileCounter = 0
+      else:
+        stackTrace(c, n, errTooManyIterations)
+        break
 
 proc evalBlock(c: PEvalContext, n: PNode): PNode =
   result = evalAux(c, n.sons[1], {})
@@ -303,9 +318,54 @@ proc evalVar(c: PEvalContext, n: PNode): PNode =
       else:
         if x.kind notin {nkEmpty..nkNilLit}:
           discardSons(x)
-          for i in countup(0, sonsLen(result) - 1): addSon(x, result.sons[i])
+          for j in countup(0, sonsLen(result) - 1): addSon(x, result.sons[j])
   result = emptyNode
 
+proc aliasNeeded(n: PNode, flags: TEvalFlags): bool = 
+  result = efLValue in flags or n.typ == nil or 
+    n.typ.kind in {tyExpr, tyStmt, tyTypeDesc}
+
+proc evalVariable(c: PStackFrame, sym: PSym, flags: TEvalFlags): PNode = 
+  # We need to return a node to the actual value,
+  # which can be modified.
+  var x = c
+  while x != nil: 
+    if sym.kind == skResult and x.params.len > 0:
+      result = x.params[0]
+      if result == nil: result = emptyNode
+      return
+    result = IdNodeTableGet(x.mapping, sym)
+    if result != nil and not aliasNeeded(result, flags): 
+      result = copyTree(result)
+    if result != nil: return 
+    x = x.next
+  #internalError(sym.info, "cannot eval " & sym.name.s)
+  result = raiseCannotEval(nil, sym.info)
+  #result = emptyNode
+
+proc evalGlobalVar(c: PEvalContext, s: PSym, flags: TEvalFlags): PNode =
+  if sfCompileTime in s.flags or c.mode == emRepl:
+    result = IdNodeTableGet(c.globals, s)
+    if result != nil: 
+      if not aliasNeeded(result, flags): 
+        result = copyTree(result)
+    else:
+      when hasFFI:
+        if sfImportc in s.flags and allowFFI in c.features:
+          result = importcSymbol(s)
+          IdNodeTablePut(c.globals, s, result)
+          return result
+      
+      result = s.ast
+      if result == nil or result.kind == nkEmpty:
+        result = getNullValue(s.typ, s.info)
+      else:
+        result = evalAux(c, result, {})
+        if isSpecial(result): return
+      IdNodeTablePut(c.globals, s, result)
+  else:
+    result = raiseCannotEval(nil, s.info)
+
 proc evalCall(c: PEvalContext, n: PNode): PNode = 
   var d = newStackFrame()
   d.call = n
@@ -336,51 +396,21 @@ proc evalCall(c: PEvalContext, n: PNode): PNode =
     if isSpecial(result): return 
     d.params[i] = result
   if n.typ != nil: d.params[0] = getNullValue(n.typ, n.info)
+  
+  when hasFFI:
+    if sfImportc in prc.sym.flags and allowFFI in c.features:
+      var newCall = newNodeI(nkCall, n.info, n.len)
+      newCall.sons[0] = evalGlobalVar(c, prc.sym, {})
+      for i in 1 .. <n.len:
+        newCall.sons[i] = d.params[i]
+      return callForeignFunction(newCall)
+  
   pushStackFrame(c, d)
   result = evalAux(c, prc.sym.getBody, {})
   if result.kind == nkExceptBranch: return 
   if n.typ != nil: result = d.params[0]
   popStackFrame(c)
 
-proc aliasNeeded(n: PNode, flags: TEvalFlags): bool = 
-  result = efLValue in flags or n.typ == nil or 
-    n.typ.kind in {tyExpr, tyStmt, tyTypeDesc}
-
-proc evalVariable(c: PStackFrame, sym: PSym, flags: TEvalFlags): PNode = 
-  # We need to return a node to the actual value,
-  # which can be modified.
-  var x = c
-  while x != nil: 
-    if sym.kind == skResult and x.params.len > 0:
-      result = x.params[0]
-      if result == nil: result = emptyNode
-      return
-    result = IdNodeTableGet(x.mapping, sym)
-    if result != nil and not aliasNeeded(result, flags): 
-      result = copyTree(result)
-    if result != nil: return 
-    x = x.next
-  #internalError(sym.info, "cannot eval " & sym.name.s)
-  result = raiseCannotEval(nil, sym.info)
-  #result = emptyNode
-
-proc evalGlobalVar(c: PEvalContext, s: PSym, flags: TEvalFlags): PNode =
-  if sfCompileTime in s.flags or c.mode == emRepl:
-    result = IdNodeTableGet(c.globals, s)
-    if result != nil: 
-      if not aliasNeeded(result, flags): 
-        result = copyTree(result)
-    else:
-      result = s.ast
-      if result == nil or result.kind == nkEmpty:
-        result = getNullValue(s.typ, s.info)
-      else:
-        result = evalAux(c, result, {})
-        if isSpecial(result): return
-      IdNodeTablePut(c.globals, s, result)
-  else:
-    result = raiseCannotEval(nil, s.info)
-
 proc evalArrayAccess(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode = 
   result = evalAux(c, n.sons[0], flags)
   if isSpecial(result): return 
@@ -519,7 +549,9 @@ proc evalSym(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
   of skConst: result = s.ast
   of skEnumField: result = newIntNodeT(s.position, n)
   else: result = nil
-  if result == nil or {sfImportc, sfForward} * s.flags != {}:
+  let mask = if hasFFI and allowFFI in c.features: {sfForward}
+             else: {sfImportc, sfForward}
+  if result == nil or mask * s.flags != {}:
     result = raiseCannotEval(c, n.info)
 
 proc evalIncDec(c: PEvalContext, n: PNode, sign: biggestInt): PNode = 
@@ -617,6 +649,18 @@ proc evalConv(c: PEvalContext, n: PNode): PNode =
       # foldConv() cannot deal with everything that we want to do here:
       result = a
 
+proc evalCast(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
+  if allowCast in c.features:
+    when hasFFI:
+      result = evalAux(c, n.sons[1], {efLValue})
+      if isSpecial(result): return
+      InternalAssert result.typ != nil
+      result = fficast(result, n.typ)
+    else:
+      result = evalConv(c, n)
+  else:
+    result = raiseCannotEval(c, n.info)
+
 proc evalCheckedFieldAccess(c: PEvalContext, n: PNode, 
                             flags: TEvalFlags): PNode = 
   result = evalAux(c, n.sons[0], flags)
@@ -1363,7 +1407,9 @@ proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
     result.typ = n.typ
   of nkPragmaBlock:
     result = evalAux(c, n.sons[1], flags)
-  of nkIdentDefs, nkCast, nkYieldStmt, nkAsmStmt, nkForStmt, nkPragmaExpr, 
+  of nkCast:
+    result = evalCast(c, n, flags)
+  of nkIdentDefs, nkYieldStmt, nkAsmStmt, nkForStmt, nkPragmaExpr, 
      nkLambdaKinds, nkContinueStmt, nkIdent, nkParForStmt, nkBindStmt:
     result = raiseCannotEval(c, n.info)
   of nkRefTy:
@@ -1388,7 +1434,7 @@ proc eval*(c: PEvalContext, n: PNode): PNode =
     if sonsLen(result) >= 1: 
       stackTrace(c, n, errUnhandledExceptionX, typeToString(result.typ))
     else:
-      stackTrace(c, n, errCannotInterpretNodeX, renderTree(n))
+      stackTrace(c, result, errCannotInterpretNodeX, renderTree(n))
 
 proc evalConstExprAux(module: PSym, e: PNode, mode: TEvalMode): PNode = 
   var p = newEvalContext(module, mode)
@@ -1433,6 +1479,7 @@ proc evalMacroCall(c: PEvalContext, n, nOrig: PNode, sym: PSym): PNode =
 
 proc myOpen(module: PSym): PPassContext =
   var c = newEvalContext(module, emRepl)
+  c.features = {allowCast, allowFFI, allowInfiniteLoops}
   pushStackFrame(c, newStackFrame())
   result = c
 

compiler/importer.nim

@@ -11,11 +11,10 @@
 
 import 
   intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups,
-  semdata, passes
+  semdata, passes, renderer
 
 proc evalImport*(c: PContext, n: PNode): PNode
 proc evalFrom*(c: PContext, n: PNode): PNode
-proc importAllSymbols*(c: PContext, fromMod: PSym)
 
 proc getModuleName*(n: PNode): string =
   # This returns a short relative module name without the nim extension
@@ -29,8 +28,11 @@ proc getModuleName*(n: PNode): string =
   of nkSym:
     result = n.sym.name.s
   else:
-    internalError(n.info, "getModuleName")
+    # hacky way to implement 'x / y /../ z':
-    result = ""
+    result = renderTree(n, {renderNoComments}).replace(" ")
+    #localError(n.info, errGenerated,
+    #  "invalide module name: '$1'" % renderTree(n))
+    #result = ""
 
 proc checkModuleName*(n: PNode): int32 =
   # This returns the full canonical path for a given module import
@@ -45,21 +47,22 @@ proc checkModuleName*(n: PNode): int32 =
 proc rawImportSymbol(c: PContext, s: PSym) =
   # This does not handle stubs, because otherwise loading on demand would be
   # pointless in practice. So importing stubs is fine here!
-  var copy = s # do not copy symbols when importing!
   # check if we have already a symbol of the same name:
   var check = StrTableGet(c.tab.stack[importTablePos], s.name)
-  if check != nil and check.id != copy.id: 
+  if check != nil and check.id != s.id:
     if s.kind notin OverloadableSyms:
       # s and check need to be qualified:
-      Incl(c.AmbiguousSymbols, copy.id)
+      Incl(c.AmbiguousSymbols, s.id)
       Incl(c.AmbiguousSymbols, check.id)
-  StrTableAdd(c.tab.stack[importTablePos], copy)
+  # thanks to 'export' feature, it could be we import the same symbol from
+  # multiple sources, so we need to call 'StrTableAdd' here:
+  StrTableAdd(c.tab.stack[importTablePos], s)
   if s.kind == skType:
     var etyp = s.typ
     if etyp.kind in {tyBool, tyEnum} and sfPure notin s.flags:
       for j in countup(0, sonsLen(etyp.n) - 1):
         var e = etyp.n.sons[j].sym
-        if (e.Kind != skEnumField): 
+        if e.Kind != skEnumField: 
           InternalError(s.info, "rawImportSymbol") 
           # BUGFIX: because of aliases for enums the symbol may already
           # have been put into the symbol table
@@ -99,19 +102,42 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
         e = NextIdentIter(it, fromMod.tab)
     else: rawImportSymbol(c, s)
 
-proc importAllSymbols(c: PContext, fromMod: PSym) = 
+proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: TIntSet) =
   var i: TTabIter
   var s = InitTabIter(i, fromMod.tab)
   while s != nil:
     if s.kind != skModule:
       if s.kind != skEnumField:
-        if not (s.Kind in ExportableSymKinds): 
+        if s.Kind notin ExportableSymKinds:
           InternalError(s.info, "importAllSymbols: " & $s.kind)
-        rawImportSymbol(c, s) # this is correct!
+        if exceptSet.empty or s.name.id notin exceptSet:
+          rawImportSymbol(c, s)
     s = NextIter(i, fromMod.tab)
 
+proc importAllSymbols*(c: PContext, fromMod: PSym) =
+  var exceptSet: TIntSet
+  importAllSymbolsExcept(c, fromMod, exceptSet)
+
+proc importForwarded(c: PContext, n: PNode, exceptSet: TIntSet) =
+  if n.isNil: return
+  case n.kind
+  of nkExportStmt:
+    for a in n:
+      assert a.kind == nkSym
+      let s = a.sym
+      if s.kind == skModule:
+        importAllSymbolsExcept(c, s, exceptSet)
+      elif exceptSet.empty or s.name.id notin exceptSet:
+        rawImportSymbol(c, s)
+  of nkExportExceptStmt:
+    localError(n.info, errGenerated, "'export except' not implemented")
+  else:
+    for i in 0 ..safeLen(n)-1:
+      importForwarded(c, n.sons[i], exceptSet)
+
 proc evalImport(c: PContext, n: PNode): PNode = 
   result = n
+  var emptySet: TIntSet
   for i in countup(0, sonsLen(n) - 1): 
     var f = checkModuleName(n.sons[i])
     if f != InvalidFileIDX:
@@ -120,7 +146,8 @@ proc evalImport(c: PContext, n: PNode): PNode =
         Message(n.sons[i].info, warnDeprecated, m.name.s) 
       # ``addDecl`` needs to be done before ``importAllSymbols``!
       addDecl(c, m)             # add symbol to symbol table of module
-      importAllSymbols(c, m)
+      importAllSymbolsExcept(c, m, emptySet)
+      importForwarded(c, m.ast, emptySet)
 
 proc evalFrom(c: PContext, n: PNode): PNode = 
   result = n
@@ -131,3 +158,18 @@ proc evalFrom(c: PContext, n: PNode): PNode =
     n.sons[0] = newSymNode(m)
     addDecl(c, m)               # add symbol to symbol table of module
     for i in countup(1, sonsLen(n) - 1): importSymbol(c, n.sons[i], m)
+
+proc evalImportExcept*(c: PContext, n: PNode): PNode = 
+  result = n
+  checkMinSonsLen(n, 2)
+  var f = checkModuleName(n.sons[0])
+  if f != InvalidFileIDX:
+    var m = gImportModule(c.module, f)
+    n.sons[0] = newSymNode(m)
+    addDecl(c, m)               # add symbol to symbol table of module
+    var exceptSet = initIntSet()
+    for i in countup(1, sonsLen(n) - 1): 
+      let ident = lookups.considerAcc(n.sons[i])
+      exceptSet.incl(ident.id)
+    importAllSymbolsExcept(c, m, exceptSet)
+    importForwarded(c, m.ast, exceptSet)

compiler/lambdalifting.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -257,6 +257,8 @@ proc captureVar(o: POuterContext, i: PInnerContext, local: PSym,
     # Currently captures are restricted to a single level of nesting:
     LocalError(info, errIllegalCaptureX, local.name.s)
   i.fn.typ.callConv = ccClosure
+  #echo "captureVar ", i.fn.name.s, i.fn.id, " ", local.name.s, local.id
+
   incl(i.fn.typ.flags, tfCapturesEnv)
 
   # we need to remember which inner most closure belongs to this lambda:
@@ -286,9 +288,10 @@ proc interestingVar(s: PSym): bool {.inline.} =
     sfGlobal notin s.flags
 
 proc semCaptureSym*(s, owner: PSym) =
-  if interestingVar(s) and owner.id != s.owner.id:
+  if interestingVar(s) and owner.id != s.owner.id and s.kind != skResult:
     if owner.typ != nil and not isGenericRoutine(owner):
       owner.typ.callConv = ccClosure
+    #echo "semCaptureSym ", owner.name.s, owner.id, " ", s.name.s, s.id
     # since the analysis is not entirely correct, we don't set 'tfCapturesEnv'
     # here
 

compiler/lists.nim

@@ -90,6 +90,22 @@ proc Remove*(list: var TLinkedList, entry: PListEntry) =
   if entry.next != nil: entry.next.prev = entry.prev
   if entry.prev != nil: entry.prev.next = entry.next
 
+proc bringToFront*(list: var TLinkedList, entry: PListEntry) =
+  if entry == list.head: return
+  if entry == list.tail: list.tail = entry.prev
+  if entry.next != nil: entry.next.prev = entry.prev
+  if entry.prev != nil: entry.prev.next = entry.next
+  entry.prev = nil
+  entry.next = list.head
+  list.head = entry
+
+proc ExcludeStr*(list: var TLinkedList, data: string) =
+  var it = list.head
+  while it != nil:
+    let nxt = it.next
+    if PStrEntry(it).data == data: remove(list, it)
+    it = nxt
+
 proc Find*(list: TLinkedList, fn: TCompareProc, closure: Pointer): PListEntry = 
   result = list.head
   while result != nil:

compiler/llstream.nim

@@ -103,17 +103,24 @@ proc continueLine(line: string, inTripleString: bool): bool {.inline.} =
       line[0] == ' ' or
       line.endsWith(LineContinuationOprs+AdditionalLineContinuationOprs)
 
+proc countTriples(s: string): int =
+  var i = 0
+  while i < s.len:
+    if s[i] == '"' and s[i+1] == '"' and s[i+2] == '"':
+      inc result
+      inc i, 2
+    inc i
+
 proc LLreadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int =
-  var inTripleString = false
   s.s = ""
   s.rd = 0
   var line = newStringOfCap(120)
+  var triples = 0
   while ReadLineFromStdin(if s.s.len == 0: ">>> " else: "... ", line): 
     add(s.s, line)
     add(s.s, "\n")
-    if line.contains("\"\"\""):
+    inc triples, countTriples(line)
-      inTripleString = not inTripleString
+    if not continueLine(line, (triples and 1) == 1): break
-    if not continueLine(line, inTripleString): break
   inc(s.lineOffset)
   result = min(bufLen, len(s.s) - s.rd)
   if result > 0: 

compiler/main.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -302,7 +302,7 @@ when has_LLVM_Backend:
     compileProject()
 
 proc CommandCompileToEcmaScript =
-  incl(gGlobalOptions, optSafeCode)
+  #incl(gGlobalOptions, optSafeCode)
   setTarget(osEcmaScript, cpuEcmaScript)
   #initDefines()
   DefineSymbol("nimrod") # 'nimrod' is always defined
@@ -316,6 +316,7 @@ proc InteractivePasses =
   #setTarget(osNimrodVM, cpuNimrodVM)
   initDefines()
   DefineSymbol("nimrodvm")
+  when hasFFI: DefineSymbol("nimffi")
   registerPass(verbosePass)
   registerPass(semPass)
   registerPass(evalPass)
@@ -527,7 +528,7 @@ proc MainCommand =
   of "dump":
     gCmd = cmdDump
     condsyms.ListSymbols()
-    for it in iterSearchPath(): MsgWriteln(it)
+    for it in iterSearchPath(searchPaths): MsgWriteln(it)
   of "check":
     gCmd = cmdCheck
     wantMainModule()

compiler/msgs.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -132,7 +132,7 @@ const
     errNumberOutOfRange: "number $1 out of valid range", 
     errNnotAllowedInCharacter: "\\n not allowed in character literal", 
     errClosingBracketExpected: "closing ']' expected, but end of file reached", 
-    errMissingFinalQuote: "missing final \'", 
+    errMissingFinalQuote: "missing final \' for character literal", 
     errIdentifierExpected: "identifier expected, but found \'$1\'", 
     errNewlineExpected: "newline expected, but found \'$1\'",
     errInvalidModuleName: "invalid module name: '$1'",
@@ -576,6 +576,14 @@ template toFilename*(info: TLineInfo): string =
 template toFullPath*(info: TLineInfo): string =
   info.fileIndex.toFullPath
 
+proc toMsgFilename*(info: TLineInfo): string =
+  if info.fileIndex < 0: result = "???"
+  else:
+    if gListFullPaths:
+      result = fileInfos[info.fileIndex].fullPath
+    else:
+      result = fileInfos[info.fileIndex].projPath
+
 proc toLinenumber*(info: TLineInfo): int {.inline.} = 
   result = info.line
 
@@ -666,7 +674,7 @@ proc writeContext(lastinfo: TLineInfo) =
   var info = lastInfo
   for i in countup(0, len(msgContext) - 1): 
     if msgContext[i] != lastInfo and msgContext[i] != info: 
-      MsgWriteln(posContextFormat % [toFilename(msgContext[i]), 
+      MsgWriteln(posContextFormat % [toMsgFilename(msgContext[i]), 
                                      coordToStr(msgContext[i].line), 
                                      coordToStr(msgContext[i].col), 
                                      getMessageStr(errInstantiationFrom, "")])
@@ -720,7 +728,7 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
     ignoreMsg = optHints notin gOptions or msg notin gNotes
     frmt = posHintFormat
     inc(gHintCounter)
-  let s = frmt % [toFilename(info), coordToStr(info.line),
+  let s = frmt % [toMsgFilename(info), coordToStr(info.line),
                   coordToStr(info.col), getMessageStr(msg, arg)]
   if not ignoreMsg:
     MsgWriteln(s)
@@ -732,6 +740,9 @@ proc Fatal*(info: TLineInfo, msg: TMsgKind, arg = "") =
 proc GlobalError*(info: TLineInfo, msg: TMsgKind, arg = "") = 
   liMessage(info, msg, arg, doRaise)
 
+proc GlobalError*(info: TLineInfo, arg: string) =
+  liMessage(info, errGenerated, arg, doRaise)
+
 proc LocalError*(info: TLineInfo, msg: TMsgKind, arg = "") =
   liMessage(info, msg, arg, doNothing)
 

compiler/nimrod.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.

compiler/options.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -13,6 +13,7 @@ import
 const
   hasTinyCBackend* = defined(tinyc)
   useEffectSystem* = true
+  hasFFI* = defined(useFFI)
 
 type                          # please make sure we have under 32 options
                               # (improves code efficiency a lot!)
@@ -91,7 +92,7 @@ var
                          optPatterns}
   gGlobalOptions*: TGlobalOptions = {optRefcGC, optThreadAnalysis}
   gExitcode*: int8
-  searchPaths*: TLinkedList
+  searchPaths*, lazyPaths*: TLinkedList
   outFile*: string = ""
   headerFile*: string = ""
   gCmd*: TCommands = cmdNone  # the command
@@ -100,6 +101,9 @@ var
   gWholeProject*: bool # for 'doc2': output any dependency
   gEvalExpr*: string          # expression for idetools --eval
   gLastCmdTime*: float        # when caas is enabled, we measure each command
+  gListFullPaths*: bool
+  
+proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools}
 
 const 
   genSubDir* = "nimcache"
@@ -199,27 +203,53 @@ proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string =
   result = joinPath(subdir, tail)
   #echo "completeGeneratedFilePath(", f, ") = ", result
 
-iterator iterSearchPath*(): string = 
+iterator iterSearchPath*(SearchPaths: TLinkedList): string = 
   var it = PStrEntry(SearchPaths.head)
   while it != nil:
     yield it.data
     it = PStrEntry(it.Next)
 
 proc rawFindFile(f: string): string =
-  for it in iterSearchPath():
+  for it in iterSearchPath(SearchPaths):
     result = JoinPath(it, f)
-    if ExistsFile(result):
+    if existsFile(result):
       return result.canonicalizePath
   result = ""
 
+proc rawFindFile2(f: string): string =
+  var it = PStrEntry(lazyPaths.head)
+  while it != nil:
+    result = JoinPath(it.data, f)
+    if existsFile(result):
+      bringToFront(lazyPaths, it)
+      return result.canonicalizePath
+    it = PStrEntry(it.Next)
+  result = ""
+
 proc FindFile*(f: string): string {.procvar.} = 
-  result = rawFindFile(f)
+  result = f.rawFindFile
-  if len(result) == 0: result = rawFindFile(toLower(f))
+  if result.len == 0:
+    result = f.toLower.rawFindFile
+    if result.len == 0:
+      result = f.rawFindFile2
+      if result.len == 0:
+        result = f.toLower.rawFindFile2
 
 proc findModule*(modulename: string): string {.inline.} =
   # returns path to module
   result = FindFile(AddFileExt(modulename, nimExt))
 
+proc libCandidates*(s: string, dest: var seq[string]) = 
+  var le = strutils.find(s, '(')
+  var ri = strutils.find(s, ')', le+1)
+  if le >= 0 and ri > le:
+    var prefix = substr(s, 0, le - 1)
+    var suffix = substr(s, ri + 1)
+    for middle in split(substr(s, le + 1, ri - 1), '|'):
+      libCandidates(prefix & middle & suffix, dest)
+  else: 
+    add(dest, s)
+
 proc binaryStrSearch*(x: openarray[string], y: string): int = 
   var a = 0
   var b = len(x) - 1

compiler/parampatterns.nim

@@ -48,8 +48,8 @@ proc add(code: var TPatternCode, op: TOpcode) {.inline.} =
   add(code, chr(ord(op)))
 
 proc whichAlias*(p: PSym): TAliasRequest =
-  if p.typ.constraint != nil:
+  if p.constraint != nil:
-    result = TAliasRequest(p.typ.constraint.strVal[0].ord)
+    result = TAliasRequest(p.constraint.strVal[0].ord)
 
 proc compileConstraints(p: PNode, result: var TPatternCode) =
   case p.kind

compiler/parser.nim

@@ -875,7 +875,7 @@ proc parseExprStmt(p: var TParser): PNode =
       getTok(p)
       skipComment(p, result)
       if p.tok.tokType == tkSad: getTok(p)
-      if not (p.tok.TokType in {tkOf, tkElif, tkElse, tkExcept}):
+      if p.tok.TokType notin {tkOf, tkElif, tkElse, tkExcept}:
         let body = parseStmt(p)
         addSon(result, newProcNode(nkDo, body.info, body))
       while true:
@@ -904,29 +904,33 @@ proc parseExprStmt(p: var TParser): PNode =
         addSon(result, b)
         if b.kind == nkElse: break
 
-proc parseImportOrIncludeStmt(p: var TParser, kind: TNodeKind): PNode = 
+proc parseImport(p: var TParser, kind: TNodeKind): PNode =
-  var a: PNode
   result = newNodeP(kind, p)
+  getTok(p)                   # skip `import` or `export`
+  optInd(p, result)
+  var a = parseExpr(p)
+  addSon(result, a)
+  if p.tok.tokType in {tkComma, tkExcept}:
+    if p.tok.tokType == tkExcept:
+      result.kind = succ(kind)
+    getTok(p)
+    optInd(p, result)
+    while p.tok.tokType notin {tkEof, tkSad, tkDed}:
+      a = parseExpr(p)
+      if a.kind == nkEmpty: break 
+      addSon(result, a)
+      if p.tok.tokType != tkComma: break 
+      getTok(p)
+      optInd(p, a)
+  expectNl(p)
+
+proc parseIncludeStmt(p: var TParser): PNode =
+  result = newNodeP(nkIncludeStmt, p)
   getTok(p)                   # skip `import` or `include`
   optInd(p, result)
-  while true: 
+  while p.tok.tokType notin {tkEof, tkSad, tkDed}:
-    case p.tok.tokType
+    var a = parseExpr(p)
-    of tkEof, tkSad, tkDed: 
+    if a.kind == nkEmpty: break
-      break 
-    of tkSymbol, tkAccent: 
-      a = parseSymbol(p)
-    of tkRStrLit: 
-      a = newStrNodeP(nkRStrLit, p.tok.literal, p)
-      getTok(p)
-    of tkStrLit: 
-      a = newStrNodeP(nkStrLit, p.tok.literal, p)
-      getTok(p)
-    of tkTripleStrLit: 
-      a = newStrNodeP(nkTripleStrLit, p.tok.literal, p)
-      getTok(p)
-    else: 
-      parMessage(p, errIdentifierExpected, p.tok)
-      break 
     addSon(result, a)
     if p.tok.tokType != tkComma: break 
     getTok(p)
@@ -934,37 +938,16 @@ proc parseImportOrIncludeStmt(p: var TParser, kind: TNodeKind): PNode =
   expectNl(p)
 
 proc parseFromStmt(p: var TParser): PNode = 
-  var a: PNode
   result = newNodeP(nkFromStmt, p)
   getTok(p)                   # skip `from`
   optInd(p, result)
-  case p.tok.tokType
+  var a = parseExpr(p)
-  of tkSymbol, tkAccent: 
-    a = parseSymbol(p)
-  of tkRStrLit: 
-    a = newStrNodeP(nkRStrLit, p.tok.literal, p)
-    getTok(p)
-  of tkStrLit: 
-    a = newStrNodeP(nkStrLit, p.tok.literal, p)
-    getTok(p)
-  of tkTripleStrLit: 
-    a = newStrNodeP(nkTripleStrLit, p.tok.literal, p)
-    getTok(p)
-  else: 
-    parMessage(p, errIdentifierExpected, p.tok)
-    return 
   addSon(result, a)           #optInd(p, a);
   eat(p, tkImport)
   optInd(p, result)
-  while true: 
+  while p.tok.tokType notin {tkEof, tkSad, tkDed}:
-    case p.tok.tokType        #optInd(p, a);
+    a = parseExpr(p)
-    of tkEof, tkSad, tkDed: 
+    if a.kind == nkEmpty: break
-      break 
-    of tkSymbol, tkAccent: 
-      a = parseSymbol(p)
-    else: 
-      parMessage(p, errIdentifierExpected, p.tok)
-      break 
     addSon(result, a)
     if p.tok.tokType != tkComma: break 
     getTok(p)
@@ -1279,14 +1262,7 @@ proc parseEnum(p: var TParser): PNode =
   result = newNodeP(nkEnumTy, p)
   a = nil
   getTok(p)
-  if false and p.tok.tokType == tkOf: 
+  addSon(result, ast.emptyNode)
-    a = newNodeP(nkOfInherit, p)
-    getTok(p)
-    optInd(p, a)
-    addSon(a, parseTypeDesc(p))
-    addSon(result, a)
-  else: 
-    addSon(result, ast.emptyNode)
   optInd(p, result)
   while true: 
     case p.tok.tokType
@@ -1425,18 +1401,6 @@ proc parseDistinct(p: var TParser): PNode =
   optInd(p, result)
   addSon(result, parseTypeDesc(p))
 
-proc parsePointerInTypeSection(p: var TParser, kind: TNodeKind): PNode =
-  result = newNodeP(kind, p)
-  getTok(p)
-  optInd(p, result)
-  if not isOperator(p.tok):
-    case p.tok.tokType
-    of tkObject: addSon(result, parseObject(p))
-    of tkTuple: addSon(result, parseTuple(p, true))
-    else:
-      if isExprStart(p):
-        addSon(result, parseTypeDesc(p))
-
 proc parseTypeDef(p: var TParser): PNode = 
   result = newNodeP(nkTypeDef, p)
   addSon(result, identWithPragma(p))
@@ -1445,15 +1409,6 @@ proc parseTypeDef(p: var TParser): PNode =
   if p.tok.tokType == tkEquals: 
     getTok(p)
     optInd(p, result)
-    #var a: PNode
-    #case p.tok.tokType
-    #of tkObject: a = parseObject(p)
-    #of tkEnum: a = parseEnum(p)
-    #of tkDistinct: a = parseDistinct(p)
-    #of tkTuple: a = parseTuple(p, true)
-    #of tkRef: a = parsePointerInTypeSection(p, nkRefTy)
-    #of tkPtr: a = parsePointerInTypeSection(p, nkPtrTy)
-    #else: a = parseTypeDesc(p)
     addSon(result, parseTypeDefAux(p))
   else:
     addSon(result, ast.emptyNode)
@@ -1511,9 +1466,10 @@ proc simpleStmt(p: var TParser): PNode =
   of tkBreak: result = parseBreakOrContinue(p, nkBreakStmt)
   of tkContinue: result = parseBreakOrContinue(p, nkContinueStmt)
   of tkCurlyDotLe: result = parseStmtPragma(p)
-  of tkImport: result = parseImportOrIncludeStmt(p, nkImportStmt)
+  of tkImport: result = parseImport(p, nkImportStmt)
+  of tkExport: result = parseImport(p, nkExportStmt)
   of tkFrom: result = parseFromStmt(p)
-  of tkInclude: result = parseImportOrIncludeStmt(p, nkIncludeStmt)
+  of tkInclude: result = parseIncludeStmt(p)
   of tkComment: result = newCommentStmt(p)
   else:
     if isExprStart(p): result = parseExprStmt(p)

compiler/patterns.nim

@@ -71,8 +71,8 @@ proc inSymChoice(sc, x: PNode): bool =
   
 proc checkTypes(c: PPatternContext, p: PSym, n: PNode): bool =
   # check param constraints first here as this is quite optimized:
-  if p.typ.constraint != nil:
+  if p.constraint != nil:
-    result = matchNodeKinds(p.typ.constraint, n)
+    result = matchNodeKinds(p.constraint, n)
     if not result: return
   if isNil(n.typ):
     result = p.typ.kind in {tyEmpty, tyStmt}
@@ -237,6 +237,15 @@ proc addToArgList(result, n: PNode) =
     else:
       for i in 0 .. <n.len: result.add(n.sons[i])
 
+when false:
+  proc procPatternMatches*(c: PContext, s: PSym, n: PNode): bool =
+    ## for AST-based overloading:
+    var ctx: TPatternContext
+    ctx.owner = s
+    ctx.c = c
+    ctx.formals = sonsLen(s.typ)-1
+    result = matches(ctx, s.ast.sons[patternPos], n)
+
 proc applyRule*(c: PContext, s: PSym, n: PNode): PNode =
   ## returns a tree to semcheck if the rule triggered; nil otherwise
   var ctx: TPatternContext

compiler/renderer.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -754,7 +754,6 @@ proc doParamsAux(g: var TSrcGen, params: PNode) =
 proc gsub(g: var TSrcGen, n: PNode, c: TContext) = 
   if isNil(n): return
   var
-    L: int
     a: TContext
   if n.comment != nil: pushCom(g, n)
   case n.kind                 # atoms:
@@ -1096,7 +1095,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     incl(a.flags, rfInConstExpr)
     gsection(g, n, a, tkConst, "const")
   of nkVarSection, nkLetSection:
-    L = sonsLen(n)
+    var L = sonsLen(n)
     if L == 0: return
     if n.kind == nkVarSection: putWithSpace(g, tkVar, "var")
     else: putWithSpace(g, tkLet, "let")
@@ -1134,13 +1133,27 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
       put(g, tkCurlyDotLe, "{.")
       gcomma(g, n, emptyContext)
       put(g, tkCurlyDotRi, ".}")
-  of nkImportStmt: 
+  of nkImportStmt, nkExportStmt:
-    putWithSpace(g, tkImport, "import")
+    if n.kind == nkImportStmt:
+      putWithSpace(g, tkImport, "import")
+    else:
+      putWithSpace(g, tkExport, "export")
     gcoms(g)
     indentNL(g)
     gcommaAux(g, n, g.indent)
     dedent(g)
     putNL(g)
+  of nkImportExceptStmt, nkExportExceptStmt:
+    if n.kind == nkImportExceptStmt:
+      putWithSpace(g, tkImport, "import")
+    else:
+      putWithSpace(g, tkExport, "export")
+    gsub(g, n.sons[0])
+    put(g, tkSpaces, Space)
+    putWithSpace(g, tkExcept, "except")
+    gcommaAux(g, n, g.indent, 1)
+    gcoms(g)
+    putNL(g)
   of nkFromStmt: 
     putWithSpace(g, tkFrom, "from")
     gsub(g, n.sons[0])

compiler/rodread.nim

@@ -330,9 +330,6 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType =
   if r.s[r.pos] == '@': 
     inc(r.pos)
     result.containerID = decodeVInt(r.s, r.pos)
-  if r.s[r.pos] == '`':
-    inc(r.pos)
-    result.constraint = decodeNode(r, UnknownLineInfo())
   decodeLoc(r, result.loc, info)
   while r.s[r.pos] == '^': 
     inc(r.pos)
@@ -423,6 +420,9 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
     result.offset = - 1
   decodeLoc(r, result.loc, result.info)
   result.annex = decodeLib(r, info)
+  if r.s[r.pos] == '#':
+    inc(r.pos)
+    result.constraint = decodeNode(r, UnknownLineInfo())
   if r.s[r.pos] == '(':
     if result.kind in routineKinds:
       result.ast = decodeNodeLazyBody(r, result.info, result)

compiler/rodwrite.nim

@@ -233,9 +233,6 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t.containerID != 0: 
     add(result, '@')
     encodeVInt(t.containerID, result)
-  if t.constraint != nil:
-    add(result, '`')
-    encodeNode(w, UnknownLineInfo(), t.constraint, result)
   encodeLoc(w, t.loc, result)
   for i in countup(0, sonsLen(t) - 1): 
     if t.sons[i] == nil: 
@@ -295,6 +292,9 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
     encodeVInt(s.offset, result)
   encodeLoc(w, s.loc, result)
   if s.annex != nil: encodeLib(w, s.annex, s.info, result)
+  if s.constraint != nil:
+    add(result, '#')
+    encodeNode(w, UnknownLineInfo(), s.constraint, result)
   # lazy loading will soon reload the ast lazily, so the ast needs to be
   # the last entry of a symbol:
   if s.ast != nil:

compiler/sem.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -38,13 +38,15 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType
 proc semStmt(c: PContext, n: PNode): PNode
 proc semParamList(c: PContext, n, genericParams: PNode, s: PSym)
 proc addParams(c: PContext, n: PNode, kind: TSymKind)
-proc addResult(c: PContext, t: PType, info: TLineInfo, owner: TSymKind)
+proc maybeAddResult(c: PContext, s: PSym, n: PNode)
-proc addResultNode(c: PContext, n: PNode)
 proc instGenericContainer(c: PContext, n: PNode, header: PType): PType
 proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode
 proc fixImmediateParams(n: PNode): PNode
 proc activate(c: PContext, n: PNode)
 proc semQuoteAst(c: PContext, n: PNode): PNode
+proc finishMethod(c: PContext, s: PSym)
+
+proc IndexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode
 
 proc typeMismatch(n: PNode, formal, actual: PType) = 
   if formal.kind != tyError and actual.kind != tyError: 

compiler/semcall.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -15,8 +15,22 @@ proc sameMethodDispatcher(a, b: PSym): bool =
   if a.kind == skMethod and b.kind == skMethod: 
     var aa = lastSon(a.ast)
     var bb = lastSon(b.ast)
-    if aa.kind == nkSym and bb.kind == nkSym and aa.sym == bb.sym: 
+    if aa.kind == nkSym and bb.kind == nkSym:
-      result = true
+      if aa.sym == bb.sym: 
+        result = true
+    else:
+      nil
+      # generics have no dispatcher yet, so we need to compare the method
+      # names; however, the names are equal anyway because otherwise we
+      # wouldn't even consider them to be overloaded. But even this does
+      # not work reliably! See tmultim6 for an example:
+      # method collide[T](a: TThing, b: TUnit[T]) is instantiated and not
+      # method collide[T](a: TUnit[T], b: TThing)! This means we need to
+      # *instantiate* every candidate! However, we don't keep more than 2-3
+      # candidated around so we cannot implement that for now. So in order
+      # to avoid subtle problems, the call remains ambiguous and needs to
+      # be disambiguated by the programmer; this way the right generic is
+      # instantiated.
   
 proc resolveOverloads(c: PContext, n, orig: PNode, 
                       filter: TSymKinds): TCandidate =
@@ -84,6 +98,35 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
         getProcHeader(best.calleeSym), getProcHeader(alt.calleeSym),
         args])
 
+
+proc instGenericConvertersArg*(c: PContext, a: PNode, x: TCandidate) =
+  if a.kind == nkHiddenCallConv and a.sons[0].kind == nkSym and
+      isGenericRoutine(a.sons[0].sym):
+    let finalCallee = generateInstance(c, a.sons[0].sym, x.bindings, a.info)
+    a.sons[0].sym = finalCallee
+    a.sons[0].typ = finalCallee.typ
+    #a.typ = finalCallee.typ.sons[0]
+
+proc instGenericConvertersSons*(c: PContext, n: PNode, x: TCandidate) =
+  assert n.kind in nkCallKinds
+  if x.genericConverter:
+    for i in 1 .. <n.len:
+      instGenericConvertersArg(c, n.sons[i], x)
+
+proc IndexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode = 
+  var m: TCandidate
+  initCandidate(m, f)
+  result = paramTypesMatch(c, m, f, a, arg, nil)
+  if m.genericConverter and result != nil:
+    instGenericConvertersArg(c, result, m)
+
+proc ConvertTo*(c: PContext, f: PType, n: PNode): PNode = 
+  var m: TCandidate
+  initCandidate(m, f)
+  result = paramTypesMatch(c, m, f, n.typ, n, nil)
+  if m.genericConverter and result != nil:
+    instGenericConvertersArg(c, result, m)
+
 proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode =
   assert x.state == csMatch
   var finalCallee = x.calleeSym
@@ -101,6 +144,7 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode =
       if ContainsGenericType(result.typ): result.typ = errorType(c)
       return
   result = x.call
+  instGenericConvertersSons(c, result, x)
   result.sons[0] = newSymNode(finalCallee, result.sons[0].info)
   result.typ = finalCallee.typ.sons[0]
 

compiler/semexprs.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -318,7 +318,7 @@ proc semIs(c: PContext, n: PNode): PNode =
     if not containsGenericType(t1): result = evalIsOp(n)
   
 proc semOpAux(c: PContext, n: PNode) =
-  let flags = {efDetermineType}
+  const flags = {efDetermineType}
   for i in countup(1, n.sonsLen- 1):
     var a = n.sons[i]
     if a.kind == nkExprEqExpr and sonsLen(a) == 2: 
@@ -589,9 +589,10 @@ proc semStaticExpr(c: PContext, n: PNode): PNode =
 
 proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
                                      flags: TExprFlags): PNode =
-  if efWantIterator in flags:
+  if flags*{efInTypeOf, efWantIterator} != {}:
-    result = semOverloadedCall(c, n, nOrig, {skIterator})
+    # consider: 'for x in pReturningArray()' --> we don't want the restriction
-  elif efInTypeOf in flags:
+    # to 'skIterator' anymore; skIterator is preferred in sigmatch already for
+    # typeof support.
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     result = semOverloadedCall(c, n, nOrig,
       {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator})
@@ -663,6 +664,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
       result = nil
     else:
       result = m.call
+      instGenericConvertersSons(c, result, m)
     # we assume that a procedure that calls something indirectly 
     # has side-effects:
     if tfNoSideEffect notin t.flags: incl(c.p.owner.flags, sfSideEffect)
@@ -1658,6 +1660,26 @@ proc fixImmediateParams(n: PNode): PNode =
   
   result = n
 
+proc semExport(c: PContext, n: PNode): PNode =
+  var x = newNodeI(n.kind, n.info)
+  #let L = if n.kind == nkExportExceptStmt: L = 1 else: n.len
+  for i in 0.. <n.len:
+    let a = n.sons[i]
+    var o: TOverloadIter
+    var s = initOverloadIter(o, c, a)
+    if s == nil:
+      localError(a.info, errGenerated, "invalid expr for 'export': " &
+          renderTree(a))
+    while s != nil:
+      if s.kind in ExportableSymKinds+{skModule}:
+        x.add(newSymNode(s, a.info))
+      s = nextOverloadIter(o, c, a)
+  if c.module.ast.isNil:
+    c.module.ast = newNodeI(nkStmtList, n.info)
+  assert c.module.ast.kind == nkStmtList
+  c.module.ast.add x
+  result = n
+
 proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = 
   result = n
   if gCmd == cmdIdeTools: suggestExpr(c, n)
@@ -1851,12 +1873,18 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   of nkImportStmt: 
     if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "import")
     result = evalImport(c, n)
+  of nkImportExceptStmt:
+    if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "import")
+    result = evalImportExcept(c, n)
   of nkFromStmt: 
     if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "from")
     result = evalFrom(c, n)
   of nkIncludeStmt: 
     if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "include")
     result = evalInclude(c, n)
+  of nkExportStmt, nkExportExceptStmt:
+    if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "export")
+    result = semExport(c, n)
   of nkPragmaBlock:
     result = semPragmaBlock(c, n)
   of nkStaticStmt:

compiler/seminst.nim

@@ -89,9 +89,7 @@ proc instantiateBody(c: PContext, n: PNode, result: PSym) =
     # add it here, so that recursive generic procs are possible:
     addDecl(c, result)
     pushProcCon(c, result)
-    if result.kind in {skProc, skMethod, skConverter, skMacro}: 
+    maybeAddResult(c, result, n)
-      addResult(c, result.typ.sons[0], n.info, result.kind)
-      addResultNode(c, n)
     var b = n.sons[bodyPos]
     var symMap: TIdTable
     InitIdTable symMap
@@ -163,6 +161,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
     result.typ = newTypeS(tyProc, c)
     rawAddSon(result.typ, nil)
   result.typ.callConv = fn.typ.callConv
+  if result.kind == skIterator: result.typ.flags.incl(tfIterator)
   var oldPrc = GenericCacheGet(fn, entry[])
   if oldPrc == nil:
     fn.procInstCache.safeAdd(entry)
@@ -182,6 +181,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   popOwner()
   c.friendModule = oldFriend
   dec(c.InstCounter)
+  if result.kind == skMethod: finishMethod(c, result)
   
 proc instGenericContainer(c: PContext, n: PNode, header: PType): PType = 
   var cl: TReplTypeVars

compiler/sempass2.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -146,7 +146,10 @@ proc catches(tracked: PEffects, e: PType) =
       dec L
     else:
       inc i
-  setLen(tracked.exc.sons, L)
+  if not isNil(tracked.exc.sons):
+    setLen(tracked.exc.sons, L)
+  else:
+    assert L == 0
 
 proc catchesAll(tracked: PEffects) =
   if not isNil(tracked.exc.sons):

compiler/semstmts.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -172,11 +172,16 @@ proc fitRemoveHiddenConv(c: PContext, typ: Ptype, n: PNode): PNode =
     changeType(result, typ)
 
 proc findShadowedVar(c: PContext, v: PSym): PSym =
-  for i in countdown(c.tab.tos - 2, 0):
+  for i in countdown(c.tab.tos - 2, ModuleTablePos+1):
     let shadowed = StrTableGet(c.tab.stack[i], v.name)
     if shadowed != nil and shadowed.kind in skLocalVars:
       return shadowed
 
+proc identWithin(n: PNode, s: PIdent): bool =
+  for i in 0 .. n.safeLen-1:
+    if identWithin(n.sons[i], s): return true
+  result = n.kind == nkSym and n.sym.name.id == s.id
+
 proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym =
   if isTopLevel(c): 
     result = semIdentWithPragma(c, kind, n, {sfExported})
@@ -239,7 +244,10 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
           let shadowed = findShadowedVar(c, v)
           if shadowed != nil:
             shadowed.flags.incl(sfShadowed)
-            Message(a.info, warnShadowIdent, v.name.s)
+            # a shadowed variable is an error unless it appears on the right
+            # side of the '=':
+            if warnShadowIdent in gNotes and not identWithin(def, v.name):
+              Message(a.info, warnShadowIdent, v.name.s)
       if def != nil and def.kind != nkEmpty:
         # this is only needed for the evaluation pass:
         v.ast = def
@@ -247,7 +255,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
       if a.kind != nkVarTuple:
         v.typ = typ
         b = newNodeI(nkIdentDefs, a.info)
-        if gCmd == cmdDoc:
+        if importantComments():
           # keep documentation information:
           b.comment = a.comment
         addSon(b, newSymNode(v))
@@ -287,7 +295,7 @@ proc semConst(c: PContext, n: PNode): PNode =
     v.ast = def               # no need to copy
     if sfGenSym notin v.flags: addInterfaceDecl(c, v)
     var b = newNodeI(nkConstDef, a.info)
-    if gCmd == cmdDoc: b.comment = a.comment
+    if importantComments(): b.comment = a.comment
     addSon(b, newSymNode(v))
     addSon(b, ast.emptyNode)            # no type description
     addSon(b, copyTree(def))
@@ -368,6 +376,15 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode =
   b.add(ast.emptyNode)
   stmts.add(b)
 
+proc addForVarDecl(c: PContext, v: PSym) =
+  if warnShadowIdent in gNotes:
+    let shadowed = findShadowedVar(c, v)
+    if shadowed != nil:
+      # XXX should we do this here?
+      #shadowed.flags.incl(sfShadowed)
+      Message(v.info, warnShadowIdent, v.name.s)
+  addDecl(c, v)
+
 proc semForVars(c: PContext, n: PNode): PNode =
   result = n
   var length = sonsLen(n)
@@ -383,7 +400,7 @@ proc semForVars(c: PContext, n: PNode): PNode =
       # for an example:
       v.typ = n.sons[length-2].typ
       n.sons[0] = newSymNode(v)
-      if sfGenSym notin v.flags: addDecl(c, v)
+      if sfGenSym notin v.flags: addForVarDecl(c, v)
     else:
       LocalError(n.info, errWrongNumberOfVariables)
   elif length-2 != sonsLen(iter):
@@ -394,7 +411,7 @@ proc semForVars(c: PContext, n: PNode): PNode =
       if getCurrOwner().kind == skModule: incl(v.flags, sfGlobal)
       v.typ = iter.sons[i]
       n.sons[i] = newSymNode(v)
-      if sfGenSym notin v.flags: addDecl(c, v)
+      if sfGenSym notin v.flags: addForVarDecl(c, v)
   Inc(c.p.nestedLoopCounter)
   n.sons[length-1] = SemStmt(c, n.sons[length-1])
   Dec(c.p.nestedLoopCounter)
@@ -672,14 +689,15 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
   if n.sons[bodyPos].kind != nkEmpty:
     if sfImportc in s.flags:
       LocalError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s)
-    if efDetermineType notin flags:
+    #if efDetermineType notin flags:
-      pushProcCon(c, s)
+    # XXX not good enough; see tnamedparamanonproc.nim
-      addResult(c, s.typ.sons[0], n.info, skProc)
+    pushProcCon(c, s)
-      let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
+    addResult(c, s.typ.sons[0], n.info, skProc)
-      n.sons[bodyPos] = transformBody(c.module, semBody, s)
+    let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
-      addResultNode(c, n)
+    n.sons[bodyPos] = transformBody(c.module, semBody, s)
-      popProcCon(c)
+    addResultNode(c, n)
-      sideEffectsCheck(c, s)
+    popProcCon(c)
+    sideEffectsCheck(c, s)
   else:
     LocalError(n.info, errImplOfXexpected, s.name.s)
   closeScope(c.tab)           # close scope for parameters
@@ -689,13 +707,16 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
 proc activate(c: PContext, n: PNode) =
   # XXX: This proc is part of my plan for getting rid of
   # forward declarations. stay tuned.
-  case n.kind
+  when false:
-  of nkLambdaKinds:
+    # well for now it breaks code ... I added the test case in main.nim of the
-    discard semLambda(c, n, {})
+    # compiler itself to break bootstrapping :P
-  of nkCallKinds:
+    case n.kind
-    for i in 1 .. <n.len: activate(c, n[i])
+    of nkLambdaKinds:
-  else:
+      discard semLambda(c, n, {})
-    nil
+    of nkCallKinds:
+      for i in 1 .. <n.len: activate(c, n[i])
+    else:
+      nil
 
 proc instantiateDestructor*(c: PContext, typ: PType): bool
 
@@ -713,6 +734,12 @@ proc doDestructorStuff(c: PContext, s: PSym, n: PNode) =
             useSym(t.sons[i].destructor),
             n.sons[paramsPos][1][0]]))
 
+proc maybeAddResult(c: PContext, s: PSym, n: PNode) =
+  if s.typ.sons[0] != nil and
+      (s.kind != skIterator or s.typ.callConv == ccClosure):
+    addResult(c, s.typ.sons[0], n.info, s.kind)
+    addResultNode(c, n)
+
 proc semProcAux(c: PContext, n: PNode, kind: TSymKind, 
                 validPragmas: TSpecialWords): PNode = 
   result = semProcAnnotation(c, n)
@@ -780,6 +807,8 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     n.sons[pragmasPos] = proto.ast.sons[pragmasPos]
     if n.sons[namePos].kind != nkSym: InternalError(n.info, "semProcAux")
     n.sons[namePos].sym = proto
+    if importantComments() and not isNil(proto.ast.comment):
+      n.comment = proto.ast.comment
     proto.ast = n             # needed for code generation
     popOwner()
     pushOwner(s)
@@ -792,17 +821,13 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     if n.sons[genericParamsPos].kind == nkEmpty: 
       ParamsTypeCheck(c, s.typ)
       pushProcCon(c, s)
-      if s.typ.sons[0] != nil and
+      maybeAddResult(c, s, n)
-          (kind != skIterator or s.typ.callConv == ccClosure):
-        addResult(c, s.typ.sons[0], n.info, kind)
-        addResultNode(c, n)
       if sfImportc notin s.flags:
         # no semantic checking for importc:
         let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
         # unfortunately we cannot skip this step when in 'system.compiles'
         # context as it may even be evaluated in 'system.compiles':
         n.sons[bodyPos] = transformBody(c.module, semBody, s)
-      #if s.typ.sons[0] != nil and kind != skIterator: addResultNode(c, n)
       popProcCon(c)
     else: 
       if s.typ.sons[0] != nil and kind != skIterator:
@@ -846,31 +871,30 @@ proc semIterator(c: PContext, n: PNode): PNode =
 proc semProc(c: PContext, n: PNode): PNode = 
   result = semProcAux(c, n, skProc, procPragmas)
 
+proc hasObjParam(s: PSym): bool =
+  var t = s.typ
+  for col in countup(1, sonsLen(t)-1):
+    if skipTypes(t.sons[col], skipPtrs).kind == tyObject:
+      return true
+
+proc finishMethod(c: PContext, s: PSym) =
+  if hasObjParam(s):
+    methodDef(s, false)
+
 proc semMethod(c: PContext, n: PNode): PNode = 
   if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "method")
   result = semProcAux(c, n, skMethod, methodPragmas)
   
   var s = result.sons[namePos].sym
-  var t = s.typ
+  if not isGenericRoutine(s):
-  var hasObjParam = false
+    if hasObjParam(s):
-  
+      methodDef(s, false)
-  for col in countup(1, sonsLen(t)-1): 
+    else:
-    if skipTypes(t.sons[col], skipPtrs).kind == tyObject: 
+      LocalError(n.info, errXNeedsParamObjectType, "method")
-      hasObjParam = true
-      break
-  
-  # XXX this not really correct way to do it: Perhaps it should be done after
-  # generic instantiation. Well it's good enough for now: 
-  if hasObjParam:
-    methodDef(s, false)
-  else:
-    LocalError(n.info, errXNeedsParamObjectType, "method")
 
 proc semConverterDef(c: PContext, n: PNode): PNode = 
   if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "converter")
   checkSonsLen(n, bodyPos + 1)
-  if n.sons[genericParamsPos].kind != nkEmpty: 
-    LocalError(n.info, errNoGenericParamsAllowedForX, "converter")
   result = semProcAux(c, n, skConverter, converterPragmas)
   var s = result.sons[namePos].sym
   var t = s.typ
@@ -1075,7 +1099,8 @@ proc insertDestructors(c: PContext, varSection: PNode):
       varTyp = varId.sym.typ
       info = varId.info
 
-    if varTyp != nil and instantiateDestructor(c, varTyp):
+    if varTyp != nil and instantiateDestructor(c, varTyp) and 
+        sfGlobal notin varId.sym.flags:
       var tryStmt = newNodeI(nkTryStmt, info)
 
       if j < totalVars - 1:

compiler/semthreads.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.

compiler/semtypes.nim

@@ -635,6 +635,13 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
         genericParams.addSon(newSymNode(s))
         result = typeClass
 
+proc semParamType(c: PContext, n: PNode, constraint: var PNode): PType =
+  if n.kind == nkCurlyExpr:
+    result = semTypeNode(c, n.sons[0], nil)
+    constraint = semNodeKindConstraints(n)
+  else:
+    result = semTypeNode(c, n, nil)
+
 proc semProcTypeNode(c: PContext, n, genericParams: PNode, 
                      prev: PType, kind: TSymKind): PType = 
   var
@@ -661,12 +668,13 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
     var
       typ: PType = nil
       def: PNode = nil
+      constraint: PNode = nil
       length = sonsLen(a)
       hasType = a.sons[length-2].kind != nkEmpty
       hasDefault = a.sons[length-1].kind != nkEmpty
 
     if hasType:
-      typ = semTypeNode(c, a.sons[length-2], nil)
+      typ = semParamType(c, a.sons[length-2], constraint)
       
     if hasDefault:
       def = semExprWithType(c, a.sons[length-1]) 
@@ -689,6 +697,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
                                     arg.name.s, arg.info).skipIntLit
       arg.typ = finalType
       arg.position = counter
+      arg.constraint = constraint
       inc(counter)
       if def != nil and def.kind != nkEmpty: arg.ast = copyTree(def)
       if ContainsOrIncl(check, arg.name.id): 
@@ -787,6 +796,12 @@ proc semTypeExpr(c: PContext, n: PNode): PType =
   else:
     LocalError(n.info, errTypeExpected, n.renderTree)
 
+proc freshType(res, prev: PType): PType {.inline.} =
+  if prev.isNil:
+    result = copyType(res, res.owner, keepId=false)
+  else:
+    result = res
+
 proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
   result = nil
   if gCmd == cmdIdeTools: suggestExpr(c, n)
@@ -825,6 +840,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
         checkSonsLen(n, 3)
         result = semTypeNode(c, n.sons[1], prev)
         if result.kind in NilableTypes and n.sons[2].kind == nkNilLit:
+          result = freshType(result, prev)
           result.flags.incl(tfNotNil)
         else:
           LocalError(n.info, errGenerated, "invalid type")
@@ -832,11 +848,6 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
         result = semTypeExpr(c, n)
     else:
       result = semTypeExpr(c, n)
-  of nkCurlyExpr:
-    result = semTypeNode(c, n.sons[0], nil)
-    if result != nil:
-      result = copyType(result, getCurrOwner(), true)
-      result.constraint = semNodeKindConstraints(n)
   of nkWhenStmt:
     var whenResult = semWhen(c, n, false)
     if whenResult.kind == nkStmtList: whenResult.kind = nkStmtListType
@@ -919,6 +930,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
   of nkSharedTy:
     checkSonsLen(n, 1)
     result = semTypeNode(c, n.sons[0], prev)
+    result = freshType(result, prev)
     result.flags.incl(tfShared)
   else:
     LocalError(n.info, errTypeExpected)

compiler/sigmatch.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -12,18 +12,19 @@
 
 import 
   intsets, ast, astalgo, semdata, types, msgs, renderer, lookups, semtypinst,
-  magicsys, condsyms, idents, lexer, options
+  magicsys, condsyms, idents, lexer, options, parampatterns, strutils,
+  docgen
 
 type
   TCandidateState* = enum 
     csEmpty, csMatch, csNoMatch
 
   TCandidate* {.final.} = object 
-    exactMatches*: int
+    exactMatches*: int       # also misused to prefer iters over procs
+    genericMatches: int      # also misused to prefer constraints
     subtypeMatches: int
     intConvMatches: int      # conversions to int are not as expensive
     convMatches: int
-    genericMatches: int
     state*: TCandidateState
     callee*: PType           # may not be nil!
     calleeSym*: PSym         # may be nil
@@ -33,6 +34,8 @@ type
     baseTypeMatch: bool      # needed for conversions from T to openarray[T]
                              # for example
     proxyMatch*: bool        # to prevent instantiations
+    genericConverter*: bool  # true if a generic converter needs to
+                             # be instantiated
     inheritancePenalty: int  # to prefer closest father object type
   
   TTypeRelation* = enum      # order is important!
@@ -57,6 +60,7 @@ proc initCandidateAux(c: var TCandidate, callee: PType) {.inline.} =
   c.callee = callee
   c.call = nil
   c.baseTypeMatch = false
+  c.genericConverter = false
   c.inheritancePenalty = 0
 
 proc initCandidate*(c: var TCandidate, callee: PType) = 
@@ -571,17 +575,27 @@ proc userConvMatch(c: PContext, m: var TCandidate, f, a: PType,
   for i in countup(0, len(c.converters) - 1): 
     var src = c.converters[i].typ.sons[1]
     var dest = c.converters[i].typ.sons[0]
-    if (typeRel(m, f, dest) == isEqual) and
+    # for generic type converters we need to check 'src <- a' before
-        (typeRel(m, src, a) == isEqual):
+    # 'f <- dest' in order to not break the unification:
+    # see tests/tgenericconverter:
+    let srca = typeRel(m, src, a)
+    if srca notin {isEqual, isGeneric}: continue
+    
+    let destIsGeneric = containsGenericType(dest)
+    if destIsGeneric:
+      dest = generateTypeInstance(c, m.bindings, arg, dest)
+    let fdest = typeRel(m, f, dest)
+    if fdest in {isEqual, isGeneric}: 
       markUsed(arg, c.converters[i])
       var s = newSymNode(c.converters[i])
       s.typ = c.converters[i].typ
       s.info = arg.info
-      result = newNodeIT(nkHiddenCallConv, arg.info, s.typ.sons[0])
+      result = newNodeIT(nkHiddenCallConv, arg.info, dest)
       addSon(result, s)
       addSon(result, copyTree(arg))
       inc(m.convMatches)
-      return
+      m.genericConverter = srca == isGeneric or destIsGeneric
+      return result
 
 proc localConvMatch(c: PContext, m: var TCandidate, f, a: PType, 
                     arg: PNode): PNode = 
@@ -692,8 +706,8 @@ proc ParamTypesMatchAux(c: PContext, m: var TCandidate, f, a: PType,
         else:
           result = userConvMatch(c, m, base(f), a, arg)
 
-proc ParamTypesMatch(c: PContext, m: var TCandidate, f, a: PType, 
+proc ParamTypesMatch*(c: PContext, m: var TCandidate, f, a: PType, 
-                     arg, argOrig: PNode): PNode =
+                      arg, argOrig: PNode): PNode =
   if arg == nil or arg.kind notin nkSymChoices:
     result = ParamTypesMatchAux(c, m, f, a, arg, argOrig)
   else: 
@@ -739,27 +753,21 @@ proc ParamTypesMatch(c: PContext, m: var TCandidate, f, a: PType,
       result = ParamTypesMatchAux(c, m, f, arg.sons[best].typ, arg.sons[best],
                                   argOrig)
 
-proc IndexTypesMatch*(c: PContext, f, a: PType, arg: PNode): PNode = 
-  var m: TCandidate
-  initCandidate(m, f)
-  result = paramTypesMatch(c, m, f, a, arg, nil)
-
-proc ConvertTo*(c: PContext, f: PType, n: PNode): PNode = 
-  var m: TCandidate
-  initCandidate(m, f)
-  result = paramTypesMatch(c, m, f, n.typ, n, nil)
-
-proc argtypeMatches*(c: PContext, f, a: PType): bool = 
-  var m: TCandidate
-  initCandidate(m, f)
-  result = paramTypesMatch(c, m, f, a, ast.emptyNode, nil) != nil  
-
 proc setSon(father: PNode, at: int, son: PNode) = 
   if sonsLen(father) <= at: setlen(father.sons, at + 1)
   father.sons[at] = son
 
-proc matchesAux*(c: PContext, n, nOrig: PNode,
+proc matchesAux(c: PContext, n, nOrig: PNode,
-                 m: var TCandidate, marker: var TIntSet) = 
+                m: var TCandidate, marker: var TIntSet) = 
+  template checkConstraint(n: expr) {.immediate, dirty.} =
+    if not formal.constraint.isNil:
+      if matchNodeKinds(formal.constraint, n):
+        # better match over other routines with no such restriction:
+        inc(m.genericMatches, 100)
+      else:
+        m.state = csNoMatch
+        return
+  
   var f = 1 # iterates over formal parameters
   var a = 1 # iterates over the actual given arguments
   m.state = csMatch           # until proven otherwise
@@ -793,6 +801,7 @@ proc matchesAux*(c: PContext, n, nOrig: PNode,
       if arg == nil: 
         m.state = csNoMatch
         return
+      checkConstraint(n.sons[a].sons[1])
       if m.baseTypeMatch: 
         assert(container == nil)
         container = newNodeI(nkBracket, n.sons[a].info)
@@ -849,6 +858,7 @@ proc matchesAux*(c: PContext, n, nOrig: PNode,
           if f != formalLen - 1: container = nil
         else: 
           setSon(m.call, formal.position + 1, arg)
+      checkConstraint(n.sons[a])
     inc(a)
     inc(f)
 
@@ -880,4 +890,13 @@ proc matches*(c: PContext, n, nOrig: PNode, m: var TCandidate) =
         setSon(m.call, formal.position + 1, copyTree(formal.ast))
     inc(f)
 
+proc argtypeMatches*(c: PContext, f, a: PType): bool = 
+  var m: TCandidate
+  initCandidate(m, f)
+  let res = paramTypesMatch(c, m, f, a, ast.emptyNode, nil)
+  #instantiateGenericConverters(c, res, m)
+  # XXX this is used by patterns.nim too; I think it's better to not
+  # instantiate generic converters for that
+  result = res != nil
+
 include suggest

compiler/suggest.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -39,6 +39,8 @@ proc SymToStr(s: PSym, isLocal: bool, section: string, li: TLineInfo): string =
   result.add($ToLinenumber(li))
   result.add(sep)
   result.add($ToColumn(li))
+  result.add(sep)
+  result.add(s.extractDocComment.escape)
 
 proc SymToStr(s: PSym, isLocal: bool, section: string): string = 
   result = SymToStr(s, isLocal, section, s.info)

compiler/transf.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -150,7 +150,7 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode =
       newVar.owner = getCurrOwner(c)
       IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
       var defs = newTransNode(nkIdentDefs, it.info, 3)
-      if gCmd == cmdDoc:
+      if importantComments():
         # keep documentation information:
         pnode(defs).comment = it.comment
       defs[0] = newSymNode(newVar).PTransNode
@@ -665,7 +665,7 @@ proc transform(c: PTransf, n: PNode): PTransNode =
   of nkIdentDefs, nkConstDef:
     result = transformSons(c, n)
     # XXX comment handling really sucks:
-    if gCmd == cmdDoc:
+    if importantComments():
       pnode(result).comment = n.comment
   else:
     result = transformSons(c, n)

compiler/types.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -632,7 +632,8 @@ proc SameTypeOrNil*(a, b: PType, flags: TTypeCmpFlags = {}): bool =
       result = SameTypeAux(a, b, c)
 
 proc equalParam(a, b: PSym): TParamsEquality = 
-  if SameTypeOrNil(a.typ, b.typ, {TypeDescExactMatch}): 
+  if SameTypeOrNil(a.typ, b.typ, {TypeDescExactMatch}) and
+      ExprStructuralEquivalent(a.constraint, b.constraint):
     if a.ast == b.ast: 
       result = paramsEqual
     elif a.ast != nil and b.ast != nil: 
@@ -875,15 +876,19 @@ proc inheritanceDiff*(a, b: PType): int =
   # | returns: -x iff `a` is the x'th direct superclass of `b`
   # | returns: +x iff `a` is the x'th direct subclass of `b`
   # | returns: `maxint` iff `a` and `b` are not compatible at all
+  assert a.kind == tyObject
+  assert b.kind == tyObject
   var x = a
   result = 0
   while x != nil:
+    x = skipTypes(x, skipPtrs)
     if sameObjectTypes(x, b): return 
     x = x.sons[0]
     dec(result)
   var y = b
   result = 0
   while y != nil:
+    y = skipTypes(y, skipPtrs)
     if sameObjectTypes(y, a): return 
     y = y.sons[0]
     inc(result)

config/nimrod.cfg

@@ -13,11 +13,6 @@ cc = gcc
 arm.linux.gcc.exe = "arm-linux-gcc"
 arm.linux.gcc.linkerexe = "arm-linux-gcc"
 
-@if nim: 
-  # use the old fixed library for bootstrapping with Nim:
-  lib = "nimlib"
-@end
-
 path="$lib/core"
 path="$lib/pure"
 path="$lib/pure/collections"
@@ -36,7 +31,10 @@ path="$lib/windows"
 path="$lib/posix"
 path="$lib/ecmas"
 path="$lib/pure/unidecode"
-#recursivePath:"$home/.babel/lib"
+
+@if nimbabel:
+  babelpath="$home/.babel/libs/"
+@end
 
 @if release or quick:
   obj_checks:off

doc/advopt.txt

@@ -22,11 +22,11 @@ Advanced options:
   -m, --mainmodule:FILE     set the project main module
   -o, --out:FILE            set the output filename
   --stdout                  output to stdout
+  --listFullPaths           list full paths in messages
   -w, --warnings:on|off     turn all warnings on|off
   --warning[X]:on|off       turn specific warning X on|off
   --hints:on|off            turn all hints on|off
   --hint[X]:on|off          turn specific hint X on|off
-  --recursivePath:PATH      add a path and all of its subdirectories
   --lib:PATH                set the system library path
   --import:PATH             add an automatically imported module
   --include:PATH            add an automatically included module
@@ -67,6 +67,8 @@ Advanced options:
   --gc:refc|boehm|none      use Nimrod's native GC|Boehm GC|no GC
   --index:on|off            turn index file generation on|off
   --putenv:key=value        set an environment variable
+  --babelPath:PATH          add a path for Babel support
+  --excludePath:PATH        exclude a path from the list of search paths
   --listCmd                 list the commands used to execute external programs
   --parallelBuild=0|1|...   perform a parallel build
                             value = number of processors (0 for auto-detect)

doc/lib.txt

@@ -38,11 +38,11 @@ Core
 
 * `threads <threads.html>`_
   Nimrod thread support. **Note**: This is part of the system module. Do not
-  import it explicitely.
+  import it explicitly.
 
 * `channels <channels.html>`_
   Nimrod message passing support for threads. **Note**: This is part of the 
-  system module. Do not import it explicitely.
+  system module. Do not import it explicitly.
 
 * `locks <locks.html>`_
   Locks and condition variables for Nimrod.

doc/tut1.txt

@@ -349,6 +349,7 @@ provides. The example uses the built-in ``countup`` iterator:
   echo("Counting to ten: ")
   for i in countup(1, 10):
     echo($i)
+  # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines
 
 The built-in ``$`` operator turns an integer (``int``) and many other types
 into a string. The variable ``i`` is implicitly declared by the ``for`` loop
@@ -362,6 +363,7 @@ the same:
   while i <= 10:
     echo($i)
     inc(i) # increment i by 1
+  # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines
 
 Counting down can be achieved as easily (but is less often needed):
 
@@ -369,6 +371,7 @@ Counting down can be achieved as easily (but is less often needed):
   echo("Counting down from 10 to 1: ")
   for i in countdown(10, 1):
     echo($i)
+  # --> Outputs 10 9 8 7 6 5 4 3 2 1 on different lines
 
 Since counting up occurs so often in programs, Nimrod also has a ``..`` iterator
 that does the same:
@@ -780,12 +783,15 @@ important differences:
 * Iterators cannot contain a ``return`` statement and procs cannot contain a
   ``yield`` statement.
 * Iterators have no implicit ``result`` variable.
-* Iterators do not support recursion. (This restriction will be gone in a
+* Iterators do not support recursion.
-  future version of the compiler.)
 * Iterators cannot be forward declared, because the compiler must be able
   to inline an iterator. (This restriction will be gone in a
   future version of the compiler.)
 
+However, you can also use a ``closure`` iterator to get a different set of
+restrictions. See `first class iterators <manual.html#first-class-iterators>`_
+for details.
+
 
 Basic types
 ===========
@@ -916,6 +922,37 @@ types automatically and vice versa. The ``toInt`` and ``toFloat`` procs can be
 used for these conversions.
 
 
+Internal type representation
+============================
+
+As mentioned earlier, the built-in ``$`` (stringify) operator turns any basic
+type into a string, which you can then print to the screen with the ``echo``
+proc. However, advanced types, or types you may define yourself won't work with
+the ``$`` operator until you define one for them. Sometimes you just want to
+debug the current value of a complex type without having to write its ``$``
+operator.  You can use then the ``repr`` proc which works with any type and
+even complex data graphs with cycles. The following example shows that even for
+basic types there is a difference between the ``$`` and ``repr`` outputs:
+
+.. code-block:: nimrod
+  var
+    myBool = true
+    myCharacter = 'n'
+    myString = "nimrod"
+    myInteger = 42
+    myFloat = 3.14
+  echo($myBool, ":", repr(myBool))
+  # --> true:true
+  echo($myCharacter, ":", repr(myCharacter))
+  # --> n:'n'
+  echo($myString, ":", repr(myString))
+  # --> nimrod:0x10fa8c050"nimrod"
+  echo($myInteger, ":", repr(myInteger))
+  # --> 42:42
+  echo($myFloat, ":", repr(myFloat))
+  # --> 3.1400000000000001e+00:3.1400000000000001e+00
+
+
 Advanced types
 ==============
 
@@ -1089,6 +1126,54 @@ copies the whole array contents.
 The built-in ``len`` proc returns the array's length. ``low(a)`` returns the
 lowest valid index for the array `a` and ``high(a)`` the highest valid index.
 
+.. code-block:: nimrod
+  type
+    TDirection = enum
+      north, east, south, west
+    TBlinkLights = enum
+      off, on, slowBlink, mediumBlink, fastBlink
+    TLevelSetting = array[north..west, TBlinkLights]
+  var
+    level : TLevelSetting
+  level[north] = on
+  level[south] = slowBlink
+  level[east] = fastBlink
+  echo repr(level)  # --> [on, fastBlink, slowBlink, off]
+  echo low(level)   # --> north
+  echo len(level)   # --> 4
+  echo high(level)  # --> west
+
+The syntax for nested arrays (multidimensional) in other languages is a matter
+of appending more brackets because usually each dimension is restricted to the
+same index type as the others. In nimrod you can have different dimensions with
+different index types, so the nesting syntax is slightly different. Building on
+the previous example where a level is defined as an array of enums indexed by
+yet another enum, we can add the following lines to add a light tower type
+subdivided in height levels accessed through their integer index:
+
+.. code-block:: nimrod
+  type
+    TLightTower = array[1..10, TLevelSetting]
+  var
+    tower: TLightTower
+  tower[1][north] = slowBlink
+  tower[1][east] = mediumBlink
+  echo len(tower)     # --> 10
+  echo len(tower[1])  # --> 4
+  echo repr(tower)    # --> [[slowBlink, mediumBlink, ...more output..
+  # The following lines don't compile due to type mistmatch errors
+  #tower[north][east] = on
+  #tower[0][1] = on
+
+Note how the built-in ``len`` proc returns only the array's first dimension
+length.  Another way of defining the ``TLightTower`` to show better its
+nested nature would be to omit the previous definition of the ``TLevelSetting``
+type and instead write it embedded directly as the type of the first dimension:
+
+.. code-block:: nimrod
+  type
+    TLightTower = array[1..10, array[north..west, TBlinkLights]]
+
 
 Sequences
 ---------
@@ -1120,6 +1205,28 @@ raised) for performance reasons. Thus one should use empty sequences ``@[]``
 rather than ``nil`` as the *empty* value. But ``@[]`` creates a sequence
 object on the heap, so there is a trade-off to be made here.
 
+The ``for`` statement can be used with one or two variables when used with a
+sequence. When you use the one variable form, the variable will hold the value
+provided by the sequence. The ``for`` statement is looping over the results
+from the ``items()`` iterator from the `system <system.html>`_ module.  But if
+you use the two variable form, the first variable will hold the index position
+and the second variable will hold the value. Here the ``for`` statement is
+looping over the results from the ``pairs()`` iterator from the `system
+<system.html>`_ module.  Examples:
+
+.. code-block:: nimrod
+  for i in @[3, 4, 5]:
+    echo($i)
+  # --> 3
+  # --> 4
+  # --> 5
+
+  for i, value in @[3, 4, 5]:
+    echo("index: ", $i, ", value:", $value)
+  # --> index: 0, value:3
+  # --> index: 1, value:4
+  # --> index: 2, value:5
+
 
 Open arrays
 -----------
@@ -1222,13 +1329,9 @@ Traced references are declared with the **ref** keyword, untraced references
 are declared with the **ptr** keyword.
 
 The empty ``[]`` subscript notation can be used to *derefer* a reference,
-meaning to retrieve the item the reference points to. The ``addr`` operator 
+meaning to retrieve the item the reference points to. The ``.`` (access a
-returns the address of an item. An address is always an untraced reference:
+tuple/object field operator) and ``[]`` (array/string/sequence index operator)
-``addr`` is an *unsafe* feature.
+operators perform implicit dereferencing operations for reference types:
-
-The ``.`` (access a tuple/object field operator)
-and ``[]`` (array/string/sequence index operator) operators perform implicit
-dereferencing operations for reference types:
 
 .. code-block:: nimrod
 
@@ -1245,8 +1348,8 @@ dereferencing operations for reference types:
 
 To allocate a new traced object, the built-in procedure ``new`` has to be used.
 To deal with untraced memory, the procedures ``alloc``, ``dealloc`` and
-``realloc`` can be used. The documentation of the system module contains
+``realloc`` can be used. The documentation of the `system <system.html>`_
-further information.
+module contains further information.
 
 If a reference points to *nothing*, it has the value ``nil``.
 

doc/tut2.txt

@@ -218,7 +218,7 @@ So "pure object oriented" code is easy to write:
   import strutils
   
   stdout.writeln("Give a list of numbers (separated by spaces): ")
-  stdout.write(stdin.readLine.split.each(parseInt).max.`$`)
+  stdout.write(stdin.readLine.split.map(parseInt).max.`$`)
   stdout.writeln(" is the maximum!")
 
 
@@ -433,45 +433,59 @@ handled, it is propagated through the call stack. This means that often
 the rest of the procedure - that is not within a ``finally`` clause -
 is not executed (if an exception occurs).
 
+If you need to *access* the actual exception object or message inside an
+``except`` branch you can use the getCurrentException() and
+getCurrentExceptionMsg() procs from the `system <system.html>`_ module.
+Example:
+
+.. code-block:: nimrod
+  try:
+    doSomethingHere()
+  except:
+    let
+      e = getCurrentException()
+      msg = getCurrentExceptionMsg()
+    echo "Got exception ", repr(e), " with message ", msg
+
 
 Exception hierarchy
 -------------------
 
 If you want to create your own exceptions you can inherit from E_Base, but you
 can also inherit from one of the existing exceptions if they fit your purpose.
-The exception tree is:
+The exception tree is::
 
-* E_Base
+  * E_Base
-	* EAsynch
+    * EAsynch
-		* EControlC
+      * EControlC
-	* ESynch
+    * ESynch
-		* ESystem
+      * ESystem
-			* EIO
+        * EIO
-			* EOS
+        * EOS
-				* EInvalidLibrary
+          * EInvalidLibrary
-		* EResourceExhausted
+      * EResourceExhausted
-		* EOutOfMemory
+      * EOutOfMemory
-		* EStackOverflow
+      * EStackOverflow
-	* EArithmetic
+    * EArithmetic
-		* EDivByZero
+      * EDivByZero
-		* EOverflow
+      * EOverflow
-	* EAccessViolation
+    * EAccessViolation
-	* EAssertionFailed
+    * EAssertionFailed
-	* EInvalidValue
+    * EInvalidValue
-		* EInvalidKey
+      * EInvalidKey
-	* EInvalidIndex
+    * EInvalidIndex
-	* EInvalidField
+    * EInvalidField
-	* EOutOfRange
+    * EOutOfRange
-	* ENoExceptionToReraise
+    * ENoExceptionToReraise
-	* EInvalidObjectAssignment
+    * EInvalidObjectAssignment
-	* EInvalidObjectConversion
+    * EInvalidObjectConversion
-	* EFloatingPoint
+    * EFloatingPoint
-		* EFloatInvalidOp
+      * EFloatInvalidOp
-		* EFloatDivByZero
+      * EFloatDivByZero
-		* EFloatOverflow
+      * EFloatOverflow
-		* EFloatUnderflow
+      * EFloatUnderflow
-		* EFloatInexact
+      * EFloatInexact
-	* EDeadThread
+    * EDeadThread
 
 See the `system <system.html>`_ module for a description of each exception.
 

examples/cross_calculator/ios/scripts/xcode_prebuild.sh

@@ -21,7 +21,6 @@ DEST_NIMBASE=build/nimcache/nimbase.h
 # Ok, are we out now?
 if [ -d src ]
 then
-	# You may want to use a wildcard here to compile all nimrod files.
 	$PATH_TO_NIMROD objc --noMain  --app:lib \
 		--nimcache:build/nimcache --compileOnly \
 		--header --cpu:i386 ../nimrod_backend/backend.nim

examples/cross_todo/readme.txt

@@ -3,3 +3,5 @@ called by different native user interfaces.
 
 This example builds on the knowledge learned from the cross_calculator example.
 Check it out first to learn how to set up nimrod on different platforms.
+Additional implementations are provided at the external
+https://github.com/gradha/nimrod-crossplatform-todo github repository.

examples/maximum.nim

@@ -3,4 +3,4 @@
 import strutils
 
 echo "Give a list of numbers (separated by spaces): "
-stdin.readLine.split.each(parseInt).max.`$`.echo(" is the maximum!")
+stdin.readLine.split.map(parseInt).max.`$`.echo(" is the maximum!")

koch.nim

@@ -1,7 +1,7 @@
 #
 #
 #         Maintenance program for Nimrod  
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -26,7 +26,7 @@ const
 +-----------------------------------------------------------------+
 |         Maintenance program for Nimrod                          |
 |             Version $1|
-|             (c) 2012 Andreas Rumpf                              |
+|             (c) 2013 Andreas Rumpf                              |
 +-----------------------------------------------------------------+
 Build time: $2, $3
 
@@ -49,6 +49,7 @@ Boot options:
   -d:tinyc                 include the Tiny C backend (not supported on Windows)
   -d:useGnuReadline        use the GNU readline library for interactive mode
                            (not needed on Windows)
+  -d:useFFI                build Nimrod with FFI support at compile time
   -d:nativeStacktrace      use native stack traces (only for Mac OS X or Linux)
 """
 
@@ -214,7 +215,7 @@ when defined(withUpdate):
       if errcode == 0:
         if output == "":
           # No changes
-          echo("No update. Exiting..")
+          echo("No update. Exiting...")
           return
         else:
           echo("Fetching updates from repo...")

lib/core/macros.nim

@@ -45,8 +45,16 @@ type
     nnkConstDef, nnkTypeDef,
     nnkYieldStmt, nnkTryStmt, nnkFinally, nnkRaiseStmt,
     nnkReturnStmt, nnkBreakStmt, nnkContinueStmt, nnkBlockStmt, nnkStaticStmt,
-    nnkDiscardStmt, nnkStmtList, nnkImportStmt, nnkFromStmt,
+    nnkDiscardStmt, nnkStmtList, 
-    nnkIncludeStmt, nnkBindStmt, nnkMixinStmt,
+    
+    nnkImportStmt,
+    nnkImportExceptStmt,
+    nnkExportStmt,
+    nnkExportExceptStmt,
+    nnkFromStmt,
+    nnkIncludeStmt,
+    
+    nnkBindStmt, nnkMixinStmt,
     nnkCommentStmt, nnkStmtListExpr, nnkBlockExpr,
     nnkStmtListType, nnkBlockType, nnkTypeOfExpr, nnkObjectTy,
     nnkTupleTy, nnkRecList, nnkRecCase, nnkRecWhen,

lib/core/typeinfo.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Dominik Picheta, Andreas Rumpf
+#        (c) Copyright 2013 Dominik Picheta, Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -139,12 +139,15 @@ proc invokeNewSeq*(x: TAny, len: int) =
   var z = newSeq(x.rawType, len)
   genericShallowAssign(x.value, addr(z), x.rawType)
 
-proc extendSeq*(x: TAny, elems = 1) =
+proc extendSeq*(x: TAny) =
-  ## performs ``setLen(x, x.len+elems)``. `x` needs to represent a ``seq``.
+  ## performs ``setLen(x, x.len+1)``. `x` needs to represent a ``seq``.
   assert x.rawType.kind == tySequence
   var y = cast[ptr PGenSeq](x.value)[]
-  var z = incrSeq(y, x.rawType.base.size * elems)
+  var z = incrSeq(y, x.rawType.base.size)
-  genericShallowAssign(x.value, addr(z), x.rawType)
+  # 'incrSeq' already freed the memory for us and copied over the RC!
+  # So we simply copy the raw pointer into 'x.value':
+  cast[ppointer](x.value)[] = z
+  #genericShallowAssign(x.value, addr(z), x.rawType)
 
 proc setObjectRuntimeType*(x: TAny) =
   ## this needs to be called to set `x`'s runtime object type field.

lib/impure/web.nim

@@ -17,7 +17,8 @@
 ## Currently only requesting URLs is implemented. The implementation depends
 ## on the libcurl library!
 ##
-## **Deprecated since version 0.8.8:** Use the ``httpclient`` module instead. 
+## **Deprecated since version 0.8.8:** Use the
+## `httpclient <httpclient.html>`_ module instead. 
 ## 
 
 {.deprecated.}

lib/pure/asyncio.nim

@@ -35,6 +35,9 @@ import sockets, os
 ## that in the future this type's fields will not be exported therefore breaking
 ## your code.
 ##
+## **Warning:** The API of this module is unstable, and therefore is subject
+## to change.
+##
 ## Asynchronous sockets
 ## ====================
 ##
@@ -123,6 +126,7 @@ type
     handleTask*: proc (s: PAsyncSocket) {.closure.}
 
     lineBuffer: TaintedString ## Temporary storage for ``recvLine``
+    sendBuffer: string ## Temporary storage for ``send``
     sslNeedAccept: bool
     proto: TProtocol
     deleg: PDelegate
@@ -152,6 +156,7 @@ proc newAsyncSocket(): PAsyncSocket =
   result.handleTask = (proc (s: PAsyncSocket) = nil)
 
   result.lineBuffer = "".TaintedString
+  result.sendBuffer = ""
 
 proc AsyncSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, 
                   protocol: TProtocol = IPPROTO_TCP, 
@@ -222,10 +227,22 @@ proc asyncSockHandleWrite(h: PObject) =
     else:
       PAsyncSocket(h).deleg.mode = fmReadWrite
   else:
-    if PAsyncSocket(h).handleWrite != nil:
+    if PAsyncSocket(h).sendBuffer != "":
-      PAsyncSocket(h).handleWrite(PAsyncSocket(h))
+      let sock = PAsyncSocket(h)
+      let bytesSent = sock.socket.sendAsync(sock.sendBuffer)
+      assert bytesSent > 0
+      if bytesSent != sock.sendBuffer.len:
+        sock.sendBuffer = sock.sendBuffer[bytesSent .. -1]
+      elif bytesSent == sock.sendBuffer.len:
+        sock.sendBuffer = ""
+      
+      if PAsyncSocket(h).handleWrite != nil:
+        PAsyncSocket(h).handleWrite(PAsyncSocket(h))
     else:
-      PAsyncSocket(h).deleg.mode = fmRead
+      if PAsyncSocket(h).handleWrite != nil:
+        PAsyncSocket(h).handleWrite(PAsyncSocket(h))
+      else:
+        PAsyncSocket(h).deleg.mode = fmRead
 
 when defined(ssl):
   proc asyncSockDoHandshake(h: PObject) =
@@ -337,7 +354,8 @@ proc acceptAddr*(server: PAsyncSocket, client: var PAsyncSocket,
   # deleg.open is set in ``toDelegate``.
   
   client.socket = c
-  client.lineBuffer = ""
+  client.lineBuffer = "".TaintedString
+  client.sendBuffer = ""
   client.info = SockConnected
 
 proc accept*(server: PAsyncSocket, client: var PAsyncSocket) =
@@ -422,6 +440,9 @@ proc recvLine*(s: PAsyncSocket, line: var TaintedString): bool =
   ## sockets properly. This function guarantees that ``line`` is a full line,
   ## if this function can only retrieve some data; it will save this data and
   ## add it to the result when a full line is retrieved.
+  ##
+  ## Unlike ``sockets.recvLine`` this function will raise an EOS or ESSL
+  ## exception if an error occurs.
   setLen(line.string, 0)
   var dataReceived = "".TaintedString
   var ret = s.socket.recvLineAsync(dataReceived)
@@ -440,8 +461,29 @@ proc recvLine*(s: PAsyncSocket, line: var TaintedString): bool =
   of RecvDisconnected:
     result = true
   of RecvFail:
+    s.SocketError(async = true)
     result = false
 
+proc send*(sock: PAsyncSocket, data: string) =
+  ## Sends ``data`` to socket ``sock``. This is basically a nicer implementation
+  ## of ``sockets.sendAsync``.
+  ##
+  ## If ``data`` cannot be sent immediately it will be buffered and sent
+  ## when ``sock`` becomes writeable (during the ``handleWrite`` event).
+  ## It's possible that only a part of ``data`` will be sent immediately, while
+  ## the rest of it will be buffered and sent later.
+  if sock.sendBuffer.len != 0:
+    sock.sendBuffer.add(data)
+    return
+  let bytesSent = sock.socket.sendAsync(data)
+  assert bytesSent >= 0
+  if bytesSent == 0:
+    sock.sendBuffer.add(data)
+    sock.deleg.mode = fmReadWrite
+  elif bytesSent != data.len:
+    sock.sendBuffer.add(data[bytesSent .. -1])
+    sock.deleg.mode = fmReadWrite
+
 proc timeValFromMilliseconds(timeout = 500): TTimeVal =
   if timeout != -1:
     var seconds = timeout div 1000
@@ -556,7 +598,9 @@ when isMainModule:
   
   proc testRead(s: PAsyncSocket, no: int) =
     echo("Reading! " & $no)
-    var data = s.getSocket.recv()
+    var data = ""
+    if not s.recvLine(data):
+      OSError()
     if data == "":
       echo("Closing connection. " & $no)
       s.close()

lib/pure/collections/intsets.nim

@@ -190,6 +190,11 @@ proc `$`*(s: TIntSet): string =
   ## The `$` operator for int sets.
   dollarImpl()
 
+proc empty*(s: TIntSet): bool {.inline.} =
+  ## returns true if `s` is empty. This is safe to call even before
+  ## the set has been initialized with `initIntSet`.
+  result = s.counter == 0
+
 when isMainModule:
   var x = initIntSet()
   x.incl(1)

lib/pure/collections/sequtils.nim

@@ -9,8 +9,16 @@
 
 ## :Author: Alex Mitchell
 ##
-## This module implements operations for the built-in `seq`:idx: type
+## This module implements operations for the built-in `seq`:idx: type which
-## which were inspired by functional programming languages.
+## were inspired by functional programming languages. If you are looking for
+## the typical `map` function which applies a function to every element in a
+## sequence, it already exists in the `system <system.html>`_ module in both
+## mutable and immutable styles.
+##
+## Also, for functional style programming you may want to pass `anonymous procs
+## <manual.html#anonymous-procs>`_ to procs like ``filter`` to reduce typing.
+## Anonymous procs can use `the special do notation <manual.html#do-notation>`_
+## which is more convenient in certain situations.
 ##
 ## **Note**: This interface will change as soon as the compiler supports
 ## closures and proper coroutines.
@@ -19,7 +27,17 @@ when not defined(nimhygiene):
   {.pragma: dirty.}
 
 proc concat*[T](seqs: varargs[seq[T]]): seq[T] =
-  ## Takes several sequences' items and returns them inside of one sequence.
+  ## Takes several sequences' items and returns them inside a new sequence.
+  ##
+  ## Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   let
+  ##     s1 = @[1, 2, 3]
+  ##     s2 = @[4, 5]
+  ##     s3 = @[6, 7]
+  ##     total = concat(s1, s2, s3)
+  ##   assert total == @[1, 2, 3, 4, 5, 6, 7]
   var L = 0
   for seqitm in items(seqs): inc(L, len(seqitm))
   newSeq(result, L)
@@ -30,14 +48,42 @@ proc concat*[T](seqs: varargs[seq[T]]): seq[T] =
       inc(i)
 
 proc distnct*[T](seq1: seq[T]): seq[T] =
-  ## Removes duplicates from a sequence and returns it.
+  ## Returns a new sequence without duplicates.
+  ##
+  ## This proc is `misspelled` on purpose to avoid a clash with the keyword
+  ## ``distinct`` used to `define a derived type incompatible with its base
+  ## type <manual.html#distinct-type>`_. Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   let
+  ##     dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4]
+  ##     dup2 = @["a", "a", "c", "d", "d"]
+  ##     unique1 = distnct(dup1)
+  ##     unique2 = distnct(dup2)
+  ##   assert unique1 == @[1, 3, 4, 2, 8]
+  ##   assert unique2 == @["a", "c", "d"]
   result = @[]
   for itm in items(seq1):
     if not result.contains(itm): result.add(itm)
     
 proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] =
-  ## Combines two sequences. If one sequence is too short,
+  ## Returns a new sequence with a combination of the two input sequences.
-  ## the remaining items in the longer sequence are discarded.
+  ##
+  ## For convenience you can access the returned tuples through the named
+  ## fields `a` and `b`. If one sequence is shorter, the remaining items in the
+  ## longer sequence are discarded. Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   let
+  ##     short = @[1, 2, 3]
+  ##     long = @[6, 5, 4, 3, 2, 1]
+  ##     words = @["one", "two", "three"]
+  ##     zip1 = zip(short, long)
+  ##     zip2 = zip(short, words)
+  ##   assert zip1 == @[(1, 6), (2, 5), (3, 4)]
+  ##   assert zip2 == @[(1, "one"), (2, "two"), (3, "three")]
+  ##   assert zip1[2].b == 4
+  ##   assert zip2[2].b == "three"
   var m = min(seq1.len, seq2.len)
   newSeq(result, m)
   for i in 0 .. m-1: result[i] = (seq1[i], seq2[i])
@@ -45,21 +91,124 @@ proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] =
 iterator filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): T =
   ## Iterates through a sequence and yields every item that fulfills the
   ## predicate.
+  ##
+  ## Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   let numbers = @[1, 4, 5, 8, 9, 7, 4]
+  ##   for n in filter(numbers, proc (x: int): bool = x mod 2 == 0):
+  ##     echo($n)
+  ##   # echoes 4, 8, 4 in separate lines
   for i in countup(0, len(seq1) -1):
     var item = seq1[i]
     if pred(item): yield seq1[i]
 
 proc filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): seq[T] =
-  ## Returns all items in a sequence that fulfilled the predicate.
+  ## Returns a new sequence with all the items that fulfilled the predicate.
+  ##
+  ## Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   let
+  ##     colors = @["red", "yellow", "black"]
+  ##     f1 = filter(colors, proc(x: string): bool = x.len < 6)
+  ##     f2 = filter(colors) do (x: string) -> bool : x.len > 5
+  ##   assert f1 == @["red", "black"]
+  ##   assert f2 == @["yellow"]
   accumulateResult(filter(seq1, pred))
 
 template filterIt*(seq1, pred: expr): expr {.immediate, dirty.} =
-  ## Finds a specific item in a sequence as long as the 
+  ## Returns a new sequence with all the items that fulfilled the predicate.
-  ## predicate returns true. The predicate needs to be an expression
+  ##
-  ## containing ``it``: ``filterIt("abcxyz", it == 'x')``.
+  ## Unlike the `proc` version, the predicate needs to be an expression using
-  block:
+  ## the ``it`` variable for testing, like: ``filterIt("abcxyz", it == 'x')``.
-    var result: type(seq1) = @[]
+  ## Example:
-    for it in items(seq1):
+  ##
-      if pred: result.add(it)
+  ## .. code-block:: nimrod
-    result
+  ##    let
+  ##      temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44]
+  ##      acceptable = filterIt(temperatures, it < 50 and it > -10)
+  ##    assert acceptable == @[-2.0, 24.5, 44.31]
+  var result {.gensym.}: type(seq1) = @[]
+  for it in items(seq1):
+    if pred: result.add(it)
+  result
 
+template toSeq*(iter: expr): expr {.immediate.} =
+  ## Transforms any iterator into a sequence.
+  ##
+  ## Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   let
+  ##     numeric = @[1, 2, 3, 4, 5, 6, 7, 8, 9]
+  ##     odd_numbers = toSeq(filter(numeric) do (x: int) -> bool:
+  ##       if x mod 2 == 1:
+  ##         result = true)
+  ##   assert odd_numbers == @[1, 3, 5, 7, 9]
+  ##
+  var result {.gensym.}: seq[type(iter)] = @[]
+  for x in iter: add(result, x)
+  result
+
+when isMainModule:
+  import strutils
+  proc toStr(x: int): string {.procvar.} = $x
+  # concat test
+  let
+    s1 = @[1, 2, 3]
+    s2 = @[4, 5]
+    s3 = @[6, 7]
+    total = concat(s1, s2, s3)
+  assert total == @[1, 2, 3, 4, 5, 6, 7]
+
+  # duplicates test
+  let
+    dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4]
+    dup2 = @["a", "a", "c", "d", "d"]
+    unique1 = distnct(dup1)
+    unique2 = distnct(dup2)
+  assert unique1 == @[1, 3, 4, 2, 8]
+  assert unique2 == @["a", "c", "d"]
+
+  # zip test
+  let
+    short = @[1, 2, 3]
+    long = @[6, 5, 4, 3, 2, 1]
+    words = @["one", "two", "three"]
+    zip1 = zip(short, long)
+    zip2 = zip(short, words)
+  assert zip1 == @[(1, 6), (2, 5), (3, 4)]
+  assert zip2 == @[(1, "one"), (2, "two"), (3, "three")]
+  assert zip1[2].b == 4
+  assert zip2[2].b == "three"
+
+  # filter proc test
+  let
+    colors = @["red", "yellow", "black"]
+    f1 = filter(colors, proc(x: string): bool = x.len < 6)
+    f2 = filter(colors) do (x: string) -> bool : x.len > 5
+  assert f1 == @["red", "black"]
+  assert f2 == @["yellow"]
+
+  # filter iterator test
+  let numbers = @[1, 4, 5, 8, 9, 7, 4]
+  for n in filter(numbers, proc (x: int): bool = x mod 2 == 0):
+    echo($n)
+  # echoes 4, 8, 4 in separate lines
+
+  # filterIt test
+  let
+    temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44]
+    acceptable = filterIt(temperatures, it < 50 and it > -10)
+  assert acceptable == @[-2.0, 24.5, 44.31]
+
+  # toSeq test
+  let
+    numeric = @[1, 2, 3, 4, 5, 6, 7, 8, 9]
+    odd_numbers = toSeq(filter(numeric) do (x: int) -> bool:
+      if x mod 2 == 1:
+        result = true)
+  assert odd_numbers == @[1, 3, 5, 7, 9]
+
+  echo "Finished doc tests"

lib/pure/collections/tables.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -308,6 +308,52 @@ proc `$`*[A, B](t: TOrderedTable[A, B]): string =
   ## The `$` operator for ordered hash tables.
   dollarImpl()
 
+proc sort*[A, B](t: var TOrderedTable[A, B], 
+                 cmp: proc (x,y: tuple[key: A, val: B]): int) =
+  ## sorts `t` according to `cmp`. This modifies the internal list
+  ## that kept the insertion order, so insertion order is lost after this
+  ## call but key lookup and insertions remain possible after `sort` (in
+  ## contrast to the `sort` for count tables).
+  var list = t.first
+  var
+    p, q, e, tail, oldhead: int
+    nmerges, psize, qsize, i: int
+  if t.counter == 0: return
+  var insize = 1
+  while true:
+    p = list; oldhead = list
+    list = -1; tail = -1; nmerges = 0
+    while p >= 0:
+      inc(nmerges)
+      q = p
+      psize = 0
+      i = 0
+      while i < insize:
+        inc(psize)
+        q = t.data[q].next
+        if q < 0: break 
+        inc(i)
+      qsize = insize
+      while psize > 0 or (qsize > 0 and q >= 0):
+        if psize == 0:
+          e = q; q = t.data[q].next; dec(qsize)
+        elif qsize == 0 or q < 0:
+          e = p; p = t.data[p].next; dec(psize)
+        elif cmp((t.data[p].key, t.data[p].val), 
+                 (t.data[q].key, t.data[q].val)) <= 0:
+          e = p; p = t.data[p].next; dec(psize)
+        else:
+          e = q; q = t.data[q].next; dec(qsize)
+        if tail >= 0: t.data[tail].next = e
+        else: list = e
+        tail = e
+      p = q
+    t.data[tail].next = -1
+    if nmerges <= 1: break
+    insize = insize * 2
+  t.first = list
+  t.last = tail
+
 # ------------------------------ count tables -------------------------------
 
 type

lib/pure/dynlib.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2009 Andreas Rumpf
+#        (c) Copyright 2012 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -18,6 +18,10 @@ proc LoadLib*(path: string): TLibHandle
   ## loads a library from `path`. Returns nil if the library could not 
   ## be loaded.
 
+proc LoadLib*(): TLibHandle
+  ## gets the handle from the current executable. Returns nil if the 
+  ## library could not be loaded.
+
 proc UnloadLib*(lib: TLibHandle)
   ## unloads the library `lib`
 
@@ -57,6 +61,7 @@ when defined(posix):
       importc, header: "<dlfcn.h>".}
 
   proc LoadLib(path: string): TLibHandle = return dlopen(path, RTLD_NOW)
+  proc LoadLib(): TLibHandle = return dlopen(nil, RTLD_NOW)
   proc UnloadLib(lib: TLibHandle) = dlclose(lib)
   proc symAddr(lib: TLibHandle, name: cstring): pointer = 
     return dlsym(lib, name)
@@ -78,6 +83,8 @@ elif defined(windows) or defined(dos):
 
   proc LoadLib(path: string): TLibHandle =
     result = cast[TLibHandle](winLoadLibrary(path))
+  proc LoadLib(): TLibHandle =
+    result = cast[TLibHandle](winLoadLibrary(nil))
   proc UnloadLib(lib: TLibHandle) = FreeLibrary(cast[THINSTANCE](lib))
 
   proc symAddr(lib: TLibHandle, name: cstring): pointer =

lib/pure/encodings.nim

@@ -254,6 +254,8 @@ when defined(windows):
 else:
   when defined(haiku):
     const iconvDll = "(libc.so.6|libiconv.so|libtextencoding.so)"
+  elif defined(macosx):
+    const iconvDll = "libiconv.dylib"
   else:
     const iconvDll = "(libc.so.6|libiconv.so)"
 
@@ -449,8 +451,13 @@ proc convert*(s: string, destEncoding = "UTF-8",
     close(c)
 
 when IsMainModule:
-  var orig = "öäüß"
+  let
-  var crap = convert(orig, "CP1252", "UTF-8")
+    orig = "öäüß"
-  echo convert(crap, "ibm850", "CP1252")
+    cp1252 = convert(orig, "CP1252", "UTF-8")
-  echo getCurrentEncoding()
+    ibm850 = convert(cp1252, "ibm850", "CP1252")
+    current = getCurrentEncoding()
+  echo "Original string from source code: ", orig
+  echo "Forced ibm850 encoding: ", ibm850
+  echo "Current encoding: ", current
+  echo "From ibm850 to current: ", convert(ibm850, current, "ibm850")
 

lib/pure/fsmonitor.nim

@@ -17,9 +17,7 @@
 ## module will therefore not work with any Linux kernel prior to that, unless
 ## it has been patched to support inotify.
 
-when defined(windows):
+when defined(linux) or defined(nimdoc):
-  {.error: "Windows is not yet supported by this module.".}
-elif defined(linux):
   from posix import read
 else:
   {.error: "Your platform is not supported.".}

lib/pure/ftpclient.nim

@@ -26,6 +26,10 @@ import sockets, strutils, parseutils, times, os, asyncio
 ##    var ftp = FTPClient("example.org", user = "user", pass = "pass")
 ##    ftp.connect()
 ##    ftp.retrFile("file.ext", "file.ext")
+##
+## **Warning:** The API of this module is unstable, and therefore is subject
+## to change.
+
 
 type
   TFTPClient* = object of TObject
@@ -92,8 +96,9 @@ type
   EFTP* = object of ESynch
 
 proc FTPClient*(address: string, port = TPort(21),
-                user, pass = ""): TFTPClient =
+                user, pass = ""): PFTPClient =
-  ## Create a ``TFTPClient`` object.
+  ## Create a ``PFTPClient`` object.
+  new(result)
   result.user = user
   result.pass = pass
   result.address = address
@@ -109,7 +114,7 @@ proc getDSock(ftp: PFTPClient): TSocket =
 proc getCSock(ftp: PFTPClient): TSocket =
   if ftp.isAsync: return ftp.asyncCSock else: return ftp.csock
 
-template blockingOperation(sock: TSocket, body: stmt) =
+template blockingOperation(sock: TSocket, body: stmt) {.immediate.} =
   if ftp.isAsync:
     sock.setBlocking(true)
   body
@@ -274,11 +279,20 @@ proc getLines(ftp: PFTPClient, async: bool = false): bool =
   ## It doesn't if `async` is true, because it doesn't check for 226 then.
   if ftp.dsockConnected:
     var r = TaintedString""
-    if getDSock(ftp).recvAsync(r):
+    if ftp.isAsync:
-      if r.string != "":
+      if ftp.asyncDSock.recvLine(r):
-        ftp.job.lines.add(r.string)
+        if r.string == "":
-      else:
+          ftp.dsockConnected = false
-        ftp.dsockConnected = False
+        else:
+          ftp.job.lines.add(r.string & "\n")
+    else:
+      assert(not async)
+      if ftp.dsock.recvLine(r):
+        if r.string == "":
+          ftp.dsockConnected = false
+        else:
+          ftp.job.lines.add(r.string & "\n")
+      else: OSError()
   
   if not async:
     var readSocks: seq[TSocket] = @[ftp.getCSock()]
@@ -385,7 +399,7 @@ proc list*(ftp: PFTPClient, dir: string = "", async = false): string =
 proc retrText*(ftp: PFTPClient, file: string, async = false): string =
   ## Retrieves ``file``. File must be ASCII text.
   ## If ``async`` is true, this function will return immediately and
-  ## it will be your job to call ``poll`` to progress this operation.
+  ## it will be your job to call asyncio's ``poll`` to progress this operation.
   ftp.createJob(getLines, JRetrText)
   ftp.pasv()
   assertReply ftp.send("RETR " & file.normalizePathSep), ["125", "150"]
@@ -400,12 +414,14 @@ proc retrText*(ftp: PFTPClient, file: string, async = false): string =
 proc getFile(ftp: PFTPClient, async = false): bool =
   if ftp.dsockConnected:
     var r = "".TaintedString
+    var bytesRead = 0
     var returned = false
     if async:
       if not ftp.isAsync: raise newException(EFTP, "FTPClient must be async.")
-      returned = ftp.AsyncDSock.recvAsync(r)
+      bytesRead = ftp.AsyncDSock.recvAsync(r, BufferSize)
+      returned = bytesRead != -1
     else: 
-      r = getDSock(ftp).recv()
+      bytesRead = getDSock(ftp).recv(r, BufferSize)
       returned = true
     let r2 = r.string
     if r2 != "":
@@ -425,8 +441,9 @@ proc getFile(ftp: PFTPClient, async = false): bool =
 proc retrFile*(ftp: PFTPClient, file, dest: string, async = false) =
   ## Downloads ``file`` and saves it to ``dest``. Usage of this function
   ## asynchronously is recommended to view the progress of the download.
-  ## The ``EvRetr`` event is given by ``poll`` when the download is finished,
+  ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function 
-  ## and the ``filename`` field will be equal to ``file``.
+  ## when the download is finished, and the ``filename`` field will be equal
+  ## to ``file``.
   ftp.createJob(getFile, JRetr)
   ftp.job.file = open(dest, mode = fmWrite)
   ftp.pasv()
@@ -450,11 +467,13 @@ proc doUpload(ftp: PFTPClient, async = false): bool =
   if ftp.dsockConnected:
     if ftp.job.toStore.len() > 0:
       assert(async)
-      if ftp.asyncDSock.sendAsync(ftp.job.toStore):
+      let bytesSent = ftp.asyncDSock.sendAsync(ftp.job.toStore)
+      if bytesSent == ftp.job.toStore.len:
         ftp.job.toStore = ""
-        ftp.job.progress.inc(ftp.job.toStore.len)
+      elif bytesSent != ftp.job.toStore.len and bytesSent != 0:
-        ftp.job.oneSecond.inc(ftp.job.toStore.len)
+        ftp.job.toStore = ftp.job.toStore[bytesSent .. -1]
-      
+      ftp.job.progress.inc(bytesSent)
+      ftp.job.oneSecond.inc(bytesSent)
     else:
       var s = newStringOfCap(4000)
       var len = ftp.job.file.readBuffer(addr(s[0]), 4000)
@@ -472,8 +491,12 @@ proc doUpload(ftp: PFTPClient, async = false): bool =
       if not async:
         getDSock(ftp).send(s)
       else:
-        if not ftp.asyncDSock.sendAsync(s):
+        let bytesSent = ftp.asyncDSock.sendAsync(s)
-          ftp.job.toStore = s
+        if bytesSent == 0:
+          ftp.job.toStore.add(s)
+        elif bytesSent != s.len:
+          ftp.job.toStore.add(s[bytesSent .. -1])
+        len = bytesSent
       
       ftp.job.progress.inc(len)
       ftp.job.oneSecond.inc(len)
@@ -482,8 +505,9 @@ proc store*(ftp: PFTPClient, file, dest: string, async = false) =
   ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this
   ## function asynchronously is recommended to view the progress of
   ## the download.
-  ## The ``EvStore`` event is given by ``poll`` when the upload is finished,
+  ## The ``EvStore`` event is passed to the specified ``handleEvent`` function 
-  ## and the ``filename`` field will be equal to ``file``.
+  ## when the upload is finished, and the ``filename`` field will be 
+  ## equal to ``file``.
   ftp.createJob(doUpload, JStore)
   ftp.job.file = open(file)
   ftp.job.total = ftp.job.file.getFileSize()
@@ -508,16 +532,6 @@ proc close*(ftp: PFTPClient) =
     ftp.csock.close()
     ftp.dsock.close()
 
-discard """proc getSocket(h: PObject): tuple[info: TInfo, sock: TSocket] =
-  result = (SockIdle, InvalidSocket)
-  var ftp = PAsyncFTPClient(h)
-  if ftp.jobInProgress:
-    case ftp.job.typ
-    of JRetrText, JRetr, JStore:
-      if ftp.dsockStatus == SockConnecting or ftp.dsockStatus == SockConnected:
-        result = (ftp.dsockStatus, ftp.dsock)
-      else: result = (SockIdle, ftp.dsock)"""
-
 proc csockHandleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) =
   if ftp.jobInProgress:
     assertReply ftp.expectReply(), "226" # Make sure the transfer completed.
@@ -540,32 +554,6 @@ proc csockHandleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) =
     
     ftp.handleEvent(ftp, r)
 
-discard """proc handleConnect(h: PObject) =
-  var ftp = PAsyncFTPClient(h)
-  ftp.dsockStatus = SockConnected
-  assert(ftp.jobInProgress)
-  if ftp.job.typ == JStore:
-    ftp.dele.mode = MWriteable
-  else: 
-    ftp.dele.mode = MReadable"""
-
-discard """proc handleRead(h: PObject) =
-  var ftp = PAsyncFTPClient(h)
-  assert(ftp.jobInProgress)
-  assert(ftp.job.typ != JStore)
-  # This can never return true, because it shouldn't check for code 
-  # 226 from csock.
-  assert(not ftp.job.prc(ftp[], true))
-"""
-
-discard """proc csockGetSocket(h: PObject): tuple[info: TInfo, sock: TSocket] =
-  # This only returns the csock if a job is in progress. Otherwise handle read
-  # would capture data which is not for it to capture.
-  result = (SockIdle, InvalidSocket)
-  var ftp = PAsyncFTPClient(h)
-  if ftp.jobInProgress:
-    result = (SockConnected, ftp.csock)"""
-
 proc AsyncFTPClient*(address: string, port = TPort(21),
                      user, pass = "",
     handleEvent: proc (ftp: PAsyncFTPClient, ev: TFTPEvent) {.closure.} = 

lib/pure/htmlparser.nim

@@ -123,7 +123,7 @@ type
     tagVar         ## the HTML ``var`` element
 
 const
-  tagStrs = [
+  tagToStr* = [
     "a", "abbr", "acronym", "address", "applet", "area", 
     "b", "base", "basefont", "bdo", "big", "blockquote", "body", 
     "br", "button", "caption", "center", "cite", "code", 
@@ -243,13 +243,13 @@ proc binaryStrSearch(x: openarray[string], y: string): int =
 proc htmlTag*(n: PXmlNode): THtmlTag = 
   ## gets `n`'s tag as a ``THtmlTag``.
   if n.clientData == 0:
-    n.clientData = binaryStrSearch(tagStrs, n.tag)+1
+    n.clientData = binaryStrSearch(tagToStr, n.tag)+1
   result = THtmlTag(n.clientData)
 
 proc htmlTag*(s: string): THtmlTag =
   ## converts `s` to a ``THtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is
   ## returned.
-  result = THtmlTag(binaryStrSearch(tagStrs, s.toLower)+1)
+  result = THtmlTag(binaryStrSearch(tagToStr, s.toLower)+1)
 
 proc entityToUtf8*(entity: string): string = 
   ## converts an HTML entity name like ``Ü`` to its UTF-8 equivalent.

lib/pure/httpclient.nim

@@ -80,62 +80,43 @@ proc fileError(msg: string) =
   e.msg = msg
   raise e
 
-proc charAt(d: var string, i: var int, s: TSocket): char {.inline.} = 
-  result = d[i]
-  while result == '\0':
-    d = string(s.recv())
-    i = 0
-    result = d[i]
-
 proc parseChunks(s: TSocket): string =
-  # get chunks:
-  var i = 0
   result = ""
-  var d = s.recv().string
+  var ri = 0
   while true:
+    var chunkSizeStr = ""
     var chunkSize = 0
-    var digitFound = false
+    if s.recvLine(chunkSizeStr):
-    while true: 
+      var i = 0
-      case d[i]
+      if chunkSizeStr == "":
-      of '0'..'9': 
+        httpError("Server terminated connection prematurely")
-        digitFound = true
+      while true:
-        chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('0'))
+        case chunkSizeStr[i]
-      of 'a'..'f': 
+        of '0'..'9':
-        digitFound = true
+          chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0'))
-        chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('a') + 10)
+        of 'a'..'f':
-      of 'A'..'F': 
+          chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10)
-        digitFound = true
+        of 'A'..'F':
-        chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('A') + 10)
+          chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10)
-      of '\0': 
+        of '\0':
-        d = string(s.recv())
+          break
-        i = -1
+        of ';':
-      else: break
+          # http://tools.ietf.org/html/rfc2616#section-3.6.1
-      inc(i)
+          # We don't care about chunk-extensions.
-    if not digitFound: httpError("Chunksize expected")
+          break
+        else:
+          httpError("Invalid chunk size: " & chunkSizeStr)
+        inc(i)
     if chunkSize <= 0: break
-    while charAt(d, i, s) notin {'\C', '\L', '\0'}: inc(i)
+    result.setLen(ri+chunkSize)
-    if charAt(d, i, s) == '\C': inc(i)
+    var bytesRead = 0
-    if charAt(d, i, s) == '\L': inc(i)
+    while bytesRead != chunkSize:
-    else: httpError("CR-LF after chunksize expected")
+      let ret = recv(s, addr(result[ri]), chunkSize-bytesRead)
-    
+      ri += ret
-    var x = substr(d, i, i+chunkSize-1)
+      bytesRead += ret
-    var size = x.len
+    s.skip(2) # Skip \c\L
-    result.add(x)
+    # Trailer headers will only be sent if the request specifies that we want
-    inc(i, size)
+    # them: http://tools.ietf.org/html/rfc2616#section-3.6.1
-    if size < chunkSize:
-      # read in the rest:
-      var missing = chunkSize - size
-      var L = result.len
-      setLen(result, L + missing)    
-      while missing > 0:
-        var bytesRead = s.recv(addr(result[L]), missing)
-        inc(L, bytesRead)
-        dec(missing, bytesRead)
-      # next chunk:
-      d = string(s.recv())
-      i = 0
-    # skip trailing CR-LF:
-    while charAt(d, i, s) in {'\C', '\L'}: inc(i)
   
 proc parseBody(s: TSocket,
                headers: PStringTable): string =
@@ -238,12 +219,18 @@ type
     httpCONNECT       ## Converts the request connection to a transparent 
                       ## TCP/IP tunnel, usually used for proxies.
 
+when not defined(ssl):
+  type PSSLContext = ref object
+  let defaultSSLContext: PSSLContext = nil
+else:
+  let defaultSSLContext = newContext(verifyMode = CVerifyNone)
+
 proc request*(url: string, httpMethod = httpGET, extraHeaders = "", 
-              body = ""): TResponse =
+              body = "",
+              sslContext: PSSLContext = defaultSSLContext): TResponse =
   ## | Requests ``url`` with the specified ``httpMethod``.
   ## | Extra headers can be specified and must be seperated by ``\c\L``
   var r = parseUrl(url)
-  
   var headers = substr($httpMethod, len("http"))
   headers.add(" /" & r.path & r.query)
 
@@ -257,7 +244,7 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "",
   var port = TPort(80)
   if r.scheme == "https":
     when defined(ssl):
-      s.wrapSocket(verifyMode = CVerifyNone)
+      sslContext.wrapSocket(s)
     else:
       raise newException(EHttpRequestErr, "SSL support was not compiled in. Cannot connect over SSL.")
     port = TPort(443)
@@ -277,32 +264,32 @@ proc redirection(status: string): bool =
     if status.startsWith(i):
       return True
   
-proc get*(url: string, maxRedirects = 5): TResponse =
+proc get*(url: string, maxRedirects = 5, sslContext: PSSLContext = defaultSSLContext): TResponse =
-  ## | GET's the ``url`` and returns a ``TResponse`` object
+  ## | GETs the ``url`` and returns a ``TResponse`` object
   ## | This proc also handles redirection
   result = request(url)
   for i in 1..maxRedirects:
     if result.status.redirection():
       var locationHeader = result.headers["Location"]
       if locationHeader == "": httpError("location header expected")
-      result = request(locationHeader)
+      result = request(locationHeader, sslContext = sslContext)
       
-proc getContent*(url: string): string =
+proc getContent*(url: string, sslContext: PSSLContext = defaultSSLContext): string =
-  ## | GET's the body and returns it as a string.
+  ## | GETs the body and returns it as a string.
   ## | Raises exceptions for the status codes ``4xx`` and ``5xx``
-  var r = get(url)
+  var r = get(url, sslContext = sslContext)
   if r.status[0] in {'4','5'}:
     raise newException(EHTTPRequestErr, r.status)
   else:
     return r.body
   
 proc post*(url: string, extraHeaders = "", body = "", 
-           maxRedirects = 5): TResponse =
+           maxRedirects = 5, sslContext: PSSLContext = defaultSSLContext): TResponse =
-  ## | POST's ``body`` to the ``url`` and returns a ``TResponse`` object.
+  ## | POSTs ``body`` to the ``url`` and returns a ``TResponse`` object.
   ## | This proc adds the necessary Content-Length header.
   ## | This proc also handles redirection.
   var xh = extraHeaders & "Content-Length: " & $len(body) & "\c\L"
-  result = request(url, httpPOST, xh, body)
+  result = request(url, httpPOST, xh, body, sslContext)
   for i in 1..maxRedirects:
     if result.status.redirection():
       var locationHeader = result.headers["Location"]
@@ -310,8 +297,9 @@ proc post*(url: string, extraHeaders = "", body = "",
       var meth = if result.status != "307": httpGet else: httpPost
       result = request(locationHeader, meth, xh, body)
   
-proc postContent*(url: string, extraHeaders = "", body = ""): string =
+proc postContent*(url: string, extraHeaders = "", body = "",
-  ## | POST's ``body`` to ``url`` and returns the response's body as a string
+                  sslContext: PSSLContext = defaultSSLContext): string =
+  ## | POSTs ``body`` to ``url`` and returns the response's body as a string
   ## | Raises exceptions for the status codes ``4xx`` and ``5xx``
   var r = post(url, extraHeaders, body)
   if r.status[0] in {'4','5'}:
@@ -319,11 +307,12 @@ proc postContent*(url: string, extraHeaders = "", body = ""): string =
   else:
     return r.body
   
-proc downloadFile*(url: string, outputFilename: string) =
+proc downloadFile*(url: string, outputFilename: string,
+                   sslContext: PSSLContext = defaultSSLContext) =
   ## Downloads ``url`` and saves it to ``outputFilename``
   var f: TFile
   if open(f, outputFilename, fmWrite):
-    f.write(getContent(url))
+    f.write(getContent(url, sslContext))
     f.close()
   else:
     fileError("Unable to open file")

lib/pure/httpserver.nim

@@ -22,6 +22,8 @@
 ##
 ##  run(handleRequest, TPort(80))
 ##
+## **Warning:** The API of this module is unstable, and therefore is subject
+## to change.
 
 import parseutils, strutils, os, osproc, strtabs, streams, sockets, asyncio
 

lib/pure/irc.nim

@@ -16,17 +16,23 @@
 ## the amount of lag.
 ##
 ## .. code-block:: Nimrod
-##   var client = irc("irc.server.net", joinChans = @["#channel"])
+##
+##   var client = irc("picheta.me", joinChans = @["#bots"])
 ##   client.connect()
 ##   while True:
 ##     var event: TIRCEvent
 ##     if client.poll(event):
 ##       case event.typ
-##       of EvDisconnected: break
+##       of EvConnected: nil
+##       of EvDisconnected:
+##         client.reconnect()
 ##       of EvMsg:
-##         # Where all the magic happens. 
+##         # Write your message reading code here.
+## 
+## **Warning:** The API of this module is unstable, and therefore is subject
+## to change.
 
-import sockets, strutils, parseutils, times, asyncio
+import sockets, strutils, parseutils, times, asyncio, os
 
 type
   TIRC* = object of TObject
@@ -34,11 +40,15 @@ type
     port: TPort
     nick, user, realname, serverPass: string
     case isAsync: bool
-    of false:
-      sock: TSocket
     of true:
-      handleEvent: proc (irc: var TAsyncIRC, ev: TIRCEvent) {.closure.}
+      handleEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure.}
       asyncSock: PAsyncSocket
+      myDispatcher: PDispatcher
+    of false:
+      dummyA: pointer
+      dummyB: pointer # workaround a Nimrod API issue
+      dummyC: pointer
+      sock: TSocket
     status: TInfo
     lastPing: float
     lastPong: float
@@ -46,6 +56,9 @@ type
     channelsToJoin: seq[string]
     msgLimit: bool
     messageBuffer: seq[tuple[timeToSend: float, m: string]]
+    lastReconnect: float
+
+  PIRC* = ref TIRC
 
   PAsyncIRC* = ref TAsyncIRC
   TAsyncIRC* = object of TIRC
@@ -68,9 +81,13 @@ type
     MError
   
   TIRCEventType* = enum
-    EvMsg, EvDisconnected
+    EvMsg, EvConnected, EvDisconnected
   TIRCEvent* = object ## IRC Event
     case typ*: TIRCEventType
+    of EvConnected:
+      ## Connected to server.
+      ## Only occurs with AsyncIRC.
+      nil
     of EvDisconnected: 
       ## Disconnected from the server
       nil
@@ -82,7 +99,7 @@ type
       origin*: string      ## The channel/user that this msg originated from
       raw*: string         ## Raw IRC message
   
-proc send*(irc: var TIRC, message: string, sendImmediately = false) =
+proc send*(irc: PIRC, message: string, sendImmediately = false) =
   ## Sends ``message`` as a raw command. It adds ``\c\L`` for you.
   var sendMsg = true
   if irc.msgLimit and not sendImmediately:
@@ -104,15 +121,15 @@ proc send*(irc: var TIRC, message: string, sendImmediately = false) =
       # but I can't exactly check for EBrokenPipe.
       irc.status = SockClosed
 
-proc privmsg*(irc: var TIRC, target, message: string) =
+proc privmsg*(irc: PIRC, target, message: string) =
   ## Sends ``message`` to ``target``. ``Target`` can be a channel, or a user.
   irc.send("PRIVMSG $1 :$2" % [target, message])
 
-proc notice*(irc: var TIRC, target, message: string) =
+proc notice*(irc: PIRC, target, message: string) =
   ## Sends ``notice`` to ``target``. ``Target`` can be a channel, or a user. 
   irc.send("NOTICE $1 :$2" % [target, message])
 
-proc join*(irc: var TIRC, channel: string, key = "") =
+proc join*(irc: PIRC, channel: string, key = "") =
   ## Joins ``channel``.
   ## 
   ## If key is not ``""``, then channel is assumed to be key protected and this
@@ -122,16 +139,19 @@ proc join*(irc: var TIRC, channel: string, key = "") =
   else:
     irc.send("JOIN " & channel & " " & key)
 
-proc part*(irc: var TIRC, channel, message: string) =
+proc part*(irc: PIRC, channel, message: string) =
   ## Leaves ``channel`` with ``message``.
   irc.send("PART " & channel & " :" & message)
 
-proc close*(irc: var TIRC) =
+proc close*(irc: PIRC) =
   ## Closes connection to an IRC server.
   ##
   ## **Warning:** This procedure does not send a ``QUIT`` message to the server.
   irc.status = SockClosed
-  irc.sock.close()
+  if irc.isAsync:
+    irc.asyncSock.close()
+  else:
+    irc.sock.close()
 
 proc isNumber(s: string): bool =
   ## Checks if `s` contains only numbers.
@@ -202,12 +222,11 @@ proc parseMessage(msg: string): TIRCEvent =
     inc(i) # Skip `:`.
     result.params.add(msg[i..msg.len-1])
 
-proc connect*(irc: var TIRC) =
+proc connect*(irc: PIRC) =
   ## Connects to an IRC server as specified by ``irc``.
   assert(irc.address != "")
   assert(irc.port != TPort(0))
   
-  irc.sock = socket()
   irc.sock.connect(irc.address, irc.port)
  
   irc.status = SockConnected
@@ -217,13 +236,28 @@ proc connect*(irc: var TIRC) =
   irc.send("NICK " & irc.nick, true)
   irc.send("USER $1 * 0 :$2" % [irc.user, irc.realname], true)
 
+proc reconnect*(irc: PIRC, timeout = 5000) =
+  ## Reconnects to an IRC server.
+  ##
+  ## ``Timeout`` specifies the time to wait in miliseconds between multiple
+  ## consecutive reconnections.
+  ##
+  ## This should be used when an ``EvDisconnected`` event occurs.
+  let secSinceReconnect = int(epochTime() - irc.lastReconnect)
+  if secSinceReconnect < timeout:
+    sleep(timeout - secSinceReconnect)
+  irc.sock = socket()
+  irc.connect()
+  irc.lastReconnect = epochTime()
+
 proc irc*(address: string, port: TPort = 6667.TPort,
          nick = "NimrodBot",
          user = "NimrodBot",
          realname = "NimrodBot", serverPass = "",
          joinChans: seq[string] = @[],
-         msgLimit: bool = true): TIRC =
+         msgLimit: bool = true): PIRC =
   ## Creates a ``TIRC`` object.
+  new(result)
   result.address = address
   result.port = port
   result.nick = nick
@@ -237,8 +271,9 @@ proc irc*(address: string, port: TPort = 6667.TPort,
   result.msgLimit = msgLimit
   result.messageBuffer = @[]
   result.status = SockIdle
+  result.sock = socket()
 
-proc processLine(irc: var TIRC, line: string): TIRCEvent =
+proc processLine(irc: PIRC, line: string): TIRCEvent =
   if line.len == 0:
     irc.close()
     result.typ = EvDisconnected
@@ -271,7 +306,7 @@ proc processLine(irc: var TIRC, line: string): TIRCEvent =
       if result.nick == irc.nick:
         irc.nick = result.params[0]
     
-proc processOther(irc: var TIRC, ev: var TIRCEvent): bool =
+proc processOther(irc: PIRC, ev: var TIRCEvent): bool =
   result = false
   if epochTime() - irc.lastPing >= 20.0:
     irc.lastPing = epochTime()
@@ -290,7 +325,7 @@ proc processOther(irc: var TIRC, ev: var TIRCEvent): bool =
       break # messageBuffer is guaranteed to be from the quickest to the
             # later-est.
 
-proc poll*(irc: var TIRC, ev: var TIRCEvent,
+proc poll*(irc: PIRC, ev: var TIRCEvent,
            timeout: int = 500): bool =
   ## This function parses a single message from the IRC server and returns 
   ## a TIRCEvent.
@@ -316,46 +351,32 @@ proc poll*(irc: var TIRC, ev: var TIRCEvent,
   
   if processOther(irc, ev): result = true
 
-proc getLag*(irc: var TIRC): float =
+proc getLag*(irc: PIRC): float =
   ## Returns the latency between this client and the IRC server in seconds.
   ## 
   ## If latency is unknown, returns -1.0.
   return irc.lag
 
-proc isConnected*(irc: var TIRC): bool =
+proc isConnected*(irc: PIRC): bool =
   ## Returns whether this IRC client is connected to an IRC server.
   return irc.status == SockConnected
 
-proc getNick*(irc: var TIRC): string =
+proc getNick*(irc: PIRC): string =
   ## Returns the current nickname of the client.
   return irc.nick
 
 # -- Asyncio dispatcher
 
-proc connect*(irc: PAsyncIRC) =
-  ## Equivalent of connect for ``TIRC`` but specifically created for asyncio.
-  assert(irc.address != "")
-  assert(irc.port != TPort(0))
-  
-  irc.asyncSock = AsyncSocket()
-  irc.asyncSock.connect(irc.address, irc.port)
-
 proc handleConnect(s: PAsyncSocket, irc: PAsyncIRC) =  
   # Greet the server :)
-  if irc.serverPass != "": irc[].send("PASS " & irc.serverPass, true)
+  if irc.serverPass != "": irc.send("PASS " & irc.serverPass, true)
-  irc[].send("NICK " & irc.nick, true)
+  irc.send("NICK " & irc.nick, true)
-  irc[].send("USER $1 * 0 :$2" % [irc.user, irc.realname], true)
+  irc.send("USER $1 * 0 :$2" % [irc.user, irc.realname], true)
-
-discard """proc handleConnect(h: PObject) =
-  var irc = PAsyncIRC(h)
-  
-  # Greet the server :)
-  if irc.serverPass != "": irc[].send("PASS " & irc.serverPass, true)
-  irc[].send("NICK " & irc.nick, true)
-  irc[].send("USER $1 * 0 :$2" % [irc.user, irc.realname], true)
-
   irc.status = SockConnected
-"""
+  
+  var ev: TIRCEvent
+  ev.typ = EvConnected
+  irc.handleEvent(irc, ev)
 
 proc handleRead(s: PAsyncSocket, irc: PAsyncIRC) =
   var line = "".TaintedString
@@ -363,42 +384,55 @@ proc handleRead(s: PAsyncSocket, irc: PAsyncIRC) =
   if ret:
     if line == "":
       var ev: TIRCEvent
-      irc[].close()
+      irc.close()
       ev.typ = EvDisconnected
-      irc[].handleEvent(irc[], ev)
+      irc.handleEvent(irc, ev)
     else:
-      var ev = irc[].processLine(line.string)
+      var ev = irc.processLine(line.string)
-      irc[].handleEvent(irc[], ev)
+      irc.handleEvent(irc, ev)
-
-discard """proc handleRead(h: PObject) =
-  var irc = PAsyncIRC(h)
-  var line = "".TaintedString
-  var ret = irc.sock.recvLineAsync(line)
-  case ret
-  of RecvFullLine:
-    var ev = irc[].processLine(irc.lineBuffer.string & line.string)
-    irc.handleEvent(irc[], ev, irc.userArg)
-    irc.lineBuffer = "".TaintedString
-  of RecvPartialLine:
-    if line.string != "":
-      string(irc.lineBuffer).add(line.string)
-  of RecvDisconnected:
-    var ev: TIRCEvent
-    irc[].close()
-    ev.typ = EvDisconnected
-    irc.handleEvent(irc[], ev, irc.userArg)
-  of RecvFail: nil"""
   
 proc handleTask(s: PAsyncSocket, irc: PAsyncIRC) =
   var ev: TIRCEvent
-  if irc[].processOther(ev):
+  if irc.processOther(ev):
-    irc.handleEvent(irc[], ev)
+    irc.handleEvent(irc, ev)
 
-discard """proc handleTask(h: PObject) =
+proc register*(d: PDispatcher, irc: PAsyncIRC) =
-  var irc = PAsyncIRC(h)
+  ## Registers ``irc`` with dispatcher ``d``.
-  var ev: TIRCEvent
+  irc.asyncSock.handleConnect =
-  if PAsyncIRC(h)[].processOther(ev):
+    proc (s: PAsyncSocket) =
-    irc.handleEvent(irc[], ev, irc.userArg)"""
+      handleConnect(s, irc)
+  irc.asyncSock.handleRead =
+    proc (s: PAsyncSocket) =
+      handleRead(s, irc)
+  irc.asyncSock.handleTask =
+    proc (s: PAsyncSocket) =
+      handleTask(s, irc)
+  d.register(irc.asyncSock)
+  irc.myDispatcher = d
+
+proc connect*(irc: PAsyncIRC) =
+  ## Equivalent of connect for ``TIRC`` but specifically created for asyncio.
+  assert(irc.address != "")
+  assert(irc.port != TPort(0))
+  
+  irc.asyncSock.connect(irc.address, irc.port)
+
+proc reconnect*(irc: PAsyncIRC, timeout = 5000) =
+  ## Reconnects to an IRC server.
+  ##
+  ## ``Timeout`` specifies the time to wait in miliseconds between multiple
+  ## consecutive reconnections.
+  ##
+  ## This should be used when an ``EvDisconnected`` event occurs.
+  ##
+  ## When successfully reconnected an ``EvConnected`` event will occur.
+  let secSinceReconnect = int(epochTime() - irc.lastReconnect)
+  if secSinceReconnect < timeout:
+    sleep(timeout - secSinceReconnect)
+  irc.asyncSock = AsyncSocket()
+  irc.myDispatcher.register(irc)
+  irc.connect()
+  irc.lastReconnect = epochTime()
 
 proc asyncIRC*(address: string, port: TPort = 6667.TPort,
               nick = "NimrodBot",
@@ -406,7 +440,7 @@ proc asyncIRC*(address: string, port: TPort = 6667.TPort,
               realname = "NimrodBot", serverPass = "",
               joinChans: seq[string] = @[],
               msgLimit: bool = true,
-              ircEvent: proc (irc: var TAsyncIRC, ev: TIRCEvent) {.closure.}
+              ircEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure.}
               ): PAsyncIRC =
   ## Use this function if you want to use asyncio's dispatcher.
   ## 
@@ -429,19 +463,7 @@ proc asyncIRC*(address: string, port: TPort = 6667.TPort,
   result.msgLimit = msgLimit
   result.messageBuffer = @[]
   result.handleEvent = ircEvent
-
+  result.asyncSock = AsyncSocket()
-proc register*(d: PDispatcher, irc: PAsyncIRC) =
-  ## Registers ``irc`` with dispatcher ``d``.
-  irc.asyncSock.handleConnect =
-    proc (s: PAsyncSocket) =
-      handleConnect(s, irc)
-  irc.asyncSock.handleRead =
-    proc (s: PAsyncSocket) =
-      handleRead(s, irc)
-  irc.asyncSock.handleTask =
-    proc (s: PAsyncSocket) =
-      handleTask(s, irc)
-  d.register(irc.asyncSock)
   
 when isMainModule:
   #var m = parseMessage("ERROR :Closing Link: dom96.co.cc (Ping timeout: 252 seconds)")
@@ -456,6 +478,8 @@ when isMainModule:
     var event: TIRCEvent
     if client.poll(event):
       case event.typ
+      of EvConnected:
+        nil
       of EvDisconnected:
         break
       of EvMsg:

lib/pure/json.nim

@@ -13,6 +13,23 @@
 ## (unlike XML). It is easy for machines to parse and generate.
 ## JSON is based on a subset of the JavaScript Programming Language,
 ## Standard ECMA-262 3rd Edition - December 1999.
+##
+## Usage example:
+##
+## .. code-block:: nimrod
+##  let
+##    small_json = """{"test": 1.3, "key2": true}"""
+##    jobj = parseJson(small_json)
+##  assert (jobj.kind == JObject)
+##  echo($jobj["test"].fnum)
+##  echo($jobj["key2"].bval)
+##
+## Results in:
+##
+## .. code-block:: nimrod
+##
+##   1.3000000000000000e+00
+##   true
 
 import 
   hashes, strutils, lexbase, streams, unicode
@@ -524,6 +541,11 @@ proc newJString*(s: String): PJsonNode =
   result.kind = JString
   result.str = s
 
+proc newJStringMove(s: String): PJsonNode =
+  new(result)
+  result.kind = JString
+  shallowCopy(result.str, s)
+
 proc newJInt*(n: biggestInt): PJsonNode =
   ## Creates a new `JInt PJsonNode`.
   new(result)
@@ -607,7 +629,7 @@ proc len*(n: PJsonNode): int =
   else: nil
 
 proc `[]`*(node: PJsonNode, name: String): PJsonNode =
-  ## Gets a field from a `JObject`.
+  ## Gets a field from a `JObject`. Returns nil if the key is not found.
   assert(node.kind == JObject)
   for key, item in items(node.fields):
     if key == name:
@@ -792,7 +814,9 @@ proc parseJson(p: var TJsonParser): PJsonNode =
   ## Parses JSON from a JSON Parser `p`.
   case p.tok
   of tkString:
-    result = newJString(p.a)
+    # we capture 'p.a' here, so we need to give it a fresh buffer afterwards:
+    result = newJStringMove(p.a)
+    p.a = ""
     discard getTok(p)
   of tkInt:
     result = newJInt(parseBiggestInt(p.a))
@@ -893,6 +917,10 @@ when isMainModule:
   echo(parsed["keyÄÖöoßß"])
   echo()
   echo(pretty(parsed2))
+  try:
+    echo(parsed["key2"][12123])
+    raise newException(EInvalidValue, "That line was expected to fail")
+  except EInvalidIndex: echo()
 
   discard """
   while true:

lib/pure/marshal.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.

lib/pure/math.nim

@@ -141,6 +141,11 @@ proc randomize*()
   ## number, i.e. a tickcount. Note: Does nothing for the ECMAScript target,
   ## as ECMAScript does not support this.
   
+proc randomize*(seed: int)
+  ## initializes the random number generator with a specific seed.
+  ## Note: Does nothing for the ECMAScript target,
+  ## as ECMAScript does not support this.
+
 when not defined(ECMAScript):
   proc sqrt*(x: float): float {.importc: "sqrt", header: "<math.h>".}
     ## computes the square root of `x`.
@@ -190,15 +195,17 @@ when not defined(ECMAScript):
   proc rand(): cint {.importc: "rand", nodecl.}
   
   when not defined(windows):
-    proc srand48(seed: cint) {.importc: "srand48", nodecl.}
+    proc srand48(seed: clong) {.importc: "srand48", nodecl.}
     proc drand48(): float {.importc: "drand48", nodecl.}
     proc random(max: float): float =
       result = drand48() * max
     
   proc randomize() =
-    let x = gettime(nil)
+    randomize(gettime(nil))
-    srand(x)
+
-    when defined(srand48): srand48(x)
+  proc randomize(seed: int) =
+    srand(cint(seed))
+    when defined(srand48): srand48(seed)
   proc random(max: int): int =
     result = int(rand()) mod max
 
@@ -217,6 +224,7 @@ else:
   proc random(max: float): float =
     result = float(mathrandom() * float(max))
   proc randomize() = nil
+  proc randomize(seed: int) = nil
   
   proc sqrt*(x: float): float {.importc: "Math.sqrt", nodecl.}
   proc ln*(x: float): float {.importc: "Math.log", nodecl.}
@@ -301,3 +309,18 @@ proc standardDeviation*(s: TRunningStat): float =
 
 {.pop.}
 {.pop.}
+
+when isMainModule and not defined(ECMAScript):
+  # Verifies random seed initialization.
+  let seed = gettime(nil)
+  randomize(seed)
+  const SIZE = 10
+  var buf : array[0..SIZE, int]
+  # Fill the buffer with random values
+  for i in 0..SIZE-1:
+    buf[i] = random(high(int))
+  # Check that the second random calls are the same for each position.
+  randomize(seed)
+  for i in 0..SIZE-1:
+    assert buf[i] == random(high(int)), "non deterministic random seeding"
+  echo "random values equal after reseeding"

lib/pure/os.nim

@@ -554,7 +554,8 @@ proc splitFile*(path: string): tuple[dir, name, ext: string] {.
     var dotPos = path.len
     for i in countdown(len(path)-1, 0):
       if path[i] == ExtSep:
-        if dotPos == path.len and i > 0: dotPos = i
+        if dotPos == path.len and i > 0 and 
+            path[i-1] notin {dirsep, altsep}: dotPos = i
       elif path[i] in {dirsep, altsep}:
         sepPos = i
         break
@@ -1448,7 +1449,7 @@ proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [FTime].} =
   else:
     var a, b: Ttimespec
     a.tv_sec = TTime(milsecs div 1000)
-    a.tv_nsec = (milsecs mod 1000) * 1000
+    a.tv_nsec = (milsecs mod 1000) * 1000 * 1000
     discard posix.nanosleep(a, b)
 
 proc getFileSize*(file: string): biggestInt {.rtl, extern: "nos$1",

lib/pure/osproc.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -201,9 +201,9 @@ proc execProcesses*(cmds: openArray[string],
             q[r] = startCmd(cmds[i], options=options)
             inc(i)
             if i > high(cmds): break
-    for i in 0..m-1:
+    for j in 0..m-1:
-      if q[i] != nil: close(q[i])
+      if q[j] != nil: close(q[j])
-      result = max(waitForExit(q[i]), result)
+      result = max(waitForExit(q[j]), result)
   else:
     for i in 0..high(cmds):
       var p = startCmd(cmds[i], options=options)

lib/pure/redis.nim

@@ -49,26 +49,35 @@ proc raiseNoOK(status: string) =
     raise newException(EInvalidReply, "Expected \"OK\" got \"$1\"" % status)
 
 proc parseStatus(r: TRedis): TRedisStatus =
-  var line = r.socket.recv.string
+  var line = ""
+  if r.socket.recvLine(line):
+    if line == "":
+      raise newException(ERedis, "Server closed connection prematurely")
   
-  if line[0] == '-':
+    if line[0] == '-':
-    raise newException(ERedis, strip(line))
+      raise newException(ERedis, strip(line))
-  if line[0] != '+':
+    if line[0] != '+':
-    raiseInvalidReply('+', line[0])
+      raiseInvalidReply('+', line[0])
     
-  return line.substr(1, line.len-3) # Strip '+' and \c\L.
+    return line.substr(1) # Strip '+'
+  else:
+    OSError()
   
 proc parseInteger(r: TRedis): TRedisInteger =
-  var line = r.socket.recv.string
+  var line = ""
+  if r.socket.recvLine(line):
+    if line == "":
+      raise newException(ERedis, "Server closed connection prematurely")
 
-  if line[0] == '-':
+    if line[0] == '-':
-    raise newException(ERedis, strip(line))
+      raise newException(ERedis, strip(line))
-  if line[0] != ':':
+    if line[0] != ':':
-    raiseInvalidReply(':', line[0])
+      raiseInvalidReply(':', line[0])
     
-  # Strip ':' and \c\L.
+    # Strip ':'
-  if parseBiggestInt(line, result, 1) == 0:
+    if parseBiggestInt(line, result, 1) == 0:
-    raise newException(EInvalidReply, "Unable to parse integer.") 
+      raise newException(EInvalidReply, "Unable to parse integer.") 
+  else: OSError()
 
 proc recv(sock: TSocket, size: int): TaintedString =
   result = newString(size).TaintedString
@@ -838,8 +847,11 @@ proc save*(r: TRedis) =
 proc shutdown*(r: TRedis) =
   ## Synchronously save the dataset to disk and then shut down the server
   r.sendCommand("SHUTDOWN")
-  var s = r.socket.recv()
+  var s = "".TaintedString
-  if s.string.len != 0: raise newException(ERedis, s.string)
+  if r.socket.recvLine(s):
+    if s.string.len != 0: raise newException(ERedis, s.string)
+  else:
+    OSError()
 
 proc slaveof*(r: TRedis, host: string, port: string) =
   ## Make the server a slave of another instance, or promote it as master

lib/pure/scgi.nim

@@ -23,6 +23,8 @@
 ##
 ##    run(handleRequest)
 ##
+## **Warning:** The API of this module is unstable, and therefore is subject
+## to change.
 
 import sockets, strutils, os, strtabs, asyncio
 

lib/pure/sockets.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -14,6 +14,8 @@
 ## For OpenSSL support compile with ``-d:ssl``. When using SSL be aware that
 ## most functions will then raise ``ESSL`` on SSL errors.
 
+{.deadCodeElim: on.}
+
 when hostos == "solaris":
   {.passl: "-lsocket -lnsl".}
 
@@ -45,12 +47,15 @@ when defined(ssl):
     TSSLAcceptResult* = enum
       AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess
 
+const
+  BufferSize*: int = 4000 ## size of a buffered socket's buffer
+
 type
   TSocketImpl = object ## socket type
     fd: cint
     case isBuffered: bool # determines whether this socket is buffered.
     of true:
-      buffer: array[0..4000, char]
+      buffer: array[0..BufferSize, char]
       currPos: int # current index in buffer
       bufLen: int # current length of buffer
     of false: nil
@@ -60,6 +65,8 @@ type
         sslHandle: PSSL
         sslContext: PSSLContext
         sslNoHandshake: bool # True if needs handshake.
+        sslHasPeekChar: bool
+        sslPeekChar: char
       of false: nil
   
   TSocket* = ref TSocketImpl
@@ -256,7 +263,10 @@ when defined(ssl):
     of protSSLv23:
       newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support.
     of protSSLv2:
-      newCTX = SSL_CTX_new(SSLv2_method())
+      when not defined(linux):
+        newCTX = SSL_CTX_new(SSLv2_method())
+      else:
+        SSLError()
     of protSSLv3:
       newCTX = SSL_CTX_new(SSLv3_method())
     of protTLSv1:
@@ -286,12 +296,55 @@ when defined(ssl):
     socket.sslContext = ctx
     socket.sslHandle = SSLNew(PSSLCTX(socket.sslContext))
     socket.sslNoHandshake = false
+    socket.sslHasPeekChar = false
     if socket.sslHandle == nil:
       SSLError()
     
     if SSLSetFd(socket.sslHandle, socket.fd) != 1:
       SSLError()
 
+proc SocketError*(socket: TSocket, err: int = -1, async = false) =
+  ## Raises proper errors based on return values of ``recv`` functions.
+  ##
+  ## If ``async`` is ``True`` no error will be thrown in the case when the
+  ## error was caused by no data being available to be read.
+  ##
+  ## If ``err`` is not lower than 0 no exception will be raised.
+  when defined(ssl):
+    if socket.isSSL:
+      if err <= 0:
+        var ret = SSLGetError(socket.sslHandle, err.cint)
+        case ret
+        of SSL_ERROR_ZERO_RETURN:
+          SSLError("TLS/SSL connection failed to initiate, socket closed prematurely.")
+        of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT:
+          if async:
+            return
+          else: SSLError("Not enough data on socket.")
+        of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ:
+          if async:
+            return
+          else: SSLError("Not enough data on socket.")
+        of SSL_ERROR_WANT_X509_LOOKUP:
+          SSLError("Function for x509 lookup has been called.")
+        of SSL_ERROR_SYSCALL, SSL_ERROR_SSL:
+          SSLError()
+        else: SSLError("Unknown Error")
+  
+  if err == -1 and not (when defined(ssl): socket.isSSL else: false):
+    if async:
+      when defined(windows):
+        # TODO: Test on Windows
+        var err = WSAGetLastError()
+        if err == WSAEWOULDBLOCK:
+          return
+        else: OSError()
+      else:
+        if errno == EAGAIN or errno == EWOULDBLOCK:
+          return
+        else: OSError()
+    else: OSError()
+
 proc listen*(socket: TSocket, backlog = SOMAXCONN) {.tags: [FReadIO].} =
   ## Marks ``socket`` as accepting connections. 
   ## ``Backlog`` specifies the maximum length of the 
@@ -844,11 +897,8 @@ proc checkBuffer(readfds: var seq[TSocket]): int =
   var res: seq[TSocket] = @[]
   result = 0
   for s in readfds:
-    if s.isBuffered:
+    if hasDataBuffered(s):
-      if s.bufLen <= 0 or s.currPos == s.bufLen:
+      inc(result)
-        res.add(s)
-      else:
-        inc(result)
     else:
       res.add(s)
   readfds = res
@@ -970,47 +1020,76 @@ template retRead(flags, readBytes: int) =
 
 proc recv*(socket: TSocket, data: pointer, size: int): int {.tags: [FReadIO].} =
   ## receives data from a socket
+  if size == 0: return
   if socket.isBuffered:
     if socket.bufLen == 0:
       retRead(0'i32, 0)
     
-    when true:
+    var read = 0
-      var read = 0
+    while read < size:
-      while read < size:
+      if socket.currPos >= socket.bufLen:
-        if socket.currPos >= socket.bufLen:
+        retRead(0'i32, read)
-          retRead(0'i32, read)
     
-        let chunk = min(socket.bufLen-socket.currPos, size-read)
+      let chunk = min(socket.bufLen-socket.currPos, size-read)
-        var d = cast[cstring](data)
+      var d = cast[cstring](data)
-        copyMem(addr(d[read]), addr(socket.buffer[socket.currPos]), chunk)
+      copyMem(addr(d[read]), addr(socket.buffer[socket.currPos]), chunk)
-        read.inc(chunk)
+      read.inc(chunk)
-        socket.currPos.inc(chunk)
+      socket.currPos.inc(chunk)
-    else:
-      var read = 0
-      while read < size:
-        if socket.currPos >= socket.bufLen:
-          retRead(0'i32, read)
-      
-        var d = cast[cstring](data)
-        d[read] = socket.buffer[socket.currPos]
-        read.inc(1)
-        socket.currPos.inc(1)
 
     result = read
   else:
     when defined(ssl):
       if socket.isSSL:
-        result = SSLRead(socket.sslHandle, data, size)
+        if socket.sslHasPeekChar:
+          copyMem(data, addr(socket.sslPeekChar), 1)
+          socket.sslHasPeekChar = false
+          if size-1 > 0:
+            var d = cast[cstring](data)
+            result = SSLRead(socket.sslHandle, addr(d[1]), size-1) + 1
+          else:
+            result = 1
+        else:
+          result = SSLRead(socket.sslHandle, data, size)
       else:
         result = recv(socket.fd, data, size.cint, 0'i32)
     else:
       result = recv(socket.fd, data, size.cint, 0'i32)
 
+proc recv*(socket: TSocket, data: var string, size: int): int =
+  ## higher-level version of the above
+  ##
+  ## When 0 is returned the socket's connection has been closed.
+  ##
+  ## This function will throw an EOS exception when an error occurs. A value
+  ## lower than 0 is never returned.
+  ##
+  ## **Note**: ``data`` must be initialised.
+  data.setLen(size)
+  result = recv(socket, cstring(data), size)
+  if result < 0:
+    data.setLen(0)
+    socket.SocketError(result)
+  data.setLen(result)
+
+proc recvAsync*(socket: TSocket, data: var string, size: int): int =
+  ## Async version of the above.
+  ##
+  ## When socket is non-blocking and no data is available on the socket,
+  ## ``-1`` will be returned and ``data`` will be ``""``.
+  ##
+  ## **Note**: ``data`` must be initialised.
+  data.setLen(size)
+  result = recv(socket, cstring(data), size)
+  if result < 0:
+    data.setLen(0)
+    socket.SocketError(async = true)
+    result = -1
+  data.setLen(result)
+
 proc waitFor(socket: TSocket, waited: var float, timeout: int): int {.
   tags: [FTime].} =
   ## returns the number of characters available to be read. In unbuffered
-  ## sockets this is always 1, otherwise this may as big as the buffer, currently
+  ## sockets this is always 1, otherwise this may as big as ``BufferSize``.
-  ## 4000.
   result = 1
   if socket.isBuffered and socket.bufLen != 0 and socket.bufLen != socket.currPos:
     result = socket.bufLen - socket.currPos
@@ -1040,6 +1119,18 @@ proc recv*(socket: TSocket, data: pointer, size: int, timeout: int): int {.
   
   result = read
 
+proc recv*(socket: TSocket, data: var string, size: int, timeout: int): int =
+  ## higher-level version of the above.
+  ##
+  ## Similar to the non-timeout version this will throw an EOS exception
+  ## when an error occurs.
+  data.setLen(size)
+  result = recv(socket, cstring(data), size, timeout)
+  if result < 0:
+    data.setLen(0)
+    socket.SocketError()
+  data.setLen(result)
+
 proc peekChar(socket: TSocket, c: var char): int {.tags: [FReadIO].} =
   if socket.isBuffered:
     result = 1
@@ -1052,8 +1143,12 @@ proc peekChar(socket: TSocket, c: var char): int {.tags: [FReadIO].} =
   else:
     when defined(ssl):
       if socket.isSSL:
-        raise newException(ESSL, "Sorry, you cannot use recvLine on an unbuffered SSL socket.")
+        if not socket.sslHasPeekChar:
+          result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1)
+          socket.sslHasPeekChar = true
         
+        c = socket.sslPeekChar
+        return
     result = recv(socket.fd, addr(c), 1, MSG_PEEK)
 
 proc recvLine*(socket: TSocket, line: var TaintedString): bool {.
@@ -1062,14 +1157,12 @@ proc recvLine*(socket: TSocket, line: var TaintedString): bool {.
   ## added to ``line``, however if solely ``\r\L`` is received then ``line``
   ## will be set to it.
   ## 
-  ## ``True`` is returned if data is available. ``False`` usually suggests an
+  ## ``True`` is returned if data is available. ``False`` suggests an
-  ## error, EOS exceptions are not raised in favour of this.
+  ## error, EOS exceptions are not raised and ``False`` is simply returned
+  ## instead.
   ## 
   ## If the socket is disconnected, ``line`` will be set to ``""`` and ``True``
   ## will be returned.
-  ##
-  ## **Warning:** Using this function on a unbuffered ssl socket will result
-  ## in an error.
   template addNLIfEmpty(): stmt =
     if line.len == 0:
       line.add("\c\L")
@@ -1096,6 +1189,8 @@ proc recvLine*(socket: TSocket, line: var TaintedString, timeout: int): bool {.
   tags: [FReadIO, FTime].} =
   ## variant with a ``timeout`` parameter, the timeout parameter specifies
   ## how many miliseconds to wait for data.
+  ##
+  ## ``ETimeout`` will be raised if ``timeout`` is exceeded.
   template addNLIfEmpty(): stmt =
     if line.len == 0:
       line.add("\c\L")
@@ -1148,11 +1243,13 @@ proc recvLineAsync*(socket: TSocket,
     elif c == '\L': return RecvFullLine
     add(line.string, c)
 
-proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO].} =
+proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO], deprecated.} =
   ## receives all the available data from the socket.
   ## Socket errors will result in an ``EOS`` error.
   ## If socket is not a connectionless socket and socket is not connected
   ## ``""`` will be returned.
+  ##
+  ## **Deprecated since version 0.9.2**: This function is not safe for use.
   const bufSize = 4000
   result = newStringOfCap(bufSize).TaintedString
   var pos = 0
@@ -1177,25 +1274,31 @@ proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO].} =
       add(result.string, buf)
       if bytesRead != bufSize-1: break
 
+{.push warning[deprecated]: off.}
 proc recvTimeout*(socket: TSocket, timeout: int): TaintedString {.
-  tags: [FReadIO].} =
+  tags: [FReadIO], deprecated.} =
   ## overloaded variant to support a ``timeout`` parameter, the ``timeout``
   ## parameter specifies the amount of miliseconds to wait for data on the
   ## socket.
+  ##
+  ## **Deprecated since version 0.9.2**: This function is not safe for use.
   if socket.bufLen == 0:
     var s = @[socket]
     if s.select(timeout) != 1:
       raise newException(ETimeout, "Call to recv() timed out.")
   
   return socket.recv
+{.pop.}
 
 proc recvAsync*(socket: TSocket, s: var TaintedString): bool {.
-  tags: [FReadIO].} =
+  tags: [FReadIO], deprecated.} =
   ## receives all the data from a non-blocking socket. If socket is non-blocking 
   ## and there are no messages available, `False` will be returned.
   ## Other socket errors will result in an ``EOS`` error.
   ## If socket is not a connectionless socket and socket is not connected
   ## ``s`` will be set to ``""``.
+  ##
+  ## **Deprecated since version 0.9.2**: This function is not safe for use.
   const bufSize = 1000
   # ensure bufSize capacity:
   setLen(s.string, bufSize)
@@ -1282,13 +1385,25 @@ proc recvFromAsync*(socket: TSocket, data: var String, length: int,
         return False
       else: OSError()
 
-proc skip*(socket: TSocket) {.tags: [FReadIO].} =
+proc skip*(socket: TSocket) {.tags: [FReadIO], deprecated.} =
   ## skips all the data that is pending for the socket
+  ##
+  ## **Deprecated since version 0.9.2**: This function is not safe for use.
   const bufSize = 1000
   var buf = alloc(bufSize)
   while recv(socket, buf, bufSize) == bufSize: nil
   dealloc(buf)
 
+proc skip*(socket: TSocket, size: int) =
+  ## Skips ``size`` amount of bytes.
+  ##
+  ## Returns the number of skipped bytes.
+  var dummy = alloc(size)
+  var bytesSkipped = 0
+  while bytesSkipped != size:
+    bytesSkipped += recv(socket, dummy, size-bytesSkipped)
+  dealloc(dummy)
+
 proc send*(socket: TSocket, data: pointer, size: int): int {.
   tags: [FWriteIO].} =
   ## sends data to a socket.
@@ -1312,21 +1427,26 @@ proc send*(socket: TSocket, data: string) {.tags: [FWriteIO].} =
     
     OSError()
 
-proc sendAsync*(socket: TSocket, data: string): bool {.tags: [FWriteIO].} =
+proc sendAsync*(socket: TSocket, data: string): int {.tags: [FWriteIO].} =
-  ## sends data to a non-blocking socket. Returns whether ``data`` was sent.
+  ## sends data to a non-blocking socket.
-  result = true
+  ## Returns ``0`` if no data could be sent, if data has been sent
-  var bytesSent = send(socket, cstring(data), data.len)
+  ## returns the amount of bytes of ``data`` that was successfully sent. This
+  ## number may not always be the length of ``data`` but typically is.
+  ##
+  ## An EOS (or ESSL if socket is an SSL socket) exception is raised if an error
+  ## occurs.
+  result = send(socket, cstring(data), data.len)
   when defined(ssl):
     if socket.isSSL:
-      if bytesSent <= 0:
+      if result <= 0:
-          let ret = SSLGetError(socket.sslHandle, bytesSent.cint)
+          let ret = SSLGetError(socket.sslHandle, result.cint)
           case ret
           of SSL_ERROR_ZERO_RETURN:
             SSLError("TLS/SSL connection failed to initiate, socket closed prematurely.")
           of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT:
             SSLError("Unexpected error occured.") # This should just not happen.
           of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ:
-            return false
+            return 0
           of SSL_ERROR_WANT_X509_LOOKUP:
             SSLError("Function for x509 lookup has been called.")
           of SSL_ERROR_SYSCALL, SSL_ERROR_SSL:
@@ -1334,18 +1454,19 @@ proc sendAsync*(socket: TSocket, data: string): bool {.tags: [FWriteIO].} =
           else: SSLError("Unknown Error")
       else:
         return
-  if bytesSent == -1:
+  if result == -1:
     when defined(windows):
       var err = WSAGetLastError()
       # TODO: Test on windows.
       if err == WSAEINPROGRESS:
-        return false
+        return 0
       else: OSError()
     else:
       if errno == EAGAIN or errno == EWOULDBLOCK:
-        return false
+        return 0
       else: OSError()
   
+
 proc trySend*(socket: TSocket, data: string): bool {.tags: [FWriteIO].} =
   ## safe alternative to ``send``. Does not raise an EOS when an error occurs,
   ## and instead returns ``false`` on failure.
@@ -1386,7 +1507,6 @@ proc sendTo*(socket: TSocket, address: string, port: TPort,
 
 when defined(Windows):
   const
-    SOCKET_ERROR = -1
     IOCPARM_MASK = 127
     IOC_IN = int(-2147483648)
     FIONBIO = int(IOC_IN or ((sizeof(int) and IOCPARM_MASK) shl 16) or 
@@ -1399,7 +1519,7 @@ when defined(Windows):
 proc setBlocking(s: TSocket, blocking: bool) =
   when defined(Windows):
     var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking
-    if SOCKET_ERROR == ioctlsocket(TWinSocket(s.fd), FIONBIO, addr(mode)):
+    if ioctlsocket(TWinSocket(s.fd), FIONBIO, addr(mode)) == -1:
       OSError()
   else: # BSD sockets
     var x: int = fcntl(s.fd, F_GETFL, 0)

lib/pure/streams.nim

@@ -267,7 +267,8 @@ proc newFileStream*(f: TFile): PFileStream =
 
 proc newFileStream*(filename: string, mode: TFileMode): PFileStream = 
   ## creates a new stream from the file named `filename` with the mode `mode`.
-  ## If the file cannot be opened, nil is returned.
+  ## If the file cannot be opened, nil is returned. See the `system
+  ## <system.html>`_ module for a list of available TFileMode enums.
   var f: TFile
   if Open(f, filename, mode): result = newFileStream(f)
 

lib/pure/strtabs.nim

@@ -36,11 +36,23 @@ proc len*(t: PStringTable): int {.rtl, extern: "nst$1".} =
   result = t.counter
 
 iterator pairs*(t: PStringTable): tuple[key, value: string] =
-  ## iterates over any (key, value) pair in the table `t`.
+  ## iterates over every (key, value) pair in the table `t`.
   for h in 0..high(t.data):
     if not isNil(t.data[h].key):
       yield (t.data[h].key, t.data[h].val)
 
+iterator keys*(t: PStringTable): string =
+  ## iterates over every key in the table `t`.
+  for h in 0..high(t.data):
+    if not isNil(t.data[h].key):
+      yield t.data[h].key
+
+iterator values*(t: PStringTable): string =
+  ## iterates over every value in the table `t`.
+  for h in 0..high(t.data):
+    if not isNil(t.data[h].key):
+      yield t.data[h].val
+
 type
   TFormatFlag* = enum         ## flags for the `%` operator
     useEnvironment,           ## use environment variable if the ``$key``

lib/pure/strutils.nim

@@ -186,7 +186,24 @@ iterator split*(s: string, seps: set[char] = Whitespace): string =
   ##   for word in split(";;this;is;an;;example;;;", {';'}):
   ##     writeln(stdout, word)
   ##
-  ## produces the same output.
+  ## produces the same output. The code:
+  ##
+  ## .. code-block:: nimrod
+  ##   let date = "2012-11-20T22:08:08.398990"
+  ##   let separators = {' ', '-', ':', 'T'}
+  ##   for number in split(date, separators):
+  ##     writeln(stdout, number)
+  ##
+  ## Results in:
+  ##
+  ## .. code-block:: nimrod
+  ##   "2012"
+  ##   "11"
+  ##   "20"
+  ##   "22"
+  ##   "08"
+  ##   "08.398990"
+  ##
   var last = 0
   assert(not ('\0' in seps))
   while last < len(s):
@@ -833,7 +850,7 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
   for c in items(s):
     case c
     of '\0'..'\31', '\128'..'\255':
-      add(result, '\\')
+      add(result, "\\x")
       add(result, toHex(ord(c), 2))
     of '\\': add(result, "\\\\")
     of '\'': add(result, "\\'")
@@ -841,6 +858,44 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
     else: add(result, c)
   add(result, suffix)
 
+proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
+  rtl, extern: "nsuUnescape".} =
+  ## Unescapes a string `s`. This complements ``escape`` as it performs the
+  ## opposite operations.
+  ##
+  ## If `s` does not begin with ``prefix`` and end with ``suffix`` a EInvalidValue
+  ## exception will be raised.
+  result = newStringOfCap(s.len)
+  var i = 0
+  if s[0 .. prefix.len-1] != prefix:
+    raise newException(EInvalidValue,
+                       "String does not start with a prefix of: " & prefix)
+  i.inc()
+  while True:
+    if i == s.len-suffix.len: break
+    case s[i]
+    of '\\':
+      case s[i+1]:
+      of 'x':
+        let j = parseHexInt(s[i+2 .. i+3])
+        result.add(chr(j))
+        inc(i, 2)
+      of '\\':
+        result.add('\\')
+      of '\'':
+        result.add('\'')
+      of '\"':
+        result.add('\"')
+      else: result.add("\\" & s[i+1])
+      inc(i)
+    of '\0': break
+    else:
+      result.add(s[i])
+    i.inc()
+  if s[i .. -1] != suffix:
+    raise newException(EInvalidValue,
+                       "String does not end with a suffix of: " & suffix)
+
 proc validIdentifier*(s: string): bool {.noSideEffect,
   rtl, extern: "nsuValidIdentifier".} =
   ## returns true if `s` is a valid identifier. A valid identifier starts

lib/system.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -38,7 +38,8 @@ type
   char* {.magic: Char.} ## built-in 8 bit character type (unsigned)
   string* {.magic: String.} ## built-in string type
   cstring* {.magic: Cstring.} ## built-in cstring (*compatible string*) type
-  pointer* {.magic: Pointer.} ## built-in pointer type
+  pointer* {.magic: Pointer.} ## built-in pointer type, use the ``addr``
+                              ## operator to get a pointer to a variable
 
 const
   on* = true    ## alias for ``true``
@@ -117,6 +118,12 @@ proc new*(T: typedesc): ref T =
   ## reference to it as result value
   new(result)
 
+proc unsafeNew*[T](a: var ref T, size: int) {.magic: "New", noSideEffect.}
+  ## creates a new object of type ``T`` and returns a safe (traced)
+  ## reference to it in ``a``. This is **unsafe** as it allocates an object
+  ## of the passed ``size``. This should only be used for optimization
+  ## purposes when you know what you're doing!
+
 proc internalNew*[T](a: var ref T) {.magic: "New", noSideEffect.}
   ## leaked implementation detail. Do not use.
 
@@ -1060,86 +1067,87 @@ proc substr*(s: string, first, last: int): string {.
   ## is used instead: This means ``substr`` can also be used to `cut`:idx:
   ## or `limit`:idx: a string's length.
 
-proc zeroMem*(p: Pointer, size: int) {.importc, noDecl.}
+when not defined(nimrodVM):
-  ## overwrites the contents of the memory at ``p`` with the value 0.
+  proc zeroMem*(p: Pointer, size: int) {.importc, noDecl.}
-  ## Exactly ``size`` bytes will be overwritten. Like any procedure
+    ## overwrites the contents of the memory at ``p`` with the value 0.
-  ## dealing with raw memory this is *unsafe*.
+    ## Exactly ``size`` bytes will be overwritten. Like any procedure
+    ## dealing with raw memory this is *unsafe*.
 
-proc copyMem*(dest, source: Pointer, size: int) {.importc: "memcpy", noDecl.}
+  proc copyMem*(dest, source: Pointer, size: int) {.importc: "memcpy", noDecl.}
-  ## copies the contents from the memory at ``source`` to the memory
+    ## copies the contents from the memory at ``source`` to the memory
-  ## at ``dest``. Exactly ``size`` bytes will be copied. The memory
+    ## at ``dest``. Exactly ``size`` bytes will be copied. The memory
-  ## regions may not overlap. Like any procedure dealing with raw
+    ## regions may not overlap. Like any procedure dealing with raw
-  ## memory this is *unsafe*.
+    ## memory this is *unsafe*.
 
-proc moveMem*(dest, source: Pointer, size: int) {.importc: "memmove", noDecl.}
+  proc moveMem*(dest, source: Pointer, size: int) {.importc: "memmove", noDecl.}
-  ## copies the contents from the memory at ``source`` to the memory
+    ## copies the contents from the memory at ``source`` to the memory
-  ## at ``dest``. Exactly ``size`` bytes will be copied. The memory
+    ## at ``dest``. Exactly ``size`` bytes will be copied. The memory
-  ## regions may overlap, ``moveMem`` handles this case appropriately
+    ## regions may overlap, ``moveMem`` handles this case appropriately
-  ## and is thus somewhat more safe than ``copyMem``. Like any procedure
+    ## and is thus somewhat more safe than ``copyMem``. Like any procedure
-  ## dealing with raw memory this is still *unsafe*, though.
+    ## dealing with raw memory this is still *unsafe*, though.
 
-proc equalMem*(a, b: Pointer, size: int): bool {.
+  proc equalMem*(a, b: Pointer, size: int): bool {.
-  importc: "equalMem", noDecl, noSideEffect.}
+    importc: "equalMem", noDecl, noSideEffect.}
-  ## compares the memory blocks ``a`` and ``b``. ``size`` bytes will
+    ## compares the memory blocks ``a`` and ``b``. ``size`` bytes will
-  ## be compared. If the blocks are equal, true is returned, false
+    ## be compared. If the blocks are equal, true is returned, false
-  ## otherwise. Like any procedure dealing with raw memory this is
+    ## otherwise. Like any procedure dealing with raw memory this is
-  ## *unsafe*.
+    ## *unsafe*.
 
-proc alloc*(size: int): pointer {.noconv, rtl, tags: [].}
+  proc alloc*(size: int): pointer {.noconv, rtl, tags: [].}
-  ## allocates a new memory block with at least ``size`` bytes. The
+    ## allocates a new memory block with at least ``size`` bytes. The
-  ## block has to be freed with ``realloc(block, 0)`` or
+    ## block has to be freed with ``realloc(block, 0)`` or
-  ## ``dealloc(block)``. The block is not initialized, so reading
+    ## ``dealloc(block)``. The block is not initialized, so reading
-  ## from it before writing to it is undefined behaviour!
+    ## from it before writing to it is undefined behaviour!
-  ## The allocated memory belongs to its allocating thread!
+    ## The allocated memory belongs to its allocating thread!
-  ## Use `allocShared` to allocate from a shared heap.
+    ## Use `allocShared` to allocate from a shared heap.
-proc alloc0*(size: int): pointer {.noconv, rtl, tags: [].}
+  proc alloc0*(size: int): pointer {.noconv, rtl, tags: [].}
-  ## allocates a new memory block with at least ``size`` bytes. The
+    ## allocates a new memory block with at least ``size`` bytes. The
-  ## block has to be freed with ``realloc(block, 0)`` or
+    ## block has to be freed with ``realloc(block, 0)`` or
-  ## ``dealloc(block)``. The block is initialized with all bytes
+    ## ``dealloc(block)``. The block is initialized with all bytes
-  ## containing zero, so it is somewhat safer than ``alloc``.
+    ## containing zero, so it is somewhat safer than ``alloc``.
-  ## The allocated memory belongs to its allocating thread!
+    ## The allocated memory belongs to its allocating thread!
-  ## Use `allocShared0` to allocate from a shared heap.
+    ## Use `allocShared0` to allocate from a shared heap.
-proc realloc*(p: Pointer, newsize: int): pointer {.noconv, rtl, tags: [].}
+  proc realloc*(p: Pointer, newsize: int): pointer {.noconv, rtl, tags: [].}
-  ## grows or shrinks a given memory block. If p is **nil** then a new
+    ## grows or shrinks a given memory block. If p is **nil** then a new
-  ## memory block is returned. In either way the block has at least
+    ## memory block is returned. In either way the block has at least
-  ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
+    ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
-  ## ``realloc`` calls ``dealloc(p)``. In other cases the block has to
+    ## ``realloc`` calls ``dealloc(p)``. In other cases the block has to
-  ## be freed with ``dealloc``.
+    ## be freed with ``dealloc``.
-  ## The allocated memory belongs to its allocating thread!
+    ## The allocated memory belongs to its allocating thread!
-  ## Use `reallocShared` to reallocate from a shared heap.
+    ## Use `reallocShared` to reallocate from a shared heap.
-proc dealloc*(p: Pointer) {.noconv, rtl, tags: [].}
+  proc dealloc*(p: Pointer) {.noconv, rtl, tags: [].}
-  ## frees the memory allocated with ``alloc``, ``alloc0`` or
+    ## frees the memory allocated with ``alloc``, ``alloc0`` or
-  ## ``realloc``. This procedure is dangerous! If one forgets to
+    ## ``realloc``. This procedure is dangerous! If one forgets to
-  ## free the memory a leak occurs; if one tries to access freed
+    ## free the memory a leak occurs; if one tries to access freed
-  ## memory (or just freeing it twice!) a core dump may happen
+    ## memory (or just freeing it twice!) a core dump may happen
-  ## or other memory may be corrupted. 
+    ## or other memory may be corrupted. 
-  ## The freed memory must belong to its allocating thread!
+    ## The freed memory must belong to its allocating thread!
-  ## Use `deallocShared` to deallocate from a shared heap.
+    ## Use `deallocShared` to deallocate from a shared heap.
 
-proc allocShared*(size: int): pointer {.noconv, rtl.}
+  proc allocShared*(size: int): pointer {.noconv, rtl.}
-  ## allocates a new memory block on the shared heap with at
+    ## allocates a new memory block on the shared heap with at
-  ## least ``size`` bytes. The block has to be freed with
+    ## least ``size`` bytes. The block has to be freed with
-  ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block
+    ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block
-  ## is not initialized, so reading from it before writing to it is 
+    ## is not initialized, so reading from it before writing to it is 
-  ## undefined behaviour!
+    ## undefined behaviour!
-proc allocShared0*(size: int): pointer {.noconv, rtl.}
+  proc allocShared0*(size: int): pointer {.noconv, rtl.}
-  ## allocates a new memory block on the shared heap with at 
+    ## allocates a new memory block on the shared heap with at 
-  ## least ``size`` bytes. The block has to be freed with
+    ## least ``size`` bytes. The block has to be freed with
-  ## ``reallocShared(block, 0)`` or ``deallocShared(block)``.
+    ## ``reallocShared(block, 0)`` or ``deallocShared(block)``.
-  ## The block is initialized with all bytes
+    ## The block is initialized with all bytes
-  ## containing zero, so it is somewhat safer than ``allocShared``.
+    ## containing zero, so it is somewhat safer than ``allocShared``.
-proc reallocShared*(p: Pointer, newsize: int): pointer {.noconv, rtl.}
+  proc reallocShared*(p: Pointer, newsize: int): pointer {.noconv, rtl.}
-  ## grows or shrinks a given memory block on the heap. If p is **nil**
+    ## grows or shrinks a given memory block on the heap. If p is **nil**
-  ## then a new memory block is returned. In either way the block has at least
+    ## then a new memory block is returned. In either way the block has at least
-  ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
+    ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
-  ## ``reallocShared`` calls ``deallocShared(p)``. In other cases the
+    ## ``reallocShared`` calls ``deallocShared(p)``. In other cases the
-  ## block has to be freed with ``deallocShared``.
+    ## block has to be freed with ``deallocShared``.
-proc deallocShared*(p: Pointer) {.noconv, rtl.}
+  proc deallocShared*(p: Pointer) {.noconv, rtl.}
-  ## frees the memory allocated with ``allocShared``, ``allocShared0`` or
+    ## frees the memory allocated with ``allocShared``, ``allocShared0`` or
-  ## ``reallocShared``. This procedure is dangerous! If one forgets to
+    ## ``reallocShared``. This procedure is dangerous! If one forgets to
-  ## free the memory a leak occurs; if one tries to access freed
+    ## free the memory a leak occurs; if one tries to access freed
-  ## memory (or just freeing it twice!) a core dump may happen
+    ## memory (or just freeing it twice!) a core dump may happen
-  ## or other memory may be corrupted.
+    ## or other memory may be corrupted.
 
 proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.}
   ## swaps the values `a` and `b`. This is often more efficient than
@@ -1215,15 +1223,16 @@ const
 
 # GC interface:
 
-proc getOccupiedMem*(): int {.rtl.}
+when not defined(nimrodVM):
-  ## returns the number of bytes that are owned by the process and hold data.
+  proc getOccupiedMem*(): int {.rtl.}
+    ## returns the number of bytes that are owned by the process and hold data.
 
-proc getFreeMem*(): int {.rtl.}
+  proc getFreeMem*(): int {.rtl.}
-  ## returns the number of bytes that are owned by the process, but do not
+    ## returns the number of bytes that are owned by the process, but do not
-  ## hold any meaningful data.
+    ## hold any meaningful data.
 
-proc getTotalMem*(): int {.rtl.}
+  proc getTotalMem*(): int {.rtl.}
-  ## returns the number of bytes that are owned by the process.
+    ## returns the number of bytes that are owned by the process.
 
 
 iterator countdown*[T](a, b: T, step = 1): T {.inline.} =
@@ -1452,15 +1461,51 @@ proc pop*[T](s: var seq[T]): T {.inline, noSideEffect.} =
   result = s[L]
   setLen(s, L)
 
-proc each*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}): seq[S] = 
+proc each*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}): seq[S] {.
+  deprecated.} =
   ## The well-known ``map`` operation from functional programming. Applies
   ## `op` to every item in `data` and returns the result as a sequence.
+  ##
+  ## **Deprecated since version 0.9:** Use the ``map`` proc instead.
   newSeq(result, data.len)
   for i in 0..data.len-1: result[i] = op(data[i])
 
-proc each*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) =
+proc each*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) {.
+  deprecated.} =
   ## The well-known ``map`` operation from functional programming. Applies
-  ## `op` to every item in `data`.
+  ## `op` to every item in `data` modifying it directly.
+  ##
+  ## **Deprecated since version 0.9:** Use the ``map`` proc instead.
+  for i in 0..data.len-1: op(data[i])
+
+proc map*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}): seq[S] =
+  ## Returns a new sequence with the results of `op` applied to every item in
+  ## `data`.
+  ##
+  ## Since the input is not modified you can use this version of ``map`` to
+  ## transform the type of the elements in the input sequence. Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   let
+  ##     a = @[1, 2, 3, 4]
+  ##     b = map(a, proc(x: int): string = $x)
+  ##   assert b == @["1", "2", "3", "4"]
+  newSeq(result, data.len)
+  for i in 0..data.len-1: result[i] = op(data[i])
+
+proc map*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) =
+  ## Applies `op` to every item in `data` modifying it directly.
+  ##
+  ## Note that this version of ``map`` requires your input and output types to
+  ## be the same, since they are modified in-place. Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   var a = @["1", "2", "3", "4"]
+  ##   echo repr(a)
+  ##   # --> ["1", "2", "3", "4"]
+  ##   map(a, proc(x: var string) = x &= "42")
+  ##   echo repr(a)
+  ##   # --> ["142", "242", "342", "442"]
   for i in 0..data.len-1: op(data[i])
 
 iterator fields*[T: tuple](x: T): TObject {.
@@ -1541,40 +1586,41 @@ when false:
 
 # ----------------- GC interface ---------------------------------------------
 
-proc GC_disable*() {.rtl, inl.}
+when not defined(nimrodVM):
-  ## disables the GC. If called n-times, n calls to `GC_enable` are needed to
+  proc GC_disable*() {.rtl, inl.}
-  ## reactivate the GC. Note that in most circumstances one should only disable
+    ## disables the GC. If called n-times, n calls to `GC_enable` are needed to
-  ## the mark and sweep phase with `GC_disableMarkAndSweep`.
+    ## reactivate the GC. Note that in most circumstances one should only disable
+    ## the mark and sweep phase with `GC_disableMarkAndSweep`.
 
-proc GC_enable*() {.rtl, inl.}
+  proc GC_enable*() {.rtl, inl.}
-  ## enables the GC again.
+    ## enables the GC again.
 
-proc GC_fullCollect*() {.rtl.}
+  proc GC_fullCollect*() {.rtl.}
-  ## forces a full garbage collection pass.
+    ## forces a full garbage collection pass.
-  ## Ordinary code does not need to call this (and should not).
+    ## Ordinary code does not need to call this (and should not).
 
-type
+  type
-  TGC_Strategy* = enum ## the strategy the GC should use for the application
+    TGC_Strategy* = enum ## the strategy the GC should use for the application
-    gcThroughput,      ## optimize for throughput
+      gcThroughput,      ## optimize for throughput
-    gcResponsiveness,  ## optimize for responsiveness (default)
+      gcResponsiveness,  ## optimize for responsiveness (default)
-    gcOptimizeTime,    ## optimize for speed
+      gcOptimizeTime,    ## optimize for speed
-    gcOptimizeSpace    ## optimize for memory footprint
+      gcOptimizeSpace    ## optimize for memory footprint
 
-proc GC_setStrategy*(strategy: TGC_Strategy) {.rtl, deprecated.}
+  proc GC_setStrategy*(strategy: TGC_Strategy) {.rtl, deprecated.}
-  ## tells the GC the desired strategy for the application.
+    ## tells the GC the desired strategy for the application.
-  ## **Deprecated** since version 0.8.14. This has always been a nop.
+    ## **Deprecated** since version 0.8.14. This has always been a nop.
 
-proc GC_enableMarkAndSweep*() {.rtl.}
+  proc GC_enableMarkAndSweep*() {.rtl.}
-proc GC_disableMarkAndSweep*() {.rtl.}
+  proc GC_disableMarkAndSweep*() {.rtl.}
-  ## the current implementation uses a reference counting garbage collector
+    ## the current implementation uses a reference counting garbage collector
-  ## with a seldomly run mark and sweep phase to free cycles. The mark and
+    ## with a seldomly run mark and sweep phase to free cycles. The mark and
-  ## sweep phase may take a long time and is not needed if the application
+    ## sweep phase may take a long time and is not needed if the application
-  ## does not create cycles. Thus the mark and sweep phase can be deactivated
+    ## does not create cycles. Thus the mark and sweep phase can be deactivated
-  ## and activated separately from the rest of the GC.
+    ## and activated separately from the rest of the GC.
 
-proc GC_getStatistics*(): string {.rtl.}
+  proc GC_getStatistics*(): string {.rtl.}
-  ## returns an informative string about the GC's activity. This may be useful
+    ## returns an informative string about the GC's activity. This may be useful
-  ## for tweaking.
+    ## for tweaking.
     
 proc GC_ref*[T](x: ref T) {.magic: "GCref".}
 proc GC_ref*[T](x: seq[T]) {.magic: "GCref".}
@@ -1701,28 +1747,29 @@ proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo".}
   ## get type information for `x`. Ordinary code should not use this, but
   ## the `typeinfo` module instead.
 
-when not defined(EcmaScript) and not defined(NimrodVM):
+when not defined(EcmaScript): #and not defined(NimrodVM):
   {.push stack_trace: off, profiler:off.}
 
-  proc initGC()
+  when not defined(NimrodVM):
-  when not defined(boehmgc) and not defined(useMalloc):
+    proc initGC()
-    proc initAllocator() {.inline.}
+    when not defined(boehmgc) and not defined(useMalloc):
+      proc initAllocator() {.inline.}
 
-  proc initStackBottom() {.inline, compilerproc.} =
+    proc initStackBottom() {.inline, compilerproc.} =
-    # WARNING: This is very fragile! An array size of 8 does not work on my
+      # WARNING: This is very fragile! An array size of 8 does not work on my
-    # Linux 64bit system. Very strange, but we are at the will of GCC's 
+      # Linux 64bit system. -- That's because the stack direction is the other
-    # optimizer...
+      # way round.
-    when defined(setStackBottom):
+      when defined(setStackBottom):
-      var locals {.volatile.}: pointer
+        var locals {.volatile.}: pointer
-      locals = addr(locals)
+        locals = addr(locals)
-      setStackBottom(locals)
+        setStackBottom(locals)
 
-  var
+    var
-    strDesc: TNimType
+      strDesc: TNimType
 
-  strDesc.size = sizeof(string)
+    strDesc.size = sizeof(string)
-  strDesc.kind = tyString
+    strDesc.kind = tyString
-  strDesc.flags = {ntfAcyclic}
+    strDesc.flags = {ntfAcyclic}
 
   include "system/ansi_c"
 
@@ -1730,28 +1777,27 @@ when not defined(EcmaScript) and not defined(NimrodVM):
     result = int(c_strcmp(x, y))
 
   const pccHack = if defined(pcc): "_" else: "" # Hack for PCC
-  when defined(windows):
+  when not defined(NimrodVM):
-    # work-around C's sucking abstraction:
+    when defined(windows):
-    # BUGFIX: stdin and stdout should be binary files!
+      # work-around C's sucking abstraction:
-    proc setmode(handle, mode: int) {.importc: pccHack & "setmode",
+      # BUGFIX: stdin and stdout should be binary files!
-                                      header: "<io.h>".}
+      proc setmode(handle, mode: int) {.importc: pccHack & "setmode",
-    proc fileno(f: C_TextFileStar): int {.importc: pccHack & "fileno",
+                                        header: "<io.h>".}
-                                          header: "<fcntl.h>".}
+      proc fileno(f: C_TextFileStar): int {.importc: pccHack & "fileno",
-    var
+                                            header: "<fcntl.h>".}
-      O_BINARY {.importc: pccHack & "O_BINARY", nodecl.}: int
+      var
+        O_BINARY {.importc: pccHack & "O_BINARY", nodecl.}: int
 
-    # we use binary mode in Windows:
+      # we use binary mode in Windows:
-    setmode(fileno(c_stdin), O_BINARY)
+      setmode(fileno(c_stdin), O_BINARY)
-    setmode(fileno(c_stdout), O_BINARY)
+      setmode(fileno(c_stdout), O_BINARY)
     
-  when defined(endb):
+    when defined(endb):
-    proc endbStep()
+      proc endbStep()
 
   # ----------------- IO Part ------------------------------------------------
-
   type
-    CFile {.importc: "FILE", nodecl, final.} = object  # empty record for
+    CFile {.importc: "FILE", nodecl, final, incompletestruct.} = object
-                                                       # data hiding
     TFile* = ptr CFile ## The type representing a file handle.
 
     TFileMode* = enum           ## The file mode when opening a file.
@@ -1777,7 +1823,7 @@ when not defined(EcmaScript) and not defined(NimrodVM):
       ## The standard error stream.
       ##
       ## Note: In my opinion, this should not be used -- the concept of a
-      ## separate error stream is a design flaw of UNIX. A seperate *message
+      ## separate error stream is a design flaw of UNIX. A separate *message
       ## stream* is a good idea, but since it is named ``stderr`` there are few
       ## programs out there that distinguish properly between ``stdout`` and
       ## ``stderr``. So, that's what you get if you don't name your variables
@@ -1934,31 +1980,32 @@ when not defined(EcmaScript) and not defined(NimrodVM):
 
   # -------------------------------------------------------------------------
 
-  proc allocCStringArray*(a: openArray[string]): cstringArray =
+  when not defined(NimrodVM):
-    ## creates a NULL terminated cstringArray from `a`. The result has to
+    proc allocCStringArray*(a: openArray[string]): cstringArray =
-    ## be freed with `deallocCStringArray` after it's not needed anymore.
+      ## creates a NULL terminated cstringArray from `a`. The result has to
-    result = cast[cstringArray](alloc0((a.len+1) * sizeof(cstring)))
+      ## be freed with `deallocCStringArray` after it's not needed anymore.
-    for i in 0 .. a.high:
+      result = cast[cstringArray](alloc0((a.len+1) * sizeof(cstring)))
-      # XXX get rid of this string copy here:
+      for i in 0 .. a.high:
-      var x = a[i]
+        # XXX get rid of this string copy here:
-      result[i] = cast[cstring](alloc0(x.len+1))
+        var x = a[i]
-      copyMem(result[i], addr(x[0]), x.len)
+        result[i] = cast[cstring](alloc0(x.len+1))
+        copyMem(result[i], addr(x[0]), x.len)
 
-  proc deallocCStringArray*(a: cstringArray) =
+    proc deallocCStringArray*(a: cstringArray) =
-    ## frees a NULL terminated cstringArray.
+      ## frees a NULL terminated cstringArray.
-    var i = 0
+      var i = 0
-    while a[i] != nil:
+      while a[i] != nil:
-      dealloc(a[i])
+        dealloc(a[i])
-      inc(i)
+        inc(i)
-    dealloc(a)
+      dealloc(a)
 
-  proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable.}
+    proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable.}
-    ## atomic increment of `memLoc`. Returns the value after the operation.
+      ## atomic increment of `memLoc`. Returns the value after the operation.
     
-  proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable.}
+    proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable.}
-    ## atomic decrement of `memLoc`. Returns the value after the operation.
+      ## atomic decrement of `memLoc`. Returns the value after the operation.
 
-  include "system/atomics"
+    include "system/atomics"
 
   type
     PSafePoint = ptr TSafePoint
@@ -1974,71 +2021,76 @@ when not defined(EcmaScript) and not defined(NimrodVM):
   when hasThreadSupport:
     include "system/syslocks"
     include "system/threads"
-  elif not defined(nogc):
+  elif not defined(nogc) and not defined(NimrodVM):
     when not defined(useNimRtl) and not defined(createNimRtl): initStackBottom()
     initGC()
 
-  proc setControlCHook*(hook: proc () {.noconv.})
+  when not defined(NimrodVM):
-    ## allows you to override the behaviour of your application when CTRL+C
+    proc setControlCHook*(hook: proc () {.noconv.})
-    ## is pressed. Only one such hook is supported.
+      ## allows you to override the behaviour of your application when CTRL+C
+      ## is pressed. Only one such hook is supported.
       
-  proc writeStackTrace*() {.tags: [FWriteIO].}
+    proc writeStackTrace*() {.tags: [FWriteIO].}
-    ## writes the current stack trace to ``stderr``. This is only works
+      ## writes the current stack trace to ``stderr``. This is only works
-    ## for debug builds.
+      ## for debug builds.
-  when hostOS != "standalone":
+    when hostOS != "standalone":
-    proc getStackTrace*(): string
+      proc getStackTrace*(): string
-      ## gets the current stack trace. This only works for debug builds.
+        ## gets the current stack trace. This only works for debug builds.
 
-    proc getStackTrace*(e: ref E_Base): string
+      proc getStackTrace*(e: ref E_Base): string
-      ## gets the stack trace associated with `e`, which is the stack that
+        ## gets the stack trace associated with `e`, which is the stack that
-      ## lead to the ``raise`` statement. This only works for debug builds.
+        ## lead to the ``raise`` statement. This only works for debug builds.
         
-  {.push stack_trace: off, profiler:off.}
+    {.push stack_trace: off, profiler:off.}
-  when hostOS == "standalone":
+    when hostOS == "standalone":
-    include "system/embedded"
+      include "system/embedded"
-  else:
-    include "system/excpt"
-    
-  # we cannot compile this with stack tracing on
-  # as it would recurse endlessly!
-  include "system/arithm"
-  {.pop.} # stack trace
-  {.pop.} # stack trace
-      
-  when hostOS != "standalone": include "system/dyncalls"
-  include "system/sets"
-
-  const
-    GenericSeqSize = (2 * sizeof(int))
-    
-  proc getDiscriminant(aa: Pointer, n: ptr TNimNode): int =
-    sysAssert(n.kind == nkCase, "getDiscriminant: node != nkCase")
-    var d: int
-    var a = cast[TAddress](aa)
-    case n.typ.size
-    of 1: d = ze(cast[ptr int8](a +% n.offset)[])
-    of 2: d = ze(cast[ptr int16](a +% n.offset)[])
-    of 4: d = int(cast[ptr int32](a +% n.offset)[])
-    else: sysAssert(false, "getDiscriminant: invalid n.typ.size")
-    return d
-
-  proc selectBranch(aa: Pointer, n: ptr TNimNode): ptr TNimNode =
-    var discr = getDiscriminant(aa, n)
-    if discr <% n.len:
-      result = n.sons[discr]
-      if result == nil: result = n.sons[n.len]
-      # n.sons[n.len] contains the ``else`` part (but may be nil)
     else:
-      result = n.sons[n.len]
+      include "system/excpt"
       
-  include "system/mmdisp"
+    # we cannot compile this with stack tracing on
-  {.push stack_trace: off, profiler:off.}
+    # as it would recurse endlessly!
-  when hostOS != "standalone": include "system/sysstr"
+    include "system/arithm"
-  {.pop.}
+    {.pop.} # stack trace
+  {.pop.} # stack trace
       
-  include "system/sysio"
+  when hostOS != "standalone" and not defined(NimrodVM):
-  when hasThreadSupport:
+    include "system/dyncalls"
-    include "system/channels"
+  when not defined(NimrodVM):
+    include "system/sets"
+
+    const
+      GenericSeqSize = (2 * sizeof(int))
+      
+    proc getDiscriminant(aa: Pointer, n: ptr TNimNode): int =
+      sysAssert(n.kind == nkCase, "getDiscriminant: node != nkCase")
+      var d: int
+      var a = cast[TAddress](aa)
+      case n.typ.size
+      of 1: d = ze(cast[ptr int8](a +% n.offset)[])
+      of 2: d = ze(cast[ptr int16](a +% n.offset)[])
+      of 4: d = int(cast[ptr int32](a +% n.offset)[])
+      else: sysAssert(false, "getDiscriminant: invalid n.typ.size")
+      return d
+
+    proc selectBranch(aa: Pointer, n: ptr TNimNode): ptr TNimNode =
+      var discr = getDiscriminant(aa, n)
+      if discr <% n.len:
+        result = n.sons[discr]
+        if result == nil: result = n.sons[n.len]
+        # n.sons[n.len] contains the ``else`` part (but may be nil)
+      else:
+        result = n.sons[n.len]
+
+    include "system/mmdisp"
+    {.push stack_trace: off, profiler:off.}
+    when hostOS != "standalone": include "system/sysstr"
+    {.pop.}
+
+    include "system/sysio"
+    when hasThreadSupport:
+      include "system/channels"
+  else:
+    include "system/sysio"
 
   iterator lines*(filename: string): TaintedString {.tags: [FReadIO].} =
     ## Iterate over any line in the file named `filename`.
@@ -2053,7 +2105,7 @@ when not defined(EcmaScript) and not defined(NimrodVM):
     var res = TaintedString(newStringOfCap(80))
     while f.readLine(res): yield TaintedString(res)
 
-  when hostOS != "standalone":
+  when hostOS != "standalone" and not defined(NimrodVM):
     include "system/assign"
     include "system/repr"
 
@@ -2078,43 +2130,46 @@ when not defined(EcmaScript) and not defined(NimrodVM):
         excHandler.raiseAction = action
 
   {.push stack_trace: off, profiler:off.}
-  when defined(endb):
+  when defined(endb) and not defined(NimrodVM):
     include "system/debugger"
 
   when defined(profiler) or defined(memProfiler):
     include "system/profiler"
   {.pop.} # stacktrace
 
-  proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.}
+  when not defined(NimrodVM):
-    ## can be used to mark a condition to be likely. This is a hint for the 
+    proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.}
-    ## optimizer.
+      ## can be used to mark a condition to be likely. This is a hint for the 
+      ## optimizer.
     
-  proc unlikely*(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.}
+    proc unlikely*(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.}
-    ## can be used to mark a condition to be unlikely. This is a hint for the 
+      ## can be used to mark a condition to be unlikely. This is a hint for the 
-    ## optimizer.
+      ## optimizer.
       
-  proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} =
+    proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} =
-    ## retrieves the raw proc pointer of the closure `x`. This is
+      ## retrieves the raw proc pointer of the closure `x`. This is
-    ## useful for interfacing closures with C.
+      ## useful for interfacing closures with C.
-    {.emit: """
+      {.emit: """
-    `result` = `x`.ClPrc;
+      `result` = `x`.ClPrc;
-    """.}
+      """.}
 
-  proc rawEnv*[T: proc](x: T): pointer {.noSideEffect, inline.} =
+    proc rawEnv*[T: proc](x: T): pointer {.noSideEffect, inline.} =
-    ## retrieves the raw environment pointer of the closure `x`. This is
+      ## retrieves the raw environment pointer of the closure `x`. This is
-    ## useful for interfacing closures with C.
+      ## useful for interfacing closures with C.
-    {.emit: """
+      {.emit: """
-    `result` = `x`.ClEnv;
+      `result` = `x`.ClEnv;
-    """.}
+      """.}
 
-  proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} =
+    proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} =
-    ## can be used to determine if a first class iterator has finished.
+      ## can be used to determine if a first class iterator has finished.
-    {.emit: """
+      {.emit: """
-    `result` = *((NI*) `x`.ClEnv) < 0;
+      `result` = *((NI*) `x`.ClEnv) < 0;
-    """.}
+      """.}
 
-elif defined(ecmaScript) or defined(NimrodVM):
+elif defined(ecmaScript):
   # Stubs:
+  proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = nil
+
   proc GC_disable() = nil
   proc GC_enable() = nil
   proc GC_fullCollect() = nil
@@ -2146,6 +2201,10 @@ elif defined(ecmaScript) or defined(NimrodVM):
       if x < y: return -1
       return 1
   
+  when defined(nimffi):
+    include "system/sysio"
+
+
 proc quit*(errormsg: string, errorcode = QuitFailure) {.noReturn.} =
   ## a shorthand for ``echo(errormsg); quit(errorcode)``.
   echo(errormsg)

lib/system/ansi_c.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -21,9 +21,8 @@ proc c_strlen(a: CString): int {.nodecl, noSideEffect, importc: "strlen".}
 proc c_memset(p: pointer, value: cint, size: int) {.nodecl, importc: "memset".}
 
 type
-  C_TextFile {.importc: "FILE", nodecl, final.} = object   # empty record for
+  C_TextFile {.importc: "FILE", nodecl, final, incompleteStruct.} = object
-                                                           # data hiding
+  C_BinaryFile {.importc: "FILE", nodecl, final, incompleteStruct.} = object
-  C_BinaryFile {.importc: "FILE", nodecl, final.} = object
   C_TextFileStar = ptr CTextFile
   C_BinaryFileStar = ptr CBinaryFile
 

lib/system/debugger.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -579,6 +579,10 @@ proc hash(Data: Pointer, Size: int): THash =
     Dec(s)
   result = !$h
 
+proc hashGcHeader(data: pointer): THash =
+  const headerSize = sizeof(int)*2
+  result = hash(cast[pointer](cast[int](data) -% headerSize), headerSize)
+
 proc genericHashAux(dest: Pointer, mt: PNimType, shallow: bool,
                     h: THash): THash
 proc genericHashAux(dest: Pointer, n: ptr TNimNode, shallow: bool,
@@ -606,20 +610,22 @@ proc genericHashAux(dest: Pointer, mt: PNimType, shallow: bool,
     result = h
     if x != nil:
       let s = cast[NimString](x)
-      when true:
+      when defined(trackGcHeaders):
-        result = result !& hash(x, s.len)
+        result = result !& hashGcHeader(x)
       else:
-        let y = cast[pointer](cast[int](x) -% 2*sizeof(int))
+        result = result !& hash(x, s.len)
-        result = result !& hash(y, s.len + 2*sizeof(int))
   of tySequence:
     var x = cast[ppointer](dest)
     var dst = cast[taddress](cast[ppointer](dest)[])
     result = h
     if dst != 0:
-      for i in 0..cast[pgenericseq](dst).len-1:
+      when defined(trackGcHeaders):
-        result = result !& genericHashAux(
+        result = result !& hashGcHeader(cast[ppointer](dest)[])
-          cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize),
+      else:
-          mt.Base, shallow, result)
+        for i in 0..cast[pgenericseq](dst).len-1:
+          result = result !& genericHashAux(
+            cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize),
+            mt.Base, shallow, result)
   of tyObject, tyTuple:
     # we don't need to copy m_type field for tyObject, as they are equal anyway
     result = genericHashAux(dest, mt.node, shallow, h)
@@ -630,13 +636,18 @@ proc genericHashAux(dest: Pointer, mt: PNimType, shallow: bool,
       result = result !& genericHashAux(cast[pointer](d +% i*% mt.base.size),
                                         mt.base, shallow, result)
   of tyRef:
-    if shallow:
+    when defined(trackGcHeaders):
-      result = h !& hash(dest, mt.size)
-    else:
-      result = h
       var s = cast[ppointer](dest)[]
       if s != nil:
-        result = result !& genericHashAux(s, mt.base, shallow, result)
+        result = result !& hashGcHeader(s)
+    else:
+      if shallow:
+        result = h !& hash(dest, mt.size)
+      else:
+        result = h
+        var s = cast[ppointer](dest)[]
+        if s != nil:
+          result = result !& genericHashAux(s, mt.base, shallow, result)
         # hash the object header:
         #const headerSize = sizeof(int)*2
         #result = result !& hash(cast[pointer](cast[int](s) -% headerSize),

lib/system/sysio.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.

lib/wrappers/gtk/gtk2.nim

@@ -16596,6 +16596,9 @@ proc message_dialog_new*(parent: PWindow, flags: TDialogFlags,
                          thetype: TMessageType, buttons: TButtonsType, 
                          message_format: cstring): PMessageDialog{.varargs, 
     cdecl, importc: "gtk_message_dialog_new", dynlib: lib.}
+proc set_markup*(msgDialog: PMessageDialog, str: cstring) {.cdecl,
+    importc: "gtk_message_dialog_set_markup", dynlib: lib.}
+
 proc signal_new*(name: cstring, signal_flags: TSignalRunType, 
                  object_type: TType, function_offset: guint, 
                  marshaller: TSignalMarshaller, return_val: TType, n_args: guint): guint{.
@@ -16893,6 +16896,15 @@ type
 proc set_tooltip_text*(w: PWidget, t: cstring){.cdecl,
   dynlib: lib, importc: "gtk_widget_set_tooltip_text".}
 
+proc get_tooltip_text*(w: PWidget): cstring{.cdecl,
+  dynlib: lib, importc: "gtk_widget_get_tooltip_text".}
+
+proc set_tooltip_markup*(w: PWidget, m: cstring) {.cdecl, dynlib: lib,
+  importc: "gtk_widget_set_tooltip_markup".}
+
+proc get_tooltip_markup*(w: PWidget): cstring {.cdecl, dynlib: lib,
+  importc: "gtk_widget_get_tooltip_markup".}
+
 proc set_tooltip_column*(w: PTreeview, column: gint){.cdecl,
   dynlib: lib, importc: "gtk_tree_view_set_tooltip_column".}
 
@@ -16905,6 +16917,9 @@ proc trigger_tooltip_query*(widg: PTooltip){.cdecl, dynlib: lib,
 proc set_has_tooltip*(widget: PWidget, b: gboolean){.cdecl, dynlib: lib, 
   importc: "gtk_widget_set_has_tooltip".}
 
+proc get_has_tooltip*(widget: PWidget): gboolean{.cdecl, dynlib: lib, 
+  importc: "gtk_widget_get_has_tooltip".}
+
 proc set_markup*(tp: PTooltip, mk: cstring){.cdecl, dynlib: lib, 
   importc: "gtk_tooltip_set_markup".}
 
@@ -17035,6 +17050,10 @@ proc remove*(combo_box: PComboBoxText; position: gint){.cdecl,
     importc: "gtk_combo_box_text_remove", dynlib: lib.}
 proc get_active_text*(combo_box: PComboBoxText): cstring{.cdecl, 
     importc: "gtk_combo_box_text_get_active_text", dynlib: lib.}
+proc is_active*(win: PWindow): gboolean{.cdecl,
+    importc: "gtk_window_is_active", dynlib: lib.}
+proc has_toplevel_focus*(win: PWindow): gboolean{.cdecl,
+    importc: "gtk_window_has_toplevel_focus", dynlib: lib.}
 
 proc nimrod_init*() =
   var

lib/wrappers/libffi.nim

@@ -0,0 +1,149 @@
+# -----------------------------------------------------------------*-C-*-
+#   libffi 3.0.10 - Copyright (c) 2011 Anthony Green
+#                    - Copyright (c) 1996-2003, 2007, 2008 Red Hat, Inc.
+#
+#   Permission is hereby granted, free of charge, to any person
+#   obtaining a copy of this software and associated documentation
+#   files (the ``Software''), to deal in the Software without
+#   restriction, including without limitation the rights to use, copy,
+#   modify, merge, publish, distribute, sublicense, and/or sell copies
+#   of the Software, and to permit persons to whom the Software is
+#   furnished to do so, subject to the following conditions:
+#
+#   The above copyright notice and this permission notice shall be
+#   included in all copies or substantial portions of the Software.
+#
+#   THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
+#   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#   NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#   DEALINGS IN THE SOFTWARE.
+#
+#   ----------------------------------------------------------------------- 
+
+{.deadCodeElim: on.}
+
+when defined(windows): 
+  const libffidll* = "libffi.dll"
+elif defined(macosx): 
+  const libffidll* = "libffi.dylib"
+else: 
+  const libffidll* = "libffi.so"
+
+type
+  TArg* = int
+  TSArg* = int
+
+when defined(windows) and defined(x86):
+  type
+    TABI* {.size: sizeof(cint).} = enum
+      FIRST_ABI, SYSV, STDCALL
+
+  const DEFAULT_ABI* = SYSV
+elif defined(amd64) and defined(windows):
+  type 
+    TABI* {.size: sizeof(cint).} = enum 
+      FIRST_ABI, WIN64
+  const DEFAULT_ABI* = WIN64
+else:
+  type 
+    TABI* {.size: sizeof(cint).} = enum
+      FIRST_ABI, SYSV, UNIX64
+
+  when defined(i386):
+    const DEFAULT_ABI* = SYSV
+  else: 
+    const DEFAULT_ABI* = UNIX64
+    
+const 
+  tkVOID* = 0
+  tkINT* = 1
+  tkFLOAT* = 2
+  tkDOUBLE* = 3
+  tkLONGDOUBLE* = 4
+  tkUINT8* = 5
+  tkSINT8* = 6
+  tkUINT16* = 7
+  tkSINT16* = 8
+  tkUINT32* = 9
+  tkSINT32* = 10
+  tkUINT64* = 11
+  tkSINT64* = 12
+  tkSTRUCT* = 13
+  tkPOINTER* = 14
+
+  tkLAST = tkPOINTER
+  tkSMALL_STRUCT_1B* = (tkLAST + 1)
+  tkSMALL_STRUCT_2B* = (tkLAST + 2)
+  tkSMALL_STRUCT_4B* = (tkLAST + 3)
+
+type
+  TType* = object
+    size*: int
+    alignment*: uint16
+    typ*: uint16
+    elements*: ptr ptr TType
+
+var
+  type_void* {.importc: "ffi_type_void", dynlib: libffidll.}: TType
+  type_uint8* {.importc: "ffi_type_uint8", dynlib: libffidll.}: TType
+  type_sint8* {.importc: "ffi_type_sint8", dynlib: libffidll.}: TType
+  type_uint16* {.importc: "ffi_type_uint16", dynlib: libffidll.}: TType
+  type_sint16* {.importc: "ffi_type_sint16", dynlib: libffidll.}: TType
+  type_uint32* {.importc: "ffi_type_uint32", dynlib: libffidll.}: TType
+  type_sint32* {.importc: "ffi_type_sint32", dynlib: libffidll.}: TType
+  type_uint64* {.importc: "ffi_type_uint64", dynlib: libffidll.}: TType
+  type_sint64* {.importc: "ffi_type_sint64", dynlib: libffidll.}: TType
+  type_float* {.importc: "ffi_type_float", dynlib: libffidll.}: TType
+  type_double* {.importc: "ffi_type_double", dynlib: libffidll.}: TType
+  type_pointer* {.importc: "ffi_type_pointer", dynlib: libffidll.}: TType
+  type_longdouble* {.importc: "ffi_type_longdouble", dynlib: libffidll.}: TType
+
+type 
+  Tstatus* {.size: sizeof(cint).} = enum 
+    OK, BAD_TYPEDEF, BAD_ABI
+  TTypeKind* = cuint
+  TCif* {.pure, final.} = object 
+    abi*: TABI
+    nargs*: cuint
+    arg_types*: ptr ptr TType
+    rtype*: ptr TType
+    bytes*: cuint
+    flags*: cuint
+
+type
+  TRaw* = object 
+    sint*: TSArg
+
+proc raw_call*(cif: var Tcif; fn: proc () {.cdecl.}; rvalue: pointer; 
+               avalue: ptr TRaw) {.cdecl, importc: "ffi_raw_call", 
+                                   dynlib: libffidll.}
+proc ptrarray_to_raw*(cif: var Tcif; args: ptr pointer; raw: ptr TRaw) {.cdecl, 
+    importc: "ffi_ptrarray_to_raw", dynlib: libffidll.}
+proc raw_to_ptrarray*(cif: var Tcif; raw: ptr TRaw; args: ptr pointer) {.cdecl, 
+    importc: "ffi_raw_to_ptrarray", dynlib: libffidll.}
+proc raw_size*(cif: var Tcif): int {.cdecl, importc: "ffi_raw_size", 
+                                     dynlib: libffidll.}
+
+proc prep_cif*(cif: var Tcif; abi: TABI; nargs: cuint; rtype: ptr TType; 
+               atypes: ptr ptr TType): TStatus {.cdecl, importc: "ffi_prep_cif", 
+    dynlib: libffidll.}
+proc call*(cif: var Tcif; fn: proc () {.cdecl.}; rvalue: pointer; 
+           avalue: ptr pointer) {.cdecl, importc: "ffi_call", dynlib: libffidll.}
+
+# the same with an easier interface:
+type
+  TParamList* = array[0..100, ptr TType]
+  TArgList* = array[0..100, pointer]
+
+proc prep_cif*(cif: var Tcif; abi: TABI; nargs: cuint; rtype: ptr TType; 
+               atypes: TParamList): TStatus {.cdecl, importc: "ffi_prep_cif",
+    dynlib: libffidll.}
+proc call*(cif: var Tcif; fn, rvalue: pointer;
+           avalue: TArgList) {.cdecl, importc: "ffi_call", dynlib: libffidll.}
+
+# Useful for eliminating compiler warnings 
+##define FFI_FN(f) ((void (*)(void))f)

lib/wrappers/lua/lua.nim

@@ -35,19 +35,32 @@
 #**   In french or in english
 #
 
-when defined(MACOSX): 
+when defined(useLuajit):
-  const 
+  when defined(MACOSX):
-    NAME* = "liblua(|5.2|5.1|5.0).dylib"
+    const
-    LIB_NAME* = "liblua(|5.2|5.1|5.0).dylib"
+      NAME* = "libluajit.dylib"
-elif defined(UNIX): 
+      LIB_NAME* = "libluajit.dylib"
-  const 
+  elif defined(UNIX):
-    NAME* = "liblua(|5.2|5.1|5.0).so(|.0)"
+    const
-    LIB_NAME* = "liblua(|5.2|5.1|5.0).so(|.0)"
+      NAME* = "libluajit.so(|.0)"
+      LIB_NAME* = "libluajit.so(|.0)"
+  else:
+    const
+      NAME* = "luajit.dll"
+      LIB_NAME* = "luajit.dll"
 else:
-  const 
+  when defined(MACOSX):
-    NAME* = "lua(|5.2|5.1|5.0).dll"
+    const
-    LIB_NAME* = "lua(|5.2|5.1|5.0).dll"
+      NAME* = "liblua(|5.2|5.1|5.0).dylib"
-
+      LIB_NAME* = "liblua(|5.2|5.1|5.0).dylib"
+  elif defined(UNIX):
+    const
+      NAME* = "liblua(|5.2|5.1|5.0).so(|.0)"
+      LIB_NAME* = "liblua(|5.2|5.1|5.0).so(|.0)"
+  else:
+    const 
+      NAME* = "lua(|5.2|5.1|5.0).dll"
+      LIB_NAME* = "lua(|5.2|5.1|5.0).dll"
 
 const 
   VERSION* = "Lua 5.1"

lib/wrappers/openssl.nim

@@ -39,6 +39,8 @@
 
 ## OpenSSL support
 
+{.deadCodeElim: on.}
+
 when defined(WINDOWS): 
   const 
     DLLSSLName = "(ssleay32|libssl32).dll"

lib/wrappers/sdl/sdl_image.nim

@@ -139,7 +139,7 @@ elif defined(macosx):
     ImageLibName = "libSDL_image-1.2.0.dylib"
 else:
   const
-    ImageLibName = "libSDL_image.so"
+    ImageLibName = "libSDL_image(.so|-1.2.so.0)"
 const
   IMAGE_MAJOR_VERSION* = 1
   IMAGE_MINOR_VERSION* = 2

packages/docutils/rst.nim

@@ -313,6 +313,10 @@ proc rstMessage(p: TRstParser, msgKind: TMsgKind, arg: string) =
   p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, 
                              p.col + p.tok[p.idx].col, msgKind, arg)
 
+proc rstMessage(p: TRstParser, msgKind: TMsgKind, arg: string, line, col: int) = 
+  p.s.msgHandler(p.filename, p.line + line, 
+                             p.col + col, msgKind, arg)
+
 proc rstMessage(p: TRstParser, msgKind: TMsgKind) = 
   p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, 
                              p.col + p.tok[p.idx].col, msgKind, 
@@ -684,6 +688,9 @@ when false:
 
 proc parseUntil(p: var TRstParser, father: PRstNode, postfix: string, 
                 interpretBackslash: bool) = 
+  let
+    line = p.tok[p.idx].line
+    col = p.tok[p.idx].col
   while true: 
     case p.tok[p.idx].kind
     of tkPunct: 
@@ -707,7 +714,7 @@ proc parseUntil(p: var TRstParser, father: PRstNode, postfix: string,
     of tkWhite: 
       add(father, newRstNode(rnLeaf, " "))
       inc(p.idx)
-    else: rstMessage(p, meExpected, postfix)
+    else: rstMessage(p, meExpected, postfix, line, col)
 
 proc parseMarkdownCodeblock(p: var TRstParser): PRstNode =
   var args = newRstNode(rnDirArg)
@@ -1014,8 +1021,8 @@ proc whichSection(p: TRstParser): TRstNodeKind =
       result = rnParagraph
   of tkWord, tkOther, tkWhite:
     if match(p, tokenAfterNewLine(p), "ai"): result = rnHeadline
-    elif isDefList(p): result = rnDefList
     elif match(p, p.idx, "e) ") or match(p, p.idx, "e. "): result = rnEnumList
+    elif isDefList(p): result = rnDefList
     else: result = rnParagraph
   else: result = rnLeaf
   

packages/docutils/rstgen.nim

@@ -390,7 +390,8 @@ proc renderField(d: PDoc, n: PRstNode, result: var string) =
   if d.target == outLatex: 
     var fieldname = addNodes(n.sons[0])
     var fieldval = esc(d.target, strip(addNodes(n.sons[1])))
-    if cmpIgnoreStyle(fieldname, "author") == 0:
+    if cmpIgnoreStyle(fieldname, "author") == 0 or 
+       cmpIgnoreStyle(fieldname, "authors") == 0:
       if d.meta[metaAuthor].len == 0:
         d.meta[metaAuthor] = fieldval
         b = true
@@ -474,8 +475,8 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
   of rnTableRow: 
     if len(n) >= 1:
       if d.target == outLatex:
-        var tmp = ""
+        #var tmp = ""
-        renderRstToOut(d, n.sons[0], tmp)
+        renderRstToOut(d, n.sons[0], result)
         for i in countup(1, len(n) - 1):
           result.add(" & ")
           renderRstToOut(d, n.sons[i], result)

readme.md

@@ -0,0 +1,66 @@
+# Nimrod Compiler
+This repo contains the Nimrod compiler, Nimrod's stdlib, tools and 
+documentation.
+
+## Compiling
+Compiling the Nimrod compiler is quite straightforward. Because
+the Nimrod compiler itself is written in the Nimrod programming language
+the C source of an older version of the compiler are needed to bootstrap the
+latest version. The C sources are however included with this repository under
+the build directory.
+
+Pre-compiled snapshots of the compiler are also available on
+[Nimbuild](http://build.nimrod-code.org/). Your platform however may not 
+currently be built for.
+
+The compiler currently supports the following platform and architecture 
+combinations:
+  
+  * Windows (Windows XP or greater) - x86 and x86_64
+  * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l
+  * Mac OS X 10.04 or higher - x86, x86_64 and ppc64
+  
+In reality a lot more are supported, however they are not tested regularly.
+
+To build from source you will need:
+
+  * gcc 3.x or later recommended. Other alternatives which may work
+    are: clang, Visual C++, Intel's C++ compiler
+  * unzip
+  * git or wget
+
+If you are on a fairly modern *nix system, the following steps should work:
+
+```
+$ git clone git://github.com/Araq/Nimrod.git
+$ cd Nimrod
+$ cd build
+$ unzip csources.zip
+$ cd ..
+$ ./build.sh
+$ bin/nimrod c koch
+$ ./koch boot -d:release
+```
+
+The install script (``install.sh``) may then be used to install Nimrod, or you
+can simply add it to your PATH.
+
+The above steps can be performed on Windows in a similar fashion, the
+``build.bat`` and ``build64.bat`` (for x86_64 systems) are provided to be used
+instead of ``build.sh``.
+
+## Getting help
+A [forum](http://forum.nimrod-code.org/) is available if you have any questions,
+and you can also get help in the IRC channel
+on [Freenode](irc://irc.freenode.net/nimrod) in #nimrod.
+
+## License
+The compiler is licensed under the GPLv2 license, the standard library is
+licensed under the LGPL license with a linking exception so that you can link
+to it statically. This means that you can use any license for your own programs 
+developed with Nimrod, allowing you to create commercial applications.
+
+Read copying.txt for more details.
+
+Copyright (c) 2004-2013 Andreas Rumpf.
+All rights reserved.

readme.txt

@@ -1,21 +1,66 @@
-===========================================================
+# Nimrod Compiler
-          Nimrod Compiler
+This repo contains the Nimrod compiler, Nimrod's stdlib, tools and 
-===========================================================
+documentation.
 
-This is the **Nimrod Compiler**. Nimrod is a new statically typed, imperative 
+## Compiling
-programming language, that supports procedural, functional, object oriented and 
+Compiling the Nimrod compiler is quite straightforward. Because
-generic programming styles while remaining simple and efficient. A special 
+the Nimrod compiler itself is written in the Nimrod programming language
-feature that Nimrod inherited from Lisp is that Nimrod's abstract syntax tree
+the C source of an older version of the compiler are needed to bootstrap the
-(AST) is part of the specification - this allows a powerful macro system which 
+latest version. The C sources are however included with this repository under
-can be used to create domain specific languages.
+the build directory.
 
-*Nimrod* is a compiled, garbage-collected systems programming language 
+Pre-compiled snapshots of the compiler are also available on
-which has an excellent productivity/performance ratio. Nimrod's design 
+[Nimbuild](http://build.nimrod-code.org/). Your platform however may not 
-focuses on efficiency, expressiveness, elegance (in the order of 
+currently be built for.
-priority). 
 
-See the file ``install.txt`` for installation instructions. See the file
+The compiler currently supports the following platform and architecture 
-``doc/intern.txt`` for the internal documentation for developers.
+combinations:
   
-Copyright (c) 2004-2012 Andreas Rumpf.
+  * Windows (Windows XP or greater) - x86 and x86_64
+  * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l
+  * Mac OS X 10.04 or higher - x86, x86_64 and ppc64
+  
+In reality a lot more are supported, however they are not tested regularly.
+
+To build from source you will need:
+
+  * gcc 3.x or later recommended. Other alternatives which may work
+    are: clang, Visual C++, Intel's C++ compiler
+  * unzip
+  * git or wget
+
+If you are on a fairly modern *nix system, the following steps should work:
+
+```
+$ git clone git://github.com/Araq/Nimrod.git
+$ cd Nimrod
+$ cd build
+$ unzip csources.zip
+$ cd ..
+$ ./build.sh
+$ bin/nimrod c koch
+$ ./koch boot -d:release
+```
+
+The install script (``install.sh``) may then be used to install Nimrod, or you
+can simply add it to your PATH.
+
+The above steps can be performed on Windows in a similar fashion, the
+``build.bat`` and ``build64.bat`` (for x86_64 systems) are provided to be used
+instead of ``build.sh``.
+
+## Getting help
+A [forum](http://forum.nimrod-code.org/) is available if you have any questions,
+and you can also get help in the IRC channel
+on [Freenode](irc://irc.freenode.net/nimrod) in #nimrod.
+
+## License
+The compiler is licensed under the GPLv2 license, the standard library is
+licensed under the LGPL license with a linking exception so that you can link
+to it statically. This means that you can use any license for your own programs 
+developed with Nimrod, allowing you to create commercial applications.
+
+Read copying.txt for more details.
+
+Copyright (c) 2004-2013 Andreas Rumpf.
 All rights reserved.

tests/compile/mexporta.nim

@@ -0,0 +1,8 @@
+# module A
+import mexportb
+export mexportb.TMyObject, mexportb.xyz
+
+export mexportb.q
+
+proc `$`*(x: TMyObject): string = "my object"
+

tests/compile/mexportb.nim

@@ -0,0 +1,7 @@
+# module B
+type TMyObject* = object
+
+const xyz* = 13
+
+proc q*(x: int): int = 6
+proc q*(x: string): string = "8"

tests/compile/tclosure4.nim

@@ -0,0 +1,13 @@
+
+import json, tables
+
+proc run(json_params: TTable) =
+  let json_elems = json_params["files"].elems
+  # These fail compilation.
+  var files = map(json_elems, proc (x: PJsonNode): string = x.str)
+  #var files = json_elems.map do (x: PJsonNode) -> string: x.str
+  echo "Hey!"
+
+when isMainModule:
+  let text = """{"files": ["a", "b", "c"]}"""
+  run(toTable((text.parseJson).fields))

tests/compile/tclosurebug2.nim

@@ -0,0 +1,194 @@
+import hashes, math
+
+type
+  TSlotEnum = enum seEmpty, seFilled, seDeleted
+  TKeyValuePair[A, B] = tuple[slot: TSlotEnum, key: A, val: B]
+  TKeyValuePairSeq[A, B] = seq[TKeyValuePair[A, B]]
+
+  TOrderedKeyValuePair[A, B] = tuple[
+    slot: TSlotEnum, next: int, key: A, val: B]
+  TOrderedKeyValuePairSeq[A, B] = seq[TOrderedKeyValuePair[A, B]]
+  TOrderedTable*[A, B] = object ## table that remembers insertion order
+    data: TOrderedKeyValuePairSeq[A, B]
+    counter, first, last: int
+
+const
+  growthFactor = 2
+
+proc mustRehash(length, counter: int): bool {.inline.} =
+  assert(length > counter)
+  result = (length * 2 < counter * 3) or (length - counter < 4)
+
+proc nextTry(h, maxHash: THash): THash {.inline.} =
+  result = ((5 * h) + 1) and maxHash
+
+template rawGetImpl() {.dirty.} =
+  var h: THash = hash(key) and high(t.data) # start with real hash value
+  while t.data[h].slot != seEmpty:
+    if t.data[h].key == key and t.data[h].slot == seFilled:
+      return h
+    h = nextTry(h, high(t.data))
+  result = -1
+
+template rawInsertImpl() {.dirty.} =
+  var h: THash = hash(key) and high(data)
+  while data[h].slot == seFilled:
+    h = nextTry(h, high(data))
+  data[h].key = key
+  data[h].val = val
+  data[h].slot = seFilled
+
+template AddImpl() {.dirty.} =
+  if mustRehash(len(t.data), t.counter): Enlarge(t)
+  RawInsert(t, t.data, key, val)
+  inc(t.counter)
+
+template PutImpl() {.dirty.} =
+  var index = RawGet(t, key)
+  if index >= 0:
+    t.data[index].val = val
+  else:
+    AddImpl()
+
+proc len*[A, B](t: TOrderedTable[A, B]): int {.inline.} =
+  ## returns the number of keys in `t`.
+  result = t.counter
+
+template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} =
+  var h = t.first
+  while h >= 0:
+    var nxt = t.data[h].next
+    if t.data[h].slot == seFilled: yieldStmt
+    h = nxt
+
+iterator pairs*[A, B](t: TOrderedTable[A, B]): tuple[key: A, val: B] =
+  ## iterates over any (key, value) pair in the table `t` in insertion
+  ## order.
+  forAllOrderedPairs:
+    yield (t.data[h].key, t.data[h].val)
+
+iterator mpairs*[A, B](t: var TOrderedTable[A, B]): tuple[key: A, val: var B] =
+  ## iterates over any (key, value) pair in the table `t` in insertion
+  ## order. The values can be modified.
+  forAllOrderedPairs:
+    yield (t.data[h].key, t.data[h].val)
+
+iterator keys*[A, B](t: TOrderedTable[A, B]): A =
+  ## iterates over any key in the table `t` in insertion order.
+  forAllOrderedPairs:
+    yield t.data[h].key
+
+iterator values*[A, B](t: TOrderedTable[A, B]): B =
+  ## iterates over any value in the table `t` in insertion order.
+  forAllOrderedPairs:
+    yield t.data[h].val
+
+iterator mvalues*[A, B](t: var TOrderedTable[A, B]): var B =
+  ## iterates over any value in the table `t` in insertion order. The values
+  ## can be modified.
+  forAllOrderedPairs:
+    yield t.data[h].val
+
+proc RawGet[A, B](t: TOrderedTable[A, B], key: A): int =
+  rawGetImpl()
+
+proc `[]`*[A, B](t: TOrderedTable[A, B], key: A): B =
+  ## retrieves the value at ``t[key]``. If `key` is not in `t`,
+  ## default empty value for the type `B` is returned
+  ## and no exception is raised. One can check with ``hasKey`` whether the key
+  ## exists.
+  var index = RawGet(t, key)
+  if index >= 0: result = t.data[index].val
+
+proc mget*[A, B](t: var TOrderedTable[A, B], key: A): var B =
+  ## retrieves the value at ``t[key]``. The value can be modified.
+  ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised.
+  var index = RawGet(t, key)
+  if index >= 0: result = t.data[index].val
+  else: raise newException(EInvalidKey, "key not found: " & $key)
+
+proc hasKey*[A, B](t: TOrderedTable[A, B], key: A): bool =
+  ## returns true iff `key` is in the table `t`.
+  result = rawGet(t, key) >= 0
+
+proc RawInsert[A, B](t: var TOrderedTable[A, B], 
+                     data: var TOrderedKeyValuePairSeq[A, B],
+                     key: A, val: B) =
+  rawInsertImpl()
+  data[h].next = -1
+  if t.first < 0: t.first = h
+  if t.last >= 0: data[t.last].next = h
+  t.last = h
+
+proc Enlarge[A, B](t: var TOrderedTable[A, B]) =
+  var n: TOrderedKeyValuePairSeq[A, B]
+  newSeq(n, len(t.data) * growthFactor)
+  var h = t.first
+  t.first = -1
+  t.last = -1
+  while h >= 0:
+    var nxt = t.data[h].next
+    if t.data[h].slot == seFilled: 
+      RawInsert(t, n, t.data[h].key, t.data[h].val)
+    h = nxt
+  swap(t.data, n)
+
+proc `[]=`*[A, B](t: var TOrderedTable[A, B], key: A, val: B) =
+  ## puts a (key, value)-pair into `t`.
+  putImpl()
+
+proc add*[A, B](t: var TOrderedTable[A, B], key: A, val: B) =
+  ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
+  AddImpl()
+
+proc initOrderedTable*[A, B](initialSize=64): TOrderedTable[A, B] =
+  ## creates a new ordered hash table that is empty. `initialSize` needs to be
+  ## a power of two.
+  assert isPowerOfTwo(initialSize)
+  result.counter = 0
+  result.first = -1
+  result.last = -1
+  newSeq(result.data, initialSize)
+
+proc toOrderedTable*[A, B](pairs: openarray[tuple[key: A, 
+                           val: B]]): TOrderedTable[A, B] =
+  ## creates a new ordered hash table that contains the given `pairs`.
+  result = initOrderedTable[A, B](nextPowerOfTwo(pairs.len+10))
+  for key, val in items(pairs): result[key] = val
+
+proc sort*[A, B](t: var TOrderedTable[A,B], 
+                 cmp: proc (x, y: tuple[key: A, val: B]): int {.closure.}) =
+  ## sorts the ordered table so that the entry with the highest counter comes
+  ## first. This is destructive (with the advantage of being efficient)! 
+  ## You must not modify `t` afterwards!
+  ## You can use the iterators `pairs`,  `keys`, and `values` to iterate over
+  ## `t` in the sorted order.
+
+  # we use shellsort here; fast enough and simple
+  var h = 1
+  while true:
+    h = 3 * h + 1
+    if h >= high(t.data): break
+  while true:
+    h = h div 3
+    for i in countup(h, high(t.data)):
+      var j = i
+      #echo(t.data.len, " ", j, " - ", h)
+      #echo(repr(t.data[j-h]))
+      proc rawCmp(x, y: TOrderedKeyValuePair[A, B]): int =
+        if x.slot in {seEmpty, seDeleted} and y.slot in {seEmpty, seDeleted}:
+          return 0
+        elif x.slot in {seEmpty, seDeleted}:
+          return -1
+        elif y.slot in {seEmpty, seDeleted}:
+          return 1
+        else:
+          let item1 = (x.key, x.val)
+          let item2 = (y.key, y.val)
+          return cmp(item1, item2)
+      
+      while rawCmp(t.data[j-h], t.data[j]) <= 0:
+        swap(t.data[j], t.data[j-h])
+        j = j-h
+        if j < h: break
+    if h == 1: break

tests/compile/tcolonisproc.nim

@@ -0,0 +1,12 @@
+
+proc p(a, b: int, c: proc ()) =
+  c()
+
+ 
+p(1, 3): 
+  echo 1
+  echo 3
+    
+p(1, 1, proc() =
+  echo 1
+  echo 2)

tests/compile/teffects1.nim

@@ -0,0 +1,17 @@
+
+type
+  PMenu = ref object
+  PMenuItem = ref object
+
+proc createMenuItem*(menu: PMenu, label: string, 
+                     action: proc (i: PMenuItem, p: pointer) {.cdecl.}) = nil
+
+var s: PMenu
+createMenuItem(s, "Go to definition...",
+      proc (i: PMenuItem, p: pointer) {.cdecl.} =
+        try:
+          echo(i.repr)
+        except EInvalidValue:
+          echo("blah")
+)
+