Merge branch 'devel' of github.com:Araq/Nimrod into seq_toString

compiler/ast.nim

@@ -62,7 +62,7 @@ type
     nkTripleStrLit,       # a triple string literal """
     nkNilLit,             # the nil literal
                           # end of atoms
-    nkMetaNode,           # difficult to explain; represents itself
+    nkMetaNode_Obsolete,  # difficult to explain; represents itself
                           # (used for macros)
     nkDotCall,            # used to temporarily flag a nkCall node; 
                           # this is used
@@ -188,6 +188,10 @@ type
     nkStmtListType,       # a statement list ending in a type; for macros
     nkBlockType,          # a statement block ending in a type; for macros
                           # types as syntactic trees:
+
+    nkWith,               # distinct with `foo`
+    nkWithout,            # distinct without `foo`
+    
     nkTypeOfExpr,         # type(1+2)
     nkObjectTy,           # object body
     nkTupleTy,            # tuple body
@@ -383,6 +387,10 @@ type
       # sons[1]: field type
       # .n: nkDotExpr storing the field name
     
+static:
+  # remind us when TTypeKind stops to fit in a single 64-bit word
+  assert TTypeKind.high.ord <= 63
+
 const
   tyPureObject* = tyTuple
   GcTypeKinds* = {tyRef, tySequence, tyString}
@@ -394,7 +402,7 @@ const
                     tyUserTypeClass, tyUserTypeClassInst,
                     tyAnd, tyOr, tyNot, tyAnything}
 
-  tyMetaTypes* = {tyGenericParam, tyTypeDesc, tyStatic, tyExpr} + tyTypeClasses
+  tyMetaTypes* = {tyGenericParam, tyTypeDesc, tyExpr} + tyTypeClasses
  
 type
   TTypeKinds* = set[TTypeKind]
@@ -409,12 +417,14 @@ type
                 # efficiency
     nfTransf,   # node has been transformed
     nfSem       # node has been checked for semantics
-    nfDelegate  # the call can use a delegator
+    nfDotField  # the call can use a dot operator
+    nfDotSetter # the call can use a setter dot operarator
+    nfExplicitCall # x.y() was used instead of x.y
     nfExprCall  # this is an attempt to call a regular expression
     nfIsRef     # this node is a 'ref' node; used for the VM
  
   TNodeFlags* = set[TNodeFlag]
-  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 23)
+  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 28)
     tfVarargs,        # procedure has C styled varargs
     tfNoSideEffect,   # procedure type does not allow side effects
     tfFinal,          # is the object final?
@@ -426,8 +436,11 @@ type
     tfFromGeneric,    # type is an instantiation of a generic; this is needed
                       # because for instantiations of objects, structural
                       # type equality has to be used
-    tfUnresolved,     # marks unresolved typedesc params: e.g.
+    tfUnresolved,     # marks unresolved typedesc/static params: e.g.
                       # proc foo(T: typedesc, list: seq[T]): var T
+                      # proc foo(L: static[int]): array[L, int]
+                      # can be attached to ranges to indicate that the range
+                      # depends on unresolved static params.
     tfRetType,        # marks return types in proc (used to detect type classes 
                       # used as return types for return type inference)
     tfCapturesEnv,    # whether proc really captures some environment
@@ -443,9 +456,16 @@ type
     tfHasMeta,        # type contains "wildcard" sub-types such as generic params
                       # or other type classes
     tfHasGCedMem,     # type contains GC'ed memory
+    tfPacked
     tfHasStatic
     tfGenericTypeParam
     tfImplicitTypeParam
+    tfWildcard        # consider a proc like foo[T, I](x: Type[T, I])
+                      # T and I here can bind to both typedesc and static types
+                      # before this is determined, we'll consider them to be a
+                      # wildcard type.
+    tfGuarded         # guarded pointer
+    tfBorrowDot       # distinct type borrows '.'
 
   TTypeFlags* = set[TTypeFlag]
 
@@ -467,7 +487,8 @@ type
     skResult,             # special 'result' variable
     skProc,               # a proc
     skMethod,             # a method
-    skIterator,           # an iterator
+    skIterator,           # an inline iterator
+    skClosureIterator,    # a resumable closure iterator
     skConverter,          # a type converter
     skMacro,              # a macro
     skTemplate,           # a template; currently also misused for user-defined
@@ -479,12 +500,15 @@ type
     skStub,               # symbol is a stub and not yet loaded from the ROD
                           # file (it is loaded on demand, which may
                           # mean: never)
+    skPackage             # symbol is a package (used for canonicalization)
   TSymKinds* = set[TSymKind]
 
 const
-  routineKinds* = {skProc, skMethod, skIterator, skConverter,
+  routineKinds* = {skProc, skMethod, skIterator, skClosureIterator,
-    skMacro, skTemplate}
+                   skConverter, skMacro, skTemplate}
   tfIncompleteStruct* = tfVarargs
+  tfUncheckedArray* = tfVarargs
+  tfUnion* = tfNoSideEffect
   skError* = skUnknown
   
   # type flags that are essential for type equality:
@@ -687,7 +711,7 @@ type
   TSym* {.acyclic.} = object of TIdObj
     # proc and type instantiations are cached in the generic symbol
     case kind*: TSymKind
-    of skType:
+    of skType, skGenericParam:
       typeInstCache*: seq[PType]
       typScope*: PScope
     of routineKinds:
@@ -812,13 +836,16 @@ type
     counter*: int
     data*: TObjectSeq
 
+  TImplication* = enum
+    impUnknown, impNo, impYes
+
 # BUGFIX: a module is overloadable so that a proc can have the
 # same name as an imported module. This is necessary because of
 # the poor naming choices in the standard library.
 
 const 
-  OverloadableSyms* = {skProc, skMethod, skIterator, skConverter,
+  OverloadableSyms* = {skProc, skMethod, skIterator, skClosureIterator,
-    skModule, skTemplate, skMacro}
+    skConverter, skModule, skTemplate, skMacro}
 
   GenericTypes*: TTypeKinds = {tyGenericInvokation, tyGenericBody, 
     tyGenericParam}
@@ -840,10 +867,12 @@ const
                                     tyTuple, tySequence}
   NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tySequence,
     tyProc, tyString, tyError}
-  ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType, skIterator, 
+  ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType,
+    skIterator, skClosureIterator,
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
-                                      nfAllConst, nfDelegate, nfIsRef}
+                                      nfDotSetter, nfDotField,
+                                      nfAllConst,nfIsRef}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -857,6 +886,7 @@ const
   nkCallKinds* = {nkCall, nkInfix, nkPrefix, nkPostfix,
                   nkCommand, nkCallStrLit, nkHiddenCallConv}
 
+  nkLiterals* = {nkCharLit..nkTripleStrLit}
   nkLambdaKinds* = {nkLambda, nkDo}
   declarativeDefs* = {nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef}
   procDefs* = nkLambdaKinds + declarativeDefs
@@ -865,7 +895,10 @@ const
   nkStrKinds* = {nkStrLit..nkTripleStrLit}
 
   skLocalVars* = {skVar, skLet, skForVar, skParam, skResult}
-  skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skMethod, skConverter}
+  skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skClosureIterator,
+                  skMethod, skConverter}
+
+  skIterators* = {skIterator, skClosureIterator}
 
   lfFullExternalName* = lfParamCopy # \
     # only used when 'gCmd == cmdPretty': Indicates that the symbol has been
@@ -946,7 +979,9 @@ var emptyNode* = newNode(nkEmpty)
 # There is a single empty node that is shared! Do not overwrite it!
 
 proc isMetaType*(t: PType): bool =
-  return t.kind in tyMetaTypes or tfHasMeta in t.flags
+  return t.kind in tyMetaTypes or
+         (t.kind == tyStatic and t.n == nil) or
+         tfHasMeta in t.flags
 
 proc linkTo*(t: PType, s: PSym): PType {.discardable.} =
   t.sym = s
@@ -1008,7 +1043,7 @@ proc discardSons(father: PNode) =
   father.sons = nil
 
 when defined(useNodeIds):
-  const nodeIdToDebug = 612777 # 612794
+  const nodeIdToDebug* = 482228 # 612794
   #612840 # 612905 # 614635 # 614637 # 614641
   # 423408
   #429107 # 430443 # 441048 # 441090 # 441153
@@ -1044,6 +1079,10 @@ proc newStrNode(kind: TNodeKind, strVal: string): PNode =
   result = newNode(kind)
   result.strVal = strVal
 
+proc withInfo*(n: PNode, info: TLineInfo): PNode =
+  n.info = info
+  return n
+
 proc newIdentNode(ident: PIdent, info: TLineInfo): PNode = 
   result = newNode(nkIdent)
   result.ident = ident
@@ -1105,10 +1144,6 @@ proc newNodeIT(kind: TNodeKind, info: TLineInfo, typ: PType): PNode =
   result.info = info
   result.typ = typ
 
-proc newMetaNodeIT*(tree: PNode, info: TLineInfo, typ: PType): PNode =
-  result = newNodeIT(nkMetaNode, info, typ)
-  result.add(tree)
-
 var emptyParams = newNode(nkFormalParams)
 emptyParams.addSon(emptyNode)
 
@@ -1276,7 +1311,7 @@ proc skipTypes*(t: PType, kinds: TTypeKinds): PType =
 proc propagateToOwner*(owner, elem: PType) =
   const HaveTheirOwnEmpty = {tySequence, tySet}
   owner.flags = owner.flags + (elem.flags * {tfHasShared, tfHasMeta,
-                                             tfHasStatic, tfHasGCedMem})
+                                             tfHasGCedMem})
   if tfNotNil in elem.flags:
     if owner.kind in {tyGenericInst, tyGenericBody, tyGenericInvokation}:
       owner.flags.incl tfNotNil
@@ -1290,12 +1325,9 @@ proc propagateToOwner*(owner, elem: PType) =
   if tfShared in elem.flags:
     owner.flags.incl tfHasShared
 
-  if elem.kind in tyMetaTypes:
+  if elem.isMetaType:
     owner.flags.incl tfHasMeta
 
-  if elem.kind == tyStatic:
-    owner.flags.incl tfHasStatic
-
   if elem.kind in {tyString, tyRef, tySequence} or
       elem.kind == tyProc and elem.callConv == ccClosure:
     owner.flags.incl tfHasGCedMem
@@ -1475,8 +1507,7 @@ proc originatingModule*(s: PSym): PSym =
   while result.kind != skModule: result = result.owner
 
 proc isRoutine*(s: PSym): bool {.inline.} =
-  result = s.kind in {skProc, skTemplate, skMacro, skIterator, skMethod,
+  result = s.kind in skProcKinds
-                      skConverter}
 
 proc hasPattern*(s: PSym): bool {.inline.} =
   result = isRoutine(s) and s.ast.sons[patternPos].kind != nkEmpty
@@ -1484,6 +1515,9 @@ proc hasPattern*(s: PSym): bool {.inline.} =
 iterator items*(n: PNode): PNode =
   for i in 0.. <n.len: yield n.sons[i]
 
+iterator pairs*(n: PNode): tuple[i: int, n: PNode] =
+  for i in 0.. <n.len: yield (i, n.sons[i])
+
 proc isAtom*(n: PNode): bool {.inline.} =
   result = n.kind >= nkNone and n.kind <= nkNilLit
 

compiler/astalgo.nim

@@ -157,6 +157,12 @@ proc leValue*(a, b: PNode): bool =
     #InternalError(a.info, "leValue")
     discard
 
+proc weakLeValue*(a, b: PNode): TImplication =
+  if a.kind notin nkLiterals or b.kind notin nkLiterals:
+    result = impUnknown
+  else:
+    result = if leValue(a, b): impYes else: impNo
+
 proc lookupInRecord(n: PNode, field: PIdent): PSym = 
   result = nil
   case n.kind
@@ -337,7 +343,10 @@ proc treeToYamlAux(n: PNode, marker: var TIntSet, indent: int,
         appf(result, ",$N$1\"floatVal\": $2", 
             [istr, toRope(n.floatVal.toStrMaxPrecision)])
       of nkStrLit..nkTripleStrLit: 
-        appf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
+        if n.strVal.isNil:
+          appf(result, ",$N$1\"strVal\": null", [istr])
+        else:
+          appf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
       of nkSym: 
         appf(result, ",$N$1\"sym\": $2", 
              [istr, symToYamlAux(n.sym, marker, indent + 2, maxRecDepth)])
@@ -407,7 +416,10 @@ proc debugTree(n: PNode, indent: int, maxRecDepth: int): PRope =
         appf(result, ",$N$1\"floatVal\": $2", 
             [istr, toRope(n.floatVal.toStrMaxPrecision)])
       of nkStrLit..nkTripleStrLit: 
-        appf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
+        if n.strVal.isNil:
+          appf(result, ",$N$1\"strVal\": null", [istr])
+        else:
+          appf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
       of nkSym: 
         appf(result, ",$N$1\"sym\": $2_$3", 
             [istr, toRope(n.sym.name.s), toRope(n.sym.id)])

compiler/c2nim/cparse.nim

@@ -540,7 +540,7 @@ proc typeAtom(p: var TParser): PNode =
       if p.tok.s == "unsigned":
         isUnsigned = true
       elif p.tok.s == "signed" or p.tok.s == "int":
-        nil
+        discard
       else:
         add(x, p.tok.s)
       getTok(p, nil)
@@ -624,10 +624,11 @@ proc parseTypeSuffix(p: var TParser, typ: PNode): PNode =
         # array type:
         result = newNodeP(nkBracketExpr, p)
         addSon(result, newIdentNodeP("array", p))
-        var r = newNodeP(nkRange, p)
+        #var r = newNodeP(nkRange, p)
-        addSon(r, newIntNodeP(nkIntLit, 0, p))
+        #addSon(r, newIntNodeP(nkIntLit, 0, p))
-        addSon(r, newBinary("-", index, newIntNodeP(nkIntLit, 1, p), p))
+        #addSon(r, newBinary("-", index, newIntNodeP(nkIntLit, 1, p), p))
-        addSon(result, r)
+        #addSon(result, r)
+        addSon(result, index)
         addSon(result, tmp)
       else:
         # pointer type:
@@ -681,11 +682,6 @@ proc parseField(p: var TParser, kind: TNodeKind): PNode =
     else: result = mangledIdent(p.tok.s, p)
     getTok(p, result)
 
-proc takeOnlyFirstField(p: TParser, isUnion: bool): bool =
-  # if we generate an interface to a header file, *all* fields can be 
-  # generated:
-  result = isUnion and p.options.header.len == 0
-
 proc parseStructBody(p: var TParser, isUnion: bool,
                      kind: TNodeKind = nkRecList): PNode =
   result = newNodeP(kind, p)
@@ -698,8 +694,7 @@ proc parseStructBody(p: var TParser, isUnion: bool,
       var i = parseField(p, kind)
       t = parseTypeSuffix(p, t)
       addSon(def, i, t, ast.emptyNode)
-      if not takeOnlyFirstField(p, isUnion) or sonsLen(result) < 1: 
+      addSon(result, def)
-        addSon(result, def)
       if p.tok.xkind != pxComma: break
       getTok(p, def)
     eat(p, pxSemicolon, lastSon(result))
@@ -710,11 +705,12 @@ proc structPragmas(p: TParser, name: PNode, origName: string): PNode =
   result = newNodeP(nkPragmaExpr, p)
   addSon(result, exportSym(p, name, origName))
   var pragmas = newNodeP(nkPragma, p)
-  addSon(pragmas, newIdentNodeP("pure", p), newIdentNodeP("final", p))
+  #addSon(pragmas, newIdentNodeP("pure", p), newIdentNodeP("final", p))
   if p.options.header.len > 0:
     addSon(pragmas, newIdentStrLitPair("importc", origName, p),
                     newIdentStrLitPair("header", p.options.header, p))
-  addSon(result, pragmas)
+  if pragmas.len > 0: addSon(result, pragmas)
+  else: addSon(result, ast.emptyNode)
 
 proc enumPragmas(p: TParser, name: PNode): PNode =
   result = newNodeP(nkPragmaExpr, p)
@@ -728,7 +724,11 @@ proc enumPragmas(p: TParser, name: PNode): PNode =
 
 proc parseStruct(p: var TParser, isUnion: bool): PNode =
   result = newNodeP(nkObjectTy, p)
-  addSon(result, ast.emptyNode, ast.emptyNode) # no pragmas, no inheritance 
+  var pragmas = ast.emptyNode
+  if isUnion:
+    pragmas = newNodeP(nkPragma, p)
+    addSon(pragmas, newIdentNodeP("union", p))
+  addSon(result, pragmas, ast.emptyNode) # no inheritance 
   if p.tok.xkind == pxCurlyLe:
     addSon(result, parseStructBody(p, isUnion))
   else: 
@@ -746,7 +746,7 @@ proc directDeclarator(p: var TParser, a: PNode, ident: ptr PNode): PNode =
       result = declarator(p, a, ident)
       eat(p, pxParRi, result)
   else:
-    nil
+    discard
   return parseTypeSuffix(p, a)
 
 proc declarator(p: var TParser, a: PNode, ident: ptr PNode): PNode =
@@ -1165,7 +1165,7 @@ proc enumSpecifier(p: var TParser): PNode =
 
 proc setBaseFlags(n: PNode, base: TNumericalBase) = 
   case base
-  of base10: nil
+  of base10: discard
   of base2: incl(n.flags, nfBase2)
   of base8: incl(n.flags, nfBase8)
   of base16: incl(n.flags, nfBase16)
@@ -1279,7 +1279,8 @@ proc leftBindingPower(p : var TParser, tok : ref TToken) : int =
   of pxComma:
     return 10
     # throw == 20
-  of pxAsgn, pxPlusAsgn, pxMinusAsgn, pxStarAsgn, pxSlashAsgn, pxModAsgn, pxShlAsgn, pxShrAsgn, pxAmpAsgn, pxHatAsgn, pxBarAsgn:
+  of pxAsgn, pxPlusAsgn, pxMinusAsgn, pxStarAsgn, pxSlashAsgn, pxModAsgn, 
+     pxShlAsgn, pxShrAsgn, pxAmpAsgn, pxHatAsgn, pxBarAsgn:
     return 30
   of pxConditional:
     return 40
@@ -1686,7 +1687,7 @@ proc switchStatement(p: var TParser): PNode =
       break
     of "case", "default":
       break
-    else: nil
+    else: discard
     addSon(result, statement(p))
   if sonsLen(result) == 0:
     # translate empty statement list to Nimrod's ``nil`` statement

compiler/c2nim/cpp.nim

@@ -103,7 +103,7 @@ proc parseDefBody(p: var TParser, m: var TMacro, params: seq[string]) =
       m.body.add(tok)
     of pxDirConc: 
       # just ignore this token: this implements token merging correctly
-      nil
+      discard
     else:
       m.body.add(p.tok)
     # we do not want macro expansion here:
@@ -166,7 +166,7 @@ proc parseStmtList(p: var TParser): PNode =
     of pxDirectiveParLe, pxDirective: 
       case p.tok.s
       of "else", "endif", "elif": break
-    else: nil
+    else: discard
     addSon(result, statement(p))
   
 proc eatEndif(p: var TParser) =

compiler/canonicalizer.nim

@@ -0,0 +1,416 @@
+#
+#
+#           The Nimrod Compiler
+#        (c) Copyright 2014 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements the canonalization for the various caching mechanisms.
+
+import strutils, db_sqlite, md5
+
+var db: TDbConn
+
+# We *hash* the relevant information into 128 bit hashes. This should be good
+# enough to prevent any collisions.
+
+type
+  TUid = distinct MD5Digest
+
+# For name mangling we encode these hashes via a variant of base64 (called
+# 'base64a') and prepend the *primary* identifier to ease the debugging pain.
+# So a signature like:
+#
+#   proc gABI(c: PCtx; n: PNode; opc: TOpcode; a, b: TRegister; imm: BiggestInt)
+#
+# is mangled into:
+#   gABI_MTdmOWY5MTQ1MDcyNGQ3ZA
+#
+# This is a good compromise between correctness and brevity. ;-)
+
+const
+  cb64 = [
+    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
+    "O", "P", "Q", "R", "S", "T" "U", "V", "W", "X", "Y", "Z", 
+    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", 
+    "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", 
+    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+    "_A", "_B"]
+
+proc toBase64a(s: cstring, len: int): string =
+  ## encodes `s` into base64 representation. After `lineLen` characters, a 
+  ## `newline` is added.
+  result = newStringOfCap(((len + 2) div 3) * 4)
+  var i = 0
+  while i < s.len - 2:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    let c = ord(s[i+2])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2) or ((c and 0xC0) shr 6)]
+    result.add cb64[c and 0x3F]
+    inc(i, 3)
+  if i < s.len-1:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2)]
+  elif i < s.len:
+    let a = ord(s[i])
+    result.add cb64[a shr 2]
+    result.add cb64[(a and 3) shl 4]
+
+proc toBase64a(u: TUid): string = toBase64a(cast[cstring](u), sizeof(u))
+
+proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len)
+
+proc hashSym(c: var MD5Context, s: PSym) =
+  if sfAnon in s.flags or s.kind == skGenericParam:
+    c &= ":anon"
+  else:
+    var it = s.owner
+    while it != nil: 
+      hashSym(c, it)
+      c &= "."
+      it = s.owner
+    c &= s.name.s
+
+proc hashTree(c: var MD5Context, n: PNode) =
+  if n == nil:
+    c &= "\255"
+    return
+  var k = n.kind
+  md5Update(c, cast[cstring](addr(k)), 1)
+  # we really must not hash line information. 'n.typ' is debatable but
+  # shouldn't be necessary for now and avoids potential infinite recursions.
+  case n.kind
+  of nkEmpty, nkNilLit, nkType: discard
+  of nkIdent:
+    c &= n.ident.s
+  of nkSym:
+    hashSym(c, n.sym)
+  of nkCharLit..nkUInt64Lit:
+    var v = n.intVal
+    md5Update(c, cast[cstring](addr(v)), sizeof(v))
+  of nkFloatLit..nkFloat64Lit:
+    var v = n.floatVal
+    md5Update(c, cast[cstring](addr(v)), sizeof(v))
+  of nkStrLit..nkTripleStrLit:
+    c &= n.strVal
+  else:
+    for i in 0.. <n.len: hashTree(c, n.sons[i])
+
+proc hashType(c: var MD5Context, t: PType) =
+  # modelled after 'typeToString'
+  if t == nil: 
+    c &= "\254"
+    return
+
+  var k = t.kind
+  md5Update(c, cast[cstring](addr(k)), 1)
+  
+  if t.sym != nil and sfAnon notin t.sym.flags:
+    # t.n for literals, but not for e.g. objects!
+    if t.kind in {tyFloat, tyInt}: c.hashNode(t.n)
+    c.hashSym(t.sym)
+    
+  case t.kind
+  of tyGenericBody, tyGenericInst, tyGenericInvokation:
+    for i in countup(0, sonsLen(t) -1 -ord(t.kind != tyGenericInvokation)):
+      c.hashType t.sons[i]
+  of tyUserTypeClass:
+    internalAssert t.sym != nil and t.sym.owner != nil
+    c &= t.sym.owner.name.s
+  of tyUserTypeClassInst:
+    let body = t.base
+    c.hashSym body.sym
+    for i in countup(1, sonsLen(t) - 2):
+      c.hashType t.sons[i]
+  of tyFromExpr, tyFieldAccessor:
+    c.hashTree(t.n)
+  of tyArrayConstr:
+    c.hashTree(t.sons[0].n)
+    c.hashType(t.sons[1])
+  of tyTuple: 
+    if t.n != nil:
+      assert(sonsLen(t.n) == sonsLen(t))
+      for i in countup(0, sonsLen(t.n) - 1): 
+        assert(t.n.sons[i].kind == nkSym)
+        c &= t.n.sons[i].sym.name.s
+        c &= ":"
+        c.hashType(t.sons[i])
+        c &= ","
+    else:
+      for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i]
+  of tyRange:
+    c.hashTree(t.n)
+    c.hashType(t.sons[0])
+  of tyProc:
+    c &= (if tfIterator in t.flags: "iterator " else: "proc ")
+    for i in 0.. <t.len: c.hashType(t.sons[i])
+    md5Update(c, cast[cstring](addr(t.callConv)), 1)
+
+    if tfNoSideEffect in t.flags: c &= ".noSideEffect"
+    if tfThread in t.flags: c &= ".thread"
+  else:
+    for i in 0.. <t.len: c.hashType(t.sons[i])
+  if tfShared in t.flags: c &= "shared"
+  if tfNotNil in t.flags: c &= "not nil"
+
+proc canonConst(n: PNode): TUid =
+  var c: MD5Context
+  md5Init(c)
+  c.hashTree(n)
+  c.hashType(n.typ)
+  md5Final(c, MD5Digest(result))
+
+proc canonSym(s: PSym): TUid =
+  var c: MD5Context
+  md5Init(c)
+  c.hashSym(s)
+  md5Final(c, MD5Digest(result))
+
+proc pushType(w: PRodWriter, t: PType) =
+  # check so that the stack does not grow too large:
+  if iiTableGet(w.index.tab, t.id) == InvalidKey:
+    w.tstack.add(t)
+
+proc pushSym(w: PRodWriter, s: PSym) =
+  # check so that the stack does not grow too large:
+  if iiTableGet(w.index.tab, s.id) == InvalidKey:
+    w.sstack.add(s)
+
+proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, 
+                result: var string) = 
+  if n == nil: 
+    # nil nodes have to be stored too:
+    result.add("()")
+    return
+  result.add('(')
+  encodeVInt(ord(n.kind), result) 
+  # we do not write comments for now
+  # Line information takes easily 20% or more of the filesize! Therefore we
+  # omit line information if it is the same as the father's line information:
+  if fInfo.fileIndex != n.info.fileIndex: 
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+    result.add(',')
+    encodeVInt(fileIdx(w, toFilename(n.info)), result)
+  elif fInfo.line != n.info.line:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+  elif fInfo.col != n.info.col:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+  var f = n.flags * PersistentNodeFlags
+  if f != {}: 
+    result.add('$')
+    encodeVInt(cast[int32](f), result)
+  if n.typ != nil:
+    result.add('^')
+    encodeVInt(n.typ.id, result)
+    pushType(w, n.typ)
+  case n.kind
+  of nkCharLit..nkInt64Lit: 
+    if n.intVal != 0:
+      result.add('!')
+      encodeVBiggestInt(n.intVal, result)
+  of nkFloatLit..nkFloat64Lit: 
+    if n.floatVal != 0.0: 
+      result.add('!')
+      encodeStr($n.floatVal, result)
+  of nkStrLit..nkTripleStrLit:
+    if n.strVal != "": 
+      result.add('!')
+      encodeStr(n.strVal, result)
+  of nkIdent:
+    result.add('!')
+    encodeStr(n.ident.s, result)
+  of nkSym:
+    result.add('!')
+    encodeVInt(n.sym.id, result)
+    pushSym(w, n.sym)
+  else:
+    for i in countup(0, sonsLen(n) - 1): 
+      encodeNode(w, n.info, n.sons[i], result)
+  add(result, ')')
+
+proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) = 
+  var oldLen = result.len
+  result.add('<')
+  if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
+  if loc.s != low(loc.s): 
+    add(result, '*')
+    encodeVInt(ord(loc.s), result)
+  if loc.flags != {}: 
+    add(result, '$')
+    encodeVInt(cast[int32](loc.flags), result)
+  if loc.t != nil:
+    add(result, '^')
+    encodeVInt(cast[int32](loc.t.id), result)
+    pushType(w, loc.t)
+  if loc.r != nil: 
+    add(result, '!')
+    encodeStr(ropeToStr(loc.r), result)
+  if loc.a != 0: 
+    add(result, '?')
+    encodeVInt(loc.a, result)
+  if oldLen + 1 == result.len:
+    # no data was necessary, so remove the '<' again:
+    setLen(result, oldLen)
+  else:
+    add(result, '>')
+  
+proc encodeType(w: PRodWriter, t: PType, result: var string) = 
+  if t == nil: 
+    # nil nodes have to be stored too:
+    result.add("[]")
+    return
+  # we need no surrounding [] here because the type is in a line of its own
+  if t.kind == tyForward: internalError("encodeType: tyForward")
+  # for the new rodfile viewer we use a preceeding [ so that the data section
+  # can easily be disambiguated:
+  add(result, '[')
+  encodeVInt(ord(t.kind), result)
+  add(result, '+')
+  encodeVInt(t.id, result)
+  if t.n != nil: 
+    encodeNode(w, unknownLineInfo(), t.n, result)
+  if t.flags != {}: 
+    add(result, '$')
+    encodeVInt(cast[int32](t.flags), result)
+  if t.callConv != low(t.callConv): 
+    add(result, '?')
+    encodeVInt(ord(t.callConv), result)
+  if t.owner != nil: 
+    add(result, '*')
+    encodeVInt(t.owner.id, result)
+    pushSym(w, t.owner)
+  if t.sym != nil: 
+    add(result, '&')
+    encodeVInt(t.sym.id, result)
+    pushSym(w, t.sym)
+  if t.size != - 1: 
+    add(result, '/')
+    encodeVBiggestInt(t.size, result)
+  if t.align != 2: 
+    add(result, '=')
+    encodeVInt(t.align, result)
+  encodeLoc(w, t.loc, result)
+  for i in countup(0, sonsLen(t) - 1): 
+    if t.sons[i] == nil: 
+      add(result, "^()")
+    else: 
+      add(result, '^') 
+      encodeVInt(t.sons[i].id, result)
+      pushType(w, t.sons[i])
+
+proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) = 
+  add(result, '|')
+  encodeVInt(ord(lib.kind), result)
+  add(result, '|')
+  encodeStr(ropeToStr(lib.name), result)
+  add(result, '|')
+  encodeNode(w, info, lib.path, result)
+
+proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
+  if s == nil:
+    # nil nodes have to be stored too:
+    result.add("{}")
+    return
+  # we need no surrounding {} here because the symbol is in a line of its own
+  encodeVInt(ord(s.kind), result)
+  result.add('+')
+  encodeVInt(s.id, result)
+  result.add('&')
+  encodeStr(s.name.s, result)
+  if s.typ != nil:
+    result.add('^')
+    encodeVInt(s.typ.id, result)
+    pushType(w, s.typ)
+  result.add('?')
+  if s.info.col != -1'i16: encodeVInt(s.info.col, result)
+  result.add(',')
+  if s.info.line != -1'i16: encodeVInt(s.info.line, result)
+  result.add(',')
+  encodeVInt(fileIdx(w, toFilename(s.info)), result)
+  if s.owner != nil:
+    result.add('*')
+    encodeVInt(s.owner.id, result)
+    pushSym(w, s.owner)
+  if s.flags != {}:
+    result.add('$')
+    encodeVInt(cast[int32](s.flags), result)
+  if s.magic != mNone:
+    result.add('@')
+    encodeVInt(ord(s.magic), result)
+  if s.options != w.options: 
+    result.add('!')
+    encodeVInt(cast[int32](s.options), result)
+  if s.position != 0: 
+    result.add('%')
+    encodeVInt(s.position, result)
+  if s.offset != - 1:
+    result.add('`')
+    encodeVInt(s.offset, result)
+  encodeLoc(w, s.loc, result)
+  if s.annex != nil: encodeLib(w, s.annex, s.info, result)
+  if s.constraint != nil:
+    add(result, '#')
+    encodeNode(w, unknownLineInfo(), s.constraint, result)
+  # lazy loading will soon reload the ast lazily, so the ast needs to be
+  # the last entry of a symbol:
+  if s.ast != nil:
+    # we used to attempt to save space here by only storing a dummy AST if
+    # it is not necessary, but Nimrod's heavy compile-time evaluation features
+    # make that unfeasible nowadays:
+    encodeNode(w, s.info, s.ast, result)
+
+
+proc createDb() =
+  db.exec(sql"""
+    create table if not exists Module(
+      id integer primary key,
+      name varchar(256) not null,
+      fullpath varchar(256) not null,
+      interfHash varchar(256) not null,
+      fullHash varchar(256) not null,
+      
+      created timestamp not null default (DATETIME('now')),
+    );""")
+
+  db.exec(sql"""
+    create table if not exists Symbol(
+      id integer primary key,
+      module integer not null,
+      name varchar(max) not null,
+      data varchar(max) not null,
+      created timestamp not null default (DATETIME('now')),
+
+      foreign key (module) references module(id)
+    );""")
+    
+  db.exec(sql"""
+    create table if not exists Type(
+      id integer primary key,
+      module integer not null,
+      name varchar(max) not null,
+      data varchar(max) not null,
+      created timestamp not null default (DATETIME('now')),
+
+      foreign key (module) references module(id)
+    );""")
+
+
+  #db.exec(sql"""
+  #  --create unique index if not exists TsstNameIx on TestResult(name);
+  #  """, [])
+

compiler/ccgexprs.nim

@@ -593,7 +593,7 @@ proc unaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
 
 proc genDeref(p: BProc, e: PNode, d: var TLoc) =
   var a: TLoc
-  if mapType(e.sons[0].typ) == ctArray:
+  if mapType(e.sons[0].typ) in {ctArray, ctPtrToArray}:
     # XXX the amount of hacks for C's arrays is incredible, maybe we should
     # simply wrap them in a struct? --> Losing auto vectorization then?
     expr(p, e.sons[0], d)
@@ -736,7 +736,7 @@ proc genArrayElem(p: BProc, e: PNode, d: var TLoc) =
   var ty = skipTypes(skipTypes(a.t, abstractVarRange), abstractPtrs)
   var first = intLiteral(firstOrd(ty))
   # emit range check:
-  if (optBoundsCheck in p.options):
+  if optBoundsCheck in p.options and tfUncheckedArray notin ty.flags:
     if not isConstExpr(e.sons[1]):
       # semantic pass has already checked for const index expressions
       if firstOrd(ty) == 0:
@@ -1803,7 +1803,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       else:
         genProc(p.module, sym)
       putLocIntoDest(p, d, sym.loc)
-    of skProc, skConverter, skIterator:
+    of skProc, skConverter, skIterators:
       genProc(p.module, sym)
       if sym.loc.r == nil or sym.loc.t == nil:
         internalError(n.info, "expr: proc not init " & sym.name.s)
@@ -1915,7 +1915,6 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       internalError(n.info, "expr: proc not init " & sym.name.s)
     putLocIntoDest(p, d, sym.loc)
   of nkClosure: genClosure(p, n, d)
-  of nkMetaNode: expr(p, n.sons[0], d)
 
   of nkEmpty: discard
   of nkWhileStmt: genWhileStmt(p, n)

compiler/ccgmerge.nim

@@ -145,33 +145,6 @@ proc atEndMark(buf: cstring, pos: int): bool =
   while s < NimMergeEndMark.len and buf[pos+s] == NimMergeEndMark[s]: inc s
   result = s == NimMergeEndMark.len
 
-when false:
-  proc readVerbatimSection(L: var TBaseLexer): PRope = 
-    var pos = L.bufpos
-    var buf = L.buf
-    result = newMutableRope(30_000)
-    while true:
-      case buf[pos]
-      of CR:
-        pos = nimlexbase.HandleCR(L, pos)
-        buf = L.buf
-        result.data.add(tnl)
-      of LF:
-        pos = nimlexbase.HandleLF(L, pos)
-        buf = L.buf
-        result.data.add(tnl)
-      of '\0':
-        InternalError("ccgmerge: expected: " & NimMergeEndMark)
-        break
-      else: 
-        if atEndMark(buf, pos):
-          inc pos, NimMergeEndMark.len
-          break
-        result.data.add(buf[pos])
-        inc pos
-    L.bufpos = pos
-    freezeMutableRope(result)
-
 proc readVerbatimSection(L: var TBaseLexer): PRope = 
   var pos = L.bufpos
   var buf = L.buf

compiler/ccgstmts.nim

@@ -65,6 +65,7 @@ proc startBlock(p: BProc, start: TFormatStr = "{$n",
   setLen(p.blocks, result + 1)
   p.blocks[result].id = p.labels
   p.blocks[result].nestedTryStmts = p.nestedTryStmts.len.int16
+  p.blocks[result].nestedExceptStmts = p.inExceptBlock.int16
 
 proc assignLabel(b: var TBlock): PRope {.inline.} =
   b.label = con("LA", b.id.toRope)
@@ -260,37 +261,57 @@ proc genIf(p: BProc, n: PNode, d: var TLoc) =
     else: internalError(n.info, "genIf()")
   if sonsLen(n) > 1: fixLabel(p, lend)
 
-proc blockLeaveActions(p: BProc, howMany: int) = 
+
-  var L = p.nestedTryStmts.len
+proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) = 
-  # danger of endless recursion! we workaround this here by a temp stack
+  # Called by return and break stmts.
+  # Deals with issues faced when jumping out of try/except/finally stmts,
+
   var stack: seq[PNode]
-  newSeq(stack, howMany)
+  newSeq(stack, 0)
-  for i in countup(1, howMany): 
-    stack[i-1] = p.nestedTryStmts[L-i]
-  setLen(p.nestedTryStmts, L-howMany)
   
   var alreadyPoppedCnt = p.inExceptBlock
-  for tryStmt in items(stack):
+  for i in countup(1, howManyTrys):
+
     if gCmd != cmdCompileToCpp:
+      # Pop safe points generated by try
       if alreadyPoppedCnt > 0:
         dec alreadyPoppedCnt
       else:
         linefmt(p, cpsStmts, "#popSafePoint();$n")
+
+    # Pop this try-stmt of the list of nested trys
+    # so we don't infinite recurse on it in the next step.
+    var tryStmt = p.nestedTryStmts.pop
+    stack.add(tryStmt)
+
+    # Find finally-stmt for this try-stmt
+    # and generate a copy of its sons
     var finallyStmt = lastSon(tryStmt)
     if finallyStmt.kind == nkFinally: 
       genStmts(p, finallyStmt.sons[0])
+
   # push old elements again:
-  for i in countdown(howMany-1, 0): 
+  for i in countdown(howManyTrys-1, 0): 
     p.nestedTryStmts.add(stack[i])
+
   if gCmd != cmdCompileToCpp:
-    for i in countdown(p.inExceptBlock-1, 0):
+    # Pop exceptions that was handled by the
+    # except-blocks we are in
+    for i in countdown(howManyExcepts-1, 0):
       linefmt(p, cpsStmts, "#popCurrentException();$n")
 
 proc genReturnStmt(p: BProc, t: PNode) =
   p.beforeRetNeeded = true
   genLineDir(p, t)
   if (t.sons[0].kind != nkEmpty): genStmts(p, t.sons[0])
-  blockLeaveActions(p, min(1, p.nestedTryStmts.len))
+  blockLeaveActions(p, 
+    howManyTrys    = p.nestedTryStmts.len,
+    howManyExcepts = p.inExceptBlock)
+  if (p.finallySafePoints.len > 0):
+    # If we're in a finally block, and we came here by exception
+    # consume it before we return.
+    var safePoint = p.finallySafePoints[p.finallySafePoints.len-1]
+    linefmt(p, cpsStmts, "if ($1.status != 0) #popCurrentException();$n", safePoint)    
   lineFF(p, cpsStmts, "goto BeforeRet;$n", "br label %BeforeRet$n", [])
 
 proc genComputedGoto(p: BProc; n: PNode) =
@@ -450,7 +471,9 @@ proc genBreakStmt(p: BProc, t: PNode) =
     if idx < 0 or not p.blocks[idx].isLoop:
       internalError(t.info, "no loop to break")
   let label = assignLabel(p.blocks[idx])
-  blockLeaveActions(p, p.nestedTryStmts.len - p.blocks[idx].nestedTryStmts)
+  blockLeaveActions(p, 
+    p.nestedTryStmts.len - p.blocks[idx].nestedTryStmts,
+    p.inExceptBlock - p.blocks[idx].nestedExceptStmts)
   genLineDir(p, t)
   lineF(p, cpsStmts, "goto $1;$n", [label])
 
@@ -827,7 +850,9 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
   discard pop(p.nestedTryStmts)
   endBlock(p) # end of else block
   if i < length and t.sons[i].kind == nkFinally:
+    p.finallySafePoints.add(safePoint)
     exprBlock(p, t.sons[i].sons[0], d)
+    discard pop(p.finallySafePoints)
   linefmt(p, cpsStmts, "if ($1.status != 0) #reraiseException();$n", safePoint)
 
 proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): PRope =
@@ -838,7 +863,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): PRope =
       res.add(t.sons[i].strVal)
     of nkSym:
       var sym = t.sons[i].sym
-      if sym.kind in {skProc, skIterator, skMethod}: 
+      if sym.kind in {skProc, skIterator, skClosureIterator, skMethod}:
         var a: TLoc
         initLocExpr(p, t.sons[i], a)
         res.add(rdLoc(a).ropeToStr)

compiler/ccgtypes.nim

@@ -69,7 +69,7 @@ proc mangleName(s: PSym): PRope =
   if result == nil: 
     if gCmd == cmdCompileToLLVM: 
       case s.kind
-      of skProc, skMethod, skConverter, skConst, skIterator: 
+      of skProc, skMethod, skConverter, skConst, skIterators:
         result = ~"@"
       of skVar, skForVar, skResult, skLet: 
         if sfGlobal in s.flags: result = ~"@"
@@ -185,7 +185,7 @@ proc mapType(typ: PType): TCTypeKind =
   of tyPtr, tyVar, tyRef:
     var base = skipTypes(typ.sons[0], typedescInst)
     case base.kind
-    of tyOpenArray, tyArrayConstr, tyArray, tyVarargs: result = ctArray
+    of tyOpenArray, tyArrayConstr, tyArray, tyVarargs: result = ctPtrToArray
     else: result = ctPtr
   of tyPointer: result = ctPtr
   of tySequence: result = ctNimSeq
@@ -377,9 +377,12 @@ proc getTypePre(m: BModule, typ: PType): PRope =
     result = getSimpleTypeDesc(m, typ)
     if result == nil: result = cacheGetType(m.typeCache, typ)
 
+proc structOrUnion(t: PType): PRope =
+  (if tfUnion in t.flags: toRope("union") else: toRope("struct"))
+
 proc getForwardStructFormat(): string = 
-  if gCmd == cmdCompileToCpp: result = "struct $1;$n"
+  if gCmd == cmdCompileToCpp: result = "$1 $2;$n"
-  else: result = "typedef struct $1 $1;$n"
+  else: result = "typedef $1 $2 $2;$n"
   
 proc getTypeForward(m: BModule, typ: PType): PRope = 
   result = cacheGetType(m.forwTypeCache, typ)
@@ -390,7 +393,8 @@ proc getTypeForward(m: BModule, typ: PType): PRope =
   of tySequence, tyTuple, tyObject: 
     result = getTypeName(typ)
     if not isImportedType(typ): 
-      appf(m.s[cfsForwardTypes], getForwardStructFormat(), [result])
+      appf(m.s[cfsForwardTypes], getForwardStructFormat(),
+          [structOrUnion(typ), result])
     idTablePut(m.forwTypeCache, typ, result)
   else: internalError("getTypeForward(" & $typ.kind & ')')
   
@@ -445,7 +449,12 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
     if accessExpr != nil: ae = ropef("$1.$2", [accessExpr, sname])
     else: ae = sname
     fillLoc(field.loc, locField, field.typ, ae, OnUnknown)
-    appf(result, "$1 $2;$n", [getTypeDescAux(m, field.loc.t, check), sname])
+    let fieldType = field.loc.t
+    if fieldType.kind == tyArray and tfUncheckedArray in fieldType.flags:
+      appf(result, "$1 $2[SEQ_DECL_SIZE];$n",
+          [getTypeDescAux(m, fieldType.elemType, check), sname])
+    else:
+      appf(result, "$1 $2;$n", [getTypeDescAux(m, fieldType, check), sname])
   else: internalError(n.info, "genRecordFieldsAux()")
   
 proc getRecordFields(m: BModule, typ: PType, check: var TIntSet): PRope = 
@@ -455,23 +464,33 @@ proc getRecordDesc(m: BModule, typ: PType, name: PRope,
                    check: var TIntSet): PRope = 
   # declare the record:
   var hasField = false
+
+  var attribute: PRope =
+    if tfPacked in typ.flags: toRope(CC[ccompiler].packedPragma)
+    else: nil
+
+  result = ropecg(m, CC[ccompiler].structStmtFmt, 
+    [structOrUnion(typ), name, attribute])
+
   if typ.kind == tyObject: 
+
     if typ.sons[0] == nil: 
       if (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags: 
-        result = ropecg(m, "struct $1 {$n", [name])
+        appcg(m, result, " {$n", [])
       else: 
-        result = ropecg(m, "struct $1 {$n#TNimType* m_type;$n", [name])
+        appcg(m, result, " {$n#TNimType* m_type;$n", [name, attribute])
         hasField = true
     elif gCmd == cmdCompileToCpp: 
-      result = ropecg(m, "struct $1 : public $2 {$n", 
+      appcg(m, result, " : public $1 {$n", 
-                      [name, getTypeDescAux(m, typ.sons[0], check)])
+                      [getTypeDescAux(m, typ.sons[0], check)])
       hasField = true
     else: 
-      result = ropecg(m, "struct $1 {$n  $2 Sup;$n", 
+      appcg(m, result, " {$n  $1 Sup;$n", 
-                      [name, getTypeDescAux(m, typ.sons[0], check)])
+                      [getTypeDescAux(m, typ.sons[0], check)])
       hasField = true
   else: 
-    result = ropef("struct $1 {$n", [name])
+    appf(result, " {$n", [name])
+
   var desc = getRecordFields(m, typ, check)
   if (desc == nil) and not hasField: 
     appf(result, "char dummy;$n", [])
@@ -481,7 +500,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: PRope,
 
 proc getTupleDesc(m: BModule, typ: PType, name: PRope, 
                   check: var TIntSet): PRope =
-  result = ropef("struct $1 {$n", [name])
+  result = ropef("$1 $2 {$n", [structOrUnion(typ), name])
   var desc: PRope = nil
   for i in countup(0, sonsLen(typ) - 1): 
     appf(desc, "$1 Field$2;$n", 
@@ -557,7 +576,8 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var TIntSet): PRope =
     if result == nil: 
       result = getTypeName(t)
       if not isImportedType(t): 
-        appf(m.s[cfsForwardTypes], getForwardStructFormat(), [result])
+        appf(m.s[cfsForwardTypes], getForwardStructFormat(),
+            [structOrUnion(t), result])
       idTablePut(m.forwTypeCache, t, result)
     assert(cacheGetType(m.typeCache, t) == nil)
     idTablePut(m.typeCache, t, con(result, "*"))
@@ -588,7 +608,8 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var TIntSet): PRope =
     if result == nil: 
       result = getTypeName(t)
       if not isImportedType(t): 
-        appf(m.s[cfsForwardTypes], getForwardStructFormat(), [result])
+        appf(m.s[cfsForwardTypes], getForwardStructFormat(),
+           [structOrUnion(t), result])
       idTablePut(m.forwTypeCache, t, result)
     idTablePut(m.typeCache, t, result) # always call for sideeffects:
     if t.kind != tyTuple: recdesc = getRecordDesc(m, t, result, check)

compiler/ccgutils.nim

@@ -87,9 +87,10 @@ proc getUniqueType*(key: PType): PType =
       gCanonicalTypes[k] = key
       result = key
   of tyTypeDesc, tyTypeClasses, tyGenericParam,
-     tyFromExpr, tyStatic, tyFieldAccessor:
+     tyFromExpr, tyFieldAccessor:
     internalError("GetUniqueType")
-  of tyGenericInst, tyDistinct, tyOrdinal, tyMutable, tyConst, tyIter:
+  of tyGenericInst, tyDistinct, tyOrdinal, tyMutable,
+     tyConst, tyIter, tyStatic:
     result = getUniqueType(lastSon(key))
   of tyArrayConstr, tyGenericInvokation, tyGenericBody,
      tyOpenArray, tyArray, tySet, tyRange, tyTuple,
@@ -130,7 +131,6 @@ proc getUniqueType*(key: PType): PType =
       idTablePut(gTypeTable[k], key, key)
       result = key
   of tyProc:
-    # tyVar is not 100% correct, but would speeds things up a little:
     if key.callConv != ccClosure:
       result = key
     else:

compiler/cgen.nim

@@ -705,7 +705,7 @@ proc cgsym(m: BModule, name: string): PRope =
   var sym = magicsys.getCompilerProc(name)
   if sym != nil: 
     case sym.kind
-    of skProc, skMethod, skConverter, skIterator: genProc(m, sym)
+    of skProc, skMethod, skConverter, skIterators: genProc(m, sym)
     of skVar, skResult, skLet: genVarPrototype(m, sym)
     of skType: discard getTypeDesc(m, sym.typ)
     else: internalError("cgsym: " & name)
@@ -761,6 +761,8 @@ proc genProcAux(m: BModule, prc: PSym) =
   var returnStmt: PRope = nil
   assert(prc.ast != nil)
   if sfPure notin prc.flags and prc.typ.sons[0] != nil:
+    if resultPos >= prc.ast.len:
+      internalError(prc.info, "proc has no result symbol")
     var res = prc.ast.sons[resultPos].sym # get result symbol
     if not isInvalidReturnType(prc.typ.sons[0]):
       if sfNoInit in prc.flags: incl(res.flags, sfNoInit)
@@ -962,8 +964,8 @@ proc genMainProc(m: BModule) =
     NimMainBody =
       "N_CDECL(void, NimMain)(void) {$N" &
         "\tPreMain();$N" &
-        "$1$N" &
+        "$1" &
-      "}$N"
+      "}$N$N"
 
     PosixNimMain =
       "int cmdCount;$N" &
@@ -977,20 +979,20 @@ proc genMainProc(m: BModule) =
         "\tcmdCount = argc;$N" &
         "\tgEnv = env;$N" &
         MainProcsWithResult &
-      "}$N"
+      "}$N$N"
   
     StandaloneCMain =
       "int main(void) {$N" &
         MainProcs &
         "\treturn 0;$N" &
-      "}$N"
+      "}$N$N"
     
     WinNimMain = NimMainBody
     
     WinCMain = "N_STDCALL(int, WinMain)(HINSTANCE hCurInstance, $N" &
       "                        HINSTANCE hPrevInstance, $N" &
       "                        LPSTR lpCmdLine, int nCmdShow) {$N" &
-      MainProcsWithResult & "}$N"
+      MainProcsWithResult & "}$N$N"
   
     WinNimDllMain = "N_LIB_EXPORT " & NimMainBody
 
@@ -998,14 +1000,14 @@ proc genMainProc(m: BModule) =
       "BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fwdreason, $N" &
       "                    LPVOID lpvReserved) {$N" &
       "\tif(fwdreason == DLL_PROCESS_ATTACH) {$N" & MainProcs & "}$N" &
-      "\treturn 1;$N}$N"
+      "\treturn 1;$N}$N$N"
 
     PosixNimDllMain = WinNimDllMain
     
     PosixCDllMain =
       "void NIM_POSIX_INIT NimMainInit(void) {$N" &
         MainProcs &
-      "}$N"
+      "}$N$N"
 
   var nimMain, otherMain: TFormatStr
   if platform.targetOS == osWindows and
@@ -1034,7 +1036,7 @@ proc genMainProc(m: BModule) =
                               platform.targetOS == osStandalone: "".toRope
                             else: ropecg(m, "\t#initStackBottom();$N")
   inc(m.labels)
-  appcg(m, m.s[cfsProcs], "void PreMain() {$N" & PreMainBody & "}$N", [
+  appcg(m, m.s[cfsProcs], "void PreMain() {$N" & PreMainBody & "}$N$N", [
     mainDatInit, initStackBottomCall, gBreakpoints, otherModsInit])
 
   appcg(m, m.s[cfsProcs], nimMain, [mainModInit, toRope(m.labels)])
@@ -1042,8 +1044,10 @@ proc genMainProc(m: BModule) =
     appcg(m, m.s[cfsProcs], otherMain, [])
 
 proc getSomeInitName(m: PSym, suffix: string): PRope =
+  assert m.kind == skModule
+  assert m.owner.kind == skPackage
   if {sfSystemModule, sfMainModule} * m.flags == {}:
-    result = m.info.toFullPath.getPackageName.mangle.toRope
+    result = m.owner.name.s.mangle.toRope
   result.app m.name.s
   result.app suffix
   
@@ -1188,7 +1192,7 @@ proc nullify[T](arr: var T) =
   for i in low(arr)..high(arr):
     arr[i] = nil
 
-proc resetModule*(m: var BModule) =
+proc resetModule*(m: BModule) =
   # between two compilations in CAAS mode, we can throw
   # away all the data that was written to disk
   initLinkedList(m.headerFiles)

compiler/cgendata.nim

@@ -41,7 +41,8 @@ type
     ctInt, ctInt8, ctInt16, ctInt32, ctInt64,
     ctFloat, ctFloat32, ctFloat64, ctFloat128,
     ctUInt, ctUInt8, ctUInt16, ctUInt32, ctUInt64,
-    ctArray, ctStruct, ctPtr, ctNimStr, ctNimSeq, ctProc, ctCString
+    ctArray, ctPtrToArray, ctStruct, ctPtr, ctNimStr, ctNimSeq, ctProc,
+    ctCString
   TCFileSections* = array[TCFileSection, PRope] # represents a generated C file
   TCProcSection* = enum       # the sections a generated C proc consists of
     cpsLocals,                # section of local variables for C proc
@@ -57,17 +58,20 @@ type
     sections*: TCProcSections # the code beloging
     isLoop*: bool             # whether block is a loop
     nestedTryStmts*: int16    # how many try statements is it nested into
+    nestedExceptStmts*: int16 # how many except statements is it nested into
     frameLen*: int16
   
   TCProc{.final.} = object    # represents C proc that is currently generated
     prc*: PSym                # the Nimrod proc that this C proc belongs to
     beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
     threadVarAccessed*: bool  # true if the proc already accessed some threadvar
-    nestedTryStmts*: seq[PNode] # in how many nested try statements we are
+    nestedTryStmts*: seq[PNode]   # in how many nested try statements we are
-                                # (the vars must be volatile then)
+                                  # (the vars must be volatile then)
     inExceptBlock*: int       # are we currently inside an except block?
                               # leaving such scopes by raise or by return must
                               # execute any applicable finally blocks
+    finallySafePoints*: seq[PRope]  # For correctly cleaning up exceptions when
+                                    # using return in finally statements
     labels*: Natural          # for generating unique labels in the C proc
     blocks*: seq[TBlock]      # nested blocks
     breakIdx*: int            # the block that will be exited
@@ -140,8 +144,9 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   else: result.options = gOptions
   newSeq(result.blocks, 1)
   result.nestedTryStmts = @[]
+  result.finallySafePoints = @[]
 
-iterator cgenModules*: var BModule =
+iterator cgenModules*: BModule =
   for i in 0..high(gModules):
     # ultimately, we are iterating over the file ids here.
     # some "files" won't have an associated cgen module (like stdin)

compiler/condsyms.nim

@@ -47,6 +47,7 @@ proc initDefines*() =
   defineSymbol("nimeffects")
   defineSymbol("nimbabel")
   defineSymbol("nimcomputedgoto")
+  defineSymbol("nimunion")
   
   # add platform specific symbols:
   case targetCPU

compiler/docgen.nim

@@ -42,17 +42,23 @@ proc compilerMsgHandler(filename: string, line, col: int,
   of mwUnsupportedLanguage: k = warnLanguageXNotSupported
   globalError(newLineInfo(filename, line, col), k, arg)
 
+proc docgenFindFile(s: string): string {.procvar.} =
+  result = options.findFile(s)
+  if result.len == 0:
+    result = getCurrentDir() / s
+    if not existsFile(result): result = ""
+
 proc parseRst(text, filename: string,
               line, column: int, hasToc: var bool,
               rstOptions: TRstParseOptions): PRstNode =
   result = rstParse(text, filename, line, column, hasToc, rstOptions,
-                    options.findFile, compilerMsgHandler)
+                    docgenFindFile, compilerMsgHandler)
 
 proc newDocumentor*(filename: string, config: PStringTable): PDoc =
   new(result)
   initRstGenerator(result[], (if gCmd != cmdRst2tex: outHtml else: outLatex),
                    options.gConfigVars, filename, {roSupportRawDirective},
-                   options.findFile, compilerMsgHandler)
+                   docgenFindFile, compilerMsgHandler)
   result.id = 100
 
 proc dispA(dest: var PRope, xml, tex: string, args: openArray[PRope]) =
@@ -360,7 +366,7 @@ proc generateJson(d: PDoc, n: PNode, jArray: PJsonNode = nil): PJsonNode =
 proc genSection(d: PDoc, kind: TSymKind) =
   const sectionNames: array[skModule..skTemplate, string] = [
     "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Methods",
-    "Iterators", "Converters", "Macros", "Templates"
+    "Iterators", "Iterators", "Converters", "Macros", "Templates"
   ]
   if d.section[kind] == nil: return
   var title = sectionNames[kind].toRope

compiler/evalffi.nim

@@ -151,7 +151,7 @@ proc getField(n: PNode; position: int): PSym =
   else: discard
 
 proc packObject(x: PNode, typ: PType, res: pointer) =
-  InternalAssert x.kind in {nkObjConstr, nkPar}
+  internalAssert x.kind in {nkObjConstr, nkPar}
   # compute the field's offsets:
   discard typ.getSize
   for i in countup(ord(x.kind == nkObjConstr), sonsLen(x) - 1):

compiler/extccomp.nim

@@ -16,7 +16,7 @@ import
 type 
   TSystemCC* = enum 
     ccNone, ccGcc, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc, 
-    ccTcc, ccPcc, ccUcc, ccIcl, ccGpp
+    ccTcc, ccPcc, ccUcc, ccIcl
   TInfoCCProp* = enum         # properties of the C compiler:
     hasSwitchRange,           # CC allows ranges in switch statements (GNU C)
     hasComputedGoto,          # CC has computed goto (GNU C extension)
@@ -33,11 +33,12 @@ type
     optSpeed: string,    # the options for optimization for speed
     optSize: string,     # the options for optimization for size
     compilerExe: string, # the compiler's executable
+    cppCompiler: string, # name of the C++ compiler's executable (if supported)
     compileTmpl: string, # the compile command template
     buildGui: string,    # command to build a GUI application
     buildDll: string,    # command to build a shared library
     buildLib: string,    # command to build a static library
-    linkerExe: string,   # the linker's executable
+    linkerExe: string,   # the linker's executable (if not matching compiler's)
     linkTmpl: string,    # command to link files to produce an exe
     includeCmd: string,  # command to add an include dir
     linkDirCmd: string,  # command to add a lib dir
@@ -46,6 +47,8 @@ type
     pic: string,         # command for position independent code
                          # used on some platforms
     asmStmtFrmt: string, # format of ASM statement
+    structStmtFmt: string, # Format for struct statement
+    packedPragma: string,  # Attribute/pragma to make struct packed (1-byte aligned)
     props: TInfoCCProps] # properties of the C compiler
 
 
@@ -63,11 +66,12 @@ compiler gcc:
     optSpeed: " -O3 -ffast-math ",
     optSize: " -Os -ffast-math ",
     compilerExe: "gcc",
+    cppCompiler: "g++",
     compileTmpl: "-c $options $include -o $objfile $file",
     buildGui: " -mwindows",
     buildDll: " -shared",
     buildLib: "ar rcs $libfile $objfiles",
-    linkerExe: "gcc",
+    linkerExe: "",
     linkTmpl: "$buildgui $builddll -o $exefile $objfiles $options",
     includeCmd: " -I",
     linkDirCmd: " -L",
@@ -75,34 +79,25 @@ compiler gcc:
     debug: "",
     pic: "-fPIC",
     asmStmtFrmt: "asm($1);$n",
+    structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
+    packedPragma: "__attribute__((__packed__))",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasNakedAttribute})
 
-compiler gpp:
-  result = gcc()
-  
-  result.name = "gpp"
-  result.compilerExe = "g++"
-  result.linkerExe = "g++"  
-
-  result.buildDll = " -mdll" 
-  # XXX: Hmm, I'm keeping this from the previos version, 
-  # but my gcc doesn't even have such an option (is this mingw?)
-
 compiler llvmGcc:
   result = gcc()
   
   result.name = "llvm_gcc"
   result.compilerExe = "llvm-gcc"
+  result.cppCompiler = "llvm-g++"
   result.buildLib = "llvm-ar rcs $libfile $objfiles"
-  result.linkerExe = "llvm-gcc"
 
 compiler clang:
   result = llvmGcc()
 
   result.name = "clang"
   result.compilerExe = "clang"
-  result.linkerExe = "clang"
+  result.cppCompiler = "clang++"
 
 compiler vcc:
   result = (
@@ -111,6 +106,7 @@ compiler vcc:
     optSpeed: " /Ogityb2 /G7 /arch:SSE2 ",
     optSize: " /O1 /G7 ",
     compilerExe: "cl",
+    cppCompiler: "cl",
     compileTmpl: "/c $options $include /Fo$objfile $file",
     buildGui: " /link /SUBSYSTEM:WINDOWS ",
     buildDll: " /LD",
@@ -123,6 +119,8 @@ compiler vcc:
     debug: " /GZ /Zi ",
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
+    structStmtFmt: "$3$n$1 $2",
+    packedPragma: "#pragma pack(1)",
     props: {hasCpp, hasAssume, hasNakedDeclspec})
 
 compiler icl:
@@ -143,6 +141,7 @@ compiler lcc:
     optSpeed: " -O -p6 ",
     optSize: " -O -p6 ",
     compilerExe: "lcc",
+    cppCompiler: "",
     compileTmpl: "$options $include -Fo$objfile $file",
     buildGui: " -subsystem windows",
     buildDll: " -dll",
@@ -155,6 +154,8 @@ compiler lcc:
     debug: " -g5 ",
     pic: "",
     asmStmtFrmt: "_asm{$n$1$n}$n",
+    structStmtFmt: "$1 $2",
+    packedPragma: "", # XXX: not supported yet
     props: {})
 
 compiler bcc:
@@ -164,6 +165,7 @@ compiler bcc:
     optSpeed: " -O2 -6 ",
     optSize: " -O1 -6 ",
     compilerExe: "bcc32",
+    cppCompiler: "",
     compileTmpl: "-c $options $include -o$objfile $file",
     buildGui: " -tW",
     buildDll: " -tWD",
@@ -176,6 +178,8 @@ compiler bcc:
     debug: "",
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
+    structStmtFmt: "$1 $2",
+    packedPragma: "", # XXX: not supported yet
     props: {hasCpp})
 
 compiler dmc:
@@ -185,6 +189,7 @@ compiler dmc:
     optSpeed: " -ff -o -6 ",
     optSize: " -ff -o -6 ",
     compilerExe: "dmc",
+    cppCompiler: "",
     compileTmpl: "-c $options $include -o$objfile $file",
     buildGui: " -L/exet:nt/su:windows",
     buildDll: " -WD",
@@ -197,6 +202,8 @@ compiler dmc:
     debug: " -g ",
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
+    structStmtFmt: "$3$n$1 $2",
+    packedPragma: "#pragma pack(1)",
     props: {hasCpp})
 
 compiler wcc:
@@ -206,6 +213,7 @@ compiler wcc:
     optSpeed: " -ox -on -6 -d0 -fp6 -zW ",
     optSize: "",
     compilerExe: "wcl386",
+    cppCompiler: "",
     compileTmpl: "-c $options $include -fo=$objfile $file",
     buildGui: " -bw",
     buildDll: " -bd",
@@ -218,6 +226,8 @@ compiler wcc:
     debug: " -d2 ",
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
+    structStmtFmt: "$1 $2",
+    packedPragma: "", # XXX: not supported yet
     props: {hasCpp})
 
 compiler tcc:
@@ -227,6 +237,7 @@ compiler tcc:
     optSpeed: "",
     optSize: "",
     compilerExe: "tcc",
+    cppCompiler: "",
     compileTmpl: "-c $options $include -o $objfile $file",
     buildGui: "UNAVAILABLE!",
     buildDll: " -shared",
@@ -239,6 +250,8 @@ compiler tcc:
     debug: " -g ",
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
+    structStmtFmt: "$1 $2",
+    packedPragma: "", # XXX: not supported yet
     props: {hasSwitchRange, hasComputedGoto})
 
 compiler pcc:
@@ -249,6 +262,7 @@ compiler pcc:
     optSpeed: " -Ox ",
     optSize: " -Os ",
     compilerExe: "cc",
+    cppCompiler: "",
     compileTmpl: "-c $options $include -Fo$objfile $file",
     buildGui: " -SUBSYSTEM:WINDOWS",
     buildDll: " -DLL",
@@ -261,6 +275,8 @@ compiler pcc:
     debug: " -Zi ",
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
+    structStmtFmt: "$1 $2",
+    packedPragma: "", # XXX: not supported yet
     props: {})
 
 compiler ucc:
@@ -270,6 +286,7 @@ compiler ucc:
     optSpeed: " -O3 ",
     optSize: " -O1 ",
     compilerExe: "cc",
+    cppCompiler: "",
     compileTmpl: "-c $options $include -o $objfile $file",
     buildGui: "",
     buildDll: " -shared ",
@@ -282,6 +299,8 @@ compiler ucc:
     debug: "",
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
+    structStmtFmt: "$1 $2",
+    packedPragma: "", # XXX: not supported yet
     props: {})
 
 const 
@@ -297,8 +316,7 @@ const
     tcc(),
     pcc(),
     ucc(),
-    icl(),
+    icl()]
-    gpp()]
 
 const
   hExt* = ".h"
@@ -471,11 +489,21 @@ proc needsExeExt(): bool {.inline.} =
   result = (optGenScript in gGlobalOptions and targetOS == osWindows) or
                                        (platform.hostOS == osWindows)
 
+proc getCompilerExe(compiler: TSystemCC): string =
+  result = if gCmd == cmdCompileToCpp: CC[compiler].cppCompiler
+           else: CC[compiler].compilerExe
+  if result.len == 0:
+    rawMessage(errCompilerDoesntSupportTarget, CC[compiler].name)
+
+proc getLinkerExe(compiler: TSystemCC): string =
+  result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
+           else: compiler.getCompilerExe
+
 proc getCompileCFileCmd*(cfilename: string, isExternal = false): string = 
   var c = cCompiler
   var options = cFileSpecificOptions(cfilename)
   var exe = getConfigVar(c, ".exe")
-  if exe.len == 0: exe = CC[c].compilerExe
+  if exe.len == 0: exe = c.getCompilerExe
   
   if needsExeExt(): exe = addFileExt(exe, "exe")
   if optGenDynLib in gGlobalOptions and
@@ -493,7 +521,7 @@ proc getCompileCFileCmd*(cfilename: string, isExternal = false): string =
     compilePattern = joinPath(ccompilerpath, exe)
   else: 
     includeCmd = ""
-    compilePattern = CC[c].compilerExe
+    compilePattern = c.getCompilerExe
   
   var cfile = if noAbsolutePaths(): extractFilename(cfilename) 
               else: cfilename
@@ -600,7 +628,7 @@ proc callCCompiler*(projectfile: string) =
       if optCompileOnly notin gGlobalOptions: execExternalProgram(linkCmd)
     else:
       var linkerExe = getConfigVar(c, ".linkerexe")
-      if len(linkerExe) == 0: linkerExe = CC[c].linkerExe
+      if len(linkerExe) == 0: linkerExe = c.getLinkerExe
       if needsExeExt(): linkerExe = addFileExt(linkerExe, "exe")
       if noAbsolutePaths(): linkCmd = quoteShell(linkerExe)
       else: linkCmd = quoteShell(joinPath(ccompilerpath, linkerExe))

compiler/guards.nim

@@ -9,7 +9,7 @@
 
 ## This module implements the 'implies' relation for guards.
 
-import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer
+import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents
 
 const
   someEq = {mEqI, mEqI64, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
@@ -69,9 +69,23 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
 
 proc interestingCaseExpr*(m: PNode): bool = isLetLocation(m, true)
 
-proc swapArgs(fact: PNode, newOp: string, m: TMagic): PNode =
+proc getMagicOp(name: string, m: TMagic): PSym =
+  result = newSym(skProc, getIdent(name), nil, unknownLineInfo())
+  result.magic = m
+
+let
+  opLe = getMagicOp("<=", mLeI)
+  opLt = getMagicOp("<", mLtI)
+  opAnd = getMagicOp("and", mAnd)
+  opOr = getMagicOp("or", mOr)
+  opNot = getMagicOp("not", mNot)
+  opIsNil = getMagicOp("isnil", mIsNil)
+  opContains = getMagicOp("contains", mInSet)
+  opEq = getMagicOp("==", mEqI)
+
+proc swapArgs(fact: PNode, newOp: PSym): PNode =
   result = newNodeI(nkCall, fact.info, 3)
-  result.sons[0] = newSymNode(getSysMagic(newOp, m))
+  result.sons[0] = newSymNode(newOp)
   result.sons[1] = fact.sons[2]
   result.sons[2] = fact.sons[1]
 
@@ -82,9 +96,9 @@ proc neg(n: PNode): PNode =
     result = n.sons[1]
   of someLt:
     # not (a < b)  ==  a >= b  ==  b <= a
-    result = swapArgs(n, "<=", mLeI)
+    result = swapArgs(n, opLe)
   of someLe:
-    result = swapArgs(n, "<", mLtI)
+    result = swapArgs(n, opLt)
   of mInSet:
     if n.sons[1].kind != nkCurly: return nil
     let t = n.sons[2].typ.skipTypes(abstractInst)
@@ -110,7 +124,7 @@ proc neg(n: PNode): PNode =
       b = n.sons[2].neg
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(getSysMagic("and", mAnd))
+      result.sons[0] = newSymNode(opAnd)
       result.sons[1] = a
       result.sons[2] = b
     elif a != nil:
@@ -120,12 +134,12 @@ proc neg(n: PNode): PNode =
   else:
     # leave  not (a == 4)  as it is
     result = newNodeI(nkCall, n.info, 2)
-    result.sons[0] = newSymNode(getSysMagic("not", mNot))
+    result.sons[0] = newSymNode(opNot)
     result.sons[1] = n
 
 proc buildIsNil(arg: PNode): PNode =
   result = newNodeI(nkCall, arg.info, 2)
-  result.sons[0] = newSymNode(getSysMagic("isNil", mIsNil))
+  result.sons[0] = newSymNode(opIsNil)
   result.sons[1] = arg
 
 proc usefulFact(n: PNode): PNode =
@@ -154,7 +168,7 @@ proc usefulFact(n: PNode): PNode =
       b = usefulFact(n.sons[2])
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(getSysMagic("and", mAnd))
+      result.sons[0] = newSymNode(opAnd)
       result.sons[1] = a
       result.sons[2] = b
     elif a != nil:
@@ -177,7 +191,7 @@ proc usefulFact(n: PNode): PNode =
       b = usefulFact(n.sons[2]).neg
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(getSysMagic("and", mAnd))
+      result.sons[0] = newSymNode(opAnd)
       result.sons[1] = a
       result.sons[2] = b
       result = result.neg
@@ -260,10 +274,6 @@ proc pred(n: PNode): PNode =
   else:
     result = n
 
-type
-  TImplication* = enum
-    impUnknown, impNo, impYes  
-
 proc impliesEq(fact, eq: PNode): TImplication =
   let (loc, val) = if isLocation(eq.sons[1]): (1, 2) else: (2, 1)
   
@@ -520,7 +530,7 @@ proc buildOf(it, loc: PNode): PNode =
   s.typ = settype(loc)
   for i in 0..it.len-2: s.sons[i] = it.sons[i]
   result = newNodeI(nkCall, it.info, 3)
-  result.sons[0] = newSymNode(getSysMagic("contains", mInSet))
+  result.sons[0] = newSymNode(opContains)
   result.sons[1] = s
   result.sons[2] = loc
 
@@ -532,20 +542,20 @@ proc buildElse(n: PNode): PNode =
     for j in 0..branch.len-2:
       s.add(branch.sons[j])
   result = newNodeI(nkCall, n.info, 3)
-  result.sons[0] = newSymNode(getSysMagic("contains", mInSet))
+  result.sons[0] = newSymNode(opContains)
   result.sons[1] = s
   result.sons[2] = n.sons[0]
 
 proc addDiscriminantFact*(m: var TModel, n: PNode) =
   var fact = newNodeI(nkCall, n.info, 3)
-  fact.sons[0] = newSymNode(getSysMagic("==", mEqI))
+  fact.sons[0] = newSymNode(opEq)
   fact.sons[1] = n.sons[0]
   fact.sons[2] = n.sons[1]
   m.add fact
 
 proc addAsgnFact*(m: var TModel, key, value: PNode) =
   var fact = newNodeI(nkCall, key.info, 3)
-  fact.sons[0] = newSymNode(getSysMagic("==", mEqI))
+  fact.sons[0] = newSymNode(opEq)
   fact.sons[1] = key
   fact.sons[2] = value
   m.add fact

compiler/idgen.nim

@@ -19,7 +19,7 @@ const
 when debugIds:
   import intsets
   
-  var usedIds = InitIntSet()
+  var usedIds = initIntSet()
 
 proc registerID*(id: PIdObj) = 
   when debugIds: 

compiler/importer.nim

@@ -103,7 +103,7 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
       internalError(n.info, "importSymbol: 2")
     # for an enumeration we have to add all identifiers
     case s.kind
-    of skProc, skMethod, skIterator, skMacro, skTemplate, skConverter:
+    of skProcKinds:
       # for a overloadable syms add all overloaded routines
       var it: TIdentIter
       var e = initIdentIter(it, fromMod.tab, s.name)

compiler/jsgen.nim

@@ -275,11 +275,11 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxI64
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinF64
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxF64
-    ["AddU", "AddU", "AddU($1, $2)", "AddU($1, $2)"], # AddU
+    ["addU", "addU", "addU($1, $2)", "addU($1, $2)"], # addU
-    ["SubU", "SubU", "SubU($1, $2)", "SubU($1, $2)"], # SubU
+    ["subU", "subU", "subU($1, $2)", "subU($1, $2)"], # subU
-    ["MulU", "MulU", "MulU($1, $2)", "MulU($1, $2)"], # MulU
+    ["mulU", "mulU", "mulU($1, $2)", "mulU($1, $2)"], # mulU
-    ["DivU", "DivU", "DivU($1, $2)", "DivU($1, $2)"], # DivU
+    ["divU", "divU", "divU($1, $2)", "divU($1, $2)"], # divU
-    ["ModU", "ModU", "ModU($1, $2)", "ModU($1, $2)"], # ModU
+    ["modU", "modU", "modU($1, $2)", "modU($1, $2)"], # modU
     ["", "", "($1 == $2)", "($1 == $2)"], # EqI
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeI
     ["", "", "($1 < $2)", "($1 < $2)"], # LtI
@@ -289,10 +289,10 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqF64
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeF64
     ["", "", "($1 < $2)", "($1 < $2)"], # LtF64
-    ["LeU", "LeU", "LeU($1, $2)", "LeU($1, $2)"], # LeU
+    ["leU", "leU", "leU($1, $2)", "leU($1, $2)"], # leU
-    ["LtU", "LtU", "LtU($1, $2)", "LtU($1, $2)"], # LtU
+    ["ltU", "ltU", "ltU($1, $2)", "ltU($1, $2)"], # ltU
-    ["LeU64", "LeU64", "LeU64($1, $2)", "LeU64($1, $2)"], # LeU64
+    ["leU64", "leU64", "leU64($1, $2)", "leU64($1, $2)"], # leU64
-    ["LtU64", "LtU64", "LtU64($1, $2)", "LtU64($1, $2)"], # LtU64
+    ["ltU64", "ltU64", "ltU64($1, $2)", "ltU64($1, $2)"], # ltU64
     ["", "", "($1 == $2)", "($1 == $2)"], # EqEnum
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeEnum
     ["", "", "($1 < $2)", "($1 < $2)"], # LtEnum
@@ -309,10 +309,10 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqCString
     ["", "", "($1 != $2)", "($1 != $2)"], # Xor
     ["", "", "($1 == $2)", "($1 == $2)"], # EqProc
-    ["NegInt", "", "NegInt($1)", "-($1)"], # UnaryMinusI
+    ["negInt", "", "negInt($1)", "-($1)"], # UnaryMinusI
-    ["NegInt64", "", "NegInt64($1)", "-($1)"], # UnaryMinusI64
+    ["negInt64", "", "negInt64($1)", "-($1)"], # UnaryMinusI64
-    ["AbsInt", "", "AbsInt($1)", "Math.abs($1)"], # AbsI
+    ["absInt", "", "absInt($1)", "Math.abs($1)"], # AbsI
-    ["AbsInt64", "", "AbsInt64($1)", "Math.abs($1)"], # AbsI64
+    ["absInt64", "", "absInt64($1)", "Math.abs($1)"], # AbsI64
     ["", "", "!($1)", "!($1)"], # Not
     ["", "", "+($1)", "+($1)"], # UnaryPlusI
     ["", "", "~($1)", "~($1)"], # BitnotI
@@ -327,9 +327,9 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["Ze16ToI64", "Ze16ToI64", "Ze16ToI64($1)", "Ze16ToI64($1)"], # mZe16ToI64
     ["Ze32ToI64", "Ze32ToI64", "Ze32ToI64($1)", "Ze32ToI64($1)"], # mZe32ToI64
     ["ZeIToI64", "ZeIToI64", "ZeIToI64($1)", "ZeIToI64($1)"], # mZeIToI64
-    ["ToU8", "ToU8", "ToU8($1)", "ToU8($1)"], # ToU8
+    ["toU8", "toU8", "toU8($1)", "toU8($1)"], # toU8
-    ["ToU16", "ToU16", "ToU16($1)", "ToU16($1)"], # ToU16
+    ["toU16", "toU16", "toU16($1)", "toU16($1)"], # toU16
-    ["ToU32", "ToU32", "ToU32($1)", "ToU32($1)"], # ToU32
+    ["toU32", "toU32", "toU32($1)", "toU32($1)"], # toU32
     ["", "", "$1", "$1"],     # ToFloat
     ["", "", "$1", "$1"],     # ToBiggestFloat
     ["", "", "Math.floor($1)", "Math.floor($1)"], # ToInt
@@ -375,11 +375,11 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxI64
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinF64
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxF64
-    ["AddU", "AddU", "AddU($1, $2)", "AddU($1, $2)"], # AddU
+    ["addU", "addU", "addU($1, $2)", "addU($1, $2)"], # addU
-    ["SubU", "SubU", "SubU($1, $2)", "SubU($1, $2)"], # SubU
+    ["subU", "subU", "subU($1, $2)", "subU($1, $2)"], # subU
-    ["MulU", "MulU", "MulU($1, $2)", "MulU($1, $2)"], # MulU
+    ["mulU", "mulU", "mulU($1, $2)", "mulU($1, $2)"], # mulU
-    ["DivU", "DivU", "DivU($1, $2)", "DivU($1, $2)"], # DivU
+    ["divU", "divU", "divU($1, $2)", "divU($1, $2)"], # divU
-    ["ModU", "ModU", "ModU($1, $2)", "ModU($1, $2)"], # ModU
+    ["modU", "modU", "modU($1, $2)", "modU($1, $2)"], # modU
     ["", "", "($1 == $2)", "($1 == $2)"], # EqI
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeI
     ["", "", "($1 < $2)", "($1 < $2)"], # LtI
@@ -389,10 +389,10 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqF64
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeF64
     ["", "", "($1 < $2)", "($1 < $2)"], # LtF64
-    ["LeU", "LeU", "LeU($1, $2)", "LeU($1, $2)"], # LeU
+    ["leU", "leU", "leU($1, $2)", "leU($1, $2)"], # leU
-    ["LtU", "LtU", "LtU($1, $2)", "LtU($1, $2)"], # LtU
+    ["ltU", "ltU", "ltU($1, $2)", "ltU($1, $2)"], # ltU
-    ["LeU64", "LeU64", "LeU64($1, $2)", "LeU64($1, $2)"], # LeU64
+    ["leU64", "leU64", "leU64($1, $2)", "leU64($1, $2)"], # leU64
-    ["LtU64", "LtU64", "LtU64($1, $2)", "LtU64($1, $2)"], # LtU64
+    ["ltU64", "ltU64", "ltU64($1, $2)", "ltU64($1, $2)"], # ltU64
     ["", "", "($1 == $2)", "($1 == $2)"], # EqEnum
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeEnum
     ["", "", "($1 < $2)", "($1 < $2)"], # LtEnum
@@ -409,10 +409,10 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqCString
     ["", "", "($1 != $2)", "($1 != $2)"], # Xor
     ["", "", "($1 == $2)", "($1 == $2)"], # EqProc
-    ["NegInt", "", "NegInt($1)", "-($1)"], # UnaryMinusI
+    ["negInt", "", "negInt($1)", "-($1)"], # UnaryMinusI
-    ["NegInt64", "", "NegInt64($1)", "-($1)"], # UnaryMinusI64
+    ["negInt64", "", "negInt64($1)", "-($1)"], # UnaryMinusI64
-    ["AbsInt", "", "AbsInt($1)", "Math.abs($1)"], # AbsI
+    ["absInt", "", "absInt($1)", "Math.abs($1)"], # AbsI
-    ["AbsInt64", "", "AbsInt64($1)", "Math.abs($1)"], # AbsI64
+    ["absInt64", "", "absInt64($1)", "Math.abs($1)"], # AbsI64
     ["", "", "not ($1)", "not ($1)"], # Not
     ["", "", "+($1)", "+($1)"], # UnaryPlusI
     ["", "", "~($1)", "~($1)"], # BitnotI
@@ -427,9 +427,9 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["Ze16ToI64", "Ze16ToI64", "Ze16ToI64($1)", "Ze16ToI64($1)"], # mZe16ToI64
     ["Ze32ToI64", "Ze32ToI64", "Ze32ToI64($1)", "Ze32ToI64($1)"], # mZe32ToI64
     ["ZeIToI64", "ZeIToI64", "ZeIToI64($1)", "ZeIToI64($1)"], # mZeIToI64
-    ["ToU8", "ToU8", "ToU8($1)", "ToU8($1)"], # ToU8
+    ["toU8", "toU8", "toU8($1)", "toU8($1)"], # toU8
-    ["ToU16", "ToU16", "ToU16($1)", "ToU16($1)"], # ToU16
+    ["toU16", "toU16", "toU16($1)", "toU16($1)"], # toU16
-    ["ToU32", "ToU32", "ToU32($1)", "ToU32($1)"], # ToU32
+    ["toU32", "toU32", "toU32($1)", "toU32($1)"], # toU32
     ["", "", "$1", "$1"],     # ToFloat
     ["", "", "$1", "$1"],     # ToBiggestFloat
     ["", "", "Math.floor($1)", "Math.floor($1)"], # ToInt
@@ -812,8 +812,8 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
     if needsNoCopy(y) or noCopyNeeded:
       appf(p.body, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
-      useMagic(p, "NimCopy")
+      useMagic(p, "nimCopy")
-      appf(p.body, "$1 = NimCopy($2, $3);$n",
+      appf(p.body, "$1 = nimCopy($2, $3);$n",
            [a.res, b.res, genTypeInfo(p, y.typ)])
   of etyBaseIndex: 
     if a.typ != etyBaseIndex or b.typ != etyBaseIndex: 
@@ -1113,8 +1113,8 @@ proc createVar(p: PProc, typ: PType, indirect: bool): PRope =
     var length = int(lengthOrd(t))
     var e = elemType(t)
     if length > 32: 
-      useMagic(p, "ArrayConstr")
+      useMagic(p, "arrayConstr")
-      result = ropef("ArrayConstr($1, $2, $3)", [toRope(length), 
+      result = ropef("arrayConstr($1, $2, $3)", [toRope(length), 
           createVar(p, e, false), genTypeInfo(p, e)])
     else: 
       result = toRope("[")
@@ -1154,7 +1154,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): PRope =
 
 proc isIndirect(v: PSym): bool = 
   result = (sfAddrTaken in v.flags) and (mapType(v.typ) != etyObject) and
-    v.kind notin {skProc, skConverter, skMethod, skIterator}
+    v.kind notin {skProc, skConverter, skMethod, skIterator, skClosureIterator}
 
 proc genVarInit(p: PProc, v: PSym, n: PNode) = 
   var 
@@ -1171,8 +1171,8 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       if needsNoCopy(n): 
         s = a.res
       else: 
-        useMagic(p, "NimCopy")
+        useMagic(p, "nimCopy")
-        s = ropef("NimCopy($1, $2)", [a.res, genTypeInfo(p, n.typ)])
+        s = ropef("nimCopy($1, $2)", [a.res, genTypeInfo(p, n.typ)])
     of etyBaseIndex: 
       if (a.typ != etyBaseIndex): internalError(n.info, "genVarInit")
       if {sfAddrTaken, sfGlobal} * v.flags != {}: 
@@ -1600,7 +1600,6 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     if lfNoDecl in s.loc.flags or s.magic != mNone: discard
     elif not p.g.generatedSyms.containsOrIncl(s.id):
       app(p.locals, genProc(p, s))
-  of nkMetaNode: gen(p, n.sons[0], r)
   of nkType: r.res = genTypeInfo(p, n.typ)
   of nkStmtList, nkStmtListExpr:
     # this shows the distinction is nice for backends and should be kept

compiler/lambdalifting.nim

@@ -11,7 +11,7 @@
 
 import 
   intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os, 
-  idents, renderer, types, magicsys, rodread
+  idents, renderer, types, magicsys, rodread, lowerings
 
 discard """
   The basic approach is that captured vars need to be put on the heap and
@@ -232,7 +232,7 @@ proc newOuterContext(fn: PSym, up: POuterContext = nil): POuterContext =
   initIdNodeTable(result.localsToAccess)
   initIdTable(result.localsToEnv)
   initIdTable(result.lambdasToEnv)
-  result.isIter = fn.kind == skIterator and fn.typ.callConv == ccClosure
+  result.isIter = fn.kind == skClosureIterator
   if result.isIter: initIterContext(result, fn)
 
 proc newInnerContext(fn: PSym): PInnerContext =
@@ -292,8 +292,7 @@ proc newCall(a, b: PSym): PNode =
   result.add newSymNode(b)
 
 proc isInnerProc(s, outerProc: PSym): bool {.inline.} =
-  result = (s.kind in {skProc, skMethod, skConverter} or
+  result = s.kind in {skProc, skMethod, skConverter, skClosureIterator} and
-            s.kind == skIterator and s.typ.callConv == ccClosure) and
            s.skipGenericOwner == outerProc
   #s.typ.callConv == ccClosure
 
@@ -357,7 +356,10 @@ proc captureVar(o: POuterContext, i: PInnerContext, local: PSym,
     # it's in some upper environment:
     access = indirectAccess(access, addDep(e, it, i.fn), info)
   access = indirectAccess(access, local, info)
-  incl(o.capturedVars, local.id)
+  if o.isIter:
+    if not containsOrIncl(o.capturedVars, local.id): addField(o.tup, local)
+  else:
+    incl(o.capturedVars, local.id)
   idNodeTablePut(i.localsToAccess, local, access)
 
 proc interestingVar(s: PSym): bool {.inline.} =
@@ -519,7 +521,7 @@ proc searchForInnerProcs(o: POuterContext, n: PNode) =
       else:
         internalError(it.info, "transformOuter")
   of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, 
-     nkClosure:
+     nkClosure, nkTypeSection:
     # don't recurse here:
     # XXX recurse here and setup 'up' pointers
     discard
@@ -537,13 +539,6 @@ proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode =
   result.sons[0] = le
   result.sons[1] = ri
 
-proc addVar*(father, v: PNode) = 
-  var vpart = newNodeI(nkIdentDefs, v.info)
-  addSon(vpart, v)
-  addSon(vpart, ast.emptyNode)
-  addSon(vpart, ast.emptyNode)
-  addSon(father, vpart)
-
 proc newClosureCreationVar(o: POuterContext; e: PEnv): PSym =
   result = newSym(skVar, getIdent(envName), o.fn, e.attachedNode.info)
   incl(result.flags, sfShadowed)
@@ -653,7 +648,7 @@ proc outerProcSons(o: POuterContext, n: PNode) =
 proc liftIterSym*(n: PNode): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env)) 
   let iter = n.sym
-  assert iter.kind == skIterator
+  assert iter.kind == skClosureIterator
 
   result = newNodeIT(nkStmtListExpr, n.info, n.typ)
   
@@ -679,7 +674,7 @@ proc transformOuterProc(o: POuterContext, n: PNode): PNode =
 
     var closure = PEnv(idTableGet(o.lambdasToEnv, local))
 
-    if local.kind == skIterator and local.typ.callConv == ccClosure:
+    if local.kind == skClosureIterator:
       # consider: [i1, i2, i1]  Since we merged the iterator's closure
       # with the captured owning variables, we need to generate the
       # closure generation code again:
@@ -843,10 +838,10 @@ proc liftForLoop*(body: PNode): PNode =
   
   # static binding?
   var env: PSym
-  if call[0].kind == nkSym and call[0].sym.kind == skIterator:
+  if call[0].kind == nkSym and call[0].sym.kind == skClosureIterator:
     # createClosure()
     let iter = call[0].sym
-    assert iter.kind == skIterator
+    assert iter.kind == skClosureIterator
     env = copySym(getHiddenParam(iter))
 
     var v = newNodeI(nkVarSection, body.info)

compiler/lexer.nim

@@ -101,6 +101,7 @@ type
     base10,                   # base10 is listed as the first element,
                               # so that it is the correct default value
     base2, base8, base16
+
   TToken* = object            # a Nimrod token
     tokType*: TTokType        # the type of the token
     indent*: int              # the indentation; != -1 if the token has been
@@ -110,6 +111,8 @@ type
     fNumber*: BiggestFloat    # the parsed floating point literal
     base*: TNumericalBase     # the numerical base; only valid for int
                               # or float literals
+    strongSpaceA*: int8       # leading spaces of an operator
+    strongSpaceB*: int8       # trailing spaces of an operator
     literal*: string          # the parsed (string) literal; and
                               # documentation comments are here too
     line*, col*: int
@@ -119,6 +122,8 @@ type
     indentAhead*: int         # if > 0 an indendation has already been read
                               # this is needed because scanning comments
                               # needs so much look-ahead
+    currLineIndent*: int
+    strongSpaces*: bool
 
 
 var gLinesCompiled*: int  # all lines that have been compiled
@@ -173,6 +178,7 @@ proc prettyTok*(tok: TToken): string =
   else: result = tokToStr(tok)
   
 proc printTok*(tok: TToken) = 
+  write(stdout, tok.line, ":", tok.col, "\t")
   write(stdout, TokTypeToStr[tok.tokType])
   write(stdout, " ")
   writeln(stdout, tokToStr(tok))
@@ -183,6 +189,7 @@ proc initToken*(L: var TToken) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
+  L.strongSpaceA = 0
   L.literal = ""
   L.fNumber = 0.0
   L.base = base10
@@ -192,6 +199,7 @@ proc fillToken(L: var TToken) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
+  L.strongSpaceA = 0
   setLen(L.literal, 0)
   L.fNumber = 0.0
   L.base = base10
@@ -201,6 +209,7 @@ proc openLexer(lex: var TLexer, fileIdx: int32, inputstream: PLLStream) =
   openBaseLexer(lex, inputstream)
   lex.fileIdx = fileidx
   lex.indentAhead = - 1
+  lex.currLineIndent = 0
   inc(lex.lineNumber, inputstream.lineOffset) 
 
 proc closeLexer(lex: var TLexer) = 
@@ -634,6 +643,14 @@ proc getOperator(L: var TLexer, tok: var TToken) =
     h = h !& ord(c)
     inc(pos)
   endOperator(L, tok, pos, h)
+  # advance pos but don't store it in L.bufpos so the next token (which might
+  # be an operator too) gets the preceeding spaces:
+  tok.strongSpaceB = 0
+  while buf[pos] == ' ':
+    inc pos
+    inc tok.strongSpaceB
+  if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
+    tok.strongSpaceB = -1
 
 proc scanComment(L: var TLexer, tok: var TToken) = 
   var pos = L.bufpos
@@ -677,10 +694,12 @@ proc scanComment(L: var TLexer, tok: var TToken) =
 proc skip(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
   var buf = L.buf
+  tok.strongSpaceA = 0
   while true:
     case buf[pos]
     of ' ':
       inc(pos)
+      inc(tok.strongSpaceA)
     of Tabulator:
       lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
       inc(pos)
@@ -691,8 +710,10 @@ proc skip(L: var TLexer, tok: var TToken) =
       while buf[pos] == ' ':
         inc(pos)
         inc(indent)
+      tok.strongSpaceA = 0
       if buf[pos] > ' ':
         tok.indent = indent
+        L.currLineIndent = indent
         break
     else:
       break                   # EndOfFile also leaves the loop
@@ -702,6 +723,7 @@ proc rawGetTok(L: var TLexer, tok: var TToken) =
   fillToken(tok)
   if L.indentAhead >= 0:
     tok.indent = L.indentAhead
+    L.currLineIndent = L.indentAhead
     L.indentAhead = -1
   else:
     tok.indent = -1

compiler/lookups.nim

@@ -32,6 +32,7 @@ proc considerAcc*(n: PNode): PIdent =
         of nkSym: id.add(x.sym.name.s)
         else: globalError(n.info, errIdentifierExpected, renderTree(n))
       result = getIdent(id)
+  of nkOpenSymChoice, nkClosedSymChoice: result = n.sons[0].sym.name
   else:
     globalError(n.info, errIdentifierExpected, renderTree(n))
  
@@ -91,7 +92,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
   result.typ = errorType(c)
   incl(result.flags, sfDiscardable)
   # pretend it's imported from some unknown module to prevent cascading errors:
-  if gCmd != cmdInteractive:
+  if gCmd != cmdInteractive and c.inCompilesContext == 0:
     c.importTable.addSym(result)
 
 type 
@@ -108,7 +109,7 @@ type
 
 proc getSymRepr*(s: PSym): string = 
   case s.kind
-  of skProc, skMethod, skConverter, skIterator: result = getProcHeader(s)
+  of skProc, skMethod, skConverter, skIterators: result = getProcHeader(s)
   else: result = s.name.s
 
 proc ensureNoMissingOrUnusedSymbols(scope: PScope) =
@@ -126,7 +127,10 @@ proc ensureNoMissingOrUnusedSymbols(scope: PScope) =
     elif {sfUsed, sfExported} * s.flags == {} and optHints in s.options: 
       # BUGFIX: check options in s!
       if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam}:
-        message(s.info, hintXDeclaredButNotUsed, getSymRepr(s))
+        # XXX: implicit type params are currently skTypes
+        # maybe they can be made skGenericParam as well.
+        if s.typ != nil and tfImplicitTypeParam notin s.typ.flags:
+          message(s.info, hintXDeclaredButNotUsed, getSymRepr(s))
     s = nextIter(it, scope.symbols)
   
 proc wrongRedefinition*(info: TLineInfo, s: string) =

compiler/lowerings.nim

@@ -0,0 +1,52 @@
+#
+#
+#           The Nimrod Compiler
+#        (c) Copyright 2014 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements common simple lowerings.
+
+const
+  genPrefix* = ":tmp"         # prefix for generated names
+
+import ast, types, idents, magicsys
+
+proc newTupleAccess*(tup: PNode, i: int): PNode =
+  result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(
+                     abstractInst).sons[i])
+  addSon(result, copyTree(tup))
+  var lit = newNodeIT(nkIntLit, tup.info, getSysType(tyInt))
+  lit.intVal = i
+  addSon(result, lit)
+
+proc addVar*(father, v: PNode) = 
+  var vpart = newNodeI(nkIdentDefs, v.info, 3)
+  vpart.sons[0] = v
+  vpart.sons[1] = ast.emptyNode
+  vpart.sons[2] = ast.emptyNode
+  addSon(father, vpart)
+
+proc newAsgnStmt(le, ri: PNode): PNode =
+  result = newNodeI(nkAsgn, le.info, 2)
+  result.sons[0] = le
+  result.sons[1] = ri
+
+proc lowerTupleUnpacking*(n: PNode; owner: PSym): PNode =
+  assert n.kind == nkVarTuple
+  let value = n.lastSon
+  result = newNodeI(nkStmtList, n.info)
+
+  var temp = newSym(skTemp, getIdent(genPrefix), owner, value.info)
+  temp.typ = skipTypes(value.typ, abstractInst)
+  incl(temp.flags, sfFromGeneric)
+
+  var v = newNodeI(nkVarSection, value.info)
+  v.addVar(newSymNode(temp))
+  result.add(v)
+  
+  result.add newAsgnStmt(newSymNode(temp), value)
+  for i in 0 .. n.len-3:
+    result.add newAsgnStmt(n.sons[i], newTupleAccess(value, i))

compiler/modules.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2013 Andreas Rumpf
+#        (c) Copyright 2014 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -20,7 +20,7 @@ type
   TModuleInMemory* = object
     compiledAt*: float
     crc*: TCrc32
-    deps*: seq[int32] ## XXX: slurped files are not currently tracked
+    deps*: seq[int32] ## XXX: slurped files are currently not tracked
     needsRecompile*: TNeedRecompile
     crcStatus*: TCrcStatus
 
@@ -83,7 +83,7 @@ proc resetAllModules* =
   for i in 0..gCompiledModules.high:
     if gCompiledModules[i] != nil:
       resetModule(i.int32)
-
+  resetPackageCache()
   # for m in cgenModules(): echo "CGEN MODULE FOUND"
 
 proc checkDepMem(fileIdx: int32): TNeedRecompile =
@@ -120,8 +120,9 @@ proc newModule(fileIdx: int32): PSym =
   if not isNimrodIdentifier(result.name.s):
     rawMessage(errInvalidModuleName, result.name.s)
   
-  result.owner = result       # a module belongs to itself
   result.info = newLineInfo(fileIdx, 1, 1)
+  result.owner = newSym(skPackage, getIdent(getPackageName(filename)), nil,
+                        result.info)
   result.position = fileIdx
   
   growCache gMemCacheData, fileIdx

compiler/msgs.nim

@@ -67,7 +67,7 @@ type
     errAmbiguousCallXYZ, errWrongNumberOfArguments, 
     errXCannotBePassedToProcVar, 
     errXCannotBeInParamDecl, errPragmaOnlyInHeaderOfProc, errImplOfXNotAllowed, 
-    errImplOfXexpected, errNoSymbolToBorrowFromFound, errDiscardValue, 
+    errImplOfXexpected, errNoSymbolToBorrowFromFound, errDiscardValueX, 
     errInvalidDiscard, errIllegalConvFromXtoY, errCannotBindXTwice, 
     errInvalidOrderInArrayConstructor,
     errInvalidOrderInEnumX, errEnumXHasHoles, errExceptExpected, errInvalidTry, 
@@ -89,12 +89,14 @@ type
     errInvalidIndexValueForTuple, errCommandExpectsFilename,
     errMainModuleMustBeSpecified,
     errXExpected,
+    errTIsNotAConcreteType,
     errInvalidSectionStart, errGridTableNotImplemented, errGeneralParseError, 
     errNewSectionExpected, errWhitespaceExpected, errXisNoValidIndexFile, 
     errCannotRenderX, errVarVarTypeNotAllowed, errInstantiateXExplicitely,
     errOnlyACallOpCanBeDelegator, errUsingNoSymbol,
+    errMacroBodyDependsOnGenericTypes,
     errDestructorNotGenericEnough,
-    
+    errInlineIteratorsAsProcParams,
     errXExpectsTwoArguments,
     errXExpectsObjectTypes, errXcanNeverBeOfThisSubtype, errTooManyIterations, 
     errCannotInterpretNodeX, errFieldXNotFound, errInvalidConversionFromTypeX, 
@@ -103,6 +105,10 @@ type
     errXhasSideEffects, errIteratorExpected, errLetNeedsInit,
     errThreadvarCannotInit, errWrongSymbolX, errIllegalCaptureX,
     errXCannotBeClosure, errXMustBeCompileTime,
+    errCannotInferTypeOfTheLiteral,
+    errCannotInferReturnType,
+    errGenericLambdaNotAllowed,
+    errCompilerDoesntSupportTarget,
     errUser,
     warnCannotOpenFile, 
     warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit, 
@@ -196,7 +202,7 @@ const
     errXExpectsArrayType: "\'$1\' expects an array type", 
     errIteratorCannotBeInstantiated: "'$1' cannot be instantiated because its body has not been compiled yet", 
     errExprXAmbiguous: "expression '$1' ambiguous in this context", 
-    errConstantDivisionByZero: "constant division by zero", 
+    errConstantDivisionByZero: "division by zero", 
     errOrdinalTypeExpected: "ordinal type expected", 
     errOrdinalOrFloatTypeExpected: "ordinal or float type expected", 
     errOverOrUnderflow: "over- or underflow", 
@@ -263,7 +269,7 @@ const
     errImplOfXNotAllowed: "implementation of \'$1\' is not allowed", 
     errImplOfXexpected: "implementation of \'$1\' expected", 
     errNoSymbolToBorrowFromFound: "no symbol to borrow from found", 
-    errDiscardValue: "value returned by statement has to be discarded", 
+    errDiscardValueX: "value of type '$1' has to be discarded", 
     errInvalidDiscard: "statement returns no value that can be discarded", 
     errIllegalConvFromXtoY: "conversion from $1 to $2 is invalid",
     errCannotBindXTwice: "cannot bind parameter \'$1\' twice", 
@@ -312,6 +318,7 @@ const
     errCommandExpectsFilename: "command expects a filename argument",
     errMainModuleMustBeSpecified: "please, specify a main module in the project configuration file",
     errXExpected: "\'$1\' expected", 
+    errTIsNotAConcreteType: "\'$1\' is not a concrete type.",
     errInvalidSectionStart: "invalid section start",
     errGridTableNotImplemented: "grid table is not implemented", 
     errGeneralParseError: "general parse error",
@@ -323,8 +330,12 @@ const
     errInstantiateXExplicitely: "instantiate '$1' explicitely",
     errOnlyACallOpCanBeDelegator: "only a call operator can be a delegator",
     errUsingNoSymbol: "'$1' is not a variable, constant or a proc name",
+    errMacroBodyDependsOnGenericTypes: "the macro body cannot be compiled, " &
+                                       "because the parameter '$1' has a generic type",
     errDestructorNotGenericEnough: "Destructor signarue is too specific. " &
                                    "A destructor must be associated will all instantiations of a generic type",
+    errInlineIteratorsAsProcParams: "inline iterators can be used as parameters only for " &
+                                    "templates, macros and other inline iterators",
     errXExpectsTwoArguments: "\'$1\' expects two arguments", 
     errXExpectsObjectTypes: "\'$1\' expects object types",
     errXcanNeverBeOfThisSubtype: "\'$1\' can never be of this subtype", 
@@ -346,6 +357,12 @@ const
     errIllegalCaptureX: "illegal capture '$1'",
     errXCannotBeClosure: "'$1' cannot have 'closure' calling convention",
     errXMustBeCompileTime: "'$1' can only be used in compile-time context",
+    errCannotInferTypeOfTheLiteral: "cannot infer the type of the $1",
+    errCannotInferReturnType: "cannot infer the return type of the proc",
+    errGenericLambdaNotAllowed: "A nested proc can have generic parameters only when " &
+                                "it is used as an operand to another routine and the types " &
+                                "of the generic paramers can be infered from the expected signature.",
+    errCompilerDoesntSupportTarget: "The current compiler \'$1\' doesn't support the requested compilation target",
     errUser: "$1", 
     warnCannotOpenFile: "cannot open \'$1\' [CannotOpenFile]",
     warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored [OctalEscape]", 

compiler/nimeval.nim

@@ -17,11 +17,11 @@ proc execute*(program: string) =
   passes.gIncludeFile = includeModule
   passes.gImportModule = importModule
   initDefines()
-  LoadConfigs(DefaultConfig)
+  loadConfigs(DefaultConfig)
 
   initDefines()
-  DefineSymbol("nimrodvm")
+  defineSymbol("nimrodvm")
-  when hasFFI: DefineSymbol("nimffi")
+  when hasFFI: defineSymbol("nimffi")
   registerPass(verbosePass)
   registerPass(semPass)
   registerPass(vmPass)
@@ -30,4 +30,4 @@ proc execute*(program: string) =
   compileSystemModule()
   var m = makeStdinModule()
   incl(m.flags, sfMainModule)
-  processModule(m, LLStreamOpen(program), nil)
+  processModule(m, llStreamOpen(program), nil)

compiler/nimrod.nimrod.cfg

@@ -17,4 +17,6 @@ import:testability
   cincludes: "$lib/wrappers/libffi/common"
 @end
 
+define:useStdoutAsStdmsg
+
 cs:partial

compiler/options.nim

@@ -209,21 +209,42 @@ proc getGeneratedPath: string =
   result = if nimcacheDir.len > 0: nimcacheDir else: gProjectPath.shortenDir /
                                                          genSubDir
 
+template newPackageCache(): expr =
+  newStringTable(when FileSystemCaseSensitive:
+                   modeCaseInsensitive
+                 else:
+                   modeCaseSensitive)
+
+var packageCache = newPackageCache()
+
+proc resetPackageCache*() = packageCache = newPackageCache()
+
+iterator myParentDirs(p: string): string =
+  # XXX os's parentDirs is stupid (multiple yields) and triggers an old bug...
+  var current = p
+  while true:
+    current = current.parentDir
+    if current.len == 0: break
+    yield current
+
 proc getPackageName*(path: string): string =
-  var q = 1
+  var parents = 0
-  var b = 0
+  block packageSearch:
-  if path[len(path)-1] in {DirSep, AltSep}: q = 2
+    for d in myParentDirs(path):
-  for i in countdown(len(path)-q, 0):
+      if packageCache.hasKey(d):
-    if path[i] in {DirSep, AltSep}:
+        #echo "from cache ", d, " |", packageCache[d], "|", path.splitFile.name
-      if b == 0: b = i
+        return packageCache[d]
-      else:
+      inc parents
-        let x = path.substr(i+1, b-1)
+      for file in walkFiles(d / "*.babel"):
-        case x.normalize
+        result = file.splitFile.name
-        of "lib", "src", "source", "package", "pckg", "library", "private":
+        break packageSearch
-          b = i
+  # we also store if we didn't find anything:
-        else:
+  if result.isNil: result = ""
-          return x.replace('.', '_')
+  for d in myParentDirs(path):
-  result = ""
+    #echo "set cache ", d, " |", result, "|", parents
+    packageCache[d] = result
+    dec parents
+    if parents <= 0: break
 
 proc withPackageName*(path: string): string =
   let x = path.getPackageName

compiler/parser.nim

@@ -38,7 +38,6 @@ type
     inSemiStmtList: int
 
 proc parseAll*(p: var TParser): PNode
-proc openParser*(p: var TParser, filename: string, inputstream: PLLStream)
 proc closeParser*(p: var TParser)
 proc parseTopLevelStmt*(p: var TParser): PNode
   # implements an iterator. Returns the next top-level statement or
@@ -50,7 +49,6 @@ proc parseString*(s: string, filename: string = "", line: int = 0): PNode
   # correct error messages referring to the original source.
   
 # helpers for the other parsers
-proc getPrecedence*(tok: TToken): int
 proc isOperator*(tok: TToken): bool
 proc getTok*(p: var TParser)
 proc parMessage*(p: TParser, msg: TMsgKind, arg: string = "")
@@ -69,7 +67,7 @@ proc optPar*(p: var TParser)
 proc optInd*(p: var TParser, n: PNode)
 proc indAndComment*(p: var TParser, n: PNode)
 proc setBaseFlags*(n: PNode, base: TNumericalBase)
-proc parseSymbol*(p: var TParser): PNode
+proc parseSymbol*(p: var TParser, allowNil = false): PNode
 proc parseTry(p: var TParser): PNode
 proc parseCase(p: var TParser): PNode
 # implementation
@@ -77,14 +75,17 @@ proc parseCase(p: var TParser): PNode
 proc getTok(p: var TParser) = 
   rawGetTok(p.lex, p.tok)
 
-proc openParser*(p: var TParser, fileIdx: int32, inputStream: PLLStream) =
+proc openParser*(p: var TParser, fileIdx: int32, inputStream: PLLStream,
+                 strongSpaces=false) =
   initToken(p.tok)
   openLexer(p.lex, fileIdx, inputStream)
   getTok(p)                   # read the first token
   p.firstTok = true
+  p.strongSpaces = strongSpaces
 
-proc openParser*(p: var TParser, filename: string, inputStream: PLLStream) =
+proc openParser*(p: var TParser, filename: string, inputStream: PLLStream,
-  openParser(p, filename.fileInfoIdx, inputstream)
+                 strongSpaces=false) =
+  openParser(p, filename.fileInfoIdx, inputstream, strongSpaces)
 
 proc closeParser(p: var TParser) = 
   closeLexer(p.lex)
@@ -193,34 +194,52 @@ proc isSigilLike(tok: TToken): bool {.inline.} =
 proc isLeftAssociative(tok: TToken): bool {.inline.} =
   result = tok.tokType != tkOpr or relevantOprChar(tok.ident) != '^'
 
-proc getPrecedence(tok: TToken): int = 
+proc getPrecedence(tok: TToken, strongSpaces: bool): int =
+  template considerStrongSpaces(x): expr =
+    x + (if strongSpaces: 100 - tok.strongSpaceA.int*10 else: 0)
+
   case tok.tokType
   of tkOpr:
     let L = tok.ident.s.len
     let relevantChar = relevantOprChar(tok.ident)
     
     template considerAsgn(value: expr) =
-      result = if tok.ident.s[L-1] == '=': 1 else: value     
+      result = if tok.ident.s[L-1] == '=': 1 else: considerStrongSpaces(value)
     
     case relevantChar
     of '$', '^': considerAsgn(10)
     of '*', '%', '/', '\\': considerAsgn(9)
-    of '~': result = 8
+    of '~': result = considerStrongSpaces(8)
     of '+', '-', '|': considerAsgn(8)
     of '&': considerAsgn(7)
-    of '=', '<', '>', '!': result = 5
+    of '=', '<', '>', '!': result = considerStrongSpaces(5)
     of '.': considerAsgn(6)
-    of '?': result = 2
+    of '?': result = considerStrongSpaces(2)
     else: considerAsgn(2)
   of tkDiv, tkMod, tkShl, tkShr: result = 9
   of tkIn, tkNotin, tkIs, tkIsnot, tkNot, tkOf, tkAs: result = 5
-  of tkDotDot: result = 6
+  of tkDotDot: result = considerStrongSpaces(6)
   of tkAnd: result = 4
   of tkOr, tkXor: result = 3
-  else: result = - 10
+  else: result = -10
 
 proc isOperator(tok: TToken): bool =
-  result = getPrecedence(tok) >= 0
+  tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
+                  tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
+
+proc isUnary(p: TParser): bool =
+  p.strongSpaces and p.tok.tokType in {tkOpr, tkDotDot} and
+    p.tok.strongSpaceB == 0 and
+    p.tok.strongSpaceA > 0
+
+proc checkBinary(p: TParser) {.inline.} =
+  # we don't check '..' here as that's too annoying
+  if p.strongSpaces and p.tok.tokType == tkOpr:
+    if p.tok.strongSpaceB > 0 and p.tok.strongSpaceA != p.tok.strongSpaceB:
+      parMessage(p, errGenerated, "number of spaces around '$#' not consistent"%
+        prettyTok(p.tok))
+    elif p.tok.strongSpaceA notin {0,1,2,4,8}:
+      parMessage(p, errGenerated, "number of spaces must be 0,1,2,4 or 8")
 
 #| module = stmt ^* (';' / IND{=})
 #|
@@ -254,7 +273,7 @@ proc colcom(p: var TParser, n: PNode) =
   eat(p, tkColon)
   skipComment(p, n)
 
-proc parseSymbol(p: var TParser): PNode =
+proc parseSymbol(p: var TParser, allowNil = false): PNode =
   #| symbol = '`' (KEYW|IDENT|operator|'(' ')'|'[' ']'|'{' '}'|'='|literal)+ '`'
   #|        | IDENT
   case p.tok.tokType
@@ -281,7 +300,7 @@ proc parseSymbol(p: var TParser): PNode =
         add(result, newIdentNodeP(getIdent"{}", p))
         getTok(p)
         eat(p, tkCurlyRi)
-      of tokKeywordLow..tokKeywordHigh, tkSymbol, tkOpr, tkDotDot:
+      of tokKeywordLow..tokKeywordHigh, tkSymbol, tkOpr, tkDot, tkDotDot:
         add(result, newIdentNodeP(p.tok.ident, p))
         getTok(p)
       of tkIntLit..tkCharLit:
@@ -293,9 +312,13 @@ proc parseSymbol(p: var TParser): PNode =
         break
     eat(p, tkAccent)
   else:
-    parMessage(p, errIdentifierExpected, p.tok)
+    if allowNil and p.tok.tokType == tkNil:
-    getTok(p) # BUGFIX: We must consume a token here to prevent endless loops!
+      result = newNodeP(nkNilLit, p)
-    result = ast.emptyNode
+      getTok(p)
+    else:
+      parMessage(p, errIdentifierExpected, p.tok)
+      getTok(p) # BUGFIX: We must consume a token here to prevent endless loops!
+      result = ast.emptyNode
 
 proc indexExpr(p: var TParser): PNode = 
   #| indexExpr = expr
@@ -502,6 +525,8 @@ proc parsePar(p: var TParser): PNode =
       asgn.sons[0] = a
       asgn.sons[1] = b
       result.add(asgn)
+      if p.tok.tokType == tkSemiColon:
+        semiStmtList(p, result)
     elif p.tok.tokType == tkSemiColon:
       # stmt context:
       result.add(a)
@@ -639,7 +664,7 @@ proc namedParams(p: var TParser, callee: PNode,
   exprColonEqExprListAux(p, endTok, result)
 
 proc parseMacroColon(p: var TParser, x: PNode): PNode
-proc primarySuffix(p: var TParser, r: PNode): PNode =
+proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
   #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks?
   #|               | doBlocks
   #|               | '.' optInd ('type' | 'addr' | symbol) generalizedLit?
@@ -647,9 +672,11 @@ proc primarySuffix(p: var TParser, r: PNode): PNode =
   #|               | '{' optInd indexExprList optPar '}'
   #|               | &( '`'|IDENT|literal|'cast') expr # command syntax
   result = r
-  while p.tok.indent < 0:
+  while p.tok.indent < 0 or
+       (p.tok.tokType == tkDot and p.tok.indent >= baseIndent):
     case p.tok.tokType
     of tkParLe:
+      if p.strongSpaces and p.tok.strongSpaceA > 0: break
       result = namedParams(p, result, nkCall, tkParRi)
       if result.len > 1 and result.sons[1].kind == nkExprColonExpr:
         result.kind = nkObjConstr
@@ -664,8 +691,10 @@ proc primarySuffix(p: var TParser, r: PNode): PNode =
       result = dotExpr(p, result)
       result = parseGStrLit(p, result)
     of tkBracketLe:
+      if p.strongSpaces and p.tok.strongSpaceA > 0: break
       result = namedParams(p, result, nkBracketExpr, tkBracketRi)
     of tkCurlyLe:
+      if p.strongSpaces and p.tok.strongSpaceA > 0: break
       result = namedParams(p, result, nkCurlyExpr, tkCurlyRi)
     of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast:
       if p.inPragma == 0:
@@ -691,14 +720,17 @@ proc primarySuffix(p: var TParser, r: PNode): PNode =
       break
     
 proc primary(p: var TParser, mode: TPrimaryMode): PNode
+proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode
 
-proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode =
+proc parseOperators(p: var TParser, headNode: PNode,
-  result = primary(p, mode)
+                    limit: int, mode: TPrimaryMode): PNode =
+  result = headNode
   # expand while operators have priorities higher than 'limit'
-  var opPrec = getPrecedence(p.tok)
+  var opPrec = getPrecedence(p.tok, p.strongSpaces)
   let modeB = if mode == pmTypeDef: pmTypeDesc else: mode
   # the operator itself must not start on a new line:
-  while opPrec >= limit and p.tok.indent < 0:
+  while opPrec >= limit and p.tok.indent < 0 and not isUnary(p):
+    checkBinary(p)
     var leftAssoc = ord(isLeftAssociative(p.tok))
     var a = newNodeP(nkInfix, p)
     var opNode = newIdentNodeP(p.tok.ident, p) # skip operator:
@@ -710,7 +742,11 @@ proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode =
     addSon(a, result)
     addSon(a, b)
     result = a
-    opPrec = getPrecedence(p.tok)
+    opPrec = getPrecedence(p.tok, p.strongSpaces)
+
+proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode =
+  result = primary(p, mode)
+  result = parseOperators(p, result, limit, mode)
   
 proc simpleExpr(p: var TParser, mode = pmNormal): PNode =
   result = simpleExprAux(p, -1, mode)
@@ -935,6 +971,15 @@ proc isExprStart(p: TParser): bool =
     result = true
   else: result = false
 
+proc parseSymbolList(p: var TParser, result: PNode, allowNil = false) =
+  while true:
+    var s = parseSymbol(p, allowNil)
+    if s.kind == nkEmpty: break
+    addSon(result, s)
+    if p.tok.tokType != tkComma: break
+    getTok(p)
+    optInd(p, s)
+
 proc parseTypeDescKAux(p: var TParser, kind: TNodeKind,
                        mode: TPrimaryMode): PNode =
   result = newNodeP(kind, p)
@@ -942,6 +987,13 @@ proc parseTypeDescKAux(p: var TParser, kind: TNodeKind,
   optInd(p, result)
   if not isOperator(p.tok) and isExprStart(p):
     addSon(result, primary(p, mode))
+  if kind == nkDistinctTy and p.tok.tokType in {tkWith, tkWithout}:
+    let nodeKind = if p.tok.tokType == tkWith: nkWith
+                   else: nkWithout
+    getTok(p)
+    let list = newNodeP(nodeKind, p)
+    result.addSon list
+    parseSymbolList(p, list, allowNil = true)
 
 proc parseExpr(p: var TParser): PNode = 
   #| expr = (ifExpr
@@ -958,7 +1010,6 @@ proc parseExpr(p: var TParser): PNode =
 
 proc parseEnum(p: var TParser): PNode
 proc parseObject(p: var TParser): PNode
-proc parseDistinct(p: var TParser): PNode
 proc parseTypeClass(p: var TParser): PNode
 
 proc primary(p: var TParser, mode: TPrimaryMode): PNode = 
@@ -978,8 +1029,9 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
     optInd(p, a)
     if isSigil: 
       #XXX prefix operators
+      let baseInd = p.lex.currLineIndent
       addSon(result, primary(p, pmSkipSuffix))
-      result = primarySuffix(p, result)
+      result = primarySuffix(p, result, baseInd)
     else:
       addSon(result, primary(p, pmNormal))
     return
@@ -1042,9 +1094,10 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
     optInd(p, result)
     addSon(result, primary(p, pmNormal))
   else:
+    let baseInd = p.lex.currLineIndent
     result = identOrLiteral(p, mode)
     if mode != pmSkipSuffix:
-      result = primarySuffix(p, result)
+      result = primarySuffix(p, result, baseInd)
 
 proc parseTypeDesc(p: var TParser): PNode =
   #| typeDesc = simpleExpr
@@ -1478,7 +1531,7 @@ proc parseSection(p: var TParser, kind: TNodeKind,
                   defparser: TDefParser): PNode =
   #| section(p) = COMMENT? p / (IND{>} (p / COMMENT)^+IND{=} DED)
   result = newNodeP(kind, p)
-  getTok(p)
+  if kind != nkTypeSection: getTok(p)
   skipComment(p, result)
   if realInd(p):
     withInd(p):
@@ -1711,13 +1764,6 @@ proc parseTypeClass(p: var TParser): PNode =
   else:
     addSon(result, parseStmt(p))
 
-proc parseDistinct(p: var TParser): PNode = 
-  #| distinct = 'distinct' optInd typeDesc
-  result = newNodeP(nkDistinctTy, p)
-  getTok(p)
-  optInd(p, result)
-  addSon(result, parseTypeDesc(p))
-
 proc parseTypeDef(p: var TParser): PNode = 
   #| typeDef = identWithPragma genericParamList? '=' optInd typeDefAux
   #|             indAndComment?
@@ -1839,7 +1885,16 @@ proc complexOrSimpleStmt(p: var TParser): PNode =
   of tkMacro: result = parseRoutine(p, nkMacroDef)
   of tkTemplate: result = parseRoutine(p, nkTemplateDef)
   of tkConverter: result = parseRoutine(p, nkConverterDef)
-  of tkType: result = parseSection(p, nkTypeSection, parseTypeDef)
+  of tkType:
+    getTok(p)
+    if p.tok.tokType == tkParLe:
+      getTok(p)
+      result = newNodeP(nkTypeOfExpr, p)
+      result.addSon(primary(p, pmTypeDesc))
+      eat(p, tkParRi)
+      result = parseOperators(p, result, -1, pmNormal)
+    else:
+      result = parseSection(p, nkTypeSection, parseTypeDef)
   of tkConst: result = parseSection(p, nkConstSection, parseConstant)
   of tkLet: result = parseSection(p, nkLetSection, parseVariable)
   of tkWhen: result = parseIfOrWhen(p, nkWhenStmt)
@@ -1863,7 +1918,7 @@ proc parseStmt(p: var TParser): PNode =
           if p.tok.indent < 0 or p.tok.indent == p.currInd: discard
           else: break
         else:
-          if p.tok.indent > p.currInd:
+          if p.tok.indent > p.currInd and p.tok.tokType != tkDot:
             parMessage(p, errInvalidIndentation)
           break
         if p.tok.tokType in {tkCurlyRi, tkParRi, tkCurlyDotRi, tkBracketRi}:
@@ -1890,7 +1945,8 @@ proc parseStmt(p: var TParser): PNode =
       else:
         result = newNodeP(nkStmtList, p)
         while true:
-          if p.tok.indent >= 0: parMessage(p, errInvalidIndentation)     
+          if p.tok.indent >= 0:
+            parMessage(p, errInvalidIndentation)
           let a = simpleStmt(p)
           if a.kind == nkEmpty: parMessage(p, errExprExpected, p.tok)
           result.add(a)
@@ -1933,7 +1989,9 @@ proc parseString(s: string, filename: string = "", line: int = 0): PNode =
   stream.lineOffset = line
 
   var parser: TParser
-  openParser(parser, filename, stream)
+  # XXX for now the builtin 'parseStmt/Expr' functions do not know about strong
+  # spaces...
+  openParser(parser, filename, stream, false)
 
   result = parser.parseAll
   closeParser(parser)

compiler/pas2nim/paslex.nim

@@ -342,7 +342,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       h = h +% ord(c)
       h = h +% h shl 10
       h = h xor (h shr 6)
-    of '_': nil
+    of '_': discard
     else: break
     inc(pos)
   h = h +% h shl 3

compiler/pas2nim/pasparse.nim

@@ -335,7 +335,7 @@ proc exprColonEqExprList(p: var TParser, kind, elemKind: TNodeKind,
 
 proc setBaseFlags(n: PNode, base: TNumericalBase) = 
   case base
-  of base10: nil
+  of base10: discard
   of base2: incl(n.flags, nfBase2)
   of base8: incl(n.flags, nfBase8)
   of base16: incl(n.flags, nfBase16)
@@ -466,7 +466,7 @@ proc lowestExprAux(p: var TParser, v: var PNode, limit: int): TTokKind =
         eat(p, pxCurlyDirRi)
         opNode.ident = getIdent("&")
       else: 
-        nil
+        discard
     of pxMinus: 
       if p.tok.xkind == pxPer: 
         getTok(p)
@@ -477,7 +477,7 @@ proc lowestExprAux(p: var TParser, v: var PNode, limit: int): TTokKind =
     of pxNeq: 
       opNode.ident = getIdent("!=")
     else: 
-      nil
+      discard
     skipCom(p, opNode)        # read sub-expression with higher priority
     nextop = lowestExprAux(p, v2, opPred)
     addSon(node, opNode)
@@ -505,7 +505,7 @@ proc fixExpr(n: PNode): PNode =
             (n.sons[2].kind in {nkCharLit, nkStrLit}): 
           n.sons[0].ident = getIdent("&") # fix operator
   else: 
-    nil
+    discard
   if not (n.kind in {nkEmpty..nkNilLit}): 
     for i in countup(0, sonsLen(n) - 1): result.sons[i] = fixExpr(n.sons[i])
   
@@ -603,7 +603,7 @@ proc parseStmtList(p: var TParser): PNode =
     of pxCurlyDirLe, pxStarDirLe: 
       if not isHandledDirective(p): break 
     else: 
-      nil
+      discard
     addSon(result, parseStmt(p))
   if sonsLen(result) == 1: result = result.sons[0]
   
@@ -732,7 +732,7 @@ proc parseRepeat(p: var TParser): PNode =
   addSon(b, c)
   addSon(a, b)
   if b.sons[0].kind == nkIdent and b.sons[0].ident.id == getIdent("false").id: 
-    nil
+    discard
   else: 
     addSon(s, a)
   addSon(result, s)
@@ -840,7 +840,7 @@ proc parseParam(p: var TParser): PNode =
     getTok(p)
     v = newNodeP(nkVarTy, p)
   else: 
-    nil
+    discard
   while true: 
     case p.tok.xkind
     of pxSymbol: a = createIdentNodeP(p.tok.ident, p)
@@ -1133,7 +1133,7 @@ proc parseRecordPart(p: var TParser): PNode =
 proc exSymbol(n: var PNode) = 
   case n.kind
   of nkPostfix: 
-    nil
+    discard
   of nkPragmaExpr: 
     exSymbol(n.sons[0])
   of nkIdent, nkAccQuoted: 
@@ -1154,7 +1154,7 @@ proc fixRecordDef(n: var PNode) =
     for i in countup(0, sonsLen(n) - 1): fixRecordDef(n.sons[i])
   of nkIdentDefs: 
     for i in countup(0, sonsLen(n) - 3): exSymbol(n.sons[i])
-  of nkNilLit, nkEmpty: nil
+  of nkNilLit, nkEmpty: discard
   else: internalError(n.info, "fixRecordDef(): " & $n.kind)
   
 proc addPragmaToIdent(ident: var PNode, pragma: PNode) = 
@@ -1191,7 +1191,7 @@ proc parseRecordBody(p: var TParser, result, definition: PNode) =
     if definition != nil: addPragmaToIdent(definition.sons[0], parseCommand(p))
     else: internalError(result.info, "anonymous record is not supported")
   else: 
-    nil
+    discard
   opt(p, pxSemicolon)
   skipCom(p, result)
 
@@ -1399,7 +1399,7 @@ proc fixVarSection(p: var TParser, counter: PNode) =
 
 proc exSymbols(n: PNode) = 
   case n.kind
-  of nkEmpty..nkNilLit: nil
+  of nkEmpty..nkNilLit: discard
   of nkProcDef..nkIteratorDef: exSymbol(n.sons[namePos])
   of nkWhenStmt, nkStmtList: 
     for i in countup(0, sonsLen(n) - 1): exSymbols(n.sons[i])
@@ -1410,7 +1410,7 @@ proc exSymbols(n: PNode) =
       exSymbol(n.sons[i].sons[0])
       if n.sons[i].sons[2].kind == nkObjectTy: 
         fixRecordDef(n.sons[i].sons[2])
-  else: nil
+  else: discard
 
 proc parseBegin(p: var TParser, result: PNode) = 
   getTok(p)

compiler/pragmas.nim

@@ -23,7 +23,7 @@ const
     wMagic, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader, 
     wCompilerproc, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, 
     wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC,
-    wNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
+    wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
     wGensym, wInject, wRaises, wTags, wOperator, wDelegator}
   converterPragmas* = procPragmas
   methodPragmas* = procPragmas
@@ -47,12 +47,13 @@ const
     wInjectStmt}
   lambdaPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl, 
     wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader, 
-    wDeprecated, wExtern, wThread, wImportCpp, wImportObjC, wNoStackFrame,
+    wDeprecated, wExtern, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame,
     wRaises, wTags}
   typePragmas* = {wImportc, wExportc, wDeprecated, wMagic, wAcyclic, wNodecl, 
     wPure, wHeader, wCompilerproc, wFinal, wSize, wExtern, wShallow,
     wImportCpp, wImportObjC, wError, wIncompleteStruct, wByCopy, wByRef,
-    wInheritable, wGensym, wInject, wRequiresInit}
+    wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
+    wBorrow}
   fieldPragmas* = {wImportc, wExportc, wDeprecated, wExtern, 
     wImportCpp, wImportObjC, wError}
   varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl, 
@@ -97,8 +98,22 @@ proc makeExternImport(s: PSym, extname: string) =
   incl(s.flags, sfImportc)
   excl(s.flags, sfForward)
 
-proc makeExternExport(s: PSym, extname: string) = 
+proc validateExternCName(s: PSym, info: TLineInfo) =
+  ## Validates that the symbol name in s.loc.r is a valid C identifier.
+  ##
+  ## Valid identifiers are those alphanumeric including the underscore not
+  ## starting with a number. If the check fails, a generic error will be
+  ## displayed to the user.
+  let target = ropeToStr(s.loc.r)
+  if target.len < 1 or target[0] notin IdentStartChars or
+      not target.allCharsInSet(IdentChars):
+    localError(info, errGenerated, "invalid exported symbol")
+
+proc makeExternExport(s: PSym, extname: string, info: TLineInfo) =
   setExternName(s, extname)
+  # XXX to fix make it work with nimrtl.
+  #if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC}:
+  #  validateExternCName(s, info)
   incl(s.flags, sfExportc)
 
 proc processImportCompilerProc(s: PSym, extname: string) =
@@ -498,6 +513,13 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) =
   else:
     invalidPragma(n)
 
+proc typeBorrow(sym: PSym, n: PNode) =
+  if n.kind == nkExprColonExpr:
+    let it = n.sons[1]
+    if it.kind != nkAccQuoted:
+      localError(n.info, "a type can only borrow `.` for now")
+  incl(sym.typ.flags, tfBorrowDot)
+
 proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
                   validPragmas: TSpecialWords): bool =
   var it = n.sons[i]
@@ -515,7 +537,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
       if k in validPragmas: 
         case k
         of wExportc: 
-          makeExternExport(sym, getOptionalStr(c, it, "$1"))
+          makeExternExport(sym, getOptionalStr(c, it, "$1"), it.info)
           incl(sym.flags, sfUsed) # avoid wrong hints
         of wImportc: makeExternImport(sym, getOptionalStr(c, it, "$1"))
         of wImportCompilerProc:
@@ -548,9 +570,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         of wNodecl: 
           noVal(it)
           incl(sym.loc.flags, lfNoDecl)
-        of wPure, wNoStackFrame:
+        of wPure, wAsmNoStackFrame:
           noVal(it)
-          if sym != nil: incl(sym.flags, sfPure)
+          if sym != nil:
+            if k == wPure and sym.kind in routineKinds: invalidPragma(it)
+            else: incl(sym.flags, sfPure)
         of wVolatile: 
           noVal(it)
           incl(sym.flags, sfVolatile)
@@ -601,7 +625,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           processDynLib(c, it, sym)
         of wCompilerproc: 
           noVal(it)           # compilerproc may not get a string!
-          makeExternExport(sym, "$1")
+          makeExternExport(sym, "$1", it.info)
           incl(sym.flags, sfCompilerProc)
           incl(sym.flags, sfUsed) # suppress all those stupid warnings
           registerCompilerProc(sym)
@@ -617,8 +641,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           if sym.typ == nil: invalidPragma(it)
           else: incl(sym.typ.flags, tfVarargs)
         of wBorrow:
-          noVal(it)
+          if sym.kind == skType:
-          incl(sym.flags, sfBorrow)
+            typeBorrow(sym, it)
+          else:
+            noVal(it)
+            incl(sym.flags, sfBorrow)
         of wFinal: 
           noVal(it)
           if sym.typ == nil: invalidPragma(it)
@@ -640,6 +667,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           incl(sym.flags, sfThread)
           incl(sym.flags, sfProcvar)
           if sym.typ != nil: incl(sym.typ.flags, tfThread)
+        of wPacked:
+          noVal(it)
+          if sym.typ == nil: invalidPragma(it)
+          else: incl(sym.typ.flags, tfPacked)
         of wHint: message(it.info, hintUser, expectStrLit(c, it))
         of wWarning: message(it.info, warnUser, expectStrLit(c, it))
         of wError: 
@@ -699,6 +730,14 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           noVal(it)
           if sym.typ == nil: invalidPragma(it)
           else: incl(sym.typ.flags, tfIncompleteStruct)
+        of wUnchecked:
+          noVal(it)
+          if sym.typ == nil: invalidPragma(it)
+          else: incl(sym.typ.flags, tfUncheckedArray)
+        of wUnion:
+          noVal(it)
+          if sym.typ == nil: invalidPragma(it)
+          else: incl(sym.typ.flags, tfUnion)
         of wRequiresInit:
           noVal(it)
           if sym.typ == nil: invalidPragma(it)
@@ -732,8 +771,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
       else: invalidPragma(it)
   else: processNote(c, it)
 
-proc implictPragmas*(c: PContext, sym: PSym, n: PNode,
+proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
-                     validPragmas: TSpecialWords) =
+                      validPragmas: TSpecialWords) =
   if sym != nil and sym.kind != skModule:
     var it = POptionEntry(c.optionStack.head)
     while it != nil:
@@ -753,8 +792,19 @@ proc implictPragmas*(c: PContext, sym: PSym, n: PNode,
       addToLib(lib, sym)
       if sym.loc.r == nil: sym.loc.r = toRope(sym.name.s)
 
+proc hasPragma*(n: PNode, pragma: TSpecialWord): bool =
+  if n == nil or n.sons == nil:
+    return false
+
+  for p in n.sons:
+    var key = if p.kind == nkExprColonExpr: p[0] else: p
+    if key.kind == nkIdent and whichKeyword(key.ident) == pragma:
+      return true
+  
+  return false
+
 proc pragma(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords) =
   if n == nil: return
   for i in countup(0, sonsLen(n) - 1):
     if singlePragma(c, sym, n, i, validPragmas): break
-  implictPragmas(c, sym, n, validPragmas)
+  implicitPragmas(c, sym, n, validPragmas)

compiler/renderer.nim

@@ -424,8 +424,11 @@ proc lsub(n: PNode): int =
   of nkRefTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("ref")
   of nkPtrTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("ptr")
   of nkVarTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("var")
-  of nkDistinctTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) +
+  of nkDistinctTy:
-                                                         len("Distinct")
+    result = len("distinct") + (if n.len > 0: lsub(n.sons[0])+1 else: 0)
+    if n.len > 1:
+      result += (if n[1].kind == nkWith: len("_with_") else: len("_without_"))
+      result += lcomma(n[1])
   of nkStaticTy: result = (if n.len > 0: lsub(n.sons[0]) else: 0) +
                                                          len("static[]")
   of nkTypeDef: result = lsons(n) + 3
@@ -558,15 +561,18 @@ proc longMode(n: PNode, start: int = 0, theEnd: int = - 1): bool =
         result = true
         break 
 
-proc gstmts(g: var TSrcGen, n: PNode, c: TContext) = 
+proc gstmts(g: var TSrcGen, n: PNode, c: TContext, doIndent=true) =
   if n.kind == nkEmpty: return
   if n.kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
-    indentNL(g)
+    if doIndent: indentNL(g)
     for i in countup(0, sonsLen(n) - 1):
       optNL(g)
-      gsub(g, n.sons[i])
+      if n.sons[i].kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
+        gstmts(g, n.sons[i], c, doIndent=false)
+      else:
+        gsub(g, n.sons[i])
       gcoms(g)
-    dedent(g)
+    if doIndent: dedent(g)
   else:
     if rfLongMode in c.flags: indentNL(g)
     gsub(g, n)
@@ -1017,9 +1023,15 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     else:
       put(g, tkVar, "var")
   of nkDistinctTy: 
-    if sonsLen(n) > 0:
+    if n.len > 0:
       putWithSpace(g, tkDistinct, "distinct")
       gsub(g, n.sons[0])
+      if n.len > 1:
+        if n[1].kind == nkWith:
+          putWithSpace(g, tkWith, " with")
+        else:
+          putWithSpace(g, tkWithout, " without")
+        gcomma(g, n[1])
     else:
       put(g, tkDistinct, "distinct")
   of nkTypeDef: 
@@ -1268,7 +1280,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
       put(g, tkBracketLe, "[")
       gcomma(g, n)
       put(g, tkBracketRi, "]")
-  of nkMetaNode:
+  of nkMetaNode_Obsolete:
     put(g, tkParLe, "(META|")
     gsub(g, n.sons[0])
     put(g, tkParRi, ")")

compiler/rodread.nim

@@ -890,7 +890,7 @@ proc loadStub*(s: PSym) =
   
   # deactivate the GC here because we do a deep recursion and generate no
   # garbage when restoring parts of the object graph anyway.
-  # Since we die with internal errors if this fails, so no try-finally is
+  # Since we die with internal errors if this fails, no try-finally is
   # necessary.
   GC_disable()
   rawLoadStub(s)

compiler/sem.nim

@@ -66,6 +66,9 @@ proc fitNode(c: PContext, formal: PType, arg: PNode): PNode =
       result = copyTree(arg)
       result.typ = formal
 
+proc inferWithMetatype(c: PContext, formal: PType,
+                       arg: PNode, coerceDistincts = false): PNode
+
 var commonTypeBegin = PType(kind: tyExpr)
 
 proc commonType*(x, y: PType): PType =
@@ -120,7 +123,8 @@ proc commonType*(x, y: PType): PType =
     if a.kind == tyObject and b.kind == tyObject:
       result = commonSuperclass(a, b)
       # this will trigger an error later:
-      if result.isNil: return x
+      if result.isNil or result == a: return x
+      if result == b: return y
       if k != tyNone:
         let r = result
         result = newType(k, r.owner)
@@ -138,6 +142,10 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
     result = n.sym
     internalAssert sfGenSym in result.flags
     internalAssert result.kind == kind
+    # when there is a nested proc inside a template, semtmpl
+    # will assign a wrong owner during the first pass over the
+    # template; we must fix it here: see #909
+    result.owner = getCurrOwner()
   else:
     result = newSym(kind, considerAcc(n), getCurrOwner(), n.info)
 
@@ -193,7 +201,8 @@ proc fixupTypeAfterEval(c: PContext, evaluated, eOrig: PNode): PNode =
       result = semExprWithType(c, evaluated)
     else:
       result = evaluated
-      semmacrosanity.annotateType(result, eOrig.typ)
+      let expectedType = eOrig.typ.skipTypes({tyStatic})
+      semmacrosanity.annotateType(result, expectedType)
   else:
     result = semExprWithType(c, evaluated)
     #result = fitNode(c, e.typ, result) inlined with special case:
@@ -213,14 +222,26 @@ proc tryConstExpr(c: PContext, n: PNode): PNode =
   result = getConstExpr(c.module, e)
   if result != nil: return
 
+  let oldErrorCount = msgs.gErrorCounter
+  let oldErrorMax = msgs.gErrorMax
+  let oldErrorOutputs = errorOutputs
+
+  errorOutputs = {}
+  msgs.gErrorMax = high(int)
+
   try:
     result = evalConstExpr(c.module, e)
     if result == nil or result.kind == nkEmpty:
-      return nil
+      result = nil
+    else:
+      result = fixupTypeAfterEval(c, result, e)
 
-    result = fixupTypeAfterEval(c, result, e)
+  except ERecoverableError:
-  except:
+    result = nil
-    return nil
+
+  msgs.gErrorCounter = oldErrorCount
+  msgs.gErrorMax = oldErrorMax
+  errorOutputs = oldErrorOutputs
 
 proc semConstExpr(c: PContext, n: PNode): PNode =
   var e = semExprWithType(c, n)
@@ -332,6 +353,8 @@ proc myOpen(module: PSym): PPassContext =
   c.semOperand = semOperand
   c.semConstBoolExpr = semConstBoolExpr
   c.semOverloadedCall = semOverloadedCall
+  c.semInferredLambda = semInferredLambda
+  c.semGenerateInstance = generateInstance
   c.semTypeNode = semTypeNode
   pushProcCon(c, module)
   pushOwner(c.module)

compiler/semcall.nim

@@ -64,7 +64,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
             errors[errors.len - 1].add("\n  " & err)
       if z.state == csMatch:
         # little hack so that iterators are preferred over everything else:
-        if sym.kind == skIterator: inc(z.exactMatches, 200)
+        if sym.kind in skIterators: inc(z.exactMatches, 200)
         case best.state
         of csEmpty, csNoMatch: best = z
         of csMatch:
@@ -82,7 +82,7 @@ proc notFoundError*(c: PContext, n: PNode, errors: seq[string]) =
     # fail fast:
     globalError(n.info, errTypeMismatch, "")
   var result = msgKindToString(errTypeMismatch)
-  add(result, describeArgs(c, n, 1 + ord(nfDelegate in n.flags)))
+  add(result, describeArgs(c, n, 1))
   add(result, ')')
   
   var candidates = ""
@@ -138,15 +138,33 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
 
   let overloadsState = result.state
   if overloadsState != csMatch:
-    if nfDelegate in n.flags:
+    if nfDotField in n.flags:
-      internalAssert f.kind == nkIdent
+      internalAssert f.kind == nkIdent and n.sonsLen >= 2
-      let calleeName = newStrNode(nkStrLit, f.ident.s)
+      let calleeName = newStrNode(nkStrLit, f.ident.s).withInfo(n.info)
-      calleeName.info = n.info
 
-      let callOp = newIdentNode(idDelegator, n.info)
+      # leave the op head symbol empty,
-      n.sons[0..0] = [callOp, calleeName]
+      # we are going to try multiple variants
-      orig.sons[0..0] = [callOp, calleeName]
+      n.sons[0..1] = [nil, n[1], calleeName]
+      orig.sons[0..1] = [nil, orig[1], calleeName]
       
+      template tryOp(x) =
+        let op = newIdentNode(getIdent(x), n.info)
+        n.sons[0] = op
+        orig.sons[0] = op
+        pickBest(op)
+
+      if nfExplicitCall in n.flags:
+        tryOp ".()"
+   
+      if result.state in {csEmpty, csNoMatch}:
+        tryOp "."
+
+    elif nfDotSetter in n.flags:
+      internalAssert f.kind == nkIdent and n.sonsLen == 3
+      let calleeName = newStrNode(nkStrLit, f.ident.s[0.. -2]).withInfo(n.info)
+      let callOp = newIdentNode(getIdent".=", n.info)
+      n.sons[0..1] = [callOp, n[1], calleeName]
+      orig.sons[0..1] = [callOp, orig[1], calleeName]
       pickBest(callOp)
    
     if overloadsState == csEmpty and result.state == csEmpty:
@@ -157,9 +175,15 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
         localError(n.info, errExprXCannotBeCalled,
                    renderTree(n, {renderNoComments}))
       else:
+        if {nfDotField, nfDotSetter} * n.flags != {}:
+          # clean up the inserted ops
+          n.sons.delete(2)
+          n.sons[0] = f
+
         errors = @[]
         pickBest(f)
         notFoundError(c, n, errors)
+
       return
 
   if alt.state == csMatch and cmpCandidates(result, alt) == 0 and
@@ -204,12 +228,25 @@ proc indexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode =
   if m.genericConverter and result != nil:
     instGenericConvertersArg(c, result, m)
 
-proc convertTo*(c: PContext, f: PType, n: PNode): PNode = 
+proc inferWithMetatype(c: PContext, formal: PType,
+                       arg: PNode, coerceDistincts = false): PNode =
   var m: TCandidate
-  initCandidate(c, m, f)
+  initCandidate(c, m, formal)
-  result = paramTypesMatch(m, f, n.typ, n, nil)
+  m.coerceDistincts = coerceDistincts
+  result = paramTypesMatch(m, formal, arg.typ, arg, nil)
   if m.genericConverter and result != nil:
     instGenericConvertersArg(c, result, m)
+  if result != nil:
+    # This almost exactly replicates the steps taken by the compiler during
+    # param matching. It performs an embarassing ammount of back-and-forth
+    # type jugling, but it's the price to pay for consistency and correctness
+    result.typ = generateTypeInstance(c, m.bindings, arg.info,
+                                      formal.skipTypes({tyCompositeTypeClass}))
+  else:
+    typeMismatch(arg, formal, arg.typ)
+    # error correction:
+    result = copyTree(arg)
+    result.typ = formal
 
 proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode =
   assert x.state == csMatch
@@ -271,8 +308,9 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     result = newNodeI(a.kind, n.info)
     for i in countup(0, len(a)-1): 
       var candidate = a.sons[i].sym
-      if candidate.kind in {skProc, skMethod, skConverter, skIterator}: 
+      if candidate.kind in {skProc, skMethod, skConverter,
-        # if suffices that the candidate has the proper number of generic 
+                            skIterator, skClosureIterator}:
+        # it suffices that the candidate has the proper number of generic 
         # type parameters:
         if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1:
           result.add(explicitGenericSym(c, n, candidate))
@@ -288,7 +326,7 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym =
   # for borrowing the sym in the symbol table is returned, else nil.
   # New approach: generate fn(x, y, z) where x, y, z have the proper types
   # and use the overloading resolution mechanism:
-  var call = newNode(nkCall)
+  var call = newNodeI(nkCall, fn.info)
   var hasDistinct = false
   call.add(newIdentNode(fn.name, fn.info))
   for i in 1.. <fn.typ.n.len:

compiler/semdata.nim

@@ -42,7 +42,7 @@ type
 
   TExprFlag* = enum 
     efLValue, efWantIterator, efInTypeof, efWantStmt, efDetermineType,
-    efAllowDestructor, efWantValue
+    efAllowDestructor, efWantValue, efOperand
   TExprFlags* = set[TExprFlag]
 
   PContext* = ref TContext
@@ -81,6 +81,9 @@ type
     semOverloadedCall*: proc (c: PContext, n, nOrig: PNode,
                               filter: TSymKinds): PNode {.nimcall.}
     semTypeNode*: proc(c: PContext, n: PNode, prev: PType): PType {.nimcall.}
+    semInferredLambda*: proc(c: PContext, pt: TIdTable, n: PNode): PNode
+    semGenerateInstance*: proc (c: PContext, fn: PSym, pt: TIdTable,
+                                info: TLineInfo): PSym
     includedFiles*: TIntSet    # used to detect recursive include files
     userPragmas*: TStrTable
     evalContext*: PEvalContext
@@ -211,7 +214,6 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType =
 
 proc makeTypeSymNode*(c: PContext, typ: PType, info: TLineInfo): PNode =
   let typedesc = makeTypeDesc(c, typ)
-  rawAddSon(typedesc, newTypeS(tyNone, c))
   let sym = newSym(skType, idAnon, getCurrOwner(), info).linkTo(typedesc)
   return newSymNode(sym, info)
 
@@ -234,17 +236,41 @@ proc makeAndType*(c: PContext, t1, t2: PType): PType =
   result.sons = @[t1, t2]
   propagateToOwner(result, t1)
   propagateToOwner(result, t2)
+  result.flags.incl((t1.flags + t2.flags) * {tfHasStatic})
 
 proc makeOrType*(c: PContext, t1, t2: PType): PType =
   result = newTypeS(tyOr, c)
   result.sons = @[t1, t2]
   propagateToOwner(result, t1)
   propagateToOwner(result, t2)
+  result.flags.incl((t1.flags + t2.flags) * {tfHasStatic})
 
 proc makeNotType*(c: PContext, t1: PType): PType =
   result = newTypeS(tyNot, c)
   result.sons = @[t1]
   propagateToOwner(result, t1)
+  result.flags.incl(t1.flags * {tfHasStatic})
+
+proc nMinusOne*(n: PNode): PNode =
+  result = newNode(nkCall, n.info, @[
+    newSymNode(getSysMagic("<", mUnaryLt)),
+    n])
+
+# Remember to fix the procs below this one when you make changes!
+proc makeRangeWithStaticExpr*(c: PContext, n: PNode): PType =
+  let intType = getSysType(tyInt)
+  result = newTypeS(tyRange, c)
+  result.sons = @[intType]
+  result.n = newNode(nkRange, n.info, @[
+    newIntTypeNode(nkIntLit, 0, intType),
+    makeStaticExpr(c, n.nMinusOne)])
+
+template rangeHasStaticIf*(t: PType): bool =
+  # this accepts the ranges's node
+  t.n[1].kind == nkStaticExpr
+
+template getStaticTypeFromRange*(t: PType): PType =
+  t.n[1][0][1].typ
 
 proc newTypeS(kind: TTypeKind, c: PContext): PType =
   result = newType(kind, getCurrOwner())
@@ -272,7 +298,7 @@ proc makeRangeType*(c: PContext; first, last: BiggestInt;
   addSonSkipIntLit(result, intType) # basetype of range
 
 proc markIndirect*(c: PContext, s: PSym) {.inline.} =
-  if s.kind in {skProc, skConverter, skMethod, skIterator}:
+  if s.kind in {skProc, skConverter, skMethod, skIterator, skClosureIterator}:
     incl(s.flags, sfAddrTaken)
     # XXX add to 'c' for global analysis
 

compiler/semdestruct.nim

@@ -55,7 +55,9 @@ proc doDestructorStuff(c: PContext, s: PSym, n: PNode) =
             useSym(destructableT.destructor),
             n.sons[paramsPos][1][0]]))
 
-proc destroyField(c: PContext, field: PSym, holder: PNode): PNode =
+proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode
+
+proc destroySym(c: PContext, field: PSym, holder: PNode): PNode =
   let destructableT = instantiateDestructor(c, field.typ)
   if destructableT != nil:
     result = newNode(nkCall, field.info, @[
@@ -70,56 +72,49 @@ proc destroyCase(c: PContext, n: PNode, holder: PNode): PNode =
   for i in countup(1, n.len - 1):
     # of A, B:
     var caseBranch = newNode(n[i].kind, n[i].info, n[i].sons[0 .. -2])
-    let recList = n[i].lastSon
-    var destroyRecList = newNode(nkStmtList, n[i].info, @[])
-    template addField(f: expr): stmt =
-      let stmt = destroyField(c, f, holder)
-      if stmt != nil:
-        destroyRecList.addSon(stmt)
-        inc nonTrivialFields
     
-    case recList.kind
+    let stmt = destroyFieldOrFields(c, n[i].lastSon, holder)
-    of nkSym:
+    if stmt == nil:
-      addField(recList.sym)
+      caseBranch.addSon(newNode(nkStmtList, n[i].info, @[]))
-    of nkRecList:
-      for j in countup(0, recList.len - 1):
-        addField(recList[j].sym)
     else:
-      internalAssert false
+      caseBranch.addSon(stmt)
+      nonTrivialFields += stmt.len
     
-    caseBranch.addSon(destroyRecList)
     result.addSon(caseBranch)
+  
   # maybe no fields were destroyed?
   if nonTrivialFields == 0:
     result = nil
 
+proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode =
+  template maybeAddLine(e: expr): stmt =
+    let stmt = e
+    if stmt != nil:
+      if result == nil: result = newNode(nkStmtList)
+      result.addSon(stmt)
+
+  case field.kind
+  of nkRecCase:
+    maybeAddLine destroyCase(c, field, holder)
+  of nkSym:
+    maybeAddLine destroySym(c, field.sym, holder)
+  of nkRecList:
+    for son in field:
+      maybeAddLine destroyFieldOrFields(c, son, holder)
+  else:
+    internalAssert false
+
 proc generateDestructor(c: PContext, t: PType): PNode =
   ## generate a destructor for a user-defined object or tuple type
   ## returns nil if the destructor turns out to be trivial
   
-  template addLine(e: expr): stmt =
-    if result == nil: result = newNode(nkStmtList)
-    result.addSon(e)
-
   # XXX: This may be true for some C-imported types such as
   # Tposix_spawnattr
   if t.n == nil or t.n.sons == nil: return
   internalAssert t.n.kind == nkRecList
   let destructedObj = newIdentNode(destructorParam, unknownLineInfo())
   # call the destructods of all fields
-  for s in countup(0, t.n.sons.len - 1):
+  result = destroyFieldOrFields(c, t.n, destructedObj)
-    case t.n.sons[s].kind
-    of nkRecCase:
-      let stmt = destroyCase(c, t.n.sons[s], destructedObj)
-      if stmt != nil: addLine(stmt)
-    of nkSym:
-      let stmt = destroyField(c, t.n.sons[s].sym, destructedObj)
-      if stmt != nil: addLine(stmt)
-    else:
-      # XXX just skip it for now so that the compiler doesn't crash, but
-      # please zahary fix it! arbitrary nesting of nkRecList/nkRecCase is
-      # possible. Any thread example seems to trigger this. 
-      discard
   # base classes' destructors will be automatically called by
   # semProcAux for both auto-generated and user-defined destructors
 

compiler/semexprs.nim

@@ -21,7 +21,7 @@ proc semFieldAccess(c: PContext, n: PNode, flags: TExprFlags = {}): PNode
 
 proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   # same as 'semExprWithType' but doesn't check for proc vars
-  result = semExpr(c, n, flags)
+  result = semExpr(c, n, flags + {efOperand})
   if result.kind == nkEmpty: 
     # do not produce another redundant error message:
     #raiseRecoverableError("")
@@ -117,10 +117,12 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     elif s.ast != nil:
       result = semExpr(c, s.ast)
     else:
-      internalError(n.info, "no default for")
+      n.typ = s.typ
-      result = emptyNode
+      return n
   of skType:
     markUsed(n, s)
+    if s.typ.kind == tyStatic and s.typ.n != nil:
+      return s.typ.n
     result = newSymNode(s, n.info)
     result.typ = makeTypeDesc(c, s.typ)
   else:
@@ -191,20 +193,47 @@ proc isCastable(dst, src: PType): bool =
 proc isSymChoice(n: PNode): bool {.inline.} =
   result = n.kind in nkSymChoices
 
+proc maybeLiftType(t: var PType, c: PContext, info: TLineInfo) =
+  # XXX: liftParamType started to perform addDecl
+  # we could do that instead in semTypeNode by snooping for added
+  # gnrc. params, then it won't be necessary to open a new scope here
+  openScope(c)
+  var lifted = liftParamType(c, skType, newNodeI(nkArgList, info),
+                         t, ":anon", info)
+  closeScope(c)
+  if lifted != nil: t = lifted
+
 proc semConv(c: PContext, n: PNode): PNode =
   if sonsLen(n) != 2:
     localError(n.info, errConvNeedsOneArg)
     return n
+
   result = newNodeI(nkConv, n.info)
-  result.typ = semTypeNode(c, n.sons[0], nil).skipTypes({tyGenericInst})
+  var targetType = semTypeNode(c, n.sons[0], nil)
-  addSon(result, copyTree(n.sons[0]))
+  maybeLiftType(targetType, c, n[0].info)
-  addSon(result, semExprWithType(c, n.sons[1]))
+  result.addSon copyTree(n.sons[0])
-  var op = result.sons[1]
+  var op = semExprWithType(c, n.sons[1])
+  
+  if targetType.isMetaType:
+    let final = inferWithMetatype(c, targetType, op, true)
+    result.addSon final
+    result.typ = final.typ
+    return
+
+  result.typ = targetType
+  addSon(result, op)
   
   if not isSymChoice(op):
     let status = checkConvertible(c, result.typ, op.typ)
     case status
-    of convOK: discard
+    of convOK:
+      # handle SomeProcType(SomeGenericProc)
+      # XXX: This needs fixing. checkConvertible uses typeRel internally, but
+      # doesn't bother to perform the work done in paramTypeMatchAux/fitNode
+      # so we are redoing the typeRel work here. Why does semConv exist as a
+      # separate proc from fitNode?
+      if op.kind == nkSym and op.sym.isGenericRoutine:
+        result.sons[1] = fitNode(c, result.typ, result.sons[1])
     of convNotNeedeed:
       message(n.info, hintConvFromXtoItselfNotNeeded, result.typ.typeToString)
     of convNotLegal:
@@ -214,7 +243,7 @@ proc semConv(c: PContext, n: PNode): PNode =
     for i in countup(0, sonsLen(op) - 1):
       let it = op.sons[i]
       let status = checkConvertible(c, result.typ, it.typ)
-      if status == convOK:
+      if status in {convOK, convNotNeedeed}:
         markUsed(n, it.sym)
         markIndirect(c, it.sym)
         return it
@@ -316,16 +345,9 @@ proc isOpImpl(c: PContext, n: PNode): PNode =
       result = newIntNode(nkIntLit, ord(t.kind == tyProc and
                                         t.callConv == ccClosure and 
                                         tfIterator notin t.flags))
-    of "iterator":
-      let t = skipTypes(t1, abstractRange)
-      result = newIntNode(nkIntLit, ord(t.kind == tyProc and
-                                        t.callConv == ccClosure and 
-                                        tfIterator in t.flags))
   else:
     var t2 = n[2].typ.skipTypes({tyTypeDesc})
-    let lifted = liftParamType(c, skType, newNodeI(nkArgList, n.info),
+    maybeLiftType(t2, c, n.info)
-                               t2, ":anon", n.info)
-    if lifted != nil: t2 = lifted
     var m: TCandidate
     initCandidate(c, m, t2)
     let match = typeRel(m, t2, t1) != isNone
@@ -340,21 +362,20 @@ proc semIs(c: PContext, n: PNode): PNode =
   result = n
   n.typ = getSysType(tyBool)
  
-  n.sons[1] = semExprWithType(c, n[1], {efDetermineType})
+  n.sons[1] = semExprWithType(c, n[1], {efDetermineType, efWantIterator})
-  
   if n[2].kind notin {nkStrLit..nkTripleStrLit}:
     let t2 = semTypeNode(c, n[2], nil)
     n.sons[2] = newNodeIT(nkType, n[2].info, t2)
 
-  if n[1].typ.kind != tyTypeDesc:
+  let lhsType = n[1].typ
-    n.sons[1] = makeTypeSymNode(c, n[1].typ, n[1].info)
+  if lhsType.kind != tyTypeDesc:
-  elif n[1].typ.sonsLen == 0:
+    n.sons[1] = makeTypeSymNode(c, lhsType, n[1].info)
+  elif lhsType.base.kind == tyNone:
     # this is a typedesc variable, leave for evals
     return
 
-  let t1 = n[1].typ.sons[0]
   # BUGFIX: don't evaluate this too early: ``T is void``
-  if not containsGenericType(t1): result = isOpImpl(c, n)
+  if not n[1].typ.base.containsGenericType: result = isOpImpl(c, n)
 
 proc semOpAux(c: PContext, n: PNode) =
   const flags = {efDetermineType}
@@ -612,6 +633,18 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
       else: return result
     result.typ = semfold.getIntervalType(callee.magic, call)
   
+  block maybeLabelAsStatic:
+    # XXX: temporary work-around needed for tlateboundstatic.
+    # This is certainly not correct, but it will get the job
+    # done until we have a more robust infrastructure for
+    # implicit statics.
+    if n.len > 1:
+      for i in 1 .. <n.len:
+        if n[i].typ.kind != tyStatic or tfUnresolved notin n[i].typ.flags:
+          break maybeLabelAsStatic
+      n.typ = newTypeWithSons(c, tyStatic, @[n.typ])
+      n.typ.flags.incl tfUnresolved
+
   # optimization pass: not necessary for correctness of the semantic pass
   if {sfNoSideEffect, sfCompileTime} * callee.flags != {} and
      {sfForward, sfImportc} * callee.flags == {}:
@@ -635,9 +668,11 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
       result = evalStaticExpr(c.module, call, c.p.owner)
       if result.isNil: 
         localError(n.info, errCannotInterpretNodeX, renderTree(call))
+      else: result = fixupTypeAfterEval(c, result, n)
     else:
       result = evalConstExpr(c.module, call)
       if result.isNil: result = n
+      else: result = fixupTypeAfterEval(c, result, n)
     #if result != n:
     #  echo "SUCCESS evaluated at compile time: ", call.renderTree
 
@@ -647,16 +682,18 @@ proc semStaticExpr(c: PContext, n: PNode): PNode =
   if result.isNil:
     localError(n.info, errCannotInterpretNodeX, renderTree(n))
     result = emptyNode
+  else:
+    result = fixupTypeAfterEval(c, result, a)
 
 proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
                                      flags: TExprFlags): PNode =
   if flags*{efInTypeof, efWantIterator} != {}:
     # consider: 'for x in pReturningArray()' --> we don't want the restriction
-    # to 'skIterator' anymore; skIterator is preferred in sigmatch already for
+    # to 'skIterators' anymore; skIterators are preferred in sigmatch already 
-    # typeof support.
+    # for typeof support.
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     result = semOverloadedCall(c, n, nOrig,
-      {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator})
+      {skProc, skMethod, skConverter, skMacro, skTemplate}+skIterators)
   else:
     result = semOverloadedCall(c, n, nOrig, 
       {skProc, skMethod, skConverter, skMacro, skTemplate})
@@ -669,7 +706,7 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
     case callee.kind
     of skMacro, skTemplate: discard
     else:
-      if (callee.kind == skIterator) and (callee.id == c.p.owner.id): 
+      if (callee.kind in skIterators) and (callee.id == c.p.owner.id): 
         localError(n.info, errRecursiveDependencyX, callee.name.s)
       if sfNoSideEffect notin callee.flags: 
         if {sfImportc, sfSideEffect} * callee.flags != {}:
@@ -687,6 +724,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
       # it is a static call!
       result = n.sons[0]
       result.kind = nkCall
+      result.flags.incl nfExplicitCall
       for i in countup(1, sonsLen(n) - 1): addSon(result, n.sons[i])
       return semExpr(c, result, flags)
   else:
@@ -893,6 +931,26 @@ proc makeDeref(n: PNode): PNode =
     addSon(result, a)
     t = skipTypes(t.sons[0], {tyGenericInst})
 
+const
+  tyTypeParamsHolders = {tyGenericInst, tyCompositeTypeClass}
+  tyDotOpTransparent = {tyVar, tyPtr, tyRef}
+
+proc readTypeParameter(c: PContext, typ: PType,
+                       paramName: PIdent, info: TLineInfo): PNode =
+  let ty = if typ.kind == tyGenericInst: typ.skipGenericAlias
+           else: (internalAssert(typ.kind == tyCompositeTypeClass); typ.sons[1])
+  
+  let tbody = ty.sons[0]
+  for s in countup(0, tbody.len-2):
+    let tParam = tbody.sons[s]
+    if tParam.sym.name == paramName:
+      let rawTyp = ty.sons[s + 1]
+      if rawTyp.kind == tyStatic:
+        return rawTyp.n
+      else:
+        let foundTyp = makeTypeDesc(c, rawTyp)
+        return newSymNode(copySym(tParam.sym).linkTo(foundTyp), info)
+
 proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   ## returns nil if it's not a built-in field access
   checkSonsLen(n, 2)
@@ -910,8 +968,9 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   var ty = n.sons[0].typ
   var f: PSym = nil
   result = nil
-  if isTypeExpr(n.sons[0]) or ty.kind == tyTypeDesc and ty.len == 1:
+  if isTypeExpr(n.sons[0]) or (ty.kind == tyTypeDesc and ty.base.kind != tyNone):
-    if ty.kind == tyTypeDesc: ty = ty.sons[0]
+    if ty.kind == tyTypeDesc: ty = ty.base
+    ty = ty.skipTypes(tyDotOpTransparent)
     case ty.kind
     of tyEnum:
       # look up if the identifier belongs to the enum:
@@ -925,22 +984,11 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
         result.typ = ty
         markUsed(n, f)
         return
-    of tyGenericInst:
+    of tyTypeParamsHolders:
-      assert ty.sons[0].kind == tyGenericBody
+      return readTypeParameter(c, ty, i, n.info)
-      let tbody = ty.sons[0]
-      for s in countup(0, tbody.len-2):
-        let tParam = tbody.sons[s]
-        if tParam.sym.name == i:
-          let rawTyp = ty.sons[s + 1]
-          if rawTyp.kind == tyStatic:
-            return rawTyp.n
-          else:
-            let foundTyp = makeTypeDesc(c, rawTyp)
-            return newSymNode(copySym(tParam.sym).linkTo(foundTyp), n.info)
-      return
     of tyObject, tyTuple:
       if ty.n.kind == nkRecList:
-        for field in ty.n.sons:
+        for field in ty.n:
           if field.sym.name == i:
             n.typ = newTypeWithSons(c, tyFieldAccessor, @[ty, field.sym.typ])
             n.typ.n = copyTree(n)
@@ -952,8 +1000,9 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
     # XXX: This is probably not relevant any more
     # reset to prevent 'nil' bug: see "tests/reject/tenumitems.nim":
     ty = n.sons[0].typ
-    
+    return nil
   ty = skipTypes(ty, {tyGenericInst, tyVar, tyPtr, tyRef})
+  while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct})
   var check: PNode = nil
   if ty.kind == tyObject:
     while true:
@@ -984,6 +1033,12 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
       n.typ = f.typ
       result = n
 
+  # we didn't find any field, let's look for a generic param
+  if result == nil:
+    let t = n.sons[0].typ.skipTypes(tyDotOpTransparent)
+    if t.kind in tyTypeParamsHolders:
+      result = readTypeParameter(c, t, i, n.info)
+
 proc dotTransformation(c: PContext, n: PNode): PNode =
   if isSymChoice(n.sons[1]):
     result = newNodeI(nkDotCall, n.info)
@@ -992,7 +1047,7 @@ proc dotTransformation(c: PContext, n: PNode): PNode =
   else:
     var i = considerAcc(n.sons[1])
     result = newNodeI(nkDotCall, n.info)
-    result.flags.incl nfDelegate
+    result.flags.incl nfDotField
     addSon(result, newIdentNode(i, n[1].info))
     addSon(result, copyTree(n[0]))
   
@@ -1075,12 +1130,13 @@ proc semArrayAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
 
 proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode =
   var id = considerAcc(a[1])
-  let setterId = newIdentNode(getIdent(id.s & '='), n.info)
+  var setterId = newIdentNode(getIdent(id.s & '='), n.info)
   # a[0] is already checked for semantics, that does ``builtinFieldAccess``
   # this is ugly. XXX Semantic checking should use the ``nfSem`` flag for
   # nodes?
   let aOrig = nOrig[0]
   result = newNode(nkCall, n.info, sons = @[setterId, a[0], semExpr(c, n[1])])
+  result.flags.incl nfDotSetter
   let orig = newNode(nkCall, n.info, sons = @[setterId, aOrig[0], nOrig[1]])
   result = semOverloadedCallAnalyseEffects(c, result, orig, {})
   
@@ -1112,6 +1168,9 @@ proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
       n.sons[0] = x # 'result[]' --> 'result'
       n.sons[1] = takeImplicitAddr(c, ri)
 
+template resultTypeIsInferrable(typ: PType): expr =
+  typ.isMetaType and typ.kind != tyTypeDesc
+
 proc semAsgn(c: PContext, n: PNode): PNode =
   checkSonsLen(n, 2)
   var a = n.sons[0]
@@ -1163,7 +1222,7 @@ proc semAsgn(c: PContext, n: PNode): PNode =
         if lhsIsResult: {efAllowDestructor} else: {})
     if lhsIsResult:
       n.typ = enforceVoidContext
-      if lhs.sym.typ.isMetaType and lhs.sym.typ.kind != tyTypeDesc:
+      if c.p.owner.kind != skMacro and resultTypeIsInferrable(lhs.sym.typ):
         if cmpTypes(c, lhs.typ, rhs.typ) == isGeneric:
           internalAssert c.p.resultSym != nil
           lhs.typ = rhs.typ
@@ -1181,7 +1240,7 @@ proc semReturn(c: PContext, n: PNode): PNode =
   result = n
   checkSonsLen(n, 1)
   if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro} or
-     (c.p.owner.kind == skIterator and c.p.owner.typ.callConv == ccClosure):
+     c.p.owner.kind == skClosureIterator:
     if n.sons[0].kind != nkEmpty:
       # transform ``return expr`` to ``result = expr; return``
       if c.p.resultSym != nil: 
@@ -1199,6 +1258,7 @@ proc semReturn(c: PContext, n: PNode): PNode =
 
 proc semProcBody(c: PContext, n: PNode): PNode =
   openScope(c)
+  
   result = semExpr(c, n)
   if c.p.resultSym != nil and not isEmptyType(result.typ):
     # transform ``expr`` to ``result = expr``, but not if the expr is already
@@ -1222,6 +1282,11 @@ proc semProcBody(c: PContext, n: PNode): PNode =
       result = semAsgn(c, a)
   else:
     discardCheck(c, result)
+  
+  if c.p.owner.kind notin {skMacro, skTemplate} and
+     c.p.resultSym != nil and c.p.resultSym.typ.isMetaType:
+    localError(c.p.resultSym.info, errCannotInferReturnType)
+
   closeScope(c)
 
 proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
@@ -1246,17 +1311,28 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
 proc semYield(c: PContext, n: PNode): PNode =
   result = n
   checkSonsLen(n, 1)
-  if c.p.owner == nil or c.p.owner.kind != skIterator:
+  if c.p.owner == nil or c.p.owner.kind notin skIterators:
     localError(n.info, errYieldNotAllowedHere)
   elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
     localError(n.info, errYieldNotAllowedInTryStmt)
   elif n.sons[0].kind != nkEmpty:
     n.sons[0] = semExprWithType(c, n.sons[0]) # check for type compatibility:
-    var restype = c.p.owner.typ.sons[0]
+    var iterType = c.p.owner.typ
+    var restype = iterType.sons[0]
     if restype != nil:
-      n.sons[0] = fitNode(c, restype, n.sons[0])
+      let adjustedRes = if c.p.owner.kind == skIterator: restype.base
+                        else: restype
+      n.sons[0] = fitNode(c, adjustedRes, n.sons[0])
       if n.sons[0].typ == nil: internalError(n.info, "semYield")
-      semYieldVarResult(c, n, restype)
+      
+      if resultTypeIsInferrable(adjustedRes):
+        let inferred = n.sons[0].typ
+        if c.p.owner.kind == skIterator:
+          iterType.sons[0].sons[0] = inferred
+        else:
+          iterType.sons[0] = inferred
+      
+      semYieldVarResult(c, n, adjustedRes)
     else:
       localError(n.info, errCannotReturnExpr)
   elif c.p.owner.typ.sons[0] != nil:
@@ -1337,7 +1413,7 @@ proc expectString(c: PContext, n: PNode): string =
     localError(n.info, errStringLiteralExpected)
 
 proc getMagicSym(magic: TMagic): PSym =
-  result = newSym(skProc, getIdent($magic), getCurrOwner(), gCodegenLineInfo)
+  result = newSym(skProc, getIdent($magic), systemModule, gCodegenLineInfo)
   result.magic = magic
 
 proc newAnonSym(kind: TSymKind, info: TLineInfo,
@@ -1405,9 +1481,8 @@ proc processQuotations(n: var PNode, op: string,
   elif n.kind == nkAccQuoted and op == "``":
     returnQuote n[0]
  
-  if not n.isAtom:
+  for i in 0 .. <n.safeLen:
-    for i in 0 .. <n.len:
+    processQuotations(n.sons[i], op, quotes, ids)
-      processQuotations(n.sons[i], op, quotes, ids)
 
 proc semQuoteAst(c: PContext, n: PNode): PNode =
   internalAssert n.len == 2 or n.len == 3
@@ -1770,22 +1845,6 @@ proc semBlock(c: PContext, n: PNode): PNode =
   closeScope(c)
   dec(c.p.nestedBlockCounter)
 
-proc buildCall(n: PNode): PNode =
-  if n.kind == nkDotExpr and n.len == 2:
-    # x.y --> y(x)
-    result = newNodeI(nkCall, n.info, 2)
-    result.sons[0] = n.sons[1]
-    result.sons[1] = n.sons[0]
-  elif n.kind in nkCallKinds and n.sons[0].kind == nkDotExpr:
-    # x.y(a) -> y(x, a)
-    let a = n.sons[0]
-    result = newNodeI(nkCall, n.info, n.len+1)
-    result.sons[0] = a.sons[1]
-    result.sons[1] = a.sons[0]
-    for i in 1 .. <n.len: result.sons[i+1] = n.sons[i]
-  else:
-    result = n
-
 proc doBlockIsStmtList(n: PNode): bool =
   result = n.kind == nkDo and
            n[paramsPos].sonsLen == 1 and
@@ -1832,13 +1891,13 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     var s = lookUp(c, n)
     semCaptureSym(s, c.p.owner)
     result = semSym(c, n, s, flags)
-    if s.kind in {skProc, skMethod, skIterator, skConverter}:
+    if s.kind in {skProc, skMethod, skConverter}+skIterators:
       #performProcvarCheck(c, n, s)
       result = symChoice(c, n, s, scClosed)
       if result.kind == nkSym:
         markIndirect(c, result.sym)
-        if isGenericRoutine(result.sym):
+        # if isGenericRoutine(result.sym):
-          localError(n.info, errInstantiateXExplicitely, s.name.s)
+        #   localError(n.info, errInstantiateXExplicitely, s.name.s)
   of nkSym:
     # because of the changed symbol binding, this does not mean that we
     # don't have to check the symbol for semantics here again!
@@ -1888,13 +1947,13 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     message(n.info, warnDeprecated, "bind")
     result = semExpr(c, n.sons[0], flags)
   of nkTypeOfExpr, nkTupleTy, nkRefTy..nkEnumTy, nkStaticTy:
-    var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc})
+    var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc, tyIter})
     result.typ = makeTypeDesc(c, typ)
     #result = symNodeFromType(c, typ, n.info)
   of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit: 
     # check if it is an expression macro:
     checkMinSonsLen(n, 1)
-    let mode = if nfDelegate in n.flags: {} else: {checkUndeclared}
+    let mode = if nfDotField in n.flags: {} else: {checkUndeclared}
     var s = qualifiedLookUp(c, n.sons[0], mode)
     if s != nil: 
       if gCmd == cmdPretty and n.sons[0].kind == nkDotExpr:
@@ -1922,7 +1981,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
           localError(n.info, errUseQualifier, s.name.s)
         elif s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
-      of skProc, skMethod, skConverter, skIterator: 
+      of skProc, skMethod, skConverter, skIterators:
         if s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
       else:
@@ -1933,7 +1992,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       # the 'newSeq[T](x)' bug
       setGenericParams(c, n.sons[0])
       result = semDirectOp(c, n, flags)
-    elif isSymChoice(n.sons[0]) or nfDelegate in n.flags:
+    elif isSymChoice(n.sons[0]) or nfDotField in n.flags:
       result = semDirectOp(c, n, flags)
     else:
       result = semIndirectOp(c, n, flags)
@@ -1946,7 +2005,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   of nkBracketExpr:
     checkMinSonsLen(n, 1)
     var s = qualifiedLookUp(c, n.sons[0], {checkUndeclared})
-    if s != nil and s.kind in {skProc, skMethod, skConverter, skIterator}: 
+    if (s != nil and s.kind in {skProc, skMethod, skConverter}+skIterators) or
+        n[0].kind in nkSymChoices:
       # type parameters: partial generic specialization
       n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s)
       result = explicitGenericInstantiation(c, n, s)

compiler/semfold.nim

@@ -537,7 +537,7 @@ proc foldArrayAccess(m: PSym, n: PNode): PNode =
       if result.kind == nkExprColonExpr: result = result.sons[1]
     else:
       localError(n.info, errIndexOutOfBounds)
-  of nkBracket, nkMetaNode: 
+  of nkBracket: 
     if (idx >= 0) and (idx < sonsLen(x)): result = x.sons[int(idx)]
     else: localError(n.info, errIndexOutOfBounds)
   of nkStrLit..nkTripleStrLit: 

compiler/semgnrc.nim

@@ -42,7 +42,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym): PNode =
   of skUnknown: 
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of skProc, skMethod, skIterator, skConverter: 
+  of skProc, skMethod, skIterators, skConverter:
     result = symChoice(c, n, s, scOpen)
   of skTemplate:
     if macroToExpand(s):
@@ -141,7 +141,7 @@ proc semGenericStmt(c: PContext, n: PNode,
         # symbol lookup ...
       of skUnknown, skParam: 
         # Leave it as an identifier.
-      of skProc, skMethod, skIterator, skConverter: 
+      of skProc, skMethod, skIterators, skConverter:
         result.sons[0] = symChoice(c, n.sons[0], s, scOption)
         first = 1
       of skGenericParam:

compiler/seminst.nim

@@ -15,12 +15,11 @@ proc instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable,
   if n.kind != nkGenericParams: 
     internalError(n.info, "instantiateGenericParamList; no generic params")
   newSeq(entry.concreteTypes, n.len)
-  for i in countup(0, n.len - 1):
+  for i, a in n.pairs:
-    var a = n.sons[i]
     if a.kind != nkSym: 
       internalError(a.info, "instantiateGenericParamList; no symbol")
     var q = a.sym
-    if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic}+tyTypeClasses:
+    if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic, tyIter}+tyTypeClasses:
       continue
     var s = newSym(skType, q.name, getCurrOwner(), q.info)
     s.flags = s.flags + {sfUsed, sfFromGeneric}
@@ -86,19 +85,21 @@ proc freshGenSyms(n: PNode, owner: PSym, symMap: var TIdTable) =
 
 proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind)
 
+proc addProcDecls(c: PContext, fn: PSym) =
+  # get the proc itself in scope (e.g. for recursion)
+  addDecl(c, fn)
+
+  for i in 1 .. <fn.typ.n.len:
+    var param = fn.typ.n.sons[i].sym
+    param.owner = fn
+    addParamOrResult(c, param, fn.kind)
+  
+  maybeAddResult(c, fn, fn.ast)
+
 proc instantiateBody(c: PContext, n: PNode, result: PSym) =
   if n.sons[bodyPos].kind != nkEmpty:
     inc c.inGenericInst
     # add it here, so that recursive generic procs are possible:
-    addDecl(c, result)
-    pushProcCon(c, result)
-    # add params to scope
-    for i in 1 .. <result.typ.n.len:
-      var param = result.typ.n.sons[i].sym
-      param.owner = result
-      addParamOrResult(c, param, result.kind)
-    # debug result.typ.n
-    maybeAddResult(c, result, n)
     var b = n.sons[bodyPos]
     var symMap: TIdTable
     initIdTable symMap
@@ -108,7 +109,6 @@ proc instantiateBody(c: PContext, n: PNode, result: PSym) =
     n.sons[bodyPos] = transformBody(c.module, b, result)
     #echo "code instantiated ", result.name.s
     excl(result.flags, sfForward)
-    popProcCon(c)
     dec c.inGenericInst
 
 proc fixupInstantiatedSymbols(c: PContext, s: PSym) =
@@ -145,11 +145,56 @@ proc instGenericContainer(c: PContext, info: TLineInfo, header: PType,
 proc instGenericContainer(c: PContext, n: PNode, header: PType): PType =
   result = instGenericContainer(c, n.info, header)
 
+proc instantiateProcType(c: PContext, pt: TIdTable,
+                          prc: PSym, info: TLineInfo) =
+  # XXX: Instantiates a generic proc signature, while at the same
+  # time adding the instantiated proc params into the current scope.
+  # This is necessary, because the instantiation process may refer to
+  # these params in situations like this:
+  # proc foo[Container](a: Container, b: a.type.Item): type(b.x)
+  #
+  # Alas, doing this here is probably not enough, because another
+  # proc signature could appear in the params:
+  # proc foo[T](a: proc (x: T, b: type(x.y))
+  #   
+  # The solution would be to move this logic into semtypinst, but
+  # at this point semtypinst have to become part of sem, because it
+  # will need to use openScope, addDecl, etc
+  #
+  addDecl(c, prc)
+  
+  pushInfoContext(info)
+  var cl = initTypeVars(c, pt, info)
+  var result = instCopyType(cl, prc.typ)
+  let originalParams = result.n
+  result.n = originalParams.shallowCopy
+  
+  for i in 1 .. <result.len:
+    result.sons[i] = replaceTypeVarsT(cl, result.sons[i])
+    propagateToOwner(result, result.sons[i])
+    let param = replaceTypeVarsN(cl, originalParams[i])
+    result.n.sons[i] = param
+    if param.kind == nkSym:
+      # XXX: this won't be true for void params
+      # implement pass-through of void params and
+      # the "sort by distance to point" container
+      param.sym.owner = prc
+      addDecl(c, param.sym)
+    
+  result.sons[0] = replaceTypeVarsT(cl, result.sons[0])
+  result.n.sons[0] = originalParams[0].copyTree
+  
+  eraseVoidParams(result)
+  skipIntLiteralParams(result)
+ 
+  prc.typ = result
+  maybeAddResult(c, prc, prc.ast)
+  popInfoContext()
+
 proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
                       info: TLineInfo): PSym =
   # no need to instantiate generic templates/macros:
   if fn.kind in {skTemplate, skMacro}: return fn
- 
   # generates an instantiated proc
   if c.instCounter > 1000: internalError(fn.ast.info, "nesting too deep")
   inc(c.instCounter)
@@ -173,7 +218,8 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   var entry = TInstantiation.new
   entry.sym = result
   instantiateGenericParamList(c, n.sons[genericParamsPos], pt, entry[])
-  result.typ = generateTypeInstance(c, pt, info, fn.typ)
+  pushProcCon(c, result)
+  instantiateProcType(c, pt, result, info)
   n.sons[genericParamsPos] = ast.emptyNode
   var oldPrc = genericCacheGet(fn, entry[])
   if oldPrc == nil:
@@ -183,12 +229,12 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
       pragma(c, result, n.sons[pragmasPos], allRoutinePragmas)
     if isNil(n.sons[bodyPos]):
       n.sons[bodyPos] = copyTree(fn.getBody)
-    if fn.kind != skTemplate:
+    instantiateBody(c, n, result)
-      instantiateBody(c, n, result)
+    sideEffectsCheck(c, result)
-      sideEffectsCheck(c, result)
     paramsTypeCheck(c, result.typ)
   else:
     result = oldPrc
+  popProcCon(c)
   popInfoContext()
   closeScope(c)           # close scope for parameters
   popOwner()

compiler/sempass2.nim

@@ -84,10 +84,10 @@ proc initVar(a: PEffects, n: PNode) =
 proc initVarViaNew(a: PEffects, n: PNode) =
   if n.kind != nkSym: return
   let s = n.sym
-  if {tfNeedsInit, tfNotNil} * s.typ.flags == {tfNotNil}:
+  if {tfNeedsInit, tfNotNil} * s.typ.flags <= {tfNotNil}:
     # 'x' is not nil, but that doesn't mean it's not nil children
     # are initialized:
-    initVarViaNew(a, n)
+    initVar(a, n)
 
 proc useVar(a: PEffects, n: PNode) =
   let s = n.sym
@@ -466,8 +466,7 @@ proc track(tracked: PEffects, n: PNode) =
         mergeEffects(tracked, effectList.sons[exceptionEffects], n)
         mergeTags(tracked, effectList.sons[tagEffects], n)
     for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i))
-    if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, 
+    if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}:
-                                           mNewSeq, mShallowCopy}:
       # may not look like an assignment, but it is:
       initVarViaNew(tracked, n.sons[1])
     for i in 0 .. <safeLen(n):
@@ -477,7 +476,6 @@ proc track(tracked: PEffects, n: PNode) =
     if warnProveField in gNotes: checkFieldAccess(tracked.guards, n)
   of nkTryStmt: trackTryStmt(tracked, n)
   of nkPragma: trackPragmaStmt(tracked, n)
-  of nkMacroDef, nkTemplateDef: discard
   of nkAsgn, nkFastAsgn:
     track(tracked, n.sons[1])
     initVar(tracked, n.sons[0])
@@ -527,7 +525,9 @@ proc track(tracked: PEffects, n: PNode) =
       if sfDiscriminant in x.sons[0].sym.flags:
         addDiscriminantFact(tracked.guards, x)
     setLen(tracked.guards, oldFacts)
-  of nkTypeSection: discard
+  of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef,
+      nkMacroDef, nkTemplateDef:
+    discard
   else:
     for i in 0 .. <safeLen(n): track(tracked, n.sons[i])
 
@@ -581,22 +581,26 @@ proc setEffectsForProcType*(t: PType, n: PNode) =
     if not isNil(tagsSpec):
       effects.sons[tagEffects] = tagsSpec
 
+proc initEffects(effects: PNode; s: PSym; t: var TEffects) =
+  newSeq(effects.sons, effectListLen)
+  effects.sons[exceptionEffects] = newNodeI(nkArgList, s.info)
+  effects.sons[tagEffects] = newNodeI(nkArgList, s.info)
+  
+  t.exc = effects.sons[exceptionEffects]
+  t.tags = effects.sons[tagEffects]
+  t.owner = s
+  t.init = @[]
+  t.guards = @[]
+  
 proc trackProc*(s: PSym, body: PNode) =
   var effects = s.typ.n.sons[0]
   internalAssert effects.kind == nkEffectList
   # effects already computed?
   if sfForward in s.flags: return
   if effects.len == effectListLen: return
-  newSeq(effects.sons, effectListLen)
-  effects.sons[exceptionEffects] = newNodeI(nkArgList, body.info)
-  effects.sons[tagEffects] = newNodeI(nkArgList, body.info)
   
   var t: TEffects
-  t.exc = effects.sons[exceptionEffects]
+  initEffects(effects, s, t)
-  t.tags = effects.sons[tagEffects]
-  t.owner = s
-  t.init = @[]
-  t.guards = @[]
   track(t, body)
   
   if not isEmptyType(s.typ.sons[0]) and tfNeedsInit in s.typ.sons[0].flags and
@@ -619,3 +623,12 @@ proc trackProc*(s: PSym, body: PNode) =
     # after the check, use the formal spec:
     effects.sons[tagEffects] = tagsSpec
     
+proc trackTopLevelStmt*(module: PSym; n: PNode) =
+  if n.kind in {nkPragma, nkMacroDef, nkTemplateDef, nkProcDef,
+                nkTypeSection, nkConverterDef, nkMethodDef, nkIteratorDef}:
+    return
+  var effects = newNode(nkEffectList, n.info)
+  var t: TEffects
+  initEffects(effects, module, t)
+
+  track(t, n)

compiler/semstmts.nim

@@ -12,9 +12,6 @@
 
 var enforceVoidContext = PType(kind: tyStmt)
 
-proc semCommand(c: PContext, n: PNode): PNode =
-  result = semExprNoType(c, n)
-  
 proc semDiscard(c: PContext, n: PNode): PNode = 
   result = n
   checkSonsLen(n, 1)
@@ -76,8 +73,8 @@ proc performProcvarCheck(c: PContext, n: PNode, s: PSym) =
 
 proc semProcvarCheck(c: PContext, n: PNode) =
   let n = n.skipConv
-  if n.kind == nkSym and n.sym.kind in {skProc, skMethod, skIterator,
+  if n.kind == nkSym and n.sym.kind in {skProc, skMethod, skConverter,
-                                        skConverter}:
+                                        skIterator, skClosureIterator}:
     performProcvarCheck(c, n, n.sym)
 
 proc semProc(c: PContext, n: PNode): PNode
@@ -126,13 +123,14 @@ proc implicitlyDiscardable(n: PNode): bool =
 proc fixNilType(n: PNode) =
   if isAtom(n):
     if n.kind != nkNilLit and n.typ != nil:
-      localError(n.info, errDiscardValue)
+      localError(n.info, errDiscardValueX, n.typ.typeToString)
   elif n.kind in {nkStmtList, nkStmtListExpr}:
     n.kind = nkStmtList
     for it in n: fixNilType(it)
   n.typ = nil
 
 proc discardCheck(c: PContext, result: PNode) =
+  if c.inTypeClass > 0: return
   if result.typ != nil and result.typ.kind notin {tyStmt, tyEmpty}:
     if result.kind == nkNilLit:
       result.typ = nil
@@ -143,10 +141,6 @@ proc discardCheck(c: PContext, result: PNode) =
       while n.kind in skipForDiscardable:
         n = n.lastSon
         n.typ = nil
-    elif c.inTypeClass > 0 and result.typ.kind == tyBool:
-      let verdict = semConstExpr(c, result)
-      if verdict.intVal == 0:
-        localError(result.info, "type class predicate failed")
     elif result.typ.kind != tyError and gCmd != cmdInteractive:
       if result.typ.kind == tyNil:
         fixNilType(result)
@@ -154,7 +148,7 @@ proc discardCheck(c: PContext, result: PNode) =
       else:
         var n = result
         while n.kind in skipForDiscardable: n = n.lastSon
-        localError(n.info, errDiscardValue)
+        localError(n.info, errDiscardValueX, result.typ.typeToString)
 
 proc semIf(c: PContext, n: PNode): PNode = 
   result = n
@@ -331,6 +325,7 @@ proc checkNilable(v: PSym) =
 proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = 
   var b: PNode
   result = copyNode(n)
+  var hasCompileTime = false
   for i in countup(0, sonsLen(n)-1): 
     var a = n.sons[i]
     if gCmd == cmdIdeTools: suggestStmt(c, a)
@@ -349,7 +344,12 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
       # BUGFIX: ``fitNode`` is needed here!
       # check type compability between def.typ and typ:
       if typ != nil: def = fitNode(c, typ, def)
-      else: typ = skipIntLit(def.typ)
+      else:
+        typ = skipIntLit(def.typ)
+        if typ.kind in {tySequence, tyArray, tySet} and
+           typ.lastSon.kind == tyEmpty:
+          localError(def.info, errCannotInferTypeOfTheLiteral,
+                     ($typ.kind).substr(2).toLower)
     else:
       def = ast.emptyNode
       if symkind == skLet: localError(a.info, errLetNeedsInit)
@@ -405,6 +405,8 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         v.typ = tup.sons[j]
         b.sons[j] = newSymNode(v)
       checkNilable(v)
+      if sfCompileTime in v.flags: hasCompileTime = true
+  if hasCompileTime: vm.setupCompileTimeVar(c.module, result)
 
 proc semConst(c: PContext, n: PNode): PNode = 
   result = copyNode(n)
@@ -607,7 +609,8 @@ proc symForVar(c: PContext, n: PNode): PSym =
 proc semForVars(c: PContext, n: PNode): PNode =
   result = n
   var length = sonsLen(n)
-  var iter = skipTypes(n.sons[length-2].typ, {tyGenericInst})
+  let iterBase = n.sons[length-2].typ.skipTypes({tyIter})
+  var iter = skipTypes(iterBase, {tyGenericInst})
   # length == 3 means that there is one for loop variable
   # and thus no tuple unpacking:
   if iter.kind != tyTuple or length == 3: 
@@ -617,7 +620,7 @@ proc semForVars(c: PContext, n: PNode): PNode =
       # BUGFIX: don't use `iter` here as that would strip away
       # the ``tyGenericInst``! See ``tests/compile/tgeneric.nim``
       # for an example:
-      v.typ = n.sons[length-2].typ
+      v.typ = iterBase
       n.sons[0] = newSymNode(v)
       if sfGenSym notin v.flags: addForVarDecl(c, v)
     else:
@@ -651,11 +654,19 @@ proc semFor(c: PContext, n: PNode): PNode =
   openScope(c)
   n.sons[length-2] = semExprNoDeref(c, n.sons[length-2], {efWantIterator})
   var call = n.sons[length-2]
-  if call.kind in nkCallKinds and call.sons[0].typ.callConv == ccClosure:
+  let isCallExpr = call.kind in nkCallKinds
+  if isCallExpr and call.sons[0].sym.magic != mNone:
+    if call.sons[0].sym.magic == mOmpParFor:
+      result = semForVars(c, n)
+      result.kind = nkParForStmt
+    else:
+      result = semForFields(c, n, call.sons[0].sym.magic)
+  elif (isCallExpr and call.sons[0].typ.callConv == ccClosure) or
+      call.typ.kind == tyIter:
     # first class iterator:
     result = semForVars(c, n)
-  elif call.kind notin nkCallKinds or call.sons[0].kind != nkSym or
+  elif not isCallExpr or call.sons[0].kind != nkSym or
-      call.sons[0].sym.kind != skIterator: 
+      call.sons[0].sym.kind notin skIterators:
     if length == 3:
       n.sons[length-2] = implicitIterator(c, "items", n.sons[length-2])
     elif length == 4:
@@ -663,12 +674,6 @@ proc semFor(c: PContext, n: PNode): PNode =
     else:
       localError(n.sons[length-2].info, errIteratorExpected)
     result = semForVars(c, n)
-  elif call.sons[0].sym.magic != mNone:
-    if call.sons[0].sym.magic == mOmpParFor:
-      result = semForVars(c, n)
-      result.kind = nkParForStmt
-    else:
-      result = semForFields(c, n, call.sons[0].sym.magic)
   else:
     result = semForVars(c, n)
   # propagate any enforced VoidContext:
@@ -764,6 +769,29 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
       s.ast = a
       popOwner()
 
+proc checkForMetaFields(n: PNode) =
+  template checkMeta(t) =
+    if t != nil and t.isMetaType and tfGenericTypeParam notin t.flags:
+      localError(n.info, errTIsNotAConcreteType, t.typeToString)
+  
+  case n.kind
+  of nkRecList, nkRecCase:
+    for s in n: checkForMetaFields(s)
+  of nkOfBranch, nkElse:
+    checkForMetaFields(n.lastSon)
+  of nkSym:
+    let t = n.sym.typ
+    case t.kind
+    of tySequence, tySet, tyArray, tyOpenArray, tyVar, tyPtr, tyRef,
+       tyProc, tyGenericInvokation, tyGenericInst:
+      let start = ord(t.kind in {tyGenericInvokation, tyGenericInst})
+      for i in start .. <t.sons.len:
+        checkMeta(t.sons[i])
+    else:
+      checkMeta(t)
+  else:
+    internalAssert false
+
 proc typeSectionFinalPass(c: PContext, n: PNode) = 
   for i in countup(0, sonsLen(n) - 1): 
     var a = n.sons[i]
@@ -780,6 +808,8 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
           assignType(s.typ, t)
           s.typ.id = t.id     # same id
       checkConstructedType(s.info, s.typ)
+      if s.typ.kind in {tyObject, tyTuple}:
+        checkForMetaFields(s.typ.n)
     let aa = a.sons[2]
     if aa.kind in {nkRefTy, nkPtrTy} and aa.len == 1 and
        aa.sons[0].kind == nkObjectTy:
@@ -883,12 +913,19 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
     s = n[namePos].sym
   pushOwner(s)
   openScope(c)
+  var gp: PNode
   if n.sons[genericParamsPos].kind != nkEmpty: 
-    illFormedAst(n)           # process parameters:
+    n.sons[genericParamsPos] = semGenericParamList(c, n.sons[genericParamsPos])
+    gp = n.sons[genericParamsPos]
+  else:
+    gp = newNodeI(nkGenericParams, n.info)
+
   if n.sons[paramsPos].kind != nkEmpty:
-    var gp = newNodeI(nkGenericParams, n.info)
     semParamList(c, n.sons[paramsPos], gp, s)
-    paramsTypeCheck(c, s.typ)
+    # paramsTypeCheck(c, s.typ)
+    if sonsLen(gp) > 0 and n.sons[genericParamsPos].kind == nkEmpty:
+      # we have a list of implicit type parameters:
+      n.sons[genericParamsPos] = gp
   else:
     s.typ = newTypeS(tyProc, c)
     rawAddSon(s.typ, nil)
@@ -900,12 +937,15 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
       localError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s)
     #if efDetermineType notin flags:
     # XXX not good enough; see tnamedparamanonproc.nim
-    pushProcCon(c, s)
+    if gp.len == 0 or (gp.len == 1 and tfRetType in gp[0].typ.flags):
-    addResult(c, s.typ.sons[0], n.info, skProc)
+      pushProcCon(c, s)
-    let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
+      addResult(c, s.typ.sons[0], n.info, skProc)
-    n.sons[bodyPos] = transformBody(c.module, semBody, s)
+      let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
-    addResultNode(c, n)
+      n.sons[bodyPos] = transformBody(c.module, semBody, s)
-    popProcCon(c)
+      addResultNode(c, n)
+      popProcCon(c)
+    elif efOperand notin flags:
+      localError(n.info, errGenericLambdaNotAllowed)
     sideEffectsCheck(c, s)
   else:
     localError(n.info, errImplOfXexpected, s.name.s)
@@ -913,6 +953,34 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
   popOwner()
   result.typ = s.typ
 
+proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
+  var n = n
+  
+  n = replaceTypesInBody(c, pt, n)
+  result = n
+  
+  n.sons[genericParamsPos] = emptyNode
+  n.sons[paramsPos] = n.typ.n
+  
+  openScope(c)
+  var s = n.sons[namePos].sym
+  addParams(c, n.typ.n, skProc)
+  pushProcCon(c, s)
+  addResult(c, n.typ.sons[0], n.info, skProc)
+  let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
+  n.sons[bodyPos] = transformBody(c.module, semBody, n.sons[namePos].sym)
+  addResultNode(c, n)
+  popProcCon(c)
+  closeScope(c)
+  
+  s.ast = result
+
+  # alternative variant (not quite working):
+  # var prc = arg[0].sym
+  # let inferred = c.semGenerateInstance(c, prc, m.bindings, arg.info)
+  # result = inferred.ast
+  # result.kind = arg.kind
+
 proc activate(c: PContext, n: PNode) =
   # XXX: This proc is part of my plan for getting rid of
   # forward declarations. stay tuned.
@@ -927,8 +995,7 @@ proc activate(c: PContext, n: PNode) =
       discard
 
 proc maybeAddResult(c: PContext, s: PSym, n: PNode) =
-  if s.typ.sons[0] != nil and
+  if s.typ.sons[0] != nil and s.kind != skIterator:
-      (s.kind != skIterator or s.typ.callConv == ccClosure):
     addResult(c, s.typ.sons[0], n.info, s.kind)
     addResultNode(c, n)
 
@@ -971,6 +1038,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
       return
   else:
     s = n[namePos].sym
+    s.owner = getCurrOwner()
     typeIsDetermined = s.typ == nil
     s.ast = n
     s.scope = c.currentScope
@@ -1003,12 +1071,12 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     rawAddSon(s.typ, nil)
   if n.sons[patternPos].kind != nkEmpty:
     n.sons[patternPos] = semPattern(c, n.sons[patternPos])
-  if s.kind == skIterator: 
+  if s.kind in skIterators:
     s.typ.flags.incl(tfIterator)
   
   var proto = searchForProc(c, s.scope, s)
   if proto == nil:
-    if s.kind == skIterator and isAnon: s.typ.callConv = ccClosure
+    if s.kind == skClosureIterator: s.typ.callConv = ccClosure
     else: s.typ.callConv = lastOptionEntry(c).defaultCC
     # add it here, so that recursive procs are possible:
     if sfGenSym in s.flags: discard
@@ -1021,7 +1089,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     if n.sons[pragmasPos].kind != nkEmpty:
       pragma(c, s, n.sons[pragmasPos], validPragmas)
     else:
-      implictPragmas(c, s, n, validPragmas)
+      implicitPragmas(c, s, n, validPragmas)
   else: 
     if n.sons[pragmasPos].kind != nkEmpty: 
       localError(n.sons[pragmasPos].info, errPragmaOnlyInHeaderOfProc)
@@ -1068,7 +1136,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         n.sons[bodyPos] = transformBody(c.module, semBody, s)
       popProcCon(c)
     else:
-      if s.typ.sons[0] != nil and kind != skIterator:
+      if s.typ.sons[0] != nil and kind notin skIterators:
         addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info))
       var toBind = initIntSet()
       n.sons[bodyPos] = semGenericStmtScope(c, n.sons[bodyPos], {}, toBind)
@@ -1095,7 +1163,15 @@ proc determineType(c: PContext, s: PSym) =
   discard semProcAux(c, s.ast, s.kind, {}, stepDetermineType)
 
 proc semIterator(c: PContext, n: PNode): PNode =
-  result = semProcAux(c, n, skIterator, iteratorPragmas)
+  let kind = if hasPragma(n[pragmasPos], wClosure) or
+                n[namePos].kind == nkEmpty: skClosureIterator
+             else: skIterator
+  # gensym'ed iterator?
+  if n[namePos].kind == nkSym:
+    # gensym'ed iterators might need to become closure iterators:
+    n[namePos].sym.owner = getCurrOwner()
+    n[namePos].sym.kind = kind
+  result = semProcAux(c, n, kind, iteratorPragmas)
   var s = result.sons[namePos].sym
   var t = s.typ
   if t.sons[0] == nil and s.typ.callConv != ccClosure:
@@ -1247,13 +1323,17 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
       return
     else:
       n.sons[i] = semExpr(c, n.sons[i])
+      if c.inTypeClass > 0 and n[i].typ != nil and n[i].typ.kind == tyBool:
+        let verdict = semConstExpr(c, n[i])
+        if verdict.intVal == 0:
+          localError(result.info, "type class predicate failed")
       if n.sons[i].typ == enforceVoidContext or usesResult(n.sons[i]):
         voidContext = true
         n.typ = enforceVoidContext
-      if i == last and efWantValue in flags:
+      if i == last and (length == 1 or efWantValue in flags):
         n.typ = n.sons[i].typ
         if not isEmptyType(n.typ): n.kind = nkStmtListExpr
-      elif i != last or voidContext or c.inTypeClass > 0:
+      elif i != last or voidContext:
         discardCheck(c, n.sons[i])
       else:
         n.typ = n.sons[i].typ
@@ -1263,8 +1343,8 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
         let (outer, inner) = insertDestructors(c, n.sons[i])
         if outer != nil:
           n.sons[i] = outer
-          for j in countup(i+1, length-1):
+          var rest = newNode(nkStmtList, n.info, n.sons[i+1 .. length-1])
-            inner.addSon(semStmt(c, n.sons[j]))
+          inner.addSon(semStmtList(c, rest, flags))
           n.sons.setLen(i+1)
           return
       of LastBlockStmts: 

compiler/semtempl.nim

@@ -155,7 +155,11 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
       of nkPragmaExpr: x = x[0]
       of nkIdent: break
       else: illFormedAst(x)
-    c.toInject.incl(x.ident.id)
+    let ident = getIdentNode(c, x)
+    if not isTemplParam(c, ident):
+      c.toInject.incl(x.ident.id)
+    else:
+      replaceIdentBySym(n, ident)
   else:
     let ident = getIdentNode(c, n)
     if not isTemplParam(c, ident):
@@ -171,7 +175,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym): PNode =
   of skUnknown: 
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of skProc, skMethod, skIterator, skConverter, skTemplate, skMacro:
+  of OverloadableSyms:
     result = symChoice(c, n, s, scOpen)
   of skGenericParam: 
     result = newSymNodeTypeDesc(s, n.info)
@@ -228,6 +232,18 @@ proc semTemplSomeDecl(c: var TemplCtx, n: PNode, symKind: TSymKind) =
     for j in countup(0, L-3):
       addLocalDecl(c, a.sons[j], symKind)
 
+proc onlyReplaceParams(c: var TemplCtx, n: PNode): PNode =
+  result = n
+  if n.kind == nkIdent:
+    let s = qualifiedLookUp(c.c, n, {})
+    if s != nil:
+      if s.owner == c.owner and s.kind == skParam:
+        incl(s.flags, sfUsed)
+        result = newSymNode(s, n.info)
+  else:
+    for i in 0 .. <n.safeLen:
+      result.sons[i] = onlyReplaceParams(c, n.sons[i])
+
 proc semPattern(c: PContext, n: PNode): PNode
 proc semTemplBody(c: var TemplCtx, n: PNode): PNode = 
   result = n
@@ -348,7 +364,9 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   of nkMethodDef:
     result = semRoutineInTemplBody(c, n, skMethod)
   of nkIteratorDef:
-    result = semRoutineInTemplBody(c, n, skIterator)
+    let kind = if hasPragma(n[pragmasPos], wClosure): skClosureIterator
+               else: skIterator
+    result = semRoutineInTemplBody(c, n, kind)
   of nkTemplateDef:
     result = semRoutineInTemplBody(c, n, skTemplate)
   of nkMacroDef:
@@ -357,8 +375,10 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
     result = semRoutineInTemplBody(c, n, skConverter)
   of nkPragmaExpr:
     result.sons[0] = semTemplBody(c, n.sons[0])
+  of nkPostfix:
+    result.sons[1] = semTemplBody(c, n.sons[1])
   of nkPragma:
-    discard
+    result = onlyReplaceParams(c, n)
   else:
     # dotExpr is ambiguous: note that we explicitely allow 'x.TemplateParam',
     # so we use the generic code for nkDotExpr too

compiler/semtypes.nim

@@ -141,11 +141,10 @@ proc semVarType(c: PContext, n: PNode, prev: PType): PType =
     result = newConstraint(c, tyVar)
 
 proc semDistinct(c: PContext, n: PNode, prev: PType): PType = 
-  if sonsLen(n) == 1:
+  if n.len == 0: return newConstraint(c, tyDistinct)
-    result = newOrPrevType(tyDistinct, prev, c)
+  result = newOrPrevType(tyDistinct, prev, c)
-    addSonSkipIntLit(result, semTypeNode(c, n.sons[0], nil))
+  addSonSkipIntLit(result, semTypeNode(c, n.sons[0], nil))
-  else:
+  if n.len > 1: result.n = n[1]
-    result = newConstraint(c, tyDistinct)
   
 proc semRangeAux(c: PContext, n: PNode, prev: PType): PType =
   assert isRange(n)
@@ -154,19 +153,31 @@ proc semRangeAux(c: PContext, n: PNode, prev: PType): PType =
   result.n = newNodeI(nkRange, n.info)
   if (n[1].kind == nkEmpty) or (n[2].kind == nkEmpty):
     localError(n.info, errRangeIsEmpty)
-  var a = semConstExpr(c, n[1])
+  
-  var b = semConstExpr(c, n[2])
+  var range: array[2, PNode]
-  if not sameType(a.typ, b.typ):
+  range[0] = semExprWithType(c, n[1], {efDetermineType})
+  range[1] = semExprWithType(c, n[2], {efDetermineType})
+  
+  var rangeT: array[2, PType]
+  for i in 0..1: rangeT[i] = range[i].typ.skipTypes({tyStatic}).skipIntLit
+
+  if not sameType(rangeT[0], rangeT[1]):
     localError(n.info, errPureTypeMismatch)
-  elif a.typ.kind notin {tyInt..tyInt64,tyEnum,tyBool,tyChar,
+  elif not rangeT[0].isOrdinalType:
-                         tyFloat..tyFloat128,tyUInt8..tyUInt32}:
     localError(n.info, errOrdinalTypeExpected)
-  elif enumHasHoles(a.typ): 
+  elif enumHasHoles(rangeT[0]):
-    localError(n.info, errEnumXHasHoles, a.typ.sym.name.s)
+    localError(n.info, errEnumXHasHoles, rangeT[0].sym.name.s)
-  elif not leValue(a, b): localError(n.info, errRangeIsEmpty)
+  
-  addSon(result.n, a)
+  for i in 0..1:
-  addSon(result.n, b)
+    if hasGenericArguments(range[i]):
-  addSonSkipIntLit(result, b.typ)
+      result.n.addSon makeStaticExpr(c, range[i])
+    else:
+      result.n.addSon semConstExpr(c, range[i])
+ 
+  if weakLeValue(result.n[0], result.n[1]) == impNo:
+    localError(n.info, errRangeIsEmpty)
+
+  addSonSkipIntLit(result, rangeT[0])
 
 proc semRange(c: PContext, n: PNode, prev: PType): PType =
   result = nil
@@ -186,11 +197,6 @@ proc semRange(c: PContext, n: PNode, prev: PType): PType =
     localError(n.info, errXExpectsOneTypeParam, "range")
     result = newOrPrevType(tyError, prev, c)
 
-proc nMinusOne(n: PNode): PNode =
-  result = newNode(nkCall, n.info, @[
-    newSymNode(getSysMagic("<", mUnaryLt)),
-    n])
-
 proc semArray(c: PContext, n: PNode, prev: PType): PType = 
   var indx, base: PType
   result = newOrPrevType(tyArray, prev, c)
@@ -200,7 +206,7 @@ proc semArray(c: PContext, n: PNode, prev: PType): PType =
     else:
       let e = semExprWithType(c, n.sons[1], {efDetermineType})
       if e.typ.kind == tyFromExpr:
-        indx = e.typ
+        indx = makeRangeWithStaticExpr(c, e.typ.n)
       elif e.kind in {nkIntLit..nkUInt64Lit}:
         indx = makeRangeType(c, 0, e.intVal-1, n.info, e.typ)
       elif e.kind == nkSym and e.typ.kind == tyStatic:
@@ -208,7 +214,8 @@ proc semArray(c: PContext, n: PNode, prev: PType): PType =
         internalAssert c.inGenericContext > 0
         if not isOrdinalType(e.typ.lastSon):
           localError(n[1].info, errOrdinalTypeExpected)
-        indx = e.typ
+        indx = makeRangeWithStaticExpr(c, e)
+        indx.flags.incl tfUnresolved
       elif e.kind in nkCallKinds and hasGenericArguments(e):
         if not isOrdinalType(e.typ):
           localError(n[1].info, errOrdinalTypeExpected)
@@ -217,12 +224,7 @@ proc semArray(c: PContext, n: PNode, prev: PType): PType =
         # We are going to construct a range type that will be
         # properly filled-out in semtypinst (see how tyStaticExpr
         # is handled there).
-        let intType = getSysType(tyInt)
+        indx = makeRangeWithStaticExpr(c, e)
-        indx = newTypeS(tyRange, c)
-        indx.sons = @[intType]
-        indx.n = newNode(nkRange, n.info, @[
-          newIntTypeNode(nkIntLit, 0, intType),
-          makeStaticExpr(c, e.nMinusOne)])
       else:
         indx = e.typ.skipTypes({tyTypeDesc})
     addSonSkipIntLit(result, indx)
@@ -271,6 +273,18 @@ proc semTypeIdent(c: PContext, n: PNode): PSym =
         result = result.typ.sym.copySym
         result.typ = copyType(result.typ, result.typ.owner, true)
         result.typ.flags.incl tfUnresolved
+      
+      if result.kind == skGenericParam:
+        if result.typ.kind == tyGenericParam and result.typ.len == 0 and
+           tfWildcard in result.typ.flags:
+          # collapse the wild-card param to a type
+          result.kind = skType
+          result.typ.flags.excl tfWildcard
+          return
+        else:
+          localError(n.info, errTypeExpected)
+          return errorSym(c, n)
+
       if result.kind != skType: 
         # this implements the wanted ``var v: V, x: V`` feature ...
         var ov: TOverloadIter
@@ -601,16 +615,36 @@ proc semObjectNode(c: PContext, n: PNode, prev: PType): PType =
   if base == nil and tfInheritable notin result.flags:
     incl(result.flags, tfFinal)
 
+proc findEnforcedStaticType(t: PType): PType =
+  # This handles types such as `static[T] and Foo`,
+  # which are subset of `static[T]`, hence they could
+  # be treated in the same way
+  if t.kind == tyStatic: return t
+  if t.kind == tyAnd:
+    for s in t.sons:
+      let t = findEnforcedStaticType(s)
+      if t != nil: return t
+
 proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind) =
-  if kind == skMacro and param.typ.kind notin {tyTypeDesc, tyStatic}:
+  template addDecl(x) =
-    # within a macro, every param has the type PNimrodNode!
+    if sfGenSym notin x.flags: addDecl(c, x)
-    # and param.typ.kind in {tyTypeDesc, tyExpr, tyStmt}:
+
-    let nn = getSysSym"PNimrodNode"
+  if kind == skMacro:
-    var a = copySym(param)
+    let staticType = findEnforcedStaticType(param.typ)
-    a.typ = nn.typ
+    if staticType != nil:
-    if sfGenSym notin a.flags: addDecl(c, a)
+      var a = copySym(param)
+      a.typ = staticType.base
+      addDecl(a)
+    elif param.typ.kind == tyTypeDesc:
+      addDecl(param)
+    else:
+      # within a macro, every param has the type PNimrodNode!
+      let nn = getSysSym"PNimrodNode"
+      var a = copySym(param)
+      a.typ = nn.typ
+      addDecl(a)
   else:
-    if sfGenSym notin param.flags: addDecl(c, param)
+    addDecl(param)
 
 let typedescId = getIdent"typedesc"
 
@@ -644,6 +678,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
     s.position = genericParams.len
     genericParams.addSon(newSymNode(s))
     result = typeClass
+    addDecl(c, s)
  
   # XXX: There are codegen errors if this is turned into a nested proc
   template liftingWalk(typ: PType, anonFlag = false): expr =
@@ -653,6 +688,10 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
   var paramTypId = if not anon and paramType.sym != nil: paramType.sym.name
                    else: nil
 
+  template maybeLift(typ: PType): expr =
+    let lifted = liftingWalk(typ)
+    (if lifted != nil: lifted else: typ)
+
   template addImplicitGeneric(e: expr): expr =
     addImplicitGenericImpl(e, paramTypId)
 
@@ -663,15 +702,21 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
   of tyStatic:
     # proc(a: expr{string}, b: expr{nkLambda})
     # overload on compile time values and AST trees
-    result = addImplicitGeneric(c.newTypeWithSons(tyStatic, paramType.sons))
+    if paramType.n != nil: return # this is a concrete type
-    result.flags.incl tfHasStatic
+    if tfUnresolved in paramType.flags: return # already lifted
+    let base = paramType.base.maybeLift
+    if base.isMetaType and procKind == skMacro:
+      localError(info, errMacroBodyDependsOnGenericTypes, paramName)
+    result = addImplicitGeneric(c.newTypeWithSons(tyStatic, @[base]))
+    result.flags.incl({tfHasStatic, tfUnresolved})
   
   of tyTypeDesc:
     if tfUnresolved notin paramType.flags:
       # naked typedescs are not bindOnce types
-      if paramType.sonsLen == 0 and paramTypId != nil and
+      if paramType.base.kind == tyNone and paramTypId != nil and
          paramTypId.id == typedescId.id: paramTypId = nil
-      result = addImplicitGeneric(c.newTypeWithSons(tyTypeDesc, paramType.sons))
+      result = addImplicitGeneric(
+        c.newTypeWithSons(tyTypeDesc, @[paramType.base]))
   
   of tyDistinct:
     if paramType.sonsLen == 1:
@@ -703,12 +748,26 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
     for i in 0 .. paramType.sonsLen - 2:
       result.rawAddSon newTypeS(tyAnything, c)
       # result.rawAddSon(copyType(paramType.sons[i], getCurrOwner(), true))
+
+    if paramType.lastSon.kind == tyUserTypeClass:
+      result.kind = tyUserTypeClassInst
+      result.rawAddSon paramType.lastSon
+      return addImplicitGeneric(result)
+   
     result = instGenericContainer(c, paramType.sym.info, result,
                                   allowMetaTypes = true)
-    result.lastSon.shouldHaveMeta
     result = newTypeWithSons(c, tyCompositeTypeClass, @[paramType, result])
     result = addImplicitGeneric(result)
 
+  of tyIter:
+    if paramType.callConv == ccInline:
+      if procKind notin {skTemplate, skMacro, skIterator}:
+        localError(info, errInlineIteratorsAsProcParams)
+      if paramType.len == 1:
+        let lifted = liftingWalk(paramType.base)
+        if lifted != nil: paramType.sons[0] = lifted
+      result = addImplicitGeneric(paramType)
+
   of tyGenericInst:
     if paramType.lastSon.kind == tyUserTypeClass:
       var cp = copyType(paramType, getCurrOwner(), false)
@@ -722,7 +781,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
         result = paramType
         result.lastSon.shouldHaveMeta
 
-    let liftBody = liftingWalk(paramType.lastSon)
+    let liftBody = liftingWalk(paramType.lastSon, true)
     if liftBody != nil:
       result = liftBody
       result.shouldHaveMeta
@@ -734,7 +793,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
 
     let expanded = instGenericContainer(c, info, paramType,
                                         allowMetaTypes = true)
-    result = liftingWalk(expanded)
+    result = liftingWalk(expanded, true)
 
   of tyUserTypeClass, tyBuiltInTypeClass, tyAnd, tyOr, tyNot:
     result = addImplicitGeneric(copyType(paramType, getCurrOwner(), true))
@@ -744,11 +803,10 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
       result = addImplicitGeneric(newTypeS(tyAnything, c))
   
   of tyGenericParam:
-    if tfGenericTypeParam in paramType.flags and false:
+    markUsed(genericParams, paramType.sym)
-      if paramType.sonsLen > 0:
+    if tfWildcard in paramType.flags:
-        result = liftingWalk(paramType.lastSon)
+      paramType.flags.excl tfWildcard
-      else:
+      paramType.sym.kind = skType
-        result = addImplicitGeneric(newTypeS(tyAnything, c))
     
   else: discard
 
@@ -827,9 +885,13 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
       addParamOrResult(c, arg, kind)
       if gCmd == cmdPretty: checkDef(a.sons[j], arg)
 
-
+  var r: PType
   if n.sons[0].kind != nkEmpty:
-    var r = semTypeNode(c, n.sons[0], nil)
+    r = semTypeNode(c, n.sons[0], nil)
+  elif kind == skIterator:
+    r = newTypeS(tyAnything, c)
+  
+  if r != nil:
     # turn explicit 'void' return type into 'nil' because the rest of the 
     # compiler only checks for 'nil':
     if skipTypes(r, {tyGenericInst}).kind != tyEmpty:
@@ -838,8 +900,20 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
                                    n.sons[0].info)
         if lifted != nil: r = lifted
         r.flags.incl tfRetType
-      result.sons[0] = skipIntLit(r)
+      r = skipIntLit(r)
-      res.typ = result.sons[0]
+      if kind == skIterator:
+        # see tchainediterators
+        # in cases like iterator foo(it: iterator): type(it)
+        # we don't need to change the return type to iter[T]
+        if not r.isInlineIterator: r = newTypeWithSons(c, tyIter, @[r])
+      result.sons[0] = r
+      res.typ = r
+
+  if genericParams != nil:
+    for n in genericParams:
+      if tfWildcard in n.sym.typ.flags:
+        n.sym.kind = skType
+        n.sym.typ.flags.excl tfWildcard
 
 proc semStmtListType(c: PContext, n: PNode, prev: PType): PType =
   checkMinSonsLen(n, 1)
@@ -869,8 +943,17 @@ proc semGenericParamInInvokation(c: PContext, n: PNode): PType =
   result = semTypeNode(c, n, nil)
 
 proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
+  if s.typ == nil:
+    localError(n.info, "cannot instantiate the '$1' $2" %
+                       [s.name.s, ($s.kind).substr(2).toLower])
+    return newOrPrevType(tyError, prev, c)
+  
+  var t = s.typ
+  if t.kind == tyCompositeTypeClass and t.base.kind == tyGenericBody:
+    t = t.base
+
   result = newOrPrevType(tyGenericInvokation, prev, c)
-  addSonSkipIntLit(result, s.typ)
+  addSonSkipIntLit(result, t)
 
   template addToResult(typ) =
     if typ.isNil:
@@ -878,27 +961,24 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
       rawAddSon(result, typ)
     else: addSonSkipIntLit(result, typ)
 
-  if s.typ == nil:
+  if t.kind == tyForward:
-    localError(n.info, errCannotInstantiateX, s.name.s)
-    return newOrPrevType(tyError, prev, c)
-  elif s.typ.kind == tyForward:
     for i in countup(1, sonsLen(n)-1):
       var elem = semGenericParamInInvokation(c, n.sons[i])
       addToResult(elem)
-  elif s.typ.kind != tyGenericBody:
+    return
+  elif t.kind != tyGenericBody:
     #we likely got code of the form TypeA[TypeB] where TypeA is
     #not generic.
     localError(n.info, errNoGenericParamsAllowedForX, s.name.s)
     return newOrPrevType(tyError, prev, c)
   else:
-
+    var m = newCandidate(c, t)
-    var m = newCandidate(c, s, n)
     matches(c, n, copyTree(n), m)
     
     if m.state != csMatch:
-      var err = "cannot instantiate " & typeToString(s.typ) & "\n" &
+      var err = "cannot instantiate " & typeToString(t) & "\n" &
                 "got: (" & describeArgs(c, n) & ")\n" &
-                "but expected: (" & describeArgs(c, s.typ.n, 0) & ")"
+                "but expected: (" & describeArgs(c, t.n, 0) & ")"
       localError(n.info, errGenerated, err)
       return newOrPrevType(tyError, prev, c)
 
@@ -910,7 +990,8 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
       addToResult(typ)
    
     if isConcrete:
-      if s.ast == nil:
+      if s.ast == nil and s.typ.kind != tyCompositeTypeClass:
+        # XXX: What kind of error is this? is it still relevant?
         localError(n.info, errCannotInstantiateX, s.name.s)
         result = newOrPrevType(tyError, prev, c)
       else:
@@ -944,6 +1025,23 @@ proc semTypeClass(c: PContext, n: PNode, prev: PType): PType =
       let typ = semTypeNode(c, n, nil)
       result.sons.safeAdd(typ)
 
+proc semProcTypeWithScope(c: PContext, n: PNode,
+                        prev: PType, kind: TSymKind): PType =
+  checkSonsLen(n, 2)
+  openScope(c)
+  result = semProcTypeNode(c, n.sons[0], nil, prev, kind)
+  # dummy symbol for `pragma`:
+  var s = newSymS(kind, newIdentNode(getIdent("dummy"), n.info), c)
+  s.typ = result
+  if n.sons[1].kind == nkEmpty or n.sons[1].len == 0:
+    if result.callConv == ccDefault:
+      result.callConv = ccClosure
+      #Message(n.info, warnImplicitClosure, renderTree(n))
+  else:
+    pragma(c, s, n.sons[1], procTypePragmas)
+    when useEffectSystem: setEffectsForProcType(result, n.sons[1])
+  closeScope(c)
+
 proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
   result = nil
   if gCmd == cmdIdeTools: suggestExpr(c, n)
@@ -952,7 +1050,8 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
   of nkTypeOfExpr:
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     checkSonsLen(n, 1)
-    result = semExprWithType(c, n.sons[0], {efInTypeof}).typ
+    let typExpr = semExprWithType(c, n.sons[0], {efInTypeof})
+    result = typExpr.typ.skipTypes({tyIter})
   of nkPar: 
     if sonsLen(n) == 1: result = semTypeNode(c, n.sons[0], prev)
     else:
@@ -1011,28 +1110,32 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     of mOrdinal: result = semOrdinal(c, n, prev)
     of mSeq: result = semContainer(c, n, tySequence, "seq", prev)
     of mVarargs: result = semVarargs(c, n, prev)
-    of mExpr, mTypeDesc:
+    of mTypeDesc: result = makeTypeDesc(c, semTypeNode(c, n[1], nil))
+    of mExpr:
       result = semTypeNode(c, n.sons[0], nil)
       if result != nil:
         result = copyType(result, getCurrOwner(), false)
         for i in countup(1, n.len - 1):
           result.rawAddSon(semTypeNode(c, n.sons[i], nil))
     else: result = semGeneric(c, n, s, prev)
-  of nkIdent, nkDotExpr, nkAccQuoted: 
+  of nkDotExpr:
-    if n.kind == nkDotExpr:
+    var typeExpr = semExpr(c, n)
-      let head = qualifiedLookUp(c, n[0], {checkAmbiguity, checkUndeclared})
+    if typeExpr.typ.kind != tyTypeDesc:
-      if head.kind in {skType}:
+      localError(n.info, errTypeExpected)
-        var toBind = initIntSet()
+      return errorType(c)
-        var preprocessed = semGenericStmt(c, n, {}, toBind)
+    result = typeExpr.typ.base
-        return makeTypeFromExpr(c, preprocessed)
+    if result.isMetaType:
+      var toBind = initIntSet()
+      var preprocessed = semGenericStmt(c, n, {}, toBind)
+      return makeTypeFromExpr(c, preprocessed)
+  of nkIdent, nkAccQuoted:
     var s = semTypeIdent(c, n)
     if s.typ == nil: 
       if s.kind != skError: localError(n.info, errTypeExpected)
       result = newOrPrevType(tyError, prev, c)
     elif s.kind == skParam and s.typ.kind == tyTypeDesc:
-      assert s.typ.len > 0
+      internalAssert s.typ.base.kind != tyNone and prev == nil
-      internalAssert prev == nil
+      result = s.typ.base
-      result = s.typ.sons[0]
     elif prev == nil:
       result = s.typ
     else: 
@@ -1066,27 +1169,22 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     var base = semTypeNode(c, n.sons[0], nil)
     result.rawAddSon(base)
     result.flags.incl tfHasStatic
-  of nkProcTy, nkIteratorTy:
+  of nkIteratorTy:
+    if n.sonsLen == 0:
+      result = newConstraint(c, tyIter)
+    else:
+      result = semProcTypeWithScope(c, n, prev, skClosureIterator)
+      if n.lastSon.kind == nkPragma and hasPragma(n.lastSon, wInline):
+        result.kind = tyIter
+        result.callConv = ccInline
+      else:
+        result.flags.incl(tfIterator)
+        result.callConv = ccClosure
+  of nkProcTy:
     if n.sonsLen == 0:
       result = newConstraint(c, tyProc)
     else:
-      checkSonsLen(n, 2)
+      result = semProcTypeWithScope(c, n, prev, skProc)
-      openScope(c)
-      result = semProcTypeNode(c, n.sons[0], nil, prev, skProc)
-      # dummy symbol for `pragma`:
-      var s = newSymS(skProc, newIdentNode(getIdent("dummy"), n.info), c)
-      s.typ = result
-      if n.sons[1].kind == nkEmpty or n.sons[1].len == 0:
-        if result.callConv == ccDefault:
-          result.callConv = ccClosure
-          #Message(n.info, warnImplicitClosure, renderTree(n))
-      else:
-        pragma(c, s, n.sons[1], procTypePragmas)
-        when useEffectSystem: setEffectsForProcType(result, n.sons[1])
-      closeScope(c)
-    if n.kind == nkIteratorTy:
-      result.flags.incl(tfIterator)
-      result.callConv = ccClosure
   of nkEnumTy: result = semEnum(c, n, prev)
   of nkType: result = n.typ
   of nkStmtListType: result = semStmtListType(c, n, prev)
@@ -1196,6 +1294,7 @@ proc semGenericParamList(c: PContext, n: PNode, father: PType = nil): PNode =
     
     if typ == nil:
       typ = newTypeS(tyGenericParam, c)
+      if father == nil: typ.flags.incl tfWildcard
 
     typ.flags.incl tfGenericTypeParam
 
@@ -1206,8 +1305,7 @@ proc semGenericParamList(c: PContext, n: PNode, father: PType = nil): PNode =
                       # type for each generic param. the index
                       # of the parameter will be stored in the
                       # attached symbol.
-      var s = case finalType.kind
+      var s = if finalType.kind == tyStatic or tfWildcard in typ.flags:
-        of tyStatic:
           newSymG(skGenericParam, a.sons[j], c).linkTo(finalType)
         else:
           newSymG(skType, a.sons[j], c).linkTo(finalType)

compiler/semtypinst.nim

@@ -29,6 +29,7 @@ proc checkConstructedType*(info: TLineInfo, typ: PType) =
     localError(info, errVarVarTypeNotAllowed)
   elif computeSize(t) == szIllegalRecursion:
     localError(info, errIllegalRecursionInTypeX, typeToString(t))
+  
   when false:
     if t.kind == tyObject and t.sons[0] != nil:
       if t.sons[0].kind != tyObject or tfFinal in t.sons[0].flags: 
@@ -79,7 +80,7 @@ type
 
 proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType
 proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym
-proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode
+proc replaceTypeVarsN*(cl: var TReplTypeVars, n: PNode): PNode
 
 template checkMetaInvariants(cl: TReplTypeVars, t: PType) =
   when false:
@@ -95,8 +96,11 @@ proc replaceTypeVarsT*(cl: var TReplTypeVars, t: PType): PType =
   checkMetaInvariants(cl, result)
 
 proc prepareNode(cl: var TReplTypeVars, n: PNode): PNode =
+  let t = replaceTypeVarsT(cl, n.typ)
+  if t != nil and t.kind == tyStatic and t.n != nil:
+    return t.n
   result = copyNode(n)
-  result.typ = replaceTypeVarsT(cl, n.typ)
+  result.typ = t
   if result.kind == nkSym: result.sym = replaceTypeVarsS(cl, n.sym)
   let isCall = result.kind in nkCallKinds
   for i in 0 .. <n.safeLen:
@@ -153,6 +157,9 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode =
     discard
   of nkSym:
     result.sym = replaceTypeVarsS(cl, n.sym)
+    if result.sym.typ.kind == tyEmpty:
+      # don't add the 'void' field
+      result = newNode(nkRecList, n.info)
   of nkRecWhen:
     var branch: PNode = nil              # the branch to take
     for i in countup(0, sonsLen(n) - 1):
@@ -177,7 +184,8 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode =
   of nkStaticExpr:
     var n = prepareNode(cl, n)
     n = reResolveCallsWithTypedescParams(cl, n)
-    result = cl.c.semExpr(cl.c, n)
+    result = if cl.allowMetaTypes: n
+             else: cl.c.semExpr(cl.c, n)
   else:
     var length = sonsLen(n)
     if length > 0:
@@ -192,8 +200,8 @@ proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym =
     result = copySym(s, false)
     incl(result.flags, sfFromGeneric)
     idTablePut(cl.symMap, s, result)
-    result.typ = replaceTypeVarsT(cl, s.typ)
     result.owner = s.owner
+    result.typ = replaceTypeVarsT(cl, s.typ)
     result.ast = replaceTypeVarsN(cl, s.ast)
     
 proc lookupTypeVar(cl: TReplTypeVars, t: PType): PType = 
@@ -205,7 +213,7 @@ proc lookupTypeVar(cl: TReplTypeVars, t: PType): PType =
   elif result.kind == tyGenericParam and not cl.allowMetaTypes:
     internalError(cl.info, "substitution with generic parameter")
 
-proc instCopyType(cl: var TReplTypeVars, t: PType): PType =
+proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
   # XXX: relying on allowMetaTypes is a kludge
   result = copyType(t, t.owner, cl.allowMetaTypes)
   result.flags.incl tfFromGeneric
@@ -216,7 +224,7 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
   # is difficult to handle: 
   var body = t.sons[0]
   if body.kind != tyGenericBody: internalError(cl.info, "no generic body")
-  var header: PType = nil
+  var header: PType = t
   # search for some instantiation here:
   if cl.allowMetaTypes:
     result = PType(idTableGet(cl.localCache, t))
@@ -228,11 +236,13 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
     if x.kind == tyGenericParam:
       x = lookupTypeVar(cl, x)
       if x != nil:
-        if header == nil: header = instCopyType(cl, t)
+        if header == t: header = instCopyType(cl, t)
         header.sons[i] = x
         propagateToOwner(header, x)
+    else:
+      propagateToOwner(header, x)
   
-  if header != nil:
+  if header != t:
     # search again after first pass:
     result = searchInstTypes(header)
     if result != nil: return
@@ -240,6 +250,7 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
     header = instCopyType(cl, t)
   
   result = newType(tyGenericInst, t.sons[0].owner)
+  result.flags = header.flags
   # be careful not to propagate unnecessary flags here (don't use rawAddSon)
   result.sons = @[header.sons[0]]
   # ugh need another pass for deeply recursive generic types (e.g. PActor)
@@ -273,7 +284,7 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
   rawAddSon(result, newbody)
   checkPartialConstructedType(cl.info, newbody)
 
-proc eraseVoidParams(t: PType) =
+proc eraseVoidParams*(t: PType) =
   if t.sons[0] != nil and t.sons[0].kind == tyEmpty:
     t.sons[0] = nil
   
@@ -290,7 +301,7 @@ proc eraseVoidParams(t: PType) =
       setLen t.n.sons, pos
       return
 
-proc skipIntLiteralParams(t: PType) =
+proc skipIntLiteralParams*(t: PType) =
   for i in 0 .. <t.sonsLen:
     let p = t.sons[i]
     if p == nil: continue
@@ -299,6 +310,11 @@ proc skipIntLiteralParams(t: PType) =
       t.sons[i] = skipped
       if i > 0: t.n.sons[i].sym.typ = skipped
   
+  # when the typeof operator is used on a static input
+  # param, the results gets infected with static as well:
+  if t.sons[0] != nil and t.sons[0].kind == tyStatic:
+    t.sons[0] = t.sons[0].base
+
 proc propagateFieldFlags(t: PType, n: PNode) =
   # This is meant for objects and tuples
   # The type must be fully instantiated!
@@ -307,16 +323,15 @@ proc propagateFieldFlags(t: PType, n: PNode) =
   of nkSym:
     propagateToOwner(t, n.sym.typ)
   of nkRecList, nkRecCase, nkOfBranch, nkElse:
-    if n.sons != nil:
+    for son in n:
-      for son in n.sons:
+      propagateFieldFlags(t, son)
-        propagateFieldFlags(t, son)
   else: discard
 
 proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
   result = t
   if t == nil: return
 
-  if t.kind in {tyStatic, tyGenericParam} + tyTypeClasses:
+  if t.kind in {tyStatic, tyGenericParam, tyIter} + tyTypeClasses:
     let lookup = PType(idTableGet(cl.typeMap, t))
     if lookup != nil: return lookup
   
@@ -329,6 +344,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
     result = replaceTypeVarsT(cl, lastSon(t))
 
   of tyFromExpr:
+    if cl.allowMetaTypes: return
     var n = prepareNode(cl, t.n)
     n = cl.c.semConstExpr(cl.c, n)
     if n.typ.kind == tyTypeDesc:
@@ -382,20 +398,10 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
 
       result.n = replaceTypeVarsN(cl, result.n)
      
-      # XXX: This is not really needed?
-      # if result.kind in GenericTypes:
-      #   localError(cl.info, errCannotInstantiateX, typeToString(t, preferName))
-
       case result.kind
       of tyArray:
         let idx = result.sons[0]
-        if idx.kind == tyStatic:
+        internalAssert idx.kind != tyStatic
-          if idx.n == nil:
-            let lookup = lookupTypeVar(cl, idx)
-            internalAssert lookup != nil
-            idx.n = lookup.n
-
-          result.sons[0] = makeRangeType(cl.c, 0, idx.n.intVal - 1, idx.n.info)
        
       of tyObject, tyTuple:
         propagateFieldFlags(result, result.n)
@@ -406,14 +412,22 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
       
       else: discard
 
+proc initTypeVars*(p: PContext, pt: TIdTable, info: TLineInfo): TReplTypeVars =
+  initIdTable(result.symMap)
+  copyIdTable(result.typeMap, pt)
+  initIdTable(result.localCache)
+  result.info = info
+  result.c = p
+
+proc replaceTypesInBody*(p: PContext, pt: TIdTable, n: PNode): PNode =
+  var cl = initTypeVars(p, pt, n.info)
+  pushInfoContext(n.info)
+  result = replaceTypeVarsN(cl, n)
+  popInfoContext()
+  
 proc generateTypeInstance*(p: PContext, pt: TIdTable, info: TLineInfo,
                            t: PType): PType =
-  var cl: TReplTypeVars
+  var cl = initTypeVars(p, pt, info)
-  initIdTable(cl.symMap)
-  copyIdTable(cl.typeMap, pt)
-  initIdTable(cl.localCache)
-  cl.info = info
-  cl.c = p
   pushInfoContext(info)
   result = replaceTypeVarsT(cl, t)
   popInfoContext()

compiler/sigmatch.nim

@@ -12,7 +12,7 @@
 
 import 
   intsets, ast, astalgo, semdata, types, msgs, renderer, lookups, semtypinst,
-  magicsys, condsyms, idents, lexer, options, parampatterns, strutils
+  magicsys, condsyms, idents, lexer, options, parampatterns, strutils, trees
 
 when not defined(noDocgen):
   import docgen
@@ -40,6 +40,8 @@ type
     proxyMatch*: bool        # to prevent instantiations
     genericConverter*: bool  # true if a generic converter needs to
                              # be instantiated
+    coerceDistincts*: bool   # this is an explicit coercion that can strip away
+                             # a distrinct type
     typedescMatched: bool
     inheritancePenalty: int  # to prefer closest father object type
     errors*: seq[string]     # additional clarifications to be displayed to the
@@ -51,6 +53,8 @@ type
     isSubtype,
     isSubrange,              # subrange of the wanted type; no type conversion
                              # but apart from that counts as ``isSubtype``
+    isInferred,              # generic proc was matched against a concrete type
+    isInferredConvertible,   # same as above, but requiring proc CC conversion
     isGeneric,
     isFromIntLit,            # conversion *from* int literal; proven safe
     isEqual
@@ -112,6 +116,9 @@ proc newCandidate*(ctx: PContext, callee: PSym,
                    binding: PNode, calleeScope = -1): TCandidate =
   initCandidate(ctx, result, callee, binding, calleeScope)
 
+proc newCandidate*(ctx: PContext, callee: PType): TCandidate =
+  initCandidate(ctx, result, callee)
+
 proc copyCandidate(a: var TCandidate, b: TCandidate) = 
   a.c = b.c
   a.exactMatches = b.exactMatches
@@ -338,34 +345,60 @@ proc recordRel(c: var TCandidate, f, a: PType): TTypeRelation =
 proc allowsNil(f: PType): TTypeRelation {.inline.} =
   result = if tfNotNil notin f.flags: isSubtype else: isNone
 
-proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
+proc inconsistentVarTypes(f, a: PType): bool {.inline.} =
-  proc inconsistentVarTypes(f, a: PType): bool {.inline.} =
+  result = f.kind != a.kind and (f.kind == tyVar or a.kind == tyVar)
-    result = f.kind != a.kind and (f.kind == tyVar or a.kind == tyVar)
 
+proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
+  var f = f
+
+  if a.isMetaType:
+    if f.isMetaType:
+      # We are matching a generic proc (as proc param)
+      # to another generic type appearing in the proc
+      # signature. There is a change that the target
+      # type is already fully-determined, so we are 
+      # going to try resolve it
+      f = generateTypeInstance(c.c, c.bindings, c.call.info, f)
+      if f == nil or f.isMetaType:
+        # no luck resolving the type, so the inference fails
+        return isNone
+    let reverseRel = typeRel(c, a, f)
+    if reverseRel == isGeneric:
+      result = isInferred
+      inc c.genericMatches
+  else:
+    result = typeRel(c, f, a)
+
+  if result <= isSubtype or inconsistentVarTypes(f, a):
+    result = isNone
+ 
+  if result == isEqual:
+    inc c.exactMatches
+    
+proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
   case a.kind
   of tyProc:
     if sonsLen(f) != sonsLen(a): return
-    # Note: We have to do unification for the parameters before the
-    # return type!
     result = isEqual      # start with maximum; also correct for no
                           # params at all
-    for i in countup(1, sonsLen(f)-1):
+    
-      var m = typeRel(c, f.sons[i], a.sons[i])
+    template checkParam(f, a) =
-      if m <= isSubtype or inconsistentVarTypes(f.sons[i], a.sons[i]):
+      result = minRel(result, procParamTypeRel(c, f, a))
-        return isNone
+      if result == isNone: return
-      else: result = minRel(m, result)
+
+    # Note: We have to do unification for the parameters before the
+    # return type!
+    for i in 1 .. <f.sonsLen:
+      checkParam(f.sons[i], a.sons[i])
+    
     if f.sons[0] != nil:
       if a.sons[0] != nil:
-        var m = typeRel(c, f.sons[0], a.sons[0])
+        checkParam(f.sons[0], a.sons[0])
-        # Subtype is sufficient for return types!
-        if m < isSubtype or inconsistentVarTypes(f.sons[0], a.sons[0]):
-          return isNone
-        elif m == isSubtype: result = isConvertible
-        else: result = minRel(m, result)
       else:
         return isNone
     elif a.sons[0] != nil:
       return isNone
+
     if tfNoSideEffect in f.flags and tfNoSideEffect notin a.flags:
       return isNone
     elif tfThread in f.flags and a.flags * {tfThread, tfNoSideEffect} == {}:
@@ -376,12 +409,16 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
     elif f.callConv != a.callConv:
       # valid to pass a 'nimcall' thingie to 'closure':
       if f.callConv == ccClosure and a.callConv == ccDefault:
-        result = isConvertible
+        result = if result != isInferred: isConvertible
+                 else: isInferredConvertible
       else:
         return isNone
     when useEffectSystem:
       if not compatibleEffects(f, a): return isNone
-  of tyNil: result = f.allowsNil
+  of tyNil:
+    result = f.allowsNil
+  of tyIter:
+    if tfIterator in f.flags: result = typeRel(c, f.base, a.base)
   else: discard
 
 proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} =
@@ -402,18 +439,8 @@ proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} =
 
 proc matchUserTypeClass*(c: PContext, m: var TCandidate,
                          ff, a: PType): TTypeRelation =
-  #if f.n == nil:
-  #  let r = typeRel(m, f, a)
-  #  return if r == isGeneric: arg else: nil
-
   var body = ff.skipTypes({tyUserTypeClassInst})
 
-  # var prev = PType(idTableGet(m.bindings, f))
-  # if prev != nil:
-  #   if sameType(prev, a): return arg
-  #   else: return nil
-
-  # pushInfoContext(arg.info)
   openScope(c)
   inc c.inTypeClass
 
@@ -438,7 +465,8 @@ proc matchUserTypeClass*(c: PContext, m: var TCandidate,
     
     if param.kind == nkVarTy:
       dummyName = param[0]
-      dummyType = makeVarType(c, a)
+      dummyType = if a.kind != tyVar: makeVarType(c, a)
+                  else: a
     else:
       dummyName = param
       dummyType = a
@@ -448,7 +476,7 @@ proc matchUserTypeClass*(c: PContext, m: var TCandidate,
     dummyParam.typ = dummyType
     addDecl(c, dummyParam)
 
-  var checkedBody = c.semTryExpr(c, copyTree(body.n[3]), bufferErrors = false)
+  var checkedBody = c.semTryExpr(c, body.n[3].copyTree, bufferErrors = false)
   m.errors = bufferedMsgs
   clearBufferedMsgs()
   if checkedBody == nil: return isNone
@@ -462,7 +490,23 @@ proc matchUserTypeClass*(c: PContext, m: var TCandidate,
       else: discard
     
   return isGeneric
-  # put(m.bindings, f, a)
+
+proc shouldSkipDistinct(rules: PNode, callIdent: PIdent): bool =
+  if rules.kind == nkWith:
+    for r in rules:
+      if r.considerAcc == callIdent: return true
+    return false
+  else:
+    for r in rules:
+      if r.considerAcc == callIdent: return false
+    return true
+
+proc maybeSkipDistinct(t: PType, callee: PSym): PType =
+  if t != nil and t.kind == tyDistinct and t.n != nil and
+     shouldSkipDistinct(t.n, callee.name):
+    result = t.base
+  else:
+    result = t
 
 proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   # typeRel can be used to establish various relationships between types:
@@ -491,7 +535,11 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   assert(aOrig != nil)
 
   # var and static arguments match regular modifier-free types
-  let a = aOrig.skipTypes({tyStatic, tyVar})
+  let a = aOrig.skipTypes({tyStatic, tyVar}).maybeSkipDistinct(c.calleeSym)
+  # XXX: Theoretically, maybeSkipDistinct could be called before we even
+  # start the param matching process. This could be done in `prepareOperand`
+  # for example, but unfortunately `prepareOperand` is not called in certain
+  # situation when nkDotExpr are rotated to nkDotCalls
   
   if a.kind == tyGenericInst and
       skipTypes(f, {tyVar}).kind notin {
@@ -501,8 +549,9 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
 
   template bindingRet(res) =
     when res == isGeneric:
-      let bound = aOrig.skipTypes({tyRange}).skipIntLit
+      if doBind:
-      put(c.bindings, f, bound)
+        let bound = aOrig.skipTypes({tyRange}).skipIntLit
+        if doBind: put(c.bindings, f, bound)
     return res
 
   template considerPreviousT(body: stmt) {.immediate.} =
@@ -599,8 +648,24 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
         else:
           fRange = prev
       result = typeRel(c, f.sons[1], a.sons[1])
-      if result < isGeneric: result = isNone
+      if result < isGeneric:
-      elif lengthOrd(fRange) != lengthOrd(a): result = isNone
+        result = isNone
+      elif tfUnresolved in fRange.flags and
+           rangeHasStaticIf(fRange):
+        # This is a range from an array instantiated with a generic
+        # static param. We must extract the static param here and bind
+        # it to the size of the currently supplied array.
+        var
+          rangeStaticT = fRange.getStaticTypeFromRange
+          replacementT = newTypeWithSons(c.c, tyStatic, @[tyInt.getSysType])
+          inputUpperBound = a.sons[0].n[1].intVal
+        # we must correct for the off-by-one discrepancy between
+        # ranges and static params:
+        replacementT.n = newIntNode(nkIntLit, inputUpperBound + 1)
+        put(c.bindings, rangeStaticT, replacementT)
+        result = isGeneric
+      elif lengthOrd(fRange) != lengthOrd(a):
+        result = isNone
     else: discard
   of tyOpenArray, tyVarargs:
     case a.kind
@@ -662,6 +727,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
           result = isSubtype
   of tyDistinct:
     if (a.kind == tyDistinct) and sameDistinctTypes(f, a): result = isEqual
+    elif c.coerceDistincts: result = typeRel(c, f.base, a)
   of tySet: 
     if a.kind == tySet: 
       if (f.sons[0].kind != tyGenericParam) and (a.sons[0].kind == tyEmpty): 
@@ -824,7 +890,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
 
   of tyUserTypeClass, tyUserTypeClassInst:
     considerPreviousT:
-      result = matchUserTypeClass(c.c, c, f, a)
+      result = matchUserTypeClass(c.c, c, f, aOrig)
       if result == isGeneric:
         put(c.bindings, f, a)
 
@@ -839,20 +905,22 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   of tyGenericParam:
     var x = PType(idTableGet(c.bindings, f))
     if x == nil:
-      if c.calleeSym != nil and c.calleeSym.kind == skType and
+      if c.callee.kind == tyGenericBody and
          f.kind == tyGenericParam and not c.typedescMatched:
         # XXX: The fact that generic types currently use tyGenericParam for 
         # their parameters is really a misnomer. tyGenericParam means "match
         # any value" and what we need is "match any type", which can be encoded
         # by a tyTypeDesc params. Unfortunately, this requires more substantial
         # changes in semtypinst and elsewhere.
-        if a.kind == tyTypeDesc:
+        if tfWildcard in a.flags:
+          result = isGeneric
+        elif a.kind == tyTypeDesc:
           if f.sonsLen == 0:
             result = isGeneric
           else:
             internalAssert a.sons != nil and a.sons.len > 0
             c.typedescMatched = true
-            result = typeRel(c, f.sons[0], a.sons[0])
+            result = typeRel(c, f.base, a.base)
         else:
           result = isNone
       else:
@@ -862,11 +930,16 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
           result = isGeneric
 
       if result == isGeneric:
-        var concrete = concreteType(c, a)
+        var concrete = a
-        if concrete == nil:
+        if tfWildcard in a.flags:
-          result = isNone
+          a.sym.kind = skType
+          a.flags.excl tfWildcard
         else:
-          if doBind: put(c.bindings, f, concrete)
+          concrete = concreteType(c, a)
+          if concrete == nil:
+            return isNone
+        if doBind:
+          put(c.bindings, f, concrete)
     elif a.kind == tyEmpty:
       result = isGeneric
     elif x.kind == tyGenericParam:
@@ -884,20 +957,32 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   of tyTypeDesc:
     var prev = PType(idTableGet(c.bindings, f))
     if prev == nil:
-      if a.kind == tyTypeDesc:
+      # proc foo(T: typedesc, x: T)
-        if f.sons[0].kind == tyNone:
+      # when `f` is an unresolved typedesc, `a` could be any
-          result = isGeneric
+      # type, so we should not perform this check earlier
-        else:
+      if a.kind != tyTypeDesc: return isNone
-          result = typeRel(c, f.sons[0], a.sons[0])
+    
-        if result != isNone:
+      if f.base.kind == tyNone:
-          put(c.bindings, f, a)
+        result = isGeneric
+      else:
+        result = typeRel(c, f.base, a.base)
+      
+      if result != isNone:
+        put(c.bindings, f, a)
+    else:
+      if tfUnresolved in f.flags:
+        result = typeRel(c, prev.base, a)
+      elif a.kind == tyTypeDesc:
+        result = typeRel(c, prev.base, a.base)
       else:
         result = isNone
+ 
+  of tyIter:
+    if a.kind == tyIter or 
+      (a.kind == tyProc and tfIterator in a.flags):
+      result = typeRel(c, f.base, a.base)
     else:
-      internalAssert prev.sonsLen == 1
+      result = isNone
-      let toMatch = if tfUnresolved in f.flags: a
-                    else: a.sons[0]
-      result = typeRel(c, prev.sons[0], toMatch)
 
   of tyStmt:
     result = isGeneric
@@ -905,7 +990,24 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   of tyProxy:
     result = isEqual
 
-  else: internalError("typeRel: " & $f.kind)
+  of tyFromExpr:
+    # fix the expression, so it contains the already instantiated types
+    let instantiated = replaceTypesInBody(c.c, c.bindings, f.n)
+    let reevaluted = c.c.semExpr(c.c, instantiated)
+    case reevaluted.typ.kind
+    of tyTypeDesc:
+      result = typeRel(c, a, reevaluted.typ.base)
+    of tyStatic:
+      result = typeRel(c, a, reevaluted.typ.base)
+      if result != isNone and reevaluted.typ.n != nil:
+        if not exprStructuralEquivalent(aOrig.n, reevaluted.typ.n):
+          result = isNone
+    else:
+      localError(f.n.info, errTypeExpected)
+      result = isNone
+  
+  else:
+    internalAssert false
   
 proc cmpTypes*(c: PContext, f, a: PType): TTypeRelation = 
   var m: TCandidate
@@ -981,6 +1083,10 @@ proc localConvMatch(c: PContext, m: var TCandidate, f, a: PType,
       result.typ = getInstantiatedType(c, arg, m, base(f))
     m.baseTypeMatch = true
 
+proc isInlineIterator*(t: PType): bool =
+  result = t.kind == tyIter or
+          (t.kind == tyBuiltInTypeClass and t.base.kind == tyIter)
+
 proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
                         argSemantized, argOrig: PNode): PNode =
   var
@@ -993,19 +1099,62 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
     # XXX: When implicit statics are the default
     # this will be done earlier - we just have to
     # make sure that static types enter here
-    var evaluated = c.semTryConstExpr(c, arg)
+     
-    if evaluated != nil:
+    # XXX: weaken tyGenericParam and call it tyGenericPlaceholder
-      arg.typ = newTypeS(tyStatic, c)
+    # and finally start using tyTypedesc for generic types properly.
-      arg.typ.sons = @[evaluated.typ]
+    if argType.kind == tyGenericParam and tfWildcard in argType.flags:
-      arg.typ.n = evaluated
+      argType.assignType(f)
-      argType = arg.typ
+      # put(m.bindings, f, argType)
+      return argSemantized
+
+    if argType.kind == tyStatic:
+      if m.callee.kind == tyGenericBody:
+        result = newNodeI(nkType, argOrig.info)
+        result.typ = makeTypeFromExpr(c, arg)
+        return
+    else:
+      var evaluated = c.semTryConstExpr(c, arg)
+      if evaluated != nil:
+        arg.typ = newTypeS(tyStatic, c)
+        arg.typ.sons = @[evaluated.typ]
+        arg.typ.n = evaluated
+        argType = arg.typ
  
   var
-    a = if c.inTypeClass > 0: argType.skipTypes({tyTypeDesc})
+    a = if c.inTypeClass > 0: argType.skipTypes({tyTypeDesc, tyFieldAccessor})
         else: argType
  
     r = typeRel(m, f, a)
 
+  if r != isNone and m.calleeSym != nil and
+     m.calleeSym.kind in {skMacro, skTemplate}:
+    # XXX: duplicating this is ugly, maybe we should move this
+    # directly into typeRel using return-like templates
+    case r
+    of isConvertible, isIntConv: inc(m.convMatches)
+    of isSubtype, isSubrange: inc(m.subtypeMatches)
+    of isGeneric, isInferred: inc(m.genericMatches)
+    of isInferredConvertible: inc(m.genericMatches); inc(m.convMatches)
+    of isFromIntLit: inc(m.intConvMatches, 256)
+    of isEqual: inc(m.exactMatches)
+    of isNone: discard
+
+    if f.kind == tyStmt and argOrig.kind == nkDo:
+      return argOrig[bodyPos]
+    elif f.kind == tyTypeDesc:
+      return arg
+    elif f.kind == tyStatic:
+      return arg.typ.n
+    else:
+      return argOrig
+
+  if r != isNone and f.isInlineIterator:
+    var inlined = newTypeS(tyStatic, c)
+    inlined.sons = @[argType]
+    inlined.n = argSemantized
+    put(m.bindings, f, inlined)
+    return argSemantized
+
   case r
   of isConvertible:
     inc(m.convMatches)
@@ -1022,24 +1171,24 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
     inc(m.subtypeMatches)
     #result = copyTree(arg)
     result = implicitConv(nkHiddenStdConv, f, copyTree(arg), m, c)
+  of isInferred, isInferredConvertible:
+    inc(m.genericMatches)
+    if arg.kind in {nkProcDef, nkIteratorDef} + nkLambdaKinds:
+      result = c.semInferredLambda(c, m.bindings, arg)
+    else:
+      let inferred = c.semGenerateInstance(c, arg.sym, m.bindings, arg.info)
+      result = newSymNode(inferred, arg.info)
+    if r == isInferredConvertible:
+      inc(m.convMatches)
+      result = implicitConv(nkHiddenStdConv, f, result, m, c)
   of isGeneric:
     inc(m.genericMatches)
-    if m.calleeSym != nil and m.calleeSym.kind in {skMacro, skTemplate}:
+    result = copyTree(arg)
-      if f.kind == tyStmt and argOrig.kind == nkDo:
+    result.typ = getInstantiatedType(c, arg, m, f)
-        result = argOrig[bodyPos]
+    # BUG: f may not be the right key!
-      elif f.kind == tyTypeDesc:
+    if skipTypes(result.typ, abstractVar-{tyTypeDesc}).kind in {tyTuple}:
-        result = arg
+      result = implicitConv(nkHiddenStdConv, f, copyTree(arg), m, c)
-      elif f.kind == tyStatic:
+      # BUGFIX: use ``result.typ`` and not `f` here
-        result = arg.typ.n
-      else:
-        result = argOrig
-    else:
-      result = copyTree(arg)
-      result.typ = getInstantiatedType(c, arg, m, f) 
-      # BUG: f may not be the right key!
-      if skipTypes(result.typ, abstractVar-{tyTypeDesc}).kind in {tyTuple}:
-        result = implicitConv(nkHiddenStdConv, f, copyTree(arg), m, c) 
-        # BUGFIX: use ``result.typ`` and not `f` here
   of isFromIntLit:
     # too lazy to introduce another ``*matches`` field, so we conflate
     # ``isIntConv`` and ``isIntLit`` here:
@@ -1082,6 +1231,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     # incorrect to simply use the first fitting match. However, to implement
     # this correctly is inefficient. We have to copy `m` here to be able to
     # roll back the side effects of the unification algorithm.
+
     let c = m.c
     var x, y, z: TCandidate
     initCandidate(c, x, m.callee)
@@ -1092,7 +1242,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     z.calleeSym = m.calleeSym
     var best = -1
     for i in countup(0, sonsLen(arg) - 1): 
-      if arg.sons[i].sym.kind in {skProc, skIterator, skMethod, skConverter}: 
+      if arg.sons[i].sym.kind in {skProc, skMethod, skConverter}+skIterators:
         copyCandidate(z, m)
         var r = typeRel(z, f, arg.sons[i].typ)
         if r != isNone: 
@@ -1132,7 +1282,9 @@ proc prepareOperand(c: PContext; formal: PType; a: PNode): PNode =
     # a.typ == nil is valid
     result = a
   elif a.typ.isNil:
-    result = c.semOperand(c, a, {efDetermineType})
+    let flags = if formal.kind == tyIter: {efDetermineType, efWantIterator}
+                else: {efDetermineType}
+    result = c.semOperand(c, a, flags)
   else:
     result = a
 

compiler/syntaxes.nim

@@ -17,12 +17,13 @@ type
   TFilterKind* = enum 
     filtNone, filtTemplate, filtReplace, filtStrip
   TParserKind* = enum 
-    skinStandard, skinBraces, skinEndX
+    skinStandard, skinStrongSpaces, skinBraces, skinEndX
 
 const 
-  parserNames*: array[TParserKind, string] = ["standard", "braces", "endx"]
+  parserNames*: array[TParserKind, string] = ["standard", "strongspaces",
+                                              "braces", "endx"]
   filterNames*: array[TFilterKind, string] = ["none", "stdtmpl", "replace",
-    "strip"]
+                                              "strip"]
 
 type
   TParsers*{.final.} = object 
@@ -54,7 +55,7 @@ proc parseFile(fileIdx: int32): PNode =
 
 proc parseAll(p: var TParsers): PNode = 
   case p.skin
-  of skinStandard: 
+  of skinStandard, skinStrongSpaces:
     result = parser.parseAll(p.parser)
   of skinBraces: 
     result = pbraces.parseAll(p.parser)
@@ -65,7 +66,7 @@ proc parseAll(p: var TParsers): PNode =
   
 proc parseTopLevelStmt(p: var TParsers): PNode = 
   case p.skin
-  of skinStandard: 
+  of skinStandard, skinStrongSpaces:
     result = parser.parseTopLevelStmt(p.parser)
   of skinBraces: 
     result = pbraces.parseTopLevelStmt(p.parser)
@@ -170,7 +171,9 @@ proc openParsers(p: var TParsers, fileIdx: int32, inputstream: PLLStream) =
   else: s = inputstream
   case p.skin
   of skinStandard, skinBraces, skinEndX:
-    parser.openParser(p.parser, fileIdx, s)
+    parser.openParser(p.parser, fileIdx, s, false)
+  of skinStrongSpaces:
+    parser.openParser(p.parser, fileIdx, s, true)
   
 proc closeParsers(p: var TParsers) =
   parser.closeParser(p.parser)

compiler/transf.nim

@@ -20,10 +20,7 @@
 import 
   intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os, 
   idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread,
-  lambdalifting, sempass2
+  lambdalifting, sempass2, lowerings
-
-const 
-  genPrefix* = ":tmp"         # prefix for generated names
 
 # implementation
 
@@ -113,7 +110,7 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode =
   result[1] = ri
 
 proc transformSymAux(c: PTransf, n: PNode): PNode =
-  #if n.sym.kind == skIterator and n.sym.typ.callConv == ccClosure:
+  #if n.sym.kind == skClosureIterator:
   #  return liftIterSym(n)
   var b: PNode
   var tc = c.transCon
@@ -241,13 +238,6 @@ proc transformLoopBody(c: PTransf, n: PNode): PTransNode =
   else: 
     result = transform(c, n)
 
-proc newTupleAccess(tup: PNode, i: int): PNode = 
-  result = newNodeIT(nkBracketExpr, tup.info, tup.typ.sons[i])
-  addSon(result, copyTree(tup))
-  var lit = newNodeIT(nkIntLit, tup.info, getSysType(tyInt))
-  lit.intVal = i
-  addSon(result, lit)
-
 proc unpackTuple(c: PTransf, n: PNode, father: PTransNode) = 
   # XXX: BUG: what if `n` is an expression with side-effects?
   for i in countup(0, sonsLen(c.transCon.forStmt) - 3): 
@@ -433,9 +423,9 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
 
   var length = sonsLen(n)
   var call = n.sons[length - 2]
-  if call.kind notin nkCallKinds or call.sons[0].kind != nkSym or 
+  if call.typ.kind != tyIter and
-      call.sons[0].typ.callConv == ccClosure or
+    (call.kind notin nkCallKinds or call.sons[0].kind != nkSym or 
-      call.sons[0].sym.kind != skIterator:
+      call.sons[0].sym.kind != skIterator):
     n.sons[length-1] = transformLoopBody(c, n.sons[length-1]).PNode
     return lambdalifting.liftForLoop(n).PTransNode
     #InternalError(call.info, "transformFor")
@@ -454,12 +444,13 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
   var newC = newTransCon(getCurrOwner(c))
   newC.forStmt = n
   newC.forLoopBody = loopBody
-  if iter.kind != skIterator: internalError(call.info, "transformFor") 
+  internalAssert iter.kind == skIterator
   # generate access statements for the parameters (unless they are constant)
   pushTransCon(c, newC)
   for i in countup(1, sonsLen(call) - 1): 
     var arg = transform(c, call.sons[i]).PNode
     var formal = skipTypes(iter.typ, abstractInst).n.sons[i].sym 
+    if arg.typ.kind == tyIter: continue
     case putArgInto(arg, formal.typ)
     of paDirectMapping: 
       idNodeTablePut(newC.mapping, formal, arg)
@@ -481,7 +472,7 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
   dec(c.inlining)
   popInfoContext()
   popTransCon(c)
-  #echo "transformed: ", renderTree(n)
+  # echo "transformed: ", result.PNode.renderTree
   
 proc getMagicOp(call: PNode): TMagic = 
   if call.sons[0].kind == nkSym and
@@ -741,7 +732,7 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
     var c = openTransf(module, "")
     result = processTransf(c, n, prc)
     result = liftLambdas(prc, result)
-    #if prc.kind == skIterator and prc.typ.callConv == ccClosure:
+    #if prc.kind == skClosureIterator:
     #  result = lambdalifting.liftIterator(prc, result)
     incl(result.flags, nfTransf)
     when useEffectSystem: trackProc(prc, result)
@@ -754,6 +745,7 @@ proc transformStmt*(module: PSym, n: PNode): PNode =
     result = processTransf(c, n, module)
     result = liftLambdasForTopLevel(module, result)
     incl(result.flags, nfTransf)
+    when useEffectSystem: trackTopLevelStmt(module, result)
 
 proc transformExpr*(module: PSym, n: PNode): PNode =
   if nfTransf in n.flags:

compiler/types.nim

@@ -431,11 +431,12 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
       add(result, typeToString(t.sons[i]))
     add(result, ']')
   of tyTypeDesc:
-    if t.len == 0: result = "typedesc"
+    if t.base.kind == tyNone: result = "typedesc"
-    else: result = "typedesc[" & typeToString(t.sons[0]) & "]"
+    else: result = "typedesc[" & typeToString(t.base) & "]"
   of tyStatic:
     internalAssert t.len > 0
     result = "static[" & typeToString(t.sons[0]) & "]"
+    if t.n != nil: result.add "(" & renderTree(t.n) & ")"
   of tyUserTypeClass:
     internalAssert t.sym != nil and t.sym.owner != nil
     return t.sym.owner.name.s
@@ -452,7 +453,8 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
       of tyProc: "proc"
       of tyObject: "object"
       of tyTuple: "tuple"
-      else: (internalAssert(false); "")
+      of tyOpenArray: "openarray"
+      else: typeToStr[t.base.kind]
   of tyUserTypeClassInst:
     let body = t.base
     result = body.sym.name.s & "["
@@ -463,7 +465,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
   of tyAnd:
     result = typeToString(t.sons[0]) & " and " & typeToString(t.sons[1])
   of tyOr:
-    result = typeToString(t.sons[0]) & " and " & typeToString(t.sons[1])
+    result = typeToString(t.sons[0]) & " or " & typeToString(t.sons[1])
   of tyNot:
     result = "not " & typeToString(t.sons[0])
   of tyExpr:
@@ -828,6 +830,12 @@ proc sameChildrenAux(a, b: PType, c: var TSameTypeClosure): bool =
     result = sameTypeOrNilAux(a.sons[i], b.sons[i], c)
     if not result: return 
 
+proc isGenericAlias*(t: PType): bool =
+  return t.kind == tyGenericInst and t.lastSon.kind == tyGenericInst
+
+proc skipGenericAlias*(t: PType): PType =
+  return if t.isGenericAlias: t.lastSon else: t
+
 proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
   template cycleCheck() =
     # believe it or not, the direct check for ``containsOrIncl(c, a, b)``
@@ -858,6 +866,19 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     of dcEqOrDistinctOf:
       while a.kind == tyDistinct: a = a.sons[0]
       if a.kind != b.kind: return false
+  
+  if x.kind == tyGenericInst:
+    let
+      lhs = x.skipGenericAlias
+      rhs = y.skipGenericAlias
+    if rhs.kind != tyGenericInst or lhs.base != rhs.base:
+      return false
+    for i in 1 .. lhs.len - 2:
+      let ff = rhs.sons[i]
+      let aa = lhs.sons[i]
+      if not sameTypeAux(ff, aa, c): return false
+    return true
+
   case a.kind
   of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCString,
      tyInt..tyBigNum, tyStmt, tyExpr:
@@ -882,8 +903,6 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
   of tyTuple:
     cycleCheck()
     result = sameTuple(a, b, c) and sameFlags(a, b)
-  of tyGenericInst:    
-    result = sameTypeAux(lastSon(a), lastSon(b), c)
   of tyTypeDesc:
     if c.cmp == dcEqIgnoreDistinct: result = false
     elif ExactTypeDescValues in c.flags:
@@ -910,6 +929,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     result = sameTypeOrNilAux(a.sons[0], b.sons[0], c) and
         sameValue(a.n.sons[0], b.n.sons[0]) and
         sameValue(a.n.sons[1], b.n.sons[1])
+  of tyGenericInst: discard
   of tyNone: result = false  
 
 proc sameBackendType*(x, y: PType): bool =
@@ -1003,12 +1023,6 @@ proc matchType*(a: PType, pattern: openArray[tuple[k:TTypeKind, i:int]],
     a = a.sons[i]
   result = a.kind == last
 
-proc isGenericAlias*(t: PType): bool =
-  return t.kind == tyGenericInst and t.lastSon.kind == tyGenericInst
-
-proc skipGenericAlias*(t: PType): PType =
-  return if t.isGenericAlias: t.lastSon else: t
-
 proc typeAllowedAux(marker: var TIntSet, typ: PType, kind: TSymKind,
                     flags: TTypeAllowedFlags = {}): bool =
   assert(kind in {skVar, skLet, skConst, skParam, skResult})
@@ -1042,7 +1056,8 @@ proc typeAllowedAux(marker: var TIntSet, typ: PType, kind: TSymKind,
   of tyEmpty:
     result = taField in flags
   of tyTypeClasses:
-    result = true
+    result = tfGenericTypeParam in t.flags or
+             taField notin flags
   of tyGenericBody, tyGenericParam, tyGenericInvokation,
      tyNone, tyForward, tyFromExpr, tyFieldAccessor:
     result = false
@@ -1231,8 +1246,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt =
   of tyGenericInst, tyDistinct, tyGenericBody, tyMutable, tyConst, tyIter:
     result = computeSizeAux(lastSon(typ), a)
   of tyTypeDesc:
-    result = if typ.len == 1: computeSizeAux(typ.sons[0], a)
+    result = computeSizeAux(typ.base, a)
-             else: szUnknownSize
   of tyForward: return szIllegalRecursion
   else:
     #internalError("computeSizeAux()")
@@ -1246,7 +1260,7 @@ proc computeSize(typ: PType): BiggestInt =
 
 proc getReturnType*(s: PSym): PType =
   # Obtains the return type of a iterator/proc/macro/template
-  assert s.kind in {skProc, skTemplate, skMacro, skIterator}
+  assert s.kind in skProcKinds
   result = s.typ.sons[0]
 
 proc getSize(typ: PType): BiggestInt = 
@@ -1254,15 +1268,18 @@ proc getSize(typ: PType): BiggestInt =
   if result < 0: internalError("getSize: " & $typ.kind)
 
 proc containsGenericTypeIter(t: PType, closure: PObject): bool =
-  if t.kind in GenericTypes + tyTypeClasses + {tyFromExpr}:
+  if t.kind == tyStatic:
-    return true
+    return t.n == nil
 
   if t.kind == tyTypeDesc:
-    if t.sons[0].kind == tyNone: return true
+    if t.base.kind == tyNone: return true
     if containsGenericTypeIter(t.base, closure): return true
     return false
 
-  return t.kind == tyStatic and t.n == nil
+  if t.kind in GenericTypes + tyTypeClasses + {tyFromExpr}:
+    return true
+
+  return false
 
 proc containsGenericType*(t: PType): bool = 
   result = iterOverType(t, containsGenericTypeIter, nil)

compiler/vmdef.nim

@@ -8,7 +8,7 @@
 #
 
 ## This module contains the type definitions for the new evaluation engine.
-## An instruction is 1-2 int32s in memory, it is a register based VM.
+## An instruction is 1-3 int32s in memory, it is a register based VM.
 
 import ast, passes, msgs, intsets
 
@@ -16,6 +16,9 @@ const
   byteExcess* = 128 # we use excess-K for immediates
   wordExcess* = 32768
 
+  MaxLoopIterations* = 500_000 # max iterations of all loops
+
+
 type
   TRegister* = range[0..255]
   TDest* = range[-1 .. 255]
@@ -32,17 +35,17 @@ type
     opcAsgnFloat,
     opcAsgnRef,
     opcAsgnComplex,
+    opcRegToNode,
+    opcNodeToReg,
 
     opcLdArr,  # a = b[c]
-    opcLdArrRef,
     opcWrArr,  # a[b] = c
-    opcWrArrRef,
     opcLdObj,  # a = b.c
-    opcLdObjRef,
     opcWrObj,  # a.b = c
-    opcWrObjRef,
+    opcAddrReg,
-    opcAddr,
+    opcAddrNode,
-    opcDeref,
+    opcLdDeref,
+    opcWrDeref,
     opcWrStrIdx,
     opcLdStrIdx, # a = b[c]
     
@@ -64,6 +67,7 @@ type
     opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq,
     opcSwap, opcIsNil, opcOf, opcIs,
     opcSubStr, opcConv, opcCast, opcQuit, opcReset,
+    opcNarrowS, opcNarrowU,
     
     opcAddStrCh,
     opcAddStrStr,
@@ -109,6 +113,7 @@ type
     opcTJmp,  # jump Bx if A != 0
     opcFJmp,  # jump Bx if A == 0
     opcJmp,   # jump Bx
+    opcJmpBack, # jump Bx; resulting from a while loop
     opcBranch,  # branch for 'case'
     opcTry,
     opcExcept,
@@ -117,15 +122,14 @@ type
     opcNew,
     opcNewSeq,
     opcLdNull,    # dest = nullvalue(types[Bx])
+    opcLdNullReg,
     opcLdConst,   # dest = constants[Bx]
     opcAsgnConst, # dest = copy(constants[Bx])
     opcLdGlobal,  # dest = globals[Bx]
+    opcLdGlobalAddr, # dest = addr(globals[Bx])
+
     opcLdImmInt,  # dest = immediate value
     opcNBindSym,
-    opcWrGlobal,
-    opcWrGlobalRef,
-    opcGlobalAlias, # load an alias to a global into a register
-    opcGlobalOnce,  # used to introduce an assignment to a global once
     opcSetType,   # dest.typ = types[Bx]
     opcTypeTrait
 
@@ -159,14 +163,13 @@ type
     slotTempInt,      # some temporary int
     slotTempFloat,    # some temporary float
     slotTempStr,      # some temporary string
-    slotTempComplex   # some complex temporary (n.sons field is used)
+    slotTempComplex   # some complex temporary (s.node field is used)
 
   PProc* = ref object
     blocks*: seq[TBlock]    # blocks; temp data structure
+    sym*: PSym
     slots*: array[TRegister, tuple[inUse: bool, kind: TSlotKind]]
     maxSlots*: int
-    globals*: array[TRegister, int] # hack: to support passing globals byref
-                                    # we map a slot persistently to a global
     
   PCtx* = ref TCtx
   TCtx* = object of passes.TPassContext # code gen context
@@ -183,6 +186,8 @@ type
     callsite*: PNode
     mode*: TEvalMode
     features*: TSandboxFlags
+    traceActive*: bool
+    loopIterations*: int
 
   TPosition* = distinct int
 
@@ -191,7 +196,7 @@ type
 proc newCtx*(module: PSym): PCtx =
   PCtx(code: @[], debug: @[],
     globals: newNode(nkStmtListExpr), constants: newNode(nkStmtList), types: @[],
-    prc: PProc(blocks: @[]), module: module)
+    prc: PProc(blocks: @[]), module: module, loopIterations: MaxLoopIterations)
 
 proc refresh*(c: PCtx, module: PSym) =
   c.module = module

compiler/vmgen.nim

@@ -11,7 +11,7 @@
 
 import
   unsigned, strutils, ast, astalgo, types, msgs, renderer, vmdef, 
-  trees, intsets, rodread, magicsys, options
+  trees, intsets, rodread, magicsys, options, lowerings
 
 from os import splitFile
 
@@ -41,7 +41,14 @@ proc codeListing(c: PCtx, result: var string, start=0) =
     let x = c.code[i]
 
     let opc = opcode(x)
-    if opc < firstABxInstr:
+    if opc in {opcConv, opcCast}:
+      let y = c.code[i+1]
+      let z = c.code[i+2]
+      result.addf("\t$#\tr$#, r$#, $#, $#", ($opc).substr(3), x.regA, x.regB,
+        c.types[y.regBx-wordExcess].typeToString, 
+        c.types[z.regBx-wordExcess].typeToString)
+      inc i, 2
+    elif opc < firstABxInstr:
       result.addf("\t$#\tr$#, r$#, r$#", ($opc).substr(3), x.regA, 
                   x.regB, x.regC)
     elif opc in relativeJumps:
@@ -92,10 +99,10 @@ proc genLabel(c: PCtx): TPosition =
   result = TPosition(c.code.len)
   #c.jumpTargets.incl(c.code.len)
 
-proc jmpBack(c: PCtx, n: PNode, opc: TOpcode, p = TPosition(0)) =
+proc jmpBack(c: PCtx, n: PNode, p = TPosition(0)) =
   let dist = p.int - c.code.len
   internalAssert(-0x7fff < dist and dist < 0x7fff)
-  gABx(c, n, opc, 0, dist)
+  gABx(c, n, opcJmpBack, 0, dist)
 
 proc patch(c: PCtx, p: TPosition) =
   # patch with current index
@@ -139,25 +146,12 @@ proc getTemp(c: PCtx; typ: PType): TRegister =
       if not c.slots[i].inUse:
         c.slots[i] = (inUse: true, kind: k)
         return TRegister(i)
+  if c.maxSlots >= high(TRegister):
+    internalError("cannot generate code; too many registers required")
   result = TRegister(c.maxSlots)
   c.slots[c.maxSlots] = (inUse: true, kind: k)
   inc c.maxSlots
 
-proc getGlobalSlot(c: PCtx; n: PNode; s: PSym): TRegister =
-  let p = c.prc
-  for i in 0 .. p.maxSlots-1:
-    if p.globals[i] == s.id: return TRegister(i)
-
-  result = TRegister(p.maxSlots)
-  p.slots[p.maxSlots] = (inUse: true, kind: slotFixedVar)
-  p.globals[p.maxSlots] = s.id
-  inc p.maxSlots
-  # XXX this is still not correct! We need to load the global in a proc init
-  # section, otherwise control flow could lead to a usage before it's been
-  # loaded.
-  c.gABx(n, opcGlobalAlias, result, s.position)
-  # XXX add some internal asserts here
-
 proc freeTemp(c: PCtx; r: TRegister) =
   let c = c.prc
   if c.slots[r].kind >= slotSomeTemp: c.slots[r].inUse = false
@@ -242,20 +236,20 @@ proc genWhile(c: PCtx; n: PNode) =
   withBlock(nil):
     if isTrue(n.sons[0]):
       c.gen(n.sons[1])
-      c.jmpBack(n, opcJmp, L1)
+      c.jmpBack(n, L1)
     elif isNotOpr(n.sons[0]):
       var tmp = c.genx(n.sons[0].sons[1])
       let L2 = c.xjmp(n, opcTJmp, tmp)
       c.freeTemp(tmp)
       c.gen(n.sons[1])
-      c.jmpBack(n, opcJmp, L1)
+      c.jmpBack(n, L1)
       c.patch(L2)
     else:
       var tmp = c.genx(n.sons[0])
       let L2 = c.xjmp(n, opcFJmp, tmp)
       c.freeTemp(tmp)
       c.gen(n.sons[1])
-      c.jmpBack(n, opcJmp, L1)
+      c.jmpBack(n, L1)
       c.patch(L2)
 
 proc genBlock(c: PCtx; n: PNode; dest: var TDest) =
@@ -321,9 +315,20 @@ proc genAndOr(c: PCtx; n: PNode; opc: TOpcode; dest: var TDest) =
   c.gen(n.sons[2], dest)
   c.patch(L1)
 
+proc canonValue*(n: PNode): PNode =
+  if n.kind == nkExprColonExpr:
+    result = n.sons[1]
+  elif n.hasSubnodeWith(nkExprColonExpr):
+    result = n.copyNode
+    newSeq(result.sons, n.len)
+    for i in 0.. <n.len:
+      result.sons[i] = canonValue(n.sons[i])
+  else:
+    result = n
+
 proc rawGenLiteral(c: PCtx; n: PNode): int =
   result = c.constants.len
-  c.constants.add n
+  c.constants.add n.canonValue
   internalAssert result < 0x7fff
 
 proc sameConstant*(a, b: PNode): bool =
@@ -461,21 +466,45 @@ proc genCall(c: PCtx; n: PNode; dest: var TDest) =
     c.gABC(n, opcIndCallAsgn, dest, x, n.len)
   c.freeTempRange(x, n.len)
 
+template isGlobal(s: PSym): bool = sfGlobal in s.flags and s.kind != skForVar
+
 proc needsAsgnPatch(n: PNode): bool = 
-  n.kind in {nkBracketExpr, nkDotExpr, nkCheckedFieldExpr}
+  n.kind in {nkBracketExpr, nkDotExpr, nkCheckedFieldExpr,
+             nkDerefExpr, nkHiddenDeref} or (n.kind == nkSym and n.sym.isGlobal)
+
+proc genField(n: PNode): TRegister =
+  if n.kind != nkSym or n.sym.kind != skField:
+    internalError(n.info, "no field symbol")
+  let s = n.sym
+  if s.position > high(result):
+      internalError(n.info,
+        "too large offset! cannot generate code for: " & s.name.s)
+  result = s.position
 
 proc genAsgnPatch(c: PCtx; le: PNode, value: TRegister) =
   case le.kind
   of nkBracketExpr:
-    let dest = c.genx(le.sons[0])
+    let dest = c.genx(le.sons[0], {gfAddrOf})
     let idx = c.genx(le.sons[1])
-    c.gABC(le, opcWrArrRef, dest, idx, value)
+    c.gABC(le, opcWrArr, dest, idx, value)
+    c.freeTemp(dest)
+    c.freeTemp(idx)
   of nkDotExpr, nkCheckedFieldExpr:
     # XXX field checks here
     let left = if le.kind == nkDotExpr: le else: le.sons[0]
-    let dest = c.genx(left.sons[0])
+    let dest = c.genx(left.sons[0], {gfAddrOf})
-    let idx = c.genx(left.sons[1])
+    let idx = genField(left.sons[1])
-    c.gABC(left, opcWrObjRef, dest, idx, value)
+    c.gABC(left, opcWrObj, dest, idx, value)
+    c.freeTemp(dest)
+  of nkDerefExpr, nkHiddenDeref:
+    let dest = c.genx(le.sons[0], {gfAddrOf})
+    c.gABC(le, opcWrDeref, dest, value)
+    c.freeTemp(dest)
+  of nkSym:
+    if le.sym.isGlobal:
+      let dest = c.genx(le, {gfAddrOf})
+      c.gABC(le, opcWrDeref, dest, value)
+      c.freeTemp(dest)
   else:
     discard
 
@@ -521,6 +550,30 @@ proc genBinaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
   c.freeTemp(tmp)
   c.freeTemp(tmp2)
 
+proc genNarrow(c: PCtx; n: PNode; dest: TDest) =
+  let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
+  # uint is uint64 in the VM, we we only need to mask the result for
+  # other unsigned types:
+  if t.kind in {tyUInt8..tyUInt32}:
+    c.gABC(n, opcNarrowU, dest, TRegister(t.size*8))
+  elif t.kind in {tyInt8..tyInt32}:
+    c.gABC(n, opcNarrowS, dest, TRegister(t.size*8))
+
+proc genNarrowU(c: PCtx; n: PNode; dest: TDest) =
+  let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
+  # uint is uint64 in the VM, we we only need to mask the result for
+  # other unsigned types:
+  if t.kind in {tyUInt8..tyUInt32, tyInt8..tyInt32}:
+    c.gABC(n, opcNarrowU, dest, TRegister(t.size*8))
+
+proc genBinaryABCnarrow(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
+  genBinaryABC(c, n, dest, opc)
+  genNarrow(c, n, dest)
+
+proc genBinaryABCnarrowU(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
+  genBinaryABC(c, n, dest, opc)
+  genNarrowU(c, n, dest)
+
 proc genSetType(c: PCtx; n: PNode; dest: TRegister) =
   let t = skipTypes(n.typ, abstractInst-{tyTypeDesc})
   if t.kind == tySet:
@@ -582,13 +635,14 @@ proc genAddSubInt(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
     c.freeTemp(tmp)
   else:
     genBinaryABC(c, n, dest, opc)
+  c.genNarrow(n, dest)
 
 proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) =  
   let tmp = c.genx(arg)
-  c.gABx(n, opcSetType, tmp, genType(c, arg.typ))
   if dest < 0: dest = c.getTemp(n.typ)
   c.gABC(n, opc, dest, tmp)
   c.gABx(n, opc, 0, genType(c, n.typ))
+  c.gABx(n, opc, 0, genType(c, arg.typ))
   c.freeTemp(tmp)
 
 proc genCard(c: PCtx; n: PNode; dest: var TDest) =
@@ -614,10 +668,17 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     c.genAddSubInt(n, dest, opcAddInt)
   of mInc, mDec:
     unused(n, dest)
-    var d = c.genx(n.sons[1]).TDest
+    let opc = if m == mInc: opcAddInt else: opcSubInt
-    c.genAddSubInt(n, d, if m == mInc: opcAddInt else: opcSubInt)
+    let d = c.genx(n.sons[1])
+    if n.sons[2].isInt8Lit:
+      c.gABI(n, succ(opc), d, d, n.sons[2].intVal)
+    else:
+      let tmp = c.genx(n.sons[2])
+      c.gABC(n, opc, d, d, tmp)
+      c.freeTemp(tmp)
+    c.genNarrow(n.sons[1], d)
     c.genAsgnPatch(n.sons[1], d)
-    c.freeTemp(d.TRegister)
+    c.freeTemp(d)
   of mOrd, mChr, mArrToSeq: c.gen(n.sons[1], dest)
   of mNew, mNewFinalize:
     unused(n, dest)
@@ -627,6 +688,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     c.genNewSeq(n)
   of mNewString:
     genUnaryABC(c, n, dest, opcNewStr)
+    # XXX buggy
   of mNewStringOfCap:
     # we ignore the 'cap' argument and translate it as 'newString(0)'.
     # eval n.sons[1] for possible side effects:
@@ -635,6 +697,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     if dest < 0: dest = c.getTemp(n.typ)
     c.gABC(n, opcNewStr, dest, tmp)
     c.freeTemp(tmp)
+    # XXX buggy
   of mLengthOpenArray, mLengthArray, mLengthSeq:
     genUnaryABI(c, n, dest, opcLenSeq)
   of mLengthStr:
@@ -648,23 +711,23 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     c.freeTemp(d)
     c.freeTemp(tmp)
   of mCard: genCard(c, n, dest)
-  of mMulI, mMulI64: genBinaryABC(c, n, dest, opcMulInt)
+  of mMulI, mMulI64: genBinaryABCnarrow(c, n, dest, opcMulInt)
-  of mDivI, mDivI64: genBinaryABC(c, n, dest, opcDivInt)
+  of mDivI, mDivI64: genBinaryABCnarrow(c, n, dest, opcDivInt)
-  of mModI, mModI64: genBinaryABC(c, n, dest, opcModInt)
+  of mModI, mModI64: genBinaryABCnarrow(c, n, dest, opcModInt)
   of mAddF64: genBinaryABC(c, n, dest, opcAddFloat)
   of mSubF64: genBinaryABC(c, n, dest, opcSubFloat)
   of mMulF64: genBinaryABC(c, n, dest, opcMulFloat)
   of mDivF64: genBinaryABC(c, n, dest, opcDivFloat)
-  of mShrI, mShrI64: genBinaryABC(c, n, dest, opcShrInt)
+  of mShrI, mShrI64: genBinaryABCnarrowU(c, n, dest, opcShrInt)
-  of mShlI, mShlI64: genBinaryABC(c, n, dest, opcShlInt)
+  of mShlI, mShlI64: genBinaryABCnarrowU(c, n, dest, opcShlInt)
-  of mBitandI, mBitandI64: genBinaryABC(c, n, dest, opcBitandInt)
+  of mBitandI, mBitandI64: genBinaryABCnarrowU(c, n, dest, opcBitandInt)
-  of mBitorI, mBitorI64: genBinaryABC(c, n, dest, opcBitorInt)
+  of mBitorI, mBitorI64: genBinaryABCnarrowU(c, n, dest, opcBitorInt)
-  of mBitxorI, mBitxorI64: genBinaryABC(c, n, dest, opcBitxorInt)
+  of mBitxorI, mBitxorI64: genBinaryABCnarrowU(c, n, dest, opcBitxorInt)
-  of mAddU: genBinaryABC(c, n, dest, opcAddu)
+  of mAddU: genBinaryABCnarrowU(c, n, dest, opcAddu)
-  of mSubU: genBinaryABC(c, n, dest, opcSubu)
+  of mSubU: genBinaryABCnarrowU(c, n, dest, opcSubu)
-  of mMulU: genBinaryABC(c, n, dest, opcMulu)
+  of mMulU: genBinaryABCnarrowU(c, n, dest, opcMulu)
-  of mDivU: genBinaryABC(c, n, dest, opcDivu)
+  of mDivU: genBinaryABCnarrowU(c, n, dest, opcDivu)
-  of mModU: genBinaryABC(c, n, dest, opcModu)
+  of mModU: genBinaryABCnarrowU(c, n, dest, opcModu)
   of mEqI, mEqI64, mEqB, mEqEnum, mEqCh:
     genBinaryABC(c, n, dest, opcEqInt)
   of mLeI, mLeI64, mLeEnum, mLeCh, mLeB:
@@ -678,12 +741,16 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
   of mLtPtr, mLtU, mLtU64: genBinaryABC(c, n, dest, opcLtu)
   of mEqProc, mEqRef, mEqUntracedRef, mEqCString:
     genBinaryABC(c, n, dest, opcEqRef)
-  of mXor: genBinaryABC(c, n, dest, opcXor)
+  of mXor: genBinaryABCnarrowU(c, n, dest, opcXor)
   of mNot: genUnaryABC(c, n, dest, opcNot)
-  of mUnaryMinusI, mUnaryMinusI64: genUnaryABC(c, n, dest, opcUnaryMinusInt)
+  of mUnaryMinusI, mUnaryMinusI64:
+    genUnaryABC(c, n, dest, opcUnaryMinusInt)
+    genNarrow(c, n, dest)
   of mUnaryMinusF64: genUnaryABC(c, n, dest, opcUnaryMinusFloat)
   of mUnaryPlusI, mUnaryPlusI64, mUnaryPlusF64: gen(c, n.sons[1], dest)
-  of mBitnotI, mBitnotI64: genUnaryABC(c, n, dest, opcBitnotInt)
+  of mBitnotI, mBitnotI64: 
+    genUnaryABC(c, n, dest, opcBitnotInt)
+    genNarrowU(c, n, dest)
   of mZe8ToI, mZe8ToI64, mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64,
      mToU8, mToU16, mToU32, mToFloat, mToBiggestFloat, mToInt, 
      mToBiggestInt, mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, 
@@ -896,6 +963,10 @@ const
     tyFloat, tyFloat32, tyFloat64, tyFloat128,
     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64}
 
+proc fitsRegister*(t: PType): bool =
+  t.skipTypes(abstractInst-{tyTypeDesc}).kind in {
+    tyRange, tyEnum, tyBool, tyInt..tyUInt64}
+
 proc requiresCopy(n: PNode): bool =
   if n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind in atomicTypes:
     result = false
@@ -907,22 +978,33 @@ proc requiresCopy(n: PNode): bool =
 proc unneededIndirection(n: PNode): bool =
   n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind == tyRef
 
-proc skipDeref(n: PNode): PNode =
-  if n.kind in {nkDerefExpr, nkHiddenDeref} and unneededIndirection(n.sons[0]):
-    result = n.sons[0]
-  else:
-    result = n
-
 proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
                   flags: TGenFlags) = 
   # a nop for certain types
-  let flags = if opc == opcAddr: flags+{gfAddrOf} else: flags
+  let isAddr = opc in {opcAddrNode, opcAddrReg}
-  if unneededIndirection(n.sons[0]):
+  let newflags = if isAddr: flags+{gfAddrOf} else: flags
-    gen(c, n.sons[0], dest, flags)
+  # consider:
+  # proc foo(f: var ref int) =
+  #   f = new(int)
+  # proc blah() =
+  #   var x: ref int
+  #   foo x
+  #
+  # The type of 'f' is 'var ref int' and of 'x' is 'ref int'. Hence for
+  # nkAddr we must not use 'unneededIndirection', but for deref we use it.
+  if not isAddr and unneededIndirection(n.sons[0]):
+    gen(c, n.sons[0], dest, newflags)
   else:
-    let tmp = c.genx(n.sons[0], flags)
+    let tmp = c.genx(n.sons[0], newflags)
     if dest < 0: dest = c.getTemp(n.typ)
-    gABC(c, n, opc, dest, tmp)
+    if not isAddr:
+      gABC(c, n, opc, dest, tmp)
+      if gfAddrOf notin flags and fitsRegister(n.typ):
+        c.gABC(n, opcNodeToReg, dest, dest)
+    elif c.prc.slots[tmp].kind >= slotTempUnknown:
+      gABC(c, n, opcAddrNode, dest, tmp)
+    else:
+      gABC(c, n, opcAddrReg, dest, tmp)
     c.freeTemp(tmp)
 
 proc whichAsgnOpc(n: PNode): TOpcode =
@@ -940,8 +1022,7 @@ proc whichAsgnOpc(n: PNode): TOpcode =
 
 proc isRef(t: PType): bool = t.skipTypes(abstractRange-{tyTypeDesc}).kind == tyRef
 
-proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode =
+proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode = opc
-  if isRef(n.typ): succ(opc) else: opc
 
 proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) =
   let tmp = c.genx(ri)
@@ -949,8 +1030,6 @@ proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) =
   gABC(c, ri, whichAsgnOpc(ri), dest, tmp)
   c.freeTemp(tmp)
 
-template isGlobal(s: PSym): bool = sfGlobal in s.flags and s.kind != skForVar
-
 proc setSlot(c: PCtx; v: PSym) =
   # XXX generate type initialization here?
   if v.position == 0:
@@ -959,40 +1038,71 @@ proc setSlot(c: PCtx; v: PSym) =
         kind: if v.kind == skLet: slotFixedLet else: slotFixedVar)
     inc c.prc.maxSlots
 
+proc cannotEval(n: PNode) {.noinline.} =
+  globalError(n.info, errGenerated, "cannot evaluate at compile time: " &
+    n.renderTree)
+
+proc isOwnedBy(a, b: PSym): bool =
+  var a = a.owner
+  while a != nil and a.kind != skModule:
+    if a == b: return true
+    a = a.owner
+
+proc getOwner(c: PCtx): PSym =
+  result = c.prc.sym
+  if result.isNil: result = c.module
+
+proc checkCanEval(c: PCtx; n: PNode) =
+  # we need to ensure that we don't evaluate 'x' here:
+  # proc foo() = var x ...
+  let s = n.sym
+  if s.position == 0:
+    if s.kind in {skVar, skTemp, skLet, skParam, skResult} and 
+        not s.isOwnedBy(c.prc.sym) and s.owner != c.module:
+      cannotEval(n)
+
 proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
   case le.kind
   of nkBracketExpr:
-    let dest = c.genx(le.sons[0])
+    let dest = c.genx(le.sons[0], {gfAddrOf})
     let idx = c.genx(le.sons[1])
     let tmp = c.genx(ri)
     if le.sons[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind in {
         tyString, tyCString}:
       c.gABC(le, opcWrStrIdx, dest, idx, tmp)
     else:
-      c.gABC(le, whichAsgnOpc(le, opcWrArr), dest, idx, tmp)
+      c.gABC(le, opcWrArr, dest, idx, tmp)
     c.freeTemp(tmp)
   of nkDotExpr, nkCheckedFieldExpr:
     # XXX field checks here
     let left = if le.kind == nkDotExpr: le else: le.sons[0]
-    let dest = c.genx(left.sons[0])
+    let dest = c.genx(left.sons[0], {gfAddrOf})
-    let idx = c.genx(left.sons[1])
+    let idx = genField(left.sons[1])
     let tmp = c.genx(ri)
-    c.gABC(left, whichAsgnOpc(left, opcWrObj), dest, idx, tmp)
+    c.gABC(left, opcWrObj, dest, idx, tmp)
+    c.freeTemp(tmp)
+  of nkDerefExpr, nkHiddenDeref:
+    let dest = c.genx(le.sons[0], {gfAddrOf})
+    let tmp = c.genx(ri)
+    c.gABC(le, opcWrDeref, dest, tmp)
     c.freeTemp(tmp)
   of nkSym:
     let s = le.sym
+    checkCanEval(c, le)
     if s.isGlobal:
       withTemp(tmp, le.typ):
-        gen(c, ri, tmp)
+        c.gen(le, tmp, {gfAddrOf})
-        c.gABx(le, whichAsgnOpc(le, opcWrGlobal), tmp, s.position)
+        let val = c.genx(ri)
+        c.gABC(le, opcWrDeref, tmp, val)
+        c.freeTemp(val)
     else:
-      if s.kind == skForVar and c.mode == emRepl: c.setSlot s
+      if s.kind == skForVar: c.setSlot s
       internalAssert s.position > 0 or (s.position == 0 and
                                         s.kind in {skParam,skResult})
       var dest: TRegister = s.position + ord(s.kind == skParam)
       gen(c, ri, dest)
   else:
-    let dest = c.genx(le)
+    let dest = c.genx(le, {gfAddrOf})
     genAsgn(c, dest, ri, requiresCopy)
 
 proc genLit(c: PCtx; n: PNode; dest: var TDest) =
@@ -1018,22 +1128,22 @@ proc importcSym(c: PCtx; info: TLineInfo; s: PSym) =
     localError(info, errGenerated,
                "cannot 'importc' variable at compile time")
 
-proc cannotEval(n: PNode) {.noinline.} =
+proc getNullValue*(typ: PType, info: TLineInfo): PNode
-  globalError(n.info, errGenerated, "cannot evaluate at compile time: " &
-    n.renderTree)
 
 proc genGlobalInit(c: PCtx; n: PNode; s: PSym) =
-  c.globals.add(emptyNode.copyNode)
+  c.globals.add(getNullValue(s.typ, n.info))
   s.position = c.globals.len
   # This is rather hard to support, due to the laziness of the VM code
   # generator. See tests/compile/tmacro2 for why this is necesary:
   #   var decls{.compileTime.}: seq[PNimrodNode] = @[]
-  c.gABx(n, opcGlobalOnce, 0, s.position)
+  let dest = c.getTemp(s.typ)
+  c.gABx(n, opcLdGlobal, dest, s.position)
   let tmp = c.genx(s.ast)
-  c.gABx(n, whichAsgnOpc(n, opcWrGlobal), tmp, s.position)
+  c.gABC(n, opcWrDeref, dest, tmp)
+  c.freeTemp(dest)
   c.freeTemp(tmp)
 
-proc genRdVar(c: PCtx; n: PNode; dest: var TDest) =
+proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
   let s = n.sym
   if s.isGlobal:
     if sfCompileTime in s.flags or c.mode == emRepl:
@@ -1043,17 +1153,21 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest) =
     if s.position == 0:
       if sfImportc in s.flags: c.importcSym(n.info, s)
       else: genGlobalInit(c, n, s)
-    if dest < 0:
+    if dest < 0: dest = c.getTemp(n.typ)
-      dest = c.getGlobalSlot(n, s)
+    if gfAddrOf notin flags and fitsRegister(s.typ):
-      #c.gABx(n, opcAliasGlobal, dest, s.position)
+      var cc = c.getTemp(n.typ)
+      c.gABx(n, opcLdGlobal, cc, s.position)
+      c.gABC(n, opcNodeToReg, dest, cc)
+      c.freeTemp(cc)
     else:
       c.gABx(n, opcLdGlobal, dest, s.position)
   else:
-    if s.kind == skForVar and c.mode == emRepl: c.setSlot s
+    if s.kind == skForVar and c.mode == emRepl: c.setSlot(s)
     if s.position > 0 or (s.position == 0 and
                           s.kind in {skParam,skResult}):
       if dest < 0:
         dest = s.position + ord(s.kind == skParam)
+        internalAssert(c.prc.slots[dest].kind < slotSomeTemp)
       else:
         # we need to generate an assignment:
         genAsgn(c, dest, n, c.prc.slots[dest].kind >= slotSomeTemp)
@@ -1061,30 +1175,45 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest) =
       # see tests/t99bott for an example that triggers it:
       cannotEval(n)
 
-proc genAccess(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
+proc genArrAccess2(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
-               flags: TGenFlags) =
+                   flags: TGenFlags) =
   let a = c.genx(n.sons[0], flags)
   let b = c.genx(n.sons[1], {})
   if dest < 0: dest = c.getTemp(n.typ)
-  c.gABC(n, (if gfAddrOf in flags: succ(opc) else: opc), dest, a, b)
+  if gfAddrOf notin flags and fitsRegister(n.typ):
+    var cc = c.getTemp(n.typ)
+    c.gABC(n, opc, cc, a, b)
+    c.gABC(n, opcNodeToReg, dest, cc)
+    c.freeTemp(cc)
+  else:
+    c.gABC(n, opc, dest, a, b)
   c.freeTemp(a)
   c.freeTemp(b)
 
 proc genObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
-  genAccess(c, n, dest, opcLdObj, flags)
+  let a = c.genx(n.sons[0], flags)
+  let b = genField(n.sons[1])
+  if dest < 0: dest = c.getTemp(n.typ)
+  if gfAddrOf notin flags and fitsRegister(n.typ.skipTypes({tyVar})):
+    var cc = c.getTemp(n.typ)
+    c.gABC(n, opcLdObj, cc, a, b)
+    c.gABC(n, opcNodeToReg, dest, cc)
+    c.freeTemp(cc)
+  else:
+    c.gABC(n, opcLdObj, dest, a, b)
+  c.freeTemp(a)
 
 proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
   # XXX implement field checks!
-  genAccess(c, n.sons[0], dest, opcLdObj, flags)
+  genObjAccess(c, n.sons[0], dest, flags)
 
 proc genArrAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
   if n.sons[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind in {
       tyString, tyCString}:
-    genAccess(c, n, dest, opcLdStrIdx, {})
+    genArrAccess2(c, n, dest, opcLdStrIdx, {})
   else:
-    genAccess(c, n, dest, opcLdArr, flags)
+    genArrAccess2(c, n, dest, opcLdArr, flags)
 
-proc getNullValue*(typ: PType, info: TLineInfo): PNode
 proc getNullValueAux(obj: PNode, result: PNode) = 
   case obj.kind
   of nkRecList:
@@ -1137,50 +1266,51 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
     result = newNodeIT(nkCurly, info, t)
   else: internalError("getNullValue: " & $t.kind)
 
+proc ldNullOpcode(t: PType): TOpcode =
+  if fitsRegister(t): opcLdNullReg else: opcLdNull
+
 proc genVarSection(c: PCtx; n: PNode) =
   for a in n:
     if a.kind == nkCommentStmt: continue
     #assert(a.sons[0].kind == nkSym) can happen for transformed vars
     if a.kind == nkVarTuple:
-      let tmp = c.genx(a.lastSon)
       for i in 0 .. a.len-3:
         setSlot(c, a[i].sym)
-        # v = t[i]
+        checkCanEval(c, a[i])
-        var v: TDest = -1
+      c.gen(lowerTupleUnpacking(a, c.getOwner))
-        genRdVar(c, a[i], v)
-        c.gABC(n, opcLdObj, v, tmp, i)
-        # XXX globals?
-      c.freeTemp(tmp)
     elif a.sons[0].kind == nkSym:
       let s = a.sons[0].sym
+      checkCanEval(c, a.sons[0])
       if s.isGlobal:
         if s.position == 0:
           if sfImportc in s.flags: c.importcSym(a.info, s)
           else:
-            let sa = if s.ast.isNil: getNullValue(s.typ, a.info) else: s.ast
+            let sa = if s.ast.isNil: getNullValue(s.typ, a.info) 
+                     else: canonValue(s.ast)
             c.globals.add(sa)
             s.position = c.globals.len
-            # "Once support" is unnecessary here
         if a.sons[2].kind == nkEmpty:
           when false:
             withTemp(tmp, s.typ):
               c.gABx(a, opcLdNull, tmp, c.genType(s.typ))
               c.gABx(a, whichAsgnOpc(a.sons[0], opcWrGlobal), tmp, s.position)
         else:
-          let tmp = genx(c, a.sons[2])
+          let tmp = c.genx(a.sons[0], {gfAddrOf})
-          c.gABx(a, whichAsgnOpc(a.sons[0], opcWrGlobal), tmp, s.position)
+          let val = c.genx(a.sons[2])
+          c.gABC(a, opcWrDeref, tmp, val)
+          c.freeTemp(val)
           c.freeTemp(tmp)
       else:
         setSlot(c, s)
         if a.sons[2].kind == nkEmpty:
-          c.gABx(a, opcLdNull, s.position, c.genType(s.typ))
+          c.gABx(a, ldNullOpcode(s.typ), s.position, c.genType(s.typ))
         else:
           gen(c, a.sons[2], s.position.TRegister)
     else:
       # assign to a.sons[0]; happens for closures
       if a.sons[2].kind == nkEmpty:
         let tmp = genx(c, a.sons[0])
-        c.gABx(a, opcLdNull, tmp, c.genType(a.sons[0].typ))
+        c.gABx(a, ldNullOpcode(a[0].typ), tmp, c.genType(a.sons[0].typ))
         c.freeTemp(tmp)
       else:
         genAsgn(c, a.sons[0], a.sons[2], true)
@@ -1188,10 +1318,19 @@ proc genVarSection(c: PCtx; n: PNode) =
 proc genArrayConstr(c: PCtx, n: PNode, dest: var TDest) =
   if dest < 0: dest = c.getTemp(n.typ)
   c.gABx(n, opcLdNull, dest, c.genType(n.typ))
+
+  let intType = getSysType(tyInt)
+  let seqType = n.typ.skipTypes(abstractVar-{tyTypeDesc})
+  if seqType.kind == tySequence:
+    var tmp = c.getTemp(intType)
+    c.gABx(n, opcLdImmInt, tmp, n.len)
+    c.gABx(n, opcNewSeq, dest, c.genType(seqType))
+    c.gABx(n, opcNewSeq, tmp, 0)
+    c.freeTemp(tmp)
+  
   if n.len > 0:
-    let intType = getSysType(tyInt)
     var tmp = getTemp(c, intType)
-    c.gABx(n, opcLdNull, tmp, c.genType(intType))
+    c.gABx(n, opcLdNullReg, tmp, c.genType(intType))
     for x in n:
       let a = c.genx(x)
       c.gABC(n, whichAsgnOpc(x, opcWrArr), dest, tmp, a)
@@ -1224,11 +1363,10 @@ proc genObjConstr(c: PCtx, n: PNode, dest: var TDest) =
   for i in 1.. <n.len:
     let it = n.sons[i]
     if it.kind == nkExprColonExpr and it.sons[0].kind == nkSym:
-      let idx = c.genx(it.sons[0])
+      let idx = genField(it.sons[0])
       let tmp = c.genx(it.sons[1])
       c.gABC(it, whichAsgnOpc(it.sons[1], opcWrObj), dest, idx, tmp)
       c.freeTemp(tmp)
-      c.freeTemp(idx)
     else:
       internalError(n.info, "invalid object constructor")
 
@@ -1239,11 +1377,10 @@ proc genTupleConstr(c: PCtx, n: PNode, dest: var TDest) =
   for i in 0.. <n.len:
     let it = n.sons[i]
     if it.kind == nkExprColonExpr:
-      let idx = c.genx(it.sons[0])
+      let idx = genField(it.sons[0])
       let tmp = c.genx(it.sons[1])
       c.gABC(it, whichAsgnOpc(it.sons[1], opcWrObj), dest, idx, tmp)
       c.freeTemp(tmp)
-      c.freeTemp(idx)
     else:
       let tmp = c.genx(it)
       c.gABC(it, whichAsgnOpc(it, opcWrObj), dest, i.TRegister, tmp)
@@ -1255,10 +1392,11 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
   case n.kind
   of nkSym:
     let s = n.sym
+    checkCanEval(c, n)
     case s.kind
     of skVar, skForVar, skTemp, skLet, skParam, skResult:
-      genRdVar(c, n, dest)
+      genRdVar(c, n, dest, flags)
-    of skProc, skConverter, skMacro, skTemplate, skMethod, skIterator:
+    of skProc, skConverter, skMacro, skTemplate, skMethod, skIterators:
       # 'skTemplate' is only allowed for 'getAst' support:
       if sfImportc in s.flags: c.importcSym(n.info, s)
       genLit(c, n, dest)
@@ -1271,12 +1409,6 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
       else:
         var lit = genLiteral(c, newIntNode(nkIntLit, s.position))
         c.gABx(n, opcLdConst, dest, lit)
-    of skField:
-      internalAssert dest < 0
-      if s.position > high(dest):
-        internalError(n.info, 
-          "too large offset! cannot generate code for: " & s.name.s)
-      dest = s.position
     of skType:
       genTypeLit(c, s.typ, dest)
     else:
@@ -1303,8 +1435,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
   of nkDotExpr: genObjAccess(c, n, dest, flags)
   of nkCheckedFieldExpr: genCheckedObjAccess(c, n, dest, flags)
   of nkBracketExpr: genArrAccess(c, n, dest, flags)
-  of nkDerefExpr, nkHiddenDeref: genAddrDeref(c, n, dest, opcDeref, flags)
+  of nkDerefExpr, nkHiddenDeref: genAddrDeref(c, n, dest, opcLdDeref, flags)
-  of nkAddr, nkHiddenAddr: genAddrDeref(c, n, dest, opcAddr, flags)
+  of nkAddr, nkHiddenAddr: genAddrDeref(c, n, dest, opcAddrNode, flags)
   of nkWhenStmt, nkIfStmt, nkIfExpr: genIf(c, n, dest)
   of nkCaseStmt: genCase(c, n, dest)
   of nkWhileStmt:
@@ -1423,7 +1555,7 @@ proc optimizeJumps(c: PCtx; start: int) =
       var d = i + c.code[i].jmpDiff
       for iters in countdown(maxIterations, 0):
         case c.code[d].opcode
-        of opcJmp:
+        of opcJmp, opcJmpBack:
           d = d + c.code[d].jmpDiff
         of opcTJmp, opcFJmp:
           if c.code[d].regA != reg: break
@@ -1441,7 +1573,7 @@ proc optimizeJumps(c: PCtx; start: int) =
         else: break
       if d != i + c.code[i].jmpDiff:
         c.finalJumpTarget(i, d - i)
-    of opcJmp:
+    of opcJmp, opcJmpBack:
       var d = i + c.code[i].jmpDiff
       var iters = maxIterations
       while c.code[d].opcode == opcJmp and iters > 0:
@@ -1472,7 +1604,7 @@ proc genProc(c: PCtx; s: PSym): int =
     # procs easily:
     let body = s.getBody
     let procStart = c.xjmp(body, opcJmp, 0)
-    var p = PProc(blocks: @[])
+    var p = PProc(blocks: @[], sym: s)
     let oldPrc = c.prc
     c.prc = p
     # iterate over the parameters and allocate space for them:
@@ -1489,9 +1621,9 @@ proc genProc(c: PCtx; s: PSym): int =
     c.gABC(body, opcEof, eofInstr.regA)
     c.optimizeJumps(result)
     s.offset = c.prc.maxSlots
-    #if s.name.s == "concatStyleInterpolation":
+    #if s.name.s == "addStuff":
+    #  echo renderTree(body)
     #  c.echoCode(result)
-    # echo renderTree(body)
     c.prc = oldPrc
   else:
     c.prc.maxSlots = s.offset

compiler/wordrecg.nim

@@ -62,8 +62,8 @@ type
     wWatchPoint, wSubsChar, 
     wAcyclic, wShallow, wUnroll, wLinearScanEnd, wComputedGoto, wInjectStmt,
     wWrite, wGensym, wInject, wDirty, wInheritable, wThreadVar, wEmit, 
-    wNoStackFrame,
+    wAsmNoStackFrame,
-    wImplicitStatic, wGlobal, wCodegenDecl,
+    wImplicitStatic, wGlobal, wCodegenDecl, wUnchecked,
 
     wAuto, wBool, wCatch, wChar, wClass,
     wConst_cast, wDefault, wDelete, wDouble, wDynamic_cast,
@@ -72,7 +72,7 @@ type
     wPrivate, wProtected, wPublic, wRegister, wReinterpret_cast,
     wShort, wSigned, wSizeof, wStatic_cast, wStruct, wSwitch,
     wThis, wThrow, wTrue, wTypedef, wTypeid, wTypename,
-    wUnion, wUnsigned, wVirtual, wVoid, wVolatile, wWchar_t,
+    wUnion, wPacked, wUnsigned, wVirtual, wVoid, wVolatile, wWchar_t,
 
     wAlignas, wAlignof, wConstexpr, wDecltype, wNullptr, wNoexcept,
     wThread_local, wStatic_assert, wChar16_t, wChar32_t,
@@ -145,7 +145,7 @@ const
     "subschar", "acyclic", "shallow", "unroll", "linearscanend",
     "computedgoto", "injectstmt",
     "write", "gensym", "inject", "dirty", "inheritable", "threadvar", "emit",
-    "nostackframe", "implicitstatic", "global", "codegendecl",
+    "asmnostackframe", "implicitstatic", "global", "codegendecl", "unchecked",
     
     "auto", "bool", "catch", "char", "class",
     "const_cast", "default", "delete", "double",
@@ -155,7 +155,7 @@ const
     "private", "protected", "public", "register", "reinterpret_cast",
     "short", "signed", "sizeof", "static_cast", "struct", "switch",
     "this", "throw", "true", "typedef", "typeid",
-    "typename", "union", "unsigned", "virtual", "void", "volatile",
+    "typename", "union", "packed", "unsigned", "virtual", "void", "volatile",
     "wchar_t",
 
     "alignas", "alignof", "constexpr", "decltype", "nullptr", "noexcept",

doc/c2nim.txt

@@ -74,7 +74,7 @@ Is translated into:
       printf("%s\x0A", x)
   else:
     template OUT*(x: expr): stmt = 
-      nil
+      discard
   
 As can been seen from the example, C's macros with parameters are mapped
 to Nimrod's templates. This mapping is the best one can do, but it is of course

doc/idetools.txt

@@ -279,8 +279,8 @@ skForVar
         col 7: ""
 
 
-skIterator
+skIterator, skClosureIterator
-----------
+-----------------------------
 
 The fourth column will be the empty string if the iterator is being
 defined, since at that point in the file the parser hasn't processed

doc/lib.txt

@@ -386,9 +386,6 @@ Database support
   A higher level SQLite database wrapper. The same interface is implemented
   for other databases too.
 
-* `db_mongo <db_mongo.html>`_
-  A higher level **mongodb** wrapper. 
-
 
 Other
 -----
@@ -503,8 +500,6 @@ Database support
   Contains a wrapper for the mySQL API.
 * `sqlite3 <sqlite3.html>`_
   Contains a wrapper for SQLite 3 API.
-* `mongodb <mongo.html>`_
-  Lower level wrapper for the **mongodb** client C library.
 * `odbcsql <odbcsql.html>`_
   interface to the ODBC driver.
 * `sphinx <sphinx.html>`_

doc/manual.txt

@@ -79,8 +79,21 @@ program execution. Unless explicitly classified, an error is a static error.
 
 A `checked runtime error`:idx: is an error that the implementation detects
 and reports at runtime. The method for reporting such errors is via *raising
-exceptions*. However, the implementation provides a means to disable these
+exceptions* or *dying with a fatal error*. However, the implementation 
-runtime checks. See the section pragmas_ for details.
+provides a means to disable these runtime checks. See the section pragmas_
+for details. 
+
+Wether a checked runtime error results in an exception or in a fatal error at
+runtime is implementation specific. Thus the following program is always
+invalid:
+
+.. code-block:: nimrod
+  var a: array[0..1, char]
+  let i = 5
+  try:
+    a[i] = 'N'
+  except EInvalidIndex:
+    echo "invalid index"
 
 An `unchecked runtime error`:idx: is an error that is not guaranteed to be
 detected, and can cause the subsequent behavior of the computation to
@@ -480,24 +493,27 @@ precedence and associativity; this is useful for meta programming.
 Associativity
 -------------
 
-All binary operators are left-associative, except binary operators whose
+Binary operators whose relevant character is ``^`` are right-associative, all
-relevant char is ``^``.
+other binary operators are left-associative.
 
 Precedence
 ----------
 
-For operators that are not keywords the precedence is determined by the
+Unary operators always bind stronger than any binary 
+operator: ``$a + b`` is ``($a) + b`` and not ``$(a + b)``.
+
+If an unary operator's relevant character is ``@`` it is a `sigil-like`:idx: 
+operator which binds stronger than a ``primarySuffix``: ``@x.abc`` is parsed
+as ``(@x).abc`` whereas ``$x.abc`` is parsed as ``$(x.abc)``.
+
+
+For binary operators that are not keywords the precedence is determined by the
 following rules:
 
 If the operator ends with ``=`` and its relevant character is none of 
 ``<``, ``>``, ``!``, ``=``, ``~``, ``?``, it is an *assignment operator* which
 has the lowest precedence.
 
-If the operator's relevant character is ``@`` it is a `sigil-like`:idx: 
-operator which binds stronger than a ``primarySuffix``: ``@x.abc`` is parsed
-as ``(@x).abc`` whereas ``$x.abc`` is parsed as ``$(x.abc)``.
-
-
 Otherwise precedence is determined by the relevant character.
 
 ================  ===============================================  ==================  ===============
@@ -508,7 +524,7 @@ Precedence level    Operators                                      Relevant char
   7               ``+    -``                                       ``+  ~  |``         OP7
   6               ``&``                                            ``&``               OP6
   5               ``..``                                           ``.``               OP5
-  4               ``==  <= < >= > !=  in not_in is isnot not of``  ``= <  > !``        OP4
+  4               ``==  <= < >= > !=  in notin is isnot not of``   ``= <  > !``        OP4
   3               ``and``                                                              OP3
   2               ``or xor``                                                           OP2
   1                                                                ``@  : ?``          OP1
@@ -516,6 +532,51 @@ Precedence level    Operators                                      Relevant char
 ================  ===============================================  ==================  ===============
 
 
+Strong spaces
+-------------
+
+The number of spaces preceeding a non-keyword operator affects precedence
+if the experimental parser directive ``#!strongSpaces`` is used. Indentation
+is not used to determine the number of spaces. If 2 or more operators have the
+same number of preceding spaces the precedence table applies, so ``1 + 3 * 4``
+is still parsed as ``1 + (3 * 4)``, but ``1+3 * 4`` is parsed as ``(1+3) * 4``:
+
+.. code-block:: nimrod
+  #! strongSpaces
+  if foo+4 * 4 == 8 and b&c | 9  ++
+      bar:
+    echo ""
+  # is parsed as
+  if ((foo+4)*4 == 8) and (((b&c) | 9) ++ bar): echo ""
+
+
+Furthermore whether an operator is used a prefix operator is affected by the
+number of spaces: 
+
+.. code-block:: nimrod
+  #! strongSpaces
+  echo $foo
+  # is parsed as
+  echo($foo)
+
+This also affects whether ``[]``, ``{}``, ``()`` are parsed as constructors
+or as accessors:
+
+.. code-block:: nimrod
+  #! strongSpaces
+  echo (1,2)
+  # is parsed as
+  echo((1,2))
+
+Only 0, 1, 2, 4 or 8 spaces are allowed to specify precedence and it is
+enforced that infix operators have the same amount of spaces before and after
+them. This rules does not apply when a newline follows after the operator,
+then only the preceding spaces are considered.
+
+
+Grammar
+-------
+
 The grammar's start symbol is ``module``.
 
 .. include:: grammar.txt
@@ -1508,6 +1569,28 @@ currency. This can be solved with templates_.
   defineCurrency(TEuro, int)
 
 
+The borrow pragma can also be used to annotate the distinct type to allow
+certain builtin operations to be lifted:
+
+.. code-block:: nimrod
+  type
+    Foo = object
+      a, b: int
+      s: string
+
+    Bar {.borrow: `.`.} = distinct Foo
+
+  var bb: ref Bar
+  new bb
+  # field access now valid
+  bb.a = 90
+  bb.s = "abc"
+
+Currently only the dot accessor can be borrowed in this way.
+
+
+
+
 Void type
 ---------
 
@@ -2822,7 +2905,6 @@ as there are components in the tuple. The i'th iteration variable's type is
 the type of the i'th component. In other words, implicit tuple unpacking in a 
 for loop context is supported.
 
-
 Implict items/pairs invocations
 -------------------------------
 
@@ -2847,10 +2929,11 @@ First class iterators
 There are 2 kinds of iterators in Nimrod: *inline* and *closure* iterators.
 An `inline iterator`:idx: is an iterator that's always inlined by the compiler 
 leading to zero overhead for the abstraction, but may result in a heavy
-increase in code size. Inline iterators are second class
+increase in code size. Inline iterators are second class citizens;
-citizens; one cannot pass them around like first class procs.
+They can be passed as parameters only to other inlining code facilities like
+templates, macros and other inline iterators.
 
-In contrast to that, a `closure iterator`:idx: can be passed around:
+In contrast to that, a `closure iterator`:idx: can be passed around more freely:
 
 .. code-block:: nimrod
   iterator count0(): int {.closure.} =
@@ -2873,9 +2956,7 @@ Closure iterators have other restrictions than inline iterators:
 1. ``yield`` in a closure iterator can not occur in a ``try`` statement.
 2. For now, a closure iterator cannot be evaluated at compile time.
 3. ``return`` is allowed in a closure iterator (but rarely useful).
-4. Since closure iterators can be used as a collaborative tasking
+4. Both inline and closure iterators cannot be recursive.
-   system, ``void`` is a valid return type for them.
-5. Both inline and closure iterators cannot be recursive.
 
 Iterators that are neither marked ``{.closure.}`` nor ``{.inline.}`` explicitly
 default to being inline, but that this may change in future versions of the
@@ -2937,6 +3018,14 @@ parameters of an outer factory proc:
   for f in foo():
     echo f
 
+Implicit return type
+--------------------
+
+Since inline interators must always produce values that will be consumed in
+a for loop, the compiler will implicity use the ``auto`` return type if no
+type is given by the user. In contrast, since closure iterators can be used
+as a collaborative tasking system, ``void`` is a valid return type for them.
+
 
 Type sections
 =============
@@ -4016,8 +4105,8 @@ Static params can also appear in the signatures of generic types:
     AffineTransform2D[T] = Matrix[3, 3, T]
     AffineTransform3D[T] = Matrix[4, 4, T]
 
-  AffineTransform3D[float]  # OK
+  var m1: AffineTransform3D[float]  # OK
-  AffineTransform2D[string] # Error, `string` is not a `Number`
+  var m2: AffineTransform2D[string] # Error, `string` is not a `Number`
 
 
 typedesc
@@ -4106,6 +4195,59 @@ types that will match the typedesc param:
 
 The constraint can be a concrete type or a type class.
 
+Special Operators
+=================
+
+dot operators
+-------------
+
+Nimrod offers a special family of dot operators that can be used to
+intercept and rewrite proc call and field access attempts, referring
+to previously undeclared symbol names. They can be used to provide a
+fluent interface to objects lying outside the static confines of the
+Nimrod's type system such as values from dynamic scripting languages
+or dynamic file formats such as JSON or XML.
+
+When Nimrod encounters an expression that cannot be resolved by the
+standard overload resolution rules, the current scope will be searched
+for a dot operator that can be matched against a re-written form of
+the expression, where the unknown field or proc name is converted to
+an additional static string parameter:
+
+.. code-block:: nimrod
+  a.b # becomes `.`(a, "b")
+  a.b(c, d) # becomes `.`(a, "b", c, d)
+
+The matched dot operators can be symbols of any callable kind (procs,
+templates and macros), depending on the desired effect:
+
+.. code-block:: nimrod
+  proc `.` (js: PJsonNode, field: string): JSON = js[field]
+
+  var js = parseJson("{ x: 1, y: 2}")
+  echo js.x # outputs 1
+  echo js.y # outputs 2
+
+The following dot operators are available:
+
+operator `.`
+------------ 
+This operator will be matched against both field accesses and method calls.
+
+operator `.()`
+---------------
+This operator will be matched exclusively against method calls. It has higher
+precedence than the `.` operator and this allows you to handle expressions like
+`x.y` and `x.y()` differently if you are interfacing with a scripting language
+for example.
+
+operator `.=`
+-------------
+This operator will be matched against assignments to missing fields.
+
+.. code-block:: nimrod
+  a.b = c # becomes `.=`(a, "b", c)
+
 
 Term rewriting macros
 =====================
@@ -4758,42 +4900,6 @@ This may change in future versions of language, but for now use
 the ``finalizer`` parameter to ``new``.
 
 
-delegator pragma
-----------------
-
-**Note**: The design of the delegator feature is subject to change.
-
-The delegator pragma can be used to intercept and rewrite proc call and field
-access attempts referring to previously undeclared symbol names. It can be used
-to provide a fluent interface to objects lying outside the static confines of
-the Nimrod's type system such as values from dynamic scripting languages or
-dynamic file formats such as JSON or XML.
-
-A delegator is a special form of the `()` operator marked with the delagator
-pragma. When Nimrod encounters an expression that cannot be resolved by the
-standard overload resolution, any delegators in the current scope will be
-matched against a rewritten form of the expression following the standard
-signature matching rules. In the rewritten expression, the name of the unknown
-proc or field name is inserted as an additional static string parameter always
-appearing in the leading position:
-
-.. code-block:: nimrod
-  a.b => delegator("b", a)
-  a.b(c, d) => delegator("b", a, c)
-  a b, c, d => delegator("a", b, c, d)
-
-
-The delegators can be any callable symbol type (procs, templates, macros)
-depending on the desired effect:
-
-.. code-block:: nimrod
-  proc `()` (field: string, js: PJsonNode): JSON {.delegator.} = js[field]
-
-  var js = parseJson("{ x: 1, y: 2}")
-  echo js.x # outputs 1
-  echo js.y # outputs 2
-
-
 procvar pragma
 --------------
 The `procvar`:idx: pragma is used to mark a proc that it can be passed to a
@@ -4876,16 +4982,16 @@ field which is used for runtime type identification is omitted. This is
 necessary for binary compatibility with other compiled languages.
 
 
-NoStackFrame pragma
+AsmNoStackFrame pragma
--------------------
+----------------------
-A proc can be marked with the `noStackFrame`:idx: pragma to tell the compiler
+A proc can be marked with the `AsmNoStackFrame`:idx: pragma to tell the compiler
 it should not generate a stack frame for the proc. There are also no exit
 statements like ``return result;`` generated and the generated C function is
 declared as ``__declspec(naked)`` or ``__attribute__((naked))`` (depending on
 the used C compiler).
 
-**Note**: This pragma should only be used by procs which consist solely of assembler
+**Note**: This pragma should only be used by procs which consist solely of
-statements.
+assembler statements.
 
 error pragma
 ------------
@@ -5303,6 +5409,63 @@ strings automatically:
   printf("hallo %s", "world") # "world" will be passed as C string
 
 
+Union pragma
+------------
+The `union`:idx: pragma can be applied to any ``object`` type. It means all
+of the object's fields are overlaid in memory. This produces a ``union``
+instead of a ``struct`` in the generated C/C++ code. The object declaration
+then must not use inheritance or any GC'ed memory but this is currently not
+checked.
+
+**Future directions**: GC'ed memory should be allowed in unions and the GC
+should scan unions conservatively.
+
+Packed pragma
+-------------
+The `packed`:idx: pragma can be applied to any ``object`` type. It ensures 
+that the fields of an object are packed back-to-back in memory. It is useful 
+to store packets or messages from/to network or hardware drivers, and for 
+interoperability with C. Combining packed pragma with inheritance is not 
+defined, and it should not be used with GC'ed memory (ref's).  
+
+**Future directions**: Using GC'ed memory in packed pragma will result in 
+compile-time error. Usage with inheritance should be defined and documented.
+
+Unchecked pragma
+----------------
+The `unchecked`:idx: pragma can be used to mark a named array as ``unchecked``
+meaning its bounds are not checked. This is often useful when one wishes to
+implement his own flexibly sized arrays. Additionally an unchecked array is
+translated into a C array of undetermined size:
+
+.. code-block:: nimrod
+  type
+    ArrayPart{.unchecked.} = array[0..0, int]
+    MySeq = object
+      len, cap: int
+      data: ArrayPart
+
+Produces roughly this C code:
+
+.. code-block:: C
+  typedef struct {
+    NI len;
+    NI cap;
+    NI data[];
+  } MySeq;
+
+The bounds checking done at compile time is not disabled for now, so to access
+``s.data[C]`` (where ``C`` is a constant) the array's index needs needs to
+include ``C``.
+
+The base type of the unchecked array may not contain any GC'ed memory but this
+is currently not checked.
+
+**Future directions**: GC'ed memory should be allowed in unchecked arrays and
+there should be an explicit annotation of how the GC is to determine the
+runtime size of the array.
+
+
 Dynlib pragma for import
 ------------------------
 With the `dynlib`:idx: pragma a procedure or a variable can be imported from

examples/htmlrefs.nim

@@ -36,7 +36,7 @@ block mainLoop:
               case x.kind
               of xmlEof: break mainLoop
               of xmlElementClose: break
-              else: nil
+              else: discard
             x.next() # skip ``xmlElementClose``
             # now we have the description for the ``a`` element
             var desc = ""

examples/htmltitle.nim

@@ -7,7 +7,7 @@ import os, streams, parsexml, strutils
 if paramCount() < 1:
   quit("Usage: htmltitle filename[.html]")
 
-var filename = addFileExt(ParamStr(1), "html")
+var filename = addFileExt(paramStr(1), "html")
 var s = newFileStream(filename, fmRead)
 if s == nil: quit("cannot open the file " & filename)
 var x: TXmlParser
@@ -23,13 +23,13 @@ while true:
         title.add(x.charData)
         x.next()
       if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0:
-        Echo("Title: " & title)
+        echo("Title: " & title)
         quit(0) # Success!
       else:
         echo(x.errorMsgExpected("/title"))
   
   of xmlEof: break # end of file reached
-  else: nil # ignore other events
+  else: discard # ignore other events
 
 x.close()
 quit("Could not determine title!")

koch.nim

@@ -58,6 +58,15 @@ Boot options:
 
 proc exe(f: string): string = return addFileExt(f, ExeExt)
 
+proc findNim(): string =
+  var nimrod = "nimrod".exe
+  result = "bin" / nimrod
+  if existsFile(result): return
+  for dir in split(getEnv("PATH"), PathSep):
+    if existsFile(dir / nimrod): return dir / nimrod
+  # assume there is a symlink to the exe or something:
+  return nimrod
+
 proc exec(cmd: string) =
   echo(cmd)
   if execShellCmd(cmd) != 0: quit("FAILURE")
@@ -70,15 +79,15 @@ const
   compileNimInst = "-d:useLibzipSrc tools/niminst/niminst"
 
 proc csource(args: string) = 
-  exec("nimrod cc $1 -r $3 --var:version=$2 csource compiler/nimrod.ini $1" %
+  exec("$4 cc $1 -r $3 --var:version=$2 csource compiler/nimrod.ini $1" %
-       [args, NimrodVersion, compileNimInst])
+       [args, NimrodVersion, compileNimInst, findNim()])
 
 proc zip(args: string) = 
-  exec("nimrod cc -r $2 --var:version=$1 zip compiler/nimrod.ini" %
+  exec("$3 cc -r $2 --var:version=$1 zip compiler/nimrod.ini" %
-       [NimrodVersion, compileNimInst])
+       [NimrodVersion, compileNimInst, findNim()])
   
 proc buildTool(toolname, args: string) = 
-  exec("nimrod cc $# $#" % [args, toolname])
+  exec("$# cc $# $#" % [findNim(), args, toolname])
   copyFile(dest="bin"/ splitFile(toolname).name.exe, source=toolname.exe)
 
 proc inno(args: string) =
@@ -90,13 +99,13 @@ proc inno(args: string) =
        NimrodVersion)
 
 proc install(args: string) = 
-  exec("nimrod cc -r $# --var:version=$# scripts compiler/nimrod.ini" %
+  exec("$# cc -r $# --var:version=$# scripts compiler/nimrod.ini" %
-       [compileNimInst, NimrodVersion])
+       [findNim(), compileNimInst, NimrodVersion])
   exec("sh ./install.sh $#" % args)
 
 proc web(args: string) =
-  exec(("nimrod cc -r tools/nimweb.nim web/nimrod --putenv:nimrodversion=$#" &
+  exec("$# cc -r tools/nimweb.nim web/nimrod --putenv:nimrodversion=$#" % 
-        " --path:$#") % [NimrodVersion, getCurrentDir()])
+       [findNim(), NimrodVersion])
 
 # -------------- boot ---------------------------------------------------------
 

lib/core/macros.nim

@@ -48,17 +48,17 @@ type
     nnkYieldStmt, nnkTryStmt, nnkFinally, nnkRaiseStmt,
     nnkReturnStmt, nnkBreakStmt, nnkContinueStmt, nnkBlockStmt, nnkStaticStmt,
     nnkDiscardStmt, nnkStmtList, 
-    
     nnkImportStmt,
     nnkImportExceptStmt,
     nnkExportStmt,
     nnkExportExceptStmt,
     nnkFromStmt,
     nnkIncludeStmt,
-    
     nnkBindStmt, nnkMixinStmt, nnkUsingStmt,
     nnkCommentStmt, nnkStmtListExpr, nnkBlockExpr,
-    nnkStmtListType, nnkBlockType, nnkTypeOfExpr, nnkObjectTy,
+    nnkStmtListType, nnkBlockType,
+    nnkWith, nnkWithout,
+    nnkTypeOfExpr, nnkObjectTy,
     nnkTupleTy, nnkTypeClassTy, nnkStaticTy,
     nnkRecList, nnkRecCase, nnkRecWhen,
     nnkRefTy, nnkPtrTy, nnkVarTy,
@@ -88,7 +88,7 @@ type
     nskUnknown, nskConditional, nskDynLib, nskParam,
     nskGenericParam, nskTemp, nskModule, nskType, nskVar, nskLet, 
     nskConst, nskResult,
-    nskProc, nskMethod, nskIterator,
+    nskProc, nskMethod, nskIterator, nskClosureIterator,
     nskConverter, nskMacro, nskTemplate, nskField,
     nskEnumField, nskForVar, nskLabel,
     nskStub
@@ -516,7 +516,7 @@ proc last*(node: PNimrodNode): PNimrodNode {.compileTime.} = node[node.high]
 
 
 const
-  RoutineNodes* = {nnkProcDef, nnkMethodDef, nnkDo, nnkLambda}
+  RoutineNodes* = {nnkProcDef, nnkMethodDef, nnkDo, nnkLambda, nnkIteratorDef}
   AtomicNodes* = {nnkNone..nnkNilLit}
   CallNodes* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, 
     nnkCallStrLit, nnkHiddenCallConv}

lib/impure/db_mongo.nim

@@ -1,227 +0,0 @@
-#
-#
-#            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements a higher level wrapper for `mongodb`:idx:. Example:
-##
-## .. code-block:: nimrod
-##
-##    import mongo, db_mongo, oids, json
-##
-##    var conn = db_mongo.open()
-##
-##    # construct JSON data:
-##    var data = %{"a": %13, "b": %"my string value", 
-##                 "inner": %{"i": %71} }
-##
-##    var id = insertID(conn, "test.test", data)
-##
-##    for v in find(conn, "test.test", "this.a == 13"):
-##      print v
-##
-##    delete(conn, "test.test", id)
-##    close(conn)
-
-import mongo, oids, json
-
-type
-  EDb* = object of EIO ## exception that is raised if a database error occurs
-  TDbConn* = TMongo    ## a database connection; alias for ``TMongo``
-
-  FDb* = object of FIO ## effect that denotes a database operation
-  FReadDb* = object of FDB   ## effect that denotes a read operation
-  FWriteDb* = object of FDB  ## effect that denotes a write operation
-
-proc dbError*(db: TDbConn, msg: string) {.noreturn.} = 
-  ## raises an EDb exception with message `msg`.
-  var e: ref EDb
-  new(e)
-  if db.errstr[0] != '\0':
-    e.msg = $db.errstr
-  else:
-    e.msg = $db.err & " " & msg
-  raise e
-
-proc close*(db: var TDbConn) {.tags: [FDB].} = 
-  ## closes the database connection.
-  disconnect(db)
-  destroy(db)
-
-proc open*(host: string = defaultHost, port: int = defaultPort): TDbConn {.
-  tags: [FDB].} =
-  ## opens a database connection. Raises `EDb` if the connection could not
-  ## be established.
-  init(result)
-  
-  let x = client(result, host, port.cint)
-  if x != 0'i32:
-    dbError(result, "cannot open: " & host)
-
-proc jsonToBSon(b: var TBSon, key: string, j: PJsonNode) =
-  case j.kind
-  of JString:
-    add(b, key, j.str)
-  of JInt:
-    add(b, key, j.num)
-  of JFloat:
-    add(b, key, j.fnum)
-  of JBool:
-    addBool(b, key, ord(j.bval))
-  of JNull:
-    addNull(b, key)
-  of JObject:
-    addStartObject(b, key)
-    for k, v in items(j.fields):
-      jsonToBSon(b, k, v)
-    addFinishObject(b)
-  of JArray:
-    addStartArray(b, key)
-    for i, e in pairs(j.elems):
-      jsonToBSon(b, $i, e)
-    addFinishArray(b)
-
-proc jsonToBSon*(j: PJsonNode, oid: TOid): TBSon =
-  ## converts a JSON value into the BSON format. The result must be
-  ## ``destroyed`` explicitely!
-  init(result)
-  assert j.kind == JObject
-  add(result, "_id", oid)
-  for key, val in items(j.fields):
-    jsonToBSon(result, key, val)
-  finish(result)
-
-proc `[]`*(obj: var TBSon, fieldname: cstring): TBSon =
-  ## retrieves the value belonging to `fieldname`. Raises `EInvalidKey` if
-  ## the attribute does not exist.
-  var it = initIter(obj)
-  let res = find(it, result, fieldname)
-  if res == bkEOO:
-    raise newException(EInvalidIndex, "key not in object")
-
-proc getId*(obj: var TBSon): TOid =
-  ## retrieves the ``_id`` attribute of `obj`.
-  var it = initIter(obj)
-  var b: TBSon
-  let res = find(it, b, "_id")
-  if res == bkOID:
-    result = oidVal(it)[]
-  else:
-    raise newException(EInvalidIndex, "_id not in object")
-
-proc insertId*(db: var TDbConn, namespace: string, data: PJsonNode): TOid {.
-  tags: [FWriteDb].} =
-  ## converts `data` to BSON format and inserts it in `namespace`. Returns
-  ## the generated OID for the ``_id`` field.
-  result = genOid()
-  var x = jsonToBSon(data, result)
-  insert(db, namespace, x, nil)
-  destroy(x)
-
-proc insert*(db: var TDbConn, namespace: string, data: PJsonNode) {.
-  tags: [FWriteDb].} =
-  ## converts `data` to BSON format and inserts it in `namespace`.  
-  discard InsertID(db, namespace, data)
-
-proc update*(db: var TDbConn, namespace: string, obj: var TBSon) {.
-  tags: [FReadDB, FWriteDb].} =
-  ## updates `obj` in `namespace`.
-  var cond: TBson
-  init(cond)
-  cond.add("_id", getId(obj))
-  finish(cond)
-  update(db, namespace, cond, obj, ord(UPDATE_UPSERT))
-  destroy(cond)
-
-proc update*(db: var TDbConn, namespace: string, oid: TOid, obj: PJsonNode) {.
-  tags: [FReadDB, FWriteDb].} =
-  ## updates the data with `oid` to have the new data `obj`.
-  var a = jsonToBSon(obj, oid)
-  Update(db, namespace, a)
-  destroy(a)
-
-proc delete*(db: var TDbConn, namespace: string, oid: TOid) {.
-  tags: [FWriteDb].} =
-  ## Deletes the object belonging to `oid`.
-  var cond: TBson
-  init(cond)
-  cond.add("_id", oid)
-  finish(cond)
-  discard remove(db, namespace, cond)
-  destroy(cond)
-
-proc delete*(db: var TDbConn, namespace: string, obj: var TBSon) {.
-  tags: [FWriteDb].} =
-  ## Deletes the object `obj`.
-  delete(db, namespace, getId(obj))
-
-iterator find*(db: var TDbConn, namespace: string): var TBSon {.
-  tags: [FReadDB].} =
-  ## iterates over any object in `namespace`.
-  var cursor: TCursor
-  init(cursor, db, namespace)
-  while next(cursor) == mongo.OK:
-    yield bson(cursor)[]
-  destroy(cursor)
-
-iterator find*(db: var TDbConn, namespace: string, 
-               query, fields: var TBSon): var TBSon {.tags: [FReadDB].} =
-  ## yields the `fields` of any document that suffices `query`.
-  var cursor = find(db, namespace, query, fields, 0'i32, 0'i32, 0'i32)
-  if cursor != nil:
-    while next(cursor[]) == mongo.OK:
-      yield bson(cursor[])[]
-    destroy(cursor[])
-
-proc setupFieldnames(fields: varargs[string]): TBSon =
-  init(result)
-  for x in fields: add(result, x, 1'i32)
-  finish(result)
-
-iterator find*(db: var TDbConn, namespace: string, 
-               query: var TBSon, fields: varargs[string]): var TBSon {.
-               tags: [FReadDB].} =
-  ## yields the `fields` of any document that suffices `query`. If `fields` 
-  ## is ``[]`` the whole document is yielded.
-  var f = setupFieldnames(fields)
-  var cursor = find(db, namespace, query, f, 0'i32, 0'i32, 0'i32)
-  if cursor != nil:
-    while next(cursor[]) == mongo.OK:
-      yield bson(cursor[])[]
-    destroy(cursor[])
-  destroy(f)
-
-proc setupQuery(query: string): TBSon =
-  init(result)
-  add(result, "$where", query)
-  finish(result)
-
-iterator find*(db: var TDbConn, namespace: string, 
-               query: string, fields: varargs[string]): var TBSon {.
-               tags: [FReadDB].} =
-  ## yields the `fields` of any document that suffices `query`. If `fields` 
-  ## is ``[]`` the whole document is yielded.
-  var f = setupFieldnames(fields)
-  var q = setupQuery(query)
-  var cursor = find(db, namespace, q, f, 0'i32, 0'i32, 0'i32)
-  if cursor != nil:
-    while next(cursor[]) == mongo.OK:
-      yield bson(cursor[])[]
-    destroy(cursor[])
-  destroy(q)
-  destroy(f)
-
-when false:
-  # this doesn't work this way; would require low level hacking
-  iterator fieldPairs*(obj: var TBSon): tuple[key: cstring, value: TBSon] =
-    ## iterates over `obj` and yields all (key, value)-Pairs.
-    var it = initIter(obj)
-    var v: TBSon
-    while next(it) != bkEOO:
-      let key = key(it)
-      discard init(v, value(it))
-      yield (key, v)

lib/impure/db_mysql.nim

@@ -87,7 +87,7 @@ proc newRow(L: int): TRow =
   
 proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) =  
   if row != nil:
-    while mysql.FetchRow(sqlres) != nil: nil
+    while mysql.FetchRow(sqlres) != nil: discard
   mysql.FreeResult(sqlres)
   
 iterator fastRows*(db: TDbConn, query: TSqlQuery,
@@ -195,8 +195,14 @@ proc open*(connection, user, password, database: string): TDbConn {.
   ## be established.
   result = mysql.Init(nil)
   if result == nil: dbError("could not open database connection") 
-  if mysql.RealConnect(result, "", user, password, database, 
+  let
-                       0'i32, nil, 0) == nil:
+    colonPos = connection.find(':')
+    host =        if colonPos < 0: connection
+                  else:            substr(connection, 0, colonPos-1)
+    port: int32 = if colonPos < 0: 0'i32
+                  else:            substr(connection, colonPos+1).parseInt.int32
+  if mysql.RealConnect(result, host, user, password, database, 
+                       port, nil, 0) == nil:
     var errmsg = $mysql.error(result)
     db_mysql.Close(result)
     dbError(errmsg)

lib/nimbase.h

@@ -285,8 +285,8 @@ static N_INLINE(NI32, float32ToInt32)(float x) {
 
 typedef struct TStringDesc* string;
 
-/* declared size of a sequence: */
+/* declared size of a sequence/variable length array: */
-#if defined(__GNUC__)
+#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
 #  define SEQ_DECL_SIZE /* empty is correct! */
 #else
 #  define SEQ_DECL_SIZE 1000000
@@ -314,6 +314,9 @@ static unsigned long nimNaN[2]={0xffffffff, 0x7fffffff};
 #    define INF INFINITY
 #  elif defined(HUGE_VAL)
 #    define INF  HUGE_VAL
+#  elif defined(_MSC_VER)
+#    include <float.h>
+#    define INF (DBL_MAX+DBL_MAX)
 #  else
 #    define INF (1.0 / 0.0)
 #  endif
@@ -373,5 +376,8 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); }
 #  define GC_GUARD
 #endif
 
+/* Test to see if nimrod and the C compiler agrees on the size of a pointer.
+   On disagreement, your C compiler will say something like: 
+   "error: 'assert_numbits' declared as an array with a negative size" */
 typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1];
 #endif

lib/packages/docutils/docutils.babel

@@ -0,0 +1,6 @@
+[Package]
+name          = "docutils"
+version       = "0.9.0"
+author        = "Andreas Rumpf"
+description   = "Nimrod's reStructuredText processor."
+license       = "MIT"

lib/packages/docutils/highlite.nim

@@ -61,9 +61,8 @@ proc getSourceLanguage*(name: string): TSourceLanguage =
     if cmpIgnoreStyle(name, sourceLanguageToStr[i]) == 0: 
       return i
   result = langNone
-
+proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: cstring) =
-proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) = 
+  g.buf = buf
-  g.buf = cstring(buf)
   g.kind = low(TTokenClass)
   g.start = 0
   g.length = 0
@@ -71,6 +70,8 @@ proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) =
   var pos = 0                     # skip initial whitespace:
   while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos)
   g.pos = pos
+proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) = 
+  initGeneralTokenizer(g, cstring(buf))
 
 proc deinitGeneralTokenizer*(g: var TGeneralTokenizer) = 
   discard

lib/packages/docutils/rst.nim

@@ -1543,7 +1543,7 @@ proc dirRaw(p: var TRstParser): PRstNode =
     elif cmpIgnoreCase(result.sons[0].sons[0].text, "latex") == 0: 
       dirRawAux(p, result, rnRawLatex, parseLiteralBlock)
     else:
-      rstMessage(p, meInvalidDirective, result.sons[0].text)
+      rstMessage(p, meInvalidDirective, result.sons[0].sons[0].text)
   else:
     dirRawAux(p, result, rnRaw, parseSectionWrapper)
 

lib/posix/epoll.nim

@@ -7,6 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
+from posix import TSocketHandle
+
 const
   EPOLLIN* = 0x00000001
   EPOLLPRI* = 0x00000002
@@ -33,8 +35,8 @@ const
 type 
   epoll_data* {.importc: "union epoll_data", 
       header: "<sys/epoll.h>", pure, final.} = object # TODO: This is actually a union.
-    thePtr* {.importc: "ptr".}: pointer # \
+    #thePtr* {.importc: "ptr".}: pointer
-    #fd*: cint
+    fd* {.importc: "fd".}: cint # \
     #u32*: uint32
     #u64*: uint64
 
@@ -54,7 +56,7 @@ proc epoll_create1*(flags: cint): cint {.importc: "epoll_create1",
   ## Same as epoll_create but with an FLAGS parameter.  The unused SIZE
   ##   parameter has been dropped.  
 
-proc epoll_ctl*(epfd: cint; op: cint; fd: cint; event: ptr epoll_event): cint {.
+proc epoll_ctl*(epfd: cint; op: cint; fd: cint | TSocketHandle; event: ptr epoll_event): cint {.
     importc: "epoll_ctl", header: "<sys/epoll.h>".}
   ## Manipulate an epoll instance "epfd". Returns 0 in case of success,
   ##   -1 in case of error ( the "errno" variable will contain the

lib/posix/linux.nim

@@ -0,0 +1,25 @@
+import posix
+
+const
+  CSIGNAL* = 0x000000FF
+  CLONE_VM* = 0x00000100
+  CLONE_FS* = 0x00000200
+  CLONE_FILES* = 0x00000400
+  CLONE_SIGHAND* = 0x00000800
+  CLONE_PTRACE* = 0x00002000
+  CLONE_VFORK* = 0x00004000
+  CLONE_PARENT* = 0x00008000
+  CLONE_THREAD* = 0x00010000
+  CLONE_NEWNS* = 0x00020000
+  CLONE_SYSVSEM* = 0x00040000
+  CLONE_SETTLS* = 0x00080000
+  CLONE_PARENT_SETTID* = 0x00100000
+  CLONE_CHILD_CLEARTID* = 0x00200000
+  CLONE_DETACHED* = 0x00400000
+  CLONE_UNTRACED* = 0x00800000
+  CLONE_CHILD_SETTID* = 0x01000000
+  CLONE_STOPPED* = 0x02000000
+
+# fn should be of type proc (a2: pointer): void {.cdecl.}
+proc clone*(fn: pointer; child_stack: pointer; flags: cint;
+            arg: pointer; ptid: ptr TPid; tls: pointer; ctid: ptr TPid): cint {.importc, header: "<sched.h>".}

lib/posix/posix.nim

@@ -2066,6 +2066,7 @@ proc pthread_spin_unlock*(a1: ptr Tpthread_spinlock): cint {.
 proc pthread_testcancel*() {.importc, header: "<pthread.h>".}
 
 
+proc exitnow*(code: int): void {.importc: "_exit", header: "<unistd.h>".}
 proc access*(a1: cstring, a2: cint): cint {.importc, header: "<unistd.h>".}
 proc alarm*(a1: cint): cint {.importc, header: "<unistd.h>".}
 proc chdir*(a1: cstring): cint {.importc, header: "<unistd.h>".}
@@ -2265,6 +2266,7 @@ proc gmtime_r*(a1: var TTime, a2: var Ttm): ptr Ttm {.importc, header: "<time.h>
 proc localtime*(a1: var TTime): ptr Ttm {.importc, header: "<time.h>".}
 proc localtime_r*(a1: var TTime, a2: var Ttm): ptr Ttm {.importc, header: "<time.h>".}
 proc mktime*(a1: var Ttm): TTime  {.importc, header: "<time.h>".}
+proc timegm*(a1: var Ttm): TTime  {.importc, header: "<time.h>".}
 proc nanosleep*(a1, a2: var Ttimespec): cint {.importc, header: "<time.h>".}
 proc strftime*(a1: cstring, a2: int, a3: cstring,
            a4: var Ttm): int {.importc, header: "<time.h>".}
@@ -2356,7 +2358,7 @@ proc FD_ZERO*(a1: var TFdSet) {.importc, header: "<sys/select.h>".}
 
 proc pselect*(a1: cint, a2, a3, a4: ptr TFdSet, a5: ptr Ttimespec,
          a6: var Tsigset): cint  {.importc, header: "<sys/select.h>".}
-proc select*(a1: cint, a2, a3, a4: ptr TFdSet, a5: ptr Ttimeval): cint {.
+proc select*(a1: cint | TSocketHandle, a2, a3, a4: ptr TFdSet, a5: ptr Ttimeval): cint {.
              importc, header: "<sys/select.h>".}
 
 when hasSpawnH:

lib/pure/algorithm.nim

@@ -55,6 +55,36 @@ proc smartBinarySearch*[T](a: openArray[T], key: T): int =
 const
   onlySafeCode = true
 
+proc lowerBound*[T](a: openarray[T], key: T, cmp: proc(x,y: T): int {.closure.}): int =
+  ## same as binarySearch except that if key is not in `a` then this 
+  ## returns the location where `key` would be if it were. In other
+  ## words if you have a sorted sequence and you call insert(thing, elm, lowerBound(thing, elm))
+  ## the sequence will still be sorted
+  ##
+  ## `cmp` is the comparator function to use, the expected return values are the same as
+  ## that of system.cmp
+  ## 
+  ## example::
+  ##
+  ##   var arr = @[1,2,3,5,6,7,8,9]
+  ##   arr.insert(4, arr.lowerBound(4))
+  ## `after running the above arr is `[1,2,3,4,5,6,7,8,9]`
+  result = a.low
+  var pos = result
+  var count, step: int
+  count = a.high - a.low + 1
+  while count != 0:
+    pos = result
+    step = count div 2
+    pos += step
+    if cmp(a[pos], key) < 0:
+      pos.inc
+      result = pos
+      count -= step + 1
+    else:
+      count = step
+
+proc lowerBound*[T](a: openarray[T], key: T): int = lowerBound(a, key, cmp[T])
 proc merge[T](a, b: var openArray[T], lo, m, hi: int, 
               cmp: proc (x, y: T): int {.closure.}, order: TSortOrder) =
   template `<-` (a, b: expr) = 

lib/pure/asyncdispatch.nim

@@ -0,0 +1,956 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2014 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import os, oids, tables, strutils, macros
+
+import rawsockets
+
+## AsyncDispatch
+## --------
+##
+## This module implements a brand new dispatcher based on Futures.
+## On Windows IOCP is used and on other operating systems the selectors module
+## is used instead.
+
+# -- Futures
+
+type
+  PFutureBase* = ref object of PObject
+    cb: proc () {.closure.}
+    finished: bool
+
+  PFuture*[T] = ref object of PFutureBase
+    value: T
+    error*: ref EBase # TODO: This shouldn't be necessary, generics bug?
+
+proc newFuture*[T](): PFuture[T] =
+  ## Creates a new future.
+  new(result)
+  result.finished = false
+
+proc complete*[T](future: PFuture[T], val: T) =
+  ## Completes ``future`` with value ``val``.
+  assert(not future.finished, "Future already finished, cannot finish twice.")
+  assert(future.error == nil)
+  future.value = val
+  future.finished = true
+  if future.cb != nil:
+    future.cb()
+
+proc complete*(future: PFuture[void]) =
+  ## Completes a void ``future``.
+  assert(not future.finished, "Future already finished, cannot finish twice.")
+  assert(future.error == nil)
+  future.finished = true
+  if future.cb != nil:
+    future.cb()
+
+proc fail*[T](future: PFuture[T], error: ref EBase) =
+  ## Completes ``future`` with ``error``.
+  assert(not future.finished, "Future already finished, cannot finish twice.")
+  future.finished = true
+  future.error = error
+  if future.cb != nil:
+    future.cb()
+
+proc `callback=`*(future: PFutureBase, cb: proc () {.closure.}) =
+  ## Sets the callback proc to be called when the future completes.
+  ##
+  ## If future has already completed then ``cb`` will be called immediately.
+  ##
+  ## **Note**: You most likely want the other ``callback`` setter which
+  ## passes ``future`` as a param to the callback.
+  future.cb = cb
+  if future.finished:
+    future.cb()
+
+proc `callback=`*[T](future: PFuture[T],
+    cb: proc (future: PFuture[T]) {.closure.}) =
+  ## Sets the callback proc to be called when the future completes.
+  ##
+  ## If future has already completed then ``cb`` will be called immediately.
+  future.callback = proc () = cb(future)
+
+proc read*[T](future: PFuture[T]): T =
+  ## Retrieves the value of ``future``. Future must be finished otherwise
+  ## this function will fail with a ``EInvalidValue`` exception.
+  ##
+  ## If the result of the future is an error then that error will be raised.
+  if future.finished:
+    if future.error != nil: raise future.error
+    when T isnot void:
+      return future.value
+  else:
+    # TODO: Make a custom exception type for this?
+    raise newException(EInvalidValue, "Future still in progress.")
+
+proc readError*[T](future: PFuture[T]): ref EBase =
+  if future.error != nil: return future.error
+  else:
+    raise newException(EInvalidValue, "No error in future.")
+
+proc finished*[T](future: PFuture[T]): bool =
+  ## Determines whether ``future`` has completed.
+  ##
+  ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish.
+  future.finished
+
+proc failed*[T](future: PFuture[T]): bool =
+  ## Determines whether ``future`` completed with an error.
+  future.error != nil
+
+when defined(windows) or defined(nimdoc):
+  import winlean, sets, hashes
+  type
+    TCompletionKey = dword
+
+    TCompletionData* = object
+      sock: TAsyncFD
+      cb: proc (sock: TAsyncFD, bytesTransferred: DWORD,
+                errcode: TOSErrorCode) {.closure.}
+
+    PDispatcher* = ref object
+      ioPort: THandle
+      handles: TSet[TAsyncFD]
+
+    TCustomOverlapped = object
+      Internal*: DWORD
+      InternalHigh*: DWORD
+      Offset*: DWORD
+      OffsetHigh*: DWORD
+      hEvent*: THANDLE
+      data*: TCompletionData
+
+    PCustomOverlapped = ptr TCustomOverlapped
+
+    TAsyncFD* = distinct int
+
+  proc hash(x: TAsyncFD): THash {.borrow.}
+  proc `==`*(x: TAsyncFD, y: TAsyncFD): bool {.borrow.}
+
+  proc newDispatcher*(): PDispatcher =
+    ## Creates a new Dispatcher instance.
+    new result
+    result.ioPort = CreateIOCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1)
+    result.handles = initSet[TAsyncFD]()
+
+  var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
+  proc getGlobalDispatcher*(): PDispatcher =
+    ## Retrieves the global thread-local dispatcher.
+    if gDisp.isNil: gDisp = newDispatcher()
+    result = gDisp
+
+  proc register*(sock: TAsyncFD) =
+    ## Registers ``sock`` with the dispatcher.
+    let p = getGlobalDispatcher()
+    if CreateIOCompletionPort(sock.THandle, p.ioPort,
+                              cast[TCompletionKey](sock), 1) == 0:
+      OSError(OSLastError())
+    p.handles.incl(sock)
+
+  proc verifyPresence(sock: TAsyncFD) =
+    ## Ensures that socket has been registered with the dispatcher.
+    let p = getGlobalDispatcher()
+    if sock notin p.handles:
+      raise newException(EInvalidValue,
+        "Operation performed on a socket which has not been registered with" &
+        " the dispatcher yet.")
+
+  proc poll*(timeout = 500) =
+    ## Waits for completion events and processes them.
+    let p = getGlobalDispatcher()
+    if p.handles.len == 0:
+      raise newException(EInvalidValue, "No handles registered in dispatcher.")
+    
+    let llTimeout =
+      if timeout ==  -1: winlean.INFINITE
+      else: timeout.int32
+    var lpNumberOfBytesTransferred: DWORD
+    var lpCompletionKey: ULONG
+    var lpOverlapped: POverlapped
+    let res = GetQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred,
+        addr lpCompletionKey, addr lpOverlapped, llTimeout).bool
+
+    # http://stackoverflow.com/a/12277264/492186
+    # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html
+    var customOverlapped = cast[PCustomOverlapped](lpOverlapped)
+    if res:
+      # This is useful for ensuring the reliability of the overlapped struct.
+      assert customOverlapped.data.sock == lpCompletionKey.TAsyncFD
+
+      customOverlapped.data.cb(customOverlapped.data.sock,
+          lpNumberOfBytesTransferred, TOSErrorCode(-1))
+      dealloc(customOverlapped)
+    else:
+      let errCode = OSLastError()
+      if lpOverlapped != nil:
+        assert customOverlapped.data.sock == lpCompletionKey.TAsyncFD
+        customOverlapped.data.cb(customOverlapped.data.sock,
+            lpNumberOfBytesTransferred, errCode)
+        dealloc(customOverlapped)
+      else:
+        if errCode.int32 == WAIT_TIMEOUT:
+          # Timed out
+          discard
+        else: OSError(errCode)
+
+  var connectExPtr: pointer = nil
+  var acceptExPtr: pointer = nil
+  var getAcceptExSockAddrsPtr: pointer = nil
+
+  proc initPointer(s: TSocketHandle, func: var pointer, guid: var TGUID): bool =
+    # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c
+    var bytesRet: DWord
+    func = nil
+    result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid,
+                      sizeof(TGUID).dword, addr func, sizeof(pointer).DWORD,
+                      addr bytesRet, nil, nil) == 0
+
+  proc initAll() =
+    let dummySock = newRawSocket()
+    if not initPointer(dummySock, connectExPtr, WSAID_CONNECTEX):
+      OSError(OSLastError())
+    if not initPointer(dummySock, acceptExPtr, WSAID_ACCEPTEX):
+      OSError(OSLastError())
+    if not initPointer(dummySock, getAcceptExSockAddrsPtr, WSAID_GETACCEPTEXSOCKADDRS):
+      OSError(OSLastError())
+
+  proc connectEx(s: TSocketHandle, name: ptr TSockAddr, namelen: cint, 
+                  lpSendBuffer: pointer, dwSendDataLength: dword,
+                  lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool =
+    if connectExPtr.isNil: raise newException(EInvalidValue, "Need to initialise ConnectEx().")
+    let func =
+      cast[proc (s: TSocketHandle, name: ptr TSockAddr, namelen: cint, 
+         lpSendBuffer: pointer, dwSendDataLength: dword,
+         lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool {.stdcall.}](connectExPtr)
+
+    result = func(s, name, namelen, lpSendBuffer, dwSendDataLength, lpdwBytesSent,
+         lpOverlapped)
+
+  proc acceptEx(listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer,
+                 dwReceiveDataLength, dwLocalAddressLength,
+                 dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD,
+                 lpOverlapped: POverlapped): bool =
+    if acceptExPtr.isNil: raise newException(EInvalidValue, "Need to initialise AcceptEx().")
+    let func =
+      cast[proc (listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer,
+                 dwReceiveDataLength, dwLocalAddressLength,
+                 dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD,
+                 lpOverlapped: POverlapped): bool {.stdcall.}](acceptExPtr)
+    result = func(listenSock, acceptSock, lpOutputBuffer, dwReceiveDataLength,
+        dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived,
+        lpOverlapped)
+
+  proc getAcceptExSockaddrs(lpOutputBuffer: pointer,
+      dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: DWORD,
+      LocalSockaddr: ptr ptr TSockAddr, LocalSockaddrLength: lpint,
+      RemoteSockaddr: ptr ptr TSockAddr, RemoteSockaddrLength: lpint) =
+    if getAcceptExSockAddrsPtr.isNil:
+      raise newException(EInvalidValue, "Need to initialise getAcceptExSockAddrs().")
+
+    let func =
+      cast[proc (lpOutputBuffer: pointer,
+                 dwReceiveDataLength, dwLocalAddressLength,
+                 dwRemoteAddressLength: DWORD, LocalSockaddr: ptr ptr TSockAddr,
+                 LocalSockaddrLength: lpint, RemoteSockaddr: ptr ptr TSockAddr,
+                RemoteSockaddrLength: lpint) {.stdcall.}](getAcceptExSockAddrsPtr)
+    
+    func(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength,
+                  dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength,
+                  RemoteSockaddr, RemoteSockaddrLength)
+
+  proc connect*(socket: TAsyncFD, address: string, port: TPort,
+    af = AF_INET): PFuture[void] =
+    ## Connects ``socket`` to server at ``address:port``.
+    ##
+    ## Returns a ``PFuture`` which will complete when the connection succeeds
+    ## or an error occurs.
+    verifyPresence(socket)
+    var retFuture = newFuture[void]()
+    # Apparently ``ConnectEx`` expects the socket to be initially bound:
+    var saddr: Tsockaddr_in
+    saddr.sin_family = int16(toInt(af))
+    saddr.sin_port = 0
+    saddr.sin_addr.s_addr = INADDR_ANY
+    if bindAddr(socket.TSocketHandle, cast[ptr TSockAddr](addr(saddr)),
+                  sizeof(saddr).TSockLen) < 0'i32:
+      OSError(OSLastError())
+
+    var aiList = getAddrInfo(address, port, af)
+    var success = false
+    var lastError: TOSErrorCode
+    var it = aiList
+    while it != nil:
+      # "the OVERLAPPED structure must remain valid until the I/O completes"
+      # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx
+      var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
+      ol.data = TCompletionData(sock: socket, cb:
+        proc (sock: TAsyncFD, bytesCount: DWord, errcode: TOSErrorCode) =
+          if not retFuture.finished:
+            if errcode == TOSErrorCode(-1):
+              retFuture.complete()
+            else:
+              retFuture.fail(newException(EOS, osErrorMsg(errcode)))
+      )
+      
+      var ret = connectEx(socket.TSocketHandle, it.ai_addr,
+                          sizeof(TSockAddrIn).cint, nil, 0, nil,
+                          cast[POverlapped](ol))
+      if ret:
+        # Request to connect completed immediately.
+        success = true
+        retFuture.complete()
+        # We don't deallocate ``ol`` here because even though this completed
+        # immediately poll will still be notified about its completion and it will
+        # free ``ol``.
+        break
+      else:
+        lastError = OSLastError()
+        if lastError.int32 == ERROR_IO_PENDING:
+          # In this case ``ol`` will be deallocated in ``poll``.
+          success = true
+          break
+        else:
+          dealloc(ol)
+          success = false
+      it = it.ai_next
+
+    dealloc(aiList)
+    if not success:
+      retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+    return retFuture
+
+  proc recv*(socket: TAsyncFD, size: int,
+             flags: int = 0): PFuture[string] =
+    ## Reads ``size`` bytes from ``socket``. Returned future will complete once
+    ## all of the requested data is read. If socket is disconnected during the
+    ## recv operation then the future may complete with only a part of the
+    ## requested data read. If socket is disconnected and no data is available
+    ## to be read then the future will complete with a value of ``""``.
+    verifyPresence(socket)
+    var retFuture = newFuture[string]()
+    
+    var dataBuf: TWSABuf
+    dataBuf.buf = newString(size)
+    dataBuf.len = size
+    
+    var bytesReceived: DWord
+    var flagsio = flags.dword
+    var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
+    ol.data = TCompletionData(sock: socket, cb:
+      proc (sock: TAsyncFD, bytesCount: DWord, errcode: TOSErrorCode) =
+        if not retFuture.finished:
+          if errcode == TOSErrorCode(-1):
+            if bytesCount == 0 and dataBuf.buf[0] == '\0':
+              retFuture.complete("")
+            else:
+              var data = newString(size)
+              copyMem(addr data[0], addr dataBuf.buf[0], size)
+              retFuture.complete($data)
+          else:
+            retFuture.fail(newException(EOS, osErrorMsg(errcode)))
+    )
+
+    let ret = WSARecv(socket.TSocketHandle, addr dataBuf, 1, addr bytesReceived,
+                      addr flagsio, cast[POverlapped](ol), nil)
+    if ret == -1:
+      let err = OSLastError()
+      if err.int32 != ERROR_IO_PENDING:
+        retFuture.fail(newException(EOS, osErrorMsg(err)))
+        dealloc(ol)
+    elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0':
+      # We have to ensure that the buffer is empty because WSARecv will tell
+      # us immediatelly when it was disconnected, even when there is still
+      # data in the buffer.
+      # We want to give the user as much data as we can. So we only return
+      # the empty string (which signals a disconnection) when there is
+      # nothing left to read.
+      retFuture.complete("")
+      # TODO: "For message-oriented sockets, where a zero byte message is often 
+      # allowable, a failure with an error code of WSAEDISCON is used to 
+      # indicate graceful closure." 
+      # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx
+    else:
+      # Request to read completed immediately.
+      var data = newString(size)
+      copyMem(addr data[0], addr dataBuf.buf[0], size)
+      retFuture.complete($data)
+      # We don't deallocate ``ol`` here because even though this completed
+      # immediately poll will still be notified about its completion and it will
+      # free ``ol``.
+    return retFuture
+
+  proc send*(socket: TAsyncFD, data: string): PFuture[void] =
+    ## Sends ``data`` to ``socket``. The returned future will complete once all
+    ## data has been sent.
+    verifyPresence(socket)
+    var retFuture = newFuture[void]()
+
+    var dataBuf: TWSABuf
+    dataBuf.buf = data
+    dataBuf.len = data.len
+
+    var bytesReceived, flags: DWord
+    var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
+    ol.data = TCompletionData(sock: socket, cb:
+      proc (sock: TAsyncFD, bytesCount: DWord, errcode: TOSErrorCode) =
+        if not retFuture.finished:
+          if errcode == TOSErrorCode(-1):
+            retFuture.complete()
+          else:
+            retFuture.fail(newException(EOS, osErrorMsg(errcode)))
+    )
+
+    let ret = WSASend(socket.TSocketHandle, addr dataBuf, 1, addr bytesReceived,
+                      flags, cast[POverlapped](ol), nil)
+    if ret == -1:
+      let err = osLastError()
+      if err.int32 != ERROR_IO_PENDING:
+        retFuture.fail(newException(EOS, osErrorMsg(err)))
+        dealloc(ol)
+    else:
+      retFuture.complete()
+      # We don't deallocate ``ol`` here because even though this completed
+      # immediately poll will still be notified about its completion and it will
+      # free ``ol``.
+    return retFuture
+
+  proc acceptAddr*(socket: TAsyncFD): 
+      PFuture[tuple[address: string, client: TAsyncFD]] =
+    ## Accepts a new connection. Returns a future containing the client socket
+    ## corresponding to that connection and the remote address of the client.
+    ## The future will complete when the connection is successfully accepted.
+    ##
+    ## The resulting client socket is automatically registered to dispatcher.
+    verifyPresence(socket)
+    var retFuture = newFuture[tuple[address: string, client: TAsyncFD]]()
+
+    var clientSock = newRawSocket()
+    if clientSock == OSInvalidSocket: osError(osLastError())
+
+    const lpOutputLen = 1024
+    var lpOutputBuf = newString(lpOutputLen)
+    var dwBytesReceived: DWORD
+    let dwReceiveDataLength = 0.DWORD # We don't want any data to be read.
+    let dwLocalAddressLength = DWORD(sizeof (TSockaddr_in) + 16)
+    let dwRemoteAddressLength = DWORD(sizeof(TSockaddr_in) + 16)
+
+    template completeAccept(): stmt {.immediate, dirty.} =
+      var listenSock = socket
+      let setoptRet = setsockopt(clientSock, SOL_SOCKET,
+          SO_UPDATE_ACCEPT_CONTEXT, addr listenSock,
+          sizeof(listenSock).TSockLen)
+      if setoptRet != 0: osError(osLastError())
+
+      var LocalSockaddr, RemoteSockaddr: ptr TSockAddr
+      var localLen, remoteLen: int32
+      getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength,
+                           dwLocalAddressLength, dwRemoteAddressLength,
+                           addr LocalSockaddr, addr localLen,
+                           addr RemoteSockaddr, addr remoteLen)
+      register(clientSock.TAsyncFD)
+      # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186
+      retFuture.complete(
+        (address: $inet_ntoa(cast[ptr Tsockaddr_in](remoteSockAddr).sin_addr),
+         client: clientSock.TAsyncFD)
+      )
+
+    var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
+    ol.data = TCompletionData(sock: socket, cb:
+      proc (sock: TAsyncFD, bytesCount: DWord, errcode: TOSErrorCode) =
+        if not retFuture.finished:
+          if errcode == TOSErrorCode(-1):
+            completeAccept()
+          else:
+            retFuture.fail(newException(EOS, osErrorMsg(errcode)))
+    )
+
+    # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx
+    let ret = acceptEx(socket.TSocketHandle, clientSock, addr lpOutputBuf[0],
+                       dwReceiveDataLength, 
+                       dwLocalAddressLength,
+                       dwRemoteAddressLength,
+                       addr dwBytesReceived, cast[POverlapped](ol))
+
+    if not ret:
+      let err = osLastError()
+      if err.int32 != ERROR_IO_PENDING:
+        retFuture.fail(newException(EOS, osErrorMsg(err)))
+        dealloc(ol)
+    else:
+      completeAccept()
+      # We don't deallocate ``ol`` here because even though this completed
+      # immediately poll will still be notified about its completion and it will
+      # free ``ol``.
+
+    return retFuture
+
+  proc newAsyncRawSocket*(domain: TDomain = AF_INET,
+               typ: TType = SOCK_STREAM,
+               protocol: TProtocol = IPPROTO_TCP): TAsyncFD =
+    ## Creates a new socket and registers it with the dispatcher implicitly.
+    result = newRawSocket(domain, typ, protocol).TAsyncFD
+    result.TSocketHandle.setBlocking(false)
+    register(result)
+
+  proc close*(socket: TAsyncFD) =
+    ## Closes a socket and ensures that it is unregistered.
+    socket.TSocketHandle.close()
+    getGlobalDispatcher().handles.excl(socket)
+
+  initAll()
+else:
+  import selectors
+  from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK
+  type
+    TAsyncFD* = distinct cint
+    TCallback = proc (sock: TAsyncFD): bool {.closure.}
+
+    PData* = ref object of PObject
+      sock: TAsyncFD
+      readCBs: seq[TCallback]
+      writeCBs: seq[TCallback]
+
+    PDispatcher* = ref object
+      selector: PSelector
+
+  proc `==`*(x, y: TAsyncFD): bool {.borrow.}
+
+  proc newDispatcher*(): PDispatcher =
+    new result
+    result.selector = newSelector()
+
+  var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
+  proc getGlobalDispatcher*(): PDispatcher =
+    if gDisp.isNil: gDisp = newDispatcher()
+    result = gDisp
+
+  proc update(sock: TAsyncFD, events: set[TEvent]) =
+    let p = getGlobalDispatcher()
+    assert sock.TSocketHandle in p.selector
+    discard p.selector.update(sock.TSocketHandle, events)
+
+  proc register(sock: TAsyncFD) =
+    let p = getGlobalDispatcher()
+    var data = PData(sock: sock, readCBs: @[], writeCBs: @[])
+    p.selector.register(sock.TSocketHandle, {}, data.PObject)
+
+  proc newAsyncRawSocket*(domain: TDomain = AF_INET,
+               typ: TType = SOCK_STREAM,
+               protocol: TProtocol = IPPROTO_TCP): TAsyncFD =
+    result = newRawSocket(domain, typ, protocol).TAsyncFD
+    result.TSocketHandle.setBlocking(false)
+    register(result)
+  
+  proc close*(sock: TAsyncFD) =
+    let disp = getGlobalDispatcher()
+    sock.TSocketHandle.close()
+    disp.selector.unregister(sock.TSocketHandle)
+
+  proc addRead(sock: TAsyncFD, cb: TCallback) =
+    let p = getGlobalDispatcher()
+    if sock.TSocketHandle notin p.selector:
+      raise newException(EInvalidValue, "File descriptor not registered.")
+    p.selector[sock.TSocketHandle].data.PData.readCBs.add(cb)
+    update(sock, p.selector[sock.TSocketHandle].events + {EvRead})
+  
+  proc addWrite(sock: TAsyncFD, cb: TCallback) =
+    let p = getGlobalDispatcher()
+    if sock.TSocketHandle notin p.selector:
+      raise newException(EInvalidValue, "File descriptor not registered.")
+    p.selector[sock.TSocketHandle].data.PData.writeCBs.add(cb)
+    update(sock, p.selector[sock.TSocketHandle].events + {EvWrite})
+  
+  proc poll*(timeout = 500) =
+    let p = getGlobalDispatcher()
+    for info in p.selector.select(timeout):
+      let data = PData(info.key.data)
+      assert data.sock == info.key.fd.TAsyncFD
+      #echo("In poll ", data.sock.cint)
+      if EvRead in info.events:
+        # Callback may add items to ``data.readCBs`` which causes issues if
+        # we are iterating over ``data.readCBs`` at the same time. We therefore
+        # make a copy to iterate over.
+        let currentCBs = data.readCBs
+        data.readCBs = @[]
+        for cb in currentCBs:
+          if not cb(data.sock):
+            # Callback wants to be called again.
+            data.readCBs.add(cb)
+      
+      if EvWrite in info.events:
+        let currentCBs = data.writeCBs
+        data.writeCBs = @[]
+        for cb in currentCBs:
+          if not cb(data.sock):
+            # Callback wants to be called again.
+            data.writeCBs.add(cb)
+      
+      if info.key in p.selector:
+        var newEvents: set[TEvent]
+        if data.readCBs.len != 0: newEvents = {EvRead}
+        if data.writeCBs.len != 0: newEvents = newEvents + {EvWrite}
+        if newEvents != info.key.events:
+          update(data.sock, newEvents)
+      else:
+        # FD no longer a part of the selector. Likely been closed
+        # (e.g. socket disconnected).
+  
+  proc connect*(socket: TAsyncFD, address: string, port: TPort,
+    af = AF_INET): PFuture[void] =
+    var retFuture = newFuture[void]()
+    
+    proc cb(sock: TAsyncFD): bool =
+      # We have connected.
+      retFuture.complete()
+      return true
+    
+    var aiList = getAddrInfo(address, port, af)
+    var success = false
+    var lastError: TOSErrorCode
+    var it = aiList
+    while it != nil:
+      var ret = connect(socket.TSocketHandle, it.ai_addr, it.ai_addrlen.TSocklen)
+      if ret == 0:
+        # Request to connect completed immediately.
+        success = true
+        retFuture.complete()
+        break
+      else:
+        lastError = osLastError()
+        if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS:
+          success = true
+          addWrite(socket, cb)
+          break
+        else:
+          success = false
+      it = it.ai_next
+
+    dealloc(aiList)
+    if not success:
+      retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+    return retFuture
+
+  proc recv*(socket: TAsyncFD, size: int,
+             flags: int = 0): PFuture[string] =
+    var retFuture = newFuture[string]()
+    
+    var readBuffer = newString(size)
+    var sizeRead = 0
+    
+    proc cb(sock: TAsyncFD): bool =
+      result = true
+      let netSize = size - sizeRead
+      let res = recv(sock.TSocketHandle, addr readBuffer[sizeRead], netSize,
+                     flags.cint)
+      #echo("recv cb res: ", res)
+      if res < 0:
+        let lastError = osLastError()
+        if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: 
+          retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+        else:
+          result = false # We still want this callback to be called.
+      elif res == 0:
+        #echo("Disconnected recv: ", sizeRead)
+        # Disconnected
+        if sizeRead == 0:
+          retFuture.complete("")
+        else:
+          readBuffer.setLen(sizeRead)
+          retFuture.complete(readBuffer)
+      else:
+        sizeRead.inc(res)
+        if res != netSize:
+          result = false # We want to read all the data requested.
+        else:
+          retFuture.complete(readBuffer)
+      #echo("Recv cb result: ", result)
+  
+    addRead(socket, cb)
+    return retFuture
+
+  proc send*(socket: TAsyncFD, data: string): PFuture[void] =
+    var retFuture = newFuture[void]()
+    
+    var written = 0
+    
+    proc cb(sock: TAsyncFD): bool =
+      result = true
+      let netSize = data.len-written
+      var d = data.cstring
+      let res = send(sock.TSocketHandle, addr d[written], netSize, 0.cint)
+      if res < 0:
+        let lastError = osLastError()
+        if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}:
+          retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+        else:
+          result = false # We still want this callback to be called.
+      else:
+        written.inc(res)
+        if res != netSize:
+          result = false # We still have data to send.
+        else:
+          retFuture.complete()
+    addWrite(socket, cb)
+    return retFuture
+
+  proc acceptAddr*(socket: TAsyncFD): 
+      PFuture[tuple[address: string, client: TAsyncFD]] =
+    var retFuture = newFuture[tuple[address: string, client: TAsyncFD]]()
+    proc cb(sock: TAsyncFD): bool =
+      result = true
+      var sockAddress: Tsockaddr_in
+      var addrLen = sizeof(sockAddress).TSocklen
+      var client = accept(sock.TSocketHandle,
+                          cast[ptr TSockAddr](addr(sockAddress)), addr(addrLen))
+      if client == osInvalidSocket:
+        let lastError = osLastError()
+        assert lastError.int32 notin {EWOULDBLOCK, EAGAIN}
+        if lastError.int32 == EINTR:
+          return false
+        else:
+          retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+      else:
+        register(client.TAsyncFD)
+        retFuture.complete(($inet_ntoa(sockAddress.sin_addr), client.TAsyncFD))
+    addRead(socket, cb)
+    return retFuture
+
+proc accept*(socket: TAsyncFD): PFuture[TAsyncFD] =
+  ## Accepts a new connection. Returns a future containing the client socket
+  ## corresponding to that connection.
+  ## The future will complete when the connection is successfully accepted.
+  var retFut = newFuture[TAsyncFD]()
+  var fut = acceptAddr(socket)
+  fut.callback =
+    proc (future: PFuture[tuple[address: string, client: TAsyncFD]]) =
+      assert future.finished
+      if future.failed:
+        retFut.fail(future.error)
+      else:
+        retFut.complete(future.read.client)
+  return retFut
+
+# -- Await Macro
+
+template createCb*(cbName, varNameIterSym, retFutureSym: expr): stmt {.immediate, dirty.} =
+  proc cbName {.closure.} =
+    if not varNameIterSym.finished:
+      var next = varNameIterSym()
+      if next == nil:
+        assert retFutureSym.finished, "Async procedure's return Future was not finished."
+      else:
+        next.callback = cbName
+
+template createVar(futSymName: string, asyncProc: PNimrodNode,
+                   valueReceiver: expr) {.immediate, dirty.} =
+  # TODO: Used template here due to bug #926
+  result = newNimNode(nnkStmtList)
+  var futSym = genSym(nskVar, "future")
+  result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y
+  result.add newNimNode(nnkYieldStmt).add(futSym) # -> yield future<x>
+  valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read
+
+proc processBody(node, retFutureSym: PNimrodNode): PNimrodNode {.compileTime.} =
+  result = node
+  case node.kind
+  of nnkReturnStmt:
+    result = newNimNode(nnkStmtList)
+    result.add newCall(newIdentNode("complete"), retFutureSym,
+      if node[0].kind == nnkEmpty: newIdentNode("result") else: node[0])
+    result.add newNimNode(nnkYieldStmt).add(newNilLit())
+  of nnkCommand:
+    if node[0].kind == nnkIdent and node[0].ident == !"await":
+      case node[1].kind
+      of nnkIdent:
+        # await x
+        result = newNimNode(nnkYieldStmt).add(node[1]) # -> yield x
+      of nnkCall:
+        # await foo(p, x)
+        var futureValue: PNimrodNode
+        createVar("future" & $node[1][0].toStrLit, node[1], futureValue)
+        result.add futureValue
+      else:
+        error("Invalid node kind in 'await', got: " & $node[1].kind)
+    elif node[1].kind == nnkCommand and node[1][0].kind == nnkIdent and
+         node[1][0].ident == !"await":
+      # foo await x
+      var newCommand = node
+      createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1])
+      result.add newCommand
+
+  of nnkVarSection, nnkLetSection:
+    case node[0][2].kind
+    of nnkCommand:
+      if node[0][2][0].ident == !"await":
+        # var x = await y
+        var newVarSection = node # TODO: Should this use copyNimNode?
+        createVar("future" & $node[0][0].ident, node[0][2][1],
+          newVarSection[0][2])
+        result.add newVarSection
+    else: discard
+  of nnkAsgn:
+    case node[1].kind
+    of nnkCommand:
+      if node[1][0].ident == !"await":
+        # x = await y
+        var newAsgn = node
+        createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1])
+        result.add newAsgn
+    else: discard
+  of nnkDiscardStmt:
+    # discard await x
+    if node[0][0].kind == nnkIdent and node[0][0].ident == !"await":
+      var dummy = newNimNode(nnkStmtList)
+      createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], dummy)
+  else: discard
+  
+  for i in 0 .. <result.len:
+    result[i] = processBody(result[i], retFutureSym)
+  #echo(treeRepr(result))
+
+proc getName(node: PNimrodNode): string {.compileTime.} =
+  case node.kind
+  of nnkPostfix:
+    return $node[1].ident
+  of nnkIdent:
+    return $node.ident
+  else:
+    assert false
+
+macro async*(prc: stmt): stmt {.immediate.} =
+  ## Macro which processes async procedures into the appropriate
+  ## iterators and yield statements.
+
+  expectKind(prc, nnkProcDef)
+
+  hint("Processing " & prc[0].getName & " as an async proc.")
+
+  let returnType = prc[3][0]
+  var subtypeName = ""
+  # Verify that the return type is a PFuture[T]
+  if returnType.kind == nnkIdent:
+    error("Expected return type of 'PFuture' got '" & $returnType & "'")
+  elif returnType.kind == nnkBracketExpr:
+    if $returnType[0] != "PFuture":
+      error("Expected return type of 'PFuture' got '" & $returnType[0] & "'")
+    subtypeName = $returnType[1].ident
+  elif returnType.kind == nnkEmpty:
+    subtypeName = "void"
+
+  var outerProcBody = newNimNode(nnkStmtList)
+
+  # -> var retFuture = newFuture[T]()
+  var retFutureSym = genSym(nskVar, "retFuture")
+  outerProcBody.add(
+    newVarStmt(retFutureSym, 
+      newCall(
+        newNimNode(nnkBracketExpr).add(
+          newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`.
+          newIdentNode(subtypeName))))) # Get type from return type of this proc
+  
+  # -> iterator nameIter(): PFutureBase {.closure.} = 
+  # ->   var result: T
+  # ->   <proc_body>
+  # ->   complete(retFuture, result)
+  var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter")
+  var procBody = prc[6].processBody(retFutureSym)
+  if subtypeName != "void":
+    procBody.insert(0, newNimNode(nnkVarSection).add(
+      newIdentDefs(newIdentNode("result"), returnType[1]))) # -> var result: T
+    procBody.add(
+      newCall(newIdentNode("complete"),
+        retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result)
+  else:
+    # -> complete(retFuture)
+    procBody.add(newCall(newIdentNode("complete"), retFutureSym))
+  
+  var closureIterator = newProc(iteratorNameSym, [newIdentNode("PFutureBase")],
+                                procBody, nnkIteratorDef)
+  closureIterator[4] = newNimNode(nnkPragma).add(newIdentNode("closure"))
+  outerProcBody.add(closureIterator)
+
+  # -> var nameIterVar = nameIter
+  # -> var first = nameIterVar()
+  var varNameIterSym = genSym(nskVar, $prc[0].getName & "IterVar")
+  var varNameIter = newVarStmt(varNameIterSym, iteratorNameSym)
+  outerProcBody.add varNameIter
+  var varFirstSym = genSym(nskVar, "first")
+  var varFirst = newVarStmt(varFirstSym, newCall(varNameIterSym))
+  outerProcBody.add varFirst
+
+  # -> createCb(cb, nameIter, retFuture)
+  var cbName = newIdentNode("cb")
+  var procCb = newCall("createCb", cbName, varNameIterSym, retFutureSym)
+  outerProcBody.add procCb
+
+  # -> first.callback = cb
+  outerProcBody.add newAssignment(
+    newDotExpr(varFirstSym, newIdentNode("callback")),
+    cbName)
+
+  # -> return retFuture
+  outerProcBody.add newNimNode(nnkReturnStmt).add(retFutureSym)
+  
+  result = prc
+
+  # Remove the 'async' pragma.
+  for i in 0 .. <result[4].len:
+    if result[4][i].ident == !"async":
+      result[4].del(i)
+  if subtypeName == "void":
+    # Add discardable pragma.
+    result[4].add(newIdentNode("discardable"))
+    if returnType.kind == nnkEmpty:
+      # Add PFuture[void]
+      result[3][0] = parseExpr("PFuture[void]")
+
+  result[6] = outerProcBody
+
+  echo(toStrLit(result))
+
+proc recvLine*(socket: TAsyncFD): PFuture[string] {.async.} =
+  ## Reads a line of data from ``socket``. Returned future will complete once
+  ## a full line is read or an error occurs.
+  ##
+  ## If a full line is read ``\r\L`` is not
+  ## added to ``line``, however if solely ``\r\L`` is read then ``line``
+  ## will be set to it.
+  ## 
+  ## If the socket is disconnected, ``line`` will be set to ``""``.
+  ##
+  ## If the socket is disconnected in the middle of a line (before ``\r\L``
+  ## is read) then line will be set to ``""``.
+  ## The partial line **will be lost**.
+  
+  template addNLIfEmpty(): stmt =
+    if result.len == 0:
+      result.add("\c\L")
+
+  result = ""
+  var c = ""
+  while true:
+    c = await recv(socket, 1)
+    if c.len == 0:
+      return ""
+    if c == "\r":
+      c = await recv(socket, 1, MSG_PEEK)
+      if c.len > 0 and c == "\L":
+        discard await recv(socket, 1)
+      addNLIfEmpty()
+      return
+    elif c == "\L":
+      addNLIfEmpty()
+      return
+    add(result, c)
+
+proc runForever*() =
+  ## Begins a never ending global dispatcher poll loop.
+  while true:
+    poll()

lib/pure/asyncdispatch.nimrod.cfg

@@ -0,0 +1,3 @@
+@if nimdoc:
+  --os:linux
+@end

lib/pure/asyncio.nim

@@ -167,7 +167,7 @@ proc asyncSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM,
   result = newAsyncSocket()
   result.socket = socket(domain, typ, protocol, buffered)
   result.proto = protocol
-  if result.socket == InvalidSocket: OSError(OSLastError())
+  if result.socket == invalidSocket: osError(osLastError())
   result.socket.setBlocking(false)
 
 proc toAsyncSocket*(sock: TSocket, state: TInfo = SockConnected): PAsyncSocket =
@@ -357,7 +357,7 @@ proc acceptAddr*(server: PAsyncSocket, client: var PAsyncSocket,
     client.sslNeedAccept = false
     client.info = SockConnected
 
-  if c == InvalidSocket: SocketError(server.socket)
+  if c == invalidSocket: socketError(server.socket)
   c.setBlocking(false) # TODO: Needs to be tested.
   
   # deleg.open is set in ``toDelegate``.
@@ -481,7 +481,7 @@ proc recvLine*(s: PAsyncSocket, line: var TaintedString): bool {.deprecated.} =
   of RecvDisconnected:
     result = true
   of RecvFail:
-    s.SocketError(async = true)
+    s.socketError(async = true)
     result = false
 {.pop.}
 
@@ -615,11 +615,11 @@ proc poll*(d: PDispatcher, timeout: int = 500): bool =
     if d.hasDataBuffered(d.deleVal):
       hasDataBufferedCount.inc()
       d.handleRead(d.deleVal)
-  if hasDataBufferedCount > 0: return True
+  if hasDataBufferedCount > 0: return true
   
   if readDg.len() == 0 and writeDg.len() == 0:
     ## TODO: Perhaps this shouldn't return if errorDg has something?
-    return False
+    return false
   
   if select(readDg, writeDg, errorDg, timeout) != 0:
     for i in 0..len(d.delegates)-1:

lib/pure/asyncio2.nim

@@ -1,485 +0,0 @@
-#
-#
-#            Nimrod's Runtime Library
-#        (c) Copyright 2014 Dominik Picheta
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import os, oids, tables, strutils
-
-import winlean
-
-import sockets2, net
-
-## Asyncio2 
-## --------
-##
-## This module implements a brand new asyncio module based on Futures.
-## IOCP is used under the hood on Windows and the selectors module is used for
-## other operating systems.
-
-# -- Futures
-
-type
-  PFutureVoid* = ref object of PObject
-    cbVoid: proc () {.closure.}
-    finished: bool
-
-  PFuture*[T] = ref object of PFutureVoid
-    value: T
-    error: ref EBase
-    cb: proc (future: PFuture[T]) {.closure.}
-
-proc newFuture*[T](): PFuture[T] =
-  ## Creates a new future.
-  new(result)
-  result.finished = false
-
-proc complete*[T](future: PFuture[T], val: T) =
-  ## Completes ``future`` with value ``val``.
-  assert(not future.finished)
-  assert(future.error == nil)
-  future.value = val
-  future.finished = true
-  if future.cb != nil:
-    future.cb(future)
-  if future.cbVoid != nil:
-    future.cbVoid()
-
-proc fail*[T](future: PFuture[T], error: ref EBase) =
-  ## Completes ``future`` with ``error``.
-  assert(not future.finished)
-  future.finished = true
-  future.error = error
-  if future.cb != nil:
-    future.cb(future)
-
-proc `callback=`*[T](future: PFuture[T],
-    cb: proc (future: PFuture[T]) {.closure.}) =
-  ## Sets the callback proc to be called when the future completes.
-  ##
-  ## If future has already completed then ``cb`` will be called immediately.
-  future.cb = cb
-  if future.finished:
-    future.cb(future)
-
-proc `callbackVoid=`*(future: PFutureVoid, cb: proc () {.closure.}) =
-  ## Sets the **void** callback proc to be called when the future completes.
-  ##
-  ## If future has already completed then ``cb`` will be called immediately.
-  ##
-  ## **Note**: This is used for the ``await`` functionality, you most likely
-  ## want to use ``callback``.
-  future.cbVoid = cb
-  if future.finished:
-    future.cbVoid()
-
-proc read*[T](future: PFuture[T]): T =
-  ## Retrieves the value of ``future``. Future must be finished otherwise
-  ## this function will fail with a ``EInvalidValue`` exception.
-  ##
-  ## If the result of the future is an error then that error will be raised.
-  if future.finished:
-    if future.error != nil: raise future.error
-    return future.value
-  else:
-    # TODO: Make a custom exception type for this?
-    raise newException(EInvalidValue, "Future still in progress.")
-
-proc finished*[T](future: PFuture[T]): bool =
-  ## Determines whether ``future`` has completed.
-  ##
-  ## ``True`` may indicate an error or a value. Use ``hasError`` to distinguish.
-  future.finished
-
-proc failed*[T](future: PFuture[T]): bool =
-  ## Determines whether ``future`` completed with an error.
-  future.error != nil
-
-when defined(windows):
-  type
-    TCompletionKey = dword
-
-    TCompletionData* = object
-      sock: TSocketHandle
-      cb: proc (sock: TSocketHandle, errcode: TOSErrorCode) {.closure.}
-
-    PDispatcher* = ref object
-      ioPort: THandle
-
-    TCustomOverlapped = object
-      Internal*: DWORD
-      InternalHigh*: DWORD
-      Offset*: DWORD
-      OffsetHigh*: DWORD
-      hEvent*: THANDLE
-      data*: TCompletionData
-
-    PCustomOverlapped = ptr TCustomOverlapped
-
-  proc newDispatcher*(): PDispatcher =
-    ## Creates a new Dispatcher instance.
-    new result
-    result.ioPort = CreateIOCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1)
-
-  proc register*(p: PDispatcher, sock: TSocketHandle) =
-    ## Registers ``sock`` with the dispatcher ``p``.
-    if CreateIOCompletionPort(sock.THandle, p.ioPort,
-                              cast[TCompletionKey](sock), 1) == 0:
-      OSError(OSLastError())
-
-  proc poll*(p: PDispatcher, timeout = 500) =
-    ## Waits for completion events and processes them.
-    let llTimeout =
-      if timeout ==  -1: winlean.INFINITE
-      else: timeout.int32
-    var lpNumberOfBytesTransferred: DWORD
-    var lpCompletionKey: ULONG
-    var lpOverlapped: POverlapped
-    let res = GetQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred,
-        addr lpCompletionKey, addr lpOverlapped, llTimeout).bool
-
-    # http://stackoverflow.com/a/12277264/492186
-    # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html
-    var customOverlapped = cast[PCustomOverlapped](lpOverlapped)
-    if res:
-      assert customOverlapped.data.sock == lpCompletionKey.TSocketHandle
-
-      customOverlapped.data.cb(customOverlapped.data.sock, TOSErrorCode(-1))
-      dealloc(customOverlapped)
-    else:
-      let errCode = OSLastError()
-      if lpOverlapped != nil:
-        assert customOverlapped.data.sock == lpCompletionKey.TSocketHandle
-        dealloc(customOverlapped)
-        customOverlapped.data.cb(customOverlapped.data.sock, errCode)
-      else:
-        if errCode.int32 == WAIT_TIMEOUT:
-          # Timed out
-          discard
-        else: OSError(errCode)
-
-  var connectExPtr: pointer = nil
-  var acceptExPtr: pointer = nil
-  var getAcceptExSockAddrsPtr: pointer = nil
-
-  proc initPointer(s: TSocketHandle, func: var pointer, guid: var TGUID): bool =
-    # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c
-    var bytesRet: DWord
-    func = nil
-    result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid,
-                      sizeof(TGUID).dword, addr func, sizeof(pointer).DWORD,
-                      addr bytesRet, nil, nil) == 0
-
-  proc initAll() =
-    let dummySock = socket()
-    if not initPointer(dummySock, connectExPtr, WSAID_CONNECTEX):
-      OSError(OSLastError())
-    if not initPointer(dummySock, acceptExPtr, WSAID_ACCEPTEX):
-      OSError(OSLastError())
-    if not initPointer(dummySock, getAcceptExSockAddrsPtr, WSAID_GETACCEPTEXSOCKADDRS):
-      OSError(OSLastError())
-
-  proc connectEx(s: TSocketHandle, name: ptr TSockAddr, namelen: cint, 
-                  lpSendBuffer: pointer, dwSendDataLength: dword,
-                  lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool =
-    if connectExPtr.isNil: raise newException(EInvalidValue, "Need to initialise ConnectEx().")
-    let func =
-      cast[proc (s: TSocketHandle, name: ptr TSockAddr, namelen: cint, 
-         lpSendBuffer: pointer, dwSendDataLength: dword,
-         lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool {.stdcall.}](connectExPtr)
-
-    result = func(s, name, namelen, lpSendBuffer, dwSendDataLength, lpdwBytesSent,
-         lpOverlapped)
-
-  proc acceptEx(listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer,
-                 dwReceiveDataLength, dwLocalAddressLength,
-                 dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD,
-                 lpOverlapped: POverlapped): bool =
-    if acceptExPtr.isNil: raise newException(EInvalidValue, "Need to initialise AcceptEx().")
-    let func =
-      cast[proc (listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer,
-                 dwReceiveDataLength, dwLocalAddressLength,
-                 dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD,
-                 lpOverlapped: POverlapped): bool {.stdcall.}](acceptExPtr)
-    result = func(listenSock, acceptSock, lpOutputBuffer, dwReceiveDataLength,
-        dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived,
-        lpOverlapped)
-
-  proc getAcceptExSockaddrs(lpOutputBuffer: pointer,
-      dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: DWORD,
-      LocalSockaddr: ptr ptr TSockAddr, LocalSockaddrLength: lpint,
-      RemoteSockaddr: ptr ptr TSockAddr, RemoteSockaddrLength: lpint) =
-    if getAcceptExSockAddrsPtr.isNil:
-      raise newException(EInvalidValue, "Need to initialise getAcceptExSockAddrs().")
-
-    let func =
-      cast[proc (lpOutputBuffer: pointer,
-                 dwReceiveDataLength, dwLocalAddressLength,
-                 dwRemoteAddressLength: DWORD, LocalSockaddr: ptr ptr TSockAddr,
-                 LocalSockaddrLength: lpint, RemoteSockaddr: ptr ptr TSockAddr,
-                RemoteSockaddrLength: lpint) {.stdcall.}](getAcceptExSockAddrsPtr)
-    
-    func(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength,
-                  dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength,
-                  RemoteSockaddr, RemoteSockaddrLength)
-
-  proc connect*(p: PDispatcher, socket: TSocketHandle, address: string, port: TPort,
-    af = AF_INET): PFuture[int] =
-    ## Connects ``socket`` to server at ``address:port``.
-    ##
-    ## Returns a ``PFuture`` which will complete when the connection succeeds
-    ## or an error occurs.
-
-    var retFuture = newFuture[int]()# TODO: Change to void when that regression is fixed.
-    # Apparently ``ConnectEx`` expects the socket to be initially bound:
-    var saddr: Tsockaddr_in
-    saddr.sin_family = int16(toInt(af))
-    saddr.sin_port = 0
-    saddr.sin_addr.s_addr = INADDR_ANY
-    if bindAddr(socket, cast[ptr TSockAddr](addr(saddr)),
-                  sizeof(saddr).TSockLen) < 0'i32:
-      OSError(OSLastError())
-
-    var aiList = getAddrInfo(address, port, af)
-    var success = false
-    var lastError: TOSErrorCode
-    var it = aiList
-    while it != nil:
-      # "the OVERLAPPED structure must remain valid until the I/O completes"
-      # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx
-      var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
-      ol.data = TCompletionData(sock: socket, cb:
-        proc (sock: TSocketHandle, errcode: TOSErrorCode) =
-          if errcode == TOSErrorCode(-1):
-            retFuture.complete(0)
-          else:
-            retFuture.fail(newException(EOS, osErrorMsg(errcode)))
-      )
-      
-      var ret = connectEx(socket, it.ai_addr, sizeof(TSockAddrIn).cint,
-                          nil, 0, nil, cast[POverlapped](ol))
-      if ret:
-        # Request to connect completed immediately.
-        success = true
-        retFuture.complete(0)
-        dealloc(ol)
-        break
-      else:
-        lastError = OSLastError()
-        if lastError.int32 == ERROR_IO_PENDING:
-          # In this case ``ol`` will be deallocated in ``poll``.
-          success = true
-          break
-        else:
-          dealloc(ol)
-          success = false
-      it = it.ai_next
-
-    dealloc(aiList)
-    if not success:
-      retFuture.fail(newException(EOS, osErrorMsg(lastError)))
-    return retFuture
-
-  proc recv*(p: PDispatcher, socket: TSocketHandle, size: int): PFuture[string] =
-    ## Reads ``size`` bytes from ``socket``. Returned future will complete once
-    ## all of the requested data is read.
-
-    var retFuture = newFuture[string]()
-    
-    var dataBuf: TWSABuf
-    dataBuf.buf = newString(size)
-    dataBuf.len = size
-    
-    var bytesReceived, flags: DWord
-    var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
-    ol.data = TCompletionData(sock: socket, cb:
-      proc (sock: TSocketHandle, errcode: TOSErrorCode) =
-        if errcode == TOSErrorCode(-1):
-          var data = newString(size)
-          copyMem(addr data[0], addr dataBuf.buf[0], size)
-          retFuture.complete($data)
-        else:
-          retFuture.fail(newException(EOS, osErrorMsg(errcode)))
-    )
-    
-    let ret = WSARecv(socket, addr dataBuf, 1, addr bytesReceived,
-                      addr flags, cast[POverlapped](ol), nil)
-    if ret == -1:
-      let err = OSLastError()
-      if err.int32 != ERROR_IO_PENDING:
-        retFuture.fail(newException(EOS, osErrorMsg(err)))
-        dealloc(ol)
-    else:
-      # Request to read completed immediately.
-      var data = newString(size)
-      copyMem(addr data[0], addr dataBuf.buf[0], size)
-      retFuture.complete($data)
-      dealloc(ol)
-    return retFuture
-
-  proc send*(p: PDispatcher, socket: TSocketHandle, data: string): PFuture[int] =
-    ## Sends ``data`` to ``socket``. The returned future will complete once all
-    ## data has been sent.
-    var retFuture = newFuture[int]()
-
-    var dataBuf: TWSABuf
-    dataBuf.buf = data
-    dataBuf.len = data.len
-
-    var bytesReceived, flags: DWord
-    var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
-    ol.data = TCompletionData(sock: socket, cb:
-      proc (sock: TSocketHandle, errcode: TOSErrorCode) =
-        if errcode == TOSErrorCode(-1):
-          retFuture.complete(0)
-        else:
-          retFuture.fail(newException(EOS, osErrorMsg(errcode)))
-    )
-
-    let ret = WSASend(socket, addr dataBuf, 1, addr bytesReceived,
-                      flags, cast[POverlapped](ol), nil)
-    if ret == -1:
-      let err = osLastError()
-      if err.int32 != ERROR_IO_PENDING:
-        retFuture.fail(newException(EOS, osErrorMsg(err)))
-        dealloc(ol)
-    else:
-      retFuture.complete(0)
-      dealloc(ol)
-    return retFuture
-
-  proc acceptAddr*(p: PDispatcher, socket: TSocketHandle): 
-      PFuture[tuple[address: string, client: TSocketHandle]] =
-    ## Accepts a new connection. Returns a future containing the client socket
-    ## corresponding to that connection and the remote address of the client.
-    ## The future will complete when the connection is successfully accepted.
-    
-    var retFuture = newFuture[tuple[address: string, client: TSocketHandle]]()
-
-    var clientSock = socket()
-    if clientSock == OSInvalidSocket: osError(osLastError())
-
-    const lpOutputLen = 1024
-    var lpOutputBuf = newString(lpOutputLen)
-    var dwBytesReceived: DWORD
-    let dwReceiveDataLength = 0.DWORD # We don't want any data to be read.
-    let dwLocalAddressLength = DWORD(sizeof (TSockaddr_in) + 16)
-    let dwRemoteAddressLength = DWORD(sizeof(TSockaddr_in) + 16)
-
-    template completeAccept(): stmt {.immediate, dirty.} =
-      var listenSock = socket
-      let setoptRet = setsockopt(clientSock, SOL_SOCKET,
-          SO_UPDATE_ACCEPT_CONTEXT, addr listenSock,
-          sizeof(listenSock).TSockLen)
-      if setoptRet != 0: osError(osLastError())
-
-      var LocalSockaddr, RemoteSockaddr: ptr TSockAddr
-      var localLen, remoteLen: int32
-      getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength,
-                           dwLocalAddressLength, dwRemoteAddressLength,
-                           addr LocalSockaddr, addr localLen,
-                           addr RemoteSockaddr, addr remoteLen)
-      # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186
-      retFuture.complete(
-        (address: $inet_ntoa(cast[ptr Tsockaddr_in](remoteSockAddr).sin_addr),
-         client: clientSock)
-      )
-
-    var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
-    ol.data = TCompletionData(sock: socket, cb:
-      proc (sock: TSocketHandle, errcode: TOSErrorCode) =
-        if errcode == TOSErrorCode(-1):
-          completeAccept()
-        else:
-          retFuture.fail(newException(EOS, osErrorMsg(errcode)))
-    )
-
-    # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx
-    let ret = acceptEx(socket, clientSock, addr lpOutputBuf[0],
-                       dwReceiveDataLength, 
-                       dwLocalAddressLength,
-                       dwRemoteAddressLength,
-                       addr dwBytesReceived, cast[POverlapped](ol))
-
-    if not ret:
-      let err = osLastError()
-      if err.int32 != ERROR_IO_PENDING:
-        retFuture.fail(newException(EOS, osErrorMsg(err)))
-        dealloc(ol)
-    else:
-      completeAccept()
-      dealloc(ol)
-
-    return retFuture
-
-  proc accept*(p: PDispatcher, socket: TSocketHandle): PFuture[TSocketHandle] =
-    ## Accepts a new connection. Returns a future containing the client socket
-    ## corresponding to that connection.
-    ## The future will complete when the connection is successfully accepted.
-    var retFut = newFuture[TSocketHandle]()
-    var fut = p.acceptAddr(socket)
-    fut.callback =
-      proc (future: PFuture[tuple[address: string, client: TSocketHandle]]) =
-        assert future.finished
-        if future.failed:
-          retFut.fail(future.error)
-        else:
-          retFut.complete(future.read.client)
-    return retFut
-
-  initAll()
-else:
-  # TODO: Selectors.
-
-
-when isMainModule:
-  
-  var p = newDispatcher()
-  var sock = socket()
-  #sock.setBlocking false
-  p.register(sock)
-
-  when true:
-
-    var f = p.connect(sock, "irc.freenode.org", TPort(6667))
-    f.callback =
-      proc (future: PFuture[int]) =
-        echo("Connected in future!")
-        echo(future.read)
-        for i in 0 .. 50:
-          var recvF = p.recv(sock, 10)
-          recvF.callback =
-            proc (future: PFuture[string]) =
-              echo("Read: ", future.read)
-
-  else:
-
-    sock.bindAddr(TPort(6667))
-    sock.listen()
-    proc onAccept(future: PFuture[TSocketHandle]) =
-      echo "Accepted"
-      var t = p.send(future.read, "test\c\L")
-      t.callback =
-        proc (future: PFuture[int]) =
-          echo(future.read)
-      
-      var f = p.accept(sock)
-      f.callback = onAccept
-      
-    var f = p.accept(sock)
-    f.callback = onAccept
-  
-  while true:
-    p.poll()
-    echo "polled"
-
-
-
-
-
-  
-
-  

lib/pure/asyncnet.nim

@@ -0,0 +1,195 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2014 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+import asyncdispatch
+import rawsockets
+import net
+
+when defined(ssl):
+  import openssl
+
+type
+  # TODO: I would prefer to just do:
+  # PAsyncSocket* {.borrow: `.`.} = distinct PSocket. But that doesn't work.
+  TAsyncSocket {.borrow: `.`.} = distinct TSocketImpl
+  PAsyncSocket* = ref TAsyncSocket
+
+# TODO: Save AF, domain etc info and reuse it in procs which need it like connect.
+
+proc newSocket(fd: TAsyncFD, isBuff: bool): PAsyncSocket =
+  assert fd != osInvalidSocket.TAsyncFD
+  new(result.PSocket)
+  result.fd = fd.TSocketHandle
+  result.isBuffered = isBuff
+  if isBuff:
+    result.currPos = 0
+
+proc newAsyncSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM,
+    protocol: TProtocol = IPPROTO_TCP, buffered = true): PAsyncSocket =
+  ## Creates a new asynchronous socket.
+  result = newSocket(newAsyncRawSocket(domain, typ, protocol), buffered)
+
+proc connect*(socket: PAsyncSocket, address: string, port: TPort,
+    af = AF_INET): PFuture[void] =
+  ## Connects ``socket`` to server at ``address:port``.
+  ##
+  ## Returns a ``PFuture`` which will complete when the connection succeeds
+  ## or an error occurs.
+  result = connect(socket.fd.TAsyncFD, address, port, af)
+
+proc recv*(socket: PAsyncSocket, size: int,
+           flags: int = 0): PFuture[string] =
+  ## Reads ``size`` bytes from ``socket``. Returned future will complete once
+  ## all of the requested data is read. If socket is disconnected during the
+  ## recv operation then the future may complete with only a part of the
+  ## requested data read. If socket is disconnected and no data is available
+  ## to be read then the future will complete with a value of ``""``.
+  result = recv(socket.fd.TAsyncFD, size, flags)
+
+proc send*(socket: PAsyncSocket, data: string): PFuture[void] =
+  ## Sends ``data`` to ``socket``. The returned future will complete once all
+  ## data has been sent.
+  result = send(socket.fd.TAsyncFD, data)
+
+proc acceptAddr*(socket: PAsyncSocket): 
+      PFuture[tuple[address: string, client: PAsyncSocket]] =
+  ## Accepts a new connection. Returns a future containing the client socket
+  ## corresponding to that connection and the remote address of the client.
+  ## The future will complete when the connection is successfully accepted.
+  var retFuture = newFuture[tuple[address: string, client: PAsyncSocket]]()
+  var fut = acceptAddr(socket.fd.TAsyncFD)
+  fut.callback =
+    proc (future: PFuture[tuple[address: string, client: TAsyncFD]]) =
+      assert future.finished
+      if future.failed:
+        retFuture.fail(future.readError)
+      else:
+        let resultTup = (future.read.address,
+                         newSocket(future.read.client, socket.isBuffered))
+        retFuture.complete(resultTup)
+  return retFuture
+
+proc accept*(socket: PAsyncSocket): PFuture[PAsyncSocket] =
+  ## Accepts a new connection. Returns a future containing the client socket
+  ## corresponding to that connection.
+  ## The future will complete when the connection is successfully accepted.
+  var retFut = newFuture[PAsyncSocket]()
+  var fut = acceptAddr(socket)
+  fut.callback =
+    proc (future: PFuture[tuple[address: string, client: PAsyncSocket]]) =
+      assert future.finished
+      if future.failed:
+        retFut.fail(future.readError)
+      else:
+        retFut.complete(future.read.client)
+  return retFut
+
+proc recvLine*(socket: PAsyncSocket): PFuture[string] {.async.} =
+  ## Reads a line of data from ``socket``. Returned future will complete once
+  ## a full line is read or an error occurs.
+  ##
+  ## If a full line is read ``\r\L`` is not
+  ## added to ``line``, however if solely ``\r\L`` is read then ``line``
+  ## will be set to it.
+  ## 
+  ## If the socket is disconnected, ``line`` will be set to ``""``.
+  ##
+  ## If the socket is disconnected in the middle of a line (before ``\r\L``
+  ## is read) then line will be set to ``""``.
+  ## The partial line **will be lost**.
+  
+  template addNLIfEmpty(): stmt =
+    if result.len == 0:
+      result.add("\c\L")
+
+  result = ""
+  var c = ""
+  while true:
+    c = await recv(socket, 1)
+    if c.len == 0:
+      return ""
+    if c == "\r":
+      c = await recv(socket, 1, MSG_PEEK)
+      if c.len > 0 and c == "\L":
+        discard await recv(socket, 1)
+      addNLIfEmpty()
+      return
+    elif c == "\L":
+      addNLIfEmpty()
+      return
+    add(result.string, c)
+
+proc bindAddr*(socket: PAsyncSocket, port = TPort(0), address = "") =
+  ## Binds ``address``:``port`` to the socket.
+  ##
+  ## If ``address`` is "" then ADDR_ANY will be bound.
+  socket.PSocket.bindAddr(port, address)
+
+proc listen*(socket: PAsyncSocket, backlog = SOMAXCONN) =
+  ## Marks ``socket`` as accepting connections.
+  ## ``Backlog`` specifies the maximum length of the
+  ## queue of pending connections.
+  ##
+  ## Raises an EOS error upon failure.
+  socket.PSocket.listen(backlog)
+
+proc close*(socket: PAsyncSocket) =
+  ## Closes the socket.
+  socket.fd.TAsyncFD.close()
+  # TODO SSL
+
+when isMainModule:
+  type
+    TestCases = enum
+      HighClient, LowClient, LowServer
+
+  const test = LowServer
+
+  when test == HighClient:
+    proc main() {.async.} =
+      var sock = newAsyncSocket()
+      await sock.connect("irc.freenode.net", TPort(6667))
+      while true:
+        let line = await sock.recvLine()
+        if line == "":
+          echo("Disconnected")
+          break
+        else:
+          echo("Got line: ", line)
+    main()
+  elif test == LowClient:
+    var sock = newAsyncSocket()
+    var f = connect(sock, "irc.freenode.net", TPort(6667))
+    f.callback =
+      proc (future: PFuture[void]) =
+        echo("Connected in future!")
+        for i in 0 .. 50:
+          var recvF = recv(sock, 10)
+          recvF.callback =
+            proc (future: PFuture[string]) =
+              echo("Read ", future.read.len, ": ", future.read.repr)
+  elif test == LowServer:
+    var sock = newAsyncSocket()
+    sock.bindAddr(TPort(6667))
+    sock.listen()
+    proc onAccept(future: PFuture[PAsyncSocket]) =
+      let client = future.read
+      echo "Accepted ", client.fd.cint
+      var t = send(client, "test\c\L")
+      t.callback =
+        proc (future: PFuture[void]) =
+          echo("Send")
+          client.close()
+      
+      var f = accept(sock)
+      f.callback = onAccept
+      
+    var f = accept(sock)
+    f.callback = onAccept
+  runForever()
+    

lib/pure/dynlib.nim

@@ -14,7 +14,7 @@
 type
   TLibHandle* = pointer ## a handle to a dynamically loaded library
 
-proc loadLib*(path: string): TLibHandle
+proc loadLib*(path: string, global_symbols=false): TLibHandle
   ## loads a library from `path`. Returns nil if the library could not 
   ## be loaded.
 
@@ -53,6 +53,7 @@ when defined(posix):
   #
   var
     RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: int
+    RTLD_GLOBAL {.importc: "RTLD_GLOBAL", header: "<dlfcn.h>".}: int
 
   proc dlclose(lib: TLibHandle) {.importc, header: "<dlfcn.h>".}
   proc dlopen(path: CString, mode: int): TLibHandle {.
@@ -60,7 +61,10 @@ when defined(posix):
   proc dlsym(lib: TLibHandle, name: cstring): pointer {.
       importc, header: "<dlfcn.h>".}
 
-  proc loadLib(path: string): TLibHandle = return dlopen(path, RTLD_NOW)
+  proc loadLib(path: string, global_symbols=false): TLibHandle = 
+    var flags = RTLD_NOW
+    if global_symbols: flags = flags or RTLD_GLOBAL
+    return dlopen(path, flags)
   proc loadLib(): TLibHandle = return dlopen(nil, RTLD_NOW)
   proc unloadLib(lib: TLibHandle) = dlclose(lib)
   proc symAddr(lib: TLibHandle, name: cstring): pointer = 
@@ -81,14 +85,14 @@ elif defined(windows) or defined(dos):
   proc getProcAddress(lib: THINSTANCE, name: cstring): pointer {.
       importc: "GetProcAddress", header: "<windows.h>", stdcall.}
 
-  proc loadLib(path: string): TLibHandle =
+  proc loadLib(path: string, global_symbols=false): TLibHandle =
     result = cast[TLibHandle](winLoadLibrary(path))
   proc loadLib(): TLibHandle =
     result = cast[TLibHandle](winLoadLibrary(nil))
   proc unloadLib(lib: TLibHandle) = FreeLibrary(cast[THINSTANCE](lib))
 
   proc symAddr(lib: TLibHandle, name: cstring): pointer =
-    result = GetProcAddress(cast[THINSTANCE](lib), name)
+    result = getProcAddress(cast[THINSTANCE](lib), name)
 
 else:
   {.error: "no implementation for dynlib".}

lib/pure/htmlgen.nim

@@ -1,12 +1,17 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2014 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
 
+## **Warning**: This module uses ``immediate`` macros which are known to
+## cause problems. Do yourself a favor and import the module
+## as ``from htmlgen import nil`` and then fully qualify the macros.
+##
+##
 ## This module implements a simple `XML`:idx: and `HTML`:idx: code 
 ## generator. Each commonly used HTML tag has a corresponding macro
 ## that generates a string with its HTML representation.
@@ -15,11 +20,11 @@
 ##
 ## .. code-block:: nimrod
 ##   var nim = "Nimrod"
-##   echo h1(a(href="http://nimrod-code.org", nim))
+##   echo h1(a(href="http://nimrod-lang.org", nim))
 ##  
 ## Writes the string::
 ##   
-##   <h1><a href="http://nimrod-code.org">Nimrod</a></h1>
+##   <h1><a href="http://nimrod-lang.org">Nimrod</a></h1>
 ##
 
 import

lib/pure/httpclient.nim

@@ -76,6 +76,8 @@
 ## currently only basic authentication is supported.
 
 import sockets, strutils, parseurl, parseutils, strtabs, base64
+import asyncnet, asyncdispatch
+import rawsockets
 
 type
   TResponse* = tuple[
@@ -286,16 +288,16 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "",
   add(headers, "\c\L")
   
   var s = socket()
-  var port = TPort(80)
+  var port = sockets.TPort(80)
   if r.scheme == "https":
     when defined(ssl):
       sslContext.wrapSocket(s)
-      port = TPort(443)
+      port = sockets.TPort(443)
     else:
       raise newException(EHttpRequestErr,
                 "SSL support is not available. Cannot connect over SSL.")
   if r.port != "":
-    port = TPort(r.port.parseInt)
+    port = sockets.TPort(r.port.parseInt)
   
   if timeout == -1:
     s.connect(r.hostname, port)
@@ -413,28 +415,217 @@ proc downloadFile*(url: string, outputFilename: string,
   else:
     fileError("Unable to open file")
 
+proc generateHeaders(r: TURL, httpMethod: THttpMethod,
+                     headers: PStringTable): string =
+  result = substr($httpMethod, len("http"))
+  # TODO: Proxies
+  result.add(" /" & r.path & r.query)
+  result.add(" HTTP/1.1\c\L")
+
+  add(result, "Host: " & r.hostname & "\c\L")
+  add(result, "Connection: Keep-Alive\c\L")
+  for key, val in headers:
+    add(result, key & ": " & val & "\c\L")
+
+  add(result, "\c\L")
+
+type
+  PAsyncHttpClient = ref object
+    socket: PAsyncSocket
+    connected: bool
+    currentURL: TURL ## Where we are currently connected.
+    headers: PStringTable
+    userAgent: string
+
+proc newAsyncHttpClient*(): PAsyncHttpClient =
+  new result
+  result.socket = newAsyncSocket()
+  result.headers = newStringTable(modeCaseInsensitive)
+  result.userAgent = defUserAgent
+
+proc parseChunks(client: PAsyncHttpClient): PFuture[string] {.async.} =
+  result = ""
+  var ri = 0
+  while true:
+    var chunkSize = 0
+    var chunkSizeStr = await client.socket.recvLine()
+    var i = 0
+    if chunkSizeStr == "":
+      httpError("Server terminated connection prematurely")
+    while true:
+      case chunkSizeStr[i]
+      of '0'..'9':
+        chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0'))
+      of 'a'..'f':
+        chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10)
+      of 'A'..'F':
+        chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10)
+      of '\0':
+        break
+      of ';':
+        # http://tools.ietf.org/html/rfc2616#section-3.6.1
+        # We don't care about chunk-extensions.
+        break
+      else:
+        httpError("Invalid chunk size: " & chunkSizeStr)
+      inc(i)
+    if chunkSize <= 0: break
+    result.add await recv(client.socket, chunkSize)
+    discard await recv(client.socket, 2) # Skip \c\L
+    # Trailer headers will only be sent if the request specifies that we want
+    # them: http://tools.ietf.org/html/rfc2616#section-3.6.1
+  
+proc parseBody(client: PAsyncHttpClient,
+               headers: PStringTable): PFuture[string] {.async.} =
+  result = ""
+  if headers["Transfer-Encoding"] == "chunked":
+    result = await parseChunks(client)
+  else:
+    # -REGION- Content-Length
+    # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.3
+    var contentLengthHeader = headers["Content-Length"]
+    if contentLengthHeader != "":
+      var length = contentLengthHeader.parseint()
+      result = await client.socket.recv(length)
+      if result == "":
+        httpError("Got disconnected while trying to recv body.")
+    else:
+      # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO
+      
+      # -REGION- Connection: Close
+      # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5
+      if headers["Connection"] == "close":
+        var buf = ""
+        while True:
+          buf = await client.socket.recv(4000)
+          if buf == "": break
+          result.add(buf)
+
+proc parseResponse(client: PAsyncHttpClient,
+                   getBody: bool): PFuture[TResponse] {.async.} =
+  var parsedStatus = false
+  var linei = 0
+  var fullyRead = false
+  var line = ""
+  result.headers = newStringTable(modeCaseInsensitive)
+  while True:
+    linei = 0
+    line = await client.socket.recvLine()
+    if line == "": break # We've been disconnected.
+    if line == "\c\L":
+      fullyRead = true
+      break
+    if not parsedStatus:
+      # Parse HTTP version info and status code.
+      var le = skipIgnoreCase(line, "HTTP/", linei)
+      if le <= 0: httpError("invalid http version")
+      inc(linei, le)
+      le = skipIgnoreCase(line, "1.1", linei)
+      if le > 0: result.version = "1.1"
+      else:
+        le = skipIgnoreCase(line, "1.0", linei)
+        if le <= 0: httpError("unsupported http version")
+        result.version = "1.0"
+      inc(linei, le)
+      # Status code
+      linei.inc skipWhitespace(line, linei)
+      result.status = line[linei .. -1]
+      parsedStatus = true
+    else:
+      # Parse headers
+      var name = ""
+      var le = parseUntil(line, name, ':', linei)
+      if le <= 0: httpError("invalid headers")
+      inc(linei, le)
+      if line[linei] != ':': httpError("invalid headers")
+      inc(linei) # Skip :
+      
+      result.headers[name] = line[linei.. -1].strip()
+  if not fullyRead:
+    httpError("Connection was closed before full request has been made")
+  if getBody:
+    result.body = await parseBody(client, result.headers)
+  else:
+    result.body = ""
+
+proc close*(client: PAsyncHttpClient) =
+  ## Closes any connections held by the HttpClient.
+  if client.connected:
+    client.socket.close()
+    client.connected = false
+    #client.socket = newAsyncSocket()
+
+proc newConnection(client: PAsyncHttpClient, url: TURL) {.async.} =
+  if not client.connected or client.currentURL.hostname != url.hostname or
+      client.currentURL.scheme != url.scheme:
+    if client.connected: client.close()
+    if url.scheme == "https":
+      assert false, "TODO SSL"
+
+    # TODO: I should be able to write 'net.TPort' here...
+    let port =
+      if url.port == "": rawsockets.TPort(80)
+      else: rawsockets.TPort(url.port.parseInt)
+    
+    await client.socket.connect(url.hostname, port)
+    client.currentURL = url
+
+proc request*(client: PAsyncHttpClient, url: string, httpMethod = httpGET,
+              body = ""): PFuture[TResponse] {.async.} =
+  let r = parseUrl(url)
+  await newConnection(client, r)
+
+  if not client.headers.hasKey("user-agent") and client.userAgent != "":
+    client.headers["User-Agent"] = client.userAgent
+  
+  var headers = generateHeaders(r, httpMethod, client.headers)
+  
+  await client.socket.send(headers)
+  if body != "":
+    await client.socket.send(body)
+  
+  result = await parseResponse(client, httpMethod != httpHEAD)
 
 when isMainModule:
-  #downloadFile("http://force7.de/nimrod/index.html", "nimrodindex.html")
+  when true:
-  #downloadFile("http://www.httpwatch.com/", "ChunkTest.html")
+    # Async
-  #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com",
+    proc main() {.async.} =
-  # "validator.html")
+      var client = newAsyncHttpClient()
+      var resp = await client.request("http://picheta.me")
       
-  #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com&
+      echo("Got response: ", resp.status)
-  #  charset=%28detect+automatically%29&doctype=Inline&group=0")
+      echo("Body:\n")
+      echo(resp.body)
 
-  var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L"
+      #var resp1 = await client.request("http://freenode.net")
-  var body: string = "--xyz\c\L"
+      #echo("Got response: ", resp1.status)
-  # soap 1.2 output
-  body.add("Content-Disposition: form-data; name=\"output\"\c\L")
-  body.add("\c\Lsoap12\c\L")
 
-  # html
+      var resp2 = await client.request("http://picheta.me/aasfasgf.html")
-  body.add("--xyz\c\L")
+      echo("Got response: ", resp2.status)
-  body.add("Content-Disposition: form-data; name=\"uploaded_file\";" &
+    main()
-           " filename=\"test.html\"\c\L")
+    runForever()
-  body.add("Content-Type: text/html\c\L")
-  body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L")
-  body.add("--xyz--")
 
-  echo(postContent("http://validator.w3.org/check", headers, body))
+  else:
+    #downloadFile("http://force7.de/nimrod/index.html", "nimrodindex.html")
+    #downloadFile("http://www.httpwatch.com/", "ChunkTest.html")
+    #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com",
+    # "validator.html")
+
+    #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com&
+    #  charset=%28detect+automatically%29&doctype=Inline&group=0")
+    
+    var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L"
+    var body: string = "--xyz\c\L"
+    # soap 1.2 output
+    body.add("Content-Disposition: form-data; name=\"output\"\c\L")
+    body.add("\c\Lsoap12\c\L")
+    
+    # html
+    body.add("--xyz\c\L")
+    body.add("Content-Disposition: form-data; name=\"uploaded_file\";" &
+             " filename=\"test.html\"\c\L")
+    body.add("Content-Type: text/html\c\L")
+    body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L")
+    body.add("--xyz--")
+
+    echo(postContent("http://validator.w3.org/check", headers, body))

lib/pure/math.nim

@@ -20,6 +20,8 @@
 when defined(Posix) and not defined(haiku):
   {.passl: "-lm".}
 
+import times
+
 const
   PI* = 3.1415926535897932384626433 ## the circle constant PI (Ludolph's number)
   E* = 2.71828182845904523536028747 ## Euler's number
@@ -201,7 +203,7 @@ when not defined(JS):
       result = drand48() * max
     
   proc randomize() =
-    randomize(gettime(nil))
+    randomize(cast[int](epochTime()))
 
   proc randomize(seed: int) =
     srand(cint(seed))

lib/pure/memfiles.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Nimrod Contributors
+#        (c) Copyright 2014 Nimrod Contributors
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -34,6 +34,43 @@ type
     else:
       handle: cint
 
+
+proc mapMem*(m: var TMemFile, mode: TFileMode = fmRead,
+             mappedSize = -1, offset = 0): pointer =
+  var readonly = mode == fmRead
+  when defined(windows):
+    result = mapViewOfFileEx(
+      m.mapHandle,
+      if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
+      int32(offset shr 32),
+      int32(offset and 0xffffffff),
+      if mappedSize == -1: 0 else: mappedSize,
+      nil)
+    if result == nil:
+      osError(osLastError())
+  else:
+    assert mappedSize > 0
+    result = mmap(
+      nil,
+      mappedSize,
+      if readonly: PROT_READ else: PROT_READ or PROT_WRITE,
+      if readonly: MAP_PRIVATE else: MAP_SHARED,
+      m.handle, offset)
+    if result == cast[pointer](MAP_FAILED):
+      osError(osLastError())
+
+
+proc unmapMem*(f: var TMemFile, p: pointer, size: int) =
+  ## unmaps the memory region ``(p, <p+size)`` of the mapped file `f`.
+  ## All changes are written back to the file system, if `f` was opened
+  ## with write access. ``size`` must be of exactly the size that was requested
+  ## via ``mapMem``.
+  when defined(windows):
+    if unmapViewOfFile(p) == 0: osError(osLastError())
+  else:
+    if munmap(p, size) != 0: osError(osLastError())
+
+
 proc open*(filename: string, mode: TFileMode = fmRead,
            mappedSize = -1, offset = 0, newFileSize = -1): TMemFile =
   ## opens a memory mapped file. If this fails, ``EOS`` is raised.
@@ -71,7 +108,7 @@ proc open*(filename: string, mode: TFileMode = fmRead,
     when useWinUnicode:
       result.fHandle = callCreateFile(createFileW, newWideCString(filename))
     else:
-      result.fHandle = callCreateFile(CreateFileA, filename)
+      result.fHandle = callCreateFile(createFileA, filename)
 
     if result.fHandle == INVALID_HANDLE_VALUE:
       fail(osLastError(), "error opening file")
@@ -170,14 +207,14 @@ proc close*(f: var TMemFile) =
 
   when defined(windows):
     if f.fHandle != INVALID_HANDLE_VALUE:
-      lastErr = osLastError()
       error = unmapViewOfFile(f.mem) == 0
+      lastErr = osLastError()
       error = (closeHandle(f.mapHandle) == 0) or error
       error = (closeHandle(f.fHandle) == 0) or error
   else:
     if f.handle != 0:
-      lastErr = osLastError()
       error = munmap(f.mem, f.size) != 0
+      lastErr = osLastError()
       error = (close(f.handle) != 0) or error
 
   f.size = 0

lib/pure/nimprof.nim

@@ -67,7 +67,7 @@ when withThreads:
 
 proc hookAux(st: TStackTrace, costs: int) =
   # this is quite performance sensitive!
-  when withThreads: Acquire profilingLock
+  when withThreads: acquire profilingLock
   inc totalCalls
   var last = high(st)
   while last > 0 and isNil(st[last]): dec last
@@ -106,7 +106,7 @@ proc hookAux(st: TStackTrace, costs: int) =
       h = ((5 * h) + 1) and high(profileData)
       inc chain
     maxChainLen = max(maxChainLen, chain)
-  when withThreads: Release profilingLock
+  when withThreads: release profilingLock
 
 when defined(memProfiler):
   const

lib/pure/os.nim

@@ -260,11 +260,12 @@ proc osError*(errorCode: TOSErrorCode) =
   ##
   ## If the error code is ``0`` or an error message could not be retrieved,
   ## the message ``unknown OS error`` will be used.
-  let msg = osErrorMsg(errorCode)
+  var e: ref EOS; new(e)
-  if msg == "":
+  e.errorCode = errorCode.int32
-    raise newException(EOS, "unknown OS error")
+  e.msg = osErrorMsg(errorCode)
-  else:
+  if e.msg == "":
-    raise newException(EOS, msg)
+    e.msg = "unknown OS error"
+  raise e
 
 {.push stackTrace:off.}
 proc osLastError*(): TOSErrorCode =
@@ -1037,7 +1038,10 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1",
   ## the process has finished. To execute a program without having a
   ## shell involved, use the `execProcess` proc of the `osproc`
   ## module.
-  result = c_system(command) shr 8
+  when defined(linux):
+    result = c_system(command) shr 8
+  else:
+    result = c_system(command)
 
 # Environment handling cannot be put into RTL, because the ``envPairs``
 # iterator depends on ``environment``.
@@ -1189,7 +1193,8 @@ iterator walkFiles*(pattern: string): string {.tags: [FReadDir].} =
     res = findFirstFile(pattern, f)
     if res != -1:
       while true:
-        if not skipFindData(f):
+        if not skipFindData(f) and
+            (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) == 0'i32:
           yield splitFile(pattern).dir / extractFilename(getFilename(f))
         if findNextFile(res, f) == 0'i32: break
       findClose(res)

lib/pure/osproc.nim

@@ -13,13 +13,16 @@
 include "system/inclrtl"
 
 import
-  strutils, os, strtabs, streams, sequtils
+  strutils, os, strtabs, streams
 
 when defined(windows):
   import winlean
 else:
   import posix
 
+when defined(linux):
+  import linux
+
 type
   TProcess = object of TObject
     when defined(windows):
@@ -44,7 +47,7 @@ type
     poStdErrToStdOut,    ## merge stdout and stderr to the stdout stream
     poParentStreams      ## use the parent's streams
 
-template poUseShell*: TProcessOption {.deprecated.} = poUsePath
+const poUseShell* {.deprecated.} = poUsePath
   ## Deprecated alias for poUsePath.
 
 proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} =
@@ -165,6 +168,9 @@ proc processID*(p: PProcess): int {.rtl, extern: "nosp$1".} =
 proc waitForExit*(p: PProcess, timeout: int = -1): int {.rtl,
   extern: "nosp$1", tags: [].}
   ## waits for the process to finish and returns `p`'s error code.
+  ##
+  ## **Warning**: Be careful when using waitForExit for processes created without
+  ## poParentStreams because they may fill output buffers, causing deadlock.
 
 proc peekExitCode*(p: PProcess): int {.tags: [].}
   ## return -1 if the process is still running. Otherwise the process' exit code
@@ -590,6 +596,24 @@ elif not defined(useNimRtl):
       copyMem(result[i], addr(x[0]), x.len+1)
       inc(i)
 
+  type TStartProcessData = object
+    sysCommand: cstring
+    sysArgs: cstringArray
+    sysEnv: cstringArray
+    workingDir: cstring
+    pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint]
+    optionPoUsePath: bool
+    optionPoParentStreams: bool
+    optionPoStdErrToStdOut: bool
+
+  when not defined(useFork):
+    proc startProcessAuxSpawn(data: TStartProcessData): TPid {.tags: [FExecIO, FReadEnv].}
+  proc startProcessAuxFork(data: TStartProcessData): TPid {.tags: [FExecIO, FReadEnv].}
+  {.push stacktrace: off, profiler: off.}
+  proc startProcessAfterFork(data: ptr TStartProcessData) {.
+    tags: [FExecIO, FReadEnv], cdecl.}
+  {.pop.}
+
   proc startProcess(command: string,
                  workingDir: string = "",
                  args: openArray[string] = [],
@@ -604,100 +628,49 @@ elif not defined(useNimRtl):
          pipe(pStderr) != 0'i32:
         osError(osLastError())
 
-    var sys_command: string
+    var sysCommand: string
-    var sys_args_raw: seq[string]
+    var sysArgsRaw: seq[string]
     if poEvalCommand in options:
-      sys_command = "/bin/sh"
+      sysCommand = "/bin/sh"
-      sys_args_raw = @[sys_command, "-c", command]
+      sysArgsRaw = @[sysCommand, "-c", command]
       assert args.len == 0
     else:
-      sys_command = command
+      sysCommand = command
-      sys_args_raw = @[command]
+      sysArgsRaw = @[command]
       for arg in args.items:
-        sys_args_raw.add arg
+        sysArgsRaw.add arg
-
-    var sys_args = allocCStringArray(sys_args_raw)
-    finally: deallocCStringArray(sys_args)
 
     var pid: TPid
-    when defined(posix_spawn) and not defined(useFork):
-      var attr: Tposix_spawnattr
-      var fops: Tposix_spawn_file_actions
 
-      template chck(e: expr) =
+    var sysArgs = allocCStringArray(sysArgsRaw)
-        if e != 0'i32: osError(osLastError())
+    finally: deallocCStringArray(sysArgs)
 
-      chck posix_spawn_file_actions_init(fops)
+    var sysEnv = if env == nil:
-      chck posix_spawnattr_init(attr)
+        envToCStringArray()
-
-      var mask: Tsigset
-      chck sigemptyset(mask)
-      chck posix_spawnattr_setsigmask(attr, mask)
-      chck posix_spawnattr_setpgroup(attr, 0'i32)
-
-      chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or
-                                          POSIX_SPAWN_SETSIGMASK or
-                                          POSIX_SPAWN_SETPGROUP)
-
-      if poParentStreams notin options:
-        chck posix_spawn_file_actions_addclose(fops, pStdin[writeIdx])
-        chck posix_spawn_file_actions_adddup2(fops, pStdin[readIdx], readIdx)
-        chck posix_spawn_file_actions_addclose(fops, pStdout[readIdx])
-        chck posix_spawn_file_actions_adddup2(fops, pStdout[writeIdx], writeIdx)
-        chck posix_spawn_file_actions_addclose(fops, pStderr[readIdx])
-        if poStdErrToStdOut in options:
-          chck posix_spawn_file_actions_adddup2(fops, pStdout[writeIdx], 2)
-        else:
-          chck posix_spawn_file_actions_adddup2(fops, p_stderr[writeIdx], 2)
-
-      var sys_env = if env == nil: envToCStringArray() else: envToCStringArray(env)
-      var res: cint
-      # This is incorrect!
-      if workingDir.len > 0: os.setCurrentDir(workingDir)
-      if poUsePath in options:
-        res = posix_spawnp(pid, sys_command, fops, attr, sys_args, sys_env)
       else:
-        res = posix_spawn(pid, sys_command, fops, attr, sys_args, sys_env)
+        envToCStringArray(env)
-      deallocCStringArray(sys_env)
-      discard posix_spawn_file_actions_destroy(fops)
-      discard posix_spawnattr_destroy(attr)
-      chck res
 
+    finally: deallocCStringArray(sysEnv)
+
+    var data: TStartProcessData
+    data.sysCommand = sysCommand
+    data.sysArgs = sysArgs
+    data.sysEnv = sysEnv
+    data.pStdin = pStdin
+    data.pStdout = pStdout
+    data.pStderr = pStderr
+    data.optionPoParentStreams = poParentStreams in options
+    data.optionPoUsePath = poUsePath in options
+    data.optionPoStdErrToStdOut = poStdErrToStdOut in options
+    data.workingDir = workingDir
+
+
+    when defined(posix_spawn) and not defined(useFork) and 
+        not defined(useClone) and not defined(linux):
+      pid = startProcessAuxSpawn(data)
     else:
-      pid = fork()
+      pid = startProcessAuxFork(data)
-      if pid < 0: osError(osLastError())
-      if pid == 0:
-        ## child process:
 
-        if poParentStreams notin options:
-          discard close(p_stdin[writeIdx])
-          if dup2(p_stdin[readIdx], readIdx) < 0: osError(osLastError())
-          discard close(p_stdout[readIdx])
-          if dup2(p_stdout[writeIdx], writeIdx) < 0: osError(osLastError())
-          discard close(p_stderr[readIdx])
-          if poStdErrToStdOut in options:
-            if dup2(p_stdout[writeIdx], 2) < 0: osError(osLastError())
-          else:
-            if dup2(p_stderr[writeIdx], 2) < 0: osError(osLastError())
-
-        # Create a new process group
-        if setpgid(0, 0) == -1: quit("setpgid call failed: " & $strerror(errno))
-
-        if workingDir.len > 0: os.setCurrentDir(workingDir)
-
-        if env == nil:
-          if poUsePath in options:
-            discard execvp(sys_command, sys_args)
-          else:
-            discard execv(sys_command, sys_args)
-        else:
-          var c_env = envToCStringArray(env)
-          if poUsePath in options:
-            discard execvpe(sys_command, sys_args, c_env)
-          else:
-            discard execve(sys_command, sys_args, c_env)
-        # too risky to raise an exception here:
-        quit("execve call failed: " & $strerror(errno))
     # Parent process. Copy process information.
     if poEchoCmd in options:
       echo(command, " ", join(args, " "))
@@ -723,6 +696,138 @@ elif not defined(useNimRtl):
       discard close(pStdin[readIdx])
       discard close(pStdout[writeIdx])
 
+  when not defined(useFork):
+    proc startProcessAuxSpawn(data: TStartProcessData): TPid =
+      var attr: Tposix_spawnattr
+      var fops: Tposix_spawn_file_actions
+
+      template chck(e: expr) =
+        if e != 0'i32: osError(osLastError())
+
+      chck posix_spawn_file_actions_init(fops)
+      chck posix_spawnattr_init(attr)
+
+      var mask: Tsigset
+      chck sigemptyset(mask)
+      chck posix_spawnattr_setsigmask(attr, mask)
+      chck posix_spawnattr_setpgroup(attr, 0'i32)
+
+      chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or
+                                          POSIX_SPAWN_SETSIGMASK or
+                                          POSIX_SPAWN_SETPGROUP)
+
+      if not data.optionPoParentStreams:
+        chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx])
+        chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx)
+        chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx])
+        chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx)
+        chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx])
+        if data.optionPoStdErrToStdOut:
+          chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2)
+        else:
+          chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2)
+
+      var res: cint
+      # FIXME: chdir is global to process
+      if data.workingDir.len > 0:
+        setCurrentDir($data.workingDir)
+      var pid: TPid
+
+      if data.optionPoUsePath:
+        res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
+      else:
+        res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
+
+      discard posix_spawn_file_actions_destroy(fops)
+      discard posix_spawnattr_destroy(attr)
+      chck res
+      return pid
+
+  proc startProcessAuxFork(data: TStartProcessData): TPid =
+    if pipe(data.pErrorPipe) != 0:
+      osError(osLastError())
+
+    finally:
+      discard close(data.pErrorPipe[readIdx])
+
+    var pid: TPid
+    var dataCopy = data
+
+    when defined(useClone):
+      const stackSize = 65536
+      let stackEnd = cast[clong](alloc(stackSize))
+      let stack = cast[pointer](stackEnd + stackSize)
+      let fn: pointer = startProcessAfterFork
+      pid = clone(fn, stack,
+                  cint(CLONE_VM or CLONE_VFORK or SIGCHLD),
+                  pointer(addr dataCopy), nil, nil, nil)
+      discard close(data.pErrorPipe[writeIdx])
+      dealloc(stack)
+    else:
+      pid = fork()
+      if pid == 0:
+        startProcessAfterFork(addr(dataCopy))
+        exitnow(1)
+
+    discard close(data.pErrorPipe[writeIdx])
+    if pid < 0: osError(osLastError())
+
+    var error: cint
+    let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error))
+    if sizeRead == sizeof(error):
+      osError($strerror(error))
+
+    return pid
+
+  {.push stacktrace: off, profiler: off.}
+  proc startProcessFail(data: ptr TStartProcessData) =
+    var error: cint = errno
+    discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error))
+    exitnow(1)
+
+  when defined(macosx):
+    var environ {.importc.}: cstringArray
+
+  proc startProcessAfterFork(data: ptr TStartProcessData) =
+    # Warning: no GC here!
+    # Or anythink that touches global structures - all called nimrod procs
+    # must be marked with noStackFrame. Inspect C code after making changes.
+    if not data.optionPoParentStreams:
+      discard close(data.pStdin[writeIdx])
+      if dup2(data.pStdin[readIdx], readIdx) < 0:
+        startProcessFail(data)
+      discard close(data.pStdout[readIdx])
+      if dup2(data.pStdout[writeIdx], writeIdx) < 0:
+        startProcessFail(data)
+      discard close(data.pStderr[readIdx])
+      if data.optionPoStdErrToStdOut:
+        if dup2(data.pStdout[writeIdx], 2) < 0:
+          startProcessFail(data)
+      else:
+        if dup2(data.pStderr[writeIdx], 2) < 0:
+          startProcessFail(data)
+
+    if data.workingDir.len > 0:
+      if chdir(data.workingDir) < 0:
+        startProcessFail(data)
+
+    discard close(data.pErrorPipe[readIdx])
+    discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC)
+
+    if data.optionPoUsePath:
+      when defined(macosx):
+        # MacOSX doesn't have execvpe, so we need workaround.
+        # On MacOSX we can arrive here only from fork, so this is safe:
+        environ = data.sysEnv
+        discard execvp(data.sysCommand, data.sysArgs)
+      else:
+        discard execvpe(data.sysCommand, data.sysArgs, data.sysEnv)
+    else:
+      discard execve(data.sysCommand, data.sysArgs, data.sysEnv)
+
+    startProcessFail(data)
+  {.pop}
+
   proc close(p: PProcess) =
     if p.inStream != nil: close(p.inStream)
     if p.outStream != nil: close(p.outStream)
@@ -791,7 +896,10 @@ elif not defined(useNimRtl):
   proc csystem(cmd: cstring): cint {.nodecl, importc: "system".}
 
   proc execCmd(command: string): int =
-    result = csystem(command) shr 8
+    when defined(linux):
+      result = csystem(command) shr 8
+    else:
+      result = csystem(command)
 
   proc createFdSet(fd: var TFdSet, s: seq[PProcess], m: var int) =
     FD_ZERO(fd)

lib/pure/parsesql.nim

@@ -267,7 +267,7 @@ proc getSymbol(c: var TSqlLexer, tok: var TToken) =
   while true: 
     add(tok.literal, buf[pos])
     Inc(pos)
-    if not (buf[pos] in {'a'..'z','A'..'Z','0'..'9','_','$', '\128'..'\255'}):
+    if buf[pos] notin {'a'..'z','A'..'Z','0'..'9','_','$', '\128'..'\255'}:
       break
   c.bufpos = pos
   tok.kind = tkIdentifier

lib/pure/pegs.nim

@@ -836,9 +836,11 @@ iterator findAll*(s: string, pattern: TPeg, start = 0): string =
   while i < s.len:
     c.ml = 0
     var L = rawMatch(s, pattern, i, c)
-    if L < 0: break
+    if L < 0:
-    yield substr(s, i, i+L-1)
+      inc(i, 1)
-    inc(i, L)
+    else:
+      yield substr(s, i, i+L-1)
+      inc(i, L)
     
 proc findAll*(s: string, pattern: TPeg, start = 0): seq[string] {.
   nosideEffect, rtl, extern: "npegs$1".} = 

lib/pure/rawsockets.nim

@@ -0,0 +1,421 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2014 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements a low-level cross-platform sockets interface. Look
+## at the ``net`` module for the higher-level version.
+
+# TODO: Clean up the exports a bit and everything else in general.
+
+import unsigned, os
+
+when hostos == "solaris":
+  {.passl: "-lsocket -lnsl".}
+
+when defined(Windows):
+  import winlean
+  export WSAEWOULDBLOCK
+else:
+  import posix
+  export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL,
+    EINTR, EINPROGRESS
+
+export TSocketHandle, TSockaddr_in, TAddrinfo, INADDR_ANY, TSockAddr, TSockLen,
+  inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto
+
+export
+  SO_ERROR,
+  SOL_SOCKET,
+  SOMAXCONN,
+  SO_ACCEPTCONN, SO_BROADCAST, SO_DEBUG, SO_DONTROUTE,
+  SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR,
+  MSG_PEEK
+
+type
+  
+  TPort* = distinct uint16  ## port type
+  
+  TDomain* = enum   ## domain, which specifies the protocol family of the
+                    ## created socket. Other domains than those that are listed
+                    ## here are unsupported.
+    AF_UNIX,        ## for local socket (using a file). Unsupported on Windows.
+    AF_INET = 2,    ## for network protocol IPv4 or
+    AF_INET6 = 23   ## for network protocol IPv6.
+
+  TType* = enum        ## second argument to `socket` proc
+    SOCK_STREAM = 1,   ## reliable stream-oriented service or Stream Sockets
+    SOCK_DGRAM = 2,    ## datagram service or Datagram Sockets
+    SOCK_RAW = 3,      ## raw protocols atop the network layer.
+    SOCK_SEQPACKET = 5 ## reliable sequenced packet service
+
+  TProtocol* = enum     ## third argument to `socket` proc
+    IPPROTO_TCP = 6,    ## Transmission control protocol. 
+    IPPROTO_UDP = 17,   ## User datagram protocol.
+    IPPROTO_IP,         ## Internet protocol. Unsupported on Windows.
+    IPPROTO_IPV6,       ## Internet Protocol Version 6. Unsupported on Windows.
+    IPPROTO_RAW,        ## Raw IP Packets Protocol. Unsupported on Windows.
+    IPPROTO_ICMP        ## Control message protocol. Unsupported on Windows.
+
+  TServent* {.pure, final.} = object ## information about a service
+    name*: string
+    aliases*: seq[string]
+    port*: TPort
+    proto*: string
+
+  Thostent* {.pure, final.} = object ## information about a given host
+    name*: string
+    aliases*: seq[string]
+    addrtype*: TDomain
+    length*: int
+    addrList*: seq[string]
+
+when defined(windows):
+  let
+    osInvalidSocket* = winlean.INVALID_SOCKET
+
+  const
+    IOCPARM_MASK* = 127
+    IOC_IN* = int(-2147483648)
+    FIONBIO* = IOC_IN.int32 or ((sizeof(int32) and IOCPARM_MASK) shl 16) or 
+                             (102 shl 8) or 126
+
+  proc ioctlsocket*(s: TSocketHandle, cmd: clong, 
+                   argptr: ptr clong): cint {.
+                   stdcall, importc: "ioctlsocket", dynlib: "ws2_32.dll".}
+else:
+  let
+    osInvalidSocket* = posix.INVALID_SOCKET
+
+proc `==`*(a, b: TPort): bool {.borrow.}
+  ## ``==`` for ports.
+
+proc `$`*(p: TPort): string {.borrow.}
+  ## returns the port number as a string
+
+proc toInt*(domain: TDomain): cint
+  ## Converts the TDomain enum to a platform-dependent ``cint``.
+
+proc toInt*(typ: TType): cint
+  ## Converts the TType enum to a platform-dependent ``cint``.
+
+proc toInt*(p: TProtocol): cint
+  ## Converts the TProtocol enum to a platform-dependent ``cint``.
+
+when defined(posix):
+  proc toInt(domain: TDomain): cint =
+    case domain
+    of AF_UNIX:        result = posix.AF_UNIX
+    of AF_INET:        result = posix.AF_INET
+    of AF_INET6:       result = posix.AF_INET6
+    else: discard
+
+  proc toInt(typ: TType): cint =
+    case typ
+    of SOCK_STREAM:    result = posix.SOCK_STREAM
+    of SOCK_DGRAM:     result = posix.SOCK_DGRAM
+    of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET
+    of SOCK_RAW:       result = posix.SOCK_RAW
+    else: discard
+
+  proc toInt(p: TProtocol): cint =
+    case p
+    of IPPROTO_TCP:    result = posix.IPPROTO_TCP
+    of IPPROTO_UDP:    result = posix.IPPROTO_UDP
+    of IPPROTO_IP:     result = posix.IPPROTO_IP
+    of IPPROTO_IPV6:   result = posix.IPPROTO_IPV6
+    of IPPROTO_RAW:    result = posix.IPPROTO_RAW
+    of IPPROTO_ICMP:   result = posix.IPPROTO_ICMP
+    else: discard
+
+else:
+  proc toInt(domain: TDomain): cint = 
+    result = toU16(ord(domain))
+
+  proc toInt(typ: TType): cint =
+    result = cint(ord(typ))
+  
+  proc toInt(p: TProtocol): cint =
+    result = cint(ord(p))
+
+
+proc newRawSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM,
+             protocol: TProtocol = IPPROTO_TCP): TSocketHandle =
+  ## Creates a new socket; returns `InvalidSocket` if an error occurs.
+  socket(toInt(domain), toInt(typ), toInt(protocol))
+
+proc close*(socket: TSocketHandle) =
+  ## closes a socket.
+  when defined(windows):
+    discard winlean.closeSocket(socket)
+  else:
+    discard posix.close(socket)
+  # TODO: These values should not be discarded. An EOS should be raised.
+  # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times
+
+proc bindAddr*(socket: TSocketHandle, name: ptr TSockAddr, namelen: TSockLen): cint =
+  result = bindSocket(socket, name, namelen)
+
+proc listen*(socket: TSocketHandle, backlog = SOMAXCONN): cint {.tags: [FReadIO].} =
+  ## Marks ``socket`` as accepting connections. 
+  ## ``Backlog`` specifies the maximum length of the 
+  ## queue of pending connections.
+  when defined(windows):
+    result = winlean.listen(socket, cint(backlog))
+  else:
+    result = posix.listen(socket, cint(backlog))
+
+proc getAddrInfo*(address: string, port: TPort, af: TDomain = AF_INET, typ: TType = SOCK_STREAM,
+                 prot: TProtocol = IPPROTO_TCP): ptr TAddrInfo =
+  ##
+  ##
+  ## **Warning**: The resulting ``ptr TAddrInfo`` must be freed using ``dealloc``!
+  var hints: TAddrInfo
+  result = nil
+  hints.ai_family = toInt(af)
+  hints.ai_socktype = toInt(typ)
+  hints.ai_protocol = toInt(prot)
+  var gaiResult = getAddrInfo(address, $port, addr(hints), result)
+  if gaiResult != 0'i32:
+    when defined(windows):
+      OSError(OSLastError())
+    else:
+      raise newException(EOS, $gai_strerror(gaiResult))
+
+proc dealloc*(ai: ptr TAddrInfo) =
+  freeaddrinfo(ai)
+
+proc ntohl*(x: int32): int32 = 
+  ## Converts 32-bit integers from network to host byte order.
+  ## On machines where the host byte order is the same as network byte order,
+  ## this is a no-op; otherwise, it performs a 4-byte swap operation.
+  when cpuEndian == bigEndian: result = x
+  else: result = (x shr 24'i32) or
+                 (x shr 8'i32 and 0xff00'i32) or
+                 (x shl 8'i32 and 0xff0000'i32) or
+                 (x shl 24'i32)
+
+proc ntohs*(x: int16): int16 =
+  ## Converts 16-bit integers from network to host byte order. On machines
+  ## where the host byte order is the same as network byte order, this is
+  ## a no-op; otherwise, it performs a 2-byte swap operation.
+  when cpuEndian == bigEndian: result = x
+  else: result = (x shr 8'i16) or (x shl 8'i16)
+
+proc htonl*(x: int32): int32 =
+  ## Converts 32-bit integers from host to network byte order. On machines
+  ## where the host byte order is the same as network byte order, this is
+  ## a no-op; otherwise, it performs a 4-byte swap operation.
+  result = rawsockets.ntohl(x)
+
+proc htons*(x: int16): int16 =
+  ## Converts 16-bit positive integers from host to network byte order.
+  ## On machines where the host byte order is the same as network byte
+  ## order, this is a no-op; otherwise, it performs a 2-byte swap operation.
+  result = rawsockets.ntohs(x)
+
+proc getServByName*(name, proto: string): TServent {.tags: [FReadIO].} =
+  ## Searches the database from the beginning and finds the first entry for 
+  ## which the service name specified by ``name`` matches the s_name member
+  ## and the protocol name specified by ``proto`` matches the s_proto member.
+  ##
+  ## On posix this will search through the ``/etc/services`` file.
+  when defined(Windows):
+    var s = winlean.getservbyname(name, proto)
+  else:
+    var s = posix.getservbyname(name, proto)
+  if s == nil: raise newException(EOS, "Service not found.")
+  result.name = $s.s_name
+  result.aliases = cstringArrayToSeq(s.s_aliases)
+  result.port = TPort(s.s_port)
+  result.proto = $s.s_proto
+  
+proc getServByPort*(port: TPort, proto: string): TServent {.tags: [FReadIO].} = 
+  ## Searches the database from the beginning and finds the first entry for 
+  ## which the port specified by ``port`` matches the s_port member and the 
+  ## protocol name specified by ``proto`` matches the s_proto member.
+  ##
+  ## On posix this will search through the ``/etc/services`` file.
+  when defined(Windows):
+    var s = winlean.getservbyport(ze(int16(port)).cint, proto)
+  else:
+    var s = posix.getservbyport(ze(int16(port)).cint, proto)
+  if s == nil: raise newException(EOS, "Service not found.")
+  result.name = $s.s_name
+  result.aliases = cstringArrayToSeq(s.s_aliases)
+  result.port = TPort(s.s_port)
+  result.proto = $s.s_proto
+
+proc getHostByAddr*(ip: string): Thostent {.tags: [FReadIO].} =
+  ## This function will lookup the hostname of an IP Address.
+  var myaddr: TInAddr
+  myaddr.s_addr = inet_addr(ip)
+  
+  when defined(windows):
+    var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint,
+                                  cint(rawsockets.AF_INET))
+    if s == nil: osError(osLastError())
+  else:
+    var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).TSocklen, 
+                                cint(posix.AF_INET))
+    if s == nil:
+      raise newException(EOS, $hstrerror(h_errno))
+  
+  result.name = $s.h_name
+  result.aliases = cstringArrayToSeq(s.h_aliases)
+  when defined(windows): 
+    result.addrtype = TDomain(s.h_addrtype)
+  else:
+    if s.h_addrtype == posix.AF_INET:
+      result.addrtype = AF_INET
+    elif s.h_addrtype == posix.AF_INET6:
+      result.addrtype = AF_INET6
+    else:
+      raise newException(EOS, "unknown h_addrtype")
+  result.addrList = cstringArrayToSeq(s.h_addr_list)
+  result.length = int(s.h_length)
+
+proc getHostByName*(name: string): Thostent {.tags: [FReadIO].} = 
+  ## This function will lookup the IP address of a hostname.
+  when defined(Windows):
+    var s = winlean.gethostbyname(name)
+  else:
+    var s = posix.gethostbyname(name)
+  if s == nil: osError(osLastError())
+  result.name = $s.h_name
+  result.aliases = cstringArrayToSeq(s.h_aliases)
+  when defined(windows): 
+    result.addrtype = TDomain(s.h_addrtype)
+  else:
+    if s.h_addrtype == posix.AF_INET:
+      result.addrtype = AF_INET
+    elif s.h_addrtype == posix.AF_INET6:
+      result.addrtype = AF_INET6
+    else:
+      raise newException(EOS, "unknown h_addrtype")
+  result.addrList = cstringArrayToSeq(s.h_addr_list)
+  result.length = int(s.h_length)
+
+proc getSockName*(socket: TSocketHandle): TPort = 
+  ## returns the socket's associated port number.
+  var name: Tsockaddr_in
+  when defined(Windows):
+    name.sin_family = int16(ord(AF_INET))
+  else:
+    name.sin_family = posix.AF_INET
+  #name.sin_port = htons(cint16(port))
+  #name.sin_addr.s_addr = htonl(INADDR_ANY)
+  var namelen = sizeof(name).TSocklen
+  if getsockname(socket, cast[ptr TSockAddr](addr(name)),
+                 addr(namelen)) == -1'i32:
+    osError(osLastError())
+  result = TPort(rawsockets.ntohs(name.sin_port))
+
+proc getSockOptInt*(socket: TSocketHandle, level, optname: int): int {.
+  tags: [FReadIO].} = 
+  ## getsockopt for integer options.
+  var res: cint
+  var size = sizeof(res).TSocklen
+  if getsockopt(socket, cint(level), cint(optname), 
+                addr(res), addr(size)) < 0'i32:
+    osError(osLastError())
+  result = int(res)
+
+proc setSockOptInt*(socket: TSocketHandle, level, optname, optval: int) {.
+  tags: [FWriteIO].} =
+  ## setsockopt for integer options.
+  var value = cint(optval)
+  if setsockopt(socket, cint(level), cint(optname), addr(value),  
+                sizeof(value).TSocklen) < 0'i32:
+    osError(osLastError())
+
+proc setBlocking*(s: TSocketHandle, blocking: bool) =
+  ## Sets blocking mode on socket.
+  ##
+  ## Raises EOS on error.
+  when defined(Windows):
+    var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking
+    if ioctlsocket(s, FIONBIO, addr(mode)) == -1:
+      osError(osLastError())
+  else: # BSD sockets
+    var x: int = fcntl(s, F_GETFL, 0)
+    if x == -1:
+      osError(osLastError())
+    else:
+      var mode = if blocking: x and not O_NONBLOCK else: x or O_NONBLOCK
+      if fcntl(s, F_SETFL, mode) == -1:
+        osError(osLastError())
+
+proc timeValFromMilliseconds(timeout = 500): Ttimeval =
+  if timeout != -1:
+    var seconds = timeout div 1000
+    result.tv_sec = seconds.int32
+    result.tv_usec = ((timeout - seconds * 1000) * 1000).int32
+
+proc createFdSet(fd: var TFdSet, s: seq[TSocketHandle], m: var int) = 
+  FD_ZERO(fd)
+  for i in items(s): 
+    m = max(m, int(i))
+    FD_SET(i, fd)
+   
+proc pruneSocketSet(s: var seq[TSocketHandle], fd: var TFdSet) = 
+  var i = 0
+  var L = s.len
+  while i < L:
+    if FD_ISSET(s[i], fd) == 0'i32:
+      s[i] = s[L-1]
+      dec(L)
+    else:
+      inc(i)
+  setLen(s, L)
+
+proc select*(readfds: var seq[TSocketHandle], timeout = 500): int =
+  ## Traditional select function. This function will return the number of
+  ## sockets that are ready to be read from, written to, or which have errors.
+  ## If there are none; 0 is returned. 
+  ## ``Timeout`` is in miliseconds and -1 can be specified for no timeout.
+  ## 
+  ## A socket is removed from the specific ``seq`` when it has data waiting to
+  ## be read/written to or has errors (``exceptfds``).
+  var tv {.noInit.}: Ttimeval = timeValFromMilliseconds(timeout)
+  
+  var rd: TFdSet
+  var m = 0
+  createFdSet((rd), readfds, m)
+  
+  if timeout != -1:
+    result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv)))
+  else:
+    result = int(select(cint(m+1), addr(rd), nil, nil, nil))
+  
+  pruneSocketSet(readfds, (rd))
+
+proc selectWrite*(writefds: var seq[TSocketHandle], 
+                  timeout = 500): int {.tags: [FReadIO].} =
+  ## When a socket in ``writefds`` is ready to be written to then a non-zero
+  ## value will be returned specifying the count of the sockets which can be
+  ## written to. The sockets which can be written to will also be removed
+  ## from ``writefds``.
+  ##
+  ## ``timeout`` is specified in miliseconds and ``-1`` can be specified for
+  ## an unlimited time.
+  var tv {.noInit.}: Ttimeval = timeValFromMilliseconds(timeout)
+  
+  var wr: TFdSet
+  var m = 0
+  createFdSet((wr), writefds, m)
+  
+  if timeout != -1:
+    result = int(select(cint(m+1), nil, addr(wr), nil, addr(tv)))
+  else:
+    result = int(select(cint(m+1), nil, addr(wr), nil, nil))
+  
+  pruneSocketSet(writefds, (wr))
+
+when defined(Windows):
+  var wsa: TWSADATA
+  if WSAStartup(0x0101'i16, addr wsa) != 0: OSError(OSLastError())

lib/pure/selectors.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2013 Dominik Picheta
+#        (c) Copyright 2014 Dominik Picheta
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -9,212 +9,230 @@
 
 # TODO: Docs.
 
-import tables, os, unsigned
+import tables, os, unsigned, hashes
-when defined(windows):
+
-  import winlean
+when defined(linux): import posix, epoll
-else:
+elif defined(windows): import winlean
-  import posix
+
+proc hash*(x: TSocketHandle): THash {.borrow.}
+proc `$`*(x: TSocketHandle): string {.borrow.}
 
 type
   TEvent* = enum
     EvRead, EvWrite
 
-  TSelectorKey* = object
+  PSelectorKey* = ref object
-    fd: cint
+    fd*: TSocketHandle
-    events: set[TEvent]
+    events*: set[TEvent] ## The events which ``fd`` listens for.
-    data: PObject
+    data*: PObject ## User object.
 
-  TReadyInfo* = tuple[key: TSelectorKey, events: set[TEvent]]
+  TReadyInfo* = tuple[key: PSelectorKey, events: set[TEvent]]
 
-  PSelector* = ref object of PObject ## Selector interface.
+when defined(linux) or defined(nimdoc):
-    fds*: TTable[cint, TSelectorKey]
-    registerImpl*: proc (s: PSelector, fd: cint, events: set[TEvent],
-                    data: PObject): TSelectorKey {.nimcall, tags: [FWriteIO].}
-    unregisterImpl*: proc (s: PSelector, fd: cint): TSelectorKey {.nimcall, tags: [FWriteIO].}
-    selectImpl*: proc (s: PSelector, timeout: int): seq[TReadyInfo] {.nimcall, tags: [FReadIO].}
-    closeImpl*: proc (s: PSelector) {.nimcall.}
-
-template initSelector(r: expr) =
-  new r
-  r.fds = initTable[cint, TSelectorKey]()
-
-proc register*(s: PSelector, fd: cint, events: set[TEvent], data: PObject):
-    TSelectorKey =
-  if not s.registerImpl.isNil: result = s.registerImpl(s, fd, events, data)
-
-proc unregister*(s: PSelector, fd: cint): TSelectorKey =
-  ##
-  ## **Note:** For the ``epoll`` implementation the resulting ``TSelectorKey``
-  ## will only have the ``fd`` field set. This is an optimisation and may
-  ## change in the future if a viable use case is presented. 
-  if not s.unregisterImpl.isNil: result = s.unregisterImpl(s, fd)
-
-proc select*(s: PSelector, timeout = 500): seq[TReadyInfo] =
-  ##
-  ## The ``events`` field of the returned ``key`` contains the original events
-  ## for which the ``fd`` was bound. This is contrary to the ``events`` field
-  ## of the ``TReadyInfo`` tuple which determines which events are ready
-  ## on the ``fd``.
-
-  if not s.selectImpl.isNil: result = s.selectImpl(s, timeout)
-
-proc close*(s: PSelector) =
-  if not s.closeImpl.isNil: s.closeImpl(s)
-
-# ---- Select() ----------------------------------------------------------------
-
-type
-  PSelectSelector* = ref object of PSelector ## Implementation of select()
-
-proc ssRegister(s: PSelector, fd: cint, events: set[TEvent],
-    data: PObject): TSelectorKey =
-  if s.fds.hasKey(fd):
-    raise newException(EInvalidValue, "FD already exists in selector.")
-  var sk = TSelectorKey(fd: fd, events: events, data: data)
-  s.fds[fd] = sk
-  result = sk
-
-proc ssUnregister(s: PSelector, fd: cint): TSelectorKey =
-  result = s.fds[fd]
-  s.fds.del(fd)
-
-proc ssClose(s: PSelector) = nil
-
-proc timeValFromMilliseconds(timeout: int): TTimeVal =
-  if timeout != -1:
-    var seconds = timeout div 1000
-    result.tv_sec = seconds.int32
-    result.tv_usec = ((timeout - seconds * 1000) * 1000).int32
-
-proc createFdSet(rd, wr: var TFdSet, fds: TTable[cint, TSelectorKey],
-    m: var int) =
-  FD_ZERO(rd); FD_ZERO(wr)
-  for k, v in pairs(fds):
-    if EvRead in v.events: 
-      m = max(m, int(k))
-      FD_SET(k, rd)
-    if EvWrite in v.events:
-      m = max(m, int(k))
-      FD_SET(k, wr)
-   
-proc getReadyFDs(rd, wr: var TFdSet, fds: TTable[cint, TSelectorKey]):
-    seq[TReadyInfo] =
-  result = @[]
-  for k, v in pairs(fds):
-    var events: set[TEvent] = {}
-    if FD_ISSET(k, rd) != 0'i32:
-      events = events + {EvRead}
-    if FD_ISSET(k, wr) != 0'i32:
-      events = events + {EvWrite}
-    result.add((v, events))
-
-proc select(fds: TTable[cint, TSelectorKey], timeout = 500):
-  seq[TReadyInfo] =
-  var tv {.noInit.}: TTimeVal = timeValFromMilliseconds(timeout)
-  
-  var rd, wr: TFdSet
-  var m = 0
-  createFdSet(rd, wr, fds, m)
-  
-  var retCode = 0
-  if timeout != -1:
-    retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, addr(tv)))
-  else:
-    retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, nil))
-  
-  if retCode < 0:
-    OSError(OSLastError())
-  elif retCode == 0:
-    return @[]
-  else:
-    return getReadyFDs(rd, wr, fds)
-
-proc ssSelect(s: PSelector, timeout: int): seq[TReadyInfo] =
-  result = select(s.fds, timeout)
-
-proc newSelectSelector*(): PSelectSelector =
-  initSelector(result)
-  result.registerImpl = ssRegister
-  result.unregisterImpl = ssUnregister
-  result.selectImpl = ssSelect
-  result.closeImpl = ssClose
-
-# ---- Epoll -------------------------------------------------------------------
-
-when defined(linux):
-  import epoll
   type
-    PEpollSelector* = ref object of PSelector
+    PSelector* = ref object
       epollFD: cint
-      events: array[64, ptr epoll_event]
+      events: array[64, epoll_event]
+      fds: TTable[TSocketHandle, PSelectorKey]
   
-    TDataWrapper = object
+  proc createEventStruct(events: set[TEvent], fd: TSocketHandle): epoll_event =
-      fd: cint
-      boundEvents: set[TEvent] ## The events which ``fd`` listens for.
-      data: PObject ## User object.
-  
-  proc esRegister(s: PSelector, fd: cint, events: set[TEvent],
-      data: PObject): TSelectorKey =
-    var es = PEpollSelector(s)
-    var event: epoll_event
     if EvRead in events:
-      event.events = EPOLLIN
+      result.events = EPOLLIN
     if EvWrite in events:
-      event.events = event.events or EPOLLOUT
+      result.events = result.events or EPOLLOUT
+    result.events = result.events or EPOLLRDHUP
+    result.data.fd = fd.cint
   
-    var dw = cast[ptr TDataWrapper](alloc0(sizeof(TDataWrapper))) # TODO: This needs to be dealloc'd
+  proc register*(s: PSelector, fd: TSocketHandle, events: set[TEvent],
-    dw.fd = fd
+      data: PObject): PSelectorKey {.discardable.} =
-    dw.boundEvents = events
+    ## Registers file descriptor ``fd`` to selector ``s`` with a set of TEvent
-    dw.data = data
+    ## ``events``.
-    event.data.thePtr = dw
+    var event = createEventStruct(events, fd)
-    
+    if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0:
-    if epoll_ctl(es.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0:
       OSError(OSLastError())
   
-    result = TSelectorKey(fd: fd, events: events, data: data)
+    var key = PSelectorKey(fd: fd, events: events, data: data)
   
-  proc esUnregister(s: PSelector, fd: cint): TSelectorKey =
+    s.fds[fd] = key
-    # We cannot find out the information about this ``fd`` from the epoll
+    result = key
-    # context. As such I will simply return an almost empty TSelectorKey.
+  
-    var es = PEpollSelector(s)
+  proc update*(s: PSelector, fd: TSocketHandle,
-    if epoll_ctl(es.epollFD, EPOLL_CTL_DEL, fd, nil) != 0:
+      events: set[TEvent]): PSelectorKey {.discardable.} =
+    ## Updates the events which ``fd`` wants notifications for.
+    if s.fds[fd].events != events:
+      var event = createEventStruct(events, fd)
+      
+      s.fds[fd].events = events
+      if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0:
+        OSError(OSLastError())
+      result = s.fds[fd]
+  
+  proc unregister*(s: PSelector, fd: TSocketHandle): PSelectorKey {.discardable.} =
+    if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0:
+      let err = OSLastError()
+      if err.cint notin {ENOENT, EBADF}: # TODO: Why do we sometimes get an EBADF? Is this normal?
+        OSError(err)
+    result = s.fds[fd]
+    s.fds.del(fd)
+
+  proc close*(s: PSelector) =
+    if s.epollFD.close() != 0: OSError(OSLastError())
+    dealloc(addr s.events) # TODO: Test this
+  
+  proc epollHasFd(s: PSelector, fd: TSocketHandle): bool =
+    result = true
+    var event = createEventStruct(s.fds[fd].events, fd)
+    if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0:
+      let err = osLastError()
+      if err.cint in {ENOENT, EBADF}:
+        return false
       OSError(OSLastError())
-    # We could fill in the ``fds`` TTable to get the info, but that wouldn't
-    # be nice for our memory.
-    result = TSelectorKey(fd: fd, events: {}, data: nil)
   
-  proc esClose(s: PSelector) =
+  proc select*(s: PSelector, timeout: int): seq[TReadyInfo] =
-    var es = PEpollSelector(s)
+    ##
-    if es.epollFD.close() != 0: OSError(OSLastError())
+    ## The ``events`` field of the returned ``key`` contains the original events
-    dealloc(addr es.events) # TODO: Test this
+    ## for which the ``fd`` was bound. This is contrary to the ``events`` field
-  
+    ## of the ``TReadyInfo`` tuple which determines which events are ready
-  proc esSelect(s: PSelector, timeout: int): seq[TReadyInfo] =
+    ## on the ``fd``.
     result = @[]
-    var es = PEpollSelector(s)
+    let evNum = epoll_wait(s.epollFD, addr s.events[0], 64.cint, timeout.cint)
-    
-    let evNum = epoll_wait(es.epollFD, es.events[0], 64.cint, timeout.cint)
     if evNum < 0: OSError(OSLastError())
     if evNum == 0: return @[]
     for i in 0 .. <evNum:
-      var evSet: set[TEvent] = {}
+      let fd = s.events[i].data.fd.TSocketHandle
-      if (es.events[i].events and EPOLLIN) != 0: evSet = evSet + {EvRead}
-      if (es.events[i].events and EPOLLOUT) != 0: evSet = evSet + {EvWrite}
-      let dw = cast[ptr TDataWrapper](es.events[i].data.thePtr)
     
-      let selectorKey = TSelectorKey(fd: dw.fd, events: dw.boundEvents, 
+      var evSet: set[TEvent] = {}
-          data: dw.data)
+      if (s.events[i].events and EPOLLIN) != 0: evSet = evSet + {EvRead}
+      if (s.events[i].events and EPOLLOUT) != 0: evSet = evSet + {EvWrite}
+      let selectorKey = s.fds[fd]
+      assert selectorKey != nil
       result.add((selectorKey, evSet))
 
-  proc newEpollSelector*(): PEpollSelector =
+      #echo("Epoll: ", result[i].key.fd, " ", result[i].events, " ", result[i].key.events)
+  
+  proc newSelector*(): PSelector =
     new result
     result.epollFD = epoll_create(64)
-    result.events = cast[array[64, ptr epoll_event]](alloc0(sizeof(epoll_event)*64))
+    result.events = cast[array[64, epoll_event]](alloc0(sizeof(epoll_event)*64))
+    result.fds = initTable[TSocketHandle, PSelectorKey]()
     if result.epollFD < 0:
       OSError(OSLastError())
-    result.registerImpl = esRegister
+
-    result.unregisterImpl = esUnregister
+  proc contains*(s: PSelector, fd: TSocketHandle): bool =
-    result.closeImpl = esClose
+    ## Determines whether selector contains a file descriptor.
-    result.selectImpl = esSelect
+    if s.fds.hasKey(fd):
+      # Ensure the underlying epoll instance still contains this fd.
+      result = epollHasFd(s, fd)
+    else:
+      return false
+
+  proc contains*(s: PSelector, key: PSelectorKey): bool =
+    ## Determines whether selector contains this selector key. More accurate
+    ## than checking if the file descriptor is in the selector because it
+    ## ensures that the keys are equal. File descriptors may not always be
+    ## unique especially when an fd is closed and then a new one is opened,
+    ## the new one may have the same value.
+    return key.fd in s and s.fds[key.fd] == key
+
+  proc `[]`*(s: PSelector, fd: TSocketHandle): PSelectorKey =
+    ## Retrieves the selector key for ``fd``.
+    return s.fds[fd]
+
+elif defined(windows):
+  type
+    PSelector* = ref object
+      fds: TTable[TSocketHandle, PSelectorKey]
+
+  proc register*(s: PSelector, fd: TSocketHandle, events: set[TEvent],
+      data: PObject): PSelectorKey {.discardable.} =
+    if s.fds.hasKey(fd):
+      raise newException(EInvalidValue, "File descriptor already exists.")
+    var sk = PSelectorKey(fd: fd, events: events, data: data)
+    s.fds[fd] = sk
+    result = sk
+
+  proc update*(s: PSelector, fd: TSocketHandle,
+      events: set[TEvent]): PSelectorKey {.discardable.} =
+    ## Updates the events which ``fd`` wants notifications for.
+    if not s.fds.hasKey(fd):
+      raise newException(EInvalidValue, "File descriptor not found.")
+
+    s.fds[fd].events = events
+    result = s.fds[fd]
+
+  proc unregister*(s: PSelector, fd: TSocketHandle): PSelectorKey {.discardable.} =
+    result = s.fds[fd]
+    s.fds.del(fd)
+
+  proc close*(s: PSelector) = nil
+
+  proc timeValFromMilliseconds(timeout: int): TTimeVal =
+    if timeout != -1:
+      var seconds = timeout div 1000
+      result.tv_sec = seconds.int32
+      result.tv_usec = ((timeout - seconds * 1000) * 1000).int32
+
+  proc createFdSet(rd, wr: var TFdSet, fds: TTable[TSocketHandle, PSelectorKey],
+      m: var int) =
+    FD_ZERO(rd); FD_ZERO(wr)
+    for k, v in pairs(fds):
+      if EvRead in v.events: 
+        m = max(m, int(k))
+        FD_SET(k, rd)
+      if EvWrite in v.events:
+        m = max(m, int(k))
+        FD_SET(k, wr)
+     
+  proc getReadyFDs(rd, wr: var TFdSet, fds: TTable[TSocketHandle, PSelectorKey]):
+      seq[TReadyInfo] =
+    result = @[]
+    for k, v in pairs(fds):
+      var events: set[TEvent] = {}
+      if FD_ISSET(k, rd) != 0'i32:
+        events = events + {EvRead}
+      if FD_ISSET(k, wr) != 0'i32:
+        events = events + {EvWrite}
+      result.add((v, events))
+
+  proc select(fds: TTable[TSocketHandle, PSelectorKey], timeout = 500):
+    seq[TReadyInfo] =
+    var tv {.noInit.}: TTimeVal = timeValFromMilliseconds(timeout)
+    
+    var rd, wr: TFdSet
+    var m = 0
+    createFdSet(rd, wr, fds, m)
+    
+    var retCode = 0
+    if timeout != -1:
+      retCode = int(select(TSocketHandle(m+1), addr(rd), addr(wr), nil, addr(tv)))
+    else:
+      retCode = int(select(TSocketHandle(m+1), addr(rd), addr(wr), nil, nil))
+    
+    if retCode < 0:
+      OSError(OSLastError())
+    elif retCode == 0:
+      return @[]
+    else:
+      return getReadyFDs(rd, wr, fds)
+
+  proc select*(s: PSelector, timeout: int): seq[TReadyInfo] =
+    result = select(s.fds, timeout)
+
+  proc newSelector*(): PSelector =
+    new result
+    result.fds = initTable[TSocketHandle, PSelectorKey]()
+
+  proc contains*(s: PSelector, fd: TSocketHandle): bool =
+    return s.fds.hasKey(fd)
+
+  proc `[]`*(s: PSelector, fd: TSocketHandle): PSelectorKey =
+    return s.fds[fd]
+
+elif defined(bsd) or defined(macosx):
+  # TODO: kqueue
+  {.error: "Sorry your platform is not supported yet.".}
+else:
+  {.error: "Sorry your platform is not supported.".}
 
 when isMainModule:
   # Select()
@@ -224,11 +242,12 @@ when isMainModule:
       sock: TSocket
   
   var sock = socket()
+  sock.setBlocking(false)
   sock.connect("irc.freenode.net", TPort(6667))
   
-  var selector = newEpollSelector()
+  var selector = newSelector()
   var data = PSockWrapper(sock: sock)
-  let key = selector.register(sock.getFD.cint, {EvRead}, data)
+  let key = selector.register(sock.getFD, {EvWrite}, data)
   var i = 0
   while true:
     let ready = selector.select(1000)
@@ -236,6 +255,7 @@ when isMainModule:
     if ready.len > 0: echo ready[0].events
     i.inc
     if i == 6:
+      assert selector.unregister(sock.getFD).fd == sock.getFD
       selector.close()
       break
   

lib/pure/sockets2.nim

@@ -1,202 +0,0 @@
-#
-#
-#            Nimrod's Runtime Library
-#        (c) Copyright 2014 Dominik Picheta
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements a low-level cross-platform sockets interface. Look
-## at the ``net`` module for the higher-level version.
-
-import unsigned, os
-
-when hostos == "solaris":
-  {.passl: "-lsocket -lnsl".}
-
-when defined(Windows):
-  import winlean
-else:
-  import posix
-
-export TSocketHandle, TSockaddr_in, TAddrinfo, INADDR_ANY, TSockAddr, TSockLen,
-  inet_ntoa
-
-type
-  
-  TPort* = distinct uint16  ## port type
-  
-  TDomain* = enum   ## domain, which specifies the protocol family of the
-                    ## created socket. Other domains than those that are listed
-                    ## here are unsupported.
-    AF_UNIX,        ## for local socket (using a file). Unsupported on Windows.
-    AF_INET = 2,    ## for network protocol IPv4 or
-    AF_INET6 = 23   ## for network protocol IPv6.
-
-  TType* = enum        ## second argument to `socket` proc
-    SOCK_STREAM = 1,   ## reliable stream-oriented service or Stream Sockets
-    SOCK_DGRAM = 2,    ## datagram service or Datagram Sockets
-    SOCK_RAW = 3,      ## raw protocols atop the network layer.
-    SOCK_SEQPACKET = 5 ## reliable sequenced packet service
-
-  TProtocol* = enum     ## third argument to `socket` proc
-    IPPROTO_TCP = 6,    ## Transmission control protocol. 
-    IPPROTO_UDP = 17,   ## User datagram protocol.
-    IPPROTO_IP,         ## Internet protocol. Unsupported on Windows.
-    IPPROTO_IPV6,       ## Internet Protocol Version 6. Unsupported on Windows.
-    IPPROTO_RAW,        ## Raw IP Packets Protocol. Unsupported on Windows.
-    IPPROTO_ICMP        ## Control message protocol. Unsupported on Windows.
-
-  TServent* {.pure, final.} = object ## information about a service
-    name*: string
-    aliases*: seq[string]
-    port*: TPort
-    proto*: string
-
-  Thostent* {.pure, final.} = object ## information about a given host
-    name*: string
-    aliases*: seq[string]
-    addrtype*: TDomain
-    length*: int
-    addrList*: seq[string]
-
-when defined(windows):
-  let
-    OSInvalidSocket* = winlean.INVALID_SOCKET
-else:
-  let
-    OSInvalidSocket* = posix.INVALID_SOCKET
-
-proc `==`*(a, b: TPort): bool {.borrow.}
-  ## ``==`` for ports.
-
-proc `$`*(p: TPort): string {.borrow.}
-  ## returns the port number as a string
-
-proc toInt*(domain: TDomain): cint
-  ## Converts the TDomain enum to a platform-dependent ``cint``.
-
-proc toInt*(typ: TType): cint
-  ## Converts the TType enum to a platform-dependent ``cint``.
-
-proc toInt*(p: TProtocol): cint
-  ## Converts the TProtocol enum to a platform-dependent ``cint``.
-
-when defined(posix):
-  proc toInt(domain: TDomain): cint =
-    case domain
-    of AF_UNIX:        result = posix.AF_UNIX
-    of AF_INET:        result = posix.AF_INET
-    of AF_INET6:       result = posix.AF_INET6
-    else: nil
-
-  proc toInt(typ: TType): cint =
-    case typ
-    of SOCK_STREAM:    result = posix.SOCK_STREAM
-    of SOCK_DGRAM:     result = posix.SOCK_DGRAM
-    of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET
-    of SOCK_RAW:       result = posix.SOCK_RAW
-    else: nil
-
-  proc toInt(p: TProtocol): cint =
-    case p
-    of IPPROTO_TCP:    result = posix.IPPROTO_TCP
-    of IPPROTO_UDP:    result = posix.IPPROTO_UDP
-    of IPPROTO_IP:     result = posix.IPPROTO_IP
-    of IPPROTO_IPV6:   result = posix.IPPROTO_IPV6
-    of IPPROTO_RAW:    result = posix.IPPROTO_RAW
-    of IPPROTO_ICMP:   result = posix.IPPROTO_ICMP
-    else: nil
-
-else:
-  proc toInt(domain: TDomain): cint = 
-    result = toU16(ord(domain))
-
-  proc toInt(typ: TType): cint =
-    result = cint(ord(typ))
-  
-  proc toInt(p: TProtocol): cint =
-    result = cint(ord(p))
-
-
-proc socket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM,
-             protocol: TProtocol = IPPROTO_TCP): TSocketHandle =
-  ## Creates a new socket; returns `InvalidSocket` if an error occurs.
-  
-  # TODO: The function which will use this will raise EOS.
-  socket(toInt(domain), toInt(typ), toInt(protocol))
-
-proc close*(socket: TSocketHandle) =
-  ## closes a socket.
-  when defined(windows):
-    discard winlean.closeSocket(socket)
-  else:
-    discard posix.close(socket)
-  # TODO: These values should not be discarded. An EOS should be raised.
-  # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times
-
-proc bindAddr*(socket: TSocketHandle, name: ptr TSockAddr, namelen: TSockLen): cint =
-  result = bindSocket(socket, name, namelen)
-
-proc listen*(socket: TSocketHandle, backlog = SOMAXCONN) {.tags: [FReadIO].} =
-  ## Marks ``socket`` as accepting connections. 
-  ## ``Backlog`` specifies the maximum length of the 
-  ## queue of pending connections.
-  when defined(windows):
-    if winlean.listen(socket, cint(backlog)) < 0'i32: osError(osLastError())
-  else:
-    if posix.listen(socket, cint(backlog)) < 0'i32: osError(osLastError())
-
-proc getAddrInfo*(address: string, port: TPort, af: TDomain = AF_INET, typ: TType = SOCK_STREAM,
-                 prot: TProtocol = IPPROTO_TCP): ptr TAddrInfo =
-  ##
-  ##
-  ## **Warning**: The resulting ``ptr TAddrInfo`` must be freed using ``dealloc``!
-  var hints: TAddrInfo
-  result = nil
-  hints.ai_family = toInt(af)
-  hints.ai_socktype = toInt(typ)
-  hints.ai_protocol = toInt(prot)
-  var gaiResult = getAddrInfo(address, $port, addr(hints), result)
-  if gaiResult != 0'i32:
-    when defined(windows):
-      OSError(OSLastError())
-    else:
-      raise newException(EOS, $gai_strerror(gaiResult))
-
-proc dealloc*(ai: ptr TAddrInfo) =
-  freeaddrinfo(ai)
-
-proc ntohl*(x: int32): int32 = 
-  ## Converts 32-bit integers from network to host byte order.
-  ## On machines where the host byte order is the same as network byte order,
-  ## this is a no-op; otherwise, it performs a 4-byte swap operation.
-  when cpuEndian == bigEndian: result = x
-  else: result = (x shr 24'i32) or
-                 (x shr 8'i32 and 0xff00'i32) or
-                 (x shl 8'i32 and 0xff0000'i32) or
-                 (x shl 24'i32)
-
-proc ntohs*(x: int16): int16 =
-  ## Converts 16-bit integers from network to host byte order. On machines
-  ## where the host byte order is the same as network byte order, this is
-  ## a no-op; otherwise, it performs a 2-byte swap operation.
-  when cpuEndian == bigEndian: result = x
-  else: result = (x shr 8'i16) or (x shl 8'i16)
-
-proc htonl*(x: int32): int32 =
-  ## Converts 32-bit integers from host to network byte order. On machines
-  ## where the host byte order is the same as network byte order, this is
-  ## a no-op; otherwise, it performs a 4-byte swap operation.
-  result = sockets2.ntohl(x)
-
-proc htons*(x: int16): int16 =
-  ## Converts 16-bit positive integers from host to network byte order.
-  ## On machines where the host byte order is the same as network byte
-  ## order, this is a no-op; otherwise, it performs a 2-byte swap operation.
-  result = sockets2.ntohs(x)
-
-when defined(Windows):
-  var wsa: TWSADATA
-  if WSAStartup(0x0101'i16, addr wsa) != 0: OSError(OSLastError())

lib/pure/times.nim

@@ -557,6 +557,119 @@ proc `$`*(m: TMonth): string =
       "November", "December"]
   return lookup[m]
 
+proc format_token(info: TTimeInfo, token: string, buf: var string) =
+  ## Helper of the format proc to parse individual tokens.
+  ##
+  ## Pass the found token in the user input string, and the buffer where the
+  ## final string is being built. This has to be a var value because certain
+  ## formatting tokens require modifying the previous characters.
+  case token
+  of "d":
+    buf.add($info.monthday)
+  of "dd":
+    if info.monthday < 10:
+      buf.add("0")
+    buf.add($info.monthday)
+  of "ddd":
+    buf.add(($info.weekday)[0 .. 2])
+  of "dddd":
+    buf.add($info.weekday)
+  of "h":
+    buf.add($(if info.hour > 12: info.hour - 12 else: info.hour))
+  of "hh":
+    let amerHour = if info.hour > 12: info.hour - 12 else: info.hour
+    if amerHour < 10:
+      buf.add('0')
+    buf.add($amerHour)
+  of "H":
+    buf.add($info.hour)
+  of "HH":
+    if info.hour < 10:
+      buf.add('0')
+    buf.add($info.hour)
+  of "m":
+    buf.add($info.minute)
+  of "mm":
+    if info.minute < 10:
+      buf.add('0')
+    buf.add($info.minute)
+  of "M":
+    buf.add($(int(info.month)+1))
+  of "MM":
+    if info.month < mOct:
+      buf.add('0')
+    buf.add($(int(info.month)+1))
+  of "MMM":
+    buf.add(($info.month)[0..2])
+  of "MMMM":
+    buf.add($info.month)
+  of "s":
+    buf.add($info.second)
+  of "ss":
+    if info.second < 10:
+      buf.add('0')
+    buf.add($info.second)
+  of "t":
+    if info.hour >= 12:
+      buf.add('P')
+    else: buf.add('A')
+  of "tt":
+    if info.hour >= 12:
+      buf.add("PM")
+    else: buf.add("AM")
+  of "y":
+    var fr = ($info.year).len()-1
+    if fr < 0: fr = 0
+    buf.add(($info.year)[fr .. ($info.year).len()-1])
+  of "yy":
+    var fr = ($info.year).len()-2
+    if fr < 0: fr = 0
+    var fyear = ($info.year)[fr .. ($info.year).len()-1]
+    if fyear.len != 2: fyear = repeatChar(2-fyear.len(), '0') & fyear
+    buf.add(fyear)
+  of "yyy":
+    var fr = ($info.year).len()-3
+    if fr < 0: fr = 0
+    var fyear = ($info.year)[fr .. ($info.year).len()-1]
+    if fyear.len != 3: fyear = repeatChar(3-fyear.len(), '0') & fyear
+    buf.add(fyear)
+  of "yyyy":
+    var fr = ($info.year).len()-4
+    if fr < 0: fr = 0
+    var fyear = ($info.year)[fr .. ($info.year).len()-1]
+    if fyear.len != 4: fyear = repeatChar(4-fyear.len(), '0') & fyear
+    buf.add(fyear)
+  of "yyyyy":
+    var fr = ($info.year).len()-5
+    if fr < 0: fr = 0
+    var fyear = ($info.year)[fr .. ($info.year).len()-1]
+    if fyear.len != 5: fyear = repeatChar(5-fyear.len(), '0') & fyear
+    buf.add(fyear)
+  of "z":
+    let hrs = (info.timezone div 60) div 60
+    buf.add($hrs)
+  of "zz":
+    let hrs = (info.timezone div 60) div 60
+
+    buf.add($hrs)
+    if hrs.abs < 10:
+      var atIndex = buf.len-(($hrs).len-(if hrs < 0: 1 else: 0))
+      buf.insert("0", atIndex)
+  of "zzz":
+    let hrs = (info.timezone div 60) div 60
+
+    buf.add($hrs & ":00")
+    if hrs.abs < 10:
+      var atIndex = buf.len-(($hrs & ":00").len-(if hrs < 0: 1 else: 0))
+      buf.insert("0", atIndex)
+  of "ZZZ":
+    buf.add(info.tzname)
+  of "":
+    discard
+  else:
+    raise newException(EInvalidValue, "Invalid format string: " & token)
+
+
 proc format*(info: TTimeInfo, f: string): string =
   ## This function formats `info` as specified by `f`. The following format
   ## specifiers are available:
@@ -591,8 +704,11 @@ proc format*(info: TTimeInfo, f: string): string =
   ##    ZZZ      Displays the name of the timezone.                                                 ``GMT -> GMT``, ``EST -> EST``
   ## ==========  =================================================================================  ================================================
   ##
-  ## Other strings can be inserted by putting them in ``''``. For example ``hh'->'mm`` will give ``01->56``.
+  ## Other strings can be inserted by putting them in ``''``. For example
-  ## The following characters can be inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` ``,``
+  ## ``hh'->'mm`` will give ``01->56``.  The following characters can be
+  ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]``
+  ## ``,``. However you don't need to necessarily separate format specifiers, a
+  ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too.
 
   result = ""
   var i = 0
@@ -600,111 +716,7 @@ proc format*(info: TTimeInfo, f: string): string =
   while true:
     case f[i]
     of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',':
-      case currentF
+      format_token(info, currentF, result)
-      of "d":
-        result.add($info.monthday)
-      of "dd":
-        if info.monthday < 10:
-          result.add("0")
-        result.add($info.monthday)
-      of "ddd":
-        result.add(($info.weekday)[0 .. 2])
-      of "dddd":
-        result.add($info.weekday)
-      of "h":
-        result.add($(if info.hour > 12: info.hour - 12 else: info.hour))
-      of "hh":
-        let amerHour = if info.hour > 12: info.hour - 12 else: info.hour
-        if amerHour < 10:
-          result.add('0')
-        result.add($amerHour)
-      of "H":
-        result.add($info.hour)
-      of "HH":
-        if info.hour < 10:
-          result.add('0')
-        result.add($info.hour)
-      of "m":
-        result.add($info.minute)
-      of "mm":
-        if info.minute < 10:
-          result.add('0')
-        result.add($info.minute)
-      of "M":
-        result.add($(int(info.month)+1))
-      of "MM":
-        if info.month < mOct:
-          result.add('0')
-        result.add($(int(info.month)+1))
-      of "MMM":
-        result.add(($info.month)[0..2])
-      of "MMMM":
-        result.add($info.month)
-      of "s":
-        result.add($info.second)
-      of "ss":
-        if info.second < 10:
-          result.add('0')
-        result.add($info.second)
-      of "t":
-        if info.hour >= 12:
-          result.add('P')
-        else: result.add('A')
-      of "tt":
-        if info.hour >= 12:
-          result.add("PM")
-        else: result.add("AM")
-      of "y":
-        var fr = ($info.year).len()-1
-        if fr < 0: fr = 0
-        result.add(($info.year)[fr .. ($info.year).len()-1])
-      of "yy":
-        var fr = ($info.year).len()-2
-        if fr < 0: fr = 0
-        var fyear = ($info.year)[fr .. ($info.year).len()-1]
-        if fyear.len != 2: fyear = repeatChar(2-fyear.len(), '0') & fyear
-        result.add(fyear)
-      of "yyy":
-        var fr = ($info.year).len()-3
-        if fr < 0: fr = 0
-        var fyear = ($info.year)[fr .. ($info.year).len()-1]
-        if fyear.len != 3: fyear = repeatChar(3-fyear.len(), '0') & fyear
-        result.add(fyear)
-      of "yyyy":
-        var fr = ($info.year).len()-4
-        if fr < 0: fr = 0
-        var fyear = ($info.year)[fr .. ($info.year).len()-1]
-        if fyear.len != 4: fyear = repeatChar(4-fyear.len(), '0') & fyear
-        result.add(fyear)
-      of "yyyyy":
-        var fr = ($info.year).len()-5
-        if fr < 0: fr = 0
-        var fyear = ($info.year)[fr .. ($info.year).len()-1]
-        if fyear.len != 5: fyear = repeatChar(5-fyear.len(), '0') & fyear
-        result.add(fyear)
-      of "z":
-        let hrs = (info.timezone div 60) div 60
-        result.add($hrs)
-      of "zz":
-        let hrs = (info.timezone div 60) div 60
-        
-        result.add($hrs)
-        if hrs.abs < 10:
-          var atIndex = result.len-(($hrs).len-(if hrs < 0: 1 else: 0))
-          result.insert("0", atIndex)
-      of "zzz":
-        let hrs = (info.timezone div 60) div 60
-        
-        result.add($hrs & ":00")
-        if hrs.abs < 10:
-          var atIndex = result.len-(($hrs & ":00").len-(if hrs < 0: 1 else: 0))
-          result.insert("0", atIndex)
-      of "ZZZ":
-        result.add(info.tzname)
-      of "":
-        discard
-      else:
-        raise newException(EInvalidValue, "Invalid format string: " & currentF)
 
       currentF = ""
       if f[i] == '\0': break
@@ -716,7 +728,15 @@ proc format*(info: TTimeInfo, f: string): string =
           inc(i)
       else: result.add(f[i])
       
-    else: currentF.add(f[i])
+    else:
+      # Check if the letter being added matches previous accumulated buffer.
+      if currentF.len < 1 or currentF[high(currentF)] == f[i]:
+        currentF.add(f[i])
+      else:
+        format_token(info, currentF, result)
+        dec(i) # Move position back to re-process the character separately.
+        currentF = ""
+
     inc(i)
 
 {.pop.}
@@ -727,12 +747,16 @@ when isMainModule:
 
   var t = getGMTime(fromSeconds(2147483647))
   echo t.format("ddd dd MMM hh:mm:ss ZZZ yyyy")
+  echo t.format("ddd ddMMMhhmmssZZZyyyy")
   assert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038"
+  assert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038"
   
   assert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" &
     " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == 
     "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 0 00 00:00 UTC"
 
+  assert t.format("yyyyMMddhhmmss") == "20380119031407"
+  
   var t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975
   assert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" &
     " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") ==

lib/stdlib.babel

@@ -0,0 +1,6 @@
+[Package]
+name          = "stdlib"
+version       = "0.9.0"
+author        = "Dominik Picheta"
+description   = "Nimrod's standard library."
+license       = "MIT"

lib/system.nim

@@ -185,6 +185,8 @@ proc `..`*[T](b: T): TSlice[T] {.noSideEffect, inline.} =
 
 when not defined(niminheritable):
   {.pragma: inheritable.}
+when not defined(nimunion):
+  {.pragma: unchecked.}
 
 const NoFakeVars* = defined(NimrodVM) ## true if the backend doesn't support \
   ## "fake variables" like 'var EBADF {.importc.}: cint'.
@@ -194,9 +196,10 @@ when not defined(JS):
     TGenericSeq {.compilerproc, pure, inheritable.} = object
       len, reserved: int
     PGenericSeq {.exportc.} = ptr TGenericSeq
+    UncheckedCharArray {.unchecked.} = array[0..100_000_000, char]
     # len and space without counting the terminating zero:
     NimStringDesc {.compilerproc, final.} = object of TGenericSeq
-      data: array[0..100_000_000, char]
+      data: UncheckedCharArray
     NimString = ptr NimStringDesc
 
 when not defined(JS) and not defined(NimrodVM):
@@ -257,6 +260,7 @@ type
                               ## system raises.
   EIO* = object of ESystem    ## raised if an IO error occured.
   EOS* = object of ESystem    ## raised if an operating system service failed.
+    errorCode*: int32 ## OS-defined error code describing this error.
   EInvalidLibrary* = object of EOS ## raised if a dynamic library
                                    ## could not be loaded.
   EResourceExhausted* = object of ESystem ## raised if a resource request
@@ -1161,6 +1165,14 @@ when not defined(nimrodVM):
       ## from it before writing to it is undefined behaviour!
       ## The allocated memory belongs to its allocating thread!
       ## Use `allocShared` to allocate from a shared heap.
+    proc createU*(T: typedesc, size = 1.Positive): ptr T {.inline.} =
+      ## allocates a new memory block with at least ``T.sizeof * size``
+      ## bytes. The block has to be freed with ``resize(block, 0)`` or
+      ## ``free(block)``. The block is not initialized, so reading
+      ## from it before writing to it is undefined behaviour!
+      ## The allocated memory belongs to its allocating thread!
+      ## Use `createSharedU` to allocate from a shared heap.
+      cast[ptr T](alloc(T.sizeof * size))
     proc alloc0*(size: int): pointer {.noconv, rtl, tags: [].}
       ## allocates a new memory block with at least ``size`` bytes. The
       ## block has to be freed with ``realloc(block, 0)`` or
@@ -1168,14 +1180,31 @@ when not defined(nimrodVM):
       ## containing zero, so it is somewhat safer than ``alloc``.
       ## The allocated memory belongs to its allocating thread!
       ## Use `allocShared0` to allocate from a shared heap.
-    proc realloc*(p: pointer, newsize: int): pointer {.noconv, rtl, tags: [].}
+    proc create*(T: typedesc, size = 1.Positive): ptr T {.inline.} =
+      ## allocates a new memory block with at least ``T.sizeof * size``
+      ## bytes. The block has to be freed with ``resize(block, 0)`` or
+      ## ``free(block)``. The block is initialized with all bytes
+      ## containing zero, so it is somewhat safer than ``createU``.
+      ## The allocated memory belongs to its allocating thread!
+      ## Use `createShared` to allocate from a shared heap.
+      cast[ptr T](alloc0(T.sizeof * size))
+    proc realloc*(p: pointer, newSize: int): pointer {.noconv, rtl, tags: [].}
       ## grows or shrinks a given memory block. If p is **nil** then a new
       ## memory block is returned. In either way the block has at least
-      ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
+      ## ``newSize`` bytes. If ``newSize == 0`` and p is not **nil**
       ## ``realloc`` calls ``dealloc(p)``. In other cases the block has to
       ## be freed with ``dealloc``.
       ## The allocated memory belongs to its allocating thread!
       ## Use `reallocShared` to reallocate from a shared heap.
+    proc resize*[T](p: ptr T, newSize: Natural): ptr T {.inline.} =
+      ## grows or shrinks a given memory block. If p is **nil** then a new
+      ## memory block is returned. In either way the block has at least
+      ## ``T.sizeof * newSize`` bytes. If ``newSize == 0`` and p is not
+      ## **nil** ``resize`` calls ``free(p)``. In other cases the block
+      ## has to be freed with ``free``. The allocated memory belongs to
+      ## its allocating thread!
+      ## Use `resizeShared` to reallocate from a shared heap.
+      cast[ptr T](realloc(p, T.sizeof * newSize))
     proc dealloc*(p: pointer) {.noconv, rtl, tags: [].}
       ## frees the memory allocated with ``alloc``, ``alloc0`` or
       ## ``realloc``. This procedure is dangerous! If one forgets to
@@ -1184,31 +1213,60 @@ when not defined(nimrodVM):
       ## or other memory may be corrupted. 
       ## The freed memory must belong to its allocating thread!
       ## Use `deallocShared` to deallocate from a shared heap.
-
+    proc free*[T](p: ptr T) {.inline.} =
+      dealloc(p)
     proc allocShared*(size: int): pointer {.noconv, rtl.}
       ## allocates a new memory block on the shared heap with at
       ## least ``size`` bytes. The block has to be freed with
       ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block
       ## is not initialized, so reading from it before writing to it is 
       ## undefined behaviour!
+    proc createSharedU*(T: typedesc, size = 1.Positive): ptr T {.inline.} =
+      ## allocates a new memory block on the shared heap with at
+      ## least ``T.sizeof * size`` bytes. The block has to be freed with
+      ## ``resizeShared(block, 0)`` or ``freeShared(block)``. The block
+      ## is not initialized, so reading from it before writing to it is 
+      ## undefined behaviour!
+      cast[ptr T](allocShared(T.sizeof * size))
     proc allocShared0*(size: int): pointer {.noconv, rtl.}
       ## allocates a new memory block on the shared heap with at 
       ## least ``size`` bytes. The block has to be freed with
       ## ``reallocShared(block, 0)`` or ``deallocShared(block)``.
       ## The block is initialized with all bytes
       ## containing zero, so it is somewhat safer than ``allocShared``.
-    proc reallocShared*(p: pointer, newsize: int): pointer {.noconv, rtl.}
+    proc createShared*(T: typedesc, size = 1.Positive): ptr T {.inline.} =
+      ## allocates a new memory block on the shared heap with at 
+      ## least ``T.sizeof * size`` bytes. The block has to be freed with
+      ## ``resizeShared(block, 0)`` or ``freeShared(block)``.
+      ## The block is initialized with all bytes
+      ## containing zero, so it is somewhat safer than ``createSharedU``.
+      cast[ptr T](allocShared0(T.sizeof * size))
+    proc reallocShared*(p: pointer, newSize: int): pointer {.noconv, rtl.}
       ## grows or shrinks a given memory block on the heap. If p is **nil**
-      ## then a new memory block is returned. In either way the block has at least
+      ## then a new memory block is returned. In either way the block has at
-      ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil**
+      ## least ``newSize`` bytes. If ``newSize == 0`` and p is not **nil**
       ## ``reallocShared`` calls ``deallocShared(p)``. In other cases the
       ## block has to be freed with ``deallocShared``.
+    proc resizeShared*[T](p: ptr T, newSize: Natural): ptr T {.inline.} =
+      ## grows or shrinks a given memory block on the heap. If p is **nil**
+      ## then a new memory block is returned. In either way the block has at
+      ## least ``T.sizeof * newSize`` bytes. If ``newSize == 0`` and p is
+      ## not **nil** ``resizeShared`` calls ``freeShared(p)``. In other
+      ## cases the block has to be freed with ``freeShared``.
+      cast[ptr T](reallocShared(p, T.sizeof * newSize))
     proc deallocShared*(p: pointer) {.noconv, rtl.}
       ## frees the memory allocated with ``allocShared``, ``allocShared0`` or
       ## ``reallocShared``. This procedure is dangerous! If one forgets to
       ## free the memory a leak occurs; if one tries to access freed
       ## memory (or just freeing it twice!) a core dump may happen
       ## or other memory may be corrupted.
+    proc freeShared*[T](p: ptr T) {.inline.} =
+      ## frees the memory allocated with ``createShared``, ``createSharedU`` or
+      ## ``resizeShared``. This procedure is dangerous! If one forgets to
+      ## free the memory a leak occurs; if one tries to access freed
+      ## memory (or just freeing it twice!) a core dump may happen
+      ## or other memory may be corrupted.
+      deallocShared(p)
 
 proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.}
   ## swaps the values `a` and `b`. This is often more efficient than
@@ -1399,20 +1457,6 @@ iterator items*[IX, T](a: array[IX, T]): T {.inline.} =
       if i >= high(IX): break
       inc(i)
 
-iterator items*[T](a: seq[T]): T {.inline.} =
-  ## iterates over each item of `a`.
-  var i = 0
-  while i < len(a):
-    yield a[i]
-    inc(i)
-
-iterator items*(a: string): char {.inline.} =
-  ## iterates over each item of `a`.
-  var i = 0
-  while i < len(a):
-    yield a[i]
-    inc(i)
-
 iterator items*[T](a: set[T]): T {.inline.} =
   ## iterates over each element of `a`. `items` iterates only over the
   ## elements that are really in the set (and not over the ones the set is
@@ -1514,7 +1558,7 @@ when not defined(NimrodVM):
     proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} =
       result = cast[pointer](x)
   else:
-    proc seqToPtr[T](x: seq[T]): pointer {.noStackFrame, nosideeffect.} =
+    proc seqToPtr[T](x: seq[T]): pointer {.asmNoStackFrame, nosideeffect.} =
       asm """return `x`"""
   
   proc `==` *[T](x, y: seq[T]): bool {.noSideEffect.} =
@@ -1802,7 +1846,7 @@ type
     len*: int           ## length of the inspectable slots
 
 when defined(JS):
-  proc add*(x: var string, y: cstring) {.noStackFrame.} =
+  proc add*(x: var string, y: cstring) {.asmNoStackFrame.} =
     asm """
       var len = `x`[0].length-1;
       for (var i = 0; i < `y`.length; ++i) {
@@ -2019,8 +2063,10 @@ when not defined(JS): #and not defined(NimrodVM):
       ## Flushes `f`'s buffer.
 
     proc readAll*(file: TFile): TaintedString {.tags: [FReadIO].}
-      ## Reads all data from the stream `file`. Raises an IO exception
+      ## Reads all data from the stream `file`.
-      ## in case of an error
+      ##
+      ## Raises an IO exception in case of an error. It is an error if the
+      ## current file position is not at the beginning of the file.
     
     proc readFile*(filename: string): TaintedString {.tags: [FReadIO].}
       ## Opens a file named `filename` for reading. Then calls `readAll`
@@ -2309,12 +2355,32 @@ when not defined(JS): #and not defined(NimrodVM):
 
   when not defined(NimrodVM):
     proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.}
-      ## can be used to mark a condition to be likely. This is a hint for the 
+      ## Hints the optimizer that `val` is likely going to be true.
-      ## optimizer.
+      ##
+      ## You can use this proc to decorate a branch condition. On certain
+      ## platforms this can help the processor predict better which branch is
+      ## going to be run. Example:
+      ##
+      ## .. code-block:: nimrod
+      ##   for value in inputValues:
+      ##     if likely(value <= 100):
+      ##       process(value)
+      ##     else:
+      ##       echo "Value too big!"
     
     proc unlikely*(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.}
-      ## can be used to mark a condition to be unlikely. This is a hint for the 
+      ## Hints the optimizer that `val` is likely going to be false.
-      ## optimizer.
+      ##
+      ## You can use this proc to decorate a branch condition. On certain
+      ## platforms this can help the processor predict better which branch is
+      ## going to be run. Example:
+      ##
+      ## .. code-block:: nimrod
+      ##   for value in inputValues:
+      ##     if unlikely(value > 100):
+      ##       echo "Value too big!"
+      ##     else:
+      ##       process(value)
       
     proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} =
       ## retrieves the raw proc pointer of the closure `x`. This is
@@ -2612,6 +2678,24 @@ template doAssert*(cond: bool, msg = "") =
     if not cond:
       raiseAssert(astToStr(cond) & ' ' & msg)
 
+iterator items*[T](a: seq[T]): T {.inline.} =
+  ## iterates over each item of `a`.
+  var i = 0
+  let L = len(a)
+  while i < L:
+    yield a[i]
+    inc(i)
+    assert(len(a) == L, "seq modified while iterating over it")
+
+iterator items*(a: string): char {.inline.} =
+  ## iterates over each item of `a`.
+  var i = 0
+  let L = len(a)
+  while i < L:
+    yield a[i]
+    inc(i)
+    assert(len(a) == L, "string modified while iterating over it")
+
 when not defined(nimhygiene):
   {.pragma: inject.}
 

lib/system/arithm.nim

@@ -111,7 +111,7 @@ const
 when asmVersion and not defined(gcc) and not defined(llvm_gcc):
   # assembler optimized versions for compilers that
   # have an intel syntax assembler:
-  proc addInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc addInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     # a in eax, and b in edx
     asm """
         mov eax, `a`
@@ -121,7 +121,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
       theEnd:
     """
 
-  proc subInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc subInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         sub eax, `b`
@@ -130,7 +130,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
       theEnd:
     """
 
-  proc negInt(a: int): int {.compilerProc, noStackFrame.} =
+  proc negInt(a: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         neg eax
@@ -139,7 +139,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
       theEnd:
     """
 
-  proc divInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc divInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         mov ecx, `b`
@@ -150,7 +150,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
       theEnd:
     """
 
-  proc modInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc modInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         mov ecx, `b`
@@ -162,7 +162,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
         mov eax, edx
     """
 
-  proc mulInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc mulInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         mov ecx, `b`

lib/system/endb.nim

@@ -179,7 +179,7 @@ proc scanAndAppendWord(src: cstring, a: var TStaticStr, start: int): int =
   while True:
     case src[result]
     of 'a'..'z', '0'..'9': add(a, src[result])
-    of '_': nil # just skip it
+    of '_': discard # just skip it
     of 'A'..'Z': add(a, chr(ord(src[result]) - ord('A') + ord('a')))
     else: break
     inc(result)
@@ -203,7 +203,7 @@ proc scanNumber(src: cstring, a: var int, start: int): int =
   while true:
     case src[result]
     of '0'..'9': a = a * 10 + ord(src[result]) - ord('0')
-    of '_': nil # skip underscores (nice for long line numbers)
+    of '_': discard # skip underscores (nice for long line numbers)
     else: break
     inc(result)
 
@@ -524,7 +524,7 @@ proc lineHookImpl() {.nimcall.} =
   of dbBreakpoints:
     # debugger is only interested in breakpoints
     checkForBreakpoint()
-  else: nil
+  else: discard
 
 proc watchpointHookImpl(name: cstring) {.nimcall.} =
   dbgWriteStackTrace(framePtr)

lib/system/jssys.nim

@@ -23,9 +23,9 @@ type
   PCallFrame = ptr TCallFrame
   TCallFrame {.importc, nodecl, final.} = object
     prev: PCallFrame
-    procname: CString
+    procname: cstring
     line: int # current line number
-    filename: CString
+    filename: cstring
 
 var
   framePtr {.importc, nodecl, volatile.}: PCallFrame
@@ -48,7 +48,7 @@ proc getCurrentExceptionMsg*(): string =
 
 proc auxWriteStackTrace(f: PCallFrame): string =
   type
-    TTempFrame = tuple[procname: CString, line: int]
+    TTempFrame = tuple[procname: cstring, line: int]
   var
     it = f
     i = 0
@@ -84,7 +84,7 @@ proc rawWriteStackTrace(): string =
     framePtr = nil
 
 proc raiseException(e: ref E_Base, ename: cstring) {.
-    compilerproc, noStackFrame.} =
+    compilerproc, asmNoStackFrame.} =
   e.name = ename
   if excHandler != nil:
     excHandler.exc = e
@@ -104,7 +104,7 @@ proc raiseException(e: ref E_Base, ename: cstring) {.
     alert(buf)
   asm """throw `e`;"""
 
-proc reraiseException() {.compilerproc, noStackFrame.} =
+proc reraiseException() {.compilerproc, asmNoStackFrame.} =
   if excHandler == nil:
     raise newException(ENoExceptionToReraise, "no exception to reraise")
   else:
@@ -125,7 +125,7 @@ proc raiseIndexError() {.compilerproc, noreturn.} =
 proc raiseFieldError(f: string) {.compilerproc, noreturn.} =
   raise newException(EInvalidField, f & " is not accessible")
 
-proc SetConstr() {.varargs, noStackFrame, compilerproc.} =
+proc SetConstr() {.varargs, asmNoStackFrame, compilerproc.} =
   asm """
     var result = {};
     for (var i = 0; i < arguments.length; ++i) {
@@ -141,7 +141,7 @@ proc SetConstr() {.varargs, noStackFrame, compilerproc.} =
     return result;
   """
 
-proc cstrToNimstr(c: cstring): string {.noStackFrame, compilerproc.} =
+proc cstrToNimstr(c: cstring): string {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = [];
     for (var i = 0; i < `c`.length; ++i) {
@@ -151,7 +151,7 @@ proc cstrToNimstr(c: cstring): string {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc toJSStr(s: string): cstring {.noStackFrame, compilerproc.} =
+proc toJSStr(s: string): cstring {.asmNoStackFrame, compilerproc.} =
   asm """
     var len = `s`.length-1;
     var result = new Array(len);
@@ -162,7 +162,7 @@ proc toJSStr(s: string): cstring {.noStackFrame, compilerproc.} =
     return result.join("");
   """
 
-proc mnewString(len: int): string {.noStackFrame, compilerproc.} =
+proc mnewString(len: int): string {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = new Array(`len`+1);
     result[0] = 0;
@@ -170,7 +170,7 @@ proc mnewString(len: int): string {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc SetCard(a: int): int {.compilerproc, noStackFrame.} =
+proc SetCard(a: int): int {.compilerproc, asmNoStackFrame.} =
   # argument type is a fake
   asm """
     var result = 0;
@@ -178,14 +178,14 @@ proc SetCard(a: int): int {.compilerproc, noStackFrame.} =
     return result;
   """
 
-proc SetEq(a, b: int): bool {.compilerproc, noStackFrame.} =
+proc SetEq(a, b: int): bool {.compilerproc, asmNoStackFrame.} =
   asm """
     for (var elem in `a`) { if (!`b`[elem]) return false; }
     for (var elem in `b`) { if (!`a`[elem]) return false; }
     return true;
   """
 
-proc SetLe(a, b: int): bool {.compilerproc, noStackFrame.} =
+proc SetLe(a, b: int): bool {.compilerproc, asmNoStackFrame.} =
   asm """
     for (var elem in `a`) { if (!`b`[elem]) return false; }
     return true;
@@ -194,7 +194,7 @@ proc SetLe(a, b: int): bool {.compilerproc, noStackFrame.} =
 proc SetLt(a, b: int): bool {.compilerproc.} =
   result = SetLe(a, b) and not SetEq(a, b)
 
-proc SetMul(a, b: int): int {.compilerproc, noStackFrame.} =
+proc SetMul(a, b: int): int {.compilerproc, asmNoStackFrame.} =
   asm """
     var result = {};
     for (var elem in `a`) {
@@ -203,7 +203,7 @@ proc SetMul(a, b: int): int {.compilerproc, noStackFrame.} =
     return result;
   """
 
-proc SetPlus(a, b: int): int {.compilerproc, noStackFrame.} =
+proc SetPlus(a, b: int): int {.compilerproc, asmNoStackFrame.} =
   asm """
     var result = {};
     for (var elem in `a`) { result[elem] = true; }
@@ -211,7 +211,7 @@ proc SetPlus(a, b: int): int {.compilerproc, noStackFrame.} =
     return result;
   """
 
-proc SetMinus(a, b: int): int {.compilerproc, noStackFrame.} =
+proc SetMinus(a, b: int): int {.compilerproc, asmNoStackFrame.} =
   asm """
     var result = {};
     for (var elem in `a`) {
@@ -220,7 +220,7 @@ proc SetMinus(a, b: int): int {.compilerproc, noStackFrame.} =
     return result;
   """
 
-proc cmpStrings(a, b: string): int {.noStackFrame, compilerProc.} =
+proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerProc.} =
   asm """
     if (`a` == `b`) return 0;
     if (!`a`) return -1;
@@ -234,7 +234,7 @@ proc cmpStrings(a, b: string): int {.noStackFrame, compilerProc.} =
 
 proc cmp(x, y: string): int = return cmpStrings(x, y)
 
-proc eqStrings(a, b: string): bool {.noStackFrame, compilerProc.} =
+proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerProc.} =
   asm """
     if (`a` == `b`) return true;
     if ((!`a`) || (!`b`)) return false;
@@ -300,7 +300,7 @@ type
     setAttributeNode*: proc (attr: ref TNode) {.nimcall.}
 
 when defined(kwin):
-  proc rawEcho {.compilerproc, nostackframe.} =
+  proc rawEcho {.compilerproc, asmNoStackFrame.} =
     asm """
       var buf = "";
       for (var i = 0; i < arguments.length; ++i) {
@@ -312,7 +312,7 @@ when defined(kwin):
 elif defined(nodejs):
   proc ewriteln(x: cstring) = log(x)
   
-  proc rawEcho {.compilerproc, nostackframe.} =
+  proc rawEcho {.compilerproc, asmNoStackFrame.} =
     asm """
       var buf = "";
       for (var i = 0; i < arguments.length; ++i) {
@@ -345,42 +345,42 @@ else:
     node.appendChild(document.createElement("br"))
 
 # Arithmetic:
-proc addInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc addInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` + `b`;
     if (result > 2147483647 || result < -2147483648) `raiseOverflow`();
     return result;
   """
 
-proc subInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc subInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` - `b`;
     if (result > 2147483647 || result < -2147483648) `raiseOverflow`();
     return result;
   """
 
-proc mulInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc mulInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` * `b`;
     if (result > 2147483647 || result < -2147483648) `raiseOverflow`();
     return result;
   """
 
-proc divInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc divInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     if (`b` == 0) `raiseDivByZero`();
     if (`b` == -1 && `a` == 2147483647) `raiseOverflow`();
     return Math.floor(`a` / `b`);
   """
 
-proc modInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc modInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     if (`b` == 0) `raiseDivByZero`();
     if (`b` == -1 && `a` == 2147483647) `raiseOverflow`();
     return Math.floor(`a` % `b`);
   """
 
-proc addInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc addInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` + `b`;
     if (result > 9223372036854775807
@@ -388,7 +388,7 @@ proc addInt64(a, b: int): int {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc subInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc subInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` - `b`;
     if (result > 9223372036854775807
@@ -396,7 +396,7 @@ proc subInt64(a, b: int): int {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc mulInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc mulInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` * `b`;
     if (result > 9223372036854775807
@@ -404,90 +404,89 @@ proc mulInt64(a, b: int): int {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc divInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc divInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     if (`b` == 0) `raiseDivByZero`();
     if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`();
     return Math.floor(`a` / `b`);
   """
 
-proc modInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc modInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     if (`b` == 0) `raiseDivByZero`();
     if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`();
     return Math.floor(`a` % `b`);
   """
 
-proc NegInt(a: int): int {.compilerproc.} =
+proc negInt(a: int): int {.compilerproc.} =
   result = a*(-1)
 
-proc NegInt64(a: int64): int64 {.compilerproc.} =
+proc negInt64(a: int64): int64 {.compilerproc.} =
   result = a*(-1)
 
-proc AbsInt(a: int): int {.compilerproc.} =
+proc absInt(a: int): int {.compilerproc.} =
   result = if a < 0: a*(-1) else: a
 
-proc AbsInt64(a: int64): int64 {.compilerproc.} =
+proc absInt64(a: int64): int64 {.compilerproc.} =
   result = if a < 0: a*(-1) else: a
 
-proc LeU(a, b: int): bool {.compilerproc.} =
+proc leU(a, b: int): bool {.compilerproc.} =
   result = abs(a) <= abs(b)
 
-proc LtU(a, b: int): bool {.compilerproc.} =
+proc ltU(a, b: int): bool {.compilerproc.} =
   result = abs(a) < abs(b)
 
-proc LeU64(a, b: int64): bool {.compilerproc.} =
+proc leU64(a, b: int64): bool {.compilerproc.} =
   result = abs(a) <= abs(b)
-
+proc ltU64(a, b: int64): bool {.compilerproc.} =
-proc LtU64(a, b: int64): bool {.compilerproc.} =
   result = abs(a) < abs(b)
 
-proc AddU(a, b: int): int {.compilerproc.} =
+proc addU(a, b: int): int {.compilerproc.} =
   result = abs(a) + abs(b)
-proc AddU64(a, b: int64): int64 {.compilerproc.} =
+proc addU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) + abs(b)
 
-proc SubU(a, b: int): int {.compilerproc.} =
+proc subU(a, b: int): int {.compilerproc.} =
   result = abs(a) - abs(b)
-proc SubU64(a, b: int64): int64 {.compilerproc.} =
+proc subU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) - abs(b)
 
-proc MulU(a, b: int): int {.compilerproc.} =
+proc mulU(a, b: int): int {.compilerproc.} =
   result = abs(a) * abs(b)
-proc MulU64(a, b: int64): int64 {.compilerproc.} =
+proc mulU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) * abs(b)
 
-proc DivU(a, b: int): int {.compilerproc.} =
+proc divU(a, b: int): int {.compilerproc.} =
   result = abs(a) div abs(b)
-proc DivU64(a, b: int64): int64 {.compilerproc.} =
+proc divU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) div abs(b)
 
-proc ModU(a, b: int): int {.compilerproc.} =
+proc modU(a, b: int): int {.compilerproc.} =
   result = abs(a) mod abs(b)
-proc ModU64(a, b: int64): int64 {.compilerproc.} =
+proc modU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) mod abs(b)
 
-proc Ze(a: int): int {.compilerproc.} =
+proc ze*(a: int): int {.compilerproc.} =
-  result = a
-proc Ze64(a: int64): int64 {.compilerproc.} =
   result = a
 
-proc ToU8(a: int): int8 {.noStackFrame, compilerproc.} =
+proc ze64*(a: int64): int64 {.compilerproc.} =
+  result = a
+
+proc ToU8(a: int): int8 {.asmNoStackFrame, compilerproc.} =
   asm """
     return `a`;
   """
 
-proc ToU16(a: int): int16 {.noStackFrame, compilerproc.} =
+proc ToU16(a: int): int16 {.asmNoStackFrame, compilerproc.} =
   asm """
     return `a`;
   """
 
-proc ToU32(a: int): int32 {.noStackFrame, compilerproc.} =
+proc ToU32(a: int): int32 {.asmNoStackFrame, compilerproc.} =
   asm """
     return `a`;
   """
 
-
 proc nimMin(a, b: int): int {.compilerproc.} = return if a <= b: a else: b
 proc nimMax(a, b: int): int {.compilerproc.} = return if a >= b: a else: b
 
@@ -500,9 +499,9 @@ proc isFatPointer(ti: PNimType): bool =
     tyArray, tyArrayConstr, tyTuple,
     tyOpenArray, tySet, tyVar, tyRef, tyPtr}
 
-proc NimCopy(x: pointer, ti: PNimType): pointer {.compilerproc.}
+proc nimCopy(x: pointer, ti: PNimType): pointer {.compilerproc.}
 
-proc NimCopyAux(dest, src: Pointer, n: ptr TNimNode) {.compilerproc.} =
+proc nimCopyAux(dest, src: Pointer, n: ptr TNimNode) {.compilerproc.} =
   case n.kind
   of nkNone: sysAssert(false, "NimCopyAux")
   of nkSlot:
@@ -518,7 +517,7 @@ proc NimCopyAux(dest, src: Pointer, n: ptr TNimNode) {.compilerproc.} =
       }
     """
 
-proc NimCopy(x: pointer, ti: PNimType): pointer =
+proc nimCopy(x: pointer, ti: PNimType): pointer =
   case ti.kind
   of tyPtr, tyRef, tyVar, tyNil:
     if not isFatPointer(ti):
@@ -586,7 +585,7 @@ proc genericReset(x: Pointer, ti: PNimType): pointer {.compilerproc.} =
     result = nil
 
 proc ArrayConstr(len: int, value: pointer, typ: PNimType): pointer {.
-                 noStackFrame, compilerproc.} =
+                 asmNoStackFrame, compilerproc.} =
   # types are fake
   asm """
     var result = new Array(`len`);
@@ -620,7 +619,7 @@ proc isObj(obj, subclass: PNimType): bool {.compilerproc.} =
     x = x.base
   return true
 
-proc addChar(x: string, c: char) {.compilerproc, noStackFrame.} =
+proc addChar(x: string, c: char) {.compilerproc, asmNoStackFrame.} =
   asm """
     `x`[`x`.length-1] = `c`; `x`.push(0);
   """

lib/system/repr.nim

@@ -59,7 +59,11 @@ proc reprChar(x: char): string {.compilerRtl.} =
 
 proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} =
   # we read an 'int' but this may have been too large, so mask the other bits:
-  let e = e and (1 shl (typ.size*8)-1)
+  let e = if typ.size == 1: e and 0xff
+          elif typ.size == 2: e and 0xffff
+          else: e
+  # XXX we need a proper narrowing based on signedness here
+  #e and ((1 shl (typ.size*8)) - 1)
   if ntfEnumHole notin typ.flags:
     if e <% typ.node.len:
       return $typ.node.sons[e].name