top level closures should work; transf is not a pass anymore; next steps for first class iterator support

compiler/ast.nim

@@ -340,7 +340,8 @@ type
                       # because for instantiations of objects, structural
                       # type equality has to be used
     tfAll,            # type class requires all constraints to be met (default)
-    tfAny             # type class requires any constraint to be met
+    tfAny,            # type class requires any constraint to be met
+    tfCapturesEnv     # whether proc really captures some environment
 
   TTypeFlags* = set[TTypeFlag]
 
@@ -867,6 +868,7 @@ proc newNode*(kind: TNodeKind, info: TLineInfo, sons: TNodeSeq = @[],
   result = newNode(kind)
   result.info = info
   result.typ = typ
+  # XXX use shallowCopy here for ownership transfer:
   result.sons = sons
 
 proc newNodeIT(kind: TNodeKind, info: TLineInfo, typ: PType): PNode = 

compiler/ccgstmts.nim

@@ -93,10 +93,9 @@ template preserveBreakIdx(body: stmt): stmt =
   p.breakIdx = oldBreakIdx
 
 proc genState(p: BProc, n: PNode) =
-  internalAssert n.len == 2 and n.sons[0].kind == nkIntLit
+  internalAssert n.len == 1 and n.sons[0].kind == nkIntLit
   let idx = n.sons[0].intVal
   lineCg(p, cpsStmts, "STATE$1: ;$n", [idx.toRope])
-  genStmts(p, n.sons[1])
 
 proc genGotoState(p: BProc, n: PNode) =
   # we resist the temptation to translate it into duff's device as it later

compiler/cgen.nim

@@ -627,7 +627,7 @@ proc deinitFrame(p: BProc): PRope =
   result = ropecg(p.module, "\t#popFrame();$n")
 
 proc closureSetup(p: BProc, prc: PSym) =
-  if prc.typ.callConv != ccClosure: return
+  if tfCapturesEnv notin prc.typ.flags: return
   # prc.ast[paramsPos].last contains the type we're after:
   var ls = lastSon(prc.ast[paramsPos])
   if ls.kind != nkSym:

compiler/evals.nim

@@ -1366,7 +1366,8 @@ proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
   inc(gNestedEvals)
 
 proc tryEval(c: PEvalContext, n: PNode): PNode =
-  var n = transform(c.module, n)
+  #internalAssert nfTransf in n.flags
+  var n = transformExpr(c.module, n)
   gWhileCounter = evalMaxIterations
   gNestedEvals = evalMaxRecDepth
   result = evalAux(c, n, {})

compiler/lambdalifting.nim

@@ -211,6 +211,7 @@ proc addHiddenParam(routine: PSym, param: PSym) =
   var params = routine.ast.sons[paramsPos]
   param.position = params.len
   addSon(params, newSymNode(param))
+  incl(routine.typ.flags, tfCapturesEnv)
   #echo "produced environment: ", param.id, " for ", routine.name.s
 
 proc isInnerProc(s, outerProc: PSym): bool {.inline.} =
@@ -234,6 +235,10 @@ proc dummyClosureParam(o: POuterContext, i: PInnerContext) =
     IdTablePut(o.lambdasToEnv, i.fn, e)
   if i.closureParam == nil: addClosureParam(i, e)
 
+proc illegalCapture(s: PSym): bool {.inline.} =
+  result = skipTypes(s.typ, abstractInst).kind in {tyVar, tyOpenArray} or
+      s.kind == skResult
+
 proc captureVar(o: POuterContext, i: PInnerContext, local: PSym, 
                 info: TLineInfo) =
   # for inlined variables the owner is still wrong, so it can happen that it's
@@ -241,6 +246,12 @@ proc captureVar(o: POuterContext, i: PInnerContext, local: PSym,
   var it = PEnv(IdTableGet(o.localsToEnv, local))
   if it == nil: return
   
+  if illegalCapture(local) or o.fn.id != local.owner.id:
+    # Currently captures are restricted to a single level of nesting:
+    LocalError(info, errIllegalCaptureX, local.name.s)
+  i.fn.typ.callConv = ccClosure
+  incl(i.fn.typ.flags, tfCapturesEnv)
+
   # we need to remember which inner most closure belongs to this lambda:
   var e = o.currentEnv
   if IdTableGet(o.lambdasToEnv, i.fn) == nil:
@@ -269,12 +280,13 @@ proc interestingVar(s: PSym): bool {.inline.} =
 
 proc gatherVars(o: POuterContext, i: PInnerContext, n: PNode) = 
   # gather used vars for closure generation
+  if n == nil: return
   case n.kind
   of nkSym:
     var s = n.sym
     if interestingVar(s) and i.fn.id != s.owner.id:
       captureVar(o, i, s, n.info)
-    elif isInnerProc(s, o.fn) and s.typ.callConv == ccClosure and s != i.fn:
+    elif isInnerProc(s, o.fn) and tfCapturesEnv in s.typ.flags and s != i.fn:
       # call to some other inner proc; we need to track the dependencies for
       # this:
       let env = PEnv(IdTableGet(o.lambdasToEnv, i.fn))
@@ -287,6 +299,28 @@ proc gatherVars(o: POuterContext, i: PInnerContext, n: PNode) =
     for k in countup(0, sonsLen(n) - 1): 
       gatherVars(o, i, n.sons[k])
 
+proc generateThunk(prc: PNode, dest: PType): PNode =
+  ## Converts 'prc' into '(thunk, nil)' so that it's compatible with
+  ## a closure.
+  
+  # we cannot generate a proper thunk here for GC-safety reasons (see internal
+  # documentation):
+  if gCmd == cmdCompileToEcmaScript: return prc
+  result = newNodeIT(nkClosure, prc.info, dest)
+  var conv = newNodeIT(nkHiddenStdConv, prc.info, dest)
+  conv.add(emptyNode)
+  conv.add(prc)
+  result.add(conv)
+  result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil)))
+
+proc transformOuterConv(n: PNode): PNode =
+  # numeric types need range checks:
+  var dest = skipTypes(n.typ, abstractVarRange)
+  var source = skipTypes(n.sons[1].typ, abstractVarRange)
+  if dest.kind == tyProc:
+    if dest.callConv == ccClosure and source.callConv == ccDefault:
+      result = generateThunk(n.sons[1], dest)
+
 proc makeClosure(prc, env: PSym, info: TLineInfo): PNode =
   result = newNodeIT(nkClosure, info, prc.typ)
   result.add(newSymNode(prc))
@@ -339,6 +373,7 @@ proc searchForInnerProcs(o: POuterContext, n: PNode) =
       gatherVars(o, inner, body)
       # dummy closure param needed?
       if inner.closureParam == nil and n.sym.typ.callConv == ccClosure:
+        assert tfCapturesEnv notin n.sym.typ.flags
         dummyClosureParam(o, inner)
       # only transform if it really needs a closure:
       if inner.closureParam != nil:
@@ -437,7 +472,7 @@ proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode =
                newSymNode(getClosureVar(o, e))))
 
 proc transformOuterProc(o: POuterContext, n: PNode): PNode =
-  # XXX I with I knew where these 'nil' nodes come from: 'array[.. |X]'
+  # XXX I wish I knew where these 'nil' nodes come from: 'array[.. |X]'
   if n == nil: return nil
   case n.kind
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: nil
@@ -481,17 +516,20 @@ proc transformOuterProc(o: POuterContext, n: PNode): PNode =
      nkIteratorDef: 
     # don't recurse here:
     nil
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    let x = transformOuterProc(o, n.sons[1])
+    if x != nil: n.sons[1] = x
+    result = transformOuterConv(n)
   else:
     for i in countup(0, sonsLen(n) - 1):
       let x = transformOuterProc(o, n.sons[i])
       if x != nil: n.sons[i] = x
 
 proc liftLambdas*(fn: PSym, body: PNode): PNode =
-  if body.kind == nkEmpty:
+  if body.kind == nkEmpty or gCmd == cmdCompileToEcmaScript:
     # ignore forward declaration:
     result = body
-  elif (fn.typ == nil or fn.typ.callConv != ccClosure) and 
+  elif not containsNode(body, procDefs):
-      not containsNode(body, procDefs):
     # fast path: no inner procs, so no closure needed:
     result = body
   else:
@@ -506,40 +544,144 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode =
       let param = params.sons[i].sym
       IdTablePut(o.localsToEnv, param, o.currentEnv)
     searchForInnerProcs(o, body)
-    let a = transformOuterProc(o, body)
+    discard transformOuterProc(o, body)
     result = ex
 
-# XXX should 's' be replaced by a tuple ('s', env)?
+proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode =
-
+  if body.kind == nkEmpty or gCmd == cmdCompileToEcmaScript:
-proc liftLambdas*(n: PNode): PNode =
-  assert n.kind in procDefs
-  var s = n.sons[namePos].sym
-  if gCmd == cmdCompileToEcmaScript: return s.getBody
-  result = liftLambdas(s, s.getBody)
-
-proc transformIterator*(fn: PSym, body: PNode): PNode =
-  if body.kind == nkEmpty:
-    # ignore forward declaration:
     result = body
-  # it(a, b) --> (it(a, b), createClosure())
+  else:
-  # it(a, b) --> ?
+    var o = newOuterContext(module)
-  discard """
+    let ex = closureCreationPoint(body)
-  let c = chain(f, g)
+    o.currentEnv = newEnv(module, nil, ex)
+    searchForInnerProcs(o, body)
+    discard transformOuterProc(o, body)
+    result = ex
 
-  for x in c: echo x
+# ------------------- iterator transformation --------------------------------
 
+discard """
   iterator chain[S, T](a, b: *S->T, args: *S): T =
     for x in a(args): yield x
     for x in b(args): yield x
 
+  let c = chain(f, g)
+  for x in c: echo x
+  
   # translated to:
-  
-  
-  
   let c = chain( (f, newClosure(f)), (g, newClosure(g)), newClosure(chain))
-  
-  
 """
 
+type
+  TIterContext {.final, pure.} = object
+    iter, closureParam, state, resultSym: PSym
+    capturedVars: TIntSet
+    tup: PType
 
+proc newIterResult(iter: PSym): PSym =
+  result = newSym(skResult, getIdent":result", iter)
+  result.info = iter.info
+  result.typ = iter.typ.sons[0]
+  incl(result.flags, sfUsed)
+
+proc transfIterBody(c: var TIterContext, n: PNode): PNode =
+  # gather used vars for closure generation
+  if n == nil: return nil
+  case n.kind
+  of nkSym:
+    var s = n.sym
+    if interestingVar(s) and c.iter.id == s.owner.id:
+      if not containsOrIncl(c.capturedVars, s.id): addField(c.tup, s)
+      result = indirectAccess(newSymNode(c.closureParam), s, n.info)
+  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: nil
+  of nkYieldStmt:
+    inc c.state.typ.n.sons[1].intVal
+    let stateNo = c.state.typ.n.sons[1].intVal
+
+    var stateAsgnStmt = newNodeI(nkAsgn, n.info)
+    stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),c.state,n.info))
+    stateAsgnStmt.add(newIntNode(nkIntLit, stateNo))
+
+    var retStmt = newNodeI(nkReturnStmt, n.info)
+    if n.sons[0].kind != nkEmpty:
+      var a = newNodeI(nkAsgn, n.sons[0].info)
+      addSon(a, newSymNode(c.resultSym))
+      addSon(a, n.sons[0])
+      retStmt.add(a)
+    else:
+      retStmt.add(emptyNode)
+    
+    var stateLabelStmt = newNodeI(nkState, n.info)
+    stateLabelStmt.add(newIntNode(nkIntLit, stateNo-1))
+    
+    result = newNodeI(nkStmtList, n.info)
+    result.add(stateAsgnStmt)
+    result.add(retStmt)
+    result.add(stateLabelStmt)
+  else:
+    for i in countup(0, sonsLen(n)-1):
+      let x = transfIterBody(c, n.sons[i])
+      if x != nil: n.sons[i] = x
+
+proc getStateType(iter: PSym): PType =
+  var n = newNodeI(nkRange, iter.info)
+  addSon(n, newIntNode(nkIntLit, -1))
+  addSon(n, newIntNode(nkIntLit, 0))
+  result = newType(tyRange, iter)
+  result.n = n
+  rawAddSon(result, getSysType(tyInt))
+
+proc liftIterator*(iter: PSym, body: PNode): PNode =
+  var c: TIterContext
+  c.iter = iter
+  c.capturedVars = initIntSet()
+
+  c.tup = newType(tyTuple, iter)
+  c.tup.n = newNodeI(nkRecList, iter.info)
+
+  var cp = newSym(skParam, getIdent(paramname), iter)
+  cp.info = iter.info
+  incl(cp.flags, sfFromGeneric)
+  cp.typ = newType(tyRef, iter)
+  rawAddSon(cp.typ, c.tup)
+  c.closureParam = cp
+  addHiddenParam(iter, cp)
+
+  c.state = newSym(skField, getIdent(":state"), iter)
+  c.state.typ = getStateType(iter)
+  addField(c.tup, c.state)
+
+  if iter.typ.sons[0] != nil:
+    c.resultSym = newIterResult(iter)
+    iter.ast.add(newSymNode(c.resultSym))
+
+  result = newNodeI(nkStmtList, iter.info)
+  var gs = newNodeI(nkGotoState, iter.info)
+  gs.add(indirectAccess(newSymNode(c.closureParam), c.state, iter.info))
+  result.add(gs)
+  var state0 = newNodeI(nkState, iter.info)
+  state0.add(newIntNode(nkIntLit, 0))
+  result.add(state0)
+  
+  let newBody = transfIterBody(c, body)
+  if newBody != nil:
+    result.add(newBody)
+  else:
+    result.add(body)
+    
+  var state1 = newNodeI(nkState, iter.info)
+  state1.add(newIntNode(nkIntLit, -1))
+  result.add(state1)
+
+proc transformForLoop*(iter: PSym, body: PNode): PNode =
+  discard """
+  for i in foo(): nil
+
+Is transformed to:
+  
+  cl = createClosure()
+  while true:
+    let i = foo(cl)
+    if cl.state == -1: break
+"""
   

compiler/main.nim

@@ -15,7 +15,7 @@ import
   os, lists, condsyms, rodread, rodwrite, ropes, trees, 
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
   cgen, ecmasgen,
-  platform, nimconf, importer, passaux, depends, transf, evals, types, idgen,
+  platform, nimconf, importer, passaux, depends, evals, types, idgen,
   tables, docgen2
 
 const
@@ -90,7 +90,6 @@ proc CompileProject(projectFile = gProjectFull) =
 proc semanticPasses =
   registerPass(verbosePass())
   registerPass(sem.semPass())
-  registerPass(transf.transfPass())
 
 proc CommandGenDepend =
   semanticPasses()

compiler/msgs.nim

@@ -25,7 +25,8 @@ type
     errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected, 
     errInvalidPragma, errUnknownPragma, errInvalidDirectiveX, 
     errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation, 
-    errExceptionExpected, errExceptionAlreadyHandled, errYieldNotAllowedHere, 
+    errExceptionExpected, errExceptionAlreadyHandled, 
+    errYieldNotAllowedHere, errYieldNotAllowedInTryStmt, 
     errInvalidNumberOfYieldExpr, errCannotReturnExpr, errAttemptToRedefine, 
     errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel, 
     errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected, 
@@ -149,7 +150,8 @@ const
     errInvalidIndentation: "invalid indentation", 
     errExceptionExpected: "exception expected", 
     errExceptionAlreadyHandled: "exception already handled", 
-    errYieldNotAllowedHere: "\'yield\' only allowed in a loop of an iterator", 
+    errYieldNotAllowedHere: "'yield' only allowed in an iterator",
+    errYieldNotAllowedInTryStmt: "'yield' cannot be used within 'try' in a non-inlined iterator",
     errInvalidNumberOfYieldExpr: "invalid number of \'yield\' expresions", 
     errCannotReturnExpr: "current routine cannot return an expression", 
     errAttemptToRedefine: "redefinition of \'$1\'", 

compiler/sem.nim

@@ -14,7 +14,7 @@ import
   wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
   magicsys, parser, nversion, nimsets, semfold, importer,
   procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
-  semthreads, intsets, transf, evals, idgen, aliases
+  semthreads, intsets, transf, evals, idgen, aliases, cgmeth
 
 proc semPass*(): TPass
 # implementation
@@ -182,6 +182,7 @@ proc myOpen(module: PSym, filename: string): PPassContext =
 proc myOpenCached(module: PSym, filename: string, 
                   rd: PRodReader): PPassContext = 
   result = myOpen(module, filename)
+  for m in items(rd.methods): methodDef(m, true)
 
 proc SemStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = 
   result = semStmt(c, n)
@@ -193,6 +194,7 @@ proc SemStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
       # a generic has been added to `a`:
       if result.kind != nkEmpty: addSon(a, result)
       result = a
+  result = transformStmt(c.module, result)
 
 proc RecoverContext(c: PContext) = 
   # clean up in case of a semantic error: We clean up the stacks, etc. This is

compiler/semdata.nim

@@ -31,6 +31,8 @@ type
     resultSym*: PSym          # the result symbol (if we are in a proc)
     nestedLoopCounter*: int   # whether we are in a loop or not
     nestedBlockCounter*: int  # whether we are in a block or not
+    InTryStmt*: int           # whether we are in a try statement; works also
+                              # in standalone ``except`` and ``finally``
     next*: PProcCon           # used for stacking procedure contexts
   
   TInstantiatedSymbol* {.final.} = object
@@ -85,36 +87,6 @@ proc newGenericsCache*(): PGenericsCache =
   initIdTable(result.InstTypes)
   result.generics = @[]
 
-proc tempContext*(c: PContext): PContext =
-  ## generates a temporary context so that side-effects can be rolled-back;
-  ## necessary for ``system.compiles``.
-  new(result)
-  result.module = c.module
-  result.p = c.p
-  # don't use the old cache:
-  result.generics = newGenericsCache()
-  result.friendModule = c.friendModule
-  result.InstCounter = c.InstCounter
-  result.threadEntries = @[]
-  # hrm, 'tab' is expensive to copy ... so we don't. We open a new scope
-  # instead to be able to undo scope changes. Not entirely correct for
-  # explicit 'global' vars though:
-  #shallowCopy(result.tab, c.tab)
-  assign(result.AmbiguousSymbols, c.AmbiguousSymbols)
-  result.InGenericContext = c.InGenericContext
-  result.InUnrolledContext = c.InUnrolledContext
-  result.InCompilesContext = c.InCompilesContext
-  result.converters = c.converters
-  result.semConstExpr = c.semConstExpr
-  result.semExpr = c.semExpr
-  result.semConstBoolExpr = c.semConstBoolExpr
-  assign(result.includedFiles, c.includedFiles) 
-  result.filename = c.filename
-  #shallowCopy(result.userPragmas, c.userPragmas) 
-  # XXX mark it as read-only:
-  result.evalContext = c.evalContext
-  
-
 proc newContext*(module: PSym, nimfile: string): PContext
 
 proc lastOptionEntry*(c: PContext): POptionEntry

compiler/semexprs.nim

@@ -65,10 +65,6 @@ proc inlineConst(n: PNode, s: PSym): PNode {.inline.} =
   result.typ = s.typ
   result.info = n.info
 
-proc illegalCapture(s: PSym): bool {.inline.} =
-  result = skipTypes(s.typ, abstractInst).kind in {tyVar, tyOpenArray} or
-      s.kind == skResult
-
 proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = 
   case s.kind
   of skProc, skMethod, skIterator, skConverter: 
@@ -111,13 +107,11 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
       incl(c.p.owner.flags, sfSideEffect)
     elif s.kind == skParam and s.typ.kind == tyExpr:
       return s.typ.n
-    elif s.owner != c.p.owner and 
-        c.p.owner.typ != nil and not IsGenericRoutine(s.owner):
-      c.p.owner.typ.callConv = ccClosure
-      if illegalCapture(s) or c.p.next.owner != s.owner:
-        # Currently captures are restricted to a single level of nesting:
-        LocalError(n.info, errIllegalCaptureX, s.name.s)
     result = newSymNode(s, n.info)
+    # We cannot check for access to outer vars for example because it's still
+    # not sure the symbol really ends up being used:
+    # var len = 0 # but won't be called
+    # genericThatUsesLen(x) # marked as taking a closure?
   of skGenericParam:
     if s.ast != nil: result = semExpr(c, s.ast)
     else:
@@ -1179,7 +1173,7 @@ proc semCompiles(c: PContext, n: PNode, flags: TExprFlags): PNode =
   
   let oldInGenericContext = c.InGenericContext
   let oldInUnrolledContext = c.InUnrolledContext
-  
+  let oldProcCon = c.p
   c.generics = newGenericsCache()
   try:
     discard semExpr(c, n.sons[1])
@@ -1190,6 +1184,7 @@ proc semCompiles(c: PContext, n: PNode, flags: TExprFlags): PNode =
   c.generics = oldGenerics
   c.InGenericContext = oldInGenericContext
   c.InUnrolledContext = oldInUnrolledContext
+  c.p = oldProcCon
   msgs.setInfoContextLen(oldContextLen)
   setlen(gOwners, oldOwnerLen)
   while c.tab.tos > oldTos: rawCloseScope(c.tab)

compiler/seminst.nim

@@ -79,8 +79,7 @@ proc instantiateBody(c: PContext, n: PNode, result: PSym) =
       addResult(c, result.typ.sons[0], n.info, result.kind)
       addResultNode(c, n)
     var b = semStmtScope(c, n.sons[bodyPos])
-    # XXX Bad hack for tests/titer2 and tests/tactiontable
+    n.sons[bodyPos] = transformBody(c.module, b, result)
-    n.sons[bodyPos] = transform(c.module, b)
     #echo "code instantiated ", result.name.s
     excl(result.flags, sfForward)
     popProcCon(c)

compiler/semstmts.nim

@@ -203,6 +203,8 @@ proc SemYield(c: PContext, n: PNode): PNode =
   checkSonsLen(n, 1)
   if c.p.owner == nil or c.p.owner.kind != skIterator:
     LocalError(n.info, errYieldNotAllowedHere)
+  elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
+    LocalError(n.info, errYieldNotAllowedInTryStmt)
   elif n.sons[0].kind != nkEmpty:
     n.sons[0] = SemExprWithType(c, n.sons[0]) # check for type compatibility:
     var restype = c.p.owner.typ.sons[0]
@@ -486,6 +488,7 @@ proc semRaise(c: PContext, n: PNode): PNode =
 
 proc semTry(c: PContext, n: PNode): PNode = 
   result = n
+  inc c.p.inTryStmt
   checkMinSonsLen(n, 2)
   n.sons[0] = semStmtScope(c, n.sons[0])
   var check = initIntSet()
@@ -511,6 +514,7 @@ proc semTry(c: PContext, n: PNode): PNode =
       illFormedAst(n) 
     # last child of an nkExcept/nkFinally branch is a statement:
     a.sons[length - 1] = semStmtScope(c, a.sons[length - 1])
+  dec c.p.inTryStmt
 
 proc addGenericParamListToScope(c: PContext, n: PNode) = 
   if n.kind != nkGenericParams: 
@@ -714,20 +718,33 @@ proc semLambda(c: PContext, n: PNode): PNode =
       LocalError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s)
     pushProcCon(c, s)
     addResult(c, s.typ.sons[0], n.info, skProc)
-    n.sons[bodyPos] = semStmtScope(c, n.sons[bodyPos])
+    let semBody = semStmtScope(c, n.sons[bodyPos])
+    n.sons[bodyPos] = transformBody(c.module, semBody, s)
     addResultNode(c, n)
     popProcCon(c)
   else:
     LocalError(n.info, errImplOfXexpected, s.name.s)
   sideEffectsCheck(c, s)
-  if s.typ.callConv == ccClosure and s.owner.kind == skModule:
-    localError(s.info, errXCannotBeClosure, s.name.s)
   closeScope(c.tab)           # close scope for parameters
   popOwner()
   result.typ = s.typ
 
 proc instantiateDestructor*(c: PContext, typ: PType): bool
 
+proc doDestructorStuff(c: PContext, s: PSym, n: PNode) =
+  let t = s.typ.sons[1].skipTypes({tyVar})
+  t.destructor = s
+  # automatically insert calls to base classes' destructors
+  if n.sons[bodyPos].kind != nkEmpty:
+    for i in countup(0, t.sonsLen - 1):
+      # when inheriting directly from object
+      # there will be a single nil son
+      if t.sons[i] == nil: continue
+      if instantiateDestructor(c, t.sons[i]):
+        n.sons[bodyPos].addSon(newNode(nkCall, t.sym.info, @[
+            useSym(t.sons[i].destructor),
+            n.sons[paramsPos][1][0]]))
+
 proc semProcAux(c: PContext, n: PNode, kind: TSymKind, 
                 validPragmas: TSpecialWords): PNode = 
   result = semProcAnnotation(c, n)
@@ -791,19 +808,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     popOwner()
     pushOwner(s)
   s.options = gOptions
-  if sfDestructor in s.flags:
+  if sfDestructor in s.flags: doDestructorStuff(c, s, n)
-    let t = s.typ.sons[1].skipTypes({tyVar})
-    t.destructor = s
-    # automatically insert calls to base classes' destructors
-    if n.sons[bodyPos].kind != nkEmpty:
-      for i in countup(0, t.sonsLen - 1):
-        # when inheriting directly from object
-        # there will be a single nil son
-        if t.sons[i] == nil: continue
-        if instantiateDestructor(c, t.sons[i]):
-          n.sons[bodyPos].addSon(newNode(nkCall, t.sym.info, @[
-              useSym(t.sons[i].destructor),
-              n.sons[paramsPos][1][0]]))
   if n.sons[bodyPos].kind != nkEmpty: 
     # for DLL generation it is annoying to check for sfImportc!
     if sfBorrow in s.flags: 
@@ -815,7 +820,10 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         addResult(c, s.typ.sons[0], n.info, kind)
       if sfImportc notin s.flags:
         # no semantic checking for importc:
-        n.sons[bodyPos] = semStmtScope(c, n.sons[bodyPos])
+        let semBody = semStmtScope(c, n.sons[bodyPos])
+        # unfortunately we cannot skip this step when in 'system.compiles'
+        # context as it may even be evaluated in 'system.compiles':
+        n.sons[bodyPos] = transformBody(c.module, semBody, s)
       if s.typ.sons[0] != nil and kind != skIterator: addResultNode(c, n)
       popProcCon(c)
     else: 
@@ -833,8 +841,6 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
       incl(s.flags, sfForward)
     elif sfBorrow in s.flags: semBorrow(c, n, s)
   sideEffectsCheck(c, s)
-  if s.typ.callConv == ccClosure and s.owner.kind == skModule:
-    localError(s.info, errXCannotBeClosure, s.name.s)
   closeScope(c.tab)           # close scope for parameters
   popOwner()
   
@@ -845,7 +851,10 @@ proc semIterator(c: PContext, n: PNode): PNode =
   if t.sons[0] == nil:
     LocalError(n.info, errXNeedsReturnType, "iterator")
   # iterators are either 'inline' or 'closure':
-  if s.typ.callConv != ccInline: s.typ.callConv = ccClosure
+  if s.typ.callConv != ccInline: 
+    s.typ.callConv = ccClosure
+    # and they always at least use the 'env' for the state field:
+    incl(s.typ.flags, tfCapturesEnv)
   if n.sons[bodyPos].kind == nkEmpty and s.magic == mNone:
     LocalError(n.info, errImplOfXexpected, s.name.s)
   

compiler/transf.nim

@@ -25,7 +25,6 @@ import
 const 
   genPrefix* = ":tmp"         # prefix for generated names
 
-proc transfPass*(): TPass
 # implementation
 
 type 
@@ -109,54 +108,6 @@ proc transformSons(c: PTransf, n: PNode): PTransNode =
   for i in countup(0, sonsLen(n)-1): 
     result[i] = transform(c, n.sons[i])
 
-# Transforming iterators into non-inlined versions is pretty hard, but
-# unavoidable for not bloating the code too much. If we had direct access to
-# the program counter, things'd be much easier.
-# ::
-#
-#  iterator items(a: string): char =
-#    var i = 0
-#    while i < length(a):
-#      yield a[i]
-#      inc(i)
-#
-#  for ch in items("hello world"): # `ch` is an iteration variable
-#    echo(ch)
-#
-# Should be transformed into::
-#
-#  type
-#    TItemsClosure = record
-#      i: int
-#      state: int
-#  proc items(a: string, c: var TItemsClosure): char =
-#    case c.state
-#    of 0: goto L0 # very difficult without goto!
-#    of 1: goto L1 # can be implemented by GCC's computed gotos
-#
-#    block L0:
-#      c.i = 0
-#      while c.i < length(a):
-#        c.state = 1
-#        return a[i]
-#        block L1: inc(c.i)
-#
-# More efficient, but not implementable::
-#
-#  type
-#    TItemsClosure = record
-#      i: int
-#      pc: pointer
-#
-#  proc items(a: string, c: var TItemsClosure): char =
-#    goto c.pc
-#    c.i = 0
-#    while c.i < length(a):
-#      c.pc = label1
-#      return a[i]
-#      label1: inc(c.i)
-#
-
 proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode = 
   result = newTransNode(nkFastAsgn, PNode(ri).info, 2)
   result[0] = PTransNode(le)
@@ -178,15 +129,6 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
     if result != nil: return
     tc = tc.next
   result = b
-  when false:
-    case b.sym.kind
-    of skConst, skEnumField: 
-      if sfFakeConst notin b.sym.flags:
-        if skipTypes(b.sym.typ, abstractInst).kind notin ConstantDataTypes: 
-          result = getConstExpr(c.module, b)
-          if result == nil: InternalError(b.info, "transformSym: const")
-    else: 
-      nil
 
 proc transformSym(c: PTransf, n: PNode): PTransNode = 
   result = PTransNode(transformSymAux(c, n))
@@ -379,20 +321,6 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode =
       # addr ( deref ( x )) --> x
       result = PTransNode(n.sons[0].sons[0])
   
-proc generateThunk(c: PTransf, prc: PNode, dest: PType): PNode =
-  ## Converts 'prc' into '(thunk, nil)' so that it's compatible with
-  ## a closure.
-  
-  # we cannot generate a proper thunk here for GC-safety reasons (see internal
-  # documentation):
-  if gCmd == cmdCompileToEcmaScript: return prc
-  result = newNodeIT(nkClosure, prc.info, dest)
-  var conv = newNodeIT(nkHiddenStdConv, prc.info, dest)
-  conv.add(emptyNode)
-  conv.add(prc)
-  result.add(conv)
-  result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil)))
-  
 proc transformConv(c: PTransf, n: PNode): PTransNode = 
   # numeric types need range checks:
   var dest = skipTypes(n.typ, abstractVarRange)
@@ -466,12 +394,6 @@ proc transformConv(c: PTransf, n: PNode): PTransNode =
       result[0] = transform(c, n.sons[1])
     else: 
       result = transform(c, n.sons[1])
-  of tyProc:
-    if dest.callConv == ccClosure and source.callConv == ccDefault:
-      let x = transform(c, n.sons[1]).pnode
-      result = generateThunk(c, x, dest).ptransnode
-    else:
-      result = transformSons(c, n)    
   of tyGenericParam, tyOrdinal, tyTypeClass:
     result = transform(c, n.sons[1])
     # happens sometimes for generated assignments, etc.
@@ -657,21 +579,23 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     result = PTransNode(n)
   of nkBracketExpr: result = transformArrayAccess(c, n)
   of procDefs:
-    if n.sons[genericParamsPos].kind == nkEmpty:
+    when false:
-      var s = n.sons[namePos].sym
+      if n.sons[genericParamsPos].kind == nkEmpty:
-      n.sons[bodyPos] = PNode(transform(c, s.getBody))
+        var s = n.sons[namePos].sym
-      if s.ast.sons[bodyPos] != n.sons[bodyPos]:
+        n.sons[bodyPos] = PNode(transform(c, s.getBody))
-        # somehow this can happen ... :-/
+        if s.ast.sons[bodyPos] != n.sons[bodyPos]:
-        s.ast.sons[bodyPos] = n.sons[bodyPos]
+          # somehow this can happen ... :-/
-      n.sons[bodyPos] = liftLambdas(n)
+          s.ast.sons[bodyPos] = n.sons[bodyPos]
-      if n.kind == nkMethodDef: methodDef(s, false)
+        #n.sons[bodyPos] = liftLambdas(s, n)
+        #if n.kind == nkMethodDef: methodDef(s, false)
     result = PTransNode(n)
   of nkMacroDef:
     # XXX no proper closure support yet:
-    if n.sons[genericParamsPos].kind == nkEmpty:
+    when false:
-      var s = n.sons[namePos].sym
+      if n.sons[genericParamsPos].kind == nkEmpty:
-      n.sons[bodyPos] = PNode(transform(c, s.getBody))
+        var s = n.sons[namePos].sym
-      if n.kind == nkMethodDef: methodDef(s, false)
+        n.sons[bodyPos] = PNode(transform(c, s.getBody))
+        if n.kind == nkMethodDef: methodDef(s, false)
     result = PTransNode(n)
   of nkForStmt: 
     inc c.inLoop
@@ -735,38 +659,57 @@ proc transform(c: PTransf, n: PNode): PTransNode =
   if cnst != nil and not dontInlineConstant(n, cnst):
     result = PTransNode(cnst) # do not miss an optimization
 
-proc processTransf(context: PPassContext, n: PNode): PNode = 
+proc processTransf(c: PTransf, n: PNode): PNode = 
   # Note: For interactive mode we cannot call 'passes.skipCodegen' and skip
   # this step! We have to rely that the semantic pass transforms too errornous
   # nodes into an empty node.
-  if passes.skipCodegen(n) or context.fromCache or nfTransf in n.flags: return n
+  if passes.skipCodegen(n) or c.fromCache or nfTransf in n.flags: return n
-  var c = PTransf(context)
   pushTransCon(c, newTransCon(getCurrOwner(c)))
   result = PNode(transform(c, n))
   popTransCon(c)
   incl(result.flags, nfTransf)
 
-proc openTransf(module: PSym, filename: string): PPassContext = 
+proc openTransf(module: PSym, filename: string): PTransf = 
-  var n: PTransf
+  new(result)
-  new(n)
+  result.contSyms = @[]
-  n.contSyms = @[]
+  result.breakSyms = @[]
-  n.breakSyms = @[]
+  result.module = module
-  n.module = module
-  result = n
 
-proc openTransfCached(module: PSym, filename: string, 
+when false:
-                      rd: PRodReader): PPassContext = 
+  proc openTransfCached(module: PSym, filename: string, 
-  result = openTransf(module, filename)
+                        rd: PRodReader): PPassContext = 
-  for m in items(rd.methods): methodDef(m, true)
+    result = openTransf(module, filename)
+    for m in items(rd.methods): methodDef(m, true)
 
-proc transfPass(): TPass = 
+  proc transfPass(): TPass = 
-  initPass(result)
+    initPass(result)
-  result.open = openTransf
+    result.open = openTransf
-  result.openCached = openTransfCached
+    result.openCached = openTransfCached
-  result.process = processTransf
+    result.process = processTransf
-  result.close = processTransf # we need to process generics too!
+    result.close = processTransf # we need to process generics too!
   
-proc transform*(module: PSym, n: PNode): PNode =
+proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
+  if nfTransf in n.flags or prc.kind in {skTemplate, skMacro}:
+    result = n
+  else:
+    var c = openTransf(module, "")
+    result = processTransf(c, n)
+    if prc.kind != skMacro:
+      # XXX no closures yet for macros:
+      result = liftLambdas(prc, result)
+    if prc.kind == skMethod: methodDef(prc, false)
+    incl(result.flags, nfTransf)
+
+proc transformStmt*(module: PSym, n: PNode): PNode =
+  if nfTransf in n.flags:
+    result = n
+  else:
+    var c = openTransf(module, "")
+    result = processTransf(c, n)
+    result = liftLambdasForTopLevel(module, result)
+    incl(result.flags, nfTransf)
+
+proc transformExpr*(module: PSym, n: PNode): PNode =
   if nfTransf in n.flags:
     result = n
   else:

lib/core/typeinfo.nim

@@ -455,7 +455,7 @@ proc getFloat*(x: TAny): float =
 
 proc getFloat32*(x: TAny): float32 = 
   ## retrieve the float32 value out of `x`. `x` needs to represent an float32.
-  assert skipRange(x.rawtype).kind == tyFloat64
+  assert skipRange(x.rawtype).kind == tyFloat32
   result = cast[ptr float32](x.value)[]
   
 proc getFloat64*(x: TAny): float64 = 

tests/run/tunittests.nim

@@ -1,3 +1 @@
-import utemplates
+import utemplates, uclosures
-
-# uclosures

tests/run/uclosures.nim

@@ -4,7 +4,8 @@ test "loop variables are captured by copy":
   var funcs: seq[proc (): int {.closure.}] = @[]
   
   for i in 0..10:
-    funcs.add do -> int: return i * i
+    let ii = i
+    funcs.add do -> int: return ii * ii
 
   check funcs[0]() == 0
   check funcs[3]() == 9

todo.txt

@@ -1,14 +1,14 @@
 version 0.9.0
 =============
 
-- closure: implement closure support for procs capturing nested
+- implement "closure tuple consists of a single 'ref'" optimization
-  module vars
+- implement for loop transformation for first class iterators
-- implicit deref for parameter matching
 
+- implicit deref for parameter matching; implement ``varargs[T, `$`]``
 - optimize genericAssign in the code generator
+- the lookup rules for generics really are too permissive
 - fix remaining closure bugs:
   - fix evals.nim with closures
-  - implement "closure tuple consists of a single 'ref'" optimization
 
 
 Bugs