bugfix: stack traces; first class iterators almost working

compiler/ast.nim

@@ -205,7 +205,8 @@ type
     nkReturnToken,        # token used for interpretation
     nkClosure,            # (prc, env)-pair (internally used for code gen)
     nkGotoState,          # used for the state machine (for iterators)
-    nkState               # give a label to a code section (for iterators)
+    nkState,              # give a label to a code section (for iterators)
+    nkBreakState          # special break statement for easier code generation
   TNodeKinds* = set[TNodeKind]
 
 type

compiler/ccgexprs.nim

@@ -1678,7 +1678,7 @@ proc expr(p: BProc, e: PNode, d: var TLoc) =
       else:
         genProc(p.module, sym)
       putLocIntoDest(p, d, sym.loc)
-    of skProc, skConverter:
+    of skProc, skConverter, skIterator:
       genProc(p.module, sym)
       if sym.loc.r == nil or sym.loc.t == nil:
         InternalError(e.info, "expr: proc not init " & sym.name.s)

compiler/ccgstmts.nim

@@ -106,10 +106,17 @@ proc genGotoState(p: BProc, n: PNode) =
   var a: TLoc
   initLocExpr(p, n.sons[0], a)
   lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)])
+  p.BeforeRetNeeded = true
+  lineF(p, cpsStmts, "case -1: goto BeforeRet;$n", [])
   for i in 0 .. lastOrd(n.sons[0].typ):
     lineF(p, cpsStmts, "case $1: goto STATE$1;$n", [toRope(i)])
   lineF(p, cpsStmts, "}$n", [])
 
+proc genBreakState(p: BProc, n: PNode) =
+  var a: TLoc
+  initLocExpr(p, n.sons[0], a)
+  lineF(p, cpsStmts, "if (($1) < 0) break;$n", [rdLoc(a)])
+
 proc genSingleVar(p: BProc, a: PNode) =
   var v = a.sons[0].sym
   if sfCompileTime in v.flags: return
@@ -713,7 +720,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode): PRope =
       app(result, t.sons[i].strVal)
     of nkSym: 
       var sym = t.sons[i].sym
-      if sym.kind in {skProc, skMethod}: 
+      if sym.kind in {skProc, skIterator, skMethod}: 
         var a: TLoc
         initLocExpr(p, t.sons[i], a)
         app(result, rdLoc(a))
@@ -884,5 +891,6 @@ proc genStmts(p: BProc, t: PNode) =
   of nkParForStmt: genParForStmt(p, t)
   of nkState: genState(p, t)
   of nkGotoState: genGotoState(p, t)
+  of nkBreakState: genBreakState(p, t)
   else: internalError(t.info, "genStmts(" & $t.kind & ')')
   

compiler/ccgtypes.nim

@@ -69,7 +69,7 @@ proc mangleName(s: PSym): PRope =
   if result == nil: 
     if gCmd == cmdCompileToLLVM: 
       case s.kind
-      of skProc, skMethod, skConverter, skConst: 
+      of skProc, skMethod, skConverter, skConst, skIterator: 
         result = toRope("@")
       of skVar, skForVar, skResult, skLet: 
         if sfGlobal in s.flags: result = toRope("@")

compiler/cgen.nim

@@ -589,7 +589,7 @@ proc cgsym(m: BModule, name: string): PRope =
   var sym = magicsys.getCompilerProc(name)
   if sym != nil: 
     case sym.kind
-    of skProc, skMethod, skConverter: genProc(m, sym)
+    of skProc, skMethod, skConverter, skIterator: genProc(m, sym)
     of skVar, skResult, skLet: genVarPrototype(m, sym)
     of skType: discard getTypeDesc(m, sym.typ)
     else: InternalError("cgsym: " & name)

compiler/lambdalifting.nim

@@ -214,8 +214,14 @@ proc addHiddenParam(routine: PSym, param: PSym) =
   incl(routine.typ.flags, tfCapturesEnv)
   #echo "produced environment: ", param.id, " for ", routine.name.s
 
+proc getHiddenParam(routine: PSym): PSym =
+  let params = routine.ast.sons[paramsPos]
+  let hidden = lastSon(params)
+  assert hidden.kind == nkSym
+  result = hidden.sym
+
 proc isInnerProc(s, outerProc: PSym): bool {.inline.} =
-  result = s.kind in {skProc, skIterator, skMethod, skConverter} and
+  result = s.kind in {skProc, skMethod, skConverter} and
     s.owner == outerProc and not isGenericRoutine(s)
   #s.typ.callConv == ccClosure
 
@@ -481,7 +487,6 @@ proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode =
                newSymNode(getClosureVar(o, e))))
 
 proc transformOuterProc(o: POuterContext, n: PNode): PNode =
-  # XXX I wish I knew where these 'nil' nodes come from: 'array[.. |X]'
   if n == nil: return nil
   case n.kind
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: nil
@@ -593,13 +598,16 @@ proc newIterResult(iter: PSym): PSym =
   result.typ = iter.typ.sons[0]
   incl(result.flags, sfUsed)
 
+proc interestingIterVar(s: PSym): bool {.inline.} =
+  result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags
+
 proc transfIterBody(c: var TIterContext, n: PNode): PNode =
   # gather used vars for closure generation
   if n == nil: return nil
   case n.kind
   of nkSym:
     var s = n.sym
-    if interestingVar(s) and c.iter.id == s.owner.id:
+    if interestingIterVar(s) and c.iter.id == s.owner.id:
       if not containsOrIncl(c.capturedVars, s.id): addField(c.tup, s)
       result = indirectAccess(newSymNode(c.closureParam), s, n.info)
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: nil
@@ -609,19 +617,20 @@ proc transfIterBody(c: var TIterContext, n: PNode): PNode =
 
     var stateAsgnStmt = newNodeI(nkAsgn, n.info)
     stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),c.state,n.info))
-    stateAsgnStmt.add(newIntNode(nkIntLit, stateNo))
+    stateAsgnStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt)))
 
     var retStmt = newNodeI(nkReturnStmt, n.info)
     if n.sons[0].kind != nkEmpty:
       var a = newNodeI(nkAsgn, n.sons[0].info)
+      var retVal = transfIterBody(c, n.sons[0])
       addSon(a, newSymNode(c.resultSym))
-      addSon(a, n.sons[0])
+      addSon(a, if retVal.isNil: n.sons[0] else: retVal)
       retStmt.add(a)
     else:
       retStmt.add(emptyNode)
     
     var stateLabelStmt = newNodeI(nkState, n.info)
-    stateLabelStmt.add(newIntNode(nkIntLit, stateNo-1))
+    stateLabelStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt)))
     
     result = newNodeI(nkStmtList, n.info)
     result.add(stateAsgnStmt)
@@ -677,22 +686,81 @@ proc liftIterator*(iter: PSym, body: PNode): PNode =
   else:
     result.add(body)
 
-  var state1 = newNodeI(nkState, iter.info)
+  var stateAsgnStmt = newNodeI(nkAsgn, iter.info)
-  state1.add(newIntNode(nkIntLit, -1))
+  stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),
-  result.add(state1)
+                    c.state,iter.info))
+  stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt)))
+  result.add(stateAsgnStmt)
 
-proc transformForLoop*(iter: PSym, body: PNode): PNode =
+proc liftForLoop*(body: PNode): PNode =
+  # BIG problem ahead: the iterator could be invoked indirectly, but then
+  # we don't know what environment to create here: 
+  # 
+  # iterator count(): int =
+  #   yield 0
+  # 
+  # iterator count2(): int =
+  #   var x = 3
+  #   yield x
+  #   inc x
+  #   yield x
+  # 
+  # proc invoke(iter: iterator(): int) =
+  #   for x in iter(): echo x
+  #
+  # --> When to create the closure? --> for the (count) occurence!
   discard """
-  for i in foo(): nil
+      for i in foo(): ...
 
-Is transformed to:
+    Is transformed to:
       
       cl = createClosure()
       while true:
         let i = foo(cl)
-    if cl.state == -1: break
+        nkBreakState(cl.state)
-"""
+        ...
-  InternalAssert body.kind == nkForStmt
+    """
-  # gather vars in a tuple:
   var L = body.len
+  InternalAssert body.kind == nkForStmt and body[L-2].kind in nkCallKinds
+  var call = body[L-2]
 
+  result = newNodeI(nkStmtList, body.info)
+  
+  # static binding?
+  var env: PSym
+  if call[0].kind == nkSym and call[0].sym.kind == skIterator:
+    # createClose()
+    let iter = call[0].sym
+    assert iter.kind == skIterator
+    env = copySym(getHiddenParam(iter))
+
+    var v = newNodeI(nkVarSection, body.info)
+    addVar(v, newSymNode(env))
+    result.add(v)
+    # add 'new' statement:
+    result.add(newCall(getSysSym"internalNew", env))
+  
+  var loopBody = newNodeI(nkStmtList, body.info, 3)
+  var whileLoop = newNodeI(nkWhileStmt, body.info, 2)
+  whileLoop.sons[0] = newIntTypeNode(nkIntLit, 1, getSysType(tyBool))
+  whileLoop.sons[1] = loopBody
+  result.add whileLoop
+  
+  # setup loopBody:
+  # gather vars in a tuple:
+  var v2 = newNodeI(nkLetSection, body.info)
+  var vpart = newNodeI(if L == 3: nkIdentDefs else: nkVarTuple, body.info)
+  for i in 0 .. L-3: addSon(vpart, body[i])
+
+  addSon(vpart, ast.emptyNode) # no explicit type
+  if not env.isnil:
+    call.sons[0] = makeClosure(call.sons[0].sym, env, body.info)
+  addSon(vpart, call)
+  addSon(v2, vpart)
+
+  loopBody.sons[0] = v2
+  var bs = newNodeI(nkBreakState, body.info)
+  bs.addSon(indirectAccess(env, 
+      newSym(skField, getIdent(":state"), env, env.info), body.info))
+  loopBody.sons[1] = bs
+  loopBody.sons[2] = body[L-1]

compiler/semstmts.nim

@@ -817,7 +817,12 @@ proc semIterator(c: PContext, n: PNode): PNode =
   var t = s.typ
   if t.sons[0] == nil:
     LocalError(n.info, errXNeedsReturnType, "iterator")
-  # iterators are either 'inline' or 'closure':
+  # iterators are either 'inline' or 'closure'; for backwards compatibility,
+  # we require first class iterators to be marked with 'closure' explicitly
+  # -- at least for 0.9.2.
+  if s.typ.callConv == ccClosure:
+    incl(s.typ.flags, tfCapturesEnv)
+  when false:
     if s.typ.callConv != ccInline: 
       s.typ.callConv = ccClosure
       # and they always at least use the 'env' for the state field:

compiler/transf.nim

@@ -415,17 +415,23 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
   # generate access statements for the parameters (unless they are constant)
   # put mapping from formal parameters to actual parameters
   if n.kind != nkForStmt: InternalError(n.info, "transformFor")
+
+  var length = sonsLen(n)
+  var call = n.sons[length - 2]
+  if call.kind notin nkCallKinds or call.sons[0].kind != nkSym or 
+      call.sons[0].typ.callConv == ccClosure or
+      call.sons[0].sym.kind != skIterator:
+    n.sons[length-1] = transformLoopBody(c, n.sons[length-1]).pnode
+    return lambdalifting.liftForLoop(n).ptransNode
+    #InternalError(call.info, "transformFor")
+
   #echo "transforming: ", renderTree(n)
   result = newTransNode(nkStmtList, n.info, 0)
-  var length = sonsLen(n)
   var loopBody = transformLoopBody(c, n.sons[length-1])
   var v = newNodeI(nkVarSection, n.info)
   for i in countup(0, length - 3): 
     addVar(v, copyTree(n.sons[i])) # declare new vars
   add(result, v.ptransNode)
-  var call = n.sons[length - 2]
-  if call.kind notin nkCallKinds or call.sons[0].kind != nkSym:
-    InternalError(call.info, "transformFor")
   
   # Bugfix: inlined locals belong to the invoking routine, not to the invoked
   # iterator!
@@ -697,6 +703,8 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
     if prc.kind != skMacro:
       # XXX no closures yet for macros:
       result = liftLambdas(prc, result)
+    if prc.kind == skIterator and prc.typ.callConv == ccClosure:
+      result = lambdalifting.liftIterator(prc, result)
     incl(result.flags, nfTransf)
 
 proc transformStmt*(module: PSym, n: PNode): PNode =

lib/system/debugger.nim

@@ -675,21 +675,27 @@ proc dbgWriteStackTrace(f: PFrame) =
     i = 0
     total = 0
     tempFrames: array [0..127, PFrame]
-  while it != nil and i <= high(tempFrames)-(firstCalls-1):
+  # setup long head:
-    # the (-1) is for a nil entry that marks where the '...' should occur
+  while it != nil and i <= high(tempFrames)-firstCalls:
     tempFrames[i] = it
     inc(i)
     inc(total)
     it = it.prev
+  # go up the stack to count 'total':
   var b = it
   while it != nil:
     inc(total)
     it = it.prev
-  for j in 1..total-i-(firstCalls-1): 
+  var skipped = 0
+  if total > len(tempFrames):
+    # skip N
+    skipped = total-i-firstCalls+1
+    for j in 1..skipped:
       if b != nil: b = b.prev
-  if total != i:
+    # create '...' entry:
     tempFrames[i] = nil
     inc(i)
+  # setup short tail:
   while b != nil and i <= high(tempFrames):
     tempFrames[i] = b
     inc(i)
@@ -697,7 +703,7 @@ proc dbgWriteStackTrace(f: PFrame) =
   for j in countdown(i-1, 0):
     if tempFrames[j] == nil: 
       write(stdout, "(")
-      write(stdout, (total-i-1))
+      write(stdout, skipped)
       write(stdout, " calls omitted) ...")
     else:
       write(stdout, tempFrames[j].filename)

lib/system/excpt.nim

@@ -135,21 +135,27 @@ proc auxWriteStackTrace(f: PFrame, s: var string) =
     it = f
     i = 0
     total = 0
-  while it != nil and i <= high(tempFrames)-(firstCalls-1):
+  # setup long head:
-    # the (-1) is for a nil entry that marks where the '...' should occur
+  while it != nil and i <= high(tempFrames)-firstCalls:
     tempFrames[i] = it
     inc(i)
     inc(total)
     it = it.prev
+  # go up the stack to count 'total':
   var b = it
   while it != nil:
     inc(total)
     it = it.prev
-  for j in 1..total-i-(firstCalls-1): 
+  var skipped = 0
+  if total > len(tempFrames):
+    # skip N
+    skipped = total-i-firstCalls+1
+    for j in 1..skipped:
       if b != nil: b = b.prev
-  if total != i:
+    # create '...' entry:
     tempFrames[i] = nil
     inc(i)
+  # setup short tail:
   while b != nil and i <= high(tempFrames):
     tempFrames[i] = b
     inc(i)
@@ -157,7 +163,7 @@ proc auxWriteStackTrace(f: PFrame, s: var string) =
   for j in countdown(i-1, 0):
     if tempFrames[j] == nil: 
       add(s, "(")
-      add(s, $(total-i-1))
+      add(s, $skipped)
       add(s, " calls omitted) ...")
     else:
       var oldLen = s.len

tests/run/titer8.nim

@@ -0,0 +1,33 @@
+discard """
+  output: '''tada
+ta da'''
+"""
+# Test first class iterator:
+
+import strutils
+
+iterator tokenize2(s: string, seps: set[char] = Whitespace): tuple[
+  token: string, isSep: bool] {.closure.} =
+  var i = 0
+  while i < s.len:
+    var j = i
+    if s[j] in seps:
+      while j < s.len and s[j] in seps: inc(j)
+      if j > i:
+        yield (substr(s, i, j-1), true)
+    else:
+      while j < s.len and s[j] notin seps: inc(j)
+      if j > i:
+        yield (substr(s, i, j-1), false)
+    i = j
+
+for word, isSep in tokenize2("ta da", whiteSpace):
+  if not isSep:
+    stdout.write(word)
+echo ""
+
+proc inProc() =
+  for word, isSep in tokenize2("ta da", whiteSpace):
+    stdout.write(word)
+
+inProc()

todo.txt

@@ -1,8 +1,8 @@
 version 0.9.2
 =============
 
-- implement the effect system
+- test&finish first class iterators
-- implement for loop transformation for first class iterators
+- fix closure bug finally
 - overloading based on ASTs: 'constraint' should not be in PType but for the
   parameter *symbol*
 
@@ -11,8 +11,9 @@ version 0.9.2
   - ``hoist`` pragma for loop hoisting: can be easily done with 
     AST overloading + global
 
-- implement the compiler as a service
+- improve the compiler as a service
 - ``=`` should be overloadable; requires specialization for ``=``
+- implement constructors and non-nil types
 - make 'bind' default for templates and introduce 'mixin'; 
   special rule for ``[]=``
 - implicit deref for parameter matching; overloading based on 'var T'
@@ -45,6 +46,7 @@ version 0.9.XX
     echo a
     echo b)
 
+- implement read/write tracking in the effect system
 - implement the "snoopResult" pragma; no, make a strutils with string append
   semantics instead ...
 - implement "closure tuple consists of a single 'ref'" optimization

web/index.txt

@@ -103,11 +103,9 @@ Roadmap to 1.0
 Version 0.9.2
   * overloading based on ASTs (like already possible for term rewriting macros)
   * better interaction between macros, templates and overloading
-  * the effect system will be extended
   * the symbol binding rules for generics and templates may change again
 
 Version 0.9.x
-  * first class iterators
   * message passing performance will be greatly improved
   * the syntactic distinction between statements and expressions will be
     removed