Merge pull request #7770 from yglukhov/yield-in-try

Yield in try

compiler/ccgexprs.nim

@@ -2326,7 +2326,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkParForStmt: genParForStmt(p, n)
   of nkState: genState(p, n)
   of nkGotoState: genGotoState(p, n)
-  of nkBreakState: genBreakState(p, n)
+  of nkBreakState: genBreakState(p, n, d)
   else: internalError(p.config, n.info, "expr(" & $n.kind & "); unknown node kind")
 
 proc genNamedConstExpr(p: BProc, n: PNode): Rope =

compiler/ccgstmts.nim

@@ -157,6 +157,39 @@ proc genState(p: BProc, n: PNode) =
   elif n0.kind == nkStrLit:
     linefmt(p, cpsStmts, "$1: ;$n", n0.strVal.rope)
 
+proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
+  # Called by return and break stmts.
+  # Deals with issues faced when jumping out of try/except/finally stmts,
+
+  var stack = newSeq[tuple[n: PNode, inExcept: bool]](0)
+
+  for i in countup(1, howManyTrys):
+    let tryStmt = p.nestedTryStmts.pop
+    if not p.module.compileToCpp or optNoCppExceptions in p.config.globalOptions:
+      # Pop safe points generated by try
+      if not tryStmt.inExcept:
+        linefmt(p, cpsStmts, "#popSafePoint();$n")
+
+    # Pop this try-stmt of the list of nested trys
+    # so we don't infinite recurse on it in the next step.
+    stack.add(tryStmt)
+
+    # Find finally-stmt for this try-stmt
+    # and generate a copy of its sons
+    var finallyStmt = lastSon(tryStmt.n)
+    if finallyStmt.kind == nkFinally:
+      genStmts(p, finallyStmt.sons[0])
+
+  # push old elements again:
+  for i in countdown(howManyTrys-1, 0):
+    p.nestedTryStmts.add(stack[i])
+
+  if not p.module.compileToCpp or optNoCppExceptions in p.config.globalOptions:
+    # Pop exceptions that was handled by the
+    # except-blocks we are in
+    for i in countdown(howManyExcepts-1, 0):
+      linefmt(p, cpsStmts, "#popCurrentException();$n")
+
 proc genGotoState(p: BProc, n: PNode) =
   # we resist the temptation to translate it into duff's device as it later
   # will be translated into computed gotos anyway for GCC at least:
@@ -167,7 +200,11 @@ proc genGotoState(p: BProc, n: PNode) =
   initLocExpr(p, n.sons[0], a)
   lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)])
   p.beforeRetNeeded = true
-  lineF(p, cpsStmts, "case -1: goto BeforeRet_;$n", [])
+  lineF(p, cpsStmts, "case -1:$n", [])
+  blockLeaveActions(p,
+    howManyTrys    = p.nestedTryStmts.len,
+    howManyExcepts = p.inExceptBlockLen)
+  lineF(p, cpsStmts, " goto BeforeRet_;$n", [])
   var statesCounter = lastOrd(n.sons[0].typ)
   if n.len >= 2 and n[1].kind == nkIntLit:
     statesCounter = n[1].intVal
@@ -177,17 +214,17 @@ proc genGotoState(p: BProc, n: PNode) =
     lineF(p, cpsStmts, "case $2: goto $1$2;$n", [prefix, rope(i)])
   lineF(p, cpsStmts, "}$n", [])
 
-proc genBreakState(p: BProc, n: PNode) =
+proc genBreakState(p: BProc, n: PNode, d: var TLoc) =
   var a: TLoc
+  initLoc(d, locExpr, n, OnUnknown)
+
   if n.sons[0].kind == nkClosure:
-    # XXX this produces quite inefficient code!
     initLocExpr(p, n.sons[0].sons[1], a)
-    lineF(p, cpsStmts, "if (((NI*) $1)[1] < 0) break;$n", [rdLoc(a)])
+    d.r = "(((NI*) $1)[1] < 0)" % [rdLoc(a)]
   else:
     initLocExpr(p, n.sons[0], a)
     # the environment is guaranteed to contain the 'state' field at offset 1:
-    lineF(p, cpsStmts, "if ((((NI*) $1.ClE_0)[1]) < 0) break;$n", [rdLoc(a)])
-  #  lineF(p, cpsStmts, "if (($1) < 0) break;$n", [rdLoc(a)])
+    d.r = "((((NI*) $1.ClE_0)[1]) < 0)" % [rdLoc(a)]
 
 proc genGotoVar(p: BProc; value: PNode) =
   if value.kind notin {nkCharLit..nkUInt64Lit}:
@@ -328,40 +365,6 @@ proc genIf(p: BProc, n: PNode, d: var TLoc) =
     else: internalError(p.config, n.info, "genIf()")
   if sonsLen(n) > 1: fixLabel(p, lend)
 
-
-proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
-  # Called by return and break stmts.
-  # Deals with issues faced when jumping out of try/except/finally stmts,
-
-  var stack = newSeq[tuple[n: PNode, inExcept: bool]](0)
-
-  for i in countup(1, howManyTrys):
-    let tryStmt = p.nestedTryStmts.pop
-    if not p.module.compileToCpp or optNoCppExceptions in p.config.globalOptions:
-      # Pop safe points generated by try
-      if not tryStmt.inExcept:
-        linefmt(p, cpsStmts, "#popSafePoint();$n")
-
-    # Pop this try-stmt of the list of nested trys
-    # so we don't infinite recurse on it in the next step.
-    stack.add(tryStmt)
-
-    # Find finally-stmt for this try-stmt
-    # and generate a copy of its sons
-    var finallyStmt = lastSon(tryStmt.n)
-    if finallyStmt.kind == nkFinally:
-      genStmts(p, finallyStmt.sons[0])
-
-  # push old elements again:
-  for i in countdown(howManyTrys-1, 0):
-    p.nestedTryStmts.add(stack[i])
-
-  if not p.module.compileToCpp or optNoCppExceptions in p.config.globalOptions:
-    # Pop exceptions that was handled by the
-    # except-blocks we are in
-    for i in countdown(howManyExcepts-1, 0):
-      linefmt(p, cpsStmts, "#popCurrentException();$n")
-
 proc genReturnStmt(p: BProc, t: PNode) =
   if nfPreventCg in t.flags: return
   p.beforeRetNeeded = true

compiler/lambdalifting.nim

@@ -7,12 +7,11 @@
 #    distribution, for details about the copyright.
 #
 
-# This include file implements lambda lifting for the transformator.
+# This file implements lambda lifting for the transformator.
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
-  idents, renderer, types, magicsys, rodread, lowerings, tables,
-  modulegraphs
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs,
+  idents, renderer, types, magicsys, rodread, lowerings, tables, modulegraphs
 
 discard """
   The basic approach is that captured vars need to be put on the heap and
@@ -126,7 +125,7 @@ proc newCall(a: PSym, b: PNode): PNode =
   result.add newSymNode(a)
   result.add b
 
-proc createStateType(g: ModuleGraph; iter: PSym): PType =
+proc createClosureIterStateType*(g: ModuleGraph; iter: PSym): PType =
   var n = newNodeI(nkRange, iter.info)
   addSon(n, newIntNode(nkIntLit, -1))
   addSon(n, newIntNode(nkIntLit, 0))
@@ -137,8 +136,8 @@ proc createStateType(g: ModuleGraph; iter: PSym): PType =
   rawAddSon(result, intType)
 
 proc createStateField(g: ModuleGraph; iter: PSym): PSym =
-  result = newSym(skField, getIdent(":state"), iter, iter.info, {})
-  result.typ = createStateType(g, iter)
+  result = newSym(skField, getIdent(":state"), iter, iter.info)
+  result.typ = createClosureIterStateType(g, iter)
 
 proc createEnvObj(g: ModuleGraph; owner: PSym; info: TLineInfo): PType =
   # YYY meh, just add the state field for every closure for now, it's too
@@ -146,7 +145,7 @@ proc createEnvObj(g: ModuleGraph; owner: PSym; info: TLineInfo): PType =
   result = createObj(g, owner, info, final=false)
   rawAddField(result, createStateField(g, owner))
 
-proc getIterResult(iter: PSym): PSym =
+proc getClosureIterResult*(iter: PSym): PSym =
   if resultPos < iter.ast.len:
     result = iter.ast.sons[resultPos].sym
   else:
@@ -400,7 +399,11 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           if not c.capturedVars.containsOrIncl(s.id):
             let obj = getHiddenParam(c.graph, owner).typ.lastSon
             #let obj = c.getEnvTypeForOwner(s.owner).lastSon
-            addField(obj, s)
+
+            if s.name == getIdent(":state"):
+              obj.n[0].sym.id = -s.id
+            else:
+              addField(obj, s)
       # but always return because the rest of the proc is only relevant when
       # ow != owner:
       return
@@ -598,7 +601,7 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
     localError(d.graph.config, n.info, "internal error: not part of closure object type")
     result = n
 
-proc getStateField(g: ModuleGraph; owner: PSym): PSym =
+proc getStateField*(g: ModuleGraph; owner: PSym): PSym =
   getHiddenParam(g, owner).typ.sons[0].n.sons[0].sym
 
 proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
@@ -625,7 +628,7 @@ proc transformYield(n: PNode; owner: PSym; d: DetectionPass;
   if n.sons[0].kind != nkEmpty:
     var a = newNodeI(nkAsgn, n.sons[0].info)
     var retVal = liftCapturedVars(n.sons[0], owner, d, c)
-    addSon(a, newSymNode(getIterResult(owner)))
+    addSon(a, newSymNode(getClosureIterResult(owner)))
     addSon(a, retVal)
     retStmt.add(a)
   else:
@@ -718,7 +721,9 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        let body = wrapIterBody(d.graph, liftCapturedVars(s.getBody, s, d, c), s)
+        var body = liftCapturedVars(s.getBody, s, d, c)
+        if oldIterTransf in d.graph.config.features:
+          body = wrapIterBody(d.graph, body, s)
         if c.envvars.getOrDefault(s.id).isNil:
           s.ast.sons[bodyPos] = body
         else:
@@ -761,9 +766,9 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       if n[1].kind == nkClosure: result = n[1]
   else:
     if owner.isIterator:
-      if n.kind == nkYieldStmt:
+      if oldIterTransf in d.graph.config.features and n.kind == nkYieldStmt:
         return transformYield(n, owner, d, c)
-      elif n.kind == nkReturnStmt:
+      elif oldIterTransf in d.graph.config.features and n.kind == nkReturnStmt:
         return transformReturn(n, owner, d, c)
       elif nfLL in n.flags:
         # special case 'when nimVm' due to bug #3636:
@@ -811,7 +816,7 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNo
   fn.typ.callConv = oldCC
 
 proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
-  # XXX conf.cmd == cmdCompileToJS does not suffice! The compiletime stuff needs
+  # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
   # However we can do lifting for the stuff which is *only* compiletime.
@@ -820,6 +825,7 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PN
   if body.kind == nkEmpty or (
       g.config.cmd == cmdCompileToJS and not isCompileTime) or
       fn.skipGenericOwner.kind != skModule:
+
     # ignore forward declaration:
     result = body
     tooEarly = true
@@ -831,10 +837,12 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PN
       d.somethingToDo = true
     if d.somethingToDo:
       var c = initLiftingPass(fn)
-      var newBody = liftCapturedVars(body, fn, d, c)
+      result = liftCapturedVars(body, fn, d, c)
       if c.envvars.getOrDefault(fn.id) != nil:
-        newBody = newTree(nkStmtList, rawClosureCreation(fn, d, c), newBody)
-      result = wrapIterBody(g, newBody, fn)
+        result = newTree(nkStmtList, rawClosureCreation(fn, d, c), result)
+
+      if oldIterTransf in g.config.features:
+        result = wrapIterBody(g, result, fn)
     else:
       result = body
     #if fn.name.s == "get2":
@@ -872,7 +880,8 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
       cl = createClosure()
       while true:
         let i = foo(cl)
-        nkBreakState(cl.state)
+        if (nkBreakState(cl.state)):
+          break
         ...
     """
   if liftingHarmful(g.config, owner): return body
@@ -932,5 +941,16 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
   loopBody.sons[0] = v2
   var bs = newNodeI(nkBreakState, body.info)
   bs.addSon(call.sons[0])
-  loopBody.sons[1] = bs
+
+  let ibs = newNode(nkIfStmt)
+  let elifBranch = newNode(nkElifBranch)
+  elifBranch.add(bs)
+
+  let br = newNode(nkBreakStmt)
+  br.add(emptyNode)
+
+  elifBranch.add(br)
+  ibs.add(elifBranch)
+
+  loopBody.sons[1] = ibs
   loopBody.sons[2] = body[L-1]

compiler/options.nim

@@ -116,7 +116,8 @@ type
     callOperator,
     parallel,
     destructor,
-    notnil
+    notnil,
+    oldIterTransf
 
   SymbolFilesOption* = enum
     disabledSf, enabledSf, writeOnlySf, readOnlySf, v2Sf

compiler/renderer.nim

@@ -1413,11 +1413,21 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     put(g, tkParLe, "(ComesFrom|")
     gsub(g, n, 0)
     put(g, tkParRi, ")")
-  of nkGotoState, nkState:
+  of nkGotoState:
     var c: TContext
     initContext c
-    putWithSpace g, tkSymbol, if n.kind == nkState: "state" else: "goto"
+    putWithSpace g, tkSymbol, "goto"
     gsons(g, n, c)
+  of nkState:
+    var c: TContext
+    initContext c
+    putWithSpace g, tkSymbol, "state"
+    gsub(g, n[0], c)
+    putWithSpace(g, tkColon, ":")
+    indentNL(g)
+    gsons(g, n, c, 1)
+    dedent(g)
+
   of nkBreakState:
     put(g, tkTuple, "breakstate")
   of nkTypeClassTy:

compiler/semexprs.nim

@@ -1556,7 +1556,7 @@ proc semYield(c: PContext, n: PNode): PNode =
   checkSonsLen(n, 1, c.config)
   if c.p.owner == nil or c.p.owner.kind != skIterator:
     localError(c.config, n.info, errYieldNotAllowedHere)
-  elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
+  elif oldIterTransf in c.features and c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
     localError(c.config, n.info, errYieldNotAllowedInTryStmt)
   elif n.sons[0].kind != nkEmpty:
     n.sons[0] = semExprWithType(c, n.sons[0]) # check for type compatibility:

compiler/transf.nim

@@ -19,9 +19,9 @@
 # * transforms 'defer' into a 'try finally' statement
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, lookups,
   idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread,
-  lambdalifting, sempass2, lowerings, lookups, destroyer, liftlocals,
+  lambdalifting, sempass2, lowerings, destroyer, liftlocals, closureiters,
   modulegraphs
 
 type
@@ -984,6 +984,10 @@ proc transformBody*(g: ModuleGraph; module: PSym, n: PNode, prc: PSym): PNode =
     result = liftLocalsIfRequested(prc, result, g.config)
     if c.needsDestroyPass: #and newDestructors:
       result = injectDestructorCalls(g, prc, result)
+
+    if prc.isIterator and oldIterTransf notin g.config.features:
+      result = g.transformClosureIterator(prc, result)
+
     incl(result.flags, nfTransf)
       #if prc.name.s == "testbody":
     #  echo renderTree(result)