first implementation of the new lambda-lifting pass; barely anything works

compiler/ast.nim

@@ -500,8 +500,7 @@ type
     skResult,             # special 'result' variable
     skProc,               # a proc
     skMethod,             # a method
-    skIterator,           # an inline iterator
+    skIterator,           # an iterator
-    skClosureIterator,    # a resumable closure iterator
     skConverter,          # a type converter
     skMacro,              # a macro
     skTemplate,           # a template; currently also misused for user-defined
@@ -518,7 +517,7 @@ type
   TSymKinds* = set[TSymKind]
 
 const
-  routineKinds* = {skProc, skMethod, skIterator, skClosureIterator,
+  routineKinds* = {skProc, skMethod, skIterator,
                    skConverter, skMacro, skTemplate}
   tfIncompleteStruct* = tfVarargs
   tfUncheckedArray* = tfVarargs
@@ -905,7 +904,7 @@ type
 # the poor naming choices in the standard library.
 
 const
-  OverloadableSyms* = {skProc, skMethod, skIterator, skClosureIterator,
+  OverloadableSyms* = {skProc, skMethod, skIterator,
     skConverter, skModule, skTemplate, skMacro}
 
   GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody,
@@ -929,7 +928,7 @@ const
   NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tySequence,
     tyProc, tyString, tyError}
   ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType,
-    skIterator, skClosureIterator,
+    skIterator,
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
                                       nfDotSetter, nfDotField,
@@ -958,11 +957,9 @@ const
   nkStrKinds* = {nkStrLit..nkTripleStrLit}
 
   skLocalVars* = {skVar, skLet, skForVar, skParam, skResult}
-  skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skClosureIterator,
+  skProcKinds* = {skProc, skTemplate, skMacro, skIterator,
                   skMethod, skConverter}
 
-  skIterators* = {skIterator, skClosureIterator}
-
 var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things
 
 proc isCallExpr*(n: PNode): bool =
@@ -1558,12 +1555,13 @@ proc isGenericRoutine*(s: PSym): bool =
   else: discard
 
 proc skipGenericOwner*(s: PSym): PSym =
-  internalAssert s.kind in skProcKinds
   ## Generic instantiations are owned by their originating generic
   ## symbol. This proc skips such owners and goes straight to the owner
   ## of the generic itself (the module or the enclosing proc).
-  result = if sfFromGeneric in s.flags: s.owner.owner
+  result = if s.kind in skProcKinds and sfFromGeneric in s.flags:
-           else: s.owner
+             s.owner.owner
+           else:
+             s.owner
 
 proc originatingModule*(s: PSym): PSym =
   result = s.owner

compiler/ccgexprs.nim

@@ -1965,7 +1965,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       else:
         genProc(p.module, sym)
       putLocIntoDest(p, d, sym.loc)
-    of skProc, skConverter, skIterators:
+    of skProc, skConverter, skIterator:
       if sfCompileTime in sym.flags:
         localError(n.info, "request to generate code for .compileTime proc: " &
            sym.name.s)

compiler/ccgstmts.nim

@@ -955,7 +955,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
       res.add(t.sons[i].strVal)
     of nkSym:
       var sym = t.sons[i].sym
-      if sym.kind in {skProc, skIterator, skClosureIterator, skMethod}:
+      if sym.kind in {skProc, skIterator, skMethod}:
         var a: TLoc
         initLocExpr(p, t.sons[i], a)
         res.add($rdLoc(a))

compiler/cgen.nim

@@ -594,7 +594,7 @@ proc cgsym(m: BModule, name: string): Rope =
   var sym = magicsys.getCompilerProc(name)
   if sym != nil:
     case sym.kind
-    of skProc, skMethod, skConverter, skIterators: genProc(m, sym)
+    of skProc, skMethod, skConverter, skIterator: genProc(m, sym)
     of skVar, skResult, skLet: genVarPrototype(m, sym)
     of skType: discard getTypeDesc(m, sym.typ)
     else: internalError("cgsym: " & name & ": " & $sym.kind)

compiler/docgen.nim

@@ -537,7 +537,7 @@ proc generateJson(d: PDoc, n: PNode, jArray: JsonNode = nil): JsonNode =
 proc genSection(d: PDoc, kind: TSymKind) =
   const sectionNames: array[skModule..skTemplate, string] = [
     "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Methods",
-    "Iterators", "Iterators", "Converters", "Macros", "Templates"
+    "Iterators", "Converters", "Macros", "Templates"
   ]
   if d.section[kind] == nil: return
   var title = sectionNames[kind].rope

compiler/jsgen.nim

@@ -931,7 +931,7 @@ proc isIndirect(v: PSym): bool =
   result = {sfAddrTaken, sfGlobal} * v.flags != {} and
     #(mapType(v.typ) != etyObject) and
     {sfImportc, sfVolatile, sfExportc} * v.flags == {} and
-    v.kind notin {skProc, skConverter, skMethod, skIterator, skClosureIterator,
+    v.kind notin {skProc, skConverter, skMethod, skIterator,
                   skConst, skTemp, skLet}
 
 proc genAddr(p: PProc, n: PNode, r: var TCompRes) =

compiler/lookups.nim

@@ -133,7 +133,7 @@ type
 
 proc getSymRepr*(s: PSym): string =
   case s.kind
-  of skProc, skMethod, skConverter, skIterators: result = getProcHeader(s)
+  of skProc, skMethod, skConverter, skIterator: result = getProcHeader(s)
   else: result = s.name.s
 
 proc ensureNoMissingOrUnusedSymbols(scope: PScope) =

compiler/lowerings.nim

@@ -585,7 +585,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType;
     objType.addField(field)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
     fn = indirectAccess(castExpr, field, n.info)
-  elif fn.kind == nkSym and fn.sym.kind in {skClosureIterator, skIterator}:
+  elif fn.kind == nkSym and fn.sym.kind == skIterator:
     localError(n.info, "iterator in spawn environment is not allowed")
   elif fn.typ.callConv == ccClosure:
     localError(n.info, "closure in spawn environment is not allowed")

compiler/semcall.nim

@@ -75,7 +75,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
           errors.add(err)
     if z.state == csMatch:
       # little hack so that iterators are preferred over everything else:
-      if sym.kind in skIterators: inc(z.exactMatches, 200)
+      if sym.kind == skIterator: inc(z.exactMatches, 200)
       case best.state
       of csEmpty, csNoMatch: best = z
       of csMatch:
@@ -395,7 +395,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     for i in countup(0, len(a)-1):
       var candidate = a.sons[i].sym
       if candidate.kind in {skProc, skMethod, skConverter,
-                            skIterator, skClosureIterator}:
+                            skIterator}:
         # it suffices that the candidate has the proper number of generic
         # type parameters:
         if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1:

compiler/semdata.nim

@@ -315,7 +315,7 @@ proc makeRangeType*(c: PContext; first, last: BiggestInt;
   addSonSkipIntLit(result, intType) # basetype of range
 
 proc markIndirect*(c: PContext, s: PSym) {.inline.} =
-  if s.kind in {skProc, skConverter, skMethod, skIterator, skClosureIterator}:
+  if s.kind in {skProc, skConverter, skMethod, skIterator}:
     incl(s.flags, sfAddrTaken)
     # XXX add to 'c' for global analysis
 

compiler/semexprs.nim

@@ -753,11 +753,11 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
                                      flags: TExprFlags): PNode =
   if flags*{efInTypeof, efWantIterator} != {}:
     # consider: 'for x in pReturningArray()' --> we don't want the restriction
-    # to 'skIterators' anymore; skIterators are preferred in sigmatch already
+    # to 'skIterator' anymore; skIterator is preferred in sigmatch already
     # for typeof support.
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     result = semOverloadedCall(c, n, nOrig,
-      {skProc, skMethod, skConverter, skMacro, skTemplate}+skIterators)
+      {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator})
   else:
     result = semOverloadedCall(c, n, nOrig,
       {skProc, skMethod, skConverter, skMacro, skTemplate})
@@ -770,7 +770,7 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
     case callee.kind
     of skMacro, skTemplate: discard
     else:
-      if callee.kind in skIterators and callee.id == c.p.owner.id:
+      if callee.kind == skIterator and callee.id == c.p.owner.id:
         localError(n.info, errRecursiveDependencyX, callee.name.s)
         # error correction, prevents endless for loop elimination in transf.
         # See bug #2051:
@@ -1201,7 +1201,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
     let s = if n.sons[0].kind == nkSym: n.sons[0].sym
             elif n[0].kind in nkSymChoices: n.sons[0][0].sym
             else: nil
-    if s != nil and s.kind in {skProc, skMethod, skConverter}+skIterators:
+    if s != nil and s.kind in {skProc, skMethod, skConverter, skIterator}:
       # type parameters: partial generic specialization
       n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s)
       result = explicitGenericInstantiation(c, n, s)
@@ -1349,8 +1349,8 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
 proc semReturn(c: PContext, n: PNode): PNode =
   result = n
   checkSonsLen(n, 1)
-  if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro,
+  if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro} or (
-                        skClosureIterator}:
+     c.p.owner.kind == skIterator and c.p.owner.typ.callConv == ccClosure):
     if n.sons[0].kind != nkEmpty:
       # transform ``return expr`` to ``result = expr; return``
       if c.p.resultSym != nil:
@@ -1426,7 +1426,7 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
 proc semYield(c: PContext, n: PNode): PNode =
   result = n
   checkSonsLen(n, 1)
-  if c.p.owner == nil or c.p.owner.kind notin skIterators:
+  if c.p.owner == nil or c.p.owner.kind != skIterator:
     localError(n.info, errYieldNotAllowedHere)
   elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
     localError(n.info, errYieldNotAllowedInTryStmt)
@@ -2124,7 +2124,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     var s = lookUp(c, n)
     if c.inTypeClass == 0: semCaptureSym(s, c.p.owner)
     result = semSym(c, n, s, flags)
-    if s.kind in {skProc, skMethod, skConverter}+skIterators:
+    if s.kind in {skProc, skMethod, skConverter, skIterator}:
       #performProcvarCheck(c, n, s)
       result = symChoice(c, n, s, scClosed)
       if result.kind == nkSym:
@@ -2212,7 +2212,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
           localError(n.info, errUseQualifier, s.name.s)
         elif s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
-      of skProc, skMethod, skConverter, skIterators:
+      of skProc, skMethod, skConverter, skIterator:
         if s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
       else:

compiler/semgnrc.nim

@@ -58,7 +58,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of skProc, skMethod, skIterators, skConverter, skModule:
+  of skProc, skMethod, skIterator, skConverter, skModule:
     result = symChoice(c, n, s, scOpen)
   of skTemplate:
     if macroToExpand(s):
@@ -226,7 +226,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       of skUnknown, skParam:
         # Leave it as an identifier.
         discard
-      of skProc, skMethod, skIterators, skConverter, skModule:
+      of skProc, skMethod, skIterator, skConverter, skModule:
         result.sons[0] = symChoice(c, fn, s, scOption)
         # do not check of 's.magic==mRoof' here because it might be some
         # other '^' but after overload resolution the proper one:

compiler/semstmts.nim

@@ -84,7 +84,7 @@ proc performProcvarCheck(c: PContext, n: PNode, s: PSym) =
 proc semProcvarCheck(c: PContext, n: PNode) =
   let n = n.skipConv
   if n.kind == nkSym and n.sym.kind in {skProc, skMethod, skConverter,
-                                        skIterator, skClosureIterator}:
+                                        skIterator}:
     performProcvarCheck(c, n, n.sym)
 
 proc semProc(c: PContext, n: PNode): PNode
@@ -598,7 +598,7 @@ proc semFor(c: PContext, n: PNode): PNode =
     # first class iterator:
     result = semForVars(c, n)
   elif not isCallExpr or call.sons[0].kind != nkSym or
-      call.sons[0].sym.kind notin skIterators:
+      call.sons[0].sym.kind != skIterator:
     if length == 3:
       n.sons[length-2] = implicitIterator(c, "items", n.sons[length-2])
     elif length == 4:
@@ -1143,13 +1143,15 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   if tfTriggersCompileTime in s.typ.flags: incl(s.flags, sfCompileTime)
   if n.sons[patternPos].kind != nkEmpty:
     n.sons[patternPos] = semPattern(c, n.sons[patternPos])
-  if s.kind in skIterators:
+  if s.kind == skIterator:
     s.typ.flags.incl(tfIterator)
 
   var proto = searchForProc(c, oldScope, s)
   if proto == nil:
-    if s.kind == skClosureIterator: s.typ.callConv = ccClosure
+    if s.kind == skIterator and s.typ.callConv == ccClosure:
-    else: s.typ.callConv = lastOptionEntry(c).defaultCC
+      discard
+    else:
+      s.typ.callConv = lastOptionEntry(c).defaultCC
     # add it here, so that recursive procs are possible:
     if sfGenSym in s.flags: discard
     elif kind in OverloadableSyms:
@@ -1209,7 +1211,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         n.sons[bodyPos] = transformBody(c.module, semBody, s)
       popProcCon(c)
     else:
-      if s.typ.sons[0] != nil and kind notin skIterators:
+      if s.typ.sons[0] != nil and kind != skIterator:
         addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info))
       openScope(c)
       n.sons[bodyPos] = semGenericStmt(c, n.sons[bodyPos])
@@ -1230,7 +1232,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   if n.sons[patternPos].kind != nkEmpty:
     c.patterns.add(s)
   if isAnon: result.typ = s.typ
-  if isTopLevel(c) and s.kind != skClosureIterator and
+  if isTopLevel(c) and s.kind != skIterator and
       s.typ.callConv == ccClosure:
     message(s.info, warnDeprecated, "top level '.closure' calling convention")
 
@@ -1240,15 +1242,12 @@ proc determineType(c: PContext, s: PSym) =
   discard semProcAux(c, s.ast, s.kind, {}, stepDetermineType)
 
 proc semIterator(c: PContext, n: PNode): PNode =
-  let kind = if hasPragma(n[pragmasPos], wClosure) or
-                n[namePos].kind == nkEmpty: skClosureIterator
-             else: skIterator
   # gensym'ed iterator?
   if n[namePos].kind == nkSym:
     # gensym'ed iterators might need to become closure iterators:
     n[namePos].sym.owner = getCurrOwner()
-    n[namePos].sym.kind = kind
+    n[namePos].sym.kind = skIterator
-  result = semProcAux(c, n, kind, iteratorPragmas)
+  result = semProcAux(c, n, skIterator, iteratorPragmas)
   var s = result.sons[namePos].sym
   var t = s.typ
   if t.sons[0] == nil and s.typ.callConv != ccClosure:

compiler/semtempl.nim

@@ -456,9 +456,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   of nkMethodDef:
     result = semRoutineInTemplBody(c, n, skMethod)
   of nkIteratorDef:
-    let kind = if hasPragma(n[pragmasPos], wClosure): skClosureIterator
+    result = semRoutineInTemplBody(c, n, skIterator)
-               else: skIterator
-    result = semRoutineInTemplBody(c, n, kind)
   of nkTemplateDef:
     result = semRoutineInTemplBody(c, n, skTemplate)
   of nkMacroDef:

compiler/semtypes.nim

@@ -1296,7 +1296,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       child.flags.incl tfIterator
       result.addSonSkipIntLit(child)
     else:
-      result = semProcTypeWithScope(c, n, prev, skClosureIterator)
+      result = semProcTypeWithScope(c, n, prev, skIterator)
       result.flags.incl(tfIterator)
       if n.lastSon.kind == nkPragma and hasPragma(n.lastSon, wInline):
         result.callConv = ccInline

compiler/sigmatch.nim

@@ -1442,7 +1442,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     z.calleeSym = m.calleeSym
     var best = -1
     for i in countup(0, sonsLen(arg) - 1):
-      if arg.sons[i].sym.kind in {skProc, skMethod, skConverter}+skIterators:
+      if arg.sons[i].sym.kind in {skProc, skMethod, skConverter, skIterator}:
         copyCandidate(z, m)
         z.callee = arg.sons[i].typ
         z.calleeSym = arg.sons[i].sym

compiler/transf.nim

@@ -348,6 +348,20 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode =
       # addr ( deref ( x )) --> x
       result = PTransNode(n.sons[0].sons[0])
 
+proc generateThunk(prc: PNode, dest: PType): PNode =
+  ## Converts 'prc' into '(thunk, nil)' so that it's compatible with
+  ## a closure.
+
+  # we cannot generate a proper thunk here for GC-safety reasons
+  # (see internal documentation):
+  if gCmd == cmdCompileToJS: return prc
+  result = newNodeIT(nkClosure, prc.info, dest)
+  var conv = newNodeIT(nkHiddenStdConv, prc.info, dest)
+  conv.add(emptyNode)
+  conv.add(prc)
+  result.add(conv)
+  result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil)))
+
 proc transformConv(c: PTransf, n: PNode): PTransNode =
   # numeric types need range checks:
   var dest = skipTypes(n.typ, abstractVarRange)
@@ -428,6 +442,10 @@ proc transformConv(c: PTransf, n: PNode): PTransNode =
   of tyGenericParam, tyOrdinal:
     result = transform(c, n.sons[1])
     # happens sometimes for generated assignments, etc.
+  of tyProc:
+    result = transformSons(c, n)
+    if dest.callConv == ccClosure and source.callConv == ccDefault:
+      result = generateThunk(result[1].PNode, dest).PTransNode
   else:
     result = transformSons(c, n)
 

compiler/vmgen.nim

@@ -1209,7 +1209,7 @@ proc checkCanEval(c: PCtx; n: PNode) =
       not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl:
     cannotEval(n)
   elif s.kind in {skProc, skConverter, skMethod,
-                  skIterator, skClosureIterator} and sfForward in s.flags:
+                  skIterator} and sfForward in s.flags:
     cannotEval(n)
 
 proc isTemp(c: PCtx; dest: TDest): bool =
@@ -1638,7 +1638,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     case s.kind
     of skVar, skForVar, skTemp, skLet, skParam, skResult:
       genRdVar(c, n, dest, flags)
-    of skProc, skConverter, skMacro, skTemplate, skMethod, skIterators:
+    of skProc, skConverter, skMacro, skTemplate, skMethod, skIterator:
       # 'skTemplate' is only allowed for 'getAst' support:
       if procIsCallback(c, s): discard
       elif sfImportc in s.flags: c.importcSym(n.info, s)

lib/core/macros.nim

@@ -97,7 +97,7 @@ type
     nskUnknown, nskConditional, nskDynLib, nskParam,
     nskGenericParam, nskTemp, nskModule, nskType, nskVar, nskLet,
     nskConst, nskResult,
-    nskProc, nskMethod, nskIterator, nskClosureIterator,
+    nskProc, nskMethod, nskIterator,
     nskConverter, nskMacro, nskTemplate, nskField,
     nskEnumField, nskForVar, nskLabel,
     nskStub