Change the order of compilation passes, transformation is made lazy at code gen (#8489)

* Ast no transformation
* Add getImplNoTransform to the macros module
* progress on delaying transf
* Fix methods tranformation
* Fix lazy lambdalifting
* fix create thread wrapper
* transform for lambda lifting
* improve getImplTransformed
* Fix destructor tests
* try to fix nimprof for linux

compiler/transf.nim

@@ -21,9 +21,13 @@
 import
   intsets, strutils, options, ast, astalgo, trees, treetab, msgs, lookups,
   idents, renderer, types, passes, semfold, magicsys, cgmeth,
-  lambdalifting, sempass2, lowerings, destroyer, liftlocals, closureiters,
+  sempass2, lowerings, destroyer, liftlocals,
   modulegraphs, lineinfos
 
+proc transformBody*(g: ModuleGraph, prc: PSym, cache = true): PNode
+
+import closureiters, lambdalifting
+
 type
   PTransNode* = distinct PNode
 
@@ -62,18 +66,25 @@ proc newTransNode(kind: TNodeKind, n: PNode,
   var x = newNodeIT(kind, n.info, n.typ)
   newSeq(x.sons, sons)
   x.typ = n.typ
+#  x.flags = n.flags
   result = x.PTransNode
 
+proc add(a, b: PTransNode) {.inline.} = addSon(PNode(a), PNode(b))
+proc len(a: PTransNode): int {.inline.} = sonsLen(a.PNode)
+
 proc `[]=`(a: PTransNode, i: int, x: PTransNode) {.inline.} =
   var n = PNode(a)
   n.sons[i] = PNode(x)
 
+proc `[]=`(a: PTransNode, i: BackwardsIndex, x: PTransNode) {.inline.} =
+  `[]=`(a, a.len - i.int, x)
+
 proc `[]`(a: PTransNode, i: int): PTransNode {.inline.} =
   var n = PNode(a)
   result = n.sons[i].PTransNode
 
-proc add(a, b: PTransNode) {.inline.} = addSon(PNode(a), PNode(b))
-proc len(a: PTransNode): int {.inline.} = result = sonsLen(a.PNode)
+proc `[]`(a: PTransNode, i: BackwardsIndex): PTransNode {.inline.} =
+  `[]`(a, a.len - i.int)
 
 proc newTransCon(owner: PSym): PTransCon =
   assert owner != nil
@@ -118,6 +129,8 @@ proc newAsgnStmt(c: PTransf, kind: TNodeKind, le: PNode, ri: PTransNode): PTrans
 proc transformSymAux(c: PTransf, n: PNode): PNode =
   let s = n.sym
   if s.typ != nil and s.typ.callConv == ccClosure:
+    if s.kind in routineKinds:
+      discard transformBody(c.graph, s)
     if s.kind == skIterator:
       if c.tooEarly: return n
       else: return liftIterSym(c.graph, n, getCurrOwner(c))
@@ -146,7 +159,7 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
       return
     tc = tc.next
   result = b
-
+  
 proc transformSym(c: PTransf, n: PNode): PTransNode =
   result = PTransNode(transformSymAux(c, n))
 
@@ -561,9 +574,10 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
   c.breakSyms.add(labl)
   if call.kind notin nkCallKinds or call.sons[0].kind != nkSym or
       call.sons[0].typ.callConv == ccClosure:
-    n.sons[length-1] = transformLoopBody(c, n.sons[length-1]).PNode
-    n.sons[length-2] = transform(c, n.sons[length-2]).PNode
-    result[1] = lambdalifting.liftForLoop(c.graph, n, getCurrOwner(c)).PTransNode
+    result[1] = n.PTransNode
+    result[1][^1] = transformLoopBody(c, n[^1])
+    result[1][^2] = transform(c, n[^2])
+    result[1] = lambdalifting.liftForLoop(c.graph, result[1].PNode, getCurrOwner(c)).PTransNode
     discard c.breakSyms.pop
     return result
 
@@ -617,7 +631,7 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
       add(stmtList, newAsgnStmt(c, nkFastAsgn, temp, arg.PTransNode))
       idNodeTablePut(newC.mapping, formal, temp)
 
-  var body = iter.getBody.copyTree
+  let body = transformBody(c.graph, iter).copyTree
   pushInfoContext(c.graph.config, n.info)
   # XXX optimize this somehow. But the check "c.inlining" is not correct:
   var symMap: TIdTable
@@ -1040,27 +1054,36 @@ template liftDefer(c, root) =
   if c.deferDetected:
     liftDeferAux(root)
 
-proc transformBody*(g: ModuleGraph; module: PSym, n: PNode, prc: PSym): PNode =
-  if nfTransf in n.flags or prc.kind in {skTemplate}:
-    result = n
+proc transformBody*(g: ModuleGraph, prc: PSym, cache = true): PNode =
+  assert prc.kind in routineKinds
+
+  if prc.transformedBody != nil:
+    result = prc.transformedBody
+  elif nfTransf in prc.ast[bodyPos].flags or prc.kind in {skTemplate}:
+    result = prc.ast[bodyPos]
   else:
-    var c = openTransf(g, module, "")
-    result = liftLambdas(g, prc, n, c.tooEarly)
-    #result = n
+
+    prc.transformedBody = newNode(nkEmpty) # protects from recursion
+    var c = openTransf(g, prc.getModule, "")
+    result = liftLambdas(g, prc, prc.ast[bodyPos], c.tooEarly)
     result = processTransf(c, result, prc)
     liftDefer(c, result)
-    #result = liftLambdas(prc, result)
-    when useEffectSystem: trackProc(g, prc, result)
     result = liftLocalsIfRequested(prc, result, g.cache, g.config)
     if c.needsDestroyPass: #and newDestructors:
       result = injectDestructorCalls(g, prc, result)
 
     if prc.isIterator:
       result = g.transformClosureIterator(prc, result)
-
+  
     incl(result.flags, nfTransf)
-      #if prc.name.s == "testbody":
-    #  echo renderTree(result)
+
+    let cache = cache or prc.typ.callConv == ccInline 
+    if cache:
+      # genProc for inline procs will be called multiple times from diffrent modules,
+      # it is important to transform exactly once to get sym ids and locations right
+      prc.transformedBody = result
+    else:
+      prc.transformedBody = nil
 
 proc transformStmt*(g: ModuleGraph; module: PSym, n: PNode): PNode =
   if nfTransf in n.flags:
@@ -1070,9 +1093,6 @@ proc transformStmt*(g: ModuleGraph; module: PSym, n: PNode): PNode =
     result = processTransf(c, n, module)
     liftDefer(c, result)
     #result = liftLambdasForTopLevel(module, result)
-    when useEffectSystem: trackTopLevelStmt(g, module, result)
-    #if n.info ?? "temp.nim":
-    #  echo renderTree(result, {renderIds})
     if c.needsDestroyPass:
       result = injectDestructorCalls(g, module, result)
     incl(result.flags, nfTransf)