IC: compilerprocs are handled correctly (#17265)

* IC: compilerprocs are handled correctly
* IC: special logic for .exportC'ed procs
* IC: 'hello world' compiles for the first round (2nd round fails)
* IC: hello world seems to work

compiler/ccgexprs.nim

@@ -2707,7 +2707,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       if sfCompileTime in sym.flags:
         localError(p.config, n.info, "request to generate code for .compileTime proc: " &
            sym.name.s)
-      if useAliveDataFromDce in p.module.flags:
+      if useAliveDataFromDce in p.module.flags and sym.typ.callConv != ccInline:
         fillProcLoc(p.module, n)
         genProcPrototype(p.module, sym)
       else:
@@ -2720,6 +2720,8 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
         putIntoDest(p, d, n, genLiteral(p, sym.ast, sym.typ), OnStatic)
       elif useAliveDataFromDce in p.module.flags:
         genConstHeader(p.module, p.module, p, sym)
+        assert((sym.loc.r != nil) and (sym.loc.t != nil))
+        putLocIntoDest(p, d, sym.loc)
       else:
         genComplexConst(p, sym, d)
     of skEnumField:
@@ -2889,7 +2891,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     if n[genericParamsPos].kind == nkEmpty:
       var prc = n[namePos].sym
       if useAliveDataFromDce in p.module.flags:
-        if p.module.alive.contains(prc.itemId.item):
+        if p.module.alive.contains(prc.itemId.item) and prc.magic in {mNone, mIsolate}:
           genProc(p.module, prc)
       elif prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags:
         if ({sfExportc, sfCompilerProc} * prc.flags == {sfExportc}) or

compiler/ic/cbackend.nim

@@ -83,6 +83,7 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool
 proc generateCode*(g: ModuleGraph) =
   ## The single entry point, generate C(++) code for the entire
   ## Nim program aka `ModuleGraph`.
+  initStrTable(g.compilerprocs)
   var alive = computeAliveSyms(g.packed, g.config)
 
   for i in 0..high(g.packed):

compiler/ic/dce.nim

@@ -47,14 +47,19 @@ proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: i
   ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will
   ## be analysed.
   if not c.alive[module].containsOrIncl(item):
-    let body = g[module].fromDisk.sh.syms[item].ast
+    var body = g[module].fromDisk.sh.syms[item].ast
     if body != emptyNodeId:
       let opt = g[module].fromDisk.sh.syms[item].options
+      if g[module].fromDisk.sh.syms[item].kind in routineKinds:
+        body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos)
       c.stack.add((module, opt, NodePos(body)))
 
     when false:
-      let name = g[module].fromDisk.sh.strings[g[module].fromDisk.sh.syms[item].name]
+      let nid = g[module].fromDisk.sh.syms[item].name
-      echo "I was called! ", name, " body exists: ", body != emptyNodeId
+      if nid != LitId(0):
+        let name = g[module].fromDisk.sh.strings[nid]
+        if name in ["nimFrame", "callDepthLimitReached"]:
+          echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item
 
 proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string) =
   let (module, item) = c.compilerProcs[name]
@@ -118,8 +123,8 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N
     rangeCheckAnalysis(c, g, tree, n)
   of nkProcDef, nkConverterDef, nkMethodDef, nkLambda, nkDo, nkFuncDef:
     if n.firstSon.kind == nkSym and isNotGeneric(n):
-      if isExportedToC(c, g, n.firstSon.operand):
+      let item = n.firstSon.operand
-        let item = n.operand
+      if isExportedToC(c, g, item):
         # This symbol is alive and everything its body references.
         followLater(c, g, c.thisModule, item)
   else:
@@ -145,6 +150,7 @@ proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms =
       c.thisModule = i
       for p in allNodes(g[i].fromDisk.topLevel):
         aliveCode(c, g, g[i].fromDisk.topLevel, p)
+
   followNow(c, g)
   result = move(c.alive)
 

compiler/ic/packed_ast.nim

@@ -464,3 +464,6 @@ iterator allNodes*(tree: PackedTree): NodePos =
     yield NodePos(p)
     let s = span(tree, p)
     inc p, s
+
+proc toPackedItemId*(item: int32): PackedItemId {.inline.} =
+  PackedItemId(module: LitId(0), item: item)

compiler/ic/replayer.nim

@@ -128,6 +128,8 @@ proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
                             PackedItemId(module: LitId(0), item: it))
     methodDef(g, g.idgen, sym)
 
+  when false:
+    # not used anymore:
     for it in mitems(g.packed[module].fromDisk.compilerProcs):
       let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it[1]))
       g.lazyCompilerprocs[g.packed[module].fromDisk.sh.strings[it[0]]] = symId

compiler/ic/to_packed_ast.nim

@@ -11,7 +11,7 @@ import std / [hashes, tables, intsets, sha1]
 import packed_ast, bitabs, rodfiles
 import ".." / [ast, idents, lineinfos, msgs, ropes, options,
   pathutils, condsyms]
-
+#import ".." / [renderer, astalgo]
 from std / os import removeFile, isAbsolute
 
 type
@@ -57,6 +57,9 @@ type
     symMarker*: IntSet #Table[ItemId, SymId]    # ItemId.item -> SymId
     config*: ConfigRef
 
+proc isActive*(e: PackedEncoder): bool = e.config != nil
+proc disable*(e: var PackedEncoder) = e.config = nil
+
 template primConfigFields(fn: untyped) {.dirty.} =
   fn backend
   fn selectedGC
@@ -216,6 +219,7 @@ proc addMissing(c: var PackedEncoder; p: PSym) =
   ## consider queuing a symbol for later addition to the packed tree
   if p != nil and p.itemId.module == c.thisModule:
     if p.itemId.item notin c.symMarker:
+      if not (sfForward in p.flags and p.kind in routineKinds):
         c.pendingSyms.add p
 
 proc addMissing(c: var PackedEncoder; p: PType) =
@@ -332,6 +336,8 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
     if s.itemId.item >= m.sh.syms.len:
       setLen m.sh.syms, s.itemId.item+1
 
+    assert sfForward notin s.flags
+
     var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
       position: s.position, offset: s.offset, options: s.options,
       name: s.name.s.toLitId(m))
@@ -418,26 +424,38 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa
       toPackedNode(n[i], ir, c, m)
     ir.patch patchPos
 
-proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
-  var t = newSeq[PackedItemId](i.concreteTypes.len)
-  for j in 0..high(i.concreteTypes):
-    t[j] = storeTypeLater(i.concreteTypes[j], c, m)
-  m.procInstCache.add PackedInstantiation(key: storeSymLater(s, c, m),
-                                          sym: storeSymLater(i.sym, c, m),
-                                          concreteTypes: t)
-
 proc storeTypeInst*(c: var PackedEncoder; m: var PackedModule; s: PSym; inst: PType) =
   m.typeInstCache.add (storeSymLater(s, c, m), storeTypeLater(inst, c, m))
 
 proc addPragmaComputation*(c: var PackedEncoder; m: var PackedModule; n: PNode) =
   toPackedNode(n, m.toReplay, c, m)
 
+proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
+  let info = toPackedInfo(n.info, c, m)
+  let patchPos = ir.prepare(n.kind, n.flags,
+                            storeTypeLater(n.typ, c, m), info)
+  for i in 0..<n.len:
+    if i != bodyPos:
+      toPackedNode(n[i], ir, c, m)
+    else:
+      # do not serialize the body of the proc, it's unnecessary since
+      # n[0].sym.ast has the sem'checked variant of it which is what
+      # everybody should use instead.
+      ir.nodes.add PackedNode(kind: nkEmpty, flags: {}, operand: 0,
+                              typeId: nilItemId, info: info)
+  ir.patch patchPos
+
 proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModule) =
   case n.kind
   of routineDefs:
+    toPackedProcDef(n, m.topLevel, encoder, m)
+    when false:
       # we serialize n[namePos].sym instead
       if n[namePos].kind == nkSym:
-      discard storeSym(n[namePos].sym, encoder, m)
+        let s = n[namePos].sym
+        discard storeSym(s, encoder, m)
+        if s.flags * {sfExportc, sfCompilerProc, sfCompileTime} == {sfExportc}:
+          m.exportCProcs.add(s.itemId.item)
       else:
         toPackedNode(n, m.topLevel, encoder, m)
   of nkStmtList, nkStmtListExpr:
@@ -450,6 +468,24 @@ proc toPackedNodeTopLevel*(n: PNode, encoder: var PackedEncoder; m: var PackedMo
   toPackedNodeIgnoreProcDefs(n, encoder, m)
   flush encoder, m
 
+proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var PackedModule) =
+  ## Generic procs and generated `=hook`'s need explicit top-level entries so
+  ## that the code generator can work without having to special case these. These
+  ## entries will also be useful for other tools and are the cleanest design
+  ## I can come up with.
+  assert s.kind in routineKinds
+  toPackedProcDef(s.ast, m.topLevel, encoder, m)
+  #flush encoder, m
+
+proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
+  var t = newSeq[PackedItemId](i.concreteTypes.len)
+  for j in 0..high(i.concreteTypes):
+    t[j] = storeTypeLater(i.concreteTypes[j], c, m)
+  m.procInstCache.add PackedInstantiation(key: storeSymLater(s, c, m),
+                                          sym: storeSymLater(i.sym, c, m),
+                                          concreteTypes: t)
+  toPackedGeneratedProcDef(i.sym, c, m)
+
 proc loadError(err: RodFileError; filename: AbsoluteFile) =
   echo "Error: ", $err, " loading file: ", filename.string
 
@@ -520,6 +556,7 @@ proc storeError(err: RodFileError; filename: AbsoluteFile) =
   removeFile(filename.string)
 
 proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var PackedModule) =
+  flush encoder, m
   #rememberConfig(encoder, encoder.config)
 
   var f = rodfiles.create(filename.string)
@@ -572,6 +609,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection enumToStringProcsSection, m.enumToStringProcs
 
   close(f)
+  encoder.disable()
   if f.err != ok:
     storeError(f.err, filename)
 
@@ -848,6 +886,8 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
                   info: newLineInfo(fileIdx, 1, 1),
                   position: int(fileIdx))
   m.module.owner = newPackage(conf, cache, fileIdx)
+  if fileIdx == conf.projectMainIdx2:
+    m.module.flags.incl sfMainModule
 
 proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                        fileIdx: FileIndex; m: var LoadedModule) =
@@ -895,6 +935,7 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache
       loadError(err, rod)
       g[m].status = outdated
       result = true
+    when false: loadError(err, rod)
   of loading, loaded:
     # For loading: Assume no recompile is required.
     result = false
@@ -1061,6 +1102,13 @@ proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
   IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len,
               typeId: int32 m.fromDisk.sh.types.len)
 
+proc searchForCompilerproc*(m: LoadedModule; name: string): int32 =
+  # slow, linear search, but the results are cached:
+  for it in items(m.fromDisk.compilerProcs):
+    if m.fromDisk.sh.strings[it[0]] == name:
+      return it[1]
+  return -1
+
 # ------------------------- .rod file viewer ---------------------------------
 
 proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
@@ -1081,6 +1129,11 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
     for ex in m.reexports:
       echo "  ", m.sh.strings[ex[0]]
     #  reexports*: seq[(LitId, PackedItemId)]
+
+  echo "all symbols"
+  for i in 0..high(m.sh.syms):
+    echo "  ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i
+
   echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len,
     " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len,
     " strings: ", m.sh.strings.len, " integers: ", m.sh.integers.len,

compiler/modulegraphs.nim

@@ -52,6 +52,7 @@ type
   ModuleGraph* = ref object
     ifaces*: seq[Iface]  ## indexed by int32 fileIdx
     packed*: PackedModuleGraph
+    encoders*: seq[PackedEncoder]
 
     typeInstCache*: Table[ItemId, seq[LazyType]] # A symbol's ItemId.
     procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId.
@@ -83,7 +84,6 @@ type
     systemModule*: PSym
     sysTypes*: array[TTypeKind, PType]
     compilerprocs*: TStrTable
-    lazyCompilerprocs*: Table[string, FullId]
     exposed*: TStrTable
     packageTypes*: TStrTable
     emptyNode*: PNode
@@ -165,6 +165,13 @@ proc simulateCachedModule*(g: ModuleGraph; moduleSym: PSym; m: PackedModule) =
     echo "simulating ", moduleSym.name.s, " ", moduleSym.position
   simulateLoadedModule(g.packed, g.config, g.cache, moduleSym, m)
 
+proc initEncoder*(g: ModuleGraph; module: PSym) =
+  let id = module.position
+  if id >= g.encoders.len:
+    setLen g.encoders, id+1
+  to_packed_ast.initEncoder(g.encoders[id],
+    g.packed[id].fromDisk, module, g.config, g.startupPackedConfig)
+
 type
   ModuleIter* = object
     fromRod: bool
@@ -260,7 +267,6 @@ proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym =
 proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
   ## we also need to record this to the packed module.
   g.attachedOps[op][t.itemId] = value
-  # XXX Also add to the packed module!
 
 proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
   ## we also need to record this to the packed module.
@@ -268,7 +274,10 @@ proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttac
   # XXX Also add to the packed module!
 
 proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
-  discard "To implement"
+  if g.config.symbolFiles != disabledSf:
+    assert module < g.encoders.len
+    assert isActive(g.encoders[module])
+    toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].fromDisk)
 
 proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
   result = resolveSym(g, g.enumToStringProcs[t.itemId])
@@ -297,12 +306,16 @@ proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) =
 
 proc loadCompilerProc*(g: ModuleGraph; name: string): PSym =
   if g.config.symbolFiles == disabledSf: return nil
-  let t = g.lazyCompilerprocs.getOrDefault(name)
+
-  if t.module != 0:
+  # slow, linear search, but the results are cached:
-    assert isCachedModule(g, t.module)
+  for module in 0..high(g.packed):
-    result = loadSymFromId(g.config, g.cache, g.packed, t.module, t.packed)
+    #if isCachedModule(g, module):
+    let x = searchForCompilerproc(g.packed[module], name)
+    if x >= 0:
+      result = loadSymFromId(g.config, g.cache, g.packed, module, toPackedItemId(x))
       if result != nil:
         strTableAdd(g.compilerprocs, result)
+      return result
 
 proc `$`*(u: SigHash): string =
   toBase64a(cast[cstring](unsafeAddr u), sizeof(u))

compiler/semdata.nim

@@ -147,7 +147,6 @@ type
     selfName*: PIdent
     cache*: IdentCache
     graph*: ModuleGraph
-    encoder*: PackedEncoder
     signatures*: TStrTable
     recursiveDep*: string
     suggestionsMade*: bool
@@ -314,9 +313,10 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext =
     assert graph.packed[id].status in {undefined, outdated}
     graph.packed[id].status = storing
     graph.packed[id].module = module
-    initEncoder result.encoder, graph.packed[id].fromDisk, module, graph.config, graph.startupPackedConfig
+    initEncoder graph, module
 
 template packedRepr*(c): untyped = c.graph.packed[c.module.position].fromDisk
+template encoder*(c): untyped = c.graph.encoders[c.module.position]
 
 proc addIncludeFileDep*(c: PContext; f: FileIndex) =
   if c.config.symbolFiles != disabledSf:
@@ -561,6 +561,7 @@ proc addToGenericCache*(c: PContext; s: PSym; inst: PType) =
 
 proc saveRodFile*(c: PContext) =
   if c.config.symbolFiles != disabledSf:
+    if c.graph.vm != nil:
       for (m, n) in PCtx(c.graph.vm).vmstateDiff:
         if m == c.module:
           addPragmaComputation(c, n)