added a 'koch ic' command for easier adhoc testing of IC (#17508)

* added a 'koch ic' command for easier adhoc testing of IC
* IC: progress
* IC: enable generics test
* make tests green

compiler/ccgtypes.nim

@@ -1357,9 +1357,6 @@ proc genTypeInfoV2Impl(m: BModule, t, origType: PType, name: Rope; info: TLineIn
   if t.kind == tyObject and t.len > 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions:
     discard genTypeInfoV1(m, t, info)
 
-proc moduleOpenForCodegen(m: BModule; module: int32): bool {.inline.} =
-  result = module < m.g.modules.len and m.g.modules[module] != nil
-
 proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
   let origType = t
   # distinct types can have their own destructors
@@ -1384,7 +1381,7 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
   m.typeInfoMarkerV2[sig] = result
 
   let owner = t.skipTypes(typedescPtrs).itemId.module
-  if owner != m.module.position and moduleOpenForCodegen(m, owner):
+  if owner != m.module.position and moduleOpenForCodegen(m.g.graph, FileIndex owner):
     # make sure the type info is created in the owner module
     discard genTypeInfoV2(m.g.modules[owner], origType, info)
     # reference the type info as extern here
@@ -1463,7 +1460,7 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
     return prefixTI.rope & result & ")".rope
 
   var owner = t.skipTypes(typedescPtrs).itemId.module
-  if owner != m.module.position and moduleOpenForCodegen(m, owner):
+  if owner != m.module.position and moduleOpenForCodegen(m.g.graph, FileIndex owner):
     # make sure the type info is created in the owner module
     discard genTypeInfoV1(m.g.modules[owner], origType, info)
     # reference the type info as extern here

compiler/cgen.nim

@@ -22,6 +22,7 @@ when not defined(leanCompiler):
 
 import strutils except `%` # collides with ropes.`%`
 
+from ic / ic import ModuleBackendFlag
 from modulegraphs import ModuleGraph, PPassContext
 from lineinfos import
   warnGcMem, errXMustBeCompileTime, hintDependency, errGenerated, errCannotOpenFile
@@ -1501,6 +1502,33 @@ proc genMainProc(m: BModule) =
     if m.config.cppCustomNamespace.len > 0:
       m.s[cfsProcs].add openNamespaceNim(m.config.cppCustomNamespace)
 
+proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) =
+  ## Called from the IC backend.
+  if HasDatInitProc in flags:
+    let datInit = getSomeNameForModule(m) & "DatInit000"
+    g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit])
+    g.mainDatInit.addf("\t$1();$N", [datInit])
+  if HasModuleInitProc in flags:
+    let init = getSomeNameForModule(m) & "Init000"
+    g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init])
+    let initCall = "\t$1();$N" % [init]
+    if sfMainModule in m.flags:
+      g.mainModInit.add(initCall)
+    elif sfSystemModule in m.flags:
+      g.mainDatInit.add(initCall) # systemInit must called right after systemDatInit if any
+    else:
+      g.otherModsInit.add(initCall)
+
+proc whichInitProcs*(m: BModule): set[ModuleBackendFlag] =
+  # called from IC.
+  result = {}
+  if m.hcrOn or m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0:
+    result.incl HasModuleInitProc
+  for i in cfsTypeInit1..cfsDynLibInit:
+    if m.s[i].len != 0:
+      result.incl HasDatInitProc
+      break
+
 proc registerModuleToMain(g: BModuleList; m: BModule) =
   let
     init = m.getInitName
@@ -1595,6 +1623,7 @@ proc genDatInitCode(m: BModule) =
 
   if moduleDatInitRequired:
     m.s[cfsDatInitProc].add(prc)
+    #rememberFlag(m.g.graph, m.module, HasDatInitProc)
 
 # Very similar to the contents of symInDynamicLib - basically only the
 # things needed for the hot code reloading runtime procs to be loaded
@@ -1725,6 +1754,7 @@ proc genInitCode(m: BModule) =
 
   if moduleInitRequired or sfMainModule in m.module.flags:
     m.s[cfsInitProc].add(prc)
+    #rememberFlag(m.g.graph, m.module, HasModuleInitProc)
 
   genDatInitCode(m)
 

compiler/ic/cbackend.nim

@@ -44,9 +44,11 @@ proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var Alive
     cgen.genTopLevelStmt(bmod, n)
 
   finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info))
+  m.fromDisk.backendFlags = cgen.whichInitProcs(bmod)
 
 proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) =
   for x in mitems(m.fromDisk.emittedTypeInfo):
+    #echo "found type ", x, " for file ", int(origin)
     g.emittedTypeInfo[x] = origin
 
 proc addFileToLink(config: ConfigRef; m: PSym) =
@@ -112,12 +114,16 @@ proc generateCode*(g: ModuleGraph) =
   resetForBackend(g)
   var alive = computeAliveSyms(g.packed, g.config)
 
+  when false:
+    for i in 0..high(g.packed):
+      echo i, " is of status ", g.packed[i].status, " ", toFullPath(g.config, FileIndex(i))
+
   for i in 0..high(g.packed):
     # case statement here to enforce exhaustive checks.
     case g.packed[i].status
     of undefined:
       discard "nothing to do"
-    of loading:
+    of loading, stored:
       assert false
     of storing, outdated:
       generateCodeForModule(g, g.packed[i], alive)
@@ -133,3 +139,7 @@ proc generateCode*(g: ModuleGraph) =
       else:
         addFileToLink(g.config, g.packed[i].module)
         replayTypeInfo(g, g.packed[i], FileIndex(i))
+
+        if g.backend == nil:
+          g.backend = cgendata.newModuleList(g)
+        registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags)

compiler/ic/ic.nim

@@ -22,6 +22,10 @@ type
     options: TOptions
     globalOptions: TGlobalOptions
 
+  ModuleBackendFlag* = enum
+    HasDatInitProc
+    HasModuleInitProc
+
   PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file
     definedSymbols: string
     includes: seq[(LitId, string)] # first entry is the module filename itself
@@ -43,6 +47,7 @@ type
     enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
 
     emittedTypeInfo*: seq[string]
+    backendFlags*: set[ModuleBackendFlag]
 
     sh*: Shared
     cfg: PackedConfig
@@ -556,6 +561,9 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadSeqSection enumToStringProcsSection, m.enumToStringProcs
   loadSeqSection typeInfoSection, m.emittedTypeInfo
 
+  f.loadSection backendFlagsSection
+  f.loadPrim m.backendFlags
+
   close(f)
   result = f.err
 
@@ -619,6 +627,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection enumToStringProcsSection, m.enumToStringProcs
   storeSeqSection typeInfoSection, m.emittedTypeInfo
 
+  f.storeSection backendFlagsSection
+  f.storePrim m.backendFlags
+
   close(f)
   encoder.disable()
   if f.err != ok:
@@ -646,7 +657,8 @@ type
     storing,  # state is strictly for stress-testing purposes
     loading,
     loaded,
-    outdated
+    outdated,
+    stored    # store is complete, no further additions possible
 
   LoadedModule* = object
     status*: ModuleStatus
@@ -673,7 +685,7 @@ proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule:
 
 proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
                        x: PackedLineInfo): TLineInfo =
-  assert g[thisModule].status in {loaded, storing}
+  assert g[thisModule].status in {loaded, storing, stored}
   result = TLineInfo(line: x.line, col: x.col,
             fileIndex: toFileIndexCached(c, g, thisModule, x.file))
 
@@ -806,7 +818,7 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s:
     result = nil
   else:
     let si = moduleIndex(c, g, thisModule, s)
-    assert g[si].status in {loaded, storing}
+    assert g[si].status in {loaded, storing, stored}
     if not g[si].symsInit:
       g[si].symsInit = true
       setLen g[si].syms, g[si].fromDisk.sh.syms.len
@@ -852,7 +864,7 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
     result = nil
   else:
     let si = moduleIndex(c, g, thisModule, t)
-    assert g[si].status in {loaded, storing}
+    assert g[si].status in {loaded, storing, stored}
     assert t.item > 0
 
     if not g[si].typesInit:
@@ -950,7 +962,7 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache
   of loading, loaded:
     # For loading: Assume no recompile is required.
     result = false
-  of outdated, storing:
+  of outdated, storing, stored:
     result = true
 
 proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;

compiler/ic/rodfiles.nim

@@ -37,6 +37,7 @@ type
     methodsPerTypeSection
     enumToStringProcsSection
     typeInfoSection  # required by the backend
+    backendFlagsSection
     aliveSymsSection # beware, this is stored in a `.alivesyms` file.
 
   RodFileError* = enum

compiler/modulegraphs.nim

@@ -455,11 +455,25 @@ proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
     elif fileIdx.int32 < g.ifaces.len:
       result = g.ifaces[fileIdx.int32].module
 
+proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} =
+  if g.config.symbolFiles == disabledSf:
+    result = true
+  else:
+    result = g.packed[m.int32].status notin {undefined, stored, loaded}
+
 proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) =
   #assert(not isCachedModule(g, m.int32))
   if g.config.symbolFiles != disabledSf:
     #assert g.encoders[m.int32].isActive
+    assert g.packed[m.int32].status != stored
     g.packed[m.int32].fromDisk.emittedTypeInfo.add ti
+    #echo "added typeinfo ", m.int32, " ", ti, " suspicious ", not g.encoders[m.int32].isActive
+
+proc rememberFlag*(g: ModuleGraph; m: PSym; flag: ModuleBackendFlag) =
+  if g.config.symbolFiles != disabledSf:
+    #assert g.encoders[m.int32].isActive
+    assert g.packed[m.position].status != stored
+    g.packed[m.position].fromDisk.backendFlags.incl flag
 
 proc closeRodFile*(g: ModuleGraph; m: PSym) =
   if g.config.symbolFiles in {readOnlySf, v2Sf}:
@@ -469,6 +483,8 @@ proc closeRodFile*(g: ModuleGraph; m: PSym) =
     let mint = m.position
     saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))),
                 g.encoders[mint], g.packed[mint].fromDisk)
+    g.packed[mint].status = stored
+
   elif g.config.symbolFiles == stressTest:
     # debug code, but maybe a good idea for production? Could reduce the compiler's
     # memory consumption considerably at the cost of more loads from disk.

koch.nim

@@ -458,8 +458,8 @@ proc temp(args: string) =
       inc i
 
   let d = getAppDir()
-  var output = d / "compiler" / "nim".exe
+  let output = d / "compiler" / "nim".exe
-  var finalDest = d / "bin" / "nim_temp".exe
+  let finalDest = d / "bin" / "nim_temp".exe
   # 125 is the magic number to tell git bisect to skip the current commit.
   var (bootArgs, programArgs) = splitArgs(args)
   if "doc" notin programArgs and
@@ -483,6 +483,22 @@ proc xtemp(cmd: string) =
   finally:
     copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe)
 
+proc icTest(args: string) =
+  temp("")
+  let inp = os.parseCmdLine(args)[0]
+  let content = readFile(inp)
+  let nimExe = getAppDir() / "bin" / "nim_temp".exe
+  var i = 0
+  for fragment in content.split("#!EDIT!#"):
+    let file = inp.replace(".nim", "_temp.nim")
+    writeFile(file, fragment)
+    var cmd = nimExe & " cpp --ic:on --listcmd "
+    if i == 0:
+      cmd.add "-f "
+    cmd.add quoteShell(file)
+    exec(cmd)
+    inc i
+
 proc buildDrNim(args: string) =
   if not dirExists("dist/nimz3"):
     exec("git clone https://github.com/zevv/nimz3.git dist/nimz3")
@@ -705,6 +721,7 @@ when isMainModule:
       of "fusion":
         let suffix = if latest: HeadHash else: FusionStableHash
         exec("nimble install -y fusion@$#" % suffix)
+      of "ic": icTest(op.cmdLineRest)
       else: showHelp()
       break
     of cmdEnd: break

tests/ic/tgenerics.nim

@@ -1,6 +1,5 @@
 discard """
   output: "bar"
-  disabled: "true"
 """
 
 import tables