IC: next steps (#16729)

* IC: dead code elimination pass
* preparations for a different codegen strategy
* added documentation to the newly written code
* IC: backend code
* IC: backend adjustments
* optimized the compiler a bit
* IC: yet another massive refactoring
* fixes regressions
* cleanups

compiler/ast.nim

@@ -720,6 +720,16 @@ type
     module*: int32
     item*: int32
 
+proc `==`*(a, b: ItemId): bool {.inline.} =
+  a.item == b.item and a.module == b.module
+
+proc hash*(x: ItemId): Hash =
+  var h: Hash = hash(x.module)
+  h = h !& hash(x.item)
+  result = !$h
+
+
+type
   TIdObj* = object of RootObj
     itemId*: ItemId
   PIdObj* = ref TIdObj
@@ -830,10 +840,8 @@ type
   TSym* {.acyclic.} = object of TIdObj # Keep in sync with PackedSym
     # proc and type instantiations are cached in the generic symbol
     case kind*: TSymKind
-    of skType, skGenericParam:
-      typeInstCache*: seq[PType]
     of routineKinds:
-      procInstCache*: seq[PInstantiation]
+      #procInstCache*: seq[PInstantiation]
       gcUnsafetyReason*: PSym  # for better error messages wrt gcsafe
       transformedBody*: PNode  # cached body after transf pass
     of skLet, skVar, skField, skForVar:
@@ -913,8 +921,6 @@ type
     owner*: PSym              # the 'owner' of the type
     sym*: PSym                # types have the sym associated with them
                               # it is used for converting types to strings
-    attachedOps*: array[TTypeAttachedOp, PSym] # destructors, etc.
-    methods*: seq[(int,PSym)] # attached methods
     size*: BiggestInt         # the size of the type in bytes
                               # -1 means that the size is unkwown
     align*: int16             # the type's alignment requirements
@@ -1069,11 +1075,7 @@ type
     module*: int32
     symId*: int32
     typeId*: int32
-
+    sealed*: bool
-proc hash*(x: ItemId): Hash =
-  var h: Hash = hash(x.module)
-  h = h !& hash(x.item)
-  result = !$h
 
 const
   PackageModuleId* = -3'i32
@@ -1083,10 +1085,12 @@ proc idGeneratorFromModule*(m: PSym): IdGenerator =
   result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0)
 
 proc nextSymId*(x: IdGenerator): ItemId {.inline.} =
+  assert(not x.sealed)
   inc x.symId
   result = ItemId(module: x.module, item: x.symId)
 
 proc nextTypeId*(x: IdGenerator): ItemId {.inline.} =
+  assert(not x.sealed)
   inc x.typeId
   result = ItemId(module: x.module, item: x.typeId)
 
@@ -1210,11 +1214,32 @@ proc newTreeIT*(kind: TNodeKind; info: TLineInfo; typ: PType; children: varargs[
 template previouslyInferred*(t: PType): PType =
   if t.sons.len > 1: t.lastSon else: nil
 
+when false:
+  import tables, strutils
+  var x: CountTable[string]
+
+  addQuitProc proc () {.noconv.} =
+    for k, v in pairs(x):
+      echo k
+      echo v
+
 proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym,
              info: TLineInfo; options: TOptions = {}): PSym =
   # generates a symbol and initializes the hash field too
   result = PSym(name: name, kind: symKind, flags: {}, info: info, itemId: id,
                 options: options, owner: owner, offset: defaultOffset)
+  when false:
+    if id.item > 2141:
+      let s = getStackTrace()
+      const words = ["createTypeBoundOps",
+        "initOperators",
+        "generateInstance",
+        "semIdentDef", "addLocalDecl"]
+      for w in words:
+        if w in s:
+          x.inc w
+          return
+      x.inc "<no category>"
 
 proc astdef*(s: PSym): PNode =
   # get only the definition (initializer) portion of the ast
@@ -1422,7 +1447,6 @@ proc assignType*(dest, src: PType) =
   dest.n = src.n
   dest.size = src.size
   dest.align = src.align
-  dest.attachedOps = src.attachedOps
   dest.lockLevel = src.lockLevel
   # this fixes 'type TLock = TSysLock':
   if src.sym != nil:
@@ -1622,11 +1646,9 @@ template transitionSymKindCommon*(k: TSymKind) =
 
 proc transitionGenericParamToType*(s: PSym) =
   transitionSymKindCommon(skType)
-  s.typeInstCache = obj.typeInstCache
 
 proc transitionRoutineSymKind*(s: PSym, kind: range[skProc..skTemplate]) =
   transitionSymKindCommon(kind)
-  s.procInstCache = obj.procInstCache
   s.gcUnsafetyReason = obj.gcUnsafetyReason
   s.transformedBody = obj.transformedBody
 
@@ -1893,10 +1915,8 @@ when false:
     for i in 0..<n.safeLen:
       if n[i].containsNil: return true
 
-template hasDestructor*(t: PType): bool = {tfHasAsgn, tfHasOwned} * t.flags != {}
 
-proc hasDisabledAsgn*(t: PType): bool =
+template hasDestructor*(t: PType): bool = {tfHasAsgn, tfHasOwned} * t.flags != {}
-  t.attachedOps[attachedAsgn] != nil and sfError in t.attachedOps[attachedAsgn].flags
 
 template incompleteType*(t: PType): bool =
   t.sym != nil and {sfForward, sfNoForward} * t.sym.flags == {sfForward}
@@ -1936,10 +1956,6 @@ proc addParam*(procType: PType; param: PSym) =
   procType.n.add newSymNode(param)
   rawAddSon(procType, param.typ)
 
-template destructor*(t: PType): PSym = t.attachedOps[attachedDestructor]
-template assignment*(t: PType): PSym = t.attachedOps[attachedAsgn]
-template asink*(t: PType): PSym = t.attachedOps[attachedSink]
-
 const magicsThatCanRaise = {
   mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst, mEcho}
 

compiler/ccgexprs.nim

@@ -1292,16 +1292,17 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) =
     genAssignment(p, a, b, {})
   else:
     let ti = genTypeInfoV1(p.module, typ, a.lode.info)
-    if bt.destructor != nil and not isTrivialProc(p.module.g.graph, bt.destructor):
+    let op = getAttachedOp(p.module.g.graph, bt, attachedDestructor)
+    if op != nil and not isTrivialProc(p.module.g.graph, op):
       # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
       # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
       # convention at least:
-      if bt.destructor.typ == nil or bt.destructor.typ.callConv != ccNimCall:
+      if op.typ == nil or op.typ.callConv != ccNimCall:
         localError(p.module.config, a.lode.info,
           "the destructor that is turned into a finalizer needs " &
           "to have the 'nimcall' calling convention")
       var f: TLoc
-      initLocExpr(p, newSymNode(bt.destructor), f)
+      initLocExpr(p, newSymNode(op), f)
       p.module.s[cfsTypeInit3].addf("$1->finalizer = (void*)$2;$n", [ti, rdLoc(f)])
 
     if a.storage == OnHeap and usesWriteBarrier(p.config):
@@ -2204,7 +2205,8 @@ proc genDestroy(p: BProc; n: PNode) =
     else: discard "nothing to do"
   else:
     let t = n[1].typ.skipTypes(abstractVar)
-    if t.destructor != nil and getBody(p.module.g.graph, t.destructor).len != 0:
+    let op = getAttachedOp(p.module.g.graph, t, attachedDestructor)
+    if op != nil and getBody(p.module.g.graph, op).len != 0:
       internalError(p.config, n.info, "destructor turned out to be not trivial")
     discard "ignore calls to the default destructor"
 
@@ -2236,14 +2238,8 @@ proc genSlice(p: BProc; e: PNode; d: var TLoc) =
 proc genEnumToStr(p: BProc, e: PNode, d: var TLoc) =
   const ToStringProcSlot = -4
   let t = e[1].typ.skipTypes(abstractInst+{tyRange})
-  var toStrProc: PSym = nil
+  let toStrProc = getToStringProc(p.module.g.graph, t)
-  for idx, p in items(t.methods):
+  # XXX need to modify this logic for IC.
-    if idx == ToStringProcSlot:
-      toStrProc = p
-      break
-  if toStrProc == nil:
-    toStrProc = genEnumToStrProc(t, e.info, p.module.g.graph, p.module.idgen)
-    t.methods.add((ToStringProcSlot, toStrProc))
   var n = copyTree(e)
   n[0] = newSymNode(toStrProc)
   expr(p, n, d)
@@ -2645,6 +2641,57 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) =
     if t.kind notin {tySequence, tyString}:
       d.storage = OnStatic
 
+proc genConstSetup(p: BProc; sym: PSym): bool =
+  let m = p.module
+  useHeader(m, sym)
+  if sym.loc.k == locNone:
+    fillLoc(sym.loc, locData, sym.ast, mangleName(p.module, sym), OnStatic)
+  if m.hcrOn: incl(sym.loc.flags, lfIndirect)
+  result = lfNoDecl notin sym.loc.flags
+
+proc genConstHeader(m, q: BModule; p: BProc, sym: PSym) =
+  assert(sym.loc.r != nil)
+  if m.hcrOn:
+    m.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]);
+    m.initProc.procSec(cpsLocals).addf(
+      "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r,
+      getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(q, sym)])
+  else:
+    let headerDecl = "extern NIM_CONST $1 $2;$n" %
+        [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]
+    m.s[cfsData].add(headerDecl)
+    if sfExportc in sym.flags and p.module.g.generatedHeader != nil:
+      p.module.g.generatedHeader.s[cfsData].add(headerDecl)
+
+proc genConstDefinition(q: BModule; p: BProc; sym: PSym) =
+  # add a suffix for hcr - will later init the global pointer with this data
+  let actualConstName = if q.hcrOn: sym.loc.r & "_const" else: sym.loc.r
+  q.s[cfsData].addf("N_LIB_PRIVATE NIM_CONST $1 $2 = $3;$n",
+      [getTypeDesc(q, sym.typ), actualConstName,
+      genBracedInit(q.initProc, sym.ast, isConst = true, sym.typ)])
+  if q.hcrOn:
+    # generate the global pointer with the real name
+    q.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(q, sym.loc.t, skVar), sym.loc.r])
+    # register it (but ignore the boolean result of hcrRegisterGlobal)
+    q.initProc.procSec(cpsLocals).addf(
+      "\thcrRegisterGlobal($1, \"$2\", sizeof($3), NULL, (void**)&$2);$n",
+      [getModuleDllPath(q, sym), sym.loc.r, rdLoc(sym.loc)])
+    # always copy over the contents of the actual constant with the _const
+    # suffix ==> this means that the constant is reloadable & updatable!
+    q.initProc.procSec(cpsLocals).add(ropecg(q,
+      "\t#nimCopyMem((void*)$1, (NIM_CONST void*)&$2, sizeof($3));$n",
+      [sym.loc.r, actualConstName, rdLoc(sym.loc)]))
+
+proc genConstStmt(p: BProc, n: PNode) =
+  # This code is only used in the new DCE implementation.
+  assert useAliveDataFromDce in p.module.flags
+  let m = p.module
+  for it in n:
+    if it[0].kind == nkSym:
+      let sym = it[0].sym
+      if not isSimpleConst(sym.typ) and sym.itemId.item in m.alive and genConstSetup(p, sym):
+        genConstDefinition(m, p, sym)
+
 proc expr(p: BProc, n: PNode, d: var TLoc) =
   when defined(nimCompilerStackraceHints):
     setFrameMsg p.config$n.info & " " & $n.kind
@@ -2655,7 +2702,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     var sym = n.sym
     case sym.kind
     of skMethod:
-      if {sfDispatcher, sfForward} * sym.flags != {}:
+      if useAliveDataFromDce in p.module.flags or {sfDispatcher, sfForward} * sym.flags != {}:
         # we cannot produce code for the dispatcher yet:
         fillProcLoc(p.module, n)
         genProcPrototype(p.module, sym)
@@ -2668,13 +2715,19 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       if sfCompileTime in sym.flags:
         localError(p.config, n.info, "request to generate code for .compileTime proc: " &
            sym.name.s)
-      genProc(p.module, sym)
+      if useAliveDataFromDce in p.module.flags:
+        fillProcLoc(p.module, n)
+        genProcPrototype(p.module, sym)
+      else:
+        genProc(p.module, sym)
       if sym.loc.r == nil or sym.loc.lode == nil:
         internalError(p.config, n.info, "expr: proc not init " & sym.name.s)
       putLocIntoDest(p, d, sym.loc)
     of skConst:
       if isSimpleConst(sym.typ):
         putIntoDest(p, d, n, genLiteral(p, sym.ast, sym.typ), OnStatic)
+      elif useAliveDataFromDce in p.module.flags:
+        genConstHeader(p.module, p.module, p, sym)
       else:
         genComplexConst(p, sym, d)
     of skEnumField:
@@ -2795,7 +2848,10 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkEmpty: discard
   of nkWhileStmt: genWhileStmt(p, n)
   of nkVarSection, nkLetSection: genVarStmt(p, n)
-  of nkConstSection: discard  # consts generated lazily on use
+  of nkConstSection:
+    if useAliveDataFromDce in p.module.flags:
+      genConstStmt(p, n)
+    # else: consts generated lazily on use
   of nkForStmt: internalError(p.config, n.info, "for statement not eliminated")
   of nkCaseStmt: genCase(p, n, d)
   of nkReturnStmt: genReturnStmt(p, n)
@@ -2840,13 +2896,16 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef:
     if n[genericParamsPos].kind == nkEmpty:
       var prc = n[namePos].sym
-      # due to a bug/limitation in the lambda lifting, unused inner procs
+      if useAliveDataFromDce in p.module.flags:
-      # are not transformed correctly. We work around this issue (#411) here
+        if p.module.alive.contains(prc.itemId.item):
-      # by ensuring it's no inner proc (owner is a module):
+          genProc(p.module, prc)
-      if prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags:
+      elif prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags:
         if ({sfExportc, sfCompilerProc} * prc.flags == {sfExportc}) or
             (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or
             (prc.kind == skMethod):
+          # due to a bug/limitation in the lambda lifting, unused inner procs
+          # are not transformed correctly. We work around this issue (#411) here
+          # by ensuring it's no inner proc (owner is a module).
           # Generate proc even if empty body, bugfix #11651.
           genProc(p.module, prc)
   of nkParForStmt: genParForStmt(p, n)

compiler/ccgstmts.nim

@@ -1530,20 +1530,21 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
         [rdLoc(a), rdLoc(tmp), discriminatorTableName(p.module, t, field),
          intLiteral(toInt64(lengthOrd(p.config, field.typ))+1)])
 
-proc genCaseObjDiscMapping(p: BProc, e: PNode, t: PType, field: PSym; d: var TLoc) =
+when false:
-  const ObjDiscMappingProcSlot = -5
+  proc genCaseObjDiscMapping(p: BProc, e: PNode, t: PType, field: PSym; d: var TLoc) =
-  var theProc: PSym = nil
+    const ObjDiscMappingProcSlot = -5
-  for idx, p in items(t.methods):
+    var theProc: PSym = nil
-    if idx == ObjDiscMappingProcSlot:
+    for idx, p in items(t.methods):
-      theProc = p
+      if idx == ObjDiscMappingProcSlot:
-      break
+        theProc = p
-  if theProc == nil:
+        break
-    theProc = genCaseObjDiscMapping(t, field, e.info, p.module.g.graph, p.module.idgen)
+    if theProc == nil:
-    t.methods.add((ObjDiscMappingProcSlot, theProc))
+      theProc = genCaseObjDiscMapping(t, field, e.info, p.module.g.graph, p.module.idgen)
-  var call = newNodeIT(nkCall, e.info, getSysType(p.module.g.graph, e.info, tyUInt8))
+      t.methods.add((ObjDiscMappingProcSlot, theProc))
-  call.add newSymNode(theProc)
+    var call = newNodeIT(nkCall, e.info, getSysType(p.module.g.graph, e.info, tyUInt8))
-  call.add e
+    call.add newSymNode(theProc)
-  expr(p, call, d)
+    call.add e
+    expr(p, call, d)
 
 proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   var a, tmp: TLoc

compiler/ccgtypes.nim

@@ -1308,7 +1308,7 @@ proc genTypeInfo2Name(m: BModule; t: PType): Rope =
 proc isTrivialProc(g: ModuleGraph; s: PSym): bool {.inline.} = getBody(g, s).len == 0
 
 proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
-  let theProc = t.attachedOps[op]
+  let theProc = getAttachedOp(m.g.graph, t, op)
   if theProc != nil and not isTrivialProc(m.g.graph, theProc):
     # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
     # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
@@ -1476,10 +1476,11 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
     genTupleInfo(m, x, origType, result, info)
   else: internalError(m.config, "genTypeInfoV1(" & $t.kind & ')')
 
-  if t.attachedOps[attachedDeepCopy] != nil:
+  var op = getAttachedOp(m.g.graph, t, attachedDeepCopy)
-    genDeepCopyProc(m, t.attachedOps[attachedDeepCopy], result)
+  if op == nil:
-  elif origType.attachedOps[attachedDeepCopy] != nil:
+    op = getAttachedOp(m.g.graph, origType, attachedDeepCopy)
-    genDeepCopyProc(m, origType.attachedOps[attachedDeepCopy], result)
+  if op != nil:
+    genDeepCopyProc(m, op, result)
 
   if optTinyRtti in m.config.globalOptions and t.kind == tyObject and sfImportc notin t.sym.flags:
     let v2info = genTypeInfoV2(m, origType, info)

compiler/cgen.nim

@@ -14,7 +14,7 @@ import
   nversion, nimsets, msgs, bitsets, idents, types,
   ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth,
   rodutils, renderer, cgendata, ccgmerge, aliases,
-  lowerings, tables, sets, ndi, lineinfos, pathutils, transf, enumtostr,
+  lowerings, tables, sets, ndi, lineinfos, pathutils, transf,
   injectdestructors
 
 when not defined(leanCompiler):
@@ -1189,48 +1189,16 @@ proc genProcNoForward(m: BModule, prc: PSym) =
     if sfInfixCall notin prc.flags: genProcPrototype(m, prc)
 
 proc requestConstImpl(p: BProc, sym: PSym) =
-  var m = p.module
+  if genConstSetup(p, sym):
-  useHeader(m, sym)
+    let m = p.module
-  if sym.loc.k == locNone:
+    # declare implementation:
-    fillLoc(sym.loc, locData, sym.ast, mangleName(p.module, sym), OnStatic)
+    var q = findPendingModule(m, sym)
-  if m.hcrOn: incl(sym.loc.flags, lfIndirect)
+    if q != nil and not containsOrIncl(q.declaredThings, sym.id):
-
+      assert q.initProc.module == q
-  if lfNoDecl in sym.loc.flags: return
+      genConstDefinition(q, p, sym)
-  # declare implementation:
+    # declare header:
-  var q = findPendingModule(m, sym)
+    if q != m and not containsOrIncl(m.declaredThings, sym.id):
-  if q != nil and not containsOrIncl(q.declaredThings, sym.id):
+      genConstHeader(m, q, p, sym)
-    assert q.initProc.module == q
-    # add a suffix for hcr - will later init the global pointer with this data
-    let actualConstName = if m.hcrOn: sym.loc.r & "_const" else: sym.loc.r
-    q.s[cfsData].addf("N_LIB_PRIVATE NIM_CONST $1 $2 = $3;$n",
-        [getTypeDesc(q, sym.typ), actualConstName,
-        genBracedInit(q.initProc, sym.ast, isConst = true, sym.typ)])
-    if m.hcrOn:
-      # generate the global pointer with the real name
-      q.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r])
-      # register it (but ignore the boolean result of hcrRegisterGlobal)
-      q.initProc.procSec(cpsLocals).addf(
-        "\thcrRegisterGlobal($1, \"$2\", sizeof($3), NULL, (void**)&$2);$n",
-        [getModuleDllPath(q, sym), sym.loc.r, rdLoc(sym.loc)])
-      # always copy over the contents of the actual constant with the _const
-      # suffix ==> this means that the constant is reloadable & updatable!
-      q.initProc.procSec(cpsLocals).add(ropecg(q,
-        "\t#nimCopyMem((void*)$1, (NIM_CONST void*)&$2, sizeof($3));$n",
-        [sym.loc.r, actualConstName, rdLoc(sym.loc)]))
-  # declare header:
-  if q != m and not containsOrIncl(m.declaredThings, sym.id):
-    assert(sym.loc.r != nil)
-    if m.hcrOn:
-      m.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]);
-      m.initProc.procSec(cpsLocals).addf(
-        "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r,
-        getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(q, sym)])
-    else:
-      let headerDecl = "extern NIM_CONST $1 $2;$n" %
-          [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]
-      m.s[cfsData].add(headerDecl)
-      if sfExportc in sym.flags and p.module.g.generatedHeader != nil:
-        p.module.g.generatedHeader.s[cfsData].add(headerDecl)
 
 proc isActivated(prc: PSym): bool = prc.typ != nil
 
@@ -1839,7 +1807,7 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule
 proc rawNewModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
   result = rawNewModule(g, module, AbsoluteFile toFullPath(conf, module.position.FileIndex))
 
-proc newModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
+proc newModule*(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
   # we should create only one cgen module for each module sym
   result = rawNewModule(g, module, conf)
   if module.position >= g.modules.len:
@@ -1922,13 +1890,9 @@ proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) =
 
     genStmts(p, n)
 
-proc myProcess(b: PPassContext, n: PNode): PNode =
+proc genTopLevelStmt*(m: BModule; n: PNode) =
-  result = n
+  ## Also called from `ic/cbackend.nim`.
-  if b == nil: return
+  if passes.skipCodegen(m.config, n): return
-  var m = BModule(b)
-  if passes.skipCodegen(m.config, n) or
-      not moduleHasChanged(m.g.graph, m.module):
-    return
   m.initProc.options = initProcOptions(m)
   #softRnl = if optLineDir in m.config.options: noRnl else: rnl
   # XXX replicate this logic!
@@ -1941,6 +1905,12 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
   else:
     genProcBody(m.initProc, transformedN)
 
+proc myProcess(b: PPassContext, n: PNode): PNode =
+  result = n
+  if b != nil:
+    var m = BModule(b)
+    genTopLevelStmt(m, n)
+
 proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
   if optForceFullMake notin m.config.globalOptions:
     if not moduleHasChanged(m.g.graph, m.module):
@@ -2013,7 +1983,7 @@ proc writeModule(m: BModule, pending: bool) =
     # that ``system.o`` is missing, so we need to call the C compiler for it:
     var cf = Cfile(nimname: m.module.name.s, cname: cfile,
                    obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
-    if not fileExists(cf.obj): cf.flags = {CfileFlag.Cached}
+    if fileExists(cf.obj): cf.flags = {CfileFlag.Cached}
     addFileToCompile(m.config, cf)
   onExit()
 
@@ -2037,10 +2007,8 @@ proc updateCachedModule(m: BModule) =
     cf.flags = {CfileFlag.Cached}
     addFileToCompile(m.config, cf)
 
-proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
+proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
-  result = n
+  ## Also called from IC.
-  if b == nil: return
-  var m = BModule(b)
   if sfMainModule in m.module.flags:
     # phase ordering problem here: We need to announce this
     # dependency to 'nimTestErrorFlag' before system.c has been written to disk.
@@ -2091,6 +2059,12 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
   let mm = m
   m.g.modulesClosed.add mm
 
+
+proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
+  result = n
+  if b == nil: return
+  finalCodegenActions(graph, BModule(b), n)
+
 proc genForwardedProcs(g: BModuleList) =
   # Forward declared proc:s lack bodies when first encountered, so they're given
   # a second pass here

compiler/cgendata.nim

@@ -112,7 +112,8 @@ type
     isHeaderFile,       # C source file is the header file
     includesStringh,    # C source file already includes ``<string.h>``
     objHasKidsValid     # whether we can rely on tfObjHasKids
-
+    useAliveDataFromDce # use the `alive: IntSet` field instead of
+                        # computing alive data on our own.
 
   BModuleList* = ref object of RootObj
     mainModProcs*, mainModInit*, otherModsInit*, mainDatInit*: Rope
@@ -154,6 +155,7 @@ type
     forwTypeCache*: TypeCache # cache for forward declarations of types
     declaredThings*: IntSet   # things we have declared in this .c file
     declaredProtos*: IntSet   # prototypes we have declared in this .c file
+    alive*: IntSet            # symbol IDs of alive data as computed by `dce.nim`
     headerFiles*: seq[string] # needed headers to include
     typeInfoMarker*: TypeCache # needed for generating type information
     typeInfoMarkerV2*: TypeCache

compiler/cgmeth.nim

@@ -107,11 +107,14 @@ proc attachDispatcher(s: PSym, dispatcher: PNode) =
       s.ast[resultPos] = newNodeI(nkEmpty, s.info)
     s.ast[dispatcherPos] = dispatcher
 
-proc createDispatcher(s: PSym; idgen: IdGenerator): PSym =
+proc createDispatcher(s: PSym; g: ModuleGraph; idgen: IdGenerator): PSym =
   var disp = copySym(s, nextSymId(idgen))
   incl(disp.flags, sfDispatcher)
   excl(disp.flags, sfExported)
+  let old = disp.typ
   disp.typ = copyType(disp.typ, nextTypeId(idgen), disp.typ.owner)
+  copyTypeProps(g, idgen.module, disp.typ, old)
+
   # we can't inline the dispatcher itself (for now):
   if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall
   disp.ast = copyTree(s.ast)
@@ -177,7 +180,7 @@ proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym, fromCache: bool) =
     of Invalid:
       if witness.isNil: witness = g.methods[i].methods[0]
   # create a new dispatcher:
-  g.methods.add((methods: @[s], dispatcher: createDispatcher(s, idgen)))
+  g.methods.add((methods: @[s], dispatcher: createDispatcher(s, g, idgen)))
   #echo "adding ", s.info
   #if fromCache:
   #  internalError(s.info, "no method dispatcher found")

compiler/guards.nim

@@ -82,26 +82,6 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
 
 proc interestingCaseExpr*(m: PNode): bool = isLetLocation(m, true)
 
-type
-  Operators* = object
-    opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym
-    opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym
-
-proc initOperators*(g: ModuleGraph): Operators =
-  result.opLe = createMagic(g, "<=", mLeI)
-  result.opLt = createMagic(g, "<", mLtI)
-  result.opAnd = createMagic(g, "and", mAnd)
-  result.opOr = createMagic(g, "or", mOr)
-  result.opIsNil = createMagic(g, "isnil", mIsNil)
-  result.opEq = createMagic(g, "==", mEqI)
-  result.opAdd = createMagic(g, "+", mAddI)
-  result.opSub = createMagic(g, "-", mSubI)
-  result.opMul = createMagic(g, "*", mMulI)
-  result.opDiv = createMagic(g, "div", mDivI)
-  result.opLen = createMagic(g, "len", mLengthSeq)
-  result.opNot = createMagic(g, "not", mNot)
-  result.opContains = createMagic(g, "contains", mInSet)
-
 proc swapArgs(fact: PNode, newOp: PSym): PNode =
   result = newNodeI(nkCall, fact.info, 3)
   result[0] = newSymNode(newOp)
@@ -404,16 +384,16 @@ proc usefulFact(n: PNode; o: Operators): PNode =
 type
   TModel* = object
     s*: seq[PNode] # the "knowledge base"
-    o*: Operators
+    g*: ModuleGraph
     beSmart*: bool
 
 proc addFact*(m: var TModel, nn: PNode) =
-  let n = usefulFact(nn, m.o)
+  let n = usefulFact(nn, m.g.operators)
   if n != nil:
     if not m.beSmart:
       m.s.add n
     else:
-      let c = canon(n, m.o)
+      let c = canon(n, m.g.operators)
       if c.getMagic == mAnd:
         addFact(m, c[1])
         addFact(m, c[2])
@@ -421,7 +401,7 @@ proc addFact*(m: var TModel, nn: PNode) =
         m.s.add c
 
 proc addFactNeg*(m: var TModel, n: PNode) =
-  let n = n.neg(m.o)
+  let n = n.neg(m.g.operators)
   if n != nil: addFact(m, n)
 
 proc sameOpr(a, b: PSym): bool =
@@ -740,7 +720,7 @@ proc doesImply*(facts: TModel, prop: PNode): TImplication =
       if result != impUnknown: return
 
 proc impliesNotNil*(m: TModel, arg: PNode): TImplication =
-  result = doesImply(m, m.o.opIsNil.buildCall(arg).neg(m.o))
+  result = doesImply(m, m.g.operators.opIsNil.buildCall(arg).neg(m.g.operators))
 
 proc simpleSlice*(a, b: PNode): BiggestInt =
   # returns 'c' if a..b matches (i+c)..(i+c), -1 otherwise. (i)..(i) is matched
@@ -833,7 +813,7 @@ proc ple(m: TModel; a, b: PNode): TImplication =
   if b.getMagic in someAdd:
     if zero() <=? b[2] and a <=? b[1]: return impYes
     # x <= y-c  if x+c <= y
-    if b[2] <=? zero() and (canon(m.o.opSub.buildCall(a, b[2]), m.o) <=? b[1]):
+    if b[2] <=? zero() and (canon(m.g.operators.opSub.buildCall(a, b[2]), m.g.operators) <=? b[1]):
       return impYes
 
   #   x+c <= y  if c <= 0 and x <= y
@@ -847,20 +827,20 @@ proc ple(m: TModel; a, b: PNode): TImplication =
   if a.getMagic in someMul and a[2].isValue and a[1].getMagic in someDiv and
       a[1][2].isValue:
     # simplify   (x div 4) * 2 <= y   to  x div (c div d)  <= y
-    if ple(m, buildCall(m.o.opDiv, a[1][1], `|div|`(a[1][2], a[2])), b) == impYes:
+    if ple(m, buildCall(m.g.operators.opDiv, a[1][1], `|div|`(a[1][2], a[2])), b) == impYes:
       return impYes
 
   # x*3 + x == x*4. It follows that:
   # x*3 + y <= x*4  if  y <= x  and 3 <= 4
   if a =~ x*dc + y and b =~ x2*ec:
     if sameTree(x, x2):
-      let ec1 = m.o.opAdd.buildCall(ec, minusOne())
+      let ec1 = m.g.operators.opAdd.buildCall(ec, minusOne())
       if x >=? 1 and ec >=? 1 and dc >=? 1 and dc <=? ec1 and y <=? x:
         return impYes
   elif a =~ x*dc and b =~ x2*ec + y:
     #echo "BUG cam ehrer e ", a, " <=? ", b
     if sameTree(x, x2):
-      let ec1 = m.o.opAdd.buildCall(ec, minusOne())
+      let ec1 = m.g.operators.opAdd.buildCall(ec, minusOne())
       if x >=? 1 and ec >=? 1 and dc >=? 1 and dc <=? ec1 and y <=? zero():
         return impYes
 
@@ -963,12 +943,12 @@ proc pleViaModel(model: TModel; aa, bb: PNode): TImplication =
   var b = bb
   if replacements.len > 0:
     m.s = @[]
-    m.o = model.o
+    m.g = model.g
     # make the other facts consistent:
     for fact in model.s:
       if fact != nil and fact.getMagic notin someEq:
         # XXX 'canon' should not be necessary here, but it is
-        m.s.add applyReplacements(fact, replacements).canon(m.o)
+        m.s.add applyReplacements(fact, replacements).canon(m.g.operators)
     a = applyReplacements(aa, replacements)
     b = applyReplacements(bb, replacements)
   else:
@@ -977,19 +957,19 @@ proc pleViaModel(model: TModel; aa, bb: PNode): TImplication =
   result = pleViaModelRec(m, a, b)
 
 proc proveLe*(m: TModel; a, b: PNode): TImplication =
-  let x = canon(m.o.opLe.buildCall(a, b), m.o)
+  let x = canon(m.g.operators.opLe.buildCall(a, b), m.g.operators)
   #echo "ROOT ", renderTree(x[1]), " <=? ", renderTree(x[2])
   result = ple(m, x[1], x[2])
   if result == impUnknown:
     # try an alternative:  a <= b  iff  not (b < a)  iff  not (b+1 <= a):
-    let y = canon(m.o.opLe.buildCall(m.o.opAdd.buildCall(b, one()), a), m.o)
+    let y = canon(m.g.operators.opLe.buildCall(m.g.operators.opAdd.buildCall(b, one()), a), m.g.operators)
     result = ~ple(m, y[1], y[2])
 
 proc addFactLe*(m: var TModel; a, b: PNode) =
-  m.s.add canon(m.o.opLe.buildCall(a, b), m.o)
+  m.s.add canon(m.g.operators.opLe.buildCall(a, b), m.g.operators)
 
 proc addFactLt*(m: var TModel; a, b: PNode) =
-  let bb = m.o.opAdd.buildCall(b, minusOne())
+  let bb = m.g.operators.opAdd.buildCall(b, minusOne())
   addFactLe(m, a, bb)
 
 proc settype(n: PNode): PType =
@@ -1021,14 +1001,14 @@ proc buildElse(n: PNode; o: Operators): PNode =
 
 proc addDiscriminantFact*(m: var TModel, n: PNode) =
   var fact = newNodeI(nkCall, n.info, 3)
-  fact[0] = newSymNode(m.o.opEq)
+  fact[0] = newSymNode(m.g.operators.opEq)
   fact[1] = n[0]
   fact[2] = n[1]
   m.s.add fact
 
 proc addAsgnFact*(m: var TModel, key, value: PNode) =
   var fact = newNodeI(nkCall, key.info, 3)
-  fact[0] = newSymNode(m.o.opEq)
+  fact[0] = newSymNode(m.g.operators.opEq)
   fact[1] = key
   fact[2] = value
   m.s.add fact
@@ -1044,7 +1024,7 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
   # However, nil checking requires exactly the same mechanism! But for now
   # we simply use sameTree and live with the unsoundness of the analysis.
   var check = newNodeI(nkCall, a.info, 3)
-  check[0] = newSymNode(m.o.opEq)
+  check[0] = newSymNode(m.g.operators.opEq)
   check[1] = a
   check[2] = b
   result = m.doesImply(check) == impYes
@@ -1052,9 +1032,9 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
 proc addCaseBranchFacts*(m: var TModel, n: PNode, i: int) =
   let branch = n[i]
   if branch.kind == nkOfBranch:
-    m.s.add buildOf(branch, n[0], m.o)
+    m.s.add buildOf(branch, n[0], m.g.operators)
   else:
-    m.s.add n.buildElse(m.o).neg(m.o)
+    m.s.add n.buildElse(m.g.operators).neg(m.g.operators)
 
 proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
   if check[1].kind == nkCurly:
@@ -1072,6 +1052,6 @@ proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
 
 proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef) =
   for i in 1..<n.len:
-    let check = buildProperFieldCheck(n[0], n[i], m.o)
+    let check = buildProperFieldCheck(n[0], n[i], m.g.operators)
     if check != nil and m.doesImply(check) != impYes:
       message(conf, n.info, warnProveField, renderTree(n[0])); break

compiler/ic/cbackend.nim

@@ -0,0 +1,102 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## New entry point into our C/C++ code generator. Ideally
+## somebody would rewrite the old backend (which is 8000 lines of crufty Nim code)
+## to work on packed trees directly and produce the C code as an AST which can
+## then be rendered to text in a very simple manner. Unfortunately nobody wrote
+## this code. So instead we wrap the existing cgen.nim and its friends so that
+## we call directly into the existing code generation logic but avoiding the
+## naive, outdated `passes` design. Thus you will see some
+## `useAliveDataFromDce in flags` checks in the old code -- the old code is
+## also doing cross-module dependency tracking and DCE that we don't need
+## anymore. DCE is now done as prepass over the entire packed module graph.
+
+import std / [intsets, algorithm]
+import ".." / [ast, options, lineinfos, modulegraphs, cgendata, cgen,
+  pathutils, extccomp, msgs]
+
+import packed_ast, to_packed_ast, bitabs, dce, rodfiles
+
+proc unpackTree(g: ModuleGraph; thisModule: int;
+                tree: PackedTree; n: NodePos): PNode =
+  var decoder = initPackedDecoder(g.config, g.cache)
+  result = loadNodes(decoder, g.packed, thisModule, tree, n)
+
+proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) =
+  if g.backend == nil:
+    g.backend = cgendata.newModuleList(g)
+
+  var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config)
+  bmod.idgen = idgenFromLoadedModule(m)
+  bmod.flags.incl useAliveDataFromDce
+  bmod.alive = move alive[m.module.position]
+
+  for p in allNodes(m.fromDisk.topLevel):
+    let n = unpackTree(g, m.module.position, m.fromDisk.topLevel, p)
+    cgen.genTopLevelStmt(bmod, n)
+
+  finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info))
+
+proc addFileToLink(config: ConfigRef; m: PSym) =
+  let filename = AbsoluteFile toFullPath(config, m.position.FileIndex)
+  let ext =
+      if config.backend == backendCpp: ".nim.cpp"
+      elif config.backend == backendObjc: ".nim.m"
+      else: ".nim.c"
+  let cfile = changeFileExt(completeCfilePath(config, withPackageName(config, filename)), ext)
+  var cf = Cfile(nimname: m.name.s, cname: cfile,
+                 obj: completeCfilePath(config, toObjFile(config, cfile)),
+                 flags: {CfileFlag.Cached})
+  addFileToCompile(config, cf)
+
+proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool =
+  let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms")
+  var s = newSeqOfCap[int32](alive[position].len)
+  for a in items(alive[position]): s.add int32(a)
+  sort(s)
+  var f2 = rodfiles.open(asymFile.string)
+  f2.loadHeader()
+  f2.loadSection aliveSymsSection
+  var oldData: seq[int32]
+  f2.loadSeq(oldData)
+  f2.close
+  if f2.err == ok and oldData == s:
+    result = false
+  else:
+    result = true
+    var f = rodfiles.create(asymFile.string)
+    f.storeHeader()
+    f.storeSection aliveSymsSection
+    f.storeSeq(s)
+    close f
+
+proc generateCode*(g: ModuleGraph) =
+  ## The single entry point, generate C(++) code for the entire
+  ## Nim program aka `ModuleGraph`.
+  var alive = computeAliveSyms(g.packed, g.config)
+
+  for i in 0..high(g.packed):
+    # case statement here to enforce exhaustive checks.
+    case g.packed[i].status
+    of undefined:
+      discard "nothing to do"
+    of loading:
+      assert false
+    of storing, outdated:
+      generateCodeForModule(g, g.packed[i], alive)
+    of loaded:
+      # Even though this module didn't change, DCE might trigger a change.
+      # Consider this case: Module A uses symbol S from B and B does not use
+      # S itself. A is then edited not to use S either. Thus we have to
+      # recompile B in order to remove S from the final result.
+      if aliveSymsChanged(g.config, g.packed[i].module.position, alive):
+        generateCodeForModule(g, g.packed[i], alive)
+      else:
+        addFileToLink(g.config, g.packed[i].module)

compiler/ic/dce.nim

@@ -0,0 +1,108 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Dead code elimination (=DCE) for IC.
+
+import std / intsets
+import ".." / [ast, options, lineinfos]
+
+import packed_ast, to_packed_ast, bitabs
+
+type
+  AliveSyms* = seq[IntSet]
+  AliveContext* = object ## Purpose is to fill the 'alive' field.
+    stack: seq[(int, NodePos)] ## A stack for marking symbols as alive.
+    decoder: PackedDecoder ## We need a PackedDecoder for module ID address translations.
+    thisModule: int  ## The module we're currently analysing for DCE.
+    alive: AliveSyms ## The final result of our computation.
+
+proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): bool =
+  ## "Exported to C" procs are special (these are marked with '.exportc') because these
+  ## must not be optimized away!
+  let symPtr = addr g[c.thisModule].fromDisk.sh.syms[symId]
+  let flags = symPtr.flags
+  # due to a bug/limitation in the lambda lifting, unused inner procs
+  # are not transformed correctly; issue (#411). However, the whole purpose here
+  # is to eliminate unused procs. So there is no special logic required for this case.
+  if sfCompileTime notin flags:
+    if ({sfExportc, sfCompilerProc} * flags == {sfExportc}) or
+        (symPtr.kind == skMethod):
+      result = true
+      # XXX: This used to be a condition to:
+      #  (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or
+
+template isNotGeneric(n: NodePos): bool = ithSon(tree, n, genericParamsPos).kind == nkEmpty
+
+proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: int32) =
+  ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will
+  ## be analysed.
+  if not c.alive[module].containsOrIncl(item):
+    let body = g[module].fromDisk.sh.syms[item].ast
+    if body != emptyNodeId:
+      c.stack.add((module, NodePos(body)))
+
+proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) =
+  ## Marks the symbols we encounter when we traverse the AST at `tree[n]` as alive, unless
+  ## it is purely in a declarative context (type section etc.).
+  case n.kind
+  of nkNone..pred(nkSym), succ(nkSym)..nkNilLit:
+    discard "ignore non-sym atoms"
+  of nkSym:
+    # This symbol is alive and everything its body references.
+    followLater(c, g, c.thisModule, n.operand)
+  of nkModuleRef:
+    let (n1, n2) = sons2(tree, n)
+    assert n1.kind == nkInt32Lit
+    assert n2.kind == nkInt32Lit
+    let m = n1.litId
+    let item = n2.operand
+    let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int
+    followLater(c, g, otherModule, item)
+  of nkMacroDef, nkTemplateDef, nkTypeSection, nkTypeOfExpr,
+     nkCommentStmt, nkIteratorDef, nkIncludeStmt,
+     nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
+     nkFromStmt, nkStaticStmt:
+    discard
+  of nkVarSection, nkLetSection, nkConstSection:
+    discard
+  of nkProcDef, nkConverterDef, nkMethodDef, nkLambda, nkDo, nkFuncDef:
+    if n.firstSon.kind == nkSym and isNotGeneric(n):
+      if isExportedToC(c, g, n.firstSon.operand):
+        let item = n.operand
+        # This symbol is alive and everything its body references.
+        followLater(c, g, c.thisModule, item)
+  else:
+    for son in sonsReadonly(tree, n):
+      aliveCode(c, g, tree, son)
+
+proc followNow(c: var AliveContext; g: PackedModuleGraph) =
+  ## Mark all entries in the stack. Marking can add more entries
+  ## to the stack but eventually we have looked at every alive symbol.
+  while c.stack.len > 0:
+    let (modId, ast) = c.stack.pop()
+    c.thisModule = modId
+    aliveCode(c, g, g[modId].fromDisk.bodies, ast)
+
+proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms =
+  ## Entry point for our DCE algorithm.
+  var c = AliveContext(stack: @[], decoder: PackedDecoder(config: conf),
+                       thisModule: -1, alive: newSeq[IntSet](g.len))
+  for i in countdown(high(g), 0):
+    if g[i].status != undefined:
+      c.thisModule = i
+      for p in allNodes(g[i].fromDisk.topLevel):
+        aliveCode(c, g, g[i].fromDisk.topLevel, p)
+  followNow(c, g)
+  result = move(c.alive)
+
+proc isAlive*(a: AliveSyms; module: int, item: int32): bool =
+  ## Backends use this to query if a symbol is `alive` which means
+  ## we need to produce (C/C++/etc) code for it.
+  result = a[module].contains(item)
+

compiler/ic/packed_ast.nim

@@ -290,6 +290,9 @@ template typ*(n: NodePos): PackedItemId =
 template flags*(n: NodePos): TNodeFlags =
   tree.nodes[n.int].flags
 
+template operand*(n: NodePos): int32 =
+  tree.nodes[n.int].operand
+
 proc span*(tree: PackedTree; pos: int): int {.inline.} =
   if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
 
@@ -451,3 +454,10 @@ when false:
     dest.add nkStrLit, msg, n.info
     copyTree(dest, tree, n)
     patch dest, patchPos
+
+iterator allNodes*(tree: PackedTree): NodePos =
+  var p = 0
+  while p < tree.len:
+    yield NodePos(p)
+    let s = span(tree, p)
+    inc p, s

compiler/ic/rodfiles.nim

@@ -31,6 +31,7 @@ type
     bodiesSection
     symsSection
     typesSection
+    aliveSymsSection # beware, this is stored in a `.alivesyms` file.
 
   RodFileError* = enum
     ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch,
@@ -134,7 +135,7 @@ proc loadHeader*(f: var RodFile) =
 
 proc storeSection*(f: var RodFile; s: RodSection) =
   if f.err != ok: return
-  assert f.currentSection == pred s
+  assert f.currentSection < s
   f.currentSection = s
   storePrim(f, s)
 

compiler/ic/to_packed_ast.nim

@@ -107,7 +107,7 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId =
     c.lastLit = result
     assert result != LitId(0)
 
-proc toFileIndex(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
+proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
   result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x])
 
 proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
@@ -280,16 +280,19 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
       paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel)
     storeNode(p, t, n)
 
-    for op, s in pairs t.attachedOps:
+    when false:
-      c.addMissing s
+      for op, s in pairs t.attachedOps:
-      p.attachedOps[op] = s.safeItemId(c, m)
+        c.addMissing s
+        p.attachedOps[op] = s.safeItemId(c, m)
 
     p.typeInst = t.typeInst.storeType(c, m)
     for kid in items t.sons:
       p.types.add kid.storeType(c, m)
-    for i, s in items t.methods:
+
-      c.addMissing s
+    when false:
-      p.methods.add (i, s.safeItemId(c, m))
+      for i, s in items t.methods:
+        c.addMissing s
+        p.methods.add (i, s.safeItemId(c, m))
     c.addMissing t.sym
     p.sym = t.sym.safeItemId(c, m)
     c.addMissing t.owner
@@ -568,11 +571,11 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
 
 type
   PackedDecoder* = object
-    lastModule*: int
+    lastModule: int
-    lastLit*: LitId
+    lastLit: LitId
-    lastFile*: FileIndex # remember the last lookup entry.
+    lastFile: FileIndex # remember the last lookup entry.
     config*: ConfigRef
-    cache: IdentCache
+    cache*: IdentCache
 
 type
   ModuleStatus* = enum
@@ -596,7 +599,7 @@ type
 proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType
 proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: PackedItemId): PSym
 
-proc toFileIndexCached(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; f: LitId): FileIndex =
+proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule: int; f: LitId): FileIndex =
   if c.lastLit == f and c.lastModule == thisModule:
     result = c.lastFile
   else:
@@ -611,8 +614,8 @@ proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; thisModul
   result = TLineInfo(line: x.line, col: x.col,
             fileIndex: toFileIndexCached(c, g, thisModule, x.file))
 
-proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
+proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
-               tree: PackedTree; n: NodePos): PNode =
+                tree: PackedTree; n: NodePos): PNode =
   let k = n.kind
   if k == nkNilRodNode:
     return nil
@@ -647,6 +650,14 @@ proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
     for n0 in sonsReadonly(tree, n):
       result.addAllowNil loadNodes(c, g, thisModule, tree, n0)
 
+proc initPackedDecoder*(config: ConfigRef; cache: IdentCache): PackedDecoder =
+  result = PackedDecoder(
+    lastModule: int32(-1),
+    lastLit: LitId(0),
+    lastFile: FileIndex(-1),
+    config: config,
+    cache: cache)
+
 proc loadProcHeader(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
                     tree: PackedTree; n: NodePos): PNode =
   # do not load the body of the proc. This will be done later in
@@ -761,14 +772,16 @@ proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
                         t: PackedType; si, item: int32; result: PType) =
   result.sym = loadSym(c, g, si, t.sym)
   result.owner = loadSym(c, g, si, t.owner)
-  for op, item in pairs t.attachedOps:
+  when false:
-    result.attachedOps[op] = loadSym(c, g, si, item)
+    for op, item in pairs t.attachedOps:
+      result.attachedOps[op] = loadSym(c, g, si, item)
   result.typeInst = loadType(c, g, si, t.typeInst)
   for son in items t.types:
     result.sons.add loadType(c, g, si, son)
   loadAstBody(t, n)
-  for gen, id in items t.methods:
+  when false:
-    result.methods.add((gen, loadSym(c, g, si, id)))
+    for gen, id in items t.methods:
+      result.methods.add((gen, loadSym(c, g, si, id)))
 
 proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType =
   if t == nilItemId:
@@ -819,11 +832,8 @@ proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
       lastFile: FileIndex(-1),
       config: conf,
       cache: cache)
-    var p = 0
+    for p in allNodes(m.fromDisk.toReplay):
-    while p < m.fromDisk.toReplay.len:
+      m.module.ast.add loadNodes(decoder, g, int(fileIdx), m.fromDisk.toReplay, p)
-      m.module.ast.add loadNodes(decoder, g, int(fileIdx), m.fromDisk.toReplay, NodePos p)
-      let s = span(m.fromDisk.toReplay, p)
-      inc p, s
 
 proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                     fileIdx: FileIndex): bool =
@@ -979,6 +989,10 @@ proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
   let values = g[int module].iface.getOrDefault(name)
   result = loadSym(decoder, g, int(module), values[0])
 
+proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
+  IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len,
+              typeId: int32 m.fromDisk.sh.types.len)
+
 # ------------------------- .rod file viewer ---------------------------------
 
 proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =

compiler/injectdestructors.nim

@@ -260,13 +260,13 @@ proc genOp(c: var Con; op: PSym; dest: PNode): PNode =
   result = newTree(nkCall, newSymNode(op), addrExp)
 
 proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
-  var op = t.attachedOps[kind]
+  var op = getAttachedOp(c.graph, t, kind)
   if op == nil or op.ast[genericParamsPos].kind != nkEmpty:
     # give up and find the canonical type instead:
     let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.graph.canonTypes.getOrDefault(h)
     if canon != nil:
-      op = canon.attachedOps[kind]
+      op = getAttachedOp(c.graph, canon, kind)
   if op == nil:
     #echo dest.typ.id
     globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
@@ -287,9 +287,9 @@ proc genDestroy(c: var Con; dest: PNode): PNode =
 proc canBeMoved(c: Con; t: PType): bool {.inline.} =
   let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
   if optOwnedRefs in c.graph.config.globalOptions:
-    result = t.kind != tyRef and t.attachedOps[attachedSink] != nil
+    result = t.kind != tyRef and getAttachedOp(c.graph, t, attachedSink) != nil
   else:
-    result = t.attachedOps[attachedSink] != nil
+    result = getAttachedOp(c.graph, t, attachedSink) != nil
 
 proc isNoInit(dest: PNode): bool {.inline.} =
   result = dest.kind == nkSym and sfNoInit in dest.sym.flags
@@ -302,7 +302,7 @@ proc genSink(c: var Con; dest, ri: PNode, isDecl = false): PNode =
     result = newTree(nkFastAsgn, dest, ri)
   else:
     let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink})
-    if t.attachedOps[attachedSink] != nil:
+    if getAttachedOp(c.graph, t, attachedSink) != nil:
       result = c.genOp(t, attachedSink, dest, ri)
       result.add ri
     else:
@@ -375,8 +375,8 @@ proc genDiscriminantAsgn(c: var Con; s: var Scope; n: PNode): PNode =
   let objType = leDotExpr[0].typ
 
   if hasDestructor(c, objType):
-    if objType.attachedOps[attachedDestructor] != nil and
+    if getAttachedOp(c.graph, objType, attachedDestructor) != nil and
-        sfOverriden in objType.attachedOps[attachedDestructor].flags:
+        sfOverriden in getAttachedOp(c.graph, objType, attachedDestructor).flags:
       localError(c.graph.config, n.info, errGenerated, """Assignment to discriminant for objects with user defined destructor is not supported, object must have default destructor.
 It is best to factor out piece of object that needs custom destructor into separate object or not use discriminator assignment""")
       result.add newTree(nkFastAsgn, le, tmp)
@@ -1095,7 +1095,7 @@ proc injectDestructorCalls*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; n:
     echo n
 
   if optCursorInference in g.config.options:
-    computeCursors(owner, n, g.config)
+    computeCursors(owner, n, g)
 
   var scope: Scope
   let body = p(n, c, scope, normal)

compiler/liftdestructors.nim

@@ -29,6 +29,10 @@ type
     c: PContext # c can be nil, then we are called from lambdalifting!
     idgen: IdGenerator
 
+template destructor*(t: PType): PSym = getAttachedOp(c.g, t, attachedDestructor)
+template assignment*(t: PType): PSym = getAttachedOp(c.g, t, attachedAsgn)
+template asink*(t: PType): PSym = getAttachedOp(c.g, t, attachedSink)
+
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
 proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
               info: TLineInfo; idgen: IdGenerator): PSym
@@ -42,8 +46,9 @@ proc at(a, i: PNode, elemType: PType): PNode =
   result[1] = i
   result.typ = elemType
 
-proc destructorOverriden(t: PType): bool =
+proc destructorOverriden(g: ModuleGraph; t: PType): bool =
-  t.attachedOps[attachedDestructor] != nil and sfOverriden in t.attachedOps[attachedDestructor].flags
+  let op = getAttachedOp(g, t, attachedDestructor)
+  op != nil and sfOverriden in op.flags
 
 proc fillBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   for i in 0..<t.len:
@@ -316,7 +321,7 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
                         field: var PSym): bool =
   if optSeqDestructors in c.g.config.globalOptions:
     var op = field
-    let destructorOverriden = destructorOverriden(t)
+    let destructorOverriden = destructorOverriden(c.g, t)
     if op != nil and op != c.fn and
         (sfOverriden in op.flags or destructorOverriden):
       if sfError in op.flags:
@@ -368,7 +373,7 @@ proc addDestructorCall(c: var TLiftCtx; orig: PType; body, x: PNode) =
     if op.ast[genericParamsPos].kind != nkEmpty:
       # patch generic destructor:
       op = instantiateGeneric(c, op, t, t.typeInst)
-      t.attachedOps[attachedDestructor] = op
+      setAttachedOp(c.g, c.idgen.module, t, attachedDestructor, op)
 
   if op == nil and (useNoGc(c, t) or requiresDestructor(c, t)):
     op = produceSym(c.g, c.c, t, attachedDestructor, c.info, c.idgen)
@@ -392,7 +397,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       if op.ast[genericParamsPos].kind != nkEmpty:
         # patch generic destructor:
         op = instantiateGeneric(c, op, t, t.typeInst)
-        t.attachedOps[attachedDestructor] = op
+        setAttachedOp(c.g, c.idgen.module, t, attachedDestructor, op)
 
       #markUsed(c.g.config, c.info, op, c.g.usageSym)
       onUse(c.info, op)
@@ -400,11 +405,17 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       result = true
     #result = addDestructorCall(c, t, body, x)
   of attachedAsgn, attachedSink, attachedTrace:
-    result = considerAsgnOrSink(c, t, body, x, y, t.attachedOps[c.kind])
+    var op = getAttachedOp(c.g, t, c.kind)
+    result = considerAsgnOrSink(c, t, body, x, y, op)
+    if op != nil:
+      setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
   of attachedDispose:
-    result = considerAsgnOrSink(c, t, body, x, nil, t.attachedOps[c.kind])
+    var op = getAttachedOp(c.g, t, c.kind)
+    result = considerAsgnOrSink(c, t, body, x, nil, op)
+    if op != nil:
+      setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
   of attachedDeepCopy:
-    let op = t.attachedOps[attachedDeepCopy]
+    let op = getAttachedOp(c.g, t, attachedDeepCopy)
     if op != nil:
       #markUsed(c.g.config, c.info, op, c.g.usageSym)
       onUse(c.info, op)
@@ -523,13 +534,15 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     doAssert t.destructor != nil
     body.add destructorCall(c, t.destructor, x)
   of attachedTrace:
-    if t.attachedOps[c.kind] == nil:
+    let op = getAttachedOp(c.g, t, c.kind)
+    if op == nil:
       return # protect from recursion
-    body.add newHookCall(c, t.attachedOps[c.kind], x, y)
+    body.add newHookCall(c, op, x, y)
   of attachedDispose:
-    if t.attachedOps[c.kind] == nil:
+    let op = getAttachedOp(c.g, t, c.kind)
+    if op == nil:
       return # protect from recursion
-    body.add newHookCall(c, t.attachedOps[c.kind], x, nil)
+    body.add newHookCall(c, op, x, nil)
 
 proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
@@ -890,10 +903,10 @@ proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
                             idgen: IdGenerator): PSym =
   assert typ.kind == tyDistinct
   let baseType = typ[0]
-  if baseType.attachedOps[kind] == nil:
+  if getAttachedOp(g, baseType, kind) == nil:
     discard produceSym(g, c, baseType, kind, info, idgen)
-  typ.attachedOps[kind] = baseType.attachedOps[kind]
+  result = getAttachedOp(g, baseType, kind)
-  result = typ.attachedOps[kind]
+  setAttachedOp(g, idgen.module, typ, kind, result)
 
 proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
               info: TLineInfo; idgen: IdGenerator): PSym =
@@ -936,7 +949,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
   if typ.kind == tyDistinct:
     return produceSymDistinctType(g, c, typ, kind, info, idgen)
 
-  result = typ.attachedOps[kind]
+  result = getAttachedOp(g, typ, kind)
   if result == nil:
     result = symPrototype(g, typ, typ.owner, kind, info, idgen)
 
@@ -949,12 +962,12 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
             else: newSymNode(result.typ.n[2].sym)
 
   # register this operation already:
-  typ.attachedOps[kind] = result
+  setAttachedOpPartial(g, idgen.module, typ, kind, result)
 
-  if kind == attachedSink and destructorOverriden(typ):
+  if kind == attachedSink and destructorOverriden(g, typ):
     ## compiler can use a combination of `=destroy` and memCopy for sink op
     dest.flags.incl sfCursor
-    result.ast[bodyPos].add newOpCall(a, typ.attachedOps[attachedDestructor], d[0])
+    result.ast[bodyPos].add newOpCall(a, getAttachedOp(g, typ, attachedDestructor), d[0])
     result.ast[bodyPos].add newAsgnStmt(d, src)
   else:
     var tk: TTypeKind
@@ -970,6 +983,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
     else:
       fillBody(a, typ, result.ast[bodyPos], d, src)
   if not a.canRaise: incl result.flags, sfNeverRaises
+  completePartialOp(g, idgen.module, typ, kind, result)
 
 
 proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym,
@@ -1002,30 +1016,34 @@ proc patchBody(g: ModuleGraph; c: PContext; n: PNode; info: TLineInfo; idgen: Id
   if n.kind in nkCallKinds:
     if n[0].kind == nkSym and n[0].sym.magic == mDestroy:
       let t = n[1].typ.skipTypes(abstractVar)
-      if t.destructor == nil:
+      if getAttachedOp(g, t, attachedDestructor) == nil:
         discard produceSym(g, c, t, attachedDestructor, info, idgen)
 
-      if t.destructor != nil:
+      let op = getAttachedOp(g, t, attachedDestructor)
-        if t.destructor.ast[genericParamsPos].kind != nkEmpty:
+      if op != nil:
+        if op.ast[genericParamsPos].kind != nkEmpty:
           internalError(g.config, info, "resolved destructor is generic")
-        if t.destructor.magic == mDestroy:
+        if op.magic == mDestroy:
           internalError(g.config, info, "patching mDestroy with mDestroy?")
-        n[0] = newSymNode(t.destructor)
+        n[0] = newSymNode(op)
   for x in n: patchBody(g, c, x, info, idgen)
 
-template inst(field, t, idgen) =
+proc inst(g: ModuleGraph; c: PContext; t: PType; kind: TTypeAttachedOp; idgen: IdGenerator;
-  if field.ast != nil and field.ast[genericParamsPos].kind != nkEmpty:
+          info: TLineInfo) =
+  let op = getAttachedOp(g, t, kind)
+  if op != nil and op.ast != nil and op.ast[genericParamsPos].kind != nkEmpty:
     if t.typeInst != nil:
       var a: TLiftCtx
       a.info = info
       a.g = g
-      a.kind = k
+      a.kind = kind
       a.c = c
       a.idgen = idgen
 
-      field = instantiateGeneric(a, field, t, t.typeInst)
+      let opInst = instantiateGeneric(a, op, t, t.typeInst)
-      if field.ast != nil:
+      if opInst.ast != nil:
-        patchBody(g, c, field.ast, info, a.idgen)
+        patchBody(g, c, opInst.ast, info, a.idgen)
+      setAttachedOp(g, idgen.module, t, kind, opInst)
     else:
       localError(g.config, info, "unresolved generic parameter")
 
@@ -1062,24 +1080,26 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
                      else: attachedSink
 
   # bug #15122: We need to produce all prototypes before entering the
-  # mind boggling recursion. Hacks like these imply we shoule rewrite
+  # mind boggling recursion. Hacks like these imply we should rewrite
   # this module.
   var generics: array[attachedDestructor..attachedDispose, bool]
   for k in attachedDestructor..lastAttached:
-    generics[k] = canon.attachedOps[k] != nil
+    generics[k] = getAttachedOp(g, canon, k) != nil
     if not generics[k]:
-      canon.attachedOps[k] = symPrototype(g, canon, canon.owner, k, info, idgen)
+      setAttachedOp(g, idgen.module, canon, k,
+          symPrototype(g, canon, canon.owner, k, info, idgen))
 
   # we generate the destructor first so that other operators can depend on it:
   for k in attachedDestructor..lastAttached:
     if not generics[k]:
       discard produceSym(g, c, canon, k, info, idgen)
     else:
-      inst(canon.attachedOps[k], canon, idgen)
+      inst(g, c, canon, k, idgen, info)
     if canon != orig:
-      orig.attachedOps[k] = canon.attachedOps[k]
+      setAttachedOp(g, idgen.module, orig, k, getAttachedOp(g, canon, k))
 
-  if not isTrival(orig.destructor):
+  if not isTrival(getAttachedOp(g, orig, attachedDestructor)):
     #or not isTrival(orig.assignment) or
     # not isTrival(orig.sink):
     orig.flags.incl tfHasAsgn
+    # ^ XXX Breaks IC!

compiler/main.nim

@@ -21,6 +21,7 @@ import
   modules,
   modulegraphs, tables, lineinfos, pathutils, vmprofiler
 
+import ic / cbackend
 from ic / to_packed_ast import rodViewer
 
 when not defined(leanCompiler):
@@ -73,14 +74,15 @@ proc commandCompileToC(graph: ModuleGraph) =
   setOutFile(conf)
   extccomp.initVars(conf)
   semanticPasses(graph)
-  registerPass(graph, cgenPass)
+  if conf.symbolFiles == disabledSf:
+    registerPass(graph, cgenPass)
 
-  if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
+    if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
-    let proj = changeFileExt(conf.projectFull, "")
+      let proj = changeFileExt(conf.projectFull, "")
-    if not changeDetectedViaJsonBuildInstructions(conf, proj):
+      if not changeDetectedViaJsonBuildInstructions(conf, proj):
-      # nothing changed
+        # nothing changed
-      graph.config.notes = graph.config.mainPackageNotes
+        graph.config.notes = graph.config.mainPackageNotes
-      return
+        return
 
   if not extccomp.ccHasSaneOverflow(conf):
     conf.symbols.defineSymbol("nimEmulateOverflowChecks")
@@ -88,8 +90,14 @@ proc commandCompileToC(graph: ModuleGraph) =
   compileProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
-  cgenWriteModules(graph.backend, conf)
+  if conf.symbolFiles == disabledSf:
-  if conf.cmd != cmdTcc:
+    cgenWriteModules(graph.backend, conf)
+  else:
+    generateCode(graph)
+    # graph.backend can be nil under IC when nothing changed at all:
+    if graph.backend != nil:
+      cgenWriteModules(graph.backend, conf)
+  if conf.cmd != cmdTcc and graph.backend != nil:
     extccomp.callCCompiler(conf)
     # for now we do not support writing out a .json file with the build instructions when HCR is on
     if not conf.hcrOn:

compiler/modulegraphs.nim

@@ -27,9 +27,20 @@ type
     pureEnums*: seq[PSym]
     interf: TStrTable
 
+  Operators* = object
+    opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym
+    opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym
+
   ModuleGraph* = ref object
     ifaces*: seq[Iface]  ## indexed by int32 fileIdx
     packed*: PackedModuleGraph
+
+    typeInstCache*: Table[ItemId, seq[PType]] # A symbol's ItemId.
+    procInstCache*: Table[ItemId, seq[PInstantiation]] # A symbol's ItemId.
+    attachedOps*: array[TTypeAttachedOp, Table[ItemId, PSym]] # Type ID, destructors, etc.
+    methodsPerType*: Table[ItemId, seq[(int, PSym)]] # Type ID, attached methods
+    enumToStringProcs*: Table[ItemId, PSym]
+
     startupPackedConfig*: PackedConfig
     packageSyms*: TStrTable
     deps*: IntSet # the dependency graph or potentially its transitive closure.
@@ -71,6 +82,7 @@ type
     strongSemCheck*: proc (graph: ModuleGraph; owner: PSym; body: PNode) {.nimcall.}
     compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
     idgen*: IdGenerator
+    operators*: Operators
 
   TPassContext* = object of RootObj # the pass's context
     idgen*: IdGenerator
@@ -85,6 +97,67 @@ type
                  close: TPassClose,
                  isFrontend: bool]
 
+iterator typeInstCacheItems*(g: ModuleGraph; s: PSym): PType =
+  if g.typeInstCache.contains(s.itemId):
+    let x = addr(g.typeInstCache[s.itemId])
+    for t in x[]:
+      yield t
+
+proc addToGenericCache*(g: ModuleGraph; module: int; s: PSym; inst: PType) =
+  g.typeInstCache.mgetOrPut(s.itemId, @[]).add inst
+  # XXX Also add to the packed module!
+
+iterator procInstCacheItems*(g: ModuleGraph; s: PSym): PInstantiation =
+  if g.procInstCache.contains(s.itemId):
+    let x = addr(g.procInstCache[s.itemId])
+    for t in x[]:
+      yield t
+
+proc addToGenericProcCache*(g: ModuleGraph; module: int; s: PSym; inst: PInstantiation) =
+  g.procInstCache.mgetOrPut(s.itemId, @[]).add inst
+  # XXX Also add to the packed module!
+
+proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym =
+  ## returns the requested attached operation for type `t`. Can return nil
+  ## if no such operation exists.
+  result = g.attachedOps[op].getOrDefault(t.itemId)
+
+proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
+  ## we also need to record this to the packed module.
+  g.attachedOps[op][t.itemId] = value
+  # XXX Also add to the packed module!
+
+proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
+  ## we also need to record this to the packed module.
+  g.attachedOps[op][t.itemId] = value
+  # XXX Also add to the packed module!
+
+proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
+  discard "To implement"
+
+proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
+  result = g.enumToStringProcs.getOrDefault(t.itemId)
+  assert result != nil
+
+proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) =
+  g.enumToStringProcs[t.itemId] = value
+
+iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) =
+  for a, b in items g.methodsPerType.getOrDefault(t.itemId):
+    yield (a, b)
+
+proc addMethodToGeneric*(g: ModuleGraph; module: int; t: PType; col: int; m: PSym) =
+  g.methodsPerType.mgetOrPut(t.itemId, @[]).add (col, m)
+
+proc hasDisabledAsgn*(g: ModuleGraph; t: PType): bool =
+  let op = getAttachedOp(g, t, attachedAsgn)
+  result = op != nil and sfError in op.flags
+
+proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) =
+  for k in low(TTypeAttachedOp)..high(TTypeAttachedOp):
+    let op = getAttachedOp(g, src, k)
+    if op != nil:
+      setAttachedOp(g, module, dest, k, op)
 
 const
   cb64 = [
@@ -241,6 +314,22 @@ proc registerModule*(g: ModuleGraph; m: PSym) =
   g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[])
   initStrTable(g.ifaces[m.position].interf)
 
+proc initOperators(g: ModuleGraph): Operators =
+  # These are safe for IC.
+  result.opLe = createMagic(g, "<=", mLeI)
+  result.opLt = createMagic(g, "<", mLtI)
+  result.opAnd = createMagic(g, "and", mAnd)
+  result.opOr = createMagic(g, "or", mOr)
+  result.opIsNil = createMagic(g, "isnil", mIsNil)
+  result.opEq = createMagic(g, "==", mEqI)
+  result.opAdd = createMagic(g, "+", mAddI)
+  result.opSub = createMagic(g, "-", mSubI)
+  result.opMul = createMagic(g, "*", mMulI)
+  result.opDiv = createMagic(g, "div", mDivI)
+  result.opLen = createMagic(g, "len", mLengthSeq)
+  result.opNot = createMagic(g, "not", mNot)
+  result.opContains = createMagic(g, "contains", mInSet)
+
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()
   # A module ID of -1 means that the symbol is not attached to a module at all,
@@ -265,6 +354,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.cacheTables = initTable[string, BTree[string, PNode]]()
   result.canonTypes = initTable[SigHash, PType]()
   result.symBodyHashes = initTable[int, SigHash]()
+  result.operators = initOperators(result)
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)

compiler/msgs.nim

@@ -460,7 +460,7 @@ proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
   if result == 0:
     try:
       for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
-        addSourceLine conf, fileIdx, line.string
+        addSourceLine conf, fileIdx, line
     except IOError:
       discard
     result = conf.m.fileInfos[fileIdx.int32].lines.len

compiler/options.nim

@@ -721,9 +721,9 @@ proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile,
   result = subdir / RelativeFile f.string.splitPath.tail
   #echo "completeGeneratedFilePath(", f, ") = ", result
 
-proc toRodFile*(conf: ConfigRef; f: AbsoluteFile): AbsoluteFile =
+proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile =
   result = changeFileExt(completeGeneratedFilePath(conf,
-    withPackageName(conf, f)), RodExt)
+    withPackageName(conf, f)), ext)
 
 proc rawFindFile(conf: ConfigRef; f: RelativeFile; suppressStdlib: bool): AbsoluteFile =
   for it in conf.searchPaths:

compiler/sem.nim

@@ -17,10 +17,7 @@ import
   intsets, transf, vmdef, vm, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity,
   lowerings, plugins/active, lineinfos, strtabs, int128,
-  isolation_check, typeallowed
+  isolation_check, typeallowed, modulegraphs, enumtostr
-
-from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward,
-  systemModuleSym, semtab, getBody, someSym, allSyms
 
 when defined(nimfix):
   import nimfix/prettybase
@@ -84,7 +81,7 @@ proc fitNodePostMatch(c: PContext, formal: PType, arg: PNode): PNode =
   if x.kind in {nkPar, nkTupleConstr, nkCurly} and formal.kind != tyUntyped:
     changeType(c, x, formal, check=true)
   result = arg
-  result = skipHiddenSubConv(result, c.idgen)
+  result = skipHiddenSubConv(result, c.graph, c.idgen)
 
 
 proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
@@ -138,7 +135,10 @@ proc commonType*(c: PContext; x, y: PType): PType =
       let aEmpty = isEmptyContainer(a[i])
       let bEmpty = isEmptyContainer(b[i])
       if aEmpty != bEmpty:
-        if nt.isNil: nt = copyType(a, nextTypeId(c.idgen), a.owner)
+        if nt.isNil:
+          nt = copyType(a, nextTypeId(c.idgen), a.owner)
+          copyTypeProps(c.graph, c.idgen.module, nt, a)
+
         nt[i] = if aEmpty: b[i] else: a[i]
     if not nt.isNil: result = nt
     #elif b[idx].kind == tyEmpty: return x

compiler/semcall.nim

@@ -685,9 +685,9 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym =
     var x: PType
     if param.typ.kind == tyVar:
       x = newTypeS(param.typ.kind, c)
-      x.addSonSkipIntLit(t.baseOfDistinct(c.idgen), c.idgen)
+      x.addSonSkipIntLit(t.baseOfDistinct(c.graph, c.idgen), c.idgen)
     else:
-      x = t.baseOfDistinct(c.idgen)
+      x = t.baseOfDistinct(c.graph, c.idgen)
     call.add(newNodeIT(nkEmpty, fn.info, x))
   if hasDistinct:
     let filter = if fn.kind in {skProc, skFunc}: {skProc, skFunc} else: {fn.kind}

compiler/semdata.nim

@@ -558,6 +558,7 @@ proc saveRodFile*(c: PContext) =
         addPragmaComputation(c, n)
     if sfSystemModule in c.module.flags:
       c.graph.systemModuleComplete = true
+    c.idgen.sealed = true # no further additions are allowed
     if c.config.symbolFiles != stressTest:
       # For stress testing we seek to reload the symbols from memory. This
       # way much of the logic is tested but the test is reproducible as it does

compiler/seminst.nim

@@ -95,9 +95,9 @@ proc sameInstantiation(a, b: TInstantiation): bool =
                                    ExactGcSafety}): return
     result = true
 
-proc genericCacheGet(genericSym: PSym, entry: TInstantiation;
+proc genericCacheGet(g: ModuleGraph; genericSym: PSym, entry: TInstantiation;
                      id: CompilesId): PSym =
-  for inst in genericSym.procInstCache:
+  for inst in procInstCacheItems(g, genericSym):
     if (inst.compilesId == 0 or inst.compilesId == id) and sameInstantiation(entry, inst[]):
       return inst.sym
 
@@ -369,7 +369,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   if tfTriggersCompileTime in result.typ.flags:
     incl(result.flags, sfCompileTime)
   n[genericParamsPos] = c.graph.emptyNode
-  var oldPrc = genericCacheGet(fn, entry[], c.compilesContextId)
+  var oldPrc = genericCacheGet(c.graph, fn, entry[], c.compilesContextId)
   if oldPrc == nil:
     # we MUST not add potentially wrong instantiations to the caching mechanism.
     # This means recursive instantiations behave differently when in
@@ -378,7 +378,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
     #if c.compilesContextId == 0:
     rawHandleSelf(c, result)
     entry.compilesId = c.compilesContextId
-    fn.procInstCache.add(entry)
+    addToGenericProcCache(c.graph, c.module.position, fn, entry)
     c.generics.add(makeInstPair(fn, entry))
     if n[pragmasPos].kind != nkEmpty:
       pragma(c, result, n[pragmasPos], allRoutinePragmas)

compiler/semmagic.nim

@@ -382,6 +382,8 @@ proc semUnown(c: PContext; n: PNode): PNode =
       let b = unownedType(c, t[^1])
       if b != t[^1]:
         result = copyType(t, nextTypeId c.idgen, t.owner)
+        copyTypeProps(c.graph, c.idgen.module, result, t)
+
         result[^1] = b
         result.flags.excl tfHasOwned
       else:
@@ -541,7 +543,8 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
 
         # check if we converted this finalizer into a destructor already:
         let t = whereToBindTypeHook(c, fin.typ[1].skipTypes(abstractInst+{tyRef}))
-        if t != nil and t.attachedOps[attachedDestructor] != nil and t.attachedOps[attachedDestructor].owner == fin:
+        if t != nil and getAttachedOp(c.graph, t, attachedDestructor) != nil and
+            getAttachedOp(c.graph, t, attachedDestructor).owner == fin:
           discard "already turned this one into a finalizer"
         else:
           bindTypeHook(c, turnFinalizerIntoDestructor(c, fin, n.info), n, attachedDestructor)
@@ -549,8 +552,9 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
   of mDestroy:
     result = n
     let t = n[1].typ.skipTypes(abstractVar)
-    if t.destructor != nil:
+    let op = getAttachedOp(c.graph, t, attachedDestructor)
-      result[0] = newSymNode(t.destructor)
+    if op != nil:
+      result[0] = newSymNode(op)
   of mUnown:
     result = semUnown(c, n)
   of mExists, mForall:

compiler/semparallel.nim

@@ -81,7 +81,7 @@ proc initAnalysisCtx(g: ModuleGraph): AnalysisCtx =
   result.slices = @[]
   result.args = @[]
   result.guards.s = @[]
-  result.guards.o = initOperators(g)
+  result.guards.g = g
   result.graph = g
 
 proc lookupSlot(c: AnalysisCtx; s: PSym): int =
@@ -138,7 +138,7 @@ proc checkLe(c: AnalysisCtx; a, b: PNode) =
 
 proc checkBounds(c: AnalysisCtx; arr, idx: PNode) =
   checkLe(c, lowBound(c.graph.config, arr), idx)
-  checkLe(c, idx, highBound(c.graph.config, arr, c.guards.o))
+  checkLe(c, idx, highBound(c.graph.config, arr, c.graph.operators))
 
 proc addLowerBoundAsFacts(c: var AnalysisCtx) =
   for v in c.locals:
@@ -147,8 +147,8 @@ proc addLowerBoundAsFacts(c: var AnalysisCtx) =
 
 proc addSlice(c: var AnalysisCtx; n: PNode; x, le, ri: PNode) =
   checkLocal(c, n)
-  let le = le.canon(c.guards.o)
+  let le = le.canon(c.graph.operators)
-  let ri = ri.canon(c.guards.o)
+  let ri = ri.canon(c.graph.operators)
   # perform static bounds checking here; and not later!
   let oldState = c.guards.s.len
   addLowerBoundAsFacts(c)
@@ -192,7 +192,7 @@ proc subStride(c: AnalysisCtx; n: PNode): PNode =
   if isLocal(n):
     let s = c.lookupSlot(n.sym)
     if s >= 0 and c.locals[s].stride != nil:
-      result = buildAdd(n, c.locals[s].stride.intVal, c.guards.o)
+      result = buildAdd(n, c.locals[s].stride.intVal, c.graph.operators)
     else:
       result = n
   elif n.safeLen > 0:
@@ -307,16 +307,16 @@ proc analyseCase(c: var AnalysisCtx; n: PNode) =
 proc analyseIf(c: var AnalysisCtx; n: PNode) =
   analyse(c, n[0][0])
   let oldFacts = c.guards.s.len
-  addFact(c.guards, canon(n[0][0], c.guards.o))
+  addFact(c.guards, canon(n[0][0], c.graph.operators))
 
   analyse(c, n[0][1])
   for i in 1..<n.len:
     let branch = n[i]
     setLen(c.guards.s, oldFacts)
     for j in 0..i-1:
-      addFactNeg(c.guards, canon(n[j][0], c.guards.o))
+      addFactNeg(c.guards, canon(n[j][0], c.graph.operators))
     if branch.len > 1:
-      addFact(c.guards, canon(branch[0], c.guards.o))
+      addFact(c.guards, canon(branch[0], c.graph.operators))
     for i in 0..<branch.len:
       analyse(c, branch[i])
   setLen(c.guards.s, oldFacts)
@@ -382,13 +382,13 @@ proc analyse(c: var AnalysisCtx; n: PNode) =
       # loop may never execute:
       let oldState = c.locals.len
       let oldFacts = c.guards.s.len
-      addFact(c.guards, canon(n[0], c.guards.o))
+      addFact(c.guards, canon(n[0], c.graph.operators))
       analyse(c, n[1])
       setLen(c.locals, oldState)
       setLen(c.guards.s, oldFacts)
       # we know after the loop the negation holds:
       if not hasSubnodeWith(n[1], nkBreakStmt):
-        addFactNeg(c.guards, canon(n[0], c.guards.o))
+        addFactNeg(c.guards, canon(n[0], c.graph.operators))
     dec c.inLoop
   of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef,
       nkMacroDef, nkTemplateDef, nkConstSection, nkPragma, nkFuncDef:

compiler/sempass2.nim

@@ -723,7 +723,7 @@ proc checkLe(c: PEffects; a, b: PNode) =
 
 proc checkBounds(c: PEffects; arr, idx: PNode) =
   checkLe(c, lowBound(c.config, arr), idx)
-  checkLe(c, idx, highBound(c.config, arr, c.guards.o))
+  checkLe(c, idx, highBound(c.config, arr, c.guards.g.operators))
 
 proc checkRange(c: PEffects; value: PNode; typ: PType) =
   let t = typ.skipTypes(abstractInst - {tyRange})
@@ -818,9 +818,9 @@ proc trackCall(tracked: PEffects; n: PNode) =
     if opKind != -1:
       # rebind type bounds operations after createTypeBoundOps call
       let t = n[1].typ.skipTypes({tyAlias, tyVar})
-      if a.sym != t.attachedOps[TTypeAttachedOp(opKind)]:
+      if a.sym != getAttachedOp(tracked.graph, t, TTypeAttachedOp(opKind)):
         createTypeBoundOps(tracked, t, n.info)
-        let op = t.attachedOps[TTypeAttachedOp(opKind)]
+        let op = getAttachedOp(tracked.graph, t, TTypeAttachedOp(opKind))
         if op != nil:
           n[0].sym = op
 
@@ -1236,7 +1236,7 @@ proc initEffects(g: ModuleGraph; effects: PNode; s: PSym; t: var TEffects; c: PC
   t.ownerModule = s.getModule
   t.init = @[]
   t.guards.s = @[]
-  t.guards.o = initOperators(g)
+  t.guards.g = g
   when defined(drnim):
     t.currOptions = g.config.options + s.options - {optStaticBoundsCheck}
   else:
@@ -1310,7 +1310,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) =
     var goals: set[Goal] = {}
     if strictFuncs in c.features: goals.incl constParameters
     if views in c.features: goals.incl borrowChecking
-    var partitions = computeGraphPartitions(s, body, g.config, goals)
+    var partitions = computeGraphPartitions(s, body, g, goals)
     if not t.hasSideEffect and t.hasDangerousAssign:
       t.hasSideEffect = varpartitions.hasSideEffect(partitions, mutationInfo)
     if views in c.features:

compiler/semstmts.nim

@@ -1693,12 +1693,13 @@ proc bindTypeHook(c: PContext; s: PSym; n: PNode; op: TTypeAttachedOp) =
       else: break
     if obj.kind in {tyObject, tyDistinct, tySequence, tyString}:
       obj = canonType(c, obj)
-      if obj.attachedOps[op] == s:
+      let ao = getAttachedOp(c.graph, obj, op)
+      if ao == s:
         discard "forward declared destructor"
-      elif obj.attachedOps[op].isNil and tfCheckedForDestructor notin obj.flags:
+      elif ao.isNil and tfCheckedForDestructor notin obj.flags:
-        obj.attachedOps[op] = s
+        setAttachedOp(c.graph, c.module.position, obj, op, s)
       else:
-        prevDestructor(c, obj.attachedOps[op], obj, n.info)
+        prevDestructor(c, ao, obj, n.info)
       noError = true
       if obj.owner.getModule != s.getModule:
         localError(c.config, n.info, errGenerated,
@@ -1726,7 +1727,8 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
         elif t.kind == tyGenericInvocation: t = t[0]
         else: break
       if t.kind in {tyObject, tyDistinct, tyEnum, tySequence, tyString}:
-        if t.attachedOps[attachedDeepCopy].isNil: t.attachedOps[attachedDeepCopy] = s
+        if getAttachedOp(c.graph, t, attachedDeepCopy).isNil:
+          setAttachedOp(c.graph, c.module.position, t, attachedDeepCopy, s)
         else:
           localError(c.config, n.info, errGenerated,
                      "cannot bind another 'deepCopy' to: " & typeToString(t))
@@ -1766,12 +1768,13 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
         obj = canonType(c, obj)
         #echo "ATTACHING TO ", obj.id, " ", s.name.s, " ", cast[int](obj)
         let k = if name == "=" or name == "=copy": attachedAsgn else: attachedSink
-        if obj.attachedOps[k] == s:
+        let ao = getAttachedOp(c.graph, obj, k)
+        if ao == s:
           discard "forward declared op"
-        elif obj.attachedOps[k].isNil and tfCheckedForDestructor notin obj.flags:
+        elif ao.isNil and tfCheckedForDestructor notin obj.flags:
-          obj.attachedOps[k] = s
+          setAttachedOp(c.graph, c.module.position, obj, k, s)
         else:
-          prevDestructor(c, obj.attachedOps[k], obj, n.info)
+          prevDestructor(c, ao, obj, n.info)
         if obj.owner.getModule != s.getModule:
           localError(c.config, n.info, errGenerated,
             "type bound operation `" & name & "` can be defined only in the same module with its type (" & obj.typeToString() & ")")
@@ -1827,7 +1830,7 @@ proc semMethodPrototype(c: PContext; s: PSym; n: PNode) =
                                       tyAlias, tySink, tyOwned})
         if x.kind == tyObject and t.len-1 == n[genericParamsPos].len:
           foundObj = true
-          x.methods.add((col,s))
+          addMethodToGeneric(c.graph, c.module.position, x, col, s)
     message(c.config, n.info, warnDeprecated, "generic methods are deprecated")
     #if not foundObj:
     #  message(c.config, n.info, warnDeprecated, "generic method not attachable to object type is deprecated")

compiler/semtypes.nim

@@ -154,6 +154,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
     addPureEnum(c, result.sym)
   if tfNotNil in e.typ.flags and not hasNull:
     result.flags.incl tfRequiresInit
+  setToStringProc(c.graph, result, genEnumToStrProc(result, n.info, c.graph, c.idgen))
 
 proc semSet(c: PContext, n: PNode, prev: PType): PType =
   result = newOrPrevType(tySet, prev, c)
@@ -1117,6 +1118,8 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
   of tyGenericInst:
     if paramType.lastSon.kind == tyUserTypeClass:
       var cp = copyType(paramType, nextTypeId c.idgen, getCurrOwner(c))
+      copyTypeProps(c.graph, c.idgen.module, cp, paramType)
+
       cp.kind = tyUserTypeClassInst
       return addImplicitGeneric(c, cp, paramTypId, info, genericParams, paramName)
 
@@ -1530,6 +1533,7 @@ proc semTypeExpr(c: PContext, n: PNode; prev: PType): PType =
 proc freshType(c: PContext; res, prev: PType): PType {.inline.} =
   if prev.isNil:
     result = copyType(res, nextTypeId c.idgen, res.owner)
+    copyTypeProps(c.graph, c.idgen.module, result, res)
   else:
     result = res
 
@@ -1843,7 +1847,9 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     of mExpr:
       result = semTypeNode(c, n[0], nil)
       if result != nil:
+        let old = result
         result = copyType(result, nextTypeId c.idgen, getCurrOwner(c))
+        copyTypeProps(c.graph, c.idgen.module, result, old)
         for i in 1..<n.len:
           result.rawAddSon(semTypeNode(c, n[i], nil))
     of mDistinct:
@@ -2117,12 +2123,16 @@ proc semGenericParamList(c: PContext, n: PNode, father: PType = nil): PNode =
       typ.flags.incl tfGenericTypeParam
 
       for j in 0..<a.len-2:
-        let finalType = if j == 0: typ
+        var finalType: PType
-                        else: copyType(typ, nextTypeId c.idgen, typ.owner)
+        if j == 0:
-                        # it's important the we create an unique
+          finalType = typ
-                        # type for each generic param. the index
+        else:
-                        # of the parameter will be stored in the
+          finalType = copyType(typ, nextTypeId c.idgen, typ.owner)
-                        # attached symbol.
+          copyTypeProps(c.graph, c.idgen.module, finalType, typ)
+        # it's important the we create an unique
+        # type for each generic param. the index
+        # of the parameter will be stored in the
+        # attached symbol.
         var paramName = a[j]
         var covarianceFlag = tfUnresolved
 

compiler/semtypinst.nim

@@ -10,7 +10,7 @@
 # This module does the instantiation of generic types.
 
 import ast, astalgo, msgs, types, magicsys, semdata, renderer, options,
-  lineinfos
+  lineinfos, modulegraphs
 
 const tfInstClearedFlags = {tfHasMeta, tfUnresolved}
 
@@ -30,15 +30,12 @@ proc checkConstructedType*(conf: ConfigRef; info: TLineInfo, typ: PType) =
       if t[0].kind != tyObject or tfFinal in t[0].flags:
         localError(info, errInheritanceOnlyWithNonFinalObjects)
 
-proc searchInstTypes*(key: PType): PType =
+proc searchInstTypes*(g: ModuleGraph; key: PType): PType =
   let genericTyp = key[0]
   if not (genericTyp.kind == tyGenericBody and
       key[0] == genericTyp and genericTyp.sym != nil): return
 
-  when not defined(nimNoNilSeqs):
+  for inst in typeInstCacheItems(g, genericTyp.sym):
-    if genericTyp.sym.typeInstCache == nil: return
-
-  for inst in genericTyp.sym.typeInstCache:
     if inst.id == key.id: return inst
     if inst.len < key.len:
       # XXX: This happens for prematurely cached
@@ -57,14 +54,12 @@ proc searchInstTypes*(key: PType): PType =
 
       return inst
 
-proc cacheTypeInst*(inst: PType) =
+proc cacheTypeInst(g: ModuleGraph; moduleId: int; inst: PType) =
-  # XXX: add to module's generics
-  #      update the refcount
   let gt = inst[0]
   let t = if gt.kind == tyGenericBody: gt.lastSon else: gt
   if t.kind in {tyStatic, tyError, tyGenericParam} + tyTypeClasses:
     return
-  gt.sym.typeInstCache.add(inst)
+  addToGenericCache(g, moduleId, gt.sym, inst)
 
 type
   LayeredIdTable* = ref object
@@ -306,6 +301,7 @@ proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
     result = t.exactReplica
   else:
     result = copyType(t, nextTypeId(cl.c.idgen), t.owner)
+    copyTypeProps(cl.c.graph, cl.c.idgen.module, result, t)
     #cl.typeMap.topLayer.idTablePut(result, t)
 
   if cl.allowMetaTypes: return
@@ -332,7 +328,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   if cl.allowMetaTypes:
     result = PType(idTableGet(cl.localCache, t))
   else:
-    result = searchInstTypes(t)
+    result = searchInstTypes(cl.c.graph, t)
 
   if result != nil and sameFlags(result, t):
     when defined(reportCacheHits):
@@ -351,7 +347,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
 
   if header != t:
     # search again after first pass:
-    result = searchInstTypes(header)
+    result = searchInstTypes(cl.c.graph, header)
     if result != nil and sameFlags(result, t):
       when defined(reportCacheHits):
         echo "Generic instantiation cached ", typeToString(result), " for ",
@@ -368,7 +364,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   # we need to add the candidate here, before it's fully instantiated for
   # recursive instantions:
   if not cl.allowMetaTypes:
-    cacheTypeInst(result)
+    cacheTypeInst(cl.c.graph, cl.c.module.position, result)
   else:
     idTablePut(cl.localCache, t, result)
 
@@ -408,13 +404,13 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   if newbody.isGenericAlias: newbody = newbody.skipGenericAlias
   rawAddSon(result, newbody)
   checkPartialConstructedType(cl.c.config, cl.info, newbody)
-  let dc = newbody.attachedOps[attachedDeepCopy]
   if not cl.allowMetaTypes:
-    if dc != nil and sfFromGeneric notin newbody.attachedOps[attachedDeepCopy].flags:
+    let dc = cl.c.graph.getAttachedOp(newbody, attachedDeepCopy)
+    if dc != nil and sfFromGeneric notin dc.flags:
       # 'deepCopy' needs to be instantiated for
       # generics *when the type is constructed*:
-      newbody.attachedOps[attachedDeepCopy] = cl.c.instTypeBoundOp(cl.c, dc, result, cl.info,
+      cl.c.graph.setAttachedOp(cl.c.module.position, newbody, attachedDeepCopy,
-                                                                   attachedDeepCopy, 1)
+          cl.c.instTypeBoundOp(cl.c, dc, result, cl.info, attachedDeepCopy, 1))
     if newbody.typeInst == nil:
       # doAssert newbody.typeInst == nil
       newbody.typeInst = result
@@ -436,8 +432,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
     if tfFromGeneric notin mm.flags:
       # bug #5479, prevent endless recursions here:
       incl mm.flags, tfFromGeneric
-      let methods = mm.methods
+      for col, meth in methodsForGeneric(cl.c.graph, mm):
-      for col, meth in items(methods):
         # we instantiate the known methods belonging to that type, this causes
         # them to be registered and that's enough, so we 'discard' the result.
         discard cl.c.instTypeBoundOp(cl.c, meth, result, cl.info,

compiler/transf.nim

@@ -496,7 +496,7 @@ proc transformConv(c: PTransf, n: PNode): PNode =
   of tyOpenArray, tyVarargs:
     result = transform(c, n[1])
     #result = transformSons(c, n)
-    result.typ = takeType(n.typ, n[1].typ, c.idgen)
+    result.typ = takeType(n.typ, n[1].typ, c.graph, c.idgen)
     #echo n.info, " came here and produced ", typeToString(result.typ),
     #   " from ", typeToString(n.typ), " and ", typeToString(n[1].typ)
   of tyCString:
@@ -612,6 +612,8 @@ proc isSimpleIteratorVar(c: PTransf; iter: PSym): bool =
   rec(getBody(c.graph, iter), iter, dangerousYields)
   result = dangerousYields == 0
 
+template destructor(t: PType): PSym = getAttachedOp(c.graph, t, attachedDestructor)
+
 proc transformFor(c: PTransf, n: PNode): PNode =
   # generate access statements for the parameters (unless they are constant)
   # put mapping from formal parameters to actual parameters

compiler/types.nim

@@ -11,7 +11,7 @@
 
 import
   intsets, ast, astalgo, trees, msgs, strutils, platform, renderer, options,
-  lineinfos, int128
+  lineinfos, int128, modulegraphs
 
 type
   TPreferedDesc* = enum
@@ -1307,11 +1307,12 @@ proc containsGenericTypeIter(t: PType, closure: RootRef): bool =
 proc containsGenericType*(t: PType): bool =
   result = iterOverType(t, containsGenericTypeIter, nil)
 
-proc baseOfDistinct*(t: PType; idgen: IdGenerator): PType =
+proc baseOfDistinct*(t: PType; g: ModuleGraph; idgen: IdGenerator): PType =
   if t.kind == tyDistinct:
     result = t[0]
   else:
     result = copyType(t, nextTypeId idgen, t.owner)
+    copyTypeProps(g, idgen.module, result, t)
     var parent: PType = nil
     var it = result
     while it.kind in {tyPtr, tyRef, tyOwned}:
@@ -1449,7 +1450,7 @@ proc isEmptyContainer*(t: PType): bool =
   of tyGenericInst, tyAlias, tySink: result = isEmptyContainer(t.lastSon)
   else: result = false
 
-proc takeType*(formal, arg: PType; idgen: IdGenerator): PType =
+proc takeType*(formal, arg: PType; g: ModuleGraph; idgen: IdGenerator): PType =
   # param: openArray[string] = []
   # [] is an array constructor of length 0 of type string!
   if arg.kind == tyNil:
@@ -1458,6 +1459,7 @@ proc takeType*(formal, arg: PType; idgen: IdGenerator): PType =
   elif formal.kind in {tyOpenArray, tyVarargs, tySequence} and
       arg.isEmptyContainer:
     let a = copyType(arg.skipTypes({tyGenericInst, tyAlias}), nextTypeId(idgen), arg.owner)
+    copyTypeProps(g, idgen.module, a, arg)
     a[ord(arg.kind == tyArray)] = formal[0]
     result = a
   elif formal.kind in {tyTuple, tySet} and arg.kind == formal.kind:
@@ -1465,14 +1467,14 @@ proc takeType*(formal, arg: PType; idgen: IdGenerator): PType =
   else:
     result = arg
 
-proc skipHiddenSubConv*(n: PNode; idgen: IdGenerator): PNode =
+proc skipHiddenSubConv*(n: PNode; g: ModuleGraph; idgen: IdGenerator): PNode =
   if n.kind == nkHiddenSubConv:
     # param: openArray[string] = []
     # [] is an array constructor of length 0 of type string!
     let formal = n.typ
     result = n[1]
     let arg = result.typ
-    let dest = takeType(formal, arg, idgen)
+    let dest = takeType(formal, arg, g, idgen)
     if dest == arg and formal.kind != tyUntyped:
       #echo n.info, " came here for ", formal.typeToString
       result = n

compiler/varpartitions.nim

@@ -28,7 +28,7 @@
 ## See https://nim-lang.github.io/Nim/manual_experimental.html#view-types-algorithm
 ## for a high-level description of how borrow checking works.
 
-import ast, types, lineinfos, options, msgs, renderer, typeallowed
+import ast, types, lineinfos, options, msgs, renderer, typeallowed, modulegraphs
 from trees import getMagic, isNoSideEffectPragma, stupidStmtListExpr
 from isolation_check import canAlias
 
@@ -99,7 +99,7 @@ type
     unanalysableMutation: bool
     inAsgnSource, inConstructor, inNoSideEffectSection: int
     owner: PSym
-    config: ConfigRef
+    g: ModuleGraph
 
 proc mutationAfterConnection(g: MutationInfo): bool {.inline.} =
   #echo g.maxMutation.int, " ", g.minConnection.int, " ", g.param
@@ -509,11 +509,11 @@ proc borrowFrom(c: var Partitions; dest: PSym; src: PNode) =
 
   let s = pathExpr(src, c.owner)
   if s == nil:
-    localError(c.config, src.info, "cannot borrow from " & $src & ", it is not a path expression; " & url)
+    localError(c.g.config, src.info, "cannot borrow from " & $src & ", it is not a path expression; " & url)
   elif s.kind == nkSym:
     if dest.kind == skResult:
       if s.sym.kind != skParam or s.sym.position != 0:
-        localError(c.config, src.info, "'result' must borrow from the first parameter")
+        localError(c.g.config, src.info, "'result' must borrow from the first parameter")
 
     let vid = variableId(c, dest)
     if vid >= 0:
@@ -539,13 +539,13 @@ proc borrowingCall(c: var Partitions; destType: PType; n: PNode; i: int) =
     when false:
       let isView = directViewType(destType) == immutableView
       if n[0].kind == nkSym and n[0].sym.name.s == "[]=":
-        localError(c.config, n[i].info, "attempt to mutate an immutable view")
+        localError(c.g.config, n[i].info, "attempt to mutate an immutable view")
 
     for j in i+1..<n.len:
       if getMagic(n[j]) == mSlice:
         borrowFrom(c, v.sym, n[j])
   else:
-    localError(c.config, n[i].info, "cannot determine the target of the borrow")
+    localError(c.g.config, n[i].info, "cannot determine the target of the borrow")
 
 proc borrowingAsgn(c: var Partitions; dest, src: PNode) =
   proc mutableParameter(n: PNode): bool {.inline.} =
@@ -569,7 +569,7 @@ proc borrowingAsgn(c: var Partitions; dest, src: PNode) =
           mutableParameter(dest[0][0]):
         discard "remains a mutable location anyhow"
       else:
-        localError(c.config, dest.info, "attempt to mutate a borrowed location from an immutable view")
+        localError(c.g.config, dest.info, "attempt to mutate a borrowed location from an immutable view")
     of noView: discard "nothing to do"
 
 proc containsPointer(t: PType): bool =
@@ -603,7 +603,7 @@ proc deps(c: var Partitions; dest, src: PNode) =
         for s in sources:
           if s == d.sym:
             discard "assignments like: it = it.next are fine"
-          elif {sfGlobal, sfThread} * s.flags != {} or hasDisabledAsgn(s.typ):
+          elif {sfGlobal, sfThread} * s.flags != {} or hasDisabledAsgn(c.g, s.typ):
             # do not borrow from a global variable or from something with a
             # disabled assignment operator.
             c.s[vid].flags.incl preventCursor
@@ -826,8 +826,8 @@ proc computeLiveRanges(c: var Partitions; n: PNode) =
   else:
     for child in n: computeLiveRanges(c, child)
 
-proc computeGraphPartitions*(s: PSym; n: PNode; config: ConfigRef; goals: set[Goal]): Partitions =
+proc computeGraphPartitions*(s: PSym; n: PNode; g: ModuleGraph; goals: set[Goal]): Partitions =
-  result = Partitions(owner: s, config: config, goals: goals)
+  result = Partitions(owner: s, g: g, goals: goals)
   if s.kind notin {skModule, skMacro}:
     let params = s.typ.n
     for i in 1..<params.len:
@@ -892,8 +892,8 @@ proc checkBorrowedLocations*(par: var Partitions; body: PNode; config: ConfigRef
       #if par.s[rid].con.kind == isRootOf and dangerousMutation(par.graphs[par.s[rid].con.graphIndex], par.s[i]):
       #  cannotBorrow(config, s, par.graphs[par.s[rid].con.graphIndex])
 
-proc computeCursors*(s: PSym; n: PNode; config: ConfigRef) =
+proc computeCursors*(s: PSym; n: PNode; g: ModuleGraph) =
-  var par = computeGraphPartitions(s, n, config, {cursorInference})
+  var par = computeGraphPartitions(s, n, g, {cursorInference})
   for i in 0 ..< par.s.len:
     let v = addr(par.s[i])
     if v.flags * {ownsData, preventCursor} == {} and v.sym.kind notin {skParam, skResult} and