ic refactoring (#17778)

* minor IC documentation update

* IC: refactoring: removed the 'shared' type and fields, these were a leftover from an earlier design
This commit is contained in:
Andreas Rumpf 2021-04-19 11:48:06 +02:00 • committed by GitHub
commit 4b0b536419
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 146 additions and 159 deletions

View file

@ -51,7 +51,12 @@ type
emittedTypeInfo*: seq[string]
backendFlags*: set[ModuleBackendFlag]
sh*: Shared
syms*: seq[PackedSym]
types*: seq[PackedType]
strings*: BiTable[string] # we could share these between modules.
numbers*: BiTable[BiggestInt] # we also store floats in here so
# that we can assure that every bit is kept
cfg: PackedConfig
PackedEncoder* = object
@ -66,6 +71,50 @@ type
symMarker*: IntSet #Table[ItemId, SymId] # ItemId.item -> SymId
config*: ConfigRef
proc toString*(tree: PackedTree; n: NodePos; m: PackedModule; nesting: int;
result: var string) =
let pos = n.int
if result.len > 0 and result[^1] notin {' ', '\n'}:
result.add ' '
result.add $tree[pos].kind
case tree.nodes[pos].kind
of nkNone, nkEmpty, nkNilLit, nkType: discard
of nkIdent, nkStrLit..nkTripleStrLit:
result.add " "
result.add m.strings[LitId tree.nodes[pos].operand]
of nkSym:
result.add " "
result.add m.strings[m.syms[tree.nodes[pos].operand].name]
of directIntLit:
result.add " "
result.addInt tree.nodes[pos].operand
of externSIntLit:
result.add " "
result.addInt m.numbers[LitId tree.nodes[pos].operand]
of externUIntLit:
result.add " "
result.add $cast[uint64](m.numbers[LitId tree.nodes[pos].operand])
of nkFloatLit..nkFloat128Lit:
result.add " "
result.add $cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand])
else:
result.add "(\n"
for i in 1..(nesting+1)*2: result.add ' '
for child in sonsReadonly(tree, n):
toString(tree, child, m, nesting + 1, result)
result.add "\n"
for i in 1..nesting*2: result.add ' '
result.add ")"
#for i in 1..nesting*2: result.add ' '
proc toString*(tree: PackedTree; n: NodePos; m: PackedModule): string =
result = ""
toString(tree, n, m, 0, result)
proc debug*(tree: PackedTree; m: PackedModule) =
stdout.write toString(tree, NodePos 0, m)
proc isActive*(e: PackedEncoder): bool = e.config != nil
proc disable(e: var PackedEncoder) = e.config = nil
@ -134,14 +183,14 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId =
result = c.filenames.getOrDefault(x)
if result == LitId(0):
let p = msgs.toFullPath(c.config, x)
result = getOrIncl(m.sh.strings, p)
result = getOrIncl(m.strings, p)
c.filenames[x] = result
c.lastFile = x
c.lastLit = result
assert result != LitId(0)
proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x])
result = msgs.fileInfoIdx(config, AbsoluteFile m.strings[x])
proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
for it in mitems(m.includes):
@ -151,7 +200,6 @@ proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) =
## setup a context for serializing to packed ast
m.sh = Shared()
c.thisModule = moduleSym.itemId.module
c.config = config
m.moduleFlags = moduleSym.flags
@ -179,11 +227,11 @@ proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex)
m.imports.add toLitId(f, c, m)
proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
let nameId = getOrIncl(m.sh.strings, s.name.s)
let nameId = getOrIncl(m.strings, s.name.s)
m.hidden.add((nameId, s.itemId.item))
proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
let nameId = getOrIncl(m.sh.strings, s.name.s)
let nameId = getOrIncl(m.strings, s.name.s)
m.exports.add((nameId, s.itemId.item))
proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
@ -201,12 +249,12 @@ proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
m.methods.add s.itemId.item
proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
let nameId = getOrIncl(m.sh.strings, s.name.s)
let nameId = getOrIncl(m.strings, s.name.s)
m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m),
item: s.itemId.item)))
proc addCompilerProc*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
let nameId = getOrIncl(m.sh.strings, s.name.s)
let nameId = getOrIncl(m.strings, s.name.s)
m.compilerProcs.add((nameId, s.itemId.item))
proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule)
@ -225,11 +273,11 @@ proc flush(c: var PackedEncoder; m: var PackedModule) =
proc toLitId(x: string; m: var PackedModule): LitId =
## store a string as a literal
result = getOrIncl(m.sh.strings, x)
result = getOrIncl(m.strings, x)
proc toLitId(x: BiggestInt; m: var PackedModule): LitId =
## store an integer as a literal
result = getOrIncl(m.sh.numbers, x)
result = getOrIncl(m.numbers, x)
proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo =
PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m))
@ -299,8 +347,8 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item)
if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item):
if t.uniqueId.item >= m.sh.types.len:
setLen m.sh.types, t.uniqueId.item+1
if t.uniqueId.item >= m.types.len:
setLen m.types, t.uniqueId.item+1
var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv,
size: t.size, align: t.align, nonUniqueId: t.itemId.item,
@ -315,7 +363,7 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
p.owner = t.owner.safeItemId(c, m)
# fill the reserved slot, nothing else:
m.sh.types[t.uniqueId.item] = p
m.types[t.uniqueId.item] = p
proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib =
## the plib hangs off the psym via the .annex field
@ -334,8 +382,8 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)
if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item):
if s.itemId.item >= m.sh.syms.len:
setLen m.sh.syms, s.itemId.item+1
if s.itemId.item >= m.syms.len:
setLen m.syms, s.itemId.item+1
assert sfForward notin s.flags
@ -363,7 +411,7 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
p.cname = toLitId(s.cname, m)
# fill the reserved slot, nothing else:
m.sh.syms[s.itemId.item] = p
m.syms[s.itemId.item] = p
proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
## add a remote symbol reference to the tree
@ -387,7 +435,7 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa
typeId: storeTypeLater(n.typ, c, m), info: info)
of nkIdent:
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
operand: int32 getOrIncl(m.sh.strings, n.ident.s),
operand: int32 getOrIncl(m.strings, n.ident.s),
typeId: storeTypeLater(n.typ, c, m), info: info)
of nkSym:
if n.sym.itemId.module == c.thisModule:
@ -405,15 +453,15 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa
typeId: storeTypeLater(n.typ, c, m), info: info)
of externIntLit:
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
operand: int32 getOrIncl(m.sh.numbers, n.intVal),
operand: int32 getOrIncl(m.numbers, n.intVal),
typeId: storeTypeLater(n.typ, c, m), info: info)
of nkStrLit..nkTripleStrLit:
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
operand: int32 getOrIncl(m.sh.strings, n.strVal),
operand: int32 getOrIncl(m.strings, n.strVal),
typeId: storeTypeLater(n.typ, c, m), info: info)
of nkFloatLit..nkFloat128Lit:
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
operand: int32 getOrIncl(m.sh.numbers, cast[BiggestInt](n.floatVal)),
operand: int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)),
typeId: storeTypeLater(n.typ, c, m), info: info)
else:
let patchPos = ir.prepare(n.kind, n.flags,
@ -499,7 +547,6 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef;
ignoreConfig = false): RodFileError =
m.sh = Shared()
var f = rodfiles.open(filename.string)
f.loadHeader()
f.loadSection configSection
@ -519,7 +566,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
f.loadSection section
f.load data
loadTabSection stringsSection, m.sh.strings
loadTabSection stringsSection, m.strings
loadSeqSection checkSumsSection, m.includes
if not includesIdentical(m, config):
@ -527,7 +574,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
loadSeqSection depsSection, m.imports
loadTabSection numbersSection, m.sh.numbers
loadTabSection numbersSection, m.numbers
loadSeqSection exportsSection, m.exports
loadSeqSection hiddenSection, m.hidden
@ -545,8 +592,8 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
loadSeqSection toReplaySection, m.toReplay.nodes
loadSeqSection topLevelSection, m.topLevel.nodes
loadSeqSection bodiesSection, m.bodies.nodes
loadSeqSection symsSection, m.sh.syms
loadSeqSection typesSection, m.sh.types
loadSeqSection symsSection, m.syms
loadSeqSection typesSection, m.types
loadSeqSection typeInstCacheSection, m.typeInstCache
loadSeqSection procInstCacheSection, m.procInstCache
@ -586,13 +633,13 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
f.storeSection section
f.store data
storeTabSection stringsSection, m.sh.strings
storeTabSection stringsSection, m.strings
storeSeqSection checkSumsSection, m.includes
storeSeqSection depsSection, m.imports
storeTabSection numbersSection, m.sh.numbers
storeTabSection numbersSection, m.numbers
storeSeqSection exportsSection, m.exports
storeSeqSection hiddenSection, m.hidden
@ -610,9 +657,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
storeSeqSection topLevelSection, m.topLevel.nodes
storeSeqSection bodiesSection, m.bodies.nodes
storeSeqSection symsSection, m.sh.syms
storeSeqSection symsSection, m.syms
storeSeqSection typesSection, m.sh.types
storeSeqSection typesSection, m.types
storeSeqSection typeInstCacheSection, m.typeInstCache
storeSeqSection procInstCacheSection, m.procInstCache
@ -700,17 +747,17 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
of nkEmpty, nkNilLit, nkType:
discard
of nkIdent:
result.ident = getIdent(c.cache, g[thisModule].fromDisk.sh.strings[n.litId])
result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId])
of nkSym:
result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand))
of directIntLit:
result.intVal = tree.nodes[n.int].operand
of externIntLit:
result.intVal = g[thisModule].fromDisk.sh.numbers[n.litId]
result.intVal = g[thisModule].fromDisk.numbers[n.litId]
of nkStrLit..nkTripleStrLit:
result.strVal = g[thisModule].fromDisk.sh.strings[n.litId]
result.strVal = g[thisModule].fromDisk.strings[n.litId]
of nkFloatLit..nkFloat128Lit:
result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.sh.numbers[n.litId])
result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.numbers[n.litId])
of nkModuleRef:
let (n1, n2) = sons2(tree, n)
assert n1.kind == nkInt32Lit
@ -767,7 +814,7 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
options: s.options,
position: if s.kind in {skForVar, skVar, skLet, skTemp}: 0 else: s.position,
offset: if s.kind in routineKinds: defaultOffset else: s.offset,
name: getIdent(c.cache, g[si].fromDisk.sh.strings[s.name])
name: getIdent(c.cache, g[si].fromDisk.strings[s.name])
)
template loadAstBody(p, field) =
@ -785,7 +832,7 @@ proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph;
result = nil
else:
result = PLib(generated: l.generated, isOverriden: l.isOverriden,
kind: l.kind, name: rope g[si].fromDisk.sh.strings[l.name])
kind: l.kind, name: rope g[si].fromDisk.strings[l.name])
loadAstBody(l, path)
proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
@ -798,14 +845,14 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
loadAstBody(s, ast)
result.annex = loadLib(c, g, si, item, s.annex)
when hasFFI:
result.cname = g[si].fromDisk.sh.strings[s.cname]
result.cname = g[si].fromDisk.strings[s.cname]
if s.kind in {skLet, skVar, skField, skForVar}:
result.guard = loadSym(c, g, si, s.guard)
result.bitsize = s.bitsize
result.alignment = s.alignment
result.owner = loadSym(c, g, si, s.owner)
let externalName = g[si].fromDisk.sh.strings[s.externalName]
let externalName = g[si].fromDisk.strings[s.externalName]
if externalName != "":
result.loc.r = rope externalName
result.loc.flags = s.locFlags
@ -818,14 +865,14 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s:
assert g[si].status in {loaded, storing, stored}
if not g[si].symsInit:
g[si].symsInit = true
setLen g[si].syms, g[si].fromDisk.sh.syms.len
setLen g[si].syms, g[si].fromDisk.syms.len
if g[si].syms[s.item] == nil:
if g[si].fromDisk.sh.syms[s.item].kind != skModule:
result = symHeaderFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item)
if g[si].fromDisk.syms[s.item].kind != skModule:
result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item)
# store it here early on, so that recursions work properly:
g[si].syms[s.item] = result
symBodyFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item, result)
symBodyFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item, result)
else:
result = g[si].module
assert result != nil
@ -866,13 +913,13 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
if not g[si].typesInit:
g[si].typesInit = true
setLen g[si].types, g[si].fromDisk.sh.types.len
setLen g[si].types, g[si].fromDisk.types.len
if g[si].types[t.item] == nil:
result = typeHeaderFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item)
result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item)
# store it here early on, so that recursions work properly:
g[si].types[t.item] = result
typeBodyFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item, result)
typeBodyFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item, result)
else:
result = g[si].types[t.item]
assert result.itemId.item > 0
@ -897,7 +944,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
let e2 =
when e[1] is PackedItemId: e[1]
else: PackedItemId(module: LitId(0), item: e[1])
iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(e2)
iface.mgetOrPut(cache.getIdent(m.fromDisk.strings[nameLit]), @[]).add(e2)
for e in m.fromDisk.exports:
m.iface.impl(e)
@ -1000,7 +1047,7 @@ proc loadProcBody*(config: ConfigRef, cache: IdentCache;
lastFile: FileIndex(-1),
config: config,
cache: cache)
let pos = g[mId].fromDisk.sh.syms[s.itemId.item].ast
let pos = g[mId].fromDisk.syms[s.itemId.item].ast
assert pos != emptyNodeId
result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos)
@ -1042,15 +1089,15 @@ proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; confi
proc checkForHoles(m: PackedModule; config: ConfigRef; moduleId: int) =
var bugs = 0
for i in 1 .. high(m.sh.syms):
if m.sh.syms[i].kind == skUnknown:
for i in 1 .. high(m.syms):
if m.syms[i].kind == skUnknown:
echo "EMPTY ID ", i, " module ", moduleId, " ", toFullPath(config, FileIndex(moduleId))
inc bugs
assert bugs == 0
when false:
var nones = 0
for i in 1 .. high(m.sh.types):
inc nones, m.sh.types[i].kind == tyNone
for i in 1 .. high(m.types):
inc nones, m.types[i].kind == tyNone
assert nones < 1
proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
@ -1133,13 +1180,13 @@ proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
result = loadSym(decoder, g, int(module), values[0])
proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len,
typeId: int32 m.fromDisk.sh.types.len)
IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.syms.len,
typeId: int32 m.fromDisk.types.len)
proc searchForCompilerproc*(m: LoadedModule; name: string): int32 =
# slow, linear search, but the results are cached:
for it in items(m.fromDisk.compilerProcs):
if m.fromDisk.sh.strings[it[0]] == name:
if m.fromDisk.strings[it[0]] == name:
return it[1]
return -1
@ -1149,31 +1196,31 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
var m: PackedModule
let err = loadRodFile(rodfile, m, config, ignoreConfig=true)
if err != ok:
config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err
config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err
when true:
echo "exports:"
for ex in m.exports:
echo " ", m.sh.strings[ex[0]], " local ID: ", ex[1]
assert ex[0] == m.sh.syms[ex[1]].name
echo " ", m.strings[ex[0]], " local ID: ", ex[1]
assert ex[0] == m.syms[ex[1]].name
# ex[1] int32
echo "reexports:"
for ex in m.reexports:
echo " ", m.sh.strings[ex[0]]
echo " ", m.strings[ex[0]]
# reexports*: seq[(LitId, PackedItemId)]
echo "hidden: " & $m.hidden.len
for ex in m.hidden:
echo " ", m.sh.strings[ex[0]], " local ID: ", ex[1]
echo " ", m.strings[ex[0]], " local ID: ", ex[1]
echo "all symbols"
for i in 0..high(m.sh.syms):
if m.sh.syms[i].name != LitId(0):
echo " ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i, " kind ", m.sh.syms[i].kind
for i in 0..high(m.syms):
if m.syms[i].name != LitId(0):
echo " ", m.strings[m.syms[i].name], " local ID: ", i, " kind ", m.syms[i].kind
else:
echo " <anon symbol?> local ID: ", i, " kind ", m.sh.syms[i].kind
echo " <anon symbol?> local ID: ", i, " kind ", m.syms[i].kind
echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len,
echo "symbols: ", m.syms.len, " types: ", m.types.len,
" top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len,
" strings: ", m.sh.strings.len, " numbers: ", m.sh.numbers.len
" strings: ", m.strings.len, " numbers: ", m.numbers.len