IC: next steps (#16729)
* IC: dead code elimination pass * preparations for a different codegen strategy * added documentation to the newly written code * IC: backend code * IC: backend adjustments * optimized the compiler a bit * IC: yet another massive refactoring * fixes regressions * cleanups
This commit is contained in:
Andreas Rumpf
GitHub
parent
eae3bdf8fe
commit
8241e55023
32 changed files with 729 additions and 323 deletions
102
compiler/ic/cbackend.nim
Normal file
102
compiler/ic/cbackend.nim
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
#
|
||||
#
|
||||
# The Nim Compiler
|
||||
# (c) Copyright 2021 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
## New entry point into our C/C++ code generator. Ideally
|
||||
## somebody would rewrite the old backend (which is 8000 lines of crufty Nim code)
|
||||
## to work on packed trees directly and produce the C code as an AST which can
|
||||
## then be rendered to text in a very simple manner. Unfortunately nobody wrote
|
||||
## this code. So instead we wrap the existing cgen.nim and its friends so that
|
||||
## we call directly into the existing code generation logic but avoiding the
|
||||
## naive, outdated `passes` design. Thus you will see some
|
||||
## `useAliveDataFromDce in flags` checks in the old code -- the old code is
|
||||
## also doing cross-module dependency tracking and DCE that we don't need
|
||||
## anymore. DCE is now done as prepass over the entire packed module graph.
|
||||
|
||||
import std / [intsets, algorithm]
|
||||
import ".." / [ast, options, lineinfos, modulegraphs, cgendata, cgen,
|
||||
pathutils, extccomp, msgs]
|
||||
|
||||
import packed_ast, to_packed_ast, bitabs, dce, rodfiles
|
||||
|
||||
proc unpackTree(g: ModuleGraph; thisModule: int;
|
||||
tree: PackedTree; n: NodePos): PNode =
|
||||
var decoder = initPackedDecoder(g.config, g.cache)
|
||||
result = loadNodes(decoder, g.packed, thisModule, tree, n)
|
||||
|
||||
proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) =
|
||||
if g.backend == nil:
|
||||
g.backend = cgendata.newModuleList(g)
|
||||
|
||||
var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config)
|
||||
bmod.idgen = idgenFromLoadedModule(m)
|
||||
bmod.flags.incl useAliveDataFromDce
|
||||
bmod.alive = move alive[m.module.position]
|
||||
|
||||
for p in allNodes(m.fromDisk.topLevel):
|
||||
let n = unpackTree(g, m.module.position, m.fromDisk.topLevel, p)
|
||||
cgen.genTopLevelStmt(bmod, n)
|
||||
|
||||
finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info))
|
||||
|
||||
proc addFileToLink(config: ConfigRef; m: PSym) =
|
||||
let filename = AbsoluteFile toFullPath(config, m.position.FileIndex)
|
||||
let ext =
|
||||
if config.backend == backendCpp: ".nim.cpp"
|
||||
elif config.backend == backendObjc: ".nim.m"
|
||||
else: ".nim.c"
|
||||
let cfile = changeFileExt(completeCfilePath(config, withPackageName(config, filename)), ext)
|
||||
var cf = Cfile(nimname: m.name.s, cname: cfile,
|
||||
obj: completeCfilePath(config, toObjFile(config, cfile)),
|
||||
flags: {CfileFlag.Cached})
|
||||
addFileToCompile(config, cf)
|
||||
|
||||
proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool =
|
||||
let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms")
|
||||
var s = newSeqOfCap[int32](alive[position].len)
|
||||
for a in items(alive[position]): s.add int32(a)
|
||||
sort(s)
|
||||
var f2 = rodfiles.open(asymFile.string)
|
||||
f2.loadHeader()
|
||||
f2.loadSection aliveSymsSection
|
||||
var oldData: seq[int32]
|
||||
f2.loadSeq(oldData)
|
||||
f2.close
|
||||
if f2.err == ok and oldData == s:
|
||||
result = false
|
||||
else:
|
||||
result = true
|
||||
var f = rodfiles.create(asymFile.string)
|
||||
f.storeHeader()
|
||||
f.storeSection aliveSymsSection
|
||||
f.storeSeq(s)
|
||||
close f
|
||||
|
||||
proc generateCode*(g: ModuleGraph) =
|
||||
## The single entry point, generate C(++) code for the entire
|
||||
## Nim program aka `ModuleGraph`.
|
||||
var alive = computeAliveSyms(g.packed, g.config)
|
||||
|
||||
for i in 0..high(g.packed):
|
||||
# case statement here to enforce exhaustive checks.
|
||||
case g.packed[i].status
|
||||
of undefined:
|
||||
discard "nothing to do"
|
||||
of loading:
|
||||
assert false
|
||||
of storing, outdated:
|
||||
generateCodeForModule(g, g.packed[i], alive)
|
||||
of loaded:
|
||||
# Even though this module didn't change, DCE might trigger a change.
|
||||
# Consider this case: Module A uses symbol S from B and B does not use
|
||||
# S itself. A is then edited not to use S either. Thus we have to
|
||||
# recompile B in order to remove S from the final result.
|
||||
if aliveSymsChanged(g.config, g.packed[i].module.position, alive):
|
||||
generateCodeForModule(g, g.packed[i], alive)
|
||||
else:
|
||||
addFileToLink(g.config, g.packed[i].module)
|
||||
108
compiler/ic/dce.nim
Normal file
108
compiler/ic/dce.nim
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
#
|
||||
#
|
||||
# The Nim Compiler
|
||||
# (c) Copyright 2021 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
## Dead code elimination (=DCE) for IC.
|
||||
|
||||
import std / intsets
|
||||
import ".." / [ast, options, lineinfos]
|
||||
|
||||
import packed_ast, to_packed_ast, bitabs
|
||||
|
||||
type
|
||||
AliveSyms* = seq[IntSet]
|
||||
AliveContext* = object ## Purpose is to fill the 'alive' field.
|
||||
stack: seq[(int, NodePos)] ## A stack for marking symbols as alive.
|
||||
decoder: PackedDecoder ## We need a PackedDecoder for module ID address translations.
|
||||
thisModule: int ## The module we're currently analysing for DCE.
|
||||
alive: AliveSyms ## The final result of our computation.
|
||||
|
||||
proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): bool =
|
||||
## "Exported to C" procs are special (these are marked with '.exportc') because these
|
||||
## must not be optimized away!
|
||||
let symPtr = addr g[c.thisModule].fromDisk.sh.syms[symId]
|
||||
let flags = symPtr.flags
|
||||
# due to a bug/limitation in the lambda lifting, unused inner procs
|
||||
# are not transformed correctly; issue (#411). However, the whole purpose here
|
||||
# is to eliminate unused procs. So there is no special logic required for this case.
|
||||
if sfCompileTime notin flags:
|
||||
if ({sfExportc, sfCompilerProc} * flags == {sfExportc}) or
|
||||
(symPtr.kind == skMethod):
|
||||
result = true
|
||||
# XXX: This used to be a condition to:
|
||||
# (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or
|
||||
|
||||
template isNotGeneric(n: NodePos): bool = ithSon(tree, n, genericParamsPos).kind == nkEmpty
|
||||
|
||||
proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: int32) =
|
||||
## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will
|
||||
## be analysed.
|
||||
if not c.alive[module].containsOrIncl(item):
|
||||
let body = g[module].fromDisk.sh.syms[item].ast
|
||||
if body != emptyNodeId:
|
||||
c.stack.add((module, NodePos(body)))
|
||||
|
||||
proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) =
|
||||
## Marks the symbols we encounter when we traverse the AST at `tree[n]` as alive, unless
|
||||
## it is purely in a declarative context (type section etc.).
|
||||
case n.kind
|
||||
of nkNone..pred(nkSym), succ(nkSym)..nkNilLit:
|
||||
discard "ignore non-sym atoms"
|
||||
of nkSym:
|
||||
# This symbol is alive and everything its body references.
|
||||
followLater(c, g, c.thisModule, n.operand)
|
||||
of nkModuleRef:
|
||||
let (n1, n2) = sons2(tree, n)
|
||||
assert n1.kind == nkInt32Lit
|
||||
assert n2.kind == nkInt32Lit
|
||||
let m = n1.litId
|
||||
let item = n2.operand
|
||||
let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int
|
||||
followLater(c, g, otherModule, item)
|
||||
of nkMacroDef, nkTemplateDef, nkTypeSection, nkTypeOfExpr,
|
||||
nkCommentStmt, nkIteratorDef, nkIncludeStmt,
|
||||
nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
|
||||
nkFromStmt, nkStaticStmt:
|
||||
discard
|
||||
of nkVarSection, nkLetSection, nkConstSection:
|
||||
discard
|
||||
of nkProcDef, nkConverterDef, nkMethodDef, nkLambda, nkDo, nkFuncDef:
|
||||
if n.firstSon.kind == nkSym and isNotGeneric(n):
|
||||
if isExportedToC(c, g, n.firstSon.operand):
|
||||
let item = n.operand
|
||||
# This symbol is alive and everything its body references.
|
||||
followLater(c, g, c.thisModule, item)
|
||||
else:
|
||||
for son in sonsReadonly(tree, n):
|
||||
aliveCode(c, g, tree, son)
|
||||
|
||||
proc followNow(c: var AliveContext; g: PackedModuleGraph) =
|
||||
## Mark all entries in the stack. Marking can add more entries
|
||||
## to the stack but eventually we have looked at every alive symbol.
|
||||
while c.stack.len > 0:
|
||||
let (modId, ast) = c.stack.pop()
|
||||
c.thisModule = modId
|
||||
aliveCode(c, g, g[modId].fromDisk.bodies, ast)
|
||||
|
||||
proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms =
|
||||
## Entry point for our DCE algorithm.
|
||||
var c = AliveContext(stack: @[], decoder: PackedDecoder(config: conf),
|
||||
thisModule: -1, alive: newSeq[IntSet](g.len))
|
||||
for i in countdown(high(g), 0):
|
||||
if g[i].status != undefined:
|
||||
c.thisModule = i
|
||||
for p in allNodes(g[i].fromDisk.topLevel):
|
||||
aliveCode(c, g, g[i].fromDisk.topLevel, p)
|
||||
followNow(c, g)
|
||||
result = move(c.alive)
|
||||
|
||||
proc isAlive*(a: AliveSyms; module: int, item: int32): bool =
|
||||
## Backends use this to query if a symbol is `alive` which means
|
||||
## we need to produce (C/C++/etc) code for it.
|
||||
result = a[module].contains(item)
|
||||
|
||||
|
|
@ -290,6 +290,9 @@ template typ*(n: NodePos): PackedItemId =
|
|||
template flags*(n: NodePos): TNodeFlags =
|
||||
tree.nodes[n.int].flags
|
||||
|
||||
template operand*(n: NodePos): int32 =
|
||||
tree.nodes[n.int].operand
|
||||
|
||||
proc span*(tree: PackedTree; pos: int): int {.inline.} =
|
||||
if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
|
||||
|
||||
|
|
@ -451,3 +454,10 @@ when false:
|
|||
dest.add nkStrLit, msg, n.info
|
||||
copyTree(dest, tree, n)
|
||||
patch dest, patchPos
|
||||
|
||||
iterator allNodes*(tree: PackedTree): NodePos =
|
||||
var p = 0
|
||||
while p < tree.len:
|
||||
yield NodePos(p)
|
||||
let s = span(tree, p)
|
||||
inc p, s
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ type
|
|||
bodiesSection
|
||||
symsSection
|
||||
typesSection
|
||||
aliveSymsSection # beware, this is stored in a `.alivesyms` file.
|
||||
|
||||
RodFileError* = enum
|
||||
ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch,
|
||||
|
|
@ -134,7 +135,7 @@ proc loadHeader*(f: var RodFile) =
|
|||
|
||||
proc storeSection*(f: var RodFile; s: RodSection) =
|
||||
if f.err != ok: return
|
||||
assert f.currentSection == pred s
|
||||
assert f.currentSection < s
|
||||
f.currentSection = s
|
||||
storePrim(f, s)
|
||||
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId =
|
|||
c.lastLit = result
|
||||
assert result != LitId(0)
|
||||
|
||||
proc toFileIndex(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
|
||||
proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
|
||||
result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x])
|
||||
|
||||
proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
|
||||
|
|
@ -280,16 +280,19 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
|
|||
paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel)
|
||||
storeNode(p, t, n)
|
||||
|
||||
for op, s in pairs t.attachedOps:
|
||||
c.addMissing s
|
||||
p.attachedOps[op] = s.safeItemId(c, m)
|
||||
when false:
|
||||
for op, s in pairs t.attachedOps:
|
||||
c.addMissing s
|
||||
p.attachedOps[op] = s.safeItemId(c, m)
|
||||
|
||||
p.typeInst = t.typeInst.storeType(c, m)
|
||||
for kid in items t.sons:
|
||||
p.types.add kid.storeType(c, m)
|
||||
for i, s in items t.methods:
|
||||
c.addMissing s
|
||||
p.methods.add (i, s.safeItemId(c, m))
|
||||
|
||||
when false:
|
||||
for i, s in items t.methods:
|
||||
c.addMissing s
|
||||
p.methods.add (i, s.safeItemId(c, m))
|
||||
c.addMissing t.sym
|
||||
p.sym = t.sym.safeItemId(c, m)
|
||||
c.addMissing t.owner
|
||||
|
|
@ -568,11 +571,11 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
|
|||
|
||||
type
|
||||
PackedDecoder* = object
|
||||
lastModule*: int
|
||||
lastLit*: LitId
|
||||
lastFile*: FileIndex # remember the last lookup entry.
|
||||
lastModule: int
|
||||
lastLit: LitId
|
||||
lastFile: FileIndex # remember the last lookup entry.
|
||||
config*: ConfigRef
|
||||
cache: IdentCache
|
||||
cache*: IdentCache
|
||||
|
||||
type
|
||||
ModuleStatus* = enum
|
||||
|
|
@ -596,7 +599,7 @@ type
|
|||
proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType
|
||||
proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: PackedItemId): PSym
|
||||
|
||||
proc toFileIndexCached(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; f: LitId): FileIndex =
|
||||
proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule: int; f: LitId): FileIndex =
|
||||
if c.lastLit == f and c.lastModule == thisModule:
|
||||
result = c.lastFile
|
||||
else:
|
||||
|
|
@ -611,8 +614,8 @@ proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; thisModul
|
|||
result = TLineInfo(line: x.line, col: x.col,
|
||||
fileIndex: toFileIndexCached(c, g, thisModule, x.file))
|
||||
|
||||
proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
|
||||
tree: PackedTree; n: NodePos): PNode =
|
||||
proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
|
||||
tree: PackedTree; n: NodePos): PNode =
|
||||
let k = n.kind
|
||||
if k == nkNilRodNode:
|
||||
return nil
|
||||
|
|
@ -647,6 +650,14 @@ proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
|
|||
for n0 in sonsReadonly(tree, n):
|
||||
result.addAllowNil loadNodes(c, g, thisModule, tree, n0)
|
||||
|
||||
proc initPackedDecoder*(config: ConfigRef; cache: IdentCache): PackedDecoder =
|
||||
result = PackedDecoder(
|
||||
lastModule: int32(-1),
|
||||
lastLit: LitId(0),
|
||||
lastFile: FileIndex(-1),
|
||||
config: config,
|
||||
cache: cache)
|
||||
|
||||
proc loadProcHeader(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
|
||||
tree: PackedTree; n: NodePos): PNode =
|
||||
# do not load the body of the proc. This will be done later in
|
||||
|
|
@ -761,14 +772,16 @@ proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
|
|||
t: PackedType; si, item: int32; result: PType) =
|
||||
result.sym = loadSym(c, g, si, t.sym)
|
||||
result.owner = loadSym(c, g, si, t.owner)
|
||||
for op, item in pairs t.attachedOps:
|
||||
result.attachedOps[op] = loadSym(c, g, si, item)
|
||||
when false:
|
||||
for op, item in pairs t.attachedOps:
|
||||
result.attachedOps[op] = loadSym(c, g, si, item)
|
||||
result.typeInst = loadType(c, g, si, t.typeInst)
|
||||
for son in items t.types:
|
||||
result.sons.add loadType(c, g, si, son)
|
||||
loadAstBody(t, n)
|
||||
for gen, id in items t.methods:
|
||||
result.methods.add((gen, loadSym(c, g, si, id)))
|
||||
when false:
|
||||
for gen, id in items t.methods:
|
||||
result.methods.add((gen, loadSym(c, g, si, id)))
|
||||
|
||||
proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType =
|
||||
if t == nilItemId:
|
||||
|
|
@ -819,11 +832,8 @@ proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
|
|||
lastFile: FileIndex(-1),
|
||||
config: conf,
|
||||
cache: cache)
|
||||
var p = 0
|
||||
while p < m.fromDisk.toReplay.len:
|
||||
m.module.ast.add loadNodes(decoder, g, int(fileIdx), m.fromDisk.toReplay, NodePos p)
|
||||
let s = span(m.fromDisk.toReplay, p)
|
||||
inc p, s
|
||||
for p in allNodes(m.fromDisk.toReplay):
|
||||
m.module.ast.add loadNodes(decoder, g, int(fileIdx), m.fromDisk.toReplay, p)
|
||||
|
||||
proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
|
||||
fileIdx: FileIndex): bool =
|
||||
|
|
@ -979,6 +989,10 @@ proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
|
|||
let values = g[int module].iface.getOrDefault(name)
|
||||
result = loadSym(decoder, g, int(module), values[0])
|
||||
|
||||
proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
|
||||
IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len,
|
||||
typeId: int32 m.fromDisk.sh.types.len)
|
||||
|
||||
# ------------------------- .rod file viewer ---------------------------------
|
||||
|
||||
proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue