Incremental compilation (IC): Improvements (#11881)
* IC: C codegen is aware of IC * manual: minor change to make VSCode's RST plugin render it properly * IC: minor refactoring * testament: code refactorings * rodutils: removed dead code * IC: always build the compiler with the IC feature * IC: C codegen improvements * IC: implement the undocumented -d:nimMustCache option for testing purposes * IC: added first basic tests * IC: extensive testing of the deserialization feature * testament: refactoring; better IC tests * IC: removes 'nimMustCache' flag; readonly does the same * testament: minor refactoring * update Nimble version * testament: removed dead code and imports; IC: added simple test * IC: progress
This commit is contained in:
Andreas Rumpf
GitHub
parent
c0d240b8cd
commit
c8cffaf420
16 changed files with 318 additions and 261 deletions
|
|
@ -1108,7 +1108,8 @@ proc genProcNoForward(m: BModule, prc: PSym) =
|
|||
# a check for ``m.declaredThings``.
|
||||
if not containsOrIncl(m.declaredThings, prc.id):
|
||||
#if prc.loc.k == locNone:
|
||||
# mangle the inline proc based on the module where it is defined - not on the first module that uses it
|
||||
# mangle the inline proc based on the module where it is defined -
|
||||
# not on the first module that uses it
|
||||
fillProcLoc(findPendingModule(m, prc), prc.ast[namePos])
|
||||
#elif {sfExportc, sfImportc} * prc.flags == {}:
|
||||
# # reset name to restore consistency in case of hashing collisions:
|
||||
|
|
@ -1781,10 +1782,6 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule
|
|||
else: AbsoluteFile""
|
||||
open(result.ndi, ndiName, g.config)
|
||||
|
||||
proc nullify[T](arr: var T) =
|
||||
for i in low(arr)..high(arr):
|
||||
arr[i] = Rope(nil)
|
||||
|
||||
proc rawNewModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
|
||||
result = rawNewModule(g, module, AbsoluteFile toFullPath(conf, module.position.FileIndex))
|
||||
|
||||
|
|
@ -1875,7 +1872,9 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
|
|||
result = n
|
||||
if b == nil: return
|
||||
var m = BModule(b)
|
||||
if passes.skipCodegen(m.config, n): return
|
||||
if passes.skipCodegen(m.config, n) or
|
||||
not moduleHasChanged(m.g.graph, m.module):
|
||||
return
|
||||
m.initProc.options = initProcOptions(m)
|
||||
#softRnl = if optLineDir in m.config.options: noRnl else: rnl
|
||||
# XXX replicate this logic!
|
||||
|
|
@ -1886,9 +1885,10 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
|
|||
genStmts(m.initProc, transformedN)
|
||||
|
||||
proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
|
||||
result = true
|
||||
if optForceFullMake notin m.config.globalOptions:
|
||||
if not equalsFile(code, cfile.cname):
|
||||
if not moduleHasChanged(m.g.graph, m.module):
|
||||
result = false
|
||||
elif not equalsFile(code, cfile.cname):
|
||||
if false:
|
||||
#m.config.symbolFiles == readOnlySf: #isDefined(m.config, "nimdiff"):
|
||||
if fileExists(cfile.cname):
|
||||
|
|
@ -1898,12 +1898,15 @@ proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
|
|||
echo "new file ", cfile.cname.string
|
||||
if not writeRope(code, cfile.cname):
|
||||
rawMessage(m.config, errCannotOpenFile, cfile.cname.string)
|
||||
return
|
||||
if fileExists(cfile.obj) and os.fileNewer(cfile.obj.string, cfile.cname.string):
|
||||
result = true
|
||||
elif fileExists(cfile.obj) and os.fileNewer(cfile.obj.string, cfile.cname.string):
|
||||
result = false
|
||||
else:
|
||||
result = true
|
||||
else:
|
||||
if not writeRope(code, cfile.cname):
|
||||
rawMessage(m.config, errCannotOpenFile, cfile.cname.string)
|
||||
result = true
|
||||
|
||||
# We need 2 different logics here: pending modules (including
|
||||
# 'nim__dat') may require file merging for the combination of dead code
|
||||
|
|
@ -1915,18 +1918,19 @@ proc writeModule(m: BModule, pending: bool) =
|
|||
let cfile = getCFile(m)
|
||||
|
||||
if true or optForceFullMake in m.config.globalOptions:
|
||||
genInitCode(m)
|
||||
finishTypeDescriptions(m)
|
||||
if sfMainModule in m.module.flags:
|
||||
# generate main file:
|
||||
genMainProc(m)
|
||||
add(m.s[cfsProcHeaders], m.g.mainModProcs)
|
||||
generateThreadVarsSize(m)
|
||||
if moduleHasChanged(m.g.graph, m.module):
|
||||
genInitCode(m)
|
||||
finishTypeDescriptions(m)
|
||||
if sfMainModule in m.module.flags:
|
||||
# generate main file:
|
||||
genMainProc(m)
|
||||
add(m.s[cfsProcHeaders], m.g.mainModProcs)
|
||||
generateThreadVarsSize(m)
|
||||
|
||||
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
|
||||
obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
|
||||
var code = genModule(m, cf)
|
||||
if code != nil:
|
||||
if code != nil or m.config.symbolFiles != disabledSf:
|
||||
when hasTinyCBackend:
|
||||
if conf.cmd == cmdRun:
|
||||
tccgen.compileCCode($code)
|
||||
|
|
@ -1983,46 +1987,47 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
|
|||
for destructorCall in graph.globalDestructors:
|
||||
n.add destructorCall
|
||||
if passes.skipCodegen(m.config, n): return
|
||||
# if the module is cached, we don't regenerate the main proc
|
||||
# nor the dispatchers? But if the dispatchers changed?
|
||||
# XXX emit the dispatchers into its own .c file?
|
||||
if n != nil:
|
||||
m.initProc.options = initProcOptions(m)
|
||||
genStmts(m.initProc, n)
|
||||
if moduleHasChanged(graph, m.module):
|
||||
# if the module is cached, we don't regenerate the main proc
|
||||
# nor the dispatchers? But if the dispatchers changed?
|
||||
# XXX emit the dispatchers into its own .c file?
|
||||
if n != nil:
|
||||
m.initProc.options = initProcOptions(m)
|
||||
genStmts(m.initProc, n)
|
||||
|
||||
if m.hcrOn:
|
||||
# make sure this is pulled in (meaning hcrGetGlobal() is called for it during init)
|
||||
discard cgsym(m, "programResult")
|
||||
if m.inHcrInitGuard:
|
||||
endBlock(m.initProc)
|
||||
|
||||
if sfMainModule in m.module.flags:
|
||||
if m.hcrOn:
|
||||
# pull ("define" since they are inline when HCR is on) these functions in the main file
|
||||
# so it can load the HCR runtime and later pass the library handle to the HCR runtime which
|
||||
# will in turn pass it to the other modules it initializes so they can initialize the
|
||||
# register/get procs so they don't have to have the definitions of these functions as well
|
||||
discard cgsym(m, "nimLoadLibrary")
|
||||
discard cgsym(m, "nimLoadLibraryError")
|
||||
discard cgsym(m, "nimGetProcAddr")
|
||||
discard cgsym(m, "procAddrError")
|
||||
discard cgsym(m, "rawWrite")
|
||||
# make sure this is pulled in (meaning hcrGetGlobal() is called for it during init)
|
||||
discard cgsym(m, "programResult")
|
||||
if m.inHcrInitGuard:
|
||||
endBlock(m.initProc)
|
||||
|
||||
# raise dependencies on behalf of genMainProc
|
||||
if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
|
||||
discard cgsym(m, "initStackBottomWith")
|
||||
if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
|
||||
discard cgsym(m, "initThreadVarsEmulation")
|
||||
if sfMainModule in m.module.flags:
|
||||
if m.hcrOn:
|
||||
# pull ("define" since they are inline when HCR is on) these functions in the main file
|
||||
# so it can load the HCR runtime and later pass the library handle to the HCR runtime which
|
||||
# will in turn pass it to the other modules it initializes so they can initialize the
|
||||
# register/get procs so they don't have to have the definitions of these functions as well
|
||||
discard cgsym(m, "nimLoadLibrary")
|
||||
discard cgsym(m, "nimLoadLibraryError")
|
||||
discard cgsym(m, "nimGetProcAddr")
|
||||
discard cgsym(m, "procAddrError")
|
||||
discard cgsym(m, "rawWrite")
|
||||
|
||||
if m.g.breakpoints != nil:
|
||||
discard cgsym(m, "dbgRegisterBreakpoint")
|
||||
if optEndb in m.config.options:
|
||||
discard cgsym(m, "dbgRegisterFilename")
|
||||
# raise dependencies on behalf of genMainProc
|
||||
if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
|
||||
discard cgsym(m, "initStackBottomWith")
|
||||
if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
|
||||
discard cgsym(m, "initThreadVarsEmulation")
|
||||
|
||||
if m.g.forwardedProcs.len == 0:
|
||||
incl m.flags, objHasKidsValid
|
||||
let disp = generateMethodDispatchers(graph)
|
||||
for x in disp: genProcAux(m, x.sym)
|
||||
if m.g.breakpoints != nil:
|
||||
discard cgsym(m, "dbgRegisterBreakpoint")
|
||||
if optEndb in m.config.options:
|
||||
discard cgsym(m, "dbgRegisterFilename")
|
||||
|
||||
if m.g.forwardedProcs.len == 0:
|
||||
incl m.flags, objHasKidsValid
|
||||
let disp = generateMethodDispatchers(graph)
|
||||
for x in disp: genProcAux(m, x.sym)
|
||||
|
||||
m.g.modulesClosed.add m
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ import
|
|||
wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
|
||||
pathutils, strtabs
|
||||
|
||||
from incremental import nimIncremental
|
||||
|
||||
# but some have deps to imported modules. Yay.
|
||||
bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
|
||||
bootSwitch(usedNativeStacktrace,
|
||||
|
|
@ -669,7 +671,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
|
|||
helpOnError(conf, pass)
|
||||
of "symbolfiles": discard "ignore for backwards compat"
|
||||
of "incremental":
|
||||
when not defined(nimIncremental):
|
||||
when not nimIncremental:
|
||||
localError(conf, info, "the compiler was not built with " &
|
||||
"incremental compilation features; bootstrap with " &
|
||||
"-d:nimIncremental to enable")
|
||||
|
|
|
|||
|
|
@ -10,13 +10,14 @@
|
|||
## Basic type definitions the module graph needs in order to support
|
||||
## incremental compilations.
|
||||
|
||||
const nimIncremental* = defined(nimIncremental)
|
||||
const nimIncremental* = true # defined(nimIncremental)
|
||||
|
||||
import options, lineinfos
|
||||
|
||||
when nimIncremental:
|
||||
import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils
|
||||
from strutils import parseInt
|
||||
from os import isAbsolute
|
||||
|
||||
type
|
||||
Writer* = object
|
||||
|
|
@ -47,7 +48,7 @@ when nimIncremental:
|
|||
|
||||
proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string =
|
||||
result = msgs.getHash(conf, fileIdx)
|
||||
if result.len == 0:
|
||||
if result.len == 0 and isAbsolute(string fullpath):
|
||||
result = $secureHashFile(string fullpath)
|
||||
msgs.setHash(conf, fileIdx, result)
|
||||
|
||||
|
|
|
|||
|
|
@ -77,8 +77,6 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
|
|||
if result == nil:
|
||||
result = newModule(graph, fileIdx)
|
||||
result.flags = result.flags + flags
|
||||
if sfMainModule in result.flags:
|
||||
graph.config.mainPackageId = result.owner.id
|
||||
result.id = id
|
||||
registerModule(graph, result)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -113,6 +113,8 @@ const
|
|||
nkExportStmt, nkExportExceptStmt, nkFromStmt, nkImportStmt, nkImportExceptStmt}
|
||||
|
||||
proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
|
||||
if sfMainModule in module.flags:
|
||||
graph.config.mainPackageId = module.owner.id
|
||||
# don't be verbose unless the module belongs to the main package:
|
||||
if module.owner.id == graph.config.mainPackageId:
|
||||
graph.config.notes = graph.config.mainPackageNotes
|
||||
|
|
@ -121,7 +123,7 @@ proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
|
|||
graph.config.notes = graph.config.foreignPackageNotes
|
||||
|
||||
proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
|
||||
result = module.id >= 0
|
||||
result = module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")
|
||||
|
||||
proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {.discardable.} =
|
||||
if graph.stopCompile(): return true
|
||||
|
|
@ -131,7 +133,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
|
|||
s: PLLStream
|
||||
fileIdx = module.fileIdx
|
||||
prepareConfigNotes(graph, module)
|
||||
if not moduleHasChanged(graph, module):
|
||||
if module.id < 0:
|
||||
# new module caching mechanism:
|
||||
for i in 0 ..< graph.passes.len:
|
||||
if not isNil(graph.passes[i].open) and not graph.passes[i].isFrontend:
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
|
|||
## - Dependency computation should use *signature* hashes in order to
|
||||
## avoid recompiling dependent modules.
|
||||
## - Patch the rest of the compiler to do lazy loading of proc bodies.
|
||||
## - Patch the C codegen to cache proc bodies and maybe types.
|
||||
|
||||
template db(): DbConn = g.incr.db
|
||||
|
||||
|
|
@ -72,9 +71,12 @@ proc getModuleId(g: ModuleGraph; fileIdx: FileIndex; fullpath: AbsoluteFile): in
|
|||
# not changed, so use the cached AST:
|
||||
doAssert(result != 0)
|
||||
var cycleCheck = initIntSet()
|
||||
if not needsRecompile(g, fileIdx, fullpath, cycleCheck) and not g.incr.configChanged:
|
||||
echo "cached successfully! ", string fullpath
|
||||
return -result
|
||||
if not needsRecompile(g, fileIdx, fullpath, cycleCheck):
|
||||
if not g.incr.configChanged or g.config.symbolFiles == readOnlySf:
|
||||
#echo "cached successfully! ", string fullpath
|
||||
return -result
|
||||
elif g.config.symbolFiles == readOnlySf:
|
||||
internalError(g.config, "file needs to be recompiled: " & (string fullpath))
|
||||
db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0])
|
||||
db.exec(sql"delete from deps where module = ?", module[0])
|
||||
db.exec(sql"delete from types where module = ?", module[0])
|
||||
|
|
@ -296,7 +298,7 @@ proc encodeSym(g: ModuleGraph, s: PSym, result: var string) =
|
|||
pushSym(w, s.owner)
|
||||
if s.flags != {}:
|
||||
result.add('$')
|
||||
encodeVInt(cast[int32](s.flags), result)
|
||||
encodeVBiggestInt(cast[int64](s.flags), result)
|
||||
if s.magic != mNone:
|
||||
result.add('@')
|
||||
encodeVInt(ord(s.magic), result)
|
||||
|
|
@ -723,7 +725,7 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
|
|||
result.owner = loadSym(g, decodeVInt(b.s, b.pos), result.info)
|
||||
if b.s[b.pos] == '$':
|
||||
inc(b.pos)
|
||||
result.flags = cast[TSymFlags](int32(decodeVInt(b.s, b.pos)))
|
||||
result.flags = cast[TSymFlags](decodeVBiggestInt(b.s, b.pos))
|
||||
if b.s[b.pos] == '@':
|
||||
inc(b.pos)
|
||||
result.magic = TMagic(decodeVInt(b.s, b.pos))
|
||||
|
|
@ -756,6 +758,7 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
|
|||
if b.s[b.pos] == '\24':
|
||||
inc b.pos
|
||||
result.transformedBody = decodeNode(g, b, result.info)
|
||||
#result.transformedBody = nil
|
||||
of skModule, skPackage:
|
||||
decodeInstantiations(g, b, result.info, result.usedGenerics)
|
||||
of skLet, skVar, skField, skForVar:
|
||||
|
|
@ -769,7 +772,7 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
|
|||
|
||||
if b.s[b.pos] == '(':
|
||||
#if result.kind in routineKinds:
|
||||
# result.ast = decodeNodeLazyBody(b, result.info, result)
|
||||
# result.ast = nil
|
||||
#else:
|
||||
result.ast = decodeNode(g, b, result.info)
|
||||
if sfCompilerProc in result.flags:
|
||||
|
|
@ -886,6 +889,10 @@ proc replay(g: ModuleGraph; module: PSym; n: PNode) =
|
|||
internalAssert g.config, imported.id < 0
|
||||
of nkStmtList, nkStmtListExpr:
|
||||
for x in n: replay(g, module, x)
|
||||
of nkExportStmt:
|
||||
for x in n:
|
||||
doAssert x.kind == nkSym
|
||||
strTableAdd(module.tab, x.sym)
|
||||
else: discard "nothing to do for this node"
|
||||
|
||||
proc loadNode*(g: ModuleGraph; module: PSym): PNode =
|
||||
|
|
|
|||
|
|
@ -46,14 +46,9 @@ proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
|
|||
of fcNegInf:
|
||||
result = "-INF"
|
||||
else:
|
||||
when defined(nimNoArrayToCstringConversion):
|
||||
result = newString(81)
|
||||
let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
|
||||
setLen(result, n)
|
||||
else:
|
||||
var buf: array[0..80, char]
|
||||
discard c_snprintf(buf.cstring, buf.len.uint, "%#.16e%s", f, literalPostfix.cstring)
|
||||
result = $buf.cstring
|
||||
result = newString(81)
|
||||
let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
|
||||
setLen(result, n)
|
||||
|
||||
proc encodeStr*(s: string, result: var string) =
|
||||
for i in 0 ..< len(s):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue