new dependency tracking for nimsuggest
This commit is contained in:
Araq
parent
33ebf3e5fb
commit
9e6fb3f696
21 changed files with 284 additions and 393 deletions
|
|
@ -7,130 +7,118 @@
|
|||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
## implements the module handling
|
||||
## Implements the module handling, including the caching of modules.
|
||||
|
||||
import
|
||||
ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options,
|
||||
idents, os, lexer, idgen, passes, syntaxes, llstream
|
||||
idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs
|
||||
|
||||
type
|
||||
TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled
|
||||
THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged
|
||||
when false:
|
||||
type
|
||||
TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled
|
||||
THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged
|
||||
|
||||
TModuleInMemory* = object
|
||||
compiledAt*: float
|
||||
hash*: SecureHash
|
||||
deps*: seq[int32] ## XXX: slurped files are currently not tracked
|
||||
needsRecompile*: TNeedRecompile
|
||||
hashStatus*: THashStatus
|
||||
TModuleInMemory* = object
|
||||
hash*: SecureHash
|
||||
deps*: seq[int32] ## XXX: slurped files are currently not tracked
|
||||
|
||||
var
|
||||
gCompiledModules: seq[PSym] = @[]
|
||||
gMemCacheData*: seq[TModuleInMemory] = @[]
|
||||
## XXX: we should implement recycling of file IDs
|
||||
## if the user keeps renaming modules, the file IDs will keep growing
|
||||
gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why.
|
||||
packageSyms: TStrTable
|
||||
needsRecompile*: TNeedRecompile
|
||||
hashStatus*: THashStatus
|
||||
|
||||
initStrTable(packageSyms)
|
||||
var
|
||||
gCompiledModules: seq[PSym] = @[]
|
||||
gMemCacheData*: seq[TModuleInMemory] = @[]
|
||||
## XXX: we should implement recycling of file IDs
|
||||
## if the user keeps renaming modules, the file IDs will keep growing
|
||||
gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why.
|
||||
|
||||
proc getModule*(fileIdx: int32): PSym =
|
||||
if fileIdx >= 0 and fileIdx < gCompiledModules.len:
|
||||
result = gCompiledModules[fileIdx]
|
||||
proc hashChanged(fileIdx: int32): bool =
|
||||
internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
|
||||
|
||||
proc hashChanged(fileIdx: int32): bool =
|
||||
internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
|
||||
template updateStatus =
|
||||
gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged
|
||||
else: hashNotChanged
|
||||
# echo "TESTING Hash: ", fileIdx.toFilename, " ", result
|
||||
|
||||
template updateStatus =
|
||||
gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged
|
||||
else: hashNotChanged
|
||||
# echo "TESTING Hash: ", fileIdx.toFilename, " ", result
|
||||
case gMemCacheData[fileIdx].hashStatus
|
||||
of hashHasChanged:
|
||||
result = true
|
||||
of hashNotChanged:
|
||||
result = false
|
||||
of hashCached:
|
||||
let newHash = secureHashFile(fileIdx.toFullPath)
|
||||
result = newHash != gMemCacheData[fileIdx].hash
|
||||
gMemCacheData[fileIdx].hash = newHash
|
||||
updateStatus()
|
||||
of hashNotTaken:
|
||||
gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
|
||||
result = true
|
||||
updateStatus()
|
||||
|
||||
case gMemCacheData[fileIdx].hashStatus
|
||||
of hashHasChanged:
|
||||
result = true
|
||||
of hashNotChanged:
|
||||
result = false
|
||||
of hashCached:
|
||||
let newHash = secureHashFile(fileIdx.toFullPath)
|
||||
result = newHash != gMemCacheData[fileIdx].hash
|
||||
gMemCacheData[fileIdx].hash = newHash
|
||||
updateStatus()
|
||||
of hashNotTaken:
|
||||
gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
|
||||
result = true
|
||||
updateStatus()
|
||||
proc doHash(fileIdx: int32) =
|
||||
if gMemCacheData[fileIdx].hashStatus == hashNotTaken:
|
||||
# echo "FIRST Hash: ", fileIdx.ToFilename
|
||||
gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
|
||||
|
||||
proc doHash(fileIdx: int32) =
|
||||
if gMemCacheData[fileIdx].hashStatus == hashNotTaken:
|
||||
# echo "FIRST Hash: ", fileIdx.ToFilename
|
||||
gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
|
||||
proc resetModule*(fileIdx: int32) =
|
||||
# echo "HARD RESETTING ", fileIdx.toFilename
|
||||
if fileIdx <% gMemCacheData.len:
|
||||
gMemCacheData[fileIdx].needsRecompile = Yes
|
||||
if fileIdx <% gCompiledModules.len:
|
||||
gCompiledModules[fileIdx] = nil
|
||||
if fileIdx <% cgendata.gModules.len:
|
||||
cgendata.gModules[fileIdx] = nil
|
||||
|
||||
proc addDep(x: PSym, dep: int32) =
|
||||
growCache gMemCacheData, dep
|
||||
gMemCacheData[x.position].deps.safeAdd(dep)
|
||||
proc resetModule*(module: PSym) =
|
||||
let conflict = getModule(module.position.int32)
|
||||
if conflict == nil: return
|
||||
doAssert conflict == module
|
||||
resetModule(module.position.int32)
|
||||
initStrTable(module.tab)
|
||||
|
||||
proc resetModule*(fileIdx: int32) =
|
||||
# echo "HARD RESETTING ", fileIdx.toFilename
|
||||
if fileIdx <% gMemCacheData.len:
|
||||
gMemCacheData[fileIdx].needsRecompile = Yes
|
||||
if fileIdx <% gCompiledModules.len:
|
||||
gCompiledModules[fileIdx] = nil
|
||||
if fileIdx <% cgendata.gModules.len:
|
||||
cgendata.gModules[fileIdx] = nil
|
||||
proc resetAllModules* =
|
||||
for i in 0..gCompiledModules.high:
|
||||
if gCompiledModules[i] != nil:
|
||||
resetModule(i.int32)
|
||||
resetPackageCache()
|
||||
# for m in cgenModules(): echo "CGEN MODULE FOUND"
|
||||
|
||||
proc resetModule*(module: PSym) =
|
||||
let conflict = getModule(module.position.int32)
|
||||
if conflict == nil: return
|
||||
doAssert conflict == module
|
||||
resetModule(module.position.int32)
|
||||
initStrTable(module.tab)
|
||||
proc resetAllModulesHard* =
|
||||
resetPackageCache()
|
||||
gCompiledModules.setLen 0
|
||||
gMemCacheData.setLen 0
|
||||
magicsys.resetSysTypes()
|
||||
# XXX
|
||||
#gOwners = @[]
|
||||
|
||||
proc resetAllModules* =
|
||||
for i in 0..gCompiledModules.high:
|
||||
if gCompiledModules[i] != nil:
|
||||
resetModule(i.int32)
|
||||
resetPackageCache()
|
||||
initStrTable(packageSyms)
|
||||
# for m in cgenModules(): echo "CGEN MODULE FOUND"
|
||||
proc checkDepMem(fileIdx: int32): TNeedRecompile =
|
||||
template markDirty =
|
||||
resetModule(fileIdx)
|
||||
return Yes
|
||||
|
||||
proc resetAllModulesHard* =
|
||||
resetPackageCache()
|
||||
gCompiledModules.setLen 0
|
||||
gMemCacheData.setLen 0
|
||||
magicsys.resetSysTypes()
|
||||
initStrTable(packageSyms)
|
||||
# XXX
|
||||
#gOwners = @[]
|
||||
if gFuzzyGraphChecking:
|
||||
if gMemCacheData[fileIdx].needsRecompile != Maybe:
|
||||
return gMemCacheData[fileIdx].needsRecompile
|
||||
else:
|
||||
# cycle detection: We claim that a cycle does no harm.
|
||||
if gMemCacheData[fileIdx].needsRecompile == Probing:
|
||||
return No
|
||||
|
||||
proc checkDepMem(fileIdx: int32): TNeedRecompile =
|
||||
template markDirty =
|
||||
resetModule(fileIdx)
|
||||
return Yes
|
||||
if optForceFullMake in gGlobalOptions or hashChanged(fileIdx):
|
||||
markDirty()
|
||||
|
||||
if gFuzzyGraphChecking:
|
||||
if gMemCacheData[fileIdx].needsRecompile != Maybe:
|
||||
return gMemCacheData[fileIdx].needsRecompile
|
||||
else:
|
||||
# cycle detection: We claim that a cycle does no harm.
|
||||
if gMemCacheData[fileIdx].needsRecompile == Probing:
|
||||
return No
|
||||
if gMemCacheData[fileIdx].deps != nil:
|
||||
gMemCacheData[fileIdx].needsRecompile = Probing
|
||||
for dep in gMemCacheData[fileIdx].deps:
|
||||
let d = checkDepMem(dep)
|
||||
if d in {Yes, Recompiled}:
|
||||
# echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
|
||||
markDirty()
|
||||
|
||||
if optForceFullMake in gGlobalOptions or hashChanged(fileIdx):
|
||||
markDirty()
|
||||
gMemCacheData[fileIdx].needsRecompile = No
|
||||
return No
|
||||
|
||||
if gMemCacheData[fileIdx].deps != nil:
|
||||
gMemCacheData[fileIdx].needsRecompile = Probing
|
||||
for dep in gMemCacheData[fileIdx].deps:
|
||||
let d = checkDepMem(dep)
|
||||
if d in {Yes, Recompiled}:
|
||||
# echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
|
||||
markDirty()
|
||||
|
||||
gMemCacheData[fileIdx].needsRecompile = No
|
||||
return No
|
||||
|
||||
proc newModule(fileIdx: int32): PSym =
|
||||
proc newModule(graph: ModuleGraph; fileIdx: int32): PSym =
|
||||
# We cannot call ``newSym`` here, because we have to circumvent the ID
|
||||
# mechanism, which we do in order to assign each module a persistent ID.
|
||||
new(result)
|
||||
|
|
@ -143,20 +131,19 @@ proc newModule(fileIdx: int32): PSym =
|
|||
|
||||
result.info = newLineInfo(fileIdx, 1, 1)
|
||||
let pack = getIdent(getPackageName(filename))
|
||||
var packSym = packageSyms.strTableGet(pack)
|
||||
var packSym = graph.packageSyms.strTableGet(pack)
|
||||
if packSym == nil:
|
||||
let pck = getPackageName(filename)
|
||||
let pck2 = if pck.len > 0: pck else: "unknown"
|
||||
packSym = newSym(skPackage, getIdent(pck2), nil, result.info)
|
||||
initStrTable(packSym.tab)
|
||||
packageSyms.strTableAdd(packSym)
|
||||
graph.packageSyms.strTableAdd(packSym)
|
||||
|
||||
result.owner = packSym
|
||||
result.position = fileIdx
|
||||
|
||||
growCache gMemCacheData, fileIdx
|
||||
growCache gCompiledModules, fileIdx
|
||||
gCompiledModules[result.position] = result
|
||||
growCache graph.modules, fileIdx
|
||||
graph.modules[result.position] = result
|
||||
|
||||
incl(result.flags, sfUsed)
|
||||
initStrTable(result.tab)
|
||||
|
|
@ -167,13 +154,12 @@ proc newModule(fileIdx: int32): PSym =
|
|||
# strTableIncl() for error corrections:
|
||||
discard strTableIncl(packSym.tab, result)
|
||||
|
||||
proc compileModule*(fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
|
||||
result = getModule(fileIdx)
|
||||
proc compileModule*(graph: ModuleGraph; fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
|
||||
result = graph.getModule(fileIdx)
|
||||
if result == nil:
|
||||
growCache gMemCacheData, fileIdx
|
||||
gMemCacheData[fileIdx].needsRecompile = Probing
|
||||
result = newModule(fileIdx)
|
||||
#var rd = handleSymbolFile(result)
|
||||
#growCache gMemCacheData, fileIdx
|
||||
#gMemCacheData[fileIdx].needsRecompile = Probing
|
||||
result = newModule(graph, fileIdx)
|
||||
var rd: PRodReader
|
||||
result.flags = result.flags + flags
|
||||
if sfMainModule in result.flags:
|
||||
|
|
@ -182,44 +168,51 @@ proc compileModule*(fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
|
|||
if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
|
||||
rd = handleSymbolFile(result, cache)
|
||||
if result.id < 0:
|
||||
internalError("handleSymbolFile should have set the module\'s ID")
|
||||
internalError("handleSymbolFile should have set the module's ID")
|
||||
return
|
||||
else:
|
||||
result.id = getID()
|
||||
let validFile = processModule(result,
|
||||
discard processModule(graph, result,
|
||||
if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil,
|
||||
rd, cache)
|
||||
if optCaasEnabled in gGlobalOptions:
|
||||
gMemCacheData[fileIdx].compiledAt = gLastCmdTime
|
||||
gMemCacheData[fileIdx].needsRecompile = Recompiled
|
||||
if validFile: doHash fileIdx
|
||||
else:
|
||||
if checkDepMem(fileIdx) == Yes:
|
||||
result = compileModule(fileIdx, cache, flags)
|
||||
else:
|
||||
result = gCompiledModules[fileIdx]
|
||||
#if optCaasEnabled in gGlobalOptions:
|
||||
# gMemCacheData[fileIdx].needsRecompile = Recompiled
|
||||
# if validFile: doHash fileIdx
|
||||
elif graph.isDirty(result):
|
||||
result.flags.excl sfDirty
|
||||
# reset module fields:
|
||||
initStrTable(result.tab)
|
||||
result.ast = nil
|
||||
discard processModule(graph, result,
|
||||
if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil,
|
||||
nil, cache)
|
||||
graph.markClientsDirty(fileIdx)
|
||||
when false:
|
||||
if checkDepMem(fileIdx) == Yes:
|
||||
result = compileModule(fileIdx, cache, flags)
|
||||
else:
|
||||
result = gCompiledModules[fileIdx]
|
||||
|
||||
proc importModule*(s: PSym, fileIdx: int32; cache: IdentCache): PSym {.procvar.} =
|
||||
proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: int32;
|
||||
cache: IdentCache): PSym {.procvar.} =
|
||||
# this is called by the semantic checking phase
|
||||
result = compileModule(fileIdx, cache, {})
|
||||
if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
|
||||
result = compileModule(graph, fileIdx, cache, {})
|
||||
graph.addDep(s, fileIdx)
|
||||
#if sfSystemModule in result.flags:
|
||||
# localError(result.info, errAttemptToRedefine, result.name.s)
|
||||
# restore the notes for outer module:
|
||||
gNotes = if s.owner.id == gMainPackageId: gMainPackageNotes
|
||||
else: ForeignPackageNotes
|
||||
|
||||
proc includeModule*(s: PSym, fileIdx: int32; cache: IdentCache): PNode {.procvar.} =
|
||||
proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: int32;
|
||||
cache: IdentCache): PNode {.procvar.} =
|
||||
result = syntaxes.parseFile(fileIdx, cache)
|
||||
if optCaasEnabled in gGlobalOptions:
|
||||
growCache gMemCacheData, fileIdx
|
||||
addDep(s, fileIdx)
|
||||
doHash(fileIdx)
|
||||
graph.addDep(s, fileIdx)
|
||||
|
||||
proc compileSystemModule*(cache: IdentCache) =
|
||||
proc compileSystemModule*(graph: ModuleGraph; cache: IdentCache) =
|
||||
if magicsys.systemModule == nil:
|
||||
systemFileIdx = fileInfoIdx(options.libpath/"system.nim")
|
||||
discard compileModule(systemFileIdx, cache, {sfSystemModule})
|
||||
discard graph.compileModule(systemFileIdx, cache, {sfSystemModule})
|
||||
|
||||
proc wantMainModule* =
|
||||
if gProjectFull.len == 0:
|
||||
|
|
@ -229,18 +222,19 @@ proc wantMainModule* =
|
|||
passes.gIncludeFile = includeModule
|
||||
passes.gImportModule = importModule
|
||||
|
||||
proc compileProject*(cache: IdentCache; projectFileIdx = -1'i32) =
|
||||
proc compileProject*(graph: ModuleGraph; cache: IdentCache;
|
||||
projectFileIdx = -1'i32) =
|
||||
wantMainModule()
|
||||
let systemFileIdx = fileInfoIdx(options.libpath / "system.nim")
|
||||
let projectFile = if projectFileIdx < 0: gProjectMainIdx else: projectFileIdx
|
||||
if projectFile == systemFileIdx:
|
||||
discard compileModule(projectFile, cache, {sfMainModule, sfSystemModule})
|
||||
discard graph.compileModule(projectFile, cache, {sfMainModule, sfSystemModule})
|
||||
else:
|
||||
compileSystemModule(cache)
|
||||
discard compileModule(projectFile, cache, {sfMainModule})
|
||||
graph.compileSystemModule(cache)
|
||||
discard graph.compileModule(projectFile, cache, {sfMainModule})
|
||||
|
||||
proc makeModule*(filename: string): PSym =
|
||||
result = newModule(fileInfoIdx filename)
|
||||
proc makeModule*(graph: ModuleGraph; filename: string): PSym =
|
||||
result = graph.newModule(fileInfoIdx filename)
|
||||
result.id = getID()
|
||||
|
||||
proc makeStdinModule*(): PSym = makeModule"stdin"
|
||||
proc makeStdinModule*(graph: ModuleGraph): PSym = graph.makeModule"stdin"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue