CaaS in-memory caching
removed some redundant filepath params and variables and switched to canonical paths in most places
This commit is contained in:
Zahary Karadjov
parent
091c1b3075
commit
36e25a6849
23 changed files with 299 additions and 215 deletions
|
|
@ -12,11 +12,11 @@
|
|||
|
||||
import
|
||||
llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs,
|
||||
os, lists, condsyms, rodread, rodwrite, ropes, trees,
|
||||
os, lists, condsyms, rodread, rodwrite, ropes, trees, times,
|
||||
wordrecg, sem, semdata, idents, passes, docgen, extccomp,
|
||||
cgen, ecmasgen,
|
||||
platform, nimconf, importer, passaux, depends, evals, types, idgen,
|
||||
tables, docgen2, service, magicsys, parser
|
||||
tables, docgen2, service, magicsys, parser, crc
|
||||
|
||||
const
|
||||
has_LLVM_Backend = false
|
||||
|
|
@ -28,60 +28,113 @@ proc MainCommand*()
|
|||
|
||||
# ------------------ module handling -----------------------------------------
|
||||
|
||||
type
|
||||
TModuleInMemory = object
|
||||
compiledAt: float
|
||||
crc: int
|
||||
deps: seq[int32] ## XXX: slurped files are not currently tracked
|
||||
needsRecompile: bool
|
||||
|
||||
var
|
||||
compMods = initTable[string, PSym]() # all compiled modules
|
||||
gCompiledModules: seq[PSym] = @[]
|
||||
gMemCacheData: seq[TModuleInMemory] = @[]
|
||||
## XXX: we should implement recycling of file IDs
|
||||
## if the user keeps renaming modules, the file IDs will keep growing
|
||||
|
||||
# This expects a normalized module path
|
||||
proc registerModule(filename: string, module: PSym) =
|
||||
compMods[filename] = module
|
||||
proc getModule(fileIdx: int32): PSym =
|
||||
if fileIdx >= 0 and fileIdx < gCompiledModules.len:
|
||||
result = gCompiledModules[fileIdx]
|
||||
else:
|
||||
result = nil
|
||||
|
||||
# This expects a normalized module path
|
||||
proc getModule(filename: string): PSym =
|
||||
result = compMods[filename]
|
||||
template compiledAt(x: PSym): expr =
|
||||
gMemCacheData[x.position].compiledAt
|
||||
|
||||
var gModulesCount = 0
|
||||
proc newModule(filename: string): PSym =
|
||||
template crc(x: PSym): expr =
|
||||
gMemCacheData[x.position].crc
|
||||
|
||||
template addDep(x: Psym, dep: int32) =
|
||||
gMemCacheData[x.position].deps.add(dep)
|
||||
|
||||
proc checkDepMem(fileIdx: int32): bool =
|
||||
template markDirty =
|
||||
gMemCacheData[fileIdx].needsRecompile = true
|
||||
return true
|
||||
|
||||
if optForceFullMake in gGlobalOptions or
|
||||
curCaasCmd != lastCaasCmd: markDirty
|
||||
|
||||
let crc = crcFromFile(fileIdx.toFilename)
|
||||
if crc != gMemCacheData[fileIdx].crc: markDirty
|
||||
|
||||
for dep in gMemCacheData[fileIdx].deps:
|
||||
if checkDepMem(dep): markDirty
|
||||
|
||||
return false
|
||||
|
||||
proc newModule(fileIdx: int32): PSym =
|
||||
# We cannot call ``newSym`` here, because we have to circumvent the ID
|
||||
# mechanism, which we do in order to assign each module a persistent ID.
|
||||
new(result)
|
||||
result.id = - 1 # for better error checking
|
||||
result.kind = skModule
|
||||
let filename = fileIdx.toFilename
|
||||
result.name = getIdent(splitFile(filename).name)
|
||||
if not isNimrodIdentifier(result.name.s):
|
||||
rawMessage(errInvalidModuleName, result.name.s)
|
||||
|
||||
result.owner = result # a module belongs to itself
|
||||
result.info = newLineInfo(filename, 1, 1)
|
||||
result.position = gModulesCount
|
||||
inc gModulesCount
|
||||
result.info = newLineInfo(fileIdx, 1, 1)
|
||||
result.position = fileIdx
|
||||
|
||||
growCache gMemCacheData, fileIdx
|
||||
growCache gCompiledModules, fileIdx
|
||||
gCompiledModules[result.position] = result
|
||||
|
||||
incl(result.flags, sfUsed)
|
||||
initStrTable(result.tab)
|
||||
RegisterModule(filename, result)
|
||||
StrTableAdd(result.tab, result) # a module knows itself
|
||||
|
||||
proc CompileModule(filename: string, flags: TSymFlags): PSym
|
||||
proc importModule(filename: string): PSym =
|
||||
proc compileModule(fileIdx: int32, flags: TSymFlags): PSym =
|
||||
result = getModule(fileIdx)
|
||||
if result == nil:
|
||||
result = newModule(fileIdx)
|
||||
var rd = handleSymbolFile(result)
|
||||
result.flags = result.flags + flags
|
||||
if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
|
||||
rd = handleSymbolFile(result)
|
||||
if result.id < 0:
|
||||
InternalError("handleSymbolFile should have set the module\'s ID")
|
||||
return
|
||||
else:
|
||||
result.id = getID()
|
||||
processModule(result, nil, rd)
|
||||
gMemCacheData[fileIdx].compiledAt = gLastCmdTime
|
||||
gMemCacheData[fileIdx].needsRecompile = false
|
||||
else:
|
||||
InternalAssert optCaasEnabled in gGlobalOptions
|
||||
if checkDepMem(fileIdx):
|
||||
gCompiledModules[fileIdx] = nil
|
||||
result = CompileModule(fileIdx, flags)
|
||||
else:
|
||||
result = gCompiledModules[fileIdx]
|
||||
|
||||
proc compileModule(filename: string, flags: TSymFlags): PSym =
|
||||
result = compileModule(filename.fileInfoIdx, flags)
|
||||
|
||||
proc importModule(s: PSym, fileIdx: int32): PSym =
|
||||
# this is called by the semantic checking phase
|
||||
result = getModule(filename)
|
||||
result = getModule(fileIdx)
|
||||
if result == nil:
|
||||
# compile the module
|
||||
result = compileModule(filename, {})
|
||||
result = compileModule(fileIdx, {})
|
||||
if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
|
||||
elif sfSystemModule in result.flags:
|
||||
LocalError(result.info, errAttemptToRedefine, result.Name.s)
|
||||
|
||||
proc CompileModule(filename: string, flags: TSymFlags): PSym =
|
||||
var rd: PRodReader = nil
|
||||
var f = addFileExt(filename, nimExt)
|
||||
result = newModule(f)
|
||||
result.flags = result.flags + flags
|
||||
if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
|
||||
rd = handleSymbolFile(result, f)
|
||||
if result.id < 0:
|
||||
InternalError("handleSymbolFile should have set the module\'s ID")
|
||||
return
|
||||
else:
|
||||
result.id = getID()
|
||||
processModule(result, f, nil, rd)
|
||||
|
||||
proc includeModule(s: PSym, fileIdx: int32): PNode =
|
||||
result = syntaxes.parseFile(fileIdx)
|
||||
if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
|
||||
|
||||
proc `==^`(a, b: string): bool =
|
||||
try:
|
||||
|
|
@ -91,7 +144,8 @@ proc `==^`(a, b: string): bool =
|
|||
|
||||
proc compileSystemModule =
|
||||
if magicsys.SystemModule == nil:
|
||||
discard CompileModule(options.libpath /"system", {sfSystemModule})
|
||||
SystemFileIdx = fileInfoIdx(options.libpath/"system.nim")
|
||||
discard CompileModule(SystemFileIdx, {sfSystemModule})
|
||||
|
||||
proc CompileProject(projectFile = gProjectFull) =
|
||||
let systemFile = options.libpath / "system"
|
||||
|
|
@ -140,6 +194,8 @@ proc CommandCompileToC =
|
|||
compileProject()
|
||||
if gCmd != cmdRun:
|
||||
extccomp.CallCCompiler(changeFileExt(gProjectFull, ""))
|
||||
# caas will keep track only of the compilation commands
|
||||
lastCaasCmd = curCaasCmd
|
||||
|
||||
when has_LLVM_Backend:
|
||||
proc CommandCompileToLLVM =
|
||||
|
|
@ -171,7 +227,7 @@ proc InteractivePasses =
|
|||
var stdinModule: PSym
|
||||
proc makeStdinModule: PSym =
|
||||
if stdinModule == nil:
|
||||
stdinModule = newModule("stdin")
|
||||
stdinModule = newModule(gCmdLineInfo.fileIndex)
|
||||
stdinModule.id = getID()
|
||||
result = stdinModule
|
||||
|
||||
|
|
@ -184,24 +240,23 @@ proc CommandInteractive =
|
|||
else:
|
||||
var m = makeStdinModule()
|
||||
incl(m.flags, sfMainModule)
|
||||
processModule(m, "stdin", LLStreamOpenStdIn(), nil)
|
||||
processModule(m, LLStreamOpenStdIn(), nil)
|
||||
|
||||
const evalPasses = [verbosePass, semPass, evalPass]
|
||||
|
||||
proc evalNim(nodes: PNode, module: PSym, filename: string) =
|
||||
# we don't want to mess with gPasses here, because in nimrod serve
|
||||
# scenario, it may be set up properly for normal cgenPass() compilation
|
||||
carryPasses(nodes, module, filename, evalPasses)
|
||||
proc evalNim(nodes: PNode, module: PSym) =
|
||||
carryPasses(nodes, module, evalPasses)
|
||||
|
||||
proc commandEval(exp: string) =
|
||||
if SystemModule == nil:
|
||||
InteractivePasses()
|
||||
compileSystemModule()
|
||||
var echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
|
||||
evalNim(echoExp.parseString, makeStdinModule(), "stdin")
|
||||
evalNim(echoExp.parseString, makeStdinModule())
|
||||
|
||||
proc CommandPretty =
|
||||
var module = parseFile(addFileExt(mainCommandArg(), NimExt))
|
||||
var projectFile = addFileExt(mainCommandArg(), NimExt)
|
||||
var module = parseFile(projectFile.fileInfoIdx)
|
||||
if module != nil:
|
||||
renderModule(module, getOutFile(mainCommandArg(), "pretty." & NimExt))
|
||||
|
||||
|
|
@ -231,16 +286,18 @@ proc CommandSuggest =
|
|||
proc wantMainModule =
|
||||
if gProjectFull.len == 0:
|
||||
Fatal(gCmdLineInfo, errCommandExpectsFilename)
|
||||
gProjectMainIdx = addFileExt(gProjectFull, nimExt).fileInfoIdx
|
||||
|
||||
proc MainCommand =
|
||||
# In "nimrod serve" scenario, each command must reset the registered passes
|
||||
clearPasses()
|
||||
gLastCmdTime = epochTime()
|
||||
appendStr(searchPaths, options.libpath)
|
||||
if gProjectFull.len != 0:
|
||||
# current path is always looked first for modules
|
||||
prependStr(searchPaths, gProjectPath)
|
||||
setID(100)
|
||||
passes.gIncludeFile = syntaxes.parseFile
|
||||
passes.gIncludeFile = includeModule
|
||||
passes.gImportModule = importModule
|
||||
case command.normalize
|
||||
of "c", "cc", "compile", "compiletoc":
|
||||
|
|
@ -323,7 +380,7 @@ proc MainCommand =
|
|||
of "parse":
|
||||
gCmd = cmdParse
|
||||
wantMainModule()
|
||||
discard parseFile(addFileExt(gProjectFull, nimExt))
|
||||
discard parseFile(gProjectMainIdx)
|
||||
of "scan":
|
||||
gCmd = cmdScan
|
||||
wantMainModule()
|
||||
|
|
@ -343,9 +400,14 @@ proc MainCommand =
|
|||
wantMainModule()
|
||||
CommandSuggest()
|
||||
of "serve":
|
||||
gCmd = cmdIdeTools
|
||||
gGlobalOptions.incl(optCaasEnabled)
|
||||
msgs.gErrorMax = high(int) # do not stop after first error
|
||||
serve(MainCommand)
|
||||
|
||||
else: rawMessage(errInvalidCommandX, command)
|
||||
|
||||
if msgs.gErrorCounter == 0 and gCmd notin {cmdInterpret, cmdRun}:
|
||||
rawMessage(hintSuccessX, [$gLinesCompiled,
|
||||
formatFloat(epochTime() - gLastCmdTime, ffDecimal, 3),
|
||||
formatSize(getTotalMem())])
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue