CRC -> Hash

This commit is contained in:
Flaviu Tamas 2015-06-29 18:23:23 -04:00
commit 62b55592e8
7 changed files with 69 additions and 69 deletions

View file

@ -429,7 +429,7 @@ proc resetCompilationLists* =
initLinkedList(toCompile) initLinkedList(toCompile)
## XXX: we must associate these with their originating module ## XXX: we must associate these with their originating module
# when the module is loaded/unloaded it adds/removes its items # when the module is loaded/unloaded it adds/removes its items
# That's because we still need to CRC check the external files # That's because we still need to hash check the external files
# Maybe we can do that in checkDep on the other hand? # Maybe we can do that in checkDep on the other hand?
initLinkedList(externalToCompile) initLinkedList(externalToCompile)
initLinkedList(toLink) initLinkedList(toLink)
@ -585,18 +585,18 @@ proc externalFileChanged(filename: string): bool =
if gCmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM}: if gCmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM}:
return false return false
var crcFile = toGeneratedFile(filename.withPackageName, "sha1") var hashFile = toGeneratedFile(filename.withPackageName, "sha1")
var currentCrc = footprint(filename) var currentHash = footprint(filename)
var f: File var f: File
if open(f, crcFile, fmRead): if open(f, hashFile, fmRead):
let oldCrc = parseSecureHash(f.readLine()) let oldHash = parseSecureHash(f.readLine())
close(f) close(f)
result = oldCrc != currentCrc result = oldHash != currentHash
else: else:
result = true result = true
if result: if result:
if open(f, crcFile, fmWrite): if open(f, hashFile, fmWrite):
f.writeLine($currentCrc) f.writeLine($currentHash)
close(f) close(f)
proc addExternalFileToCompile*(filename: string) = proc addExternalFileToCompile*(filename: string) =

View file

@ -70,7 +70,7 @@ proc commandCompileToC =
lastCaasCmd = curCaasCmd lastCaasCmd = curCaasCmd
resetCgenModules() resetCgenModules()
for i in 0 .. <gMemCacheData.len: for i in 0 .. <gMemCacheData.len:
gMemCacheData[i].crcStatus = crcCached gMemCacheData[i].hashStatus = hashCached
gMemCacheData[i].needsRecompile = Maybe gMemCacheData[i].needsRecompile = Maybe
# XXX: clean these global vars # XXX: clean these global vars

View file

@ -15,14 +15,14 @@ import
type type
TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled
TCrcStatus* = enum crcNotTaken, crcCached, crcHasChanged, crcNotChanged THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged
TModuleInMemory* = object TModuleInMemory* = object
compiledAt*: float compiledAt*: float
crc*: SecureHash hash*: SecureHash
deps*: seq[int32] ## XXX: slurped files are currently not tracked deps*: seq[int32] ## XXX: slurped files are currently not tracked
needsRecompile*: TNeedRecompile needsRecompile*: TNeedRecompile
crcStatus*: TCrcStatus hashStatus*: THashStatus
var var
gCompiledModules: seq[PSym] = @[] gCompiledModules: seq[PSym] = @[]
@ -34,36 +34,36 @@ proc getModule(fileIdx: int32): PSym =
if fileIdx >= 0 and fileIdx < gCompiledModules.len: if fileIdx >= 0 and fileIdx < gCompiledModules.len:
result = gCompiledModules[fileIdx] result = gCompiledModules[fileIdx]
template crc(x: PSym): expr = template hash(x: PSym): expr =
gMemCacheData[x.position].crc gMemCacheData[x.position].hash
proc crcChanged(fileIdx: int32): bool = proc hashChanged(fileIdx: int32): bool =
internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
template updateStatus = template updateStatus =
gMemCacheData[fileIdx].crcStatus = if result: crcHasChanged gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged
else: crcNotChanged else: hashNotChanged
# echo "TESTING CRC: ", fileIdx.toFilename, " ", result # echo "TESTING Hash: ", fileIdx.toFilename, " ", result
case gMemCacheData[fileIdx].crcStatus: case gMemCacheData[fileIdx].hashStatus:
of crcHasChanged: of hashHasChanged:
result = true result = true
of crcNotChanged: of hashNotChanged:
result = false result = false
of crcCached: of hashCached:
let newCrc = secureHashFile(fileIdx.toFullPath) let newHash = secureHashFile(fileIdx.toFullPath)
result = newCrc != gMemCacheData[fileIdx].crc result = newHash != gMemCacheData[fileIdx].hash
gMemCacheData[fileIdx].crc = newCrc gMemCacheData[fileIdx].hash = newHash
updateStatus() updateStatus()
of crcNotTaken: of hashNotTaken:
gMemCacheData[fileIdx].crc = secureHashFile(fileIdx.toFullPath) gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
result = true result = true
updateStatus() updateStatus()
proc doCRC(fileIdx: int32) = proc doHash(fileIdx: int32) =
if gMemCacheData[fileIdx].crcStatus == crcNotTaken: if gMemCacheData[fileIdx].hashStatus == hashNotTaken:
# echo "FIRST CRC: ", fileIdx.ToFilename # echo "FIRST Hash: ", fileIdx.ToFilename
gMemCacheData[fileIdx].crc = secureHashFile(fileIdx.toFullPath) gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
proc addDep(x: PSym, dep: int32) = proc addDep(x: PSym, dep: int32) =
growCache gMemCacheData, dep growCache gMemCacheData, dep
@ -94,7 +94,7 @@ proc checkDepMem(fileIdx: int32): TNeedRecompile =
return gMemCacheData[fileIdx].needsRecompile return gMemCacheData[fileIdx].needsRecompile
if optForceFullMake in gGlobalOptions or if optForceFullMake in gGlobalOptions or
crcChanged(fileIdx): hashChanged(fileIdx):
markDirty markDirty
if gMemCacheData[fileIdx].deps != nil: if gMemCacheData[fileIdx].deps != nil:
@ -155,7 +155,7 @@ proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym =
if optCaasEnabled in gGlobalOptions: if optCaasEnabled in gGlobalOptions:
gMemCacheData[fileIdx].compiledAt = gLastCmdTime gMemCacheData[fileIdx].compiledAt = gLastCmdTime
gMemCacheData[fileIdx].needsRecompile = Recompiled gMemCacheData[fileIdx].needsRecompile = Recompiled
doCRC fileIdx doHash fileIdx
else: else:
if checkDepMem(fileIdx) == Yes: if checkDepMem(fileIdx) == Yes:
result = compileModule(fileIdx, flags) result = compileModule(fileIdx, flags)
@ -174,7 +174,7 @@ proc includeModule*(s: PSym, fileIdx: int32): PNode {.procvar.} =
if optCaasEnabled in gGlobalOptions: if optCaasEnabled in gGlobalOptions:
growCache gMemCacheData, fileIdx growCache gMemCacheData, fileIdx
addDep(s, fileIdx) addDep(s, fileIdx)
doCRC(fileIdx) doHash(fileIdx)
proc `==^`(a, b: string): bool = proc `==^`(a, b: string): bool =
try: try:

View file

@ -19,8 +19,8 @@
# NIM:$fileversion\n # NIM:$fileversion\n
# - the module's id (even if the module changed, its ID will not!): # - the module's id (even if the module changed, its ID will not!):
# ID:Ax3\n # ID:Ax3\n
# - CRC value of this module: # - HASH value of this module:
# CRC:CRC-val\n # HASH:HASH-val\n
# - a section containing the compiler options and defines this # - a section containing the compiler options and defines this
# module has been compiled with: # module has been compiled with:
# OPTIONS:options\n # OPTIONS:options\n
@ -33,7 +33,7 @@
# ) # )
# - an include file dependency section: # - an include file dependency section:
# INCLUDES( # INCLUDES(
# <fileidx> <CRC of myfile.inc>\n # fileidx is the LINE in the file section! # <fileidx> <Hash of myfile.inc>\n # fileidx is the LINE in the file section!
# ) # )
# - a module dependency section: # - a module dependency section:
# DEPS: <fileidx> <fileidx>\n # DEPS: <fileidx> <fileidx>\n
@ -98,7 +98,7 @@ type
rrNone, # no need to recompile rrNone, # no need to recompile
rrRodDoesNotExist, # rod file does not exist rrRodDoesNotExist, # rod file does not exist
rrRodInvalid, # rod file is invalid rrRodInvalid, # rod file is invalid
rrCrcChange, # file has been edited since last recompilation rrHashChange, # file has been edited since last recompilation
rrDefines, # defines have changed rrDefines, # defines have changed
rrOptions, # options have changed rrOptions, # options have changed
rrInclDeps, # an include has changed rrInclDeps, # an include has changed
@ -538,20 +538,20 @@ proc cmdChangeTriggersRecompilation(old, new: TCommands): bool =
# else: trigger recompilation: # else: trigger recompilation:
result = true result = true
proc processRodFile(r: PRodReader, crc: SecureHash) = proc processRodFile(r: PRodReader, hash: SecureHash) =
var var
w: string w: string
d: int d: int
var inclCrc: SecureHash var inclHash: SecureHash
while r.s[r.pos] != '\0': while r.s[r.pos] != '\0':
var section = rdWord(r) var section = rdWord(r)
if r.reason != rrNone: if r.reason != rrNone:
break # no need to process this file further break # no need to process this file further
case section case section
of "CRC": of "HASH":
inc(r.pos) # skip ':' inc(r.pos) # skip ':'
if crc != parseSecureHash(decodeStr(r.s, r.pos)): if hash != parseSecureHash(decodeStr(r.s, r.pos)):
r.reason = rrCrcChange r.reason = rrHashChange
of "ID": of "ID":
inc(r.pos) # skip ':' inc(r.pos) # skip ':'
r.moduleID = decodeVInt(r.s, r.pos) r.moduleID = decodeVInt(r.s, r.pos)
@ -598,9 +598,9 @@ proc processRodFile(r: PRodReader, crc: SecureHash) =
while r.s[r.pos] != ')': while r.s[r.pos] != ')':
w = r.files[decodeVInt(r.s, r.pos)].toFullPath w = r.files[decodeVInt(r.s, r.pos)].toFullPath
inc(r.pos) # skip ' ' inc(r.pos) # skip ' '
inclCrc = parseSecureHash(decodeStr(r.s, r.pos)) inclHash = parseSecureHash(decodeStr(r.s, r.pos))
if r.reason == rrNone: if r.reason == rrNone:
if not existsFile(w) or (inclCrc != secureHashFile(w)): if not existsFile(w) or (inclHash != secureHashFile(w)):
r.reason = rrInclDeps r.reason = rrInclDeps
if r.s[r.pos] == '\x0A': if r.s[r.pos] == '\x0A':
inc(r.pos) inc(r.pos)
@ -651,7 +651,7 @@ proc startsWith(buf: cstring, token: string, pos = 0): bool =
while s < token.len and buf[pos+s] == token[s]: inc s while s < token.len and buf[pos+s] == token[s]: inc s
result = s == token.len result = s == token.len
proc newRodReader(modfilename: string, crc: SecureHash, proc newRodReader(modfilename: string, hash: SecureHash,
readerIndex: int): PRodReader = readerIndex: int): PRodReader =
new(result) new(result)
try: try:
@ -703,8 +703,8 @@ type
filename*: string filename*: string
reason*: TReasonForRecompile reason*: TReasonForRecompile
rd*: PRodReader rd*: PRodReader
crc*: SecureHash hash*: SecureHash
crcDone*: bool hashDone*: bool
TFileModuleMap = seq[TFileModuleRec] TFileModuleMap = seq[TFileModuleRec]
@ -796,14 +796,14 @@ proc loadMethods(r: PRodReader) =
r.methods.add(rrGetSym(r, d, unknownLineInfo())) r.methods.add(rrGetSym(r, d, unknownLineInfo()))
if r.s[r.pos] == ' ': inc(r.pos) if r.s[r.pos] == ' ': inc(r.pos)
proc getCRC*(fileIdx: int32): SecureHash = proc getHash*(fileIdx: int32): SecureHash =
internalAssert fileIdx >= 0 and fileIdx < gMods.len internalAssert fileIdx >= 0 and fileIdx < gMods.len
if gMods[fileIdx].crcDone: if gMods[fileIdx].hashDone:
return gMods[fileIdx].crc return gMods[fileIdx].hash
result = secureHashFile(fileIdx.toFilename) result = secureHashFile(fileIdx.toFilename)
gMods[fileIdx].crc = result gMods[fileIdx].hash = result
template growCache*(cache, pos) = template growCache*(cache, pos) =
if cache.len <= pos: cache.setLen(pos+1) if cache.len <= pos: cache.setLen(pos+1)
@ -815,16 +815,16 @@ proc checkDep(fileIdx: int32): TReasonForRecompile =
# reason has already been computed for this module: # reason has already been computed for this module:
return gMods[fileIdx].reason return gMods[fileIdx].reason
let filename = fileIdx.toFilename let filename = fileIdx.toFilename
var crc = getCRC(fileIdx) var hash = getHash(fileIdx)
gMods[fileIdx].reason = rrNone # we need to set it here to avoid cycles gMods[fileIdx].reason = rrNone # we need to set it here to avoid cycles
result = rrNone result = rrNone
var r: PRodReader = nil var r: PRodReader = nil
var rodfile = toGeneratedFile(filename.withPackageName, RodExt) var rodfile = toGeneratedFile(filename.withPackageName, RodExt)
r = newRodReader(rodfile, crc, fileIdx) r = newRodReader(rodfile, hash, fileIdx)
if r == nil: if r == nil:
result = (if existsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist) result = (if existsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist)
else: else:
processRodFile(r, crc) processRodFile(r, hash)
result = r.reason result = r.reason
if result == rrNone: if result == rrNone:
# check modules it depends on # check modules it depends on
@ -1029,9 +1029,9 @@ proc viewFile(rodfile: string) =
while r.s[r.pos] != '\0': while r.s[r.pos] != '\0':
let section = rdWord(r) let section = rdWord(r)
case section case section
of "CRC": of "HASH":
inc(r.pos) # skip ':' inc(r.pos) # skip ':'
outf.writeLine("CRC:", $decodeVInt(r.s, r.pos)) outf.writeLine("HASH:", $decodeVInt(r.s, r.pos))
of "ID": of "ID":
inc(r.pos) # skip ':' inc(r.pos) # skip ':'
r.moduleID = decodeVInt(r.s, r.pos) r.moduleID = decodeVInt(r.s, r.pos)
@ -1084,11 +1084,11 @@ proc viewFile(rodfile: string) =
while r.s[r.pos] != ')': while r.s[r.pos] != ')':
let w = r.files[decodeVInt(r.s, r.pos)] let w = r.files[decodeVInt(r.s, r.pos)]
inc(r.pos) # skip ' ' inc(r.pos) # skip ' '
let inclCrc = decodeVInt(r.s, r.pos) let inclHash = decodeVInt(r.s, r.pos)
if r.s[r.pos] == '\x0A': if r.s[r.pos] == '\x0A':
inc(r.pos) inc(r.pos)
inc(r.line) inc(r.line)
outf.write(w, " ", inclCrc, "\n") outf.write(w, " ", inclHash, "\n")
if r.s[r.pos] == ')': inc(r.pos) if r.s[r.pos] == ')': inc(r.pos)
outf.write(")\n") outf.write(")\n")
of "DEPS": of "DEPS":

View file

@ -21,7 +21,7 @@ import
type type
TRodWriter = object of TPassContext TRodWriter = object of TPassContext
module: PSym module: PSym
crc: SecureHash hash: SecureHash
options: TOptions options: TOptions
defines: string defines: string
inclDeps: string inclDeps: string
@ -39,7 +39,7 @@ type
PRodWriter = ref TRodWriter PRodWriter = ref TRodWriter
proc newRodWriter(crc: SecureHash, module: PSym): PRodWriter proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter
proc addModDep(w: PRodWriter, dep: string) proc addModDep(w: PRodWriter, dep: string)
proc addInclDep(w: PRodWriter, dep: string) proc addInclDep(w: PRodWriter, dep: string)
proc addInterfaceSym(w: PRodWriter, s: PSym) proc addInterfaceSym(w: PRodWriter, s: PSym)
@ -63,7 +63,7 @@ proc fileIdx(w: PRodWriter, filename: string): int =
template filename*(w: PRodWriter): string = template filename*(w: PRodWriter): string =
w.module.filename w.module.filename
proc newRodWriter(crc: SecureHash, module: PSym): PRodWriter = proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter =
new(result) new(result)
result.sstack = @[] result.sstack = @[]
result.tstack = @[] result.tstack = @[]
@ -71,7 +71,7 @@ proc newRodWriter(crc: SecureHash, module: PSym): PRodWriter =
initIiTable(result.imports.tab) initIiTable(result.imports.tab)
result.index.r = "" result.index.r = ""
result.imports.r = "" result.imports.r = ""
result.crc = crc result.hash = hash
result.module = module result.module = module
result.defines = getDefines() result.defines = getDefines()
result.options = options.gOptions result.options = options.gOptions
@ -440,9 +440,9 @@ proc writeRod(w: PRodWriter) =
f.write(orig) f.write(orig)
f.write(rodNL) f.write(rodNL)
var crc = "CRC:" var hash = "HASH:"
encodeStr($w.crc, crc) encodeStr($w.hash, hash)
f.write(crc) f.write(hash)
f.write(rodNL) f.write(rodNL)
var options = "OPTIONS:" var options = "OPTIONS:"
@ -575,7 +575,7 @@ proc process(c: PPassContext, n: PNode): PNode =
proc myOpen(module: PSym): PPassContext = proc myOpen(module: PSym): PPassContext =
if module.id < 0: internalError("rodwrite: module ID not set") if module.id < 0: internalError("rodwrite: module ID not set")
var w = newRodWriter(module.fileIdx.getCRC, module) var w = newRodWriter(module.fileIdx.getHash, module)
rawAddInterfaceSym(w, module) rawAddInterfaceSym(w, module)
result = w result = w

View file

@ -18,7 +18,7 @@ when useCaas:
# We cache modules and the dependency graph. However, we don't check for # We cache modules and the dependency graph. However, we don't check for
# file changes but expect the client to tell us about them, otherwise the # file changes but expect the client to tell us about them, otherwise the
# repeated CRC calculations may turn out to be too slow. # repeated hash calculations may turn out to be too slow.
var var
curCaasCmd* = "" curCaasCmd* = ""

View file

@ -318,7 +318,7 @@ with the project:
.. code-block:: Nim .. code-block:: Nim
{.compile: "myfile.cpp".} {.compile: "myfile.cpp".}
**Note**: Nim computes a CRC checksum and only recompiles the file if it **Note**: Nim computes a SHA1 checksum and only recompiles the file if it
has changed. You can use the ``-f`` command line option to force recompilation has changed. You can use the ``-f`` command line option to force recompilation
of the file. of the file.