jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

niminst: faster way to deduplicate files

Browse source
This commit is contained in:
Araq 2017-01-03 17:03:30 +01:00
commit 8413c9f15e
1 changed files with 18 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

35
tools/niminst/niminst.nim
View file

@ -14,7 +14,8 @@ when haveZipLib:
import zipfiles import zipfiles
import import
os, osproc, strutils, parseopt, parsecfg, strtabs, streams, debcreation os, osproc, strutils, parseopt, parsecfg, strtabs, streams, debcreation,
securehash
const const
maxOS = 20 # max number of OSes maxOS = 20 # max number of OSes
@ -476,23 +477,23 @@ proc writeFile(filename, content, newline: string) =
else: else:
quit("Cannot open for writing: " & filename) quit("Cannot open for writing: " & filename)
proc removeDuplicateFiles(c: var ConfigData) = proc deduplicateFiles(c: var ConfigData) =
for osA in countdown(c.oses.len, 1): var tab = newStringTable()
for cpuA in countdown(c.cpus.len, 1): let build = getOutputDir(c)
for osA in countup(1, c.oses.len):
for cpuA in countup(1, c.cpus.len):
if c.cfiles[osA][cpuA].isNil: c.cfiles[osA][cpuA] = @[] if c.cfiles[osA][cpuA].isNil: c.cfiles[osA][cpuA] = @[]
if c.explicitPlatforms and not c.platforms[osA][cpuA]: continue if c.explicitPlatforms and not c.platforms[osA][cpuA]: continue
for i in 0..c.cfiles[osA][cpuA].len-1: for dup in mitems(c.cfiles[osA][cpuA]):
var dup = c.cfiles[osA][cpuA][i] let key = $secureHashFile(build / dup)
var f = extractFilename(dup) let val = build / buildDir(osA, cpuA) / extractFilename(dup)
for osB in 1..c.oses.len: let orig = tab.getOrDefault(key)
for cpuB in 1..c.cpus.len: if orig.len > 0:
if osB != osA or cpuB != cpuA: # file is identical, so delete duplicate:
var orig = buildDir(osB, cpuB) / f removeFile(dup)
if existsFile(orig) and existsFile(dup) and dup = orig
sameFileContent(orig, dup): else:
# file is identical, so delete duplicate: tab[key] = val
removeFile(dup)
c.cfiles[osA][cpuA][i] = orig
proc writeInstallScripts(c: var ConfigData) = proc writeInstallScripts(c: var ConfigData) =
if c.installScript: if c.installScript:
@ -536,7 +537,7 @@ proc srcdist(c: var ConfigData) =
copyFile(dest=dest, source=c.cfiles[osA][cpuA][i]) copyFile(dest=dest, source=c.cfiles[osA][cpuA][i])
c.cfiles[osA][cpuA][i] = relDest c.cfiles[osA][cpuA][i] = relDest
# second pass: remove duplicate files # second pass: remove duplicate files
removeDuplicateFiles(c) deduplicateFiles(c)
writeFile(getOutputDir(c) / buildShFile, generateBuildShellScript(c), "\10") writeFile(getOutputDir(c) / buildShFile, generateBuildShellScript(c), "\10")
inclFilePermissions(getOutputDir(c) / buildShFile, {fpUserExec, fpGroupExec, fpOthersExec}) inclFilePermissions(getOutputDir(c) / buildShFile, {fpUserExec, fpGroupExec, fpOthersExec})
writeFile(getOutputDir(c) / makeFile, generateMakefile(c), "\10") writeFile(getOutputDir(c) / makeFile, generateMakefile(c), "\10")