Improve the trimcc tool's batching algorithm.

tools/fakedeps.nim

@@ -0,0 +1,18 @@
+import strutils, os, pegs, strtabs, math, threadpool, times
+
+proc fakeCppDep(x: ptr float) {.importcpp: "fakeCppDep", header: "<vector>".}
+proc fakeTimeDep() = echo(times.getDateStr())
+proc fakedeps() =
+  var x = 0.4
+  {.emit: "#if 0\n".}
+  fakeCppDep(addr x)
+  {.emit: "#endif\n".}
+
+  # this is not true:
+  if math.sin(x) > 0.6:
+    spawn(fakeTimeDep())
+
+proc main =
+  fakedeps()
+when isMainModule:
+  main()

tools/trimcc.nim

@@ -2,33 +2,16 @@
 
 import strutils, os, pegs, strtabs, math, threadpool, times
 
-proc fakeCppDep(x: ptr float) {.importcpp: "fakeCppDep", header: "<vector>".}
-
 const
   Essential = """gcc.exe g++.exe gdb.exe ld.exe as.exe c++.exe cpp.exe cc1.exe
-crtbegin.o crtend.o crt2.o dllcrt2.o
+crtbegin.o crtend.o crt2.o dllcrt2.o libgcc_s_dw2-1.dll libgcc_s_sjlj-1.dll
-libexpat-1.dll libwinpthread-1.dll
+libgcc_s_seh-1.dll libexpat-1.dll libwinpthread-1.dll aio.h dlfcn.h fcntl.h
-
+fenv.h fmtmsg.h fnmatch.h ftw.h errno.h glob.h gtmath.h if.h in.h ipc.h
-aio.h
+langinfo.h locale.h math.h mman.h netdb.h nl_types.h poll.h pthread.h pwd.h
-dlfcn.h
+sched.h select.h semaphore.h signal.h socket.h spawn.h stat.h statvfs.h stdio.h
-fcntl.h fenv.h fmtmsg.h fnmatch.h ftw.h
+stdlib.h string.h strings.h tcp.h time.h types.h ucontext.h uio.h utsname.h
-errno.h
+unistd.h wait.h varargs.h windows.h zlib.h
-glob.h gtmath.h
-if.h in.h ipc.h
-langinfo.h locale.h
-math.h mman.h
-netdb.h nl_types.h
-poll.h pthread.h pwd.h
-sched.h select.h semaphore.h signal.h
-socket.h spawn.h stat.h statvfs.h stdio.h stdlib.h string.h strings.h
-tcp.h time.h types.h
-ucontext.h uio.h utsname.h unistd.h
-wait.h
-varargs.h
-windows.h
-zlib.h
 """.split
-  BucketSize = 40
 
 proc includes(headerpath, headerfile: string, whitelist: StringTableRef) =
   whitelist[headerfile] = "processed"
@@ -61,8 +44,26 @@ proc gatherFiles(dir: string, whitelist: StringTableRef, result: var seq[string]
       let name = extractFilename(path)
       if not whitelist.hasKey(name):
         result.add(path)
-    of pcDir: gatherFiles(path, whitelist, result)
+    of pcDir:
-    else: discard
+      gatherFiles(path, whitelist, result)
+    else:
+      discard
+
+proc gatherEmptyFolders(dir: string, whitelist: StringTableRef, result: var seq[string]) =
+  var empty = true
+  for kind, path in walkDir(dir):
+    case kind
+    of pcFile:
+      empty = false
+    of pcDir:
+      let (none, name) = splitPath(path)
+      if not whitelist.hasKey(name):
+        gatherEmptyFolders(path, whitelist, result)
+      empty = false
+    else:
+      discard
+  if empty:
+    result.add(dir)
 
 proc newName(f: string): string =
   let (dir, name, ext) = splitFile(f)
@@ -70,58 +71,104 @@ proc newName(f: string): string =
 
 proc ccStillWorks(): bool =
   const
-    c1 = r"nim c --force_build koch"
+    c1 = r"nim c --verbosity:0 --force_build koch"
-    c2 = r"nim c --force_build --threads:on --out:temp.exe tools/trimcc"
+    c2 = r"nim c --verbosity:0 --force_build --threads:on --out:tempOne.exe trimcc"
-  result = execShellCmd(c1) == 0 and execShellCmd(c2) == 0
+    c3 = r"nim c --verbosity:0 --force_build --threads:on --out:tempTwo.exe fakeDeps"
+    c4 = r".\koch.exe"
+    c5 = r".\tempOne.exe"
+    c6 = r".\tempTwo.exe"
+  result = execShellCmd(c1) == 0 and execShellCmd(c2) == 0 and
+           execShellCmd(c3) == 0 and execShellCmd(c4) == 0 and
+           execShellCmd(c5) == 0 and execShellCmd(c6) == 0
 
-proc trialDeletion(files: seq[string], a, b: int) =
+proc trialDeletion(files: seq[string], a, b: int, whitelist: StringTableRef): bool =
-  for i in a .. min(b, files.high):
+  result = true
-    let path = files[i]
+  var single = (a == min(b, files.high))
+  for path in files[a .. min(b, files.high)]:
+    try:
       moveFile(dest=newName(path), source=path)
+    except OSError:
+      return false
+
+  # Test if compilation still works, even with the moved files.
   if ccStillWorks():
-    for i in a .. min(b, files.high):
+    for path in files[a .. min(b, files.high)]:
-      let path = files[i]
+      try:
-      echo "Optional: ", path
         removeFile(newName(path))
-  else:
    for i in a .. min(b, files.high):
+        echo "Optional: ", path
-      let path = files[i]
+      except OSError:
-      echo "Required: ", path
+        echo "Warning, couldn't move ", path
-      # copy back:
         moveFile(dest=path, source=newName(path))
+        return false
+  else:
+    for path in files[a .. min(b, files.high)]:
+      echo "Required: ", path
+      moveFile(dest=path, source=newName(path))
+      if single:
+        whitelist[path] = "found"
+      result = false
 
 proc main(dir: string) =
+  # Construct a whitelist of files to not remove
   var whitelist = newStringTable(modeCaseInsensitive)
   for e in Essential:
     whitelist[e] = "found"
   while true:
     let oldLen = whitelist.len
     processIncludes(dir, whitelist)
-    if oldLen == whitelist.len: break
+    if oldLen == whitelist.len:
-  var allFiles: seq[string] = @[]
+      break
+
+  # Remove batches of files
+  var nearlyDone = false
+  while true:
+    # Gather files to test
+    var allFiles = newSeq[string]()
     gatherFiles(dir, whitelist, allFiles)
-  when true:
+
+    # Determine the initial size of groups to check
+    var
+      maxBucketSize = len(allFiles)
+      bucketSize = 1
+
+    # Loop through the list of files, deleting batches
     var i = 0
     while i < allFiles.len:
-      trialDeletion(allFiles, i, i+BucketSize-1)
+      var success = trialDeletion(allFiles, i, i+bucketSize-1, whitelist)
-      inc i, BucketSize
+      inc i, bucketSize
+
+      # If we aren't on the last pass, adjust the batch size based on success
+      if not nearlyDone:
+        if success:
+          bucketSize = min(bucketSize * 2, maxBucketSize)
         else:
-    for x in allFiles: echo x
+          bucketSize = max(bucketSize div 2, 1)
+      echo "Bucket size is now ", bucketSize
 
-proc fakeTimeDep() = echo(times.getDateStr())
+    # After looping through all the files, check if we need to break.
+    if nearlyDone:
+      break
+    if bucketSize == 1:
+      nearlyDone = true
 
-proc fakedeps() =
+  while true:
-  var x = 0.4
+    var
-  {.emit: "#if 0\n".}
+      emptyFolders = newSeq[string]()
-  fakeCppDep(addr x)
+      changed = false
-  {.emit: "#endif\n".}
 
-  # this is not true:
+    gatherEmptyFolders(dir, whitelist, emptyFolders)
-  if math.sin(x) > 0.6:
+    for path in emptyFolders:
-    spawn(fakeTimeDep())
+      removeDir(path)
+      if not ccStillWorks():
+        createDir(path)
+        whitelist[path] = "found"
+      else:
+        changed = true
+    if not changed:
+      break
 
 if paramCount() == 1:
   doAssert ccStillWorks()
-  fakedeps()
   main(paramStr(1))
 else:
-  quit "Usage: trimcc c_compiler_directory"
+  quit "Usage: trimcc c_compiler_directory", QuitSuccess