testament: generic N-fold batching: windows CI 37mn=>16m (#14823)

* testament: run CI faster thanks to batching
* move ta_in, tstdin into existing tosproc
* move ta_out,tafalse,texitcode,tstderr into existing tosproc
* joinable osproc
* move tstdout into existing tosproc
* spec: batchable; fix tests
* fixup

testament/categories.nim

@@ -679,8 +679,9 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
     quit 1
   else:
     echo "megatest output OK"
-    removeFile(outputGottenFile)
-    removeFile(megatestFile)
+    when false: # no point removing those, always good for debugging
+      removeFile(outputGottenFile)
+      removeFile(megatestFile) # keep it around
   #testSpec r, makeTest("megatest", options, cat)
 
 # ---------------------------------------------------------------------------

testament/specs.nim

@@ -8,6 +8,15 @@
 #
 
 import sequtils, parseutils, strutils, os, streams, parsecfg
+from hashes import hash
+
+type TestamentData* = ref object
+  # better to group globals under 1 object; could group the other ones here too
+  batchArg*: string
+  testamentNumBatch*: int
+  testamentBatch*: int
+
+let testamentData0* = TestamentData()
 
 var compilerPrefix* = findExe("nim")
 
@@ -68,11 +77,13 @@ type
     ccodeCheck*: string
     maxCodeSize*: int
     err*: TResultEnum
+    inCurrentBatch*: bool
     targets*: set[TTarget]
     matrix*: seq[string]
     nimout*: string
     parseErrors*: string # when the spec definition is invalid, this is not empty.
     unjoinable*: bool
+    unbatchable*: bool
     useValgrind*: bool
     timeout*: float # in seconds, fractions possible,
                     # but don't rely on much precision
@@ -138,6 +149,12 @@ proc addLine*(self: var string; a,b: string) =
 proc initSpec*(filename: string): TSpec =
   result.file = filename
 
+proc isCurrentBatch(testamentData: TestamentData, filename: string): bool =
+  if testamentData.testamentNumBatch != 0:
+    hash(filename) mod testamentData.testamentNumBatch == testamentData.testamentBatch
+  else:
+    true
+
 proc parseSpec*(filename: string): TSpec =
   result.file = filename
   let specStr = extractSpec(filename)
@@ -203,6 +220,8 @@ proc parseSpec*(filename: string): TSpec =
         result.action = actionReject
       of "nimout":
         result.nimout = e.value
+      of "batchable":
+        result.unbatchable = not parseCfgBool(e.value)
       of "joinable":
         result.unjoinable = not parseCfgBool(e.value)
       of "valgrind":
@@ -297,3 +316,7 @@ proc parseSpec*(filename: string): TSpec =
 
   if skips.anyIt(it in result.file):
     result.err = reDisabled
+
+  result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable
+  if not result.inCurrentBatch:
+    result.err = reDisabled

testament/testament.nim

@@ -273,12 +273,14 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
                             expected = expected,
                             given = given)
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
+  template disp(msg) =
+    maybeStyledEcho styleDim, fgYellow, msg & " ", styleBright, fgCyan, name
   if success == reSuccess:
     maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)"
   elif success == reDisabled:
-    maybeStyledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
-  elif success == reJoined:
-    maybeStyledEcho styleDim, fgYellow, "JOINED: ", styleBright, fgCyan, name
+    if test.spec.inCurrentBatch: disp("SKIP:")
+    else: disp("NOTINBATCH:")
+  elif success == reJoined: disp("JOINED:")
   else:
     maybeStyledEcho styleBright, fgRed, failString, fgCyan, name
     maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
@@ -645,6 +647,15 @@ proc main() =
         useColors = false
       else:
         quit Usage
+    of "batch":
+      testamentData0.batchArg = p.val
+      if p.val != "_":
+        let s = p.val.split("_")
+        doAssert s.len == 2, $(p.val, s)
+        testamentData0.testamentBatch = s[0].parseInt
+        testamentData0.testamentNumBatch = s[1].parseInt
+        doAssert testamentData0.testamentNumBatch > 0
+        doAssert testamentData0.testamentBatch >= 0 and testamentData0.testamentBatch < testamentData0.testamentNumBatch
     of "simulate":
       simulate = true
     of "megatest":
@@ -682,6 +693,7 @@ proc main() =
       myself &= " " & quoteShell("--targets:" & targetsStr)
 
     myself &= " " & quoteShell("--nim:" & compilerPrefix)
+    myself &= " --batch:" & testamentData0.batchArg
 
     if skipFrom.len > 0:
       myself &= " " & quoteShell("--skipFrom:" & skipFrom)