tester: add test skipping capability (#11080)

testament/categories.nim

@@ -11,6 +11,7 @@
 ## of the compiler.
 
 import important_packages
+import sequtils
 
 const
   specialCategories = [
@@ -688,8 +689,22 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
 
 # ---------------------------------------------------------------------------
 
-proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
+proc loadSkipFrom(name: string): seq[string] =
+  # One skip per line, comments start with #
+  # used by `nlvm` (at least)
+  try:
+    for line in lines(name):
+      let sline = line.strip()
+      if sline.len > 0 and not sline.startsWith("#"):
+        result.add sline
+  except:
+    echo "Could not load " & name & ", ignoring"
+
+proc processCategory(r: var TResults, cat: Category,
+                     options, testsDir, skipFrom: string,
                      runJoinableTests: bool) =
+  let skips = loadSkipFrom(skipFrom)
+
   case cat.string.normalize
   of "rodfiles":
     when false:
@@ -745,6 +760,9 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
 
     for i, name in files:
       var test = makeTest(name, options, cat)
+      if skips.anyIt(it in name):
+        test.spec.err = reDisabled
+
       if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
         discard "run the test"
       else:

testament/tester.nim

@@ -48,6 +48,7 @@ Options:
   --directory:dir           Change to directory dir before reading the tests or doing anything else.
   --colors:on|off           Turn messagescoloring on|off.
   --backendLogging:on|off   Disable or enable backend logging. By default turned on.
+  --skipFrom:file           Read tests to skip from `file` - one test per line, # comments ignored
 """ % resultsFile
 
 type
@@ -545,6 +546,7 @@ proc main() =
   var optFailing = false
   var targetsStr = ""
   var isMainProcess = true
+  var skipFrom = ""
 
   var p = initOptParser()
   p.next()
@@ -580,6 +582,8 @@ proc main() =
         backendLogging = false
       else:
         quit Usage
+    of "skipfrom":
+      skipFrom = p.val.string
     else:
       quit Usage
     p.next()
@@ -598,6 +602,9 @@ proc main() =
 
     myself &= " " & quoteShell("--nim:" & compilerPrefix)
 
+    if skipFrom.len > 0:
+      myself &= " " & quoteShell("--skipFrom:" & skipFrom)
+
     var cats: seq[string]
     let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: ""
     for kind, dir in walkDir(testsDir):
@@ -617,13 +624,13 @@ proc main() =
     if simulate:
       for i, cati in cats:
         progressStatus(i)
-        processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false)
+        processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = false)
     else:
       quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
   of "c", "cat", "category":
     var cat = Category(p.key)
     p.next
-    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true)
+    processCategory(r, cat, p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = true)
   of "pcat":
     # 'pcat' is used for running a category in parallel. Currently the only
     # difference is that we don't want to run joinable tests here as they
@@ -631,7 +638,7 @@ proc main() =
     isMainProcess = false
     var cat = Category(p.key)
     p.next
-    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false)
+    processCategory(r, cat, p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = false)
   of "r", "run":
     # at least one directory is required in the path, to use as a category name
     let pathParts = split(p.key.string, {DirSep, AltSep})