testament: apply skip to hard-coded categories (#11168)

* testament: apply skip to hard-coded categories

* allow disabling megatest

testament/categories.nim

@@ -551,7 +551,8 @@ proc testNimblePackages(r: var TResults, cat: Category) =
 
 # ----------------------------------------------------------------------------
 
-const AdditionalCategories = ["debugger", "examples", "lib", "megatest"]
+const AdditionalCategories = ["debugger", "examples", "lib"]
+const MegaTestCat = "megatest"
 
 proc `&.?`(a, b: string): string =
   # candidate for the stdlib?
@@ -679,22 +680,9 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
 
 # ---------------------------------------------------------------------------
 
-proc loadSkipFrom(name: string): seq[string] =
-  # One skip per line, comments start with #
-  # used by `nlvm` (at least)
-  try:
-    for line in lines(name):
-      let sline = line.strip()
-      if sline.len > 0 and not sline.startsWith("#"):
-        result.add sline
-  except:
-    echo "Could not load " & name & ", ignoring"
-
 proc processCategory(r: var TResults, cat: Category,
-                     options, testsDir, skipFrom: string,
+                     options, testsDir: string,
                      runJoinableTests: bool) =
-  let skips = loadSkipFrom(skipFrom)
-
   case cat.string.normalize
   of "rodfiles":
     when false:
@@ -750,9 +738,6 @@ proc processCategory(r: var TResults, cat: Category,
 
     for i, name in files:
       var test = makeTest(name, options, cat)
-      if skips.anyIt(it in name):
-        test.spec.err = reDisabled
-
       if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
         discard "run the test"
       else:

testament/specs.nim

@@ -7,13 +7,15 @@
 #    distribution, for details about the copyright.
 #
 
-import parseutils, strutils, os, osproc, streams, parsecfg
+import sequtils, parseutils, strutils, os, osproc, streams, parsecfg
 
 var compilerPrefix* = findExe("nim")
 
 let isTravis* = existsEnv("TRAVIS")
 let isAppVeyor* = existsEnv("APPVEYOR")
 
+var skips*: seq[string]
+
 type
   TTestAction* = enum
     actionRun = "run"
@@ -245,3 +247,6 @@ proc parseSpec*(filename: string): TSpec =
     of cfgEof:
       break
   close(p)
+
+  if skips.anyIt(it in result.file):
+    result.err = reDisabled

testament/tester.nim

@@ -44,6 +44,7 @@ Options:
   --directory:dir           Change to directory dir before reading the tests or doing anything else.
   --colors:on|off           Turn messagescoloring on|off.
   --backendLogging:on|off   Disable or enable backend logging. By default turned on.
+  --megatest:on|off         Enable or disable megatest. Default is on.
   --skipFrom:file           Read tests to skip from `file` - one test per line, # comments ignored
 """ % resultsFile
 
@@ -380,6 +381,15 @@ proc getTestSpecTarget(): TTarget =
   else:
     return targetC
 
+proc checkDisabled(r: var TResults, test: TTest): bool =
+  if test.spec.err in {reDisabled, reJoined}:
+    # targetC is a lie, but parameter is required
+    r.addResult(test, targetC, "", "", test.spec.err)
+    inc(r.skipped)
+    inc(r.total)
+    return
+  true
+
 proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
   var expected = test.spec
   if expected.parseErrors.len > 0:
@@ -387,12 +397,8 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
     r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec)
     inc(r.total)
     return
-  if expected.err in {reDisabled, reJoined}:
+  if not checkDisabled(r, test): return
-    # targetC is a lie, but parameter is required
+
-    r.addResult(test, targetC, "", "", expected.err)
-    inc(r.skipped)
-    inc(r.total)
-    return
   expected.targets.incl targets
   # still no target specified at all
   if expected.targets == {}:
@@ -472,6 +478,8 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
 
 proc testC(r: var TResults, test: TTest, action: TTestAction) =
   # runs C code. Doesn't support any specs, just goes by exit code.
+  if not checkDisabled(r, test): return
+
   let tname = test.name.addFileExt(".c")
   inc(r.total)
   maybeStyledEcho "Processing ", fgCyan, extractFilename(tname)
@@ -486,6 +494,8 @@ proc testC(r: var TResults, test: TTest, action: TTestAction) =
 
 proc testExec(r: var TResults, test: TTest) =
   # runs executable or script, just goes by exit code
+  if not checkDisabled(r, test): return
+
   inc(r.total)
   let (outp, errC) = execCmdEx(test.options.strip())
   var given: TSpec
@@ -534,6 +544,19 @@ else:
 
 include categories
 
+proc loadSkipFrom(name: string): seq[string] =
+  if name.len() == 0: return
+
+  # One skip per line, comments start with #
+  # used by `nlvm` (at least)
+  try:
+    for line in lines(name):
+      let sline = line.strip()
+      if sline.len > 0 and not sline.startsWith("#"):
+        result.add sline
+  except:
+    echo "Could not load " & name & ", ignoring"
+
 proc main() =
   os.putenv "NIMTEST_COLOR", "never"
   os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
@@ -544,6 +567,7 @@ proc main() =
   var targetsStr = ""
   var isMainProcess = true
   var skipFrom = ""
+  var useMegatest = true
 
   var p = initOptParser()
   p.next()
@@ -569,6 +593,14 @@ proc main() =
         quit Usage
     of "simulate":
       simulate = true
+    of "megatest":
+      case p.val.string:
+      of "on":
+        useMegatest = true
+      of "off":
+        useMegatest = false
+      else:
+        quit Usage
     of "backendlogging":
       case p.val.string:
       of "on":
@@ -608,32 +640,37 @@ proc main() =
       if kind == pcDir and cat notin ["testdata", "nimcache"]:
         cats.add cat
     cats.add AdditionalCategories
+    if useMegatest: cats.add MegaTestCat
 
     var cmds: seq[string]
     for cat in cats:
-      cmds.add(myself & " pcat " & quoteShell(cat) & rest)
+      let runtype = if useMegatest: " pcat " else: " cat "
+      cmds.add(myself & runtype & quoteShell(cat) & rest)
 
     proc progressStatus(idx: int) =
       echo "progress[all]: i: " & $idx & " / " & $cats.len & " cat: " & cats[idx]
 
     if simulate:
+      skips = loadSkipFrom(skipFrom)
       for i, cati in cats:
         progressStatus(i)
-        processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = false)
+        processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false)
     else:
       quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
   of "c", "cat", "category":
+    skips = loadSkipFrom(skipFrom)
     var cat = Category(p.key)
     p.next
-    processCategory(r, cat, p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = true)
+    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true)
   of "pcat":
+    skips = loadSkipFrom(skipFrom)
     # 'pcat' is used for running a category in parallel. Currently the only
     # difference is that we don't want to run joinable tests here as they
     # are covered by the 'megatest' category.
     isMainProcess = false
     var cat = Category(p.key)
     p.next
-    processCategory(r, cat, p.cmdLineRest.string, testsDir, skipFrom, runJoinableTests = false)
+    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false)
   of "r", "run":
     # at least one directory is required in the path, to use as a category name
     let pathParts = split(p.key.string, {DirSep, AltSep})