testament: fix #16829, fix partially #16830 (#16831)

* fix #16829
* address comments

testament/categories.nim

@@ -644,7 +644,11 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
 proc processCategory(r: var TResults, cat: Category,
                      options, testsDir: string,
                      runJoinableTests: bool) =
-  case cat.string.normalize
+  let cat2 = cat.string.normalize
+  var handled = false
+  if isNimRepoTests():
+    handled = true
+    case cat2
     of "js":
       # only run the JS tests on Windows or Linux because Travis is bad
       # and other OSes like Haiku might lack nodejs:
@@ -686,6 +690,10 @@ proc processCategory(r: var TResults, cat: Category,
     of "untestable":
       # We can't test it because it depends on a third party.
       discard # TODO: Move untestable tests to someplace else, i.e. nimble repo.
+    else:
+      handled = false
+  if not handled:
+    case cat2
     of "megatest":
       runJoinedTest(r, cat, testsDir)
     else:
@@ -694,7 +702,6 @@ proc processCategory(r: var TResults, cat: Category,
       for file in walkDirRec(testsDir &.? cat.string):
         if isTestFile(file): files.add file
       files.sort # give reproducible order
-
       for i, name in files:
         var test = makeTest(name, options, cat)
         if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:

testament/testament.nim

@@ -54,6 +54,13 @@ On Azure Pipelines, testament will also publish test results via Azure Pipelines
 provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN.
 """ % resultsFile
 
+proc isNimRepoTests(): bool =
+  # this logic could either be specific to cwd, or to some file derived from
+  # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose
+  # the former since it's simpler and also works with `testament all`.
+  let file = "testament"/"testament.nim.cfg"
+  result = file.fileExists
+
 type
   Category = distinct string
   TResults = object
@@ -86,6 +93,11 @@ var targetsSet = false
 
 proc isSuccess(input: string): bool =
   # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs
+  # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we
+  # could handle colors, there would be other issues such as handling other flags
+  # that may appear in user config (eg: `--listFullPaths`).
+  # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config
+  # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory
   input.startsWith("Hint: ") and input.endsWith("[SuccessX]")
 
 proc getFileDir(filename: string): string =
@@ -744,6 +756,7 @@ proc main() =
       let cat = dir[testsDir.len .. ^1]
       if kind == pcDir and cat notin ["testdata", "nimcache"]:
         cats.add cat
+    if isNimRepoTests():
       cats.add AdditionalCategories
     if useMegatest: cats.add MegaTestCat
 

testament/testament.nim.cfg

@@ -1,3 +1,5 @@
+# don't move this file without updating the logic in `isNimRepoTests`
+
 path = "$nim" # For compiler/nodejs
 -d:ssl # For azure
 # my SSL doesn't have this feature and I don't care: