testament: fix #16829, fix partially #16830 (#16831)

* fix #16829
* address comments

testament/categories.nim

@@ -644,71 +644,78 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
 proc processCategory(r: var TResults, cat: Category,
                      options, testsDir: string,
                      runJoinableTests: bool) =
-  case cat.string.normalize
+  let cat2 = cat.string.normalize
-  of "js":
+  var handled = false
-    # only run the JS tests on Windows or Linux because Travis is bad
+  if isNimRepoTests():
-    # and other OSes like Haiku might lack nodejs:
+    handled = true
-    if not defined(linux) and isTravis:
+    case cat2
-      discard
+    of "js":
-    else:
+      # only run the JS tests on Windows or Linux because Travis is bad
-      jsTests(r, cat, options)
+      # and other OSes like Haiku might lack nodejs:
-  of "dll":
+      if not defined(linux) and isTravis:
-    dllTests(r, cat, options)
+        discard
-  of "flags":
-    flagTests(r, cat, options)
-  of "gc":
-    gcTests(r, cat, options)
-  of "longgc":
-    longGCTests(r, cat, options)
-  of "debugger":
-    debuggerTests(r, cat, options)
-  of "manyloc":
-    manyLoc r, cat, options
-  of "threads":
-    threadTests r, cat, options & " --threads:on"
-  of "io":
-    ioTests r, cat, options
-  of "async":
-    asyncTests r, cat, options
-  of "lib":
-    testStdlib(r, "lib/pure/", options, cat)
-    testStdlib(r, "lib/packages/docutils/", options, cat)
-  of "examples":
-    compileExample(r, "examples/*.nim", options, cat)
-    compileExample(r, "examples/gtk/*.nim", options, cat)
-    compileExample(r, "examples/talk/*.nim", options, cat)
-  of "nimble-packages-1":
-    testNimblePackages(r, cat, options, ppOne)
-  of "nimble-packages-2":
-    testNimblePackages(r, cat, options, ppTwo)
-  of "niminaction":
-    testNimInAction(r, cat, options)
-  of "untestable":
-    # We can't test it because it depends on a third party.
-    discard # TODO: Move untestable tests to someplace else, i.e. nimble repo.
-  of "megatest":
-    runJoinedTest(r, cat, testsDir)
-  else:
-    var testsRun = 0
-    var files: seq[string]
-    for file in walkDirRec(testsDir &.? cat.string):
-      if isTestFile(file): files.add file
-    files.sort # give reproducible order
-
-    for i, name in files:
-      var test = makeTest(name, options, cat)
-      if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
-        discard "run the test"
       else:
-        test.spec.err = reJoined
+        jsTests(r, cat, options)
-      testSpec r, test
+    of "dll":
-      inc testsRun
+      dllTests(r, cat, options)
-    if testsRun == 0:
+    of "flags":
-      const whiteListedDirs = ["deps", "htmldocs", "pkgs"]
+      flagTests(r, cat, options)
-        # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users
+    of "gc":
-        # that try an old version of choosenim.
+      gcTests(r, cat, options)
-      doAssert cat.string in whiteListedDirs,
+    of "longgc":
-        "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs]
+      longGCTests(r, cat, options)
+    of "debugger":
+      debuggerTests(r, cat, options)
+    of "manyloc":
+      manyLoc r, cat, options
+    of "threads":
+      threadTests r, cat, options & " --threads:on"
+    of "io":
+      ioTests r, cat, options
+    of "async":
+      asyncTests r, cat, options
+    of "lib":
+      testStdlib(r, "lib/pure/", options, cat)
+      testStdlib(r, "lib/packages/docutils/", options, cat)
+    of "examples":
+      compileExample(r, "examples/*.nim", options, cat)
+      compileExample(r, "examples/gtk/*.nim", options, cat)
+      compileExample(r, "examples/talk/*.nim", options, cat)
+    of "nimble-packages-1":
+      testNimblePackages(r, cat, options, ppOne)
+    of "nimble-packages-2":
+      testNimblePackages(r, cat, options, ppTwo)
+    of "niminaction":
+      testNimInAction(r, cat, options)
+    of "untestable":
+      # We can't test it because it depends on a third party.
+      discard # TODO: Move untestable tests to someplace else, i.e. nimble repo.
+    else:
+      handled = false
+  if not handled:
+    case cat2
+    of "megatest":
+      runJoinedTest(r, cat, testsDir)
+    else:
+      var testsRun = 0
+      var files: seq[string]
+      for file in walkDirRec(testsDir &.? cat.string):
+        if isTestFile(file): files.add file
+      files.sort # give reproducible order
+      for i, name in files:
+        var test = makeTest(name, options, cat)
+        if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
+          discard "run the test"
+        else:
+          test.spec.err = reJoined
+        testSpec r, test
+        inc testsRun
+      if testsRun == 0:
+        const whiteListedDirs = ["deps", "htmldocs", "pkgs"]
+          # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users
+          # that try an old version of choosenim.
+        doAssert cat.string in whiteListedDirs,
+          "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs]
 
 proc processPattern(r: var TResults, pattern, options: string; simulate: bool) =
   var testsRun = 0

testament/testament.nim

@@ -54,6 +54,13 @@ On Azure Pipelines, testament will also publish test results via Azure Pipelines
 provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN.
 """ % resultsFile
 
+proc isNimRepoTests(): bool =
+  # this logic could either be specific to cwd, or to some file derived from
+  # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose
+  # the former since it's simpler and also works with `testament all`.
+  let file = "testament"/"testament.nim.cfg"
+  result = file.fileExists
+
 type
   Category = distinct string
   TResults = object
@@ -86,6 +93,11 @@ var targetsSet = false
 
 proc isSuccess(input: string): bool =
   # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs
+  # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we
+  # could handle colors, there would be other issues such as handling other flags
+  # that may appear in user config (eg: `--listFullPaths`).
+  # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config
+  # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory
   input.startsWith("Hint: ") and input.endsWith("[SuccessX]")
 
 proc getFileDir(filename: string): string =
@@ -744,7 +756,8 @@ proc main() =
       let cat = dir[testsDir.len .. ^1]
       if kind == pcDir and cat notin ["testdata", "nimcache"]:
         cats.add cat
-    cats.add AdditionalCategories
+    if isNimRepoTests():
+      cats.add AdditionalCategories
     if useMegatest: cats.add MegaTestCat
 
     var cmds: seq[string]

testament/testament.nim.cfg

@@ -1,3 +1,5 @@
+# don't move this file without updating the logic in `isNimRepoTests`
+
 path = "$nim" # For compiler/nodejs
 -d:ssl # For azure
 # my SSL doesn't have this feature and I don't care: