testament: catch failing nimble tests (#10832)

* testament: catch failing nimble tests and clean up a bit
* fix name collision

koch.nim

@@ -474,7 +474,7 @@ proc runCI(cmd: string) =
     kochExecFold("boot -d:release -d:nimHasLibFFI", "boot -d:release -d:nimHasLibFFI")
 
   if getEnv("NIM_TEST_PACKAGES", "false") == "true":
-    execFold("Test selected Nimble packages", "nim c -r testament/tester cat nimble-extra")
+    execFold("Test selected Nimble packages", "nim c -r testament/tester cat nimble-packages")
   else:
     buildTools() # altenatively, kochExec "tools --toolsNoNimble"
 

testament/categories.nim

@@ -26,9 +26,7 @@ const
     "lib",
     "longgc",
     "manyloc",
-    "nimble-all",
+    "nimble-packages",
-    "nimble-core",
-    "nimble-extra",
     "niminaction",
     "rodfiles",
     "threads",
@@ -443,17 +441,11 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
     testSpec r, testObj
 
 # ----------------------------- nimble ----------------------------------------
-type
-  PackageFilter = enum
-    pfCoreOnly
-    pfExtraOnly
-    pfAll
 
 var nimbleDir = getEnv("NIMBLE_DIR").string
 if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
 let
   nimbleExe = findExe("nimble")
-  #packageDir = nimbleDir / "pkgs" # not used
   packageIndex = nimbleDir / "packages_official.json"
 
 proc waitForExitEx(p: Process): int =
@@ -475,26 +467,21 @@ proc getPackageDir(package: string): string =
   else:
     result = commandOutput[0].string
 
-iterator listPackages(filter: PackageFilter):
+iterator listPackages(): tuple[name, url, cmd: string, hasDeps: bool] =
-                      tuple[name, url, cmd: string, hasDeps: bool] =
+  let defaultCmd = "nimble test"
-  const defaultCmd = "nimble test"
   let packageList = parseFile(packageIndex)
-  for package in packageList.items:
-    if package.hasKey("url"):
-      let name = package["name"].str
-      if name notin ["nimble", "compiler"]:
-        let url = package["url"].str
-        case filter
-        of pfCoreOnly:
-          if "nim-lang" in normalize(url):
-            yield (name, url, defaultCmd, false)
-        of pfExtraOnly:
   for n, cmd, commit, hasDeps in important_packages.packages.items:
+    var found = false
+    for package in packageList.items:
+      let name = package["name"].str
       if name == n:
+        found = true
+        let url = package["url"].str
         let cmd = if cmd.len == 0: defaultCmd else: cmd
         yield (name, url, cmd, hasDeps)
-        of pfAll:
+        break
-          yield (name, url, defaultCmd, false)
+    if not found:
+      raise newException(ValueError, "Cannot find package '$#'." % n)
 
 proc makeSupTest(test, options: string, cat: Category): TTest =
   result.cat = cat
@@ -502,20 +489,20 @@ proc makeSupTest(test, options: string, cat: Category): TTest =
   result.options = options
   result.startTime = epochTime()
 
-proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
+proc testNimblePackages(r: var TResults, cat: Category) =
   if nimbleExe == "":
-    echo("[Warning] - Cannot run nimble tests: Nimble binary not found.")
+    echo "[Warning] - Cannot run nimble tests: Nimble binary not found."
     return
-
   if execCmd("$# update" % nimbleExe) == QuitFailure:
-    echo("[Warning] - Cannot run nimble tests: Nimble update failed.")
+    echo "[Warning] - Cannot run nimble tests: Nimble update failed."
     return
 
   let packageFileTest = makeSupTest("PackageFileParsed", "", cat)
-  var keepDir = false
+  let packagesDir = "pkgstemp"
-  var packagesDir = "pkgstemp"
+  var errors = 0
   try:
-    for name, url, cmd, hasDep in listPackages(filter):
+    for name, url, cmd, hasDep in listPackages():
+      inc r.total
       var test = makeSupTest(url, "", cat)
       let buildPath = packagesDir / name
       if not existsDir(buildPath):
@@ -525,8 +512,7 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
           let nimbleStatus = waitForExitEx(nimbleProcess)
           nimbleProcess.close
           if nimbleStatus != QuitSuccess:
-            r.addResult(test, targetC, "", "", reInstallFailed)
+            r.addResult(test, targetC, "", "'nimble install' failed", reInstallFailed)
-            keepDir = true
             continue
 
         let installProcess = startProcess("git", "", ["clone", url, buildPath],
@@ -534,8 +520,7 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
         let installStatus = waitForExitEx(installProcess)
         installProcess.close
         if installStatus != QuitSuccess:
-          r.addResult(test, targetC, "", "", reInstallFailed)
+          r.addResult(test, targetC, "", "'git clone' failed", reInstallFailed)
-          keepDir = true
           continue
 
       let cmdArgs = parseCmdLine(cmd)
@@ -543,17 +528,26 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
                                       options = {poUsePath, poStdErrToStdOut})
       let buildStatus = waitForExitEx(buildProcess)
       buildProcess.close
+
       if buildStatus != QuitSuccess:
-        r.addResult(test, targetC, "", "", reBuildFailed)
+        r.addResult(test, targetC, "", "package test failed", reBuildFailed)
-        keepDir = true
       else:
+        inc r.passed
         r.addResult(test, targetC, "", "", reSuccess)
+    errors = r.total - r.passed
+    if errors == 0:
       r.addResult(packageFileTest, targetC, "", "", reSuccess)
-  except JsonParsingError:
+    else:
-    echo("[Warning] - Cannot run nimble tests: Invalid package file.")
       r.addResult(packageFileTest, targetC, "", "", reBuildFailed)
+
+  except JsonParsingError:
+    echo "[Warning] - Cannot run nimble tests: Invalid package file."
+    r.addResult(packageFileTest, targetC, "", "Invalid package file", reBuildFailed)
+  except ValueError:
+    echo "[Warning] - $#" % getCurrentExceptionMsg()
+    r.addResult(packageFileTest, targetC, "", "Unknown package", reBuildFailed)
   finally:
-    if not keepDir: removeDir(packagesDir)
+    if errors == 0: removeDir(packagesDir)
 
 
 # ----------------------------------------------------------------------------
@@ -726,12 +720,8 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
     compileExample(r, "examples/*.nim", options, cat)
     compileExample(r, "examples/gtk/*.nim", options, cat)
     compileExample(r, "examples/talk/*.nim", options, cat)
-  of "nimble-core":
+  of "nimble-packages":
-    testNimblePackages(r, cat, pfCoreOnly)
+    testNimblePackages(r, cat)
-  of "nimble-extra":
-    testNimblePackages(r, cat, pfExtraOnly)
-  of "nimble-all":
-    testNimblePackages(r, cat, pfAll)
   of "niminaction":
     testNimInAction(r, cat, options)
   of "untestable":

testament/important_packages.nim

@@ -10,9 +10,9 @@ pkg "arraymancer", "nim c src/arraymancer.nim", "", true
 pkg "ast_pattern_matching", "nim c tests/test1.nim"
 pkg "blscurve", "", "", true
 pkg "bncurve", "nim c tests/tarith.nim", "", true
-pkg "c2nim"
+pkg "c2nim", "nim c testsuite/tester.nim"
 pkg "chronicles", "nim c -o:chr chronicles.nim"
-pkg "chronos", "nim c tests/teststream.nim"
+# pkg "chronos", "nim c tests/teststream.nim"
 pkg "cligen", "nim c -o:cligenn cligen.nim"
 pkg "compactdict", "nim c tests/test1.nim"
 pkg "criterion"
@@ -37,7 +37,7 @@ pkg "nimongo", "nimble test_ci", "", true
 pkg "nimpy", "nim c tests/nimfrompy.nim"
 pkg "nimsl", "nim c test.nim"
 pkg "nimsvg"
-pkg "nimx", "nim c --threads:on test/main.nim", "", true
+# pkg "nimx", "nim c --threads:on test/main.nim", "", true
 pkg "parsetoml"
 pkg "patty"
 pkg "plotly", "nim c examples/all.nim", "", true