From 091da5c9792b9400ee64618774f2ee736b0d5132 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 13 Mar 2019 11:52:15 +0100 Subject: [PATCH] testament: catch failing nimble tests (#10832) * testament: catch failing nimble tests and clean up a bit * fix name collision --- koch.nim | 2 +- testament/categories.nim | 88 ++++++++++++++------------------ testament/important_packages.nim | 6 +-- 3 files changed, 43 insertions(+), 53 deletions(-) diff --git a/koch.nim b/koch.nim index 3698db212..2a65b8c34 100644 --- a/koch.nim +++ b/koch.nim @@ -474,7 +474,7 @@ proc runCI(cmd: string) = kochExecFold("boot -d:release -d:nimHasLibFFI", "boot -d:release -d:nimHasLibFFI") if getEnv("NIM_TEST_PACKAGES", "false") == "true": - execFold("Test selected Nimble packages", "nim c -r testament/tester cat nimble-extra") + execFold("Test selected Nimble packages", "nim c -r testament/tester cat nimble-packages") else: buildTools() # altenatively, kochExec "tools --toolsNoNimble" diff --git a/testament/categories.nim b/testament/categories.nim index 1d6c342a1..eecdd7bc9 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -26,9 +26,7 @@ const "lib", "longgc", "manyloc", - "nimble-all", - "nimble-core", - "nimble-extra", + "nimble-packages", "niminaction", "rodfiles", "threads", @@ -443,17 +441,11 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = testSpec r, testObj # ----------------------------- nimble ---------------------------------------- -type - PackageFilter = enum - pfCoreOnly - pfExtraOnly - pfAll var nimbleDir = getEnv("NIMBLE_DIR").string if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" let nimbleExe = findExe("nimble") - #packageDir = nimbleDir / "pkgs" # not used packageIndex = nimbleDir / "packages_official.json" proc waitForExitEx(p: Process): int = @@ -475,26 +467,21 @@ proc getPackageDir(package: string): string = else: result = commandOutput[0].string -iterator listPackages(filter: PackageFilter): - tuple[name, url, cmd: string, hasDeps: bool] = - const defaultCmd = "nimble test" +iterator listPackages(): tuple[name, url, cmd: string, hasDeps: bool] = + let defaultCmd = "nimble test" let packageList = parseFile(packageIndex) - for package in packageList.items: - if package.hasKey("url"): + for n, cmd, commit, hasDeps in important_packages.packages.items: + var found = false + for package in packageList.items: let name = package["name"].str - if name notin ["nimble", "compiler"]: + if name == n: + found = true let url = package["url"].str - case filter - of pfCoreOnly: - if "nim-lang" in normalize(url): - yield (name, url, defaultCmd, false) - of pfExtraOnly: - for n, cmd, commit, hasDeps in important_packages.packages.items: - if name == n: - let cmd = if cmd.len == 0: defaultCmd else: cmd - yield (name, url, cmd, hasDeps) - of pfAll: - yield (name, url, defaultCmd, false) + let cmd = if cmd.len == 0: defaultCmd else: cmd + yield (name, url, cmd, hasDeps) + break + if not found: + raise newException(ValueError, "Cannot find package '$#'." % n) proc makeSupTest(test, options: string, cat: Category): TTest = result.cat = cat @@ -502,20 +489,20 @@ proc makeSupTest(test, options: string, cat: Category): TTest = result.options = options result.startTime = epochTime() -proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = +proc testNimblePackages(r: var TResults, cat: Category) = if nimbleExe == "": - echo("[Warning] - Cannot run nimble tests: Nimble binary not found.") + echo "[Warning] - Cannot run nimble tests: Nimble binary not found." return - if execCmd("$# update" % nimbleExe) == QuitFailure: - echo("[Warning] - Cannot run nimble tests: Nimble update failed.") + echo "[Warning] - Cannot run nimble tests: Nimble update failed." return let packageFileTest = makeSupTest("PackageFileParsed", "", cat) - var keepDir = false - var packagesDir = "pkgstemp" + let packagesDir = "pkgstemp" + var errors = 0 try: - for name, url, cmd, hasDep in listPackages(filter): + for name, url, cmd, hasDep in listPackages(): + inc r.total var test = makeSupTest(url, "", cat) let buildPath = packagesDir / name if not existsDir(buildPath): @@ -525,8 +512,7 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = let nimbleStatus = waitForExitEx(nimbleProcess) nimbleProcess.close if nimbleStatus != QuitSuccess: - r.addResult(test, targetC, "", "", reInstallFailed) - keepDir = true + r.addResult(test, targetC, "", "'nimble install' failed", reInstallFailed) continue let installProcess = startProcess("git", "", ["clone", url, buildPath], @@ -534,8 +520,7 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = let installStatus = waitForExitEx(installProcess) installProcess.close if installStatus != QuitSuccess: - r.addResult(test, targetC, "", "", reInstallFailed) - keepDir = true + r.addResult(test, targetC, "", "'git clone' failed", reInstallFailed) continue let cmdArgs = parseCmdLine(cmd) @@ -543,17 +528,26 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = options = {poUsePath, poStdErrToStdOut}) let buildStatus = waitForExitEx(buildProcess) buildProcess.close + if buildStatus != QuitSuccess: - r.addResult(test, targetC, "", "", reBuildFailed) - keepDir = true + r.addResult(test, targetC, "", "package test failed", reBuildFailed) else: + inc r.passed r.addResult(test, targetC, "", "", reSuccess) - r.addResult(packageFileTest, targetC, "", "", reSuccess) + errors = r.total - r.passed + if errors == 0: + r.addResult(packageFileTest, targetC, "", "", reSuccess) + else: + r.addResult(packageFileTest, targetC, "", "", reBuildFailed) + except JsonParsingError: - echo("[Warning] - Cannot run nimble tests: Invalid package file.") - r.addResult(packageFileTest, targetC, "", "", reBuildFailed) + echo "[Warning] - Cannot run nimble tests: Invalid package file." + r.addResult(packageFileTest, targetC, "", "Invalid package file", reBuildFailed) + except ValueError: + echo "[Warning] - $#" % getCurrentExceptionMsg() + r.addResult(packageFileTest, targetC, "", "Unknown package", reBuildFailed) finally: - if not keepDir: removeDir(packagesDir) + if errors == 0: removeDir(packagesDir) # ---------------------------------------------------------------------------- @@ -726,12 +720,8 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string, compileExample(r, "examples/*.nim", options, cat) compileExample(r, "examples/gtk/*.nim", options, cat) compileExample(r, "examples/talk/*.nim", options, cat) - of "nimble-core": - testNimblePackages(r, cat, pfCoreOnly) - of "nimble-extra": - testNimblePackages(r, cat, pfExtraOnly) - of "nimble-all": - testNimblePackages(r, cat, pfAll) + of "nimble-packages": + testNimblePackages(r, cat) of "niminaction": testNimInAction(r, cat, options) of "untestable": diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 44a429a99..39bcba2f2 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -10,9 +10,9 @@ pkg "arraymancer", "nim c src/arraymancer.nim", "", true pkg "ast_pattern_matching", "nim c tests/test1.nim" pkg "blscurve", "", "", true pkg "bncurve", "nim c tests/tarith.nim", "", true -pkg "c2nim" +pkg "c2nim", "nim c testsuite/tester.nim" pkg "chronicles", "nim c -o:chr chronicles.nim" -pkg "chronos", "nim c tests/teststream.nim" +# pkg "chronos", "nim c tests/teststream.nim" pkg "cligen", "nim c -o:cligenn cligen.nim" pkg "compactdict", "nim c tests/test1.nim" pkg "criterion" @@ -37,7 +37,7 @@ pkg "nimongo", "nimble test_ci", "", true pkg "nimpy", "nim c tests/nimfrompy.nim" pkg "nimsl", "nim c test.nim" pkg "nimsvg" -pkg "nimx", "nim c --threads:on test/main.nim", "", true +# pkg "nimx", "nim c --threads:on test/main.nim", "", true pkg "parsetoml" pkg "patty" pkg "plotly", "nim c examples/all.nim", "", true