testament :show duration also for failed tests; improve tshould_not_work; mitigate #17946 tchannels timeouts (#17947)

* refs #17946; refactor testament test summary, show test duration for failures; increase timeout tchannels

* revert workarounds from https://github.com/nim-lang/Nim/pull/16698 and add allowPrefixMatch optional param to greedyOrderedSubsetLines

* add test

* workaround for yet another testament bug

tests/misc/trunner.nim

@@ -6,6 +6,8 @@ discard """
 ## tests that don't quite fit the mold and are easier to handle via `execCmdEx`
 ## A few others could be added to here to simplify code.
 ## Note: this test is a bit slow but tests a lot of things; please don't disable.
+## Note: if needed, we could use `matrix: "-d:case1; -d:case2"` to split this
+## into several independent tests while retaining the common test helpers.
 
 import std/[strformat,os,osproc,unittest,compilesettings]
 from std/sequtils import toSeq,mapIt

tests/stdlib/tchannels.nim

@@ -1,5 +1,5 @@
 discard """
-  timeout:  5.0 # but typically < 1s
+  timeout:  20.0 # but typically < 1s (in isolation but other tests running in parallel can affect this since based on epochTime)
   disabled: "freebsd"
   matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger"
 """

tests/stdlib/ttestutils.nim

@@ -22,3 +22,18 @@ block: # greedyOrderedSubsetLines
     not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\na5:suffix")
     not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\nprefix:a5")
     not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\na5:suffix")
+
+block: # greedyOrderedSubsetLines with allowPrefixMatch = true
+  template fn(a, b): bool =
+    greedyOrderedSubsetLines(a, b, allowPrefixMatch = true)
+  assertAll:
+    fn("a1\na3", "a0\na1\na2\na3_suffix\na4")
+    not fn("a1\na3", "a0\na1\na2\nprefix_a3\na4")
+    # these are same as above, could be refactored
+    not fn("a3\na1", "a0\na1\na2\na3\na4") # out of order
+    not fn("a1\na5", "a0\na1\na2\na3\na4") # a5 not in lhs
+
+    not fn("a1\na5", "a0\na1\na2\na3\na4\nprefix:a5")
+    fn("a1\na5", "a0\na1\na2\na3\na4\na5:suffix")
+    not fn("a5", "a0\na1\na2\na3\na4\nprefix:a5")
+    fn("a5", "a0\na1\na2\na3\na4\na5:suffix")

tests/testament/tshould_not_work.nim

@@ -1,39 +1,54 @@
 discard """
-cmd: "testament/testament --directory:testament --colors:off --backendLogging:off --nim:$nim category shouldfail"
-action: compile
-nimout: '''
-FAIL: tests/shouldfail/tccodecheck.nim c
+  joinable: false
+"""
+
+const expected = """
+FAIL: tests/shouldfail/tccodecheck.nim
 Failure: reCodegenFailure
 Expected:
 baz
-FAIL: tests/shouldfail/tcolumn.nim c
+FAIL: tests/shouldfail/tcolumn.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/terrormsg.nim c
+FAIL: tests/shouldfail/terrormsg.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/texitcode1.nim c
+FAIL: tests/shouldfail/texitcode1.nim
 Failure: reExitcodesDiffer
-FAIL: tests/shouldfail/tfile.nim c
+FAIL: tests/shouldfail/tfile.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tline.nim c
+FAIL: tests/shouldfail/tline.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/tmaxcodesize.nim c
+FAIL: tests/shouldfail/tmaxcodesize.nim
 Failure: reCodegenFailure
 max allowed size: 1
-FAIL: tests/shouldfail/tnimout.nim c
+FAIL: tests/shouldfail/tnimout.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/tnimoutfull.nim c
+FAIL: tests/shouldfail/tnimoutfull.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/toutput.nim c
+FAIL: tests/shouldfail/toutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/toutputsub.nim c
+FAIL: tests/shouldfail/toutputsub.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/treject.nim c
+FAIL: tests/shouldfail/treject.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tsortoutput.nim c
+FAIL: tests/shouldfail/tsortoutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/ttimeout.nim c
+FAIL: tests/shouldfail/ttimeout.nim
 Failure: reTimeout
-FAIL: tests/shouldfail/tvalgrind.nim c
+FAIL: tests/shouldfail/tvalgrind.nim
 Failure: reExitcodesDiffer
-'''
 """
+
+import std/[os,strformat,osproc]
+import stdtest/testutils
+
+proc main =
+  const nim = getCurrentCompilerExe()
+  # TODO: bin/testament instead? like other tools (eg bin/nim, bin/nimsuggest etc)
+  let testamentExe = "testament/testament"
+  let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail"
+  let (outp, status) = execCmdEx(cmd)
+  doAssert status == 1, $status
+
+  let ok = greedyOrderedSubsetLines(expected, outp, allowPrefixMatch = true)
+  doAssert ok, &"\nexpected:\n{expected}\noutp:\n{outp}"
+main()