From b809562c7cc38e391c1e3240d8ca77ba035c3f75 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 28 Nov 2020 00:09:31 -0800 Subject: [PATCH] make megatest consistent with unjoined tests wrt newlines, honor newlines in output spec (#16151) * fix megatest newlines * still allow missing trailing newline for now but in a more strict way than before --- testament/categories.nim | 22 ++++---- testament/specs.nim | 2 +- testament/testament.nim | 14 +++-- tests/arc/tstrformat.nim | 4 +- tests/async/tasyncfilewrite.nim | 3 +- tests/ccgbugs/tret_arg_init.nim | 2 + tests/concepts/treversable.nim | 2 +- tests/destructor/ttuple.nim | 2 +- tests/exception/texcas.nim | 5 +- tests/exception/texceptions.nim | 1 + tests/generics/treentranttypes.nim | 8 --- tests/js/tunittests.nim | 4 +- tests/lookups/test.nim | 2 +- tests/macros/tastrepr.nim | 1 + tests/magics/t10307.nim | 5 +- tests/metatype/tmatrix3.nim | 2 +- tests/metatype/ttensor.nim | 6 +- tests/misc/temptyecho.nim | 4 ++ tests/objects/tobject.nim | 2 +- tests/overload/tstatic_with_converter.nim | 5 +- tests/specialops/tdotops.nim | 4 +- tests/statictypes/tpassthruarith.nim | 4 -- tests/stdlib/talgorithm.nim | 2 +- tests/stdlib/tcgi.nim | 1 + tests/stdlib/tmath.nim | 5 +- tests/stdlib/tnet_ll.nim | 1 + tests/stdlib/tnre.nim | 4 +- tests/stdlib/tparseuints.nim | 13 +++-- tests/stdlib/trst.nim | 1 + tests/stdlib/tsequtils.nim | 3 - tests/stdlib/tunittest.nim | 68 +++++++++++------------ tests/system/tenum_array_repr.nim | 2 - tests/template/tgensymregression.nim | 2 - 33 files changed, 110 insertions(+), 96 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 0d19842ec..4f6e40659 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -591,12 +591,14 @@ proc isJoinableSpec(spec: TSpec): bool = if spec.file.readFile.contains "when isMainModule": result = false -proc norm(s: var string) = - while true: - let tmp = s.replace("\n\n", "\n") - if tmp == s: break - s = tmp - s = s.strip +when false: + proc norm(s: var string) = + ## strip empty newlines + while true: + let tmp = s.replace("\n\n", "\n") + if tmp == s: break + s = tmp + s = s.strip proc quoted(a: string): string = # todo: consider moving to system.nim @@ -654,16 +656,16 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = echo buf.string quit(failString & "megatest execution failed") - norm buf.string const outputExceptedFile = "outputExpected.txt" const outputGottenFile = "outputGotten.txt" writeFile(outputGottenFile, buf.string) var outputExpected = "" for i, runSpec in specs: outputExpected.add marker & runSpec.file & "\n" - outputExpected.add runSpec.output.strip - outputExpected.add '\n' - norm outputExpected + if runSpec.output.len > 0: + outputExpected.add runSpec.output + if not runSpec.output.endsWith "\n": + outputExpected.add '\n' if buf.string != outputExpected: writeFile(outputExceptedFile, outputExpected) diff --git a/testament/specs.nim b/testament/specs.nim index eaae598bb..37fe12275 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -280,7 +280,7 @@ proc parseSpec*(filename: string): TSpec = of "output": if result.outputCheck != ocSubstr: result.outputCheck = ocEqual - result.output = strip(e.value) + result.output = e.value of "input": result.input = e.value of "outputsub": diff --git a/testament/testament.nim b/testament/testament.nim index 2912b03ab..94f66c2db 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -469,6 +469,10 @@ proc checkDisabled(r: var TResults, test: TTest): bool = var count = 0 +proc equalModuloLastNewline(a, b: string): bool = + # allow lazy output spec that omits last newline, but really those should be fixed instead + result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] + proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, target: TTarget, nimcache: string, extraOptions = "") = test.startTime = epochTime() @@ -513,16 +517,18 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, if exitCode != 0: exitCode = 1 let bufB = if expected.sortoutput: - var x = splitLines(strip(buf.string)) + var buf2 = buf.string + buf2.stripLineEnd + var x = splitLines(buf2) sort(x, system.cmp) - join(x, "\n") + join(x, "\n") & "\n" else: - strip(buf.string) + buf.string if exitCode != expected.exitCode: r.addResult(test, target, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & bufB, reExitcodesDiffer) - elif (expected.outputCheck == ocEqual and expected.output != bufB) or + elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or (expected.outputCheck == ocSubstr and expected.output notin bufB): given.err = reOutputsDiffer r.addResult(test, target, expected.output, bufB, reOutputsDiffer) diff --git a/tests/arc/tstrformat.nim b/tests/arc/tstrformat.nim index 00d7a6679..641f323da 100644 --- a/tests/arc/tstrformat.nim +++ b/tests/arc/tstrformat.nim @@ -1,5 +1,7 @@ discard """ - output: '''verstuff''' + output: ''' +verstuff +''' cmd: "nim c --gc:arc $file" """ diff --git a/tests/async/tasyncfilewrite.nim b/tests/async/tasyncfilewrite.nim index 3baf2bbc6..72a2df0b0 100644 --- a/tests/async/tasyncfilewrite.nim +++ b/tests/async/tasyncfilewrite.nim @@ -1,7 +1,8 @@ discard """ output: '''string 1 string 2 -string 3''' +string 3 +''' """ # bug #5532 import os, asyncfile, asyncdispatch diff --git a/tests/ccgbugs/tret_arg_init.nim b/tests/ccgbugs/tret_arg_init.nim index 5cd67de3e..e39e5a0de 100644 --- a/tests/ccgbugs/tret_arg_init.nim +++ b/tests/ccgbugs/tret_arg_init.nim @@ -1,6 +1,8 @@ discard """ output: ''' + + ''' """ diff --git a/tests/concepts/treversable.nim b/tests/concepts/treversable.nim index 6ebc077d9..d30ba0a3c 100644 --- a/tests/concepts/treversable.nim +++ b/tests/concepts/treversable.nim @@ -3,7 +3,7 @@ discard """ output: ''' z e - ''' +''' """ type diff --git a/tests/destructor/ttuple.nim b/tests/destructor/ttuple.nim index 5a2126105..d0ea72c60 100644 --- a/tests/destructor/ttuple.nim +++ b/tests/destructor/ttuple.nim @@ -2,7 +2,7 @@ discard """ output: '''5.0 10.0 =destroy -=destroy +=destroy ''' """ diff --git a/tests/exception/texcas.nim b/tests/exception/texcas.nim index 7108e334c..ad6819f11 100644 --- a/tests/exception/texcas.nim +++ b/tests/exception/texcas.nim @@ -1,8 +1,9 @@ discard """ targets: "c cpp" - output: '''Hello + output: ''' Hello - ''' +Hello +''' """ proc test[T]() = try: diff --git a/tests/exception/texceptions.nim b/tests/exception/texceptions.nim index 5d4d0fa9e..adee5d1d5 100644 --- a/tests/exception/texceptions.nim +++ b/tests/exception/texceptions.nim @@ -1,5 +1,6 @@ discard """ output: ''' + BEFORE FINALLY diff --git a/tests/generics/treentranttypes.nim b/tests/generics/treentranttypes.nim index 31fa25293..40ff1647b 100644 --- a/tests/generics/treentranttypes.nim +++ b/tests/generics/treentranttypes.nim @@ -2,21 +2,13 @@ discard """ output: ''' (10, ("test", 1.2)) 3x3 Matrix [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0], [2.0, 0.0, 5.0]] - 2x3 Matrix [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0]] - 2x3 Literal [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0]] - 2x3 Matrix [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] - 2x2 ArrayArray[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] - 2x3 ArrayVector[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] - 2x3 VectorVector [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] - 2x3 VectorArray [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] - @[1, 2] @[1, 2] @[1, 2]@[3, 4] diff --git a/tests/js/tunittests.nim b/tests/js/tunittests.nim index 7c2e70563..0d934a82b 100644 --- a/tests/js/tunittests.nim +++ b/tests/js/tunittests.nim @@ -1,7 +1,9 @@ discard """ output: ''' + [Suite] Bacon - [OK] >:)''' + [OK] >:) +''' """ import unittest diff --git a/tests/lookups/test.nim b/tests/lookups/test.nim index 97dfa131a..dfacaf5b5 100644 --- a/tests/lookups/test.nim +++ b/tests/lookups/test.nim @@ -1,7 +1,7 @@ discard """ output: ''' -[Suite] memoization +[Suite] memoization ''' """ diff --git a/tests/macros/tastrepr.nim b/tests/macros/tastrepr.nim index e4271f019..759ca55b5 100644 --- a/tests/macros/tastrepr.nim +++ b/tests/macros/tastrepr.nim @@ -1,5 +1,6 @@ discard """ output: ''' + var data = @[(1, "one"), (2, "two")] for (i, d) in pairs(data): discard diff --git a/tests/magics/t10307.nim b/tests/magics/t10307.nim index 111b0d221..b5bbfdfa8 100644 --- a/tests/magics/t10307.nim +++ b/tests/magics/t10307.nim @@ -4,7 +4,8 @@ discard """ res: yes yes running someProc(false) -res: +res: + ''' """ @@ -20,4 +21,4 @@ echo "running someProc(true)" echo someProc(true) echo "running someProc(false)" -echo someProc(false) \ No newline at end of file +echo someProc(false) diff --git a/tests/metatype/tmatrix3.nim b/tests/metatype/tmatrix3.nim index a143e2bc9..28e85fcee 100644 --- a/tests/metatype/tmatrix3.nim +++ b/tests/metatype/tmatrix3.nim @@ -1,5 +1,5 @@ discard """ - output: "" + output: "\n" """ type Matrix[M,N: static[int]] = array[M, array[N, float]] diff --git a/tests/metatype/ttensor.nim b/tests/metatype/ttensor.nim index f2f24ba8c..89a0bf007 100644 --- a/tests/metatype/ttensor.nim +++ b/tests/metatype/ttensor.nim @@ -1,13 +1,11 @@ discard """ - output: '''before tensor2: + output: ''' +before tensor2: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0] - before tensor3: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0] - after tensor3: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0] - a3: [1000.0, 1001.0, 1002.0, 1003.0, 1004.0, 1005.0, 1006.0, 1007.0, 1008.0, 1009.0, 1010.0, 1011.0, 1012.0, 1013.0, 1014.0, 1015.0, 1016.0, 1017.0, 1018.0, 1019.0, 1020.0, 1021.0, 1022.0, 1023.0, 1024.0, 1025.0, 1026.0]''' """ diff --git a/tests/misc/temptyecho.nim b/tests/misc/temptyecho.nim index 5f1aa6515..a3c407897 100644 --- a/tests/misc/temptyecho.nim +++ b/tests/misc/temptyecho.nim @@ -1,2 +1,6 @@ +discard """ +output: "\n" +""" + echo() diff --git a/tests/objects/tobject.nim b/tests/objects/tobject.nim index a79c2bdfd..d166d5385 100644 --- a/tests/objects/tobject.nim +++ b/tests/objects/tobject.nim @@ -1,5 +1,5 @@ discard """ -output: "[Suite] object basic methods" +output: "\n[Suite] object basic methods" """ import unittest diff --git a/tests/overload/tstatic_with_converter.nim b/tests/overload/tstatic_with_converter.nim index 2871744eb..8209e8399 100644 --- a/tests/overload/tstatic_with_converter.nim +++ b/tests/overload/tstatic_with_converter.nim @@ -1,6 +1,8 @@ discard """ output: ''' -9.0''' +9.0 + +''' """ ### bug #6773 @@ -41,6 +43,7 @@ proc `^`(x: vfloat, exp: static[float]): vfloat = proc `$`(x: vfloat): string = let y = cast[ptr float](unsafeAddr x) + # xxx not sure if intentional in this issue, but this returns "" echo y[] let x = set1(9.0) diff --git a/tests/specialops/tdotops.nim b/tests/specialops/tdotops.nim index 227204f51..b1c75ab33 100644 --- a/tests/specialops/tdotops.nim +++ b/tests/specialops/tdotops.nim @@ -1,5 +1,6 @@ discard """ output: ''' + 10 assigning z = 20 reading field y @@ -12,7 +13,8 @@ no params call to b 100 one param call to c with 10 100 -0 4''' +0 4 +''' """ type diff --git a/tests/statictypes/tpassthruarith.nim b/tests/statictypes/tpassthruarith.nim index 857e5b0af..33e4e0303 100644 --- a/tests/statictypes/tpassthruarith.nim +++ b/tests/statictypes/tpassthruarith.nim @@ -1,13 +1,9 @@ discard """ output: ''' [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]] - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - [1, 2, 3, 4] ''' """ diff --git a/tests/stdlib/talgorithm.nim b/tests/stdlib/talgorithm.nim index 59336e70d..9dec68f03 100644 --- a/tests/stdlib/talgorithm.nim +++ b/tests/stdlib/talgorithm.nim @@ -1,6 +1,6 @@ discard """ output:'''@["3", "2", "1"] - ''' +''' """ #12928,10456 import sequtils, strutils, algorithm, json diff --git a/tests/stdlib/tcgi.nim b/tests/stdlib/tcgi.nim index 222a3bf6d..cec188e35 100644 --- a/tests/stdlib/tcgi.nim +++ b/tests/stdlib/tcgi.nim @@ -1,5 +1,6 @@ discard """ output: ''' + [Suite] Test cgi module (key: "a", value: "1") (key: "b", value: "0") diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index ff1f32d36..ac5ce4c90 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -1,6 +1,8 @@ discard """ action: run - output: '''[Suite] random int + output: ''' + +[Suite] random int [Suite] random float @@ -9,7 +11,6 @@ discard """ [Suite] random sample [Suite] ^ - ''' """ diff --git a/tests/stdlib/tnet_ll.nim b/tests/stdlib/tnet_ll.nim index 2ac272fd1..affa21947 100644 --- a/tests/stdlib/tnet_ll.nim +++ b/tests/stdlib/tnet_ll.nim @@ -1,6 +1,7 @@ discard """ action: run output: ''' + [Suite] inet_ntop tests ''' """ diff --git a/tests/stdlib/tnre.nim b/tests/stdlib/tnre.nim index 0929956cb..d2dc1a7c5 100644 --- a/tests/stdlib/tnre.nim +++ b/tests/stdlib/tnre.nim @@ -4,7 +4,9 @@ discard """ # codes to differ and be reported as a failure output: - '''[Suite] Test NRE initialization + ''' + +[Suite] Test NRE initialization [Suite] captures diff --git a/tests/stdlib/tparseuints.nim b/tests/stdlib/tparseuints.nim index 6b228d933..72041da66 100644 --- a/tests/stdlib/tparseuints.nim +++ b/tests/stdlib/tparseuints.nim @@ -1,13 +1,14 @@ discard """ action: run output: ''' -[Suite] parseutils''' + +[Suite] parseutils +''' """ import unittest, strutils suite "parseutils": - test "uint": - check: parseBiggestUInt("0") == 0'u64 - check: parseBiggestUInt("18446744073709551615") == 0xFFFF_FFFF_FFFF_FFFF'u64 - expect(ValueError): - discard parseBiggestUInt("18446744073709551616") + check: parseBiggestUInt("0") == 0'u64 + check: parseBiggestUInt("18446744073709551615") == 0xFFFF_FFFF_FFFF_FFFF'u64 + expect(ValueError): + discard parseBiggestUInt("18446744073709551616") diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 797010a22..6a6e6fdc0 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -1,5 +1,6 @@ discard """ output: ''' + [Suite] RST include directive ''' """ diff --git a/tests/stdlib/tsequtils.nim b/tests/stdlib/tsequtils.nim index ea3f06982..efcc9f126 100644 --- a/tests/stdlib/tsequtils.nim +++ b/tests/stdlib/tsequtils.nim @@ -455,6 +455,3 @@ block: yield i doAssert: iter(3).mapIt(2*it).foldl(a + b) == 6 - -when not defined(testing): - echo "Finished doc tests" diff --git a/tests/stdlib/tunittest.nim b/tests/stdlib/tunittest.nim index 9ef689e32..8e82a0c97 100644 --- a/tests/stdlib/tunittest.nim +++ b/tests/stdlib/tunittest.nim @@ -1,5 +1,7 @@ discard """ - output: '''[Suite] suite with only teardown + output: ''' + +[Suite] suite with only teardown [Suite] suite with only setup @@ -16,7 +18,6 @@ discard """ [Suite] test suite [Suite] test name filtering - ''' """ @@ -141,38 +142,37 @@ suite "test suite": check(a == b) -when defined(testing): - suite "test name filtering": - test "test name": - check matchFilter("suite1", "foo", "") - check matchFilter("suite1", "foo", "foo") - check matchFilter("suite1", "foo", "::") - check matchFilter("suite1", "foo", "*") - check matchFilter("suite1", "foo", "::foo") - check matchFilter("suite1", "::foo", "::foo") +suite "test name filtering": + test "test name": + check matchFilter("suite1", "foo", "") + check matchFilter("suite1", "foo", "foo") + check matchFilter("suite1", "foo", "::") + check matchFilter("suite1", "foo", "*") + check matchFilter("suite1", "foo", "::foo") + check matchFilter("suite1", "::foo", "::foo") - test "test name - glob": - check matchFilter("suite1", "foo", "f*") - check matchFilter("suite1", "foo", "*oo") - check matchFilter("suite1", "12345", "12*345") - check matchFilter("suite1", "q*wefoo", "q*wefoo") - check false == matchFilter("suite1", "foo", "::x") - check false == matchFilter("suite1", "foo", "::x*") - check false == matchFilter("suite1", "foo", "::*x") - # overlap - check false == matchFilter("suite1", "12345", "123*345") - check matchFilter("suite1", "ab*c::d*e::f", "ab*c::d*e::f") + test "test name - glob": + check matchFilter("suite1", "foo", "f*") + check matchFilter("suite1", "foo", "*oo") + check matchFilter("suite1", "12345", "12*345") + check matchFilter("suite1", "q*wefoo", "q*wefoo") + check false == matchFilter("suite1", "foo", "::x") + check false == matchFilter("suite1", "foo", "::x*") + check false == matchFilter("suite1", "foo", "::*x") + # overlap + check false == matchFilter("suite1", "12345", "123*345") + check matchFilter("suite1", "ab*c::d*e::f", "ab*c::d*e::f") - test "suite name": - check matchFilter("suite1", "foo", "suite1::") - check false == matchFilter("suite1", "foo", "suite2::") - check matchFilter("suite1", "qwe::foo", "qwe::foo") - check matchFilter("suite1", "qwe::foo", "suite1::qwe::foo") + test "suite name": + check matchFilter("suite1", "foo", "suite1::") + check false == matchFilter("suite1", "foo", "suite2::") + check matchFilter("suite1", "qwe::foo", "qwe::foo") + check matchFilter("suite1", "qwe::foo", "suite1::qwe::foo") - test "suite name - glob": - check matchFilter("suite1", "foo", "::*") - check matchFilter("suite1", "foo", "*::*") - check matchFilter("suite1", "foo", "*::foo") - check false == matchFilter("suite1", "foo", "*ite2::") - check matchFilter("suite1", "q**we::foo", "q**we::foo") - check matchFilter("suite1", "a::b*c::d*e", "a::b*c::d*e") + test "suite name - glob": + check matchFilter("suite1", "foo", "::*") + check matchFilter("suite1", "foo", "*::*") + check matchFilter("suite1", "foo", "*::foo") + check false == matchFilter("suite1", "foo", "*ite2::") + check matchFilter("suite1", "q**we::foo", "q**we::foo") + check matchFilter("suite1", "a::b*c::d*e", "a::b*c::d*e") diff --git a/tests/system/tenum_array_repr.nim b/tests/system/tenum_array_repr.nim index 3634692e3..39b1a5f9a 100644 --- a/tests/system/tenum_array_repr.nim +++ b/tests/system/tenum_array_repr.nim @@ -2,10 +2,8 @@ discard """ output: ''' 1 [a, b] - 2 [c, d] - 4 [e, f]''' """ diff --git a/tests/template/tgensymregression.nim b/tests/template/tgensymregression.nim index f84c96403..2a5dca934 100644 --- a/tests/template/tgensymregression.nim +++ b/tests/template/tgensymregression.nim @@ -1,8 +1,6 @@ discard """ output: '''[0.0, 0.0, 0.0] - [0.0, 0.0, 0.0, 0.0] - 5050 123''' """