megatest checks output and nimout

testament/categories.nim

@@ -591,9 +591,10 @@ const specialDisabedTests = [
   "tests/system/trealloc.nim",       # out of memory
   "tests/system/t7894.nim",          # causes out of memory in later tests
   "tests/types/tissues_types.nim",   # causes out of memory with --gc:boehm
+  "tests/pragmas/tused.nim",         # paths in nimout differ when imported
 ]
 
-proc parseAllSpecs(): void =
+proc runJoinedTest(): void =
   var specs: array[TTestAction, seq[TSpec]]
   var specialTests = 0
   var ignoredTests = 0
@@ -601,6 +602,7 @@ proc parseAllSpecs(): void =
   var specsWithCustomCmd: seq[TSpec]
   var specsEarlyExit: seq[TSpec]
   var specsWithInput: seq[TSpec]
+  var specsNonCtarget: seq[TSpec]
 
   for file in os.walkFiles("tests/*/t*.nim"):
 
@@ -643,6 +645,10 @@ proc parseAllSpecs(): void =
       specsWithInput.add spec
       continue
 
+    if card(spec.targets) > 0 and spec.targets != {targetC}:
+      specsNonCtarget.add spec
+      continue
+
     specs[spec.action].add spec
 
   for action, specs in specs.pairs:
@@ -653,16 +659,52 @@ proc parseAllSpecs(): void =
   echo "special: ", specialTests
   echo "ignored: ", ignoredTests
   echo "withInput: ", specsWithInput.len
+  echo "nonCtarget: ", specsNonCtarget.len
 
   var megatest: string
-
   for runSpec in specs[actionRun]:
-    if targetC in runSpec.targets or runSpec.targets == {}:
-      megatest.add "echo \"------------------------------: "
-      megatest.add runSpec.file
-      megatest.add "\"\n"
     megatest.add "import \""
     megatest.add runSpec.file
     megatest.add "\"\n"
 
   writeFile("megatest.nim", megatest)
+
+  let args = ["c", "-d:testing", "--gc:boehm", "megatest.nim"]
+  var (buf, exitCode) = execCmdEx2(command = "nim", args = args, options = {poStdErrToStdOut, poUsePath}, input = "")
+  if exitCode != 0:
+    quit("megatest compilation failed")
+
+  echo "compilation ok"
+
+  var nimoutOK = true
+  for runSpec in specs[actionRun]:
+    for line in runSpec.nimout.splitLines:
+      if buf.find(line) < 0:
+        echo "could not find: ", line
+        echo runSpec.file
+        nimoutOK = false
+
+  if nimoutOK:
+    echo "nimout OK"
+  else:
+    echo "nimout FAIL"
+
+  (buf, exitCode) = execCmdEx2("./megatest", [], {}, "")
+  if exitCode != 0:
+    quit("megatest execution failed")
+
+  echo "run ok"
+
+  var outputOK = true
+  for runSpec in specs[actionRun]:
+    for line in runSpec.output.splitLines:
+      if buf.find(line) < 0:
+        echo "could not find: ", line
+        echo runSpec.file
+        outputOK = false
+  if outputOK:
+    echo "output OK"
+  else:
+    echo "output FAIL"
+
+  removeFile("megatest.nim")

testament/specs.nim

@@ -57,7 +57,7 @@ type
     input*: string
     outputCheck*: TOutputCheck
     sortoutput*: bool
-    outp*: string
+    output*: string
     line*, column*: int
     tfile*: string
     tline*, tcolumn*: int
@@ -161,12 +161,12 @@ proc parseSpec*(filename: string): TSpec =
         discard parseInt(e.value, result.tcolumn)
       of "output":
         result.outputCheck = ocEqual
-        result.outp = strip(e.value)
+        result.output = strip(e.value)
       of "input":
         result.input = e.value
       of "outputsub":
         result.outputCheck = ocSubstr
-        result.outp = strip(e.value)
+        result.output = strip(e.value)
       of "sortoutput":
         try:
           result.sortoutput  = parseCfgBool(e.value)

testament/tester.nim

@@ -141,7 +141,7 @@ proc callCompiler(cmdTemplate, filename, options: string,
   close(p)
   result.msg = ""
   result.file = ""
-  result.outp = ""
+  result.output = ""
   result.line = 0
   result.column = 0
   result.tfile = ""
@@ -173,7 +173,7 @@ proc callCCompiler(cmdTemplate, filename, options: string,
   result.nimout = ""
   result.msg = ""
   result.file = ""
-  result.outp = ""
+  result.output = ""
   result.line = -1
   while outp.readLine(x.TaintedString) or running(p):
     result.nimout.add(x & "\n")
@@ -400,12 +400,12 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
         exeFile = changeFileExt(test.name, ExeExt)
 
       if not existsFile(exeFile):
-        r.addResult(test, target, expected.outp, "executable not found", reExeNotFound)
+        r.addResult(test, target, expected.output, "executable not found", reExeNotFound)
         continue
 
       let nodejs = if isJsTarget: findNodeJs() else: ""
       if isJsTarget and nodejs == "":
-        r.addResult(test, target, expected.outp, "nodejs binary not in PATH",
+        r.addResult(test, target, expected.output, "nodejs binary not in PATH",
                     reExeNotFound)
         continue
 
@@ -438,10 +438,10 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
                           bufB, reExitCodesDiffer)
         continue
 
-      if (expected.outputCheck == ocEqual  and expected.outp !=  bufB) or
+      if (expected.outputCheck == ocEqual  and expected.output !=  bufB) or
-         (expected.outputCheck == ocSubstr and expected.outp notin bufB):
+         (expected.outputCheck == ocSubstr and expected.output notin bufB):
         given.err = reOutputsDiffer
-        r.addResult(test, target, expected.outp, bufB, reOutputsDiffer)
+        r.addResult(test, target, expected.output, bufB, reOutputsDiffer)
         continue
 
       compilerOutputTests(test, target, given, expected, r)
@@ -582,7 +582,7 @@ proc main() =
   of "html":
     generateHtml(resultsFile, optFailing)
   of "stats":
-    parseAllSpecs()
+    runJoinedTest()
   else:
     quit Usage
 

tests/ccgbugs/tmissinginit.nim

@@ -27,4 +27,4 @@ echo bug()[0]
 echo bug()[0]
 echo bug()[0]
 
-when isMainModule: test()
+test()

tests/ccgbugs/tobjconstr_bad_aliasing.nim

@@ -22,5 +22,4 @@ proc dosomething(): seq[TThing] =
 
   result = @[TThing(data: 10, children: result)]
 
-when isMainModule:
 echo($dosomething()[0])

tests/concepts/trandomvars.nim

@@ -41,7 +41,8 @@ proc lift1[A, B](f: proc(a: A): B, r: RandomVar[A]): ClosureVar[B] =
 
   return inner
 
-when isMainModule:
+
+proc main() =
   proc sq(x: float): float = x * x
 
   let
@@ -59,3 +60,4 @@ when isMainModule:
   echo rng.sample(u)
   echo rng.sample(t)
 
+main()

tests/errmsgs/treportunused.nim

@@ -12,6 +12,7 @@ treportunused.nim(27, 5) Hint: 's9' is declared but not used [XDeclaredButNotUse
 treportunused.nim(21, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed]
 treportunused.nim(28, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed]
 '''
+action: compile
 """
 
 # bug #9764

tests/generics/tgeneric3.nim

@@ -430,9 +430,6 @@ iterator keys* [T,D] (n: PNode[T,D]): T =
         i = Path[level].Xi
         inc(i)
 
-
-when isMainModule:
-
 proc test() =
   var oldvalue: int
   var root = internalPut[int, int](nil, 312, 312, oldvalue)
@@ -479,6 +476,4 @@ when isMainModule:
     echo cnt
     #traceTree(root)
 
-
-
 test()

tests/generics/tgenericvariant.nim

@@ -26,9 +26,11 @@ proc safeReadLine(): TMaybe[string] =
   if r == "": return Nothing[string]()
   else: return Just(r)
 
-when isMainModule:
+proc main() =
   var Test = Just("Test")
   echo(Test.value)
   var mSomething = safeReadLine()
   echo(mSomething.value)
   mSomething = safeReadLine()
+
+main()

tests/generics/tparser_generator.nim

@@ -397,7 +397,6 @@ template grammar*[K](Kind, Text, Symbol: typedesc; default: K, code: untyped): t
 template grammar*[K](Kind: typedesc; default: K, code: untyped): typed {.hint[XDeclaredButNotUsed]: off.} =
   grammar(Kind, string, char, default, code)
 
-when isMainModule:
 block:
   type DummyKind = enum dkDefault
   grammar(DummyKind, string, char, dkDefault):

tests/generics/trtree.nim

@@ -589,7 +589,6 @@ proc delete*[M, D: Dim; RT, LT](t: RTree[M, D, RT, LT]; leaf: L[D, RT, LT]): boo
       t.root.parent = nil
     return true
 
-when isMainModule:
 
 var t = [4, 1, 3, 2]
 var xt = 7
@@ -655,4 +654,3 @@ when isMainModule:
 test(1500)
 
 # 651 lines
-

tests/misc/tshadow_magic_type.nim

@@ -26,7 +26,6 @@ proc lrange*(key: string): TRedisList =
   foo.str = key
   result = @[foo]
 
-when isMainModule:
 var p = lrange("mylist")
 for i in items(p):
   echo(i.str)

tests/objects/tobjconstr.nim

@@ -1,14 +1,15 @@
 discard """
-  output: '''(k: kindA, a: (x: "abc", z: [1, 1, 3]), method: ())
+  output: '''
-(k: kindA, a: (x: "abc", z: [1, 2, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 1, 3]), method: ())
-(k: kindA, a: (x: "abc", z: [1, 3, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 2, 3]), method: ())
-(k: kindA, a: (x: "abc", z: [1, 4, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 3, 3]), method: ())
-(k: kindA, a: (x: "abc", z: [1, 5, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 4, 3]), method: ())
-(k: kindA, a: (x: "abc", z: [1, 6, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 5, 3]), method: ())
-(k: kindA, a: (x: "abc", z: [1, 7, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 6, 3]), method: ())
-(k: kindA, a: (x: "abc", z: [1, 8, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 7, 3]), method: ())
-(k: kindA, a: (x: "abc", z: [1, 9, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 8, 3]), method: ())
-(k: kindA, a: (x: "abc", z: [1, 10, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 9, 3]), method: ())
+(k: kindA, a: (x: "abc", z: @[1, 10, 3]), method: ())
 (y: 0, x: 123)
 (y: 678, x: 123)
 (z: 89, y: 0, x: 128)
@@ -16,7 +17,8 @@ discard """
 (y: 678, x: 123)
 (y: 0, x: 123)
 (y: 678, x: 123)
-(y: 123, x: 678)'''
+(y: 123, x: 678)
+'''
 """
 
 type
@@ -32,13 +34,6 @@ type
       a: TArg
       `method`: TEmpty # bug #1791
 
-proc `$`[T](s: seq[T]): string =
-  result = "["
-  for i, x in s:
-    if i > 0: result.add(", ")
-    result.add($x)
-  result.add("]")
-
 proc main() =
   for i in 1..10:
     let d = TDummy(k: kindA, a: TArg(x: "abc", z: @[1,i,3]), `method`: TEmpty())

tests/overload/toverload_issues.nim

@@ -119,7 +119,6 @@ template test(loopCount: int, testBody: untyped): typed =
     test(loopCount, 0, testBody)
     echo "done extraI passed 0"
 
-when isMainModule:
 var
   loops = 0
 
@@ -139,9 +138,6 @@ when isMainModule:
     loops += 1
   echo "test no extra complete, loops=", loops
 
-
-
-
 # bug #2229
 type
   Type1 = object

tests/stdlib/tnativesockets.nim

@@ -1,11 +0,0 @@
-discard """
-outputsub: ""
-"""
-
-import nativesockets, unittest
-
-suite "nativesockets":
-  test "getHostname":
-    let hostname = getHostname()
-    check hostname.len > 0
-    check hostname.len < 64

tests/typerel/texplicitcmp.nim

@@ -24,7 +24,6 @@ proc weird(json_params: Table) =
   sort(f, system.cmp[int])
   outp(f)
 
-when isMainModule:
 var t = @[3, 2, 1]
 sort(t, system.cmp[int])
 outp(t)