megatest checks output and nimout

testament/categories.nim

@@ -591,9 +591,10 @@ const specialDisabedTests = [
   "tests/system/trealloc.nim",       # out of memory
   "tests/system/t7894.nim",          # causes out of memory in later tests
   "tests/types/tissues_types.nim",   # causes out of memory with --gc:boehm
+  "tests/pragmas/tused.nim",         # paths in nimout differ when imported
 ]
 
-proc parseAllSpecs(): void =
+proc runJoinedTest(): void =
   var specs: array[TTestAction, seq[TSpec]]
   var specialTests = 0
   var ignoredTests = 0
@@ -601,6 +602,7 @@ proc parseAllSpecs(): void =
   var specsWithCustomCmd: seq[TSpec]
   var specsEarlyExit: seq[TSpec]
   var specsWithInput: seq[TSpec]
+  var specsNonCtarget: seq[TSpec]
 
   for file in os.walkFiles("tests/*/t*.nim"):
 
@@ -643,6 +645,10 @@ proc parseAllSpecs(): void =
       specsWithInput.add spec
       continue
 
+    if card(spec.targets) > 0 and spec.targets != {targetC}:
+      specsNonCtarget.add spec
+      continue
+
     specs[spec.action].add spec
 
   for action, specs in specs.pairs:
@@ -653,16 +659,52 @@ proc parseAllSpecs(): void =
   echo "special: ", specialTests
   echo "ignored: ", ignoredTests
   echo "withInput: ", specsWithInput.len
+  echo "nonCtarget: ", specsNonCtarget.len
 
   var megatest: string
-
   for runSpec in specs[actionRun]:
-    if targetC in runSpec.targets or runSpec.targets == {}:
-      megatest.add "echo \"------------------------------: "
-      megatest.add runSpec.file
-      megatest.add "\"\n"
-      megatest.add "import \""
-      megatest.add runSpec.file
-      megatest.add "\"\n"
+    megatest.add "import \""
+    megatest.add runSpec.file
+    megatest.add "\"\n"
 
   writeFile("megatest.nim", megatest)
+
+  let args = ["c", "-d:testing", "--gc:boehm", "megatest.nim"]
+  var (buf, exitCode) = execCmdEx2(command = "nim", args = args, options = {poStdErrToStdOut, poUsePath}, input = "")
+  if exitCode != 0:
+    quit("megatest compilation failed")
+
+  echo "compilation ok"
+
+  var nimoutOK = true
+  for runSpec in specs[actionRun]:
+    for line in runSpec.nimout.splitLines:
+      if buf.find(line) < 0:
+        echo "could not find: ", line
+        echo runSpec.file
+        nimoutOK = false
+
+  if nimoutOK:
+    echo "nimout OK"
+  else:
+    echo "nimout FAIL"
+
+  (buf, exitCode) = execCmdEx2("./megatest", [], {}, "")
+  if exitCode != 0:
+    quit("megatest execution failed")
+
+  echo "run ok"
+
+  var outputOK = true
+  for runSpec in specs[actionRun]:
+    for line in runSpec.output.splitLines:
+      if buf.find(line) < 0:
+        echo "could not find: ", line
+        echo runSpec.file
+        outputOK = false
+  if outputOK:
+    echo "output OK"
+  else:
+    echo "output FAIL"
+
+  removeFile("megatest.nim")

testament/specs.nim

@@ -57,7 +57,7 @@ type
     input*: string
     outputCheck*: TOutputCheck
     sortoutput*: bool
-    outp*: string
+    output*: string
     line*, column*: int
     tfile*: string
     tline*, tcolumn*: int
@@ -161,12 +161,12 @@ proc parseSpec*(filename: string): TSpec =
         discard parseInt(e.value, result.tcolumn)
       of "output":
         result.outputCheck = ocEqual
-        result.outp = strip(e.value)
+        result.output = strip(e.value)
       of "input":
         result.input = e.value
       of "outputsub":
         result.outputCheck = ocSubstr
-        result.outp = strip(e.value)
+        result.output = strip(e.value)
       of "sortoutput":
         try:
           result.sortoutput  = parseCfgBool(e.value)

testament/tester.nim

@@ -141,7 +141,7 @@ proc callCompiler(cmdTemplate, filename, options: string,
   close(p)
   result.msg = ""
   result.file = ""
-  result.outp = ""
+  result.output = ""
   result.line = 0
   result.column = 0
   result.tfile = ""
@@ -173,7 +173,7 @@ proc callCCompiler(cmdTemplate, filename, options: string,
   result.nimout = ""
   result.msg = ""
   result.file = ""
-  result.outp = ""
+  result.output = ""
   result.line = -1
   while outp.readLine(x.TaintedString) or running(p):
     result.nimout.add(x & "\n")
@@ -400,12 +400,12 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
         exeFile = changeFileExt(test.name, ExeExt)
 
       if not existsFile(exeFile):
-        r.addResult(test, target, expected.outp, "executable not found", reExeNotFound)
+        r.addResult(test, target, expected.output, "executable not found", reExeNotFound)
         continue
 
       let nodejs = if isJsTarget: findNodeJs() else: ""
       if isJsTarget and nodejs == "":
-        r.addResult(test, target, expected.outp, "nodejs binary not in PATH",
+        r.addResult(test, target, expected.output, "nodejs binary not in PATH",
                     reExeNotFound)
         continue
 
@@ -438,10 +438,10 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
                           bufB, reExitCodesDiffer)
         continue
 
-      if (expected.outputCheck == ocEqual  and expected.outp !=  bufB) or
-         (expected.outputCheck == ocSubstr and expected.outp notin bufB):
+      if (expected.outputCheck == ocEqual  and expected.output !=  bufB) or
+         (expected.outputCheck == ocSubstr and expected.output notin bufB):
         given.err = reOutputsDiffer
-        r.addResult(test, target, expected.outp, bufB, reOutputsDiffer)
+        r.addResult(test, target, expected.output, bufB, reOutputsDiffer)
         continue
 
       compilerOutputTests(test, target, given, expected, r)
@@ -582,7 +582,7 @@ proc main() =
   of "html":
     generateHtml(resultsFile, optFailing)
   of "stats":
-    parseAllSpecs()
+    runJoinedTest()
   else:
     quit Usage