fixes #2517

tests/testament/categories.nim

@@ -226,7 +226,7 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
   for test in os.walkFiles(pattern):
     let contents = readFile(test).string
     if contents.contains("when isMainModule"):
-      testSpec r, makeTest(test, options, cat, actionRun)
+      testSpec r, makeTest(test, options, cat, actionRunNoSpec)
     else:
       testNoSpec r, makeTest(test, options, cat, actionCompile)
 

tests/testament/specs.nim

@@ -10,13 +10,14 @@
 import parseutils, strutils, os, osproc, streams, parsecfg
 
 const
-  cmdTemplate* = r"nim $target --hints:on $options $file"
+  cmdTemplate* = r"nim $target --hints:on -d:testing $options $file"
 
 type
   TTestAction* = enum
     actionCompile = "compile"
     actionRun = "run"
     actionReject = "reject"
+    actionRunNoSpec = "runNoSpec"
   TResultEnum* = enum
     reNimcCrash,     # nim compiler seems to have crashed
     reMsgsDiffer,       # error messages differ
@@ -78,7 +79,7 @@ proc extractSpec(filename: string): string =
 when not defined(nimhygiene):
   {.pragma: inject.}
 
-template parseSpecAux(fillResult: stmt) {.immediate.} =
+template parseSpecAux(fillResult: untyped) =
   var ss = newStringStream(extractSpec(filename))
   var p {.inject.}: CfgParser
   open(p, ss, filename, 1)
@@ -92,8 +93,7 @@ template parseSpecAux(fillResult: stmt) {.immediate.} =
       fillResult
   close(p)
 
-proc parseSpec*(filename: string): TSpec =
-  result.file = filename
+proc specDefaults*(result: var TSpec) =
   result.msg = ""
   result.outp = ""
   result.nimout = ""
@@ -101,6 +101,10 @@ proc parseSpec*(filename: string): TSpec =
   result.cmd = cmdTemplate
   result.line = 0
   result.column = 0
+
+proc parseSpec*(filename: string): TSpec =
+  specDefaults(result)
+  result.file = filename
   parseSpecAux:
     case normalize(e.key)
     of "action":
@@ -112,7 +116,7 @@ proc parseSpec*(filename: string): TSpec =
     of "file": result.file = e.value
     of "line": discard parseInt(e.value, result.line)
     of "column": discard parseInt(e.value, result.column)
-    of "output": 
+    of "output":
       result.action = actionRun
       result.outp = e.value
     of "outputsub":
@@ -121,7 +125,7 @@ proc parseSpec*(filename: string): TSpec =
       result.substr = true
     of "sortoutput":
       result.sortoutput = parseCfgBool(e.value)
-    of "exitcode": 
+    of "exitcode":
       discard parseInt(e.value, result.exitCode)
     of "msg":
       result.msg = e.value

tests/testament/tester.nim

@@ -196,7 +196,12 @@ proc testSpec(r: var TResults, test: TTest) =
   let tname = test.name.addFileExt(".nim")
   inc(r.total)
   styledEcho "Processing ", fgCyan, extractFilename(tname)
-  var expected = parseSpec(tname)
+  var expected: TSpec
+  if test.action != actionRunNoSpec:
+    expected = parseSpec(tname)
+  else:
+    specDefaults expected
+    expected.action = actionRunNoSpec
   if expected.err == reIgnored:
     r.addResult(test, "", "", reIgnored)
     inc(r.skipped)
@@ -206,7 +211,7 @@ proc testSpec(r: var TResults, test: TTest) =
       var given = callCompiler(expected.cmd, test.name,
         test.options & " --hint[Path]:off --hint[Processing]:off", test.target)
       compilerOutputTests(test, given, expected, r)
-    of actionRun:
+    of actionRun, actionRunNoSpec:
       var given = callCompiler(expected.cmd, test.name, test.options,
                                test.target)
       if given.err != reSuccess: