bugfixes for the tester; the tester now supports running of single tests

lib/pure/osproc.nim

@@ -568,7 +568,7 @@ elif not defined(useNimRtl):
     if waitPid(p.id, p.exitCode, 0) < 0:
       p.exitCode = -3
       OSError()
-    result = int(p.exitCode)
+    result = int(p.exitCode) shr 8
 
   proc peekExitCode(p: PProcess): int =
     if p.exitCode != -3: return p.exitCode
@@ -576,7 +576,7 @@ elif not defined(useNimRtl):
     var b = ret == int(p.id)
     if b: result = -1
     if p.exitCode == -3: result = -1
-    else: result = p.exitCode
+    else: result = p.exitCode.int shr 8
 
   proc inputStream(p: PProcess): PStream =
     var f: TFile
@@ -641,14 +641,12 @@ proc execCmdEx*(command: string, options: set[TProcessOption] = {
   var p = startCmd(command, options)
   var outp = outputStream(p)
   result = (TaintedString"", -1)
-  while not outp.atEnd(outp):
+  while true:
+    result[1] = peekExitCode(p)
+    if result[1] != -1 and outp.atEnd(outp): break
     result[0].string.add(outp.readLine().string)
     result[0].string.add("\n")
-    result[1] = peekExitCode(p)
-    if result[1] != -1: break
   outp.close(outp)
-  if result[1] == -1:
-    result[1] = peekExitCode(p)
   close(p)
 
 

tests/accept/run/tassert.nim

@@ -1,6 +1,7 @@
 discard """
   file: "tassert.nim"
   outputsub: "assertion failure!this shall be always written"
+  exitcode: "1"
 """
 # test assert and exception handling
 

tests/accept/run/tcontinuexc.nim

@@ -1,6 +1,7 @@
 discard """
   file: "tcontinuexc.nim"
   outputsub: "ECcaught"
+  exitcode: "1"
 """
 type
   ESomething = object of E_Base

tests/accept/run/tfloat1.nim

@@ -1,6 +1,7 @@
 discard """
   file: "tfloat1.nim"
   outputsub: "Error: unhandled exception: FPU operation caused an overflow [EFloatOverflow]"
+  exitcode: "1"
 """
 # Test new floating point exceptions
 

tests/accept/run/tfloat2.nim

@@ -1,6 +1,7 @@
 discard """
   file: "tfloat2.nim"
   outputsub: "Error: unhandled exception: FPU operation caused a NaN result [EFloatInvalidOp]"
+  exitcode: "1"
 """
 # Test new floating point exceptions
 

tests/accept/run/toverflw2.nim

@@ -1,11 +1,10 @@
 discard """
   file: "toverflw2.nim"
   outputsub: "Error: unhandled exception: over- or underflow [EOverflow]"
+  exitcode: "1"
 """
 var a : int32 = 2147483647
 var b : int32 = 2147483647
 var c = a + b
 
 
-
-

tests/accept/run/treraise.nim

@@ -1,6 +1,7 @@
 discard """
   file: "treraise.nim"
   outputsub: "Error: unhandled exception: bla [ESomeOtherErr]"
+  exitcode: "1"
 """
 type
   ESomething = object of E_Base

tests/accept/run/tunhandledexc.nim

@@ -1,6 +1,7 @@
 discard """
   file: "tunhandledexc.nim"
   outputsub: "Error: unhandled exception: bla [ESomeOtherErr]"
+  exitcode: "1"
 """
 type
   ESomething = object of E_Base

tests/accept/run/twrongexc.nim

@@ -1,6 +1,7 @@
 discard """
   file: "twrongexc.nim"
   outputsub: "Error: unhandled exception:  [EInvalidValue]"
+  exitcode: "1"
 """
 try:
   raise newException(EInvalidValue, "")

tests/tester.nim

@@ -11,13 +11,14 @@
 
 import
   parseutils, strutils, pegs, os, osproc, streams, parsecfg, browsers, json,
-  marshal, cgi
+  marshal, cgi, parseopt
 
 const
   cmdTemplate = r"nimrod cc --hints:on $# $#"
   resultsFile = "testresults.html"
   jsonFile = "testresults.json"
-  Usage = "usage: tester reject|compile|examples|run|merge [nimrod options]"
+  Usage = "usage: tester reject|compile|examples|run|merge [nimrod options]\n" &
+          "   or: tester test singleTest"
 
 type
   TTestAction = enum
@@ -26,7 +27,7 @@ type
     action: TTestAction
     file, cmd: string
     outp: string
-    line: int
+    line, exitCode: int
     msg: string
     err: bool
     disabled: bool
@@ -93,6 +94,8 @@ proc parseSpec(filename: string): TSpec =
     of "outputsub":
       result.outp = e.value
       result.substr = true
+    of "exitcode": 
+      discard parseInt(e.value, result.exitCode)
     of "errormsg", "msg": result.msg = e.value
     of "disabled": result.disabled = parseCfgBool(e.value)
     of "cmd": result.cmd = e.value
@@ -108,9 +111,7 @@ var
 
 proc callCompiler(cmdTemplate, filename, options: string): TSpec =
   var c = parseCmdLine(cmdTemplate % [options, filename])
-  var a: seq[string] = @[] # slicing is not yet implemented :-(
+  var p = startProcess(command=c[0], args=c[1.. -1],
-  for i in 1 .. c.len-1: add(a, c[i])
-  var p = startProcess(command=c[0], args=a,
                        options={poStdErrToStdOut, poUseShell})
   var outp = p.outputStream
   var s = ""
@@ -200,19 +201,21 @@ proc cmpMsgs(r: var TResults, expected, given: TSpec, test: string) =
     r.addResult(test, expected.msg, given.msg, reSuccess)
     inc(r.passed)
 
+proc rejectSingleTest(r: var TResults, test, options: string) =
+  var t = extractFilename(test)
+  inc(r.total)
+  echo t
+  var expected = parseSpec(test)
+  if expected.disabled:
+    r.addResult(t, "", "", reIgnored)
+    inc(r.skipped)
+  else:
+    var given = callCompiler(expected.cmd, test, options)
+    cmpMsgs(r, expected, given, t)
+
 proc reject(r: var TResults, dir, options: string) =
   ## handle all the tests that the compiler should reject
-  for test in os.walkFiles(dir / "t*.nim"):
+  for test in os.walkFiles(dir / "t*.nim"): rejectSingleTest(r, test, options)
-    var t = extractFilename(test)
-    inc(r.total)
-    echo t
-    var expected = parseSpec(test)
-    if expected.disabled:
-      r.addResult(t, "", "", reIgnored)
-      inc(r.skipped)
-    else:
-      var given = callCompiler(expected.cmd, test, options)
-      cmpMsgs(r, expected, given, t)
 
 proc compile(r: var TResults, pattern, options: string) =
   for test in os.walkFiles(pattern):
@@ -252,8 +255,9 @@ proc runSingleTest(r: var TResults, test, options: string) =
       var exeFile = changeFileExt(test, ExeExt)
       if existsFile(exeFile):
         var (buf, exitCode) = execCmdEx(exeFile)
-        if exitCode != 0:
+        if exitCode != expected.ExitCode:
-          r.addResult(t, expected.outp, "exitCode: " & $exitCode, reFailure)
+          r.addResult(t, "exitcode: " & $expected.ExitCode,
+                         "exitcode: " & $exitCode, reFailure)
         else:
           var success = strip(buf.string) == strip(expected.outp)
           if expected.substr and not success: 
@@ -322,9 +326,9 @@ proc compileRodFiles(r: var TResults, options: string) =
 
 # --------------------- DLL generation tests ----------------------------------
 
-proc runBasicDLLTest(r: var TResults, options: string) =
+proc runBasicDLLTest(c, r: var TResults, options: string) =
-  compileSingleTest r, "lib/nimrtl.nim", options & " --app:lib -d:createNimRtl"
+  compileSingleTest c, "lib/nimrtl.nim", options & " --app:lib -d:createNimRtl"
-  compileSingleTest r, "tests/dll/server.nim", 
+  compileSingleTest c, "tests/dll/server.nim", 
     options & " --app:lib -d:useNimRtl"
   
   when defined(Windows): 
@@ -342,10 +346,13 @@ proc runBasicDLLTest(r: var TResults, options: string) =
   runSingleTest r, "tests/dll/client.nim", options & " -d:useNimRtl"
 
 proc runDLLTests(r: var TResults, options: string) =
-  runBasicDLLTest(r, options)
+  # dummy compile result:
-  runBasicDLLTest(r, options & " -d:release")
+  var c = initResults()
-  runBasicDLLTest(r, options & " --gc:boehm")
+  
-  runBasicDLLTest(r, options & " -d:release --gc:boehm")
+  runBasicDLLTest c, r, options
+  runBasicDLLTest c, r, options & " -d:release"
+  runBasicDLLTest c, r, options & " --gc:boehm"
+  runBasicDLLTest c, r, options & " -d:release --gc:boehm"
   
   
 # -----------------------------------------------------------------------------
@@ -367,38 +374,39 @@ proc outputJSON(reject, compile, run: TResults) =
   var s = pretty(doc)
   writeFile(jsonFile, s)
 
-proc main(action: string) =
+proc main() =
   const
     compileJson = "compile.json"
     runJson = "run.json"
     rejectJson = "reject.json"
-  var options = ""
+  
-  for i in 2.. paramCount():
+  var p = initOptParser()
-    add(options, " ")
+  p.next()
-    add(options, paramStr(i).string)
+  if p.kind != cmdArgument: quit usage
-
+  var action = p.key.string.normalize
+  p.next()
   case action
   of "reject":
     var rejectRes = initResults()
-    reject(rejectRes, "tests/reject", options)
+    reject(rejectRes, "tests/reject", p.cmdLineRest.string)
     writeResults(rejectJson, rejectRes)
   of "compile":
     var compileRes = initResults()
-    compile(compileRes, "tests/accept/compile/t*.nim", options)
+    compile(compileRes, "tests/accept/compile/t*.nim", p.cmdLineRest.string)
-    compile(compileRes, "tests/ecmas.nim", options)
+    compile(compileRes, "tests/ecmas.nim", p.cmdLineRest.string)
-    compileRodFiles(compileRes, options)
+    compileRodFiles(compileRes, p.cmdLineRest.string)
     writeResults(compileJson, compileRes)
   of "examples":
     var compileRes = readResults(compileJson)
-    compileExample(compileRes, "lib/pure/*.nim", options)
+    compileExample(compileRes, "lib/pure/*.nim", p.cmdLineRest.string)
-    compileExample(compileRes, "examples/*.nim", options)
+    compileExample(compileRes, "examples/*.nim", p.cmdLineRest.string)
-    compileExample(compileRes, "examples/gtk/*.nim", options)
+    compileExample(compileRes, "examples/gtk/*.nim", p.cmdLineRest.string)
     writeResults(compileJson, compileRes)
   of "run":
     var runRes = initResults()
-    run(runRes, "tests/accept/run", options)
+    run(runRes, "tests/accept/run", p.cmdLineRest.string)
-    runRodFiles(runRes, options)
+    runRodFiles(runRes, p.cmdLineRest.string)
-    runDLLTests(runRes, options)
+    runDLLTests(runRes, p.cmdLineRest.string)
     writeResults(runJson, runRes)
   of "merge":
     var rejectRes = readResults(rejectJson)
@@ -406,14 +414,22 @@ proc main(action: string) =
     var runRes = readResults(runJson)
     listResults(rejectRes, compileRes, runRes)
     outputJSON(rejectRes, compileRes, runRes)
-  of "dll":
+  of "test":
-    var runRes = initResults()
+    var r = initResults()
-    runDLLTests runRes, ""
+    if p.kind != cmdArgument: quit usage
-    writeResults(runJson, runRes)
+    var testFile = p.key.string
+    p.next()
+    if peg"'/reject/'" in testFile:
+      reject(r, testFile, p.cmdLineRest.string)
+    elif peg"'/compile/'" in testFile:
+       compileSingleTest(r, testFile, p.cmdLineRest.string)
+    else:
+      runSingleTest(r, testFile, p.cmdLineRest.string)
+    echo r
   else:
     quit usage
 
 if paramCount() == 0:
   quit usage
-main(paramStr(1).string)
+main()