Merge pull request #2862 from ozra/bugfix-2858-testament-sources-system-wide

Bugfix 2858 Testament Sources Files System Wide

compiler/nodejs.nim

@@ -4,3 +4,5 @@ proc findNodeJs*(): string =
   result = findExe("nodejs")
   if result == "":
     result = findExe("node")
+  if result == "":
+    result = findExe("iojs")

koch.nim

@@ -333,6 +333,9 @@ proc tests(args: string) =
   # we compile the tester with taintMode:on to have a basic
   # taint mode test :-)
   exec "nim cc --taintMode:on tests/testament/tester"
+  # Since tests take a long time (on my machine), and we want to defy Murhpys 
+  # law - lets make sure the compiler really is freshly compiled!
+  exec "nim c --lib:lib -d:release --opt:speed compiler/nim.nim"
   let tester = quoteShell(getCurrentDir() / "tests/testament/tester".exe)
   let success = tryExec tester & " " & (args|"all")
   exec tester & " html"

tests/testament/specs.nim

@@ -10,7 +10,7 @@
 import parseutils, strutils, os, osproc, streams, parsecfg
 
 const
-  cmdTemplate* = r"nim $target --hints:on -d:testing $options $file"
+  cmdTemplate* = r"compiler" / "nim $target --lib:lib --hints:on -d:testing $options $file"
 
 type
   TTestAction* = enum

tests/testament/tester.nim

@@ -12,7 +12,7 @@
 import
   parseutils, strutils, pegs, os, osproc, streams, parsecfg, json,
   marshal, backend, parseopt, specs, htmlgen, browsers, terminal,
-  algorithm, compiler/nodejs
+  algorithm, compiler/nodejs, re
 
 const
   resultsFile = "testresults.html"
@@ -137,14 +137,18 @@ proc addResult(r: var TResults, test: TTest,
                           expected = expected,
                           given = given)
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
-  if success == reIgnored:
+  if success == reSuccess:
-    styledEcho styleBright, name, fgYellow, " [", $success, "]"
+    styledEcho fgGreen, "PASS: ", fgCyan, name
-  elif success != reSuccess:
+  elif success == reIgnored:
-    styledEcho styleBright, name, fgRed, " [", $success, "]"
+    styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
-    echo"Expected:"
+  else:
-    styledEcho styleBright, expected
+    styledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
-    echo"Given:"
+    styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
-    styledEcho styleBright, given
+    styledEcho styleBright, fgRed, "Failure: ", $success
+    styledEcho fgYellow, "Expected:"
+    styledEcho styleBright, expected, "\n"
+    styledEcho fgYellow, "Gotten:"
+    styledEcho styleBright, given, "\n"
 
 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) =
   if strip(expected.msg) notin strip(given.msg):
@@ -211,68 +215,101 @@ proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec;
   if given.err == reSuccess: inc(r.passed)
   r.addResult(test, expectedmsg, givenmsg, given.err)
 
+proc analyzeAndConsolidateOutput(s: string): string =
+  result = ""
+  let rows = s.splitLines
+  for i in 0 ..< rows.len:
+    if (let pos = find(rows[i], "Traceback (most recent call last)"); pos != -1):
+      result = substr(rows[i], pos) & "\n"
+      for i in i+1 ..< rows.len:
+        result.add rows[i] & "\n"
+        if not (rows[i] =~ re"^[^(]+\(\d+\)\s+"):
+          return
+    elif (let pos = find(rows[i], "SIGSEGV: Illegal storage access."); pos != -1):
+      result = substr(rows[i], pos)
+      return
+
 proc testSpec(r: var TResults, test: TTest) =
   # major entry point for a single test
   let tname = test.name.addFileExt(".nim")
   inc(r.total)
-  styledEcho "Processing ", fgCyan, extractFilename(tname)
   var expected: TSpec
   if test.action != actionRunNoSpec:
     expected = parseSpec(tname)
   else:
     specDefaults expected
     expected.action = actionRunNoSpec
+
   if expected.err == reIgnored:
     r.addResult(test, "", "", reIgnored)
     inc(r.skipped)
-  else:
+    return
+
   case expected.action
   of actionCompile:
     var given = callCompiler(expected.cmd, test.name,
       test.options & " --hint[Path]:off --hint[Processing]:off", test.target)
     compilerOutputTests(test, given, expected, r)
   of actionRun, actionRunNoSpec:
+    # In this branch of code "early return" pattern is clearer than deep
+    # nested conditionals - the empty rows in between to clarify the "danger"
     var given = callCompiler(expected.cmd, test.name, test.options,
                              test.target)
+
     if given.err != reSuccess:
       r.addResult(test, "", given.msg, given.err)
-      else:
+      return
+
+    let isJsTarget = test.target == targetJS
     var exeFile: string
-        if test.target == targetJS:
+    if isJsTarget:
       let (dir, file, ext) = splitFile(tname)
-          exeFile = dir / "nimcache" / file & ".js"
+      exeFile = dir / "nimcache" / file & ".js" # *TODO* hardcoded "nimcache"
     else:
       exeFile = changeFileExt(tname, ExeExt)
-        if existsFile(exeFile):
+
-          let nodejs = findNodeJs()
+    if not existsFile(exeFile):
-          if test.target == targetJS and nodejs == "":
+      r.addResult(test, expected.outp, "executable not found", reExeNotFound)
+      return
+
+    let nodejs = if isJsTarget: findNodeJs() else: ""
+    if isJsTarget and nodejs == "":
       r.addResult(test, expected.outp, "nodejs binary not in PATH",
                   reExeNotFound)
       return
-          var (buf, exitCode) = execCmdEx(
+
-            (if test.target == targetJS: nodejs & " " else: "") & exeFile)
+    let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile
+    let (buf, exitCode) = execCmdEx(exeCmd)
+    let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string))
+               else: strip(buf.string)
+    let expectedOut = strip(expected.outp)
+
     if exitCode != expected.exitCode:
       r.addResult(test, "exitcode: " & $expected.exitCode,
-                              "exitcode: " & $exitCode, reExitCodesDiffer)
+                        "exitcode: " & $exitCode & "\n\nOutput:\n" & 
-          else:
+                        analyzeAndConsolidateOutput(bufB), 
-            var bufB = strip(buf.string)
+                        reExitCodesDiffer)
-            if expected.sortoutput: bufB = makeDeterministic(bufB)
+      return
-            if bufB != strip(expected.outp):
+
-              if not (expected.substr and expected.outp in bufB):
+    if bufB != expectedOut:
+      if not (expected.substr and expectedOut in bufB):
         given.err = reOutputsDiffer
+        r.addResult(test, expected.outp, bufB, reOutputsDiffer)
+        return
+
     compilerOutputTests(test, given, expected, r)
-        else:
+    return
-          r.addResult(test, expected.outp, "executable not found", reExeNotFound)
+
   of actionReject:
     var given = callCompiler(expected.cmd, test.name, test.options,
                              test.target)
     cmpMsgs(r, expected, given, test)
+    return
 
 proc testNoSpec(r: var TResults, test: TTest) =
   # does not extract the spec because the file is not supposed to have any
   let tname = test.name.addFileExt(".nim")
   inc(r.total)
-  styledEcho "Processing ", fgCyan, extractFilename(tname)
   let given = callCompiler(cmdTemplate, test.name, test.options, test.target)
   r.addResult(test, "", given.msg, given.err)
   if given.err == reSuccess: inc(r.passed)