Add "safety compilation". Improved, fixed messages

- node ext searcher also looks for "iojs" if no node(js?) found.
- koch ensures the local work dir compiler is freshly compiled before executing the tests.
- the tester can only run in the repo dir - now the compiler and libs used are also explicitly the local dirs - so no confusion from system wide pathing ("sufficiently sandboxed").
- fixed expectations in tmitems.nim test to match changes in json mod.
- re-layouted the columns / row printing slightly, making test fails pop out more, and everything "normal" layed back.

compiler/nodejs.nim

@@ -4,3 +4,5 @@ proc findNodeJs*(): string =
   result = findExe("nodejs")
   if result == "":
     result = findExe("node")
+  if result == "":
+    result = findExe("iojs")

koch.nim

@@ -330,12 +330,12 @@ proc winRelease() =
 template `|`(a, b): expr = (if a.len > 0: a else: b)
 
 proc tests(args: string) =
-  # Since tests take a long time (on my machine) - lets make sure a stupid 
-  # mistake - like forgetting to compile the compiler - isn't made...
-  exec "nim c --lib:lib -d:release compiler/nim.nim"
   # we compile the tester with taintMode:on to have a basic
   # taint mode test :-)
   exec "nim cc --taintMode:on tests/testament/tester"
+  # Since tests take a long time (on my machine), and we want to defy Murhpys 
+  # law - lets make sure the compiler really is freshly compiled!
+  exec "nim c --lib:lib -d:release --opt:speed compiler/nim.nim"
   let tester = quoteShell(getCurrentDir() / "tests/testament/tester".exe)
   let success = tryExec tester & " " & (args|"all")
   exec tester & " html"

tests/stdlib/tmitems.nim

@@ -11,8 +11,8 @@ fpqeew
 [11, 12, 13]
 [11, 12, 13]
 [11, 12, 13]
-{"key1": 11, "key2": 12, "key3": 13}
+{"key1":11,"key2":12,"key3":13}
-[11, 12, 13]
+[11,12,13]
 <Students>
   <Student Name="Aprilfoo" />
   <Student Name="bar" />

tests/testament/tester.nim

@@ -12,7 +12,7 @@
 import
   parseutils, strutils, pegs, os, osproc, streams, parsecfg, json,
   marshal, backend, parseopt, specs, htmlgen, browsers, terminal,
-  algorithm, compiler/nodejs
+  algorithm, compiler/nodejs, re
 
 const
   resultsFile = "testresults.html"
@@ -126,6 +126,9 @@ proc `$`(x: TResults): string =
             "Tests skipped: $2 / $3 <br />\n") %
             [$x.passed, $x.skipped, $x.total]
 
+proc leftAlign(s: string, count: Natural, padding = ' '): string =
+  s & repeat(padding, max(0, count - s.len))
+
 proc addResult(r: var TResults, test: TTest,
                expected, given: string, success: TResultEnum) =
   let name = test.name.extractFilename & test.options
@@ -137,14 +140,19 @@ proc addResult(r: var TResults, test: TTest,
                           expected = expected,
                           given = given)
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
-  if success == reIgnored:
+  let alignedName = leftAlign(name, 72) # Make the total line length 78 chars
-    styledEcho styleBright, name, fgYellow, " [", $success, "]"
+  if success == reSuccess:
-  elif success != reSuccess:
+    styledEcho fgCyan, alignedName, fgGreen, " [PASS]"
-    styledEcho styleBright, name, fgRed, " [", $success, "]"
+  elif success == reIgnored:
-    echo"Expected:"
+    styledEcho styleBright, fgCyan, alignedName, styleDim, fgYellow, " [SKIP]"
-    styledEcho styleBright, expected
+  else:
-    echo"Given:"
+    styledEcho styleBright, fgCyan, alignedName, fgRed, " [FAIL]"
-    styledEcho styleBright, given
+    styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
+    styledEcho styleBright, fgRed, "Failure: ", $success
+    styledEcho fgYellow, "Expected:"
+    styledEcho styleBright, expected, "\n"
+    styledEcho fgYellow, "Gotten:"
+    styledEcho styleBright, given, "\n"
 
 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) =
   if strip(expected.msg) notin strip(given.msg):
@@ -211,68 +219,100 @@ proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec;
   if given.err == reSuccess: inc(r.passed)
   r.addResult(test, expectedmsg, givenmsg, given.err)
 
+proc analyzeAndConsolidateOutput(s: string): string =
+  result = ""
+  let rows = s.splitLines
+  for i in 0 ..< rows.len:
+    if (let pos = find(rows[i], "Traceback (most recent call last)"); pos != -1):
+      result = substr(rows[i], pos) & "\n"
+      for i in i+1 ..< rows.len:
+        result.add rows[i] & "\n"
+        if not (rows[i] =~ re"^[^(]+\(\d+\)\s+"):
+          return
+    elif (let pos = find(rows[i], "SIGSEGV: Illegal storage access."); pos != -1):
+      result = substr(rows[i], pos)
+      return
+
 proc testSpec(r: var TResults, test: TTest) =
   # major entry point for a single test
   let tname = test.name.addFileExt(".nim")
   inc(r.total)
-  styledEcho "Processing ", fgCyan, extractFilename(tname)
   var expected: TSpec
   if test.action != actionRunNoSpec:
     expected = parseSpec(tname)
   else:
     specDefaults expected
     expected.action = actionRunNoSpec
+
   if expected.err == reIgnored:
     r.addResult(test, "", "", reIgnored)
     inc(r.skipped)
-  else:
+    return
-    case expected.action
+
-    of actionCompile:
+  case expected.action
-      var given = callCompiler(expected.cmd, test.name,
+  of actionCompile:
-        test.options & " --hint[Path]:off --hint[Processing]:off", test.target)
+    var given = callCompiler(expected.cmd, test.name,
-      compilerOutputTests(test, given, expected, r)
+      test.options & " --hint[Path]:off --hint[Processing]:off", test.target)
-    of actionRun, actionRunNoSpec:
+    compilerOutputTests(test, given, expected, r)
-      var given = callCompiler(expected.cmd, test.name, test.options,
+  of actionRun, actionRunNoSpec:
-                               test.target)
+    # In this branch of code "early return" pattern is clearer than deep
-      if given.err != reSuccess:
+    # nested conditionals - the empty rows in between to clarify the "danger"
-        r.addResult(test, "", given.msg, given.err)
+    var given = callCompiler(expected.cmd, test.name, test.options,
-      else:
+                             test.target)
-        var exeFile: string
+
-        if test.target == targetJS:
+    if given.err != reSuccess:
-          let (dir, file, ext) = splitFile(tname)
+      r.addResult(test, "", given.msg, given.err)
-          exeFile = dir / "nimcache" / file & ".js"
+      return
-        else:
+
-          exeFile = changeFileExt(tname, ExeExt)
+    let isJsTarget = test.target == targetJS
-        if existsFile(exeFile):
+    var exeFile: string
-          let nodejs = findNodeJs()
+    if isJsTarget:
-          if test.target == targetJS and nodejs == "":
+      let (dir, file, ext) = splitFile(tname)
-            r.addResult(test, expected.outp, "nodejs binary not in PATH",
+      exeFile = dir / "nimcache" / file & ".js" # *TODO* hardcoded "nimcache"
-                        reExeNotFound)
+    else:
-            return
+      exeFile = changeFileExt(tname, ExeExt)
-          var (buf, exitCode) = execCmdEx(
+
-            (if test.target == targetJS: nodejs & " " else: "") & exeFile)
+    if not existsFile(exeFile):
-          if exitCode != expected.exitCode:
+      r.addResult(test, expected.outp, "executable not found", reExeNotFound)
-            r.addResult(test, "exitcode: " & $expected.exitCode,
+      return
-                              "exitcode: " & $exitCode, reExitCodesDiffer)
+
-          else:
+    let nodejs = if isJsTarget: findNodeJs() else: ""
-            var bufB = strip(buf.string)
+    if isJsTarget and nodejs == "":
-            if expected.sortoutput: bufB = makeDeterministic(bufB)
+      r.addResult(test, expected.outp, "nodejs binary not in PATH",
-            if bufB != strip(expected.outp):
+                  reExeNotFound)
-              if not (expected.substr and expected.outp in bufB):
+      return
-                given.err = reOutputsDiffer
+
-            compilerOutputTests(test, given, expected, r)
+    let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile
-        else:
+    let (buf, exitCode) = execCmdEx(exeCmd)
-          r.addResult(test, expected.outp, "executable not found", reExeNotFound)
+    let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string))
-    of actionReject:
+               else: strip(buf.string)
-      var given = callCompiler(expected.cmd, test.name, test.options,
+
-                               test.target)
+    if exitCode != expected.exitCode:
-      cmpMsgs(r, expected, given, test)
+      r.addResult(test, "exitcode: " & $expected.exitCode,
+                        "exitcode: " & $exitCode & "\n\nOutput:\n" & 
+                        analyzeAndConsolidateOutput(bufB), 
+                        reExitCodesDiffer)
+      return
+
+    if bufB != strip(expected.outp):
+      if not (expected.substr and expected.outp in bufB):
+        given.err = reOutputsDiffer
+        r.addResult(test, expected.outp, bufB, reOutputsDiffer)
+        return
+
+    compilerOutputTests(test, given, expected, r)
+    return
+
+  of actionReject:
+    var given = callCompiler(expected.cmd, test.name, test.options,
+                             test.target)
+    cmpMsgs(r, expected, given, test)
+    return
 
 proc testNoSpec(r: var TResults, test: TTest) =
   # does not extract the spec because the file is not supposed to have any
   let tname = test.name.addFileExt(".nim")
   inc(r.total)
-  styledEcho "Processing ", fgCyan, extractFilename(tname)
   let given = callCompiler(cmdTemplate, test.name, test.options, test.target)
   r.addResult(test, "", given.msg, given.err)
   if given.err == reSuccess: inc(r.passed)