new tester; all tests categorized

tests/testament/backend.nim

@@ -0,0 +1,120 @@
+#
+#
+#              The Nimrod Tester
+#        (c) Copyright 2013 Andreas Rumpf
+#
+#    Look at license.txt for more info.
+#    All rights reserved.
+
+import strutils, db_sqlite, os, osproc
+
+var db: TDbConn
+
+proc createDb() =
+  db.exec(sql"""
+    create table if not exists Machine(
+      id integer primary key,
+      name varchar(100) not null,
+      os varchar(20) not null,
+      cpu varchar(20) not null
+    );""")
+
+  db.exec(sql"""
+    create table if not exists [Commit](
+      id integer primary key,
+      hash varchar(256) not null,
+      branch varchar(50) not null
+    );""")
+
+  db.exec(sql"""
+    create table if not exists TestResult(
+      id integer primary key,
+      name varchar(100) not null,
+      category varchar(100) not null,
+      target varchar(20) not null,
+      action varchar(10) not null,
+      result varchar(30) not null,
+      [commit] int not null,
+      machine int not null,
+      expected varchar(10000) not null,
+      given varchar(10000) not null,
+      created timestamp not null default (DATETIME('now')),
+
+      foreign key ([commit]) references [commit](id),
+      foreign key (machine) references machine(id)
+    );""")
+
+  #db.exec(sql"""
+  #  --create unique index if not exists TsstNameIx on TestResult(name);
+  #  """, [])
+
+type
+  MachineId = distinct int64
+  CommitId = distinct int64
+
+proc `$`(id: MachineId): string {.borrow.}
+proc `$`(id: CommitId): string {.borrow.}
+
+var
+  thisMachine: MachineId
+  thisCommit: CommitId
+
+proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip
+
+proc getMachine: MachineId =
+  var name = "hostname"()
+  if name.len == 0:
+    name = when defined(posix): getenv"HOSTNAME".string
+           else: getenv"COMPUTERNAME".string
+  if name.len == 0:
+    quit "cannot determine the machine name"
+
+  let id = db.getValue(sql"select id from Machine where name = ?", name)
+  if id.len > 0:
+    result = id.parseInt.MachineId
+  else:
+    result = db.insertId(sql"insert into Machine(name, os, cpu) values (?,?,?)", 
+                         name, system.hostOS, system.hostCPU).MachineId
+
+proc getCommit: CommitId =
+  let hash = "git log -n 1"()
+  let branch = "git symbolic-ref --short HEAD"()
+  if hash.len == 0 or branch.len == 0: quit "cannot determine git HEAD"
+  
+  let id = db.getValue(sql"select id from [Commit] where hash = ? and branch = ?",
+                       hash, branch)
+  if id.len > 0:
+    result = id.parseInt.CommitId
+  else:
+    result = db.insertId(sql"insert into [Commit](hash, branch) values (?, ?)", 
+                         hash, branch).CommitId
+
+proc writeTestResult*(name, category, target, 
+                      action, result, expected, given: string) =
+  let id = db.getValue(sql"""select id from TestResult 
+                             where name = ? and category = ? and target = ? and
+                                machine = ? and [commit] = ?""", 
+                                name, category, target,
+                                thisMachine, thisCommit)
+  if id.len > 0:
+    db.exec(sql"""update TestResult
+                  set action = ?, result = ?, expected = ?, given = ? 
+                  where id = ?""", action, result, expected, given, id)
+  else:
+    db.exec(sql"""insert into TestResult(name, category, target, 
+                                         action, 
+                                         result, expected, given,
+                                         [commit], machine)
+                  values (?,?,?,?,?,?,?,?,?) """, name, category, target, 
+                                        action,
+                                        result, expected, given, 
+                                        thisCommit, thisMachine)
+
+proc open*() =
+  db = open(connection="testament.db", user="testament", password="",
+            database="testament")
+  createDb()
+  thisMachine = getMachine()
+  thisCommit = getCommit()
+
+proc close*() = close(db)

tests/testament/categories.nim

@@ -0,0 +1,256 @@
+#
+#
+#            Nimrod Tester
+#        (c) Copyright 2014 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Include for the tester that contains test suites that test special features
+## of the compiler.
+
+# included from tester.nim
+# ---------------- ROD file tests ---------------------------------------------
+
+const
+  rodfilesDir = "tests/rodfiles"
+  nimcacheDir = rodfilesDir / "nimcache"
+
+proc delNimCache() =
+  try:
+    removeDir(nimcacheDir)
+  except EOS:
+    echo "[Warning] could not delete: ", nimcacheDir
+    
+proc runRodFiles(r: var TResults, cat: Category, options: string) =
+  template test(filename: expr): stmt =
+    testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun)
+  
+  delNimCache()
+  
+  # test basic recompilation scheme:
+  test "hallo"
+  test "hallo"
+  # test incremental type information:
+  test "hallo2"
+  delNimCache()
+  
+  # test type converters:
+  test "aconv"
+  test "bconv"
+  delNimCache()
+  
+  # test G, A, B example from the documentation; test init sections:
+  test "deada"
+  test "deada2"
+  delNimCache()
+  
+  # test method generation:
+  test "bmethods"
+  test "bmethods2"
+  delNimCache()
+  
+  # test generics:
+  test "tgeneric1"
+  test "tgeneric2"
+  delNimCache()
+
+proc compileRodFiles(r: var TResults, cat: Category, options: string) =
+  template test(filename: expr): stmt =
+    testSpec r, makeTest(rodfilesDir / filename, options, cat)
+
+  delNimCache()
+  # test DLL interfacing:
+  test "gtkex1"
+  test "gtkex2"
+  delNimCache()
+
+# --------------------- DLL generation tests ----------------------------------
+
+proc safeCopyFile(src, dest: string) =
+  try:
+    copyFile(src, dest)
+  except EOS:
+    echo "[Warning] could not copy: ", src, " to ", dest
+
+proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
+  testSpec c, makeTest("lib/nimrtl.nim",
+    options & " --app:lib -d:createNimRtl", cat)
+  testSpec c, makeTest("tests/dll/server.nim",
+    options & " --app:lib -d:useNimRtl", cat)
+  
+  when defined(Windows): 
+    # windows looks in the dir of the exe (yay!):
+    var nimrtlDll = DynlibFormat % "nimrtl"
+    safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
+  else:
+    # posix relies on crappy LD_LIBRARY_PATH (ugh!):
+    var libpath = getenv"LD_LIBRARY_PATH".string
+    if peg"\i '/nimrod' (!'/')* '/lib'" notin libpath:
+      echo "[Warning] insufficient LD_LIBRARY_PATH"
+    var serverDll = DynlibFormat % "server"
+    safeCopyFile("tests/dll" / serverDll, "lib" / serverDll)
+  
+  testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl", 
+                       cat, actionRun)
+
+proc dllTests(r: var TResults, cat: Category, options: string) =
+  # dummy compile result:
+  var c = initResults()
+  
+  runBasicDLLTest c, r, cat, options
+  runBasicDLLTest c, r, cat, options & " -d:release"
+  runBasicDLLTest c, r, cat, options & " --gc:boehm"
+  runBasicDLLTest c, r, cat, options & " -d:release --gc:boehm"
+
+# ------------------------------ GC tests -------------------------------------
+
+proc gcTests(r: var TResults, cat: Category, options: string) =
+  template test(filename: expr): stmt =
+    testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " -d:release", cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " -d:release -d:useRealtimeGC", cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " --gc:markAndSweep", cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " -d:release --gc:markAndSweep", cat, actionRun)
+  
+  test "gcbench"
+  test "gcleak"
+  test "gcleak2"
+  test "gctest"
+  test "gcleak3"
+  test "weakrefs"
+  test "cycleleak"
+  test "closureleak"
+
+# ------------------------- threading tests -----------------------------------
+
+proc threadTests(r: var TResults, cat: Category, options: string) =
+  template test(filename: expr): stmt =
+    testSpec r, makeTest("tests/threads" / filename, options, cat, actionRun)
+    testSpec r, makeTest("tests/threads" / filename, options &
+      " -d:release", cat, actionRun)
+    testSpec r, makeTest("tests/threads" / filename, options &
+      " --tlsEmulation:on", cat, actionRun)
+  
+  test "tactors"
+  test "tactors2"
+  test "threadex"
+  # deactivated because output capturing still causes problems sometimes:
+  #test "trecursive_actor"
+  #test "threadring"
+  #test "tthreadanalysis"
+  #test "tthreadsort"
+  test "tthreadanalysis2"
+  test "tthreadanalysis3"
+  test "tthreadheapviolation1"
+
+# ------------------------- IO tests ------------------------------------------
+
+proc ioTests(r: var TResults, cat: Category, options: string) =
+  # We need readall_echo to be compiled for this test to run.
+  # dummy compile result:
+  var c = initResults()
+  testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat)
+  testSpec r, makeTest("tests/system/io", options, cat)
+
+# ------------------------- debugger tests ------------------------------------
+
+proc debuggerTests(r: var TResults, cat: Category, options: string) =
+  testNoSpec r, makeTest("tools/nimgrep", options & " --debugger:on", cat)
+
+# ------------------------- JS tests ------------------------------------------
+
+proc jsTests(r: var TResults, cat: Category, options: string) =
+  template test(filename: expr): stmt =
+    testSpec r, makeTest(filename, options & " -d:nodejs", cat,
+                         actionRun, targetJS)
+    testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat,
+                         actionRun, targetJS)
+    
+  for t in os.walkFiles("tests/js/t*.nim"):
+    test(t)
+  for testfile in ["texceptions", "texcpt1", "texcsub", "tfinally",
+                   "tfinally2", "tfinally3", "tactiontable", "tmultim1",
+                   "tmultim3", "tmultim4"]:
+    test "tests/run/" & testfile & ".nim"
+
+# ------------------------- manyloc -------------------------------------------
+#proc runSpecialTests(r: var TResults, options: string) =
+#  for t in ["lib/packages/docutils/highlite"]:
+#    testSpec(r, t, options)
+
+proc findMainFile(dir: string): string =
+  # finds the file belonging to ".nimrod.cfg"; if there is no such file
+  # it returns the some ".nim" file if there is only one: 
+  const cfgExt = ".nimrod.cfg"
+  result = ""
+  var nimFiles = 0
+  for kind, file in os.walkDir(dir):
+    if kind == pcFile:
+      if file.endsWith(cfgExt): return file[.. -(cfgExt.len+1)] & ".nim"
+      elif file.endsWith(".nim"):
+        if result.len == 0: result = file
+        inc nimFiles
+  if nimFiles != 1: result.setlen(0)
+
+proc manyLoc(r: var TResults, cat: Category, options: string) =
+  for kind, dir in os.walkDir("tests/manyloc"):
+    if kind == pcDir:
+      let mainfile = findMainFile(dir)
+      if mainfile != ".nim":
+        testNoSpec r, makeTest(mainfile, options, cat)
+
+proc compileExample(r: var TResults, pattern, options: string, cat: Category) =
+  for test in os.walkFiles(pattern):
+    testNoSpec r, makeTest(test, options, cat)
+
+proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
+  for test in os.walkFiles(pattern):
+    let contents = readFile(test).string
+    if contents.contains("when isMainModule"):
+      testSpec r, makeTest(test, options, cat, actionRun)
+    else:
+      testNoSpec r, makeTest(test, options, cat, actionCompile)
+
+# ----------------------------------------------------------------------------
+
+const AdditionalCategories = ["debugger", "tools", "examples", "stdlib"]
+
+proc processCategory(r: var TResults, cat: Category, options: string) =
+  case cat.string.normalize
+  of "rodfiles":
+    compileRodFiles(r, cat, options)
+    runRodFiles(r, cat, options)
+  of "js":
+    # XXX JS doesn't need to be special anymore
+    jsTests(r, cat, options)
+  of "dll":
+    dllTests(r, cat, options)
+  of "gc":
+    gcTests(r, cat, options)
+  of "debugger":
+    debuggerTests(r, cat, options)
+  of "tools":
+    testSpec r, makeTest("compiler/c2nim/c2nim.nim", options, cat)
+    testSpec r, makeTest("compiler/pas2nim/pas2nim.nim", options, cat)
+  of "manyloc":
+    manyLoc r, cat, options
+  of "threads":
+    threadTests r, cat, options & " --threads:on"
+  of "io":
+    ioTests r, cat, options
+  of "stdlib":
+    testStdlib(r, "lib/pure/*.nim", options, cat)
+    testStdlib(r, "lib/packages/docutils/highlite", options, cat)
+  of "examples":
+    compileExample(r, "examples/*.nim", options, cat)
+    compileExample(r, "examples/gtk/*.nim", options, cat)
+    compileExample(r, "examples/talk/*.nim", options, cat)
+  else:
+    for name in os.walkFiles(cat.string / "t*.nim"):
+      testSpec r, makeTest(name, options, cat)

tests/testament/specs.nim

@@ -0,0 +1,130 @@
+#
+#
+#            Nimrod Tester
+#        (c) Copyright 2014 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import parseutils, strutils, os, osproc, streams, parsecfg
+
+const
+  cmdTemplate* = r"nimrod cc --hints:on $# $#"
+
+type
+  TTestAction* = enum
+    actionCompile = "compile"
+    actionRun = "run"
+    actionReject = "reject"
+  TResultEnum* = enum
+    reNimrodcCrash,     # nimrod compiler seems to have crashed
+    reMsgsDiffer,       # error messages differ
+    reFilesDiffer,      # expected and given filenames differ
+    reLinesDiffer,      # expected and given line numbers differ
+    reOutputsDiffer,
+    reExitcodesDiffer,
+    reInvalidPeg,
+    reCodegenFailure,
+    reCodeNotFound,
+    reExeNotFound,
+    reIgnored,          # test is ignored
+    reSuccess           # test was successful
+  TTarget* = enum
+    targetC = "C"
+    targetCpp = "C++"
+    targetObjC = "ObjC"
+    targetJS = "JS"
+
+  TSpec* = object
+    action*: TTestAction
+    file*, cmd*: string
+    outp*: string
+    line*, exitCode*: int
+    msg*: string
+    ccodeCheck*: string
+    err*: TResultEnum
+    substr*: bool
+    targets*: set[TTarget]
+
+const
+  targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"]
+
+when not defined(parseCfgBool):
+  # candidate for the stdlib:
+  proc parseCfgBool(s: string): bool =
+    case normalize(s)
+    of "y", "yes", "true", "1", "on": result = true
+    of "n", "no", "false", "0", "off": result = false
+    else: raise newException(EInvalidValue, "cannot interpret as a bool: " & s)
+
+proc extractSpec(filename: string): string =
+  const tripleQuote = "\"\"\""
+  var x = readFile(filename).string
+  var a = x.find(tripleQuote)
+  var b = x.find(tripleQuote, a+3)
+  # look for """ only in the first section
+  if a >= 0 and b > a and a < 40:
+    result = x.substr(a+3, b-1).replace("'''", tripleQuote)
+  else:
+    #echo "warning: file does not contain spec: " & filename
+    result = ""
+
+when not defined(nimhygiene):
+  {.pragma: inject.}
+
+template parseSpecAux(fillResult: stmt) {.immediate.} =
+  var ss = newStringStream(extractSpec(filename))
+  var p {.inject.}: TCfgParser
+  open(p, ss, filename, 1)
+  while true:
+    var e {.inject.} = next(p)
+    case e.kind
+    of cfgEof: break
+    of cfgSectionStart, cfgOption, cfgError:
+      echo ignoreMsg(p, e)
+    of cfgKeyValuePair:
+      fillResult
+  close(p)
+
+proc parseSpec*(filename: string): TSpec =
+  result.file = filename
+  result.msg = ""
+  result.outp = ""
+  result.ccodeCheck = ""
+  result.cmd = cmdTemplate
+  parseSpecAux:
+    case normalize(e.key)
+    of "action":
+      case e.value.normalize
+      of "compile": result.action = actionCompile
+      of "run": result.action = actionRun
+      of "reject": result.action = actionReject
+      else: echo ignoreMsg(p, e)
+    of "file": result.file = e.value
+    of "line": discard parseInt(e.value, result.line)
+    of "output": 
+      result.action = actionRun
+      result.outp = e.value
+    of "outputsub":
+      result.action = actionRun
+      result.outp = e.value
+      result.substr = true
+    of "exitcode": 
+      discard parseInt(e.value, result.exitCode)
+    of "errormsg", "msg":
+      result.msg = e.value
+      result.action = actionReject
+    of "disabled":
+      if parseCfgBool(e.value): result.err = reIgnored
+    of "cmd": result.cmd = e.value
+    of "ccodecheck": result.ccodeCheck = e.value
+    of "target", "targets":
+      for v in e.value.normalize.split:
+        case v
+        of "c": result.targets.incl(targetC)
+        of "cpp", "c++": result.targets.incl(targetCpp)
+        of "objc": result.targets.incl(targetObjC)
+        of "js": result.targets.incl(targetJS)
+        else: echo ignoreMsg(p, e)
+    else: echo ignoreMsg(p, e)

tests/testament/tester.nim

@@ -0,0 +1,267 @@
+#
+#
+#            Nimrod Tester
+#        (c) Copyright 2014 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This program verifies Nimrod against the testcases.
+
+import
+  parseutils, strutils, pegs, os, osproc, streams, parsecfg, browsers, json,
+  marshal, cgi, backend, parseopt, specs #, caas
+
+const
+  resultsFile = "testresults.html"
+  jsonFile = "testresults.json"
+  Usage = """Usage:
+  tester [options] command [arguments]
+
+Command:
+  all                         run all tests
+  c|category <category>       run all the tests of a certain category
+  html                        generate $1 from the database
+Arguments:
+  arguments are passed to the compiler
+Options:
+  --print                   also print results to the console
+""" % resultsFile
+
+type
+  Category = distinct string
+  TResults = object
+    total, passed, skipped: int
+    data: string
+
+  TTest = object
+    name: string
+    cat: Category
+    options: string
+    target: TTarget
+    action: TTestAction
+
+# ----------------------------------------------------------------------------
+
+let
+  pegLineError = 
+    peg"{[^(]*} '(' {\d+} ', ' \d+ ') ' ('Error'/'Warning') ':' \s* {.*}"
+  pegOtherError = peg"'Error:' \s* {.*}"
+  pegSuccess = peg"'Hint: operation successful'.*"
+  pegOfInterest = pegLineError / pegOtherError
+
+proc callCompiler(cmdTemplate, filename, options: string): TSpec =
+  let c = parseCmdLine(cmdTemplate % [options, filename])
+  var p = startProcess(command=c[0], args=c[1.. -1],
+                       options={poStdErrToStdOut, poUseShell})
+  let outp = p.outputStream
+  var suc = ""
+  var err = ""
+  var x = newStringOfCap(120)
+  while outp.readLine(x.TaintedString) or running(p):
+    if x =~ pegOfInterest:
+      # `err` should contain the last error/warning message
+      err = x
+    elif x =~ pegSuccess:
+      suc = x
+  close(p)
+  result.msg = ""
+  result.file = ""
+  result.outp = ""
+  result.line = -1
+  if err =~ pegLineError:
+    result.file = extractFilename(matches[0])
+    result.line = parseInt(matches[1])
+    result.msg = matches[2]
+  elif err =~ pegOtherError:
+    result.msg = matches[0]
+  elif suc =~ pegSuccess:
+    result.err = reSuccess
+
+proc initResults: TResults =
+  result.total = 0
+  result.passed = 0
+  result.skipped = 0
+  result.data = ""
+
+proc readResults(filename: string): TResults =
+  result = marshal.to[TResults](readFile(filename).string)
+
+proc writeResults(filename: string, r: TResults) =
+  writeFile(filename, $$r)
+
+proc `$`(x: TResults): string =
+  result = ("Tests passed: $1 / $3 <br />\n" &
+            "Tests skipped: $2 / $3 <br />\n") %
+            [$x.passed, $x.skipped, $x.total]
+
+proc addResult(r: var TResults, test: TTest,
+               expected, given: string, success: TResultEnum) =
+  let name = test.name.extractFilename & test.options
+  backend.writeTestResult(name = name,
+                          category = test.cat.string, 
+                          target = $test.target,
+                          action = $test.action,
+                          result = $success,
+                          expected = expected,
+                          given = given)
+  r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
+
+proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) =
+  if strip(expected.msg) notin strip(given.msg):
+    r.addResult(test, expected.msg, given.msg, reMsgsDiffer)
+  elif extractFilename(expected.file) != extractFilename(given.file) and
+      "internal error:" notin expected.msg:
+    r.addResult(test, expected.file, given.file, reFilesDiffer)
+  elif expected.line != given.line and expected.line != 0:
+    r.addResult(test, $expected.line, $given.line, reLinesDiffer)
+  else:
+    r.addResult(test, expected.msg, given.msg, reSuccess)
+    inc(r.passed)
+
+proc generatedFile(path, name: string, target: TTarget): string =
+  let ext = targetToExt[target]
+  result = path / "nimcache" /
+    (if target == targetJS: path.splitPath.tail & "_" else: "") &
+    name.changeFileExt(ext)
+
+proc codegenCheck(test: TTest, check: string, given: var TSpec) =
+  if check.len > 0:
+    try:
+      let (path, name, ext2) = test.name.splitFile
+      let genFile = generatedFile(path, name, test.target)
+      echo genFile
+      let contents = readFile(genFile).string
+      if contents.find(check.peg) < 0:
+        given.err = reCodegenFailure
+    except EInvalidValue:
+      given.err = reInvalidPeg
+    except EIO:
+      given.err = reCodeNotFound
+
+proc testSpec(r: var TResults, test: TTest) =
+  # major entry point for a single test
+  let tname = test.name.addFileExt(".nim")
+  inc(r.total)
+  echo extractFilename(tname)
+  var expected = parseSpec(tname)
+  if expected.err == reIgnored:
+    r.addResult(test, "", "", reIgnored)
+    inc(r.skipped)
+  else:
+    case expected.action
+    of actionCompile:
+      var given = callCompiler(expected.cmd, test.name, test.options)
+      if given.err == reSuccess:
+        codegenCheck(test, expected.ccodeCheck, given)
+      r.addResult(test, "", given.msg, given.err)
+      if given.err == reSuccess: inc(r.passed)
+    of actionRun:
+      var given = callCompiler(expected.cmd, test.name, test.options)
+      if given.err != reSuccess:
+        r.addResult(test, "", given.msg, given.err)
+      else:
+        var exeFile: string
+        if test.target == targetJS:
+          let (dir, file, ext) = splitFile(tname)
+          exeFile = dir / "nimcache" / file & ".js"
+        else:
+          exeFile = changeFileExt(tname, ExeExt)
+        
+        if existsFile(exeFile):
+          var (buf, exitCode) = execCmdEx(
+            (if test.target==targetJS: "node " else: "") & exeFile)
+          if exitCode != expected.ExitCode:
+            r.addResult(test, "exitcode: " & $expected.exitCode,
+                              "exitcode: " & $exitCode, reExitCodesDiffer)
+          else:
+            if strip(buf.string) != strip(expected.outp):
+              if not (expected.substr and expected.outp in buf.string):
+                given.err = reOutputsDiffer
+            if given.err == reSuccess:
+              codeGenCheck(test, expected.ccodeCheck, given)
+            if given.err == reSuccess: inc(r.passed)
+            r.addResult(test, expected.outp, buf.string, given.err)
+        else:
+          r.addResult(test, expected.outp, "executable not found", reExeNotFound)
+    of actionReject:
+      var given = callCompiler(expected.cmd, test.name, test.options)
+      cmpMsgs(r, expected, given, test)
+
+proc testNoSpec(r: var TResults, test: TTest) =
+  # does not extract the spec because the file is not supposed to have any
+  let tname = test.name.addFileExt(".nim")
+  inc(r.total)
+  echo extractFilename(tname)
+  let given = callCompiler(cmdTemplate, test.name, test.options)
+  r.addResult(test, "", given.msg, given.err)
+  if given.err == reSuccess: inc(r.passed)
+
+proc makeTest(test, options: string, cat: Category, action = actionCompile,
+              target = targetC): TTest =
+  # start with 'actionCompile', will be overwritten in the spec:
+  result = TTest(cat: cat, name: test, options: options,
+                 target: target, action: action)
+
+include categories
+
+proc toJson(res: TResults): PJsonNode =
+  result = newJObject()
+  result["total"] = newJInt(res.total)
+  result["passed"] = newJInt(res.passed)
+  result["skipped"] = newJInt(res.skipped)
+
+proc outputJson(reject, compile, run: TResults) =
+  var doc = newJObject()
+  doc["reject"] = toJson(reject)
+  doc["compile"] = toJson(compile)
+  doc["run"] = toJson(run)
+  var s = pretty(doc)
+  writeFile(jsonFile, s)
+
+# proc runCaasTests(r: var TResults) =
+#   for test, output, status, mode in caasTestsRunner():
+#     r.addResult(test, "", output & "-> " & $mode,
+#                 if status: reSuccess else: reOutputsDiffer)
+
+proc main() =
+  os.putenv "NIMTEST_NO_COLOR", "1"
+  os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
+
+  backend.open()  
+  var optPrintResults = false
+  var p = initOptParser()
+  p.next()
+  if p.kind == cmdLongoption:
+    case p.key.string.normalize
+    of "print", "verbose": optPrintResults = true
+    else: quit usage
+    p.next()
+  if p.kind != cmdArgument: quit usage
+  var action = p.key.string.normalize
+  p.next()
+  var r = initResults()
+  case action
+  of "all":
+    for kind, dir in walkDir("tests"):
+      if kind == pcDir and dir != "testament":
+        processCategory(r, Category(dir), p.cmdLineRest.string)
+    for a in AdditionalCategories:
+      processCategory(r, Category(a), p.cmdLineRest.string)
+  of "c", "category":
+    var cat = Category(p.key)
+    p.next
+    processCategory(r, cat, p.cmdLineRest.string)
+  of "html":
+    quit "too implement"
+  else:
+    quit usage
+
+  if optPrintResults: echo r, r.data
+  backend.close()
+  
+if paramCount() == 0:
+  quit usage
+main()
+