test-runner for caas multi-step tests

tests/caas/basic-recompile.txt

@@ -0,0 +1,7 @@
+main.nim
+> c
+SuccessX
+> c
+! Processing
+SuccessX
+

tests/caas/imported.nim

@@ -0,0 +1,3 @@
+proc `+++`*(a,b: string): string =
+  return a & "  " & b
+

tests/caas/main.nim

@@ -0,0 +1,7 @@
+import imported, strutils
+
+proc main =
+  var t1 = "text"
+  var t2 = t1.toUpper
+  echo(t1 +++ t2)
+

tests/caasdriver.nim

@@ -0,0 +1,83 @@
+import osproc, streams, os, strutils, re
+
+type
+  TNimrodSession* = object
+    nim: PProcess
+
+proc dirname(path: string): string = path.splitPath()[0]
+
+const
+  TesterDir = CurrentSourcePath.dirname
+  NimrodBin = TesterDir / "../bin/nimrod"
+
+proc startNimrodSession*(project: string): TNimrodSession =
+  result.nim = startProcess(NimrodBin,
+    workingDir = project.dirname,
+    args = ["serve", "--server.type:stdin", project])
+
+proc doCommand*(session: var TNimrodSession, command: string): string =
+  session.nim.inputStream.write(command & "\n")
+  session.nim.inputStream.flush
+  
+  result = ""
+  
+  while true:
+    var line = TaintedString("")
+    if session.nim.outputStream.readLine(line):
+      if line.string == "": break
+      result.add(line.string & "\n")
+
+proc close(session: var TNimrodSession) {.destructor.} =
+  session.nim.close
+
+proc doScenario(script: string, output: PStream): bool =
+  result = true
+
+  var f = open(script)
+  var project = TaintedString("")
+  
+  if f.readLine(project):
+    var
+      s = startNimrodSession(script.dirname / project.string)
+      tline = TaintedString("")
+      lastOutput = ""
+      ln = 1
+
+    while f.readLine(tline):
+      var line = tline.string
+      inc ln
+      if line.strip.len == 0: continue
+
+      if line.startsWith(">"):
+        lastOutput = s.doCommand(line.substr(1).strip)
+        output.writeln line, "\n", lastOutput
+      else:
+        var expectMatch = true
+        var pattern = line
+        if line.startsWith("!"):
+          pattern = line.substr(1).strip
+          expectMatch = false
+
+        var actualMatch = lastOutput.find(re(pattern)) != -1
+
+        if expectMatch == actualMatch:
+          output.writeln "SUCCESS ", line
+        else:
+          output.writeln "FAILURE ", line
+          result = false
+
+iterator caasTestsRunner*(filter = ""): tuple[test, output: string,
+                                              status: bool] =
+  for scenario in os.walkFiles(TesterDir / "caas/*.txt"):
+    if filter.len > 0 and find(scenario, filter) == -1: continue
+    var outStream = newStringStream()
+    let r = doScenario(scenario, outStream)
+    yield (scenario, outStream.data, r)
+
+when isMainModule:
+  var filter = ""
+  if paramCount() > 0: filter = paramStr(1)
+  
+  for t, o, r in caasTestsRunner(filter):
+    echo t, "\n", o
+    

tests/tester.nim

@@ -11,7 +11,7 @@
 
 import
   parseutils, strutils, pegs, os, osproc, streams, parsecfg, browsers, json,
-  marshal, cgi, parseopt
+  marshal, cgi, parseopt, caasdriver
 
 const
   cmdTemplate = r"nimrod cc --hints:on $# $#"
@@ -38,6 +38,7 @@ type
     reExeNotFound,
     reIgnored,          # test is ignored
     reSuccess           # test was successful
+
   TTarget = enum
     targetC, targetCpp, targetObjC, targetJS
 
@@ -363,6 +364,10 @@ proc outputJSON(reject, compile, run: TResults) =
   var s = pretty(doc)
   writeFile(jsonFile, s)
 
+proc runCaasTests(r: var TResults) =
+  for test, output, status in caasTestsRunner():
+    r.addResult(test, "", output, if status: reSuccess else: reOutputsDiffer)
+
 proc main() =
   os.putenv "NIMTEST_NO_COLOR", "1"
   os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
@@ -404,6 +409,7 @@ proc main() =
     writeResults(runJson, r)
   of "special":
     runSpecialTests(r, p.cmdLineRest.string)
+    runCaasTests(r)
     writeResults(runJson, r)
   of "rodfiles":
     runRodFiles(r, p.cmdLineRest.string)