version 0.7.2

tests/tester.nim

@@ -0,0 +1,161 @@
+#
+#
+#            Nimrod Tester
+#        (c) Copyright 2008 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This program verifies Nimrod against the testcases.
+## The testcases may contain the directives '#ERROR' or '#ERROR_IN'.
+## '#ERROR' is used to indicate that the compiler should report
+## an error in the marked line (the line that contains the '#ERROR'
+## directive.)
+## The format for '#ERROR_IN' is:
+##      #ERROR_IN filename linenumber
+## One can omit the extension of the filename ('.nim' is then assumed).
+## Tests which contain none of the two directives should compile. Thus they
+## are executed after successful compilation and their output is verified
+## against the results specified with the '#OUT' directive.
+## (Tests which require user interaction are not possible.)
+## Tests can have an #ERROR_MSG directive specifiying the error message
+## Nimrod shall produce.
+
+import
+  strutils, regexprs, os, osproc, streams
+
+const
+  cmdTemplate = r"nimrod cc --hints:on $options $filename"
+
+type
+  TSpec = object of TObject ## specification object
+    line: int    ## line number where compiler should throw an error
+    file: string ## file where compiler should throw an error
+    err: bool    ## true if the specification says there should be an error
+    outp: string ## output that should be produced
+    puremsg: string ## pure message of compiler
+
+proc myExec(cmd: string): string =
+  #echo("Executing: " & cmd)
+  result = osproc.executeProcess(cmd)
+  #echo("Received: " & result)
+
+proc parseTest(filename: string): TSpec =
+  var i = 0 # the line counter
+  var matches: array [0..2, string]
+  result.outp = ""
+  result.puremsg = ""
+  result.file = filename
+  for s in lines(filename):
+    inc(i)
+    if find(s, r"\#OUT\s*(.*)", matches):
+      result.outp = matches[1]
+      break
+    if find(s, r"\#ERROR_IN\s*(\S*)\s*(\d+)", matches):
+      result.file = matches[1]
+      result.line = parseInt(matches[2])
+      result.err = true
+      break
+    if find(s, r"\#ERROR_MSG\s*(.*)", matches):
+      result.line = i
+      result.outp = matches[1]
+      result.err = True
+      break
+    if find(s, r"\#ERROR$", matches):
+      result.line = i
+      result.err = true
+      break
+
+proc callCompiler(filename, options: string): TSpec =
+  var c = parseCmdLine(cmdTemplate % ["filename", filename, "options", options])
+  var a: seq[string] = @[] # slicing is not yet implemented :-(
+  for i in 1 .. c.len-1: add(a, c[i])
+  var p = startProcess(command=c[0], args=a,
+                       options={poStdErrToStdOut, poUseShell})
+  var outp = p.outputStream
+  while running(p) or not outp.atEnd(outp):
+    var s = outp.readLine()
+    var matches: array [0..3, string]
+    result.outp = ""
+    result.puremsg = ""
+    result.file = ""
+    result.err = true
+    if match(s, r"(.*)\((\d+), \d+\) Error\: (.*)", matches):
+      result.file = matches[1]
+      result.line = parseInt(matches[2])
+      result.outp = matches[0]
+      result.puremsg = matches[3]
+      break
+    elif match(s, r"Error\: (.*)", matches):
+      result.puremsg = matches[1]
+      result.outp = matches[0]
+      result.line = 1
+      break
+    elif match(s, r"Hint\: operation successful", matches):
+      result.outp = matches[0]
+      result.err = false
+      break
+
+proc cmpResults(filename: string, spec, comp: TSpec): bool =
+  # short filename for messages (better readability):
+  var shortfile = os.extractFilename(filename)
+
+  if comp.err and comp.outp == "":
+    # the compiler did not say "[Error]" nor "operation sucessful"
+    Echo("[Tester] $1 -- FAILED; COMPILER BROKEN" % shortfile)
+  elif spec.err != comp.err:
+    Echo(("[Tester] $1 -- FAILED\n" &
+         "Compiler says: $2\n" &
+         "But specification says: $3") %
+         [shortfile, comp.outp, spec.outp])
+  elif spec.err:
+    if extractFilename(comp.file) != extractFilename(spec.file):
+      Echo(("[Tester] $1 -- FAILED: file names do not match:\n" &
+           "Compiler: $2\nSpec: $3") % [shortfile, comp.file, spec.file])
+    elif strip(spec.outp) notin strip(comp.puremsg):
+      Echo(("[Tester] $1 -- FAILED: error messages do not match:\n" &
+           "Compiler: $2\nSpec: $3") % [shortfile, comp.pureMsg, spec.outp])
+    elif comp.line != spec.line:
+      Echo(("[Tester] $1 -- FAILED: line numbers do not match:\n" &
+           "Compiler: $2\nSpec: $3") % [shortfile, $comp.line, $spec.line])
+    else:
+      Echo("[Tester] $1 -- OK" % shortfile)
+      result = true
+  else:
+    # we have to run the executable and check its output:
+    var exeFile = changeFileExt(filename, ExeExt)
+    if ExistsFile(exeFile):
+      if len(spec.outp) == 0:
+        # we have no output to validate against, but compilation succeeded,
+        # so it's okay:
+        Echo("[Tester] $1 -- OK" % shortfile)
+        result = true
+      else:
+        var buf = myExec(exeFile)
+        result = strip(buf) == strip(spec.outp)
+        if result:
+          Echo("[Tester] $1 -- compiled program OK" % shortfile)
+        else:
+          Echo("[Tester] $1 -- compiled program FAILED" % shortfile)
+    else:
+      Echo("[Tester] $1 -- FAILED; executable not found" % shortfile)
+
+proc main(options: string) =
+  # runs the complete testsuite
+  var total = 0
+  var passed = 0
+  for filename in os.walkFiles("tests/t*.nim"):
+    if extractFilename(filename) == "tester.nim": continue
+    var spec = parseTest(filename)
+    var comp = callCompiler(filename, options)
+    if cmpResults(filename, spec, comp): inc(passed)
+    inc(total)
+  Echo("[Tester] $1/$2 tests passed\n" % [$passed, $total])
+
+var
+  options = ""
+for i in 1.. paramCount():
+  add(options, " ")
+  add(options, paramStr(i))
+main(options)