jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use targets from test spec when running testament tests (#6687)

Browse source
This commit is contained in:
Jacek Sieka 2017-11-16 05:37:51 +08:00 • committed by Andreas Rumpf
commit b122836acf
21 changed files with 124 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/cpp/tasync_cpp.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
output: "hello" output: "hello"
""" """

2
tests/cpp/tcovariancerules.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
output: ''' output: '''
cat cat
cat cat

2
tests/cpp/tcppraise.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
output: '''foo output: '''foo
bar bar
Need odd and >= 3 digits## Need odd and >= 3 digits##

2
tests/cpp/tdont_init_instantiation.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
output: '''''' output: ''''''
disabled: true disabled: true
""" """

1
tests/cpp/tembarrassing_generic_failure.nim
View file

@ -1,4 +1,5 @@
discard """ discard """
targets: "cpp"
cmd: "nim cpp --threads:on $file" cmd: "nim cpp --threads:on $file"
""" """

2
tests/cpp/temitlist.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
output: '''6.0''' output: '''6.0'''
""" """

2
tests/cpp/tempty_generic_obj.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
output: '''int output: '''int
float''' float'''
""" """

2
tests/cpp/tgen_prototype_for_importc.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
output: '''Hello world''' output: '''Hello world'''
""" """

2
tests/cpp/tget_subsystem.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
""" """
{.emit: """ {.emit: """

2
tests/cpp/tnativesockets.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
""" """
import nativesockets import nativesockets

3
tests/cpp/treturn_array.nim
View file

@ -1,3 +1,6 @@
discard """
targets: "cpp"
"""
# bug #2259 # bug #2259
type Mat4f* = array[0..15, float] type Mat4f* = array[0..15, float]

2
tests/cpp/tsigbreak.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
""" """
import tables, lists import tables, lists

2
tests/cpp/tstaticvar_via_typedesc.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
output: "42" output: "42"
""" """

4
tests/cpp/ttemplatetype.nim
View file

@ -1,3 +1,7 @@
discard """
targets: "cpp"
"""
type type
Map {.importcpp: "std::map", header: "<map>".} [T,U] = object Map {.importcpp: "std::map", header: "<map>".} [T,U] = object

1
tests/cpp/tthread_createthread.nim
View file

@ -1,4 +1,5 @@
discard """ discard """
targets: "cpp"
cmd: "nim cpp --hints:on --threads:on $options $file" cmd: "nim cpp --hints:on --threads:on $options $file"
""" """

2
tests/cpp/ttypeinfo.nim
View file

@ -1,6 +1,6 @@
discard """ discard """
targets: "cpp"
output: '''100''' output: '''100'''
cmd: "nim cpp $file"
""" """
import typeinfo import typeinfo

2
tests/cpp/ttypeinfo2.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
""" """
# bug #2841 # bug #2841
import typeinfo import typeinfo

2
tests/cpp/tvector_iterator.nim
View file

@ -1,5 +1,5 @@
discard """ discard """
cmd: "nim cpp $file" targets: "cpp"
""" """
{.emit: """ {.emit: """

2
tests/cpp/tvectorseq.nim
View file

@ -1,7 +1,7 @@
discard """ discard """
targets: "cpp"
output: '''(x: 1.0) output: '''(x: 1.0)
(x: 0.0)''' (x: 0.0)'''
cmd: "nim cpp $file"
disabled: "true" disabled: "true"
""" """

20
tests/testament/categories.nim
View file

@ -219,9 +219,9 @@ proc debuggerTests(r: var TResults, cat: Category, options: string) =
proc jsTests(r: var TResults, cat: Category, options: string) = proc jsTests(r: var TResults, cat: Category, options: string) =
template test(filename: untyped) = template test(filename: untyped) =
testSpec r, makeTest(filename, options & " -d:nodejs", cat, testSpec r, makeTest(filename, options & " -d:nodejs", cat,
actionRun, targetJS) actionRun), targetJS
testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat, testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat,
actionRun, targetJS) actionRun), targetJS
for t in os.walkFiles("tests/js/t*.nim"): for t in os.walkFiles("tests/js/t*.nim"):
test(t) test(t)
@ -245,10 +245,10 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
testSpec r, makeTest(filename, options, cat, action) testSpec r, makeTest(filename, options, cat, action)
template testJS(filename: untyped) = template testJS(filename: untyped) =
testSpec r, makeTest(filename, options, cat, actionCompile, targetJS) testSpec r, makeTest(filename, options, cat, actionCompile), targetJS
template testCPP(filename: untyped) = template testCPP(filename: untyped) =
testSpec r, makeTest(filename, options, cat, actionCompile, targetCPP) testSpec r, makeTest(filename, options, cat, actionCompile), targetCPP
let tests = [ let tests = [
"niminaction/Chapter3/ChatApp/src/server", "niminaction/Chapter3/ChatApp/src/server",
@ -387,7 +387,7 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
installStatus = waitForExitEx(installProcess) installStatus = waitForExitEx(installProcess)
installProcess.close installProcess.close
if installStatus != QuitSuccess: if installStatus != QuitSuccess:
r.addResult(test, "", "", reInstallFailed) r.addResult(test, targetC, "", "", reInstallFailed)
continue continue
let let
@ -396,12 +396,12 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
buildStatus = waitForExitEx(buildProcess) buildStatus = waitForExitEx(buildProcess)
buildProcess.close buildProcess.close
if buildStatus != QuitSuccess: if buildStatus != QuitSuccess:
r.addResult(test, "", "", reBuildFailed) r.addResult(test, targetC, "", "", reBuildFailed)
r.addResult(test, "", "", reSuccess) r.addResult(test, targetC, "", "", reSuccess)
r.addResult(packageFileTest, "", "", reSuccess) r.addResult(packageFileTest, targetC, "", "", reSuccess)
except JsonParsingError: except JsonParsingError:
echo("[Warning] - Cannot run nimble tests: Invalid package file.") echo("[Warning] - Cannot run nimble tests: Invalid package file.")
r.addResult(packageFileTest, "", "", reBuildFailed) r.addResult(packageFileTest, targetC, "", "", reBuildFailed)
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
@ -420,7 +420,7 @@ proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
let test = "tests" & DirSep &.? cat.string / test let test = "tests" & DirSep &.? cat.string / test
let target = if cat.string.normalize == "js": targetJS else: targetC let target = if cat.string.normalize == "js": targetJS else: targetC
if existsFile(test): testSpec r, makeTest(test, options, cat, target = target) if existsFile(test): testSpec r, makeTest(test, options, cat), target
else: echo "[Warning] - ", test, " test does not exist" else: echo "[Warning] - ", test, " test does not exist"
proc processCategory(r: var TResults, cat: Category, options: string) = proc processCategory(r: var TResults, cat: Category, options: string) =

106
tests/testament/tester.nim
View file

@ -44,7 +44,6 @@ type
name: string name: string
cat: Category cat: Category
options: string options: string
target: TTarget
action: TTestAction action: TTestAction
startTime: float startTime: float
@ -155,13 +154,13 @@ proc `$`(x: TResults): string =
"Tests skipped: $2 / $3 <br />\n") % "Tests skipped: $2 / $3 <br />\n") %
[$x.passed, $x.skipped, $x.total] [$x.passed, $x.skipped, $x.total]
proc addResult(r: var TResults, test: TTest, proc addResult(r: var TResults, test: TTest, target: TTarget,
expected, given: string, success: TResultEnum) = expected, given: string, success: TResultEnum) =
let name = test.name.extractFilename & test.options let name = test.name.extractFilename & " " & $target & test.options
let duration = epochTime() - test.startTime let duration = epochTime() - test.startTime
backend.writeTestResult(name = name, backend.writeTestResult(name = name,
category = test.cat.string, category = test.cat.string,
target = $test.target, target = $target,
action = $test.action, action = $test.action,
result = $success, result = $success,
expected = expected, expected = expected,
@ -197,29 +196,29 @@ proc addResult(r: var TResults, test: TTest,
discard waitForExit(p) discard waitForExit(p)
close(p) close(p)
proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) = proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
if strip(expected.msg) notin strip(given.msg): if strip(expected.msg) notin strip(given.msg):
r.addResult(test, expected.msg, given.msg, reMsgsDiffer) r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer)
elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg: elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg:
r.addResult(test, expected.nimout, given.nimout, reMsgsDiffer) r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer)
elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and
"internal error:" notin expected.msg: "internal error:" notin expected.msg:
r.addResult(test, expected.file, given.file, reFilesDiffer) r.addResult(test, target, expected.file, given.file, reFilesDiffer)
elif expected.line != given.line and expected.line != 0 or elif expected.line != given.line and expected.line != 0 or
expected.column != given.column and expected.column != 0: expected.column != given.column and expected.column != 0:
r.addResult(test, $expected.line & ':' & $expected.column, r.addResult(test, target, $expected.line & ':' & $expected.column,
$given.line & ':' & $given.column, $given.line & ':' & $given.column,
reLinesDiffer) reLinesDiffer)
elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and
"internal error:" notin expected.msg: "internal error:" notin expected.msg:
r.addResult(test, expected.tfile, given.tfile, reFilesDiffer) r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer)
elif expected.tline != given.tline and expected.tline != 0 or elif expected.tline != given.tline and expected.tline != 0 or
expected.tcolumn != given.tcolumn and expected.tcolumn != 0: expected.tcolumn != given.tcolumn and expected.tcolumn != 0:
r.addResult(test, $expected.tline & ':' & $expected.tcolumn, r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn,
$given.tline & ':' & $given.tcolumn, $given.tline & ':' & $given.tcolumn,
reLinesDiffer) reLinesDiffer)
else: else:
r.addResult(test, expected.msg, given.msg, reSuccess) r.addResult(test, target, expected.msg, given.msg, reSuccess)
inc(r.passed) inc(r.passed)
proc generatedFile(path, name: string, target: TTarget): string = proc generatedFile(path, name: string, target: TTarget): string =
@ -231,11 +230,11 @@ proc generatedFile(path, name: string, target: TTarget): string =
proc needsCodegenCheck(spec: TSpec): bool = proc needsCodegenCheck(spec: TSpec): bool =
result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0 result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0
proc codegenCheck(test: TTest, spec: TSpec, expectedMsg: var string, proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string,
given: var TSpec) = given: var TSpec) =
try: try:
let (path, name, _) = test.name.splitFile let (path, name, _) = test.name.splitFile
let genFile = generatedFile(path, name, test.target) let genFile = generatedFile(path, name, target)
let contents = readFile(genFile).string let contents = readFile(genFile).string
let check = spec.ccodeCheck let check = spec.ccodeCheck
if check.len > 0: if check.len > 0:
@ -267,13 +266,13 @@ proc makeDeterministic(s: string): string =
sort(x, system.cmp) sort(x, system.cmp)
result = join(x, "\n") result = join(x, "\n")
proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec; proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
r: var TResults) = expected: TSpec; r: var TResults) =
var expectedmsg: string = "" var expectedmsg: string = ""
var givenmsg: string = "" var givenmsg: string = ""
if given.err == reSuccess: if given.err == reSuccess:
if expected.needsCodegenCheck: if expected.needsCodegenCheck:
codegenCheck(test, expected, expectedmsg, given) codegenCheck(test, target, expected, expectedmsg, given)
givenmsg = given.msg givenmsg = given.msg
if expected.nimout.len > 0: if expected.nimout.len > 0:
expectedmsg = expected.nimout expectedmsg = expected.nimout
@ -282,7 +281,7 @@ proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec;
else: else:
givenmsg = given.nimout.strip givenmsg = given.nimout.strip
if given.err == reSuccess: inc(r.passed) if given.err == reSuccess: inc(r.passed)
r.addResult(test, expectedmsg, givenmsg, given.err) r.addResult(test, target, expectedmsg, givenmsg, given.err)
proc analyzeAndConsolidateOutput(s: string): string = proc analyzeAndConsolidateOutput(s: string): string =
result = "" result = ""
@ -298,13 +297,7 @@ proc analyzeAndConsolidateOutput(s: string): string =
result = substr(rows[i], pos) result = substr(rows[i], pos)
return return
proc testSpec(r: var TResults, test: TTest) = proc testSpec(r: var TResults, test: TTest, target = targetC) =
# major entry point for a single test
if test.target notin targets:
r.addResult(test, "", "", reIgnored)
inc(r.skipped)
return
let tname = test.name.addFileExt(".nim") let tname = test.name.addFileExt(".nim")
#echo "TESTING ", tname #echo "TESTING ", tname
inc(r.total) inc(r.total)
@ -316,27 +309,36 @@ proc testSpec(r: var TResults, test: TTest) =
expected.action = actionRunNoSpec expected.action = actionRunNoSpec
if expected.err == reIgnored: if expected.err == reIgnored:
r.addResult(test, "", "", reIgnored) r.addResult(test, target, "", "", reIgnored)
inc(r.skipped) inc(r.skipped)
return return
if expected.targets == {}:
expected.targets.incl(target)
for target in expected.targets:
if target notin targets:
r.addResult(test, target, "", "", reIgnored)
inc(r.skipped)
continue
case expected.action case expected.action
of actionCompile: of actionCompile:
var given = callCompiler(expected.cmd, test.name, var given = callCompiler(expected.cmd, test.name,
test.options & " --stdout --hint[Path]:off --hint[Processing]:off", test.options & " --stdout --hint[Path]:off --hint[Processing]:off",
test.target) target)
compilerOutputTests(test, given, expected, r) compilerOutputTests(test, target, given, expected, r)
of actionRun, actionRunNoSpec: of actionRun, actionRunNoSpec:
# In this branch of code "early return" pattern is clearer than deep # In this branch of code "early return" pattern is clearer than deep
# nested conditionals - the empty rows in between to clarify the "danger" # nested conditionals - the empty rows in between to clarify the "danger"
var given = callCompiler(expected.cmd, test.name, test.options, var given = callCompiler(expected.cmd, test.name, test.options,
test.target) target)
if given.err != reSuccess: if given.err != reSuccess:
r.addResult(test, "", given.msg, given.err) r.addResult(test, target, "", given.msg, given.err)
return continue
let isJsTarget = test.target == targetJS let isJsTarget = target == targetJS
var exeFile: string var exeFile: string
if isJsTarget: if isJsTarget:
let (dir, file, _) = splitFile(tname) let (dir, file, _) = splitFile(tname)
@ -345,14 +347,14 @@ proc testSpec(r: var TResults, test: TTest) =
exeFile = changeFileExt(tname, ExeExt) exeFile = changeFileExt(tname, ExeExt)
if not existsFile(exeFile): if not existsFile(exeFile):
r.addResult(test, expected.outp, "executable not found", reExeNotFound) r.addResult(test, target, expected.outp, "executable not found", reExeNotFound)
return continue
let nodejs = if isJsTarget: findNodeJs() else: "" let nodejs = if isJsTarget: findNodeJs() else: ""
if isJsTarget and nodejs == "": if isJsTarget and nodejs == "":
r.addResult(test, expected.outp, "nodejs binary not in PATH", r.addResult(test, target, expected.outp, "nodejs binary not in PATH",
reExeNotFound) reExeNotFound)
return continue
let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile
var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut}) var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut})
@ -366,33 +368,33 @@ proc testSpec(r: var TResults, test: TTest) =
let expectedOut = strip(expected.outp) let expectedOut = strip(expected.outp)
if exitCode != expected.exitCode: if exitCode != expected.exitCode:
r.addResult(test, "exitcode: " & $expected.exitCode, r.addResult(test, target, "exitcode: " & $expected.exitCode,
"exitcode: " & $exitCode & "\n\nOutput:\n" & "exitcode: " & $exitCode & "\n\nOutput:\n" &
analyzeAndConsolidateOutput(bufB), analyzeAndConsolidateOutput(bufB),
reExitCodesDiffer) reExitCodesDiffer)
return continue
if bufB != expectedOut and expected.action != actionRunNoSpec: if bufB != expectedOut and expected.action != actionRunNoSpec:
if not (expected.substr and expectedOut in bufB): if not (expected.substr and expectedOut in bufB):
given.err = reOutputsDiffer given.err = reOutputsDiffer
r.addResult(test, expected.outp, bufB, reOutputsDiffer) r.addResult(test, target, expected.outp, bufB, reOutputsDiffer)
return continue
compilerOutputTests(test, given, expected, r) compilerOutputTests(test, target, given, expected, r)
return continue
of actionReject: of actionReject:
var given = callCompiler(expected.cmd, test.name, test.options, var given = callCompiler(expected.cmd, test.name, test.options,
test.target) target)
cmpMsgs(r, expected, given, test) cmpMsgs(r, expected, given, test, target)
return continue
proc testNoSpec(r: var TResults, test: TTest) = proc testNoSpec(r: var TResults, test: TTest, target = targetC) =
# does not extract the spec because the file is not supposed to have any # does not extract the spec because the file is not supposed to have any
#let tname = test.name.addFileExt(".nim") #let tname = test.name.addFileExt(".nim")
inc(r.total) inc(r.total)
let given = callCompiler(cmdTemplate(), test.name, test.options, test.target) let given = callCompiler(cmdTemplate(), test.name, test.options, target)
r.addResult(test, "", given.msg, given.err) r.addResult(test, target, "", given.msg, given.err)
if given.err == reSuccess: inc(r.passed) if given.err == reSuccess: inc(r.passed)
proc testC(r: var TResults, test: TTest) = proc testC(r: var TResults, test: TTest) =
@ -400,9 +402,9 @@ proc testC(r: var TResults, test: TTest) =
let tname = test.name.addFileExt(".c") let tname = test.name.addFileExt(".c")
inc(r.total) inc(r.total)
styledEcho "Processing ", fgCyan, extractFilename(tname) styledEcho "Processing ", fgCyan, extractFilename(tname)
var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, test.target) var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC)
if given.err != reSuccess: if given.err != reSuccess:
r.addResult(test, "", given.msg, given.err) r.addResult(test, targetC, "", given.msg, given.err)
elif test.action == actionRun: elif test.action == actionRun:
let exeFile = changeFileExt(test.name, ExeExt) let exeFile = changeFileExt(test.name, ExeExt)
var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath}) var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath})
@ -410,10 +412,10 @@ proc testC(r: var TResults, test: TTest) =
if given.err == reSuccess: inc(r.passed) if given.err == reSuccess: inc(r.passed)
proc makeTest(test, options: string, cat: Category, action = actionCompile, proc makeTest(test, options: string, cat: Category, action = actionCompile,
target = targetC, env: string = ""): TTest = env: string = ""): TTest =
# start with 'actionCompile', will be overwritten in the spec: # start with 'actionCompile', will be overwritten in the spec:
result = TTest(cat: cat, name: test, options: options, result = TTest(cat: cat, name: test, options: options,
target: target, action: action, startTime: epochTime()) action: action, startTime: epochTime())
when defined(windows): when defined(windows):
const const