Testament: refactoring; makes the test joiner green
This commit is contained in:
Araq
parent
8896770f8f
commit
3b783f7e13
28 changed files with 176 additions and 230 deletions
|
|
@ -10,6 +10,58 @@
|
|||
## Include for the tester that contains test suites that test special features
|
||||
## of the compiler.
|
||||
|
||||
const
|
||||
specialCategories = [
|
||||
"assert",
|
||||
"async",
|
||||
"debugger",
|
||||
"dll",
|
||||
"examples",
|
||||
"flags",
|
||||
"gc",
|
||||
"io",
|
||||
"js",
|
||||
"lib",
|
||||
"longgc",
|
||||
"manyloc",
|
||||
"nimble-all",
|
||||
"nimble-core",
|
||||
"nimble-extra",
|
||||
"niminaction",
|
||||
"rodfiles",
|
||||
"threads",
|
||||
"untestable",
|
||||
"stdlib",
|
||||
"testdata",
|
||||
"nimcache",
|
||||
"coroutines",
|
||||
"osproc"
|
||||
]
|
||||
|
||||
|
||||
# these tests still have bugs. At some point when the bugs are fixd
|
||||
# this should become empty.
|
||||
|
||||
# exclude for various reasons
|
||||
const
|
||||
specialDisabedTests = [
|
||||
"tests/dir with space/tspace.nim", # can't import dir with spaces.
|
||||
"tests/method/tmultim.nim", # (77, 8) Error: method is not a base
|
||||
"tests/system/talloc2.nim", # too much memory
|
||||
"tests/collections/ttables.nim", # takes too long
|
||||
"tests/system/tparams.nim", # executes itself with parameters
|
||||
"tests/stdlib/tquit.nim", # not testing for obvious reasons
|
||||
"tests/system/trealloc.nim", # out of memory
|
||||
"tests/system/t7894.nim", # causes out of memory in later tests
|
||||
"tests/types/tissues_types.nim", # causes out of memory with --gc:boehm
|
||||
"tests/pragmas/tused.nim", # paths in nimout differ when imported
|
||||
"tests/generics/trtree.nim", # very very ugly test
|
||||
"tests/array/tarray.nim", #
|
||||
"tests/destructor/turn_destroy_into_finalizer.nim", # fails when imported
|
||||
"tests/misc/tnew.nim",
|
||||
"tests/misc/tcmdline.nim"
|
||||
]
|
||||
|
||||
# included from tester.nim
|
||||
# ---------------- ROD file tests ---------------------------------------------
|
||||
|
||||
|
|
@ -327,24 +379,15 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
|
|||
let filename = "tests" / test.addFileExt("nim")
|
||||
let testHash = getMD5(readFile(filename).string)
|
||||
doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed."
|
||||
|
||||
|
||||
|
||||
# Run the tests.
|
||||
for testfile in tests:
|
||||
test "tests/" & testfile & ".nim"
|
||||
|
||||
let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim"
|
||||
testJS jsFile
|
||||
|
||||
let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim"
|
||||
testCPP cppFile
|
||||
|
||||
|
||||
# ------------------------- manyloc -------------------------------------------
|
||||
#proc runSpecialTests(r: var TResults, options: string) =
|
||||
# for t in ["lib/packages/docutils/highlite"]:
|
||||
# testSpec(r, t, options)
|
||||
|
||||
proc findMainFile(dir: string): string =
|
||||
# finds the file belonging to ".nim.cfg"; if there is no such file
|
||||
|
|
@ -382,20 +425,18 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
|
|||
for testFile in os.walkFiles(pattern):
|
||||
let name = extractFilename(testFile)
|
||||
if name notin disabledFiles:
|
||||
|
||||
|
||||
let contents = readFile(testFile).string
|
||||
|
||||
var testObj = makeTest(testFile, options, cat)
|
||||
if "when isMainModule" notin contents:
|
||||
testObj.spec.action = actionCompile
|
||||
testSpec r, testObj
|
||||
|
||||
# ----------------------------- nimble ----------------------------------------
|
||||
type PackageFilter = enum
|
||||
pfCoreOnly
|
||||
pfExtraOnly
|
||||
pfAll
|
||||
type
|
||||
PackageFilter = enum
|
||||
pfCoreOnly
|
||||
pfExtraOnly
|
||||
pfAll
|
||||
|
||||
var nimbleDir = getEnv("NIMBLE_DIR").string
|
||||
if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
|
||||
|
|
@ -425,7 +466,6 @@ proc getPackageDir(package: string): string =
|
|||
|
||||
iterator listPackages(filter: PackageFilter): tuple[name, url: string] =
|
||||
let packageList = parseFile(packageIndex)
|
||||
|
||||
for package in packageList.items():
|
||||
let
|
||||
name = package["name"].str
|
||||
|
|
@ -485,19 +525,28 @@ proc `&.?`(a, b: string): string =
|
|||
# candidate for the stdlib?
|
||||
result = if b.startswith(a): b else: a & b
|
||||
|
||||
#proc `&?.`(a, b: string): string = # not used
|
||||
# candidate for the stdlib?
|
||||
#result = if a.endswith(b): a else: a & b
|
||||
|
||||
proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
|
||||
let test = "tests" & DirSep &.? cat.string / test
|
||||
let target = if cat.string.normalize == "js": targetJS else: targetC
|
||||
|
||||
if existsFile(test):
|
||||
testSpec r, makeTest(test, options, cat), {target}
|
||||
else: echo "[Warning] - ", test, " test does not exist"
|
||||
else:
|
||||
echo "[Warning] - ", test, " test does not exist"
|
||||
|
||||
proc isJoinableSpec(spec: TSpec): bool
|
||||
proc isJoinableSpec(spec: TSpec): bool =
|
||||
result = not spec.sortoutput and
|
||||
spec.action == actionRun and
|
||||
spec.file.replace('\\', '/') notin specialDisabedTests and
|
||||
not fileExists(spec.file.changeFileExt("cfg")) and
|
||||
not fileExists(parentDir(spec.file) / "nim.cfg") and
|
||||
spec.cmd.len == 0 and
|
||||
spec.err != reIgnored and
|
||||
spec.exitCode == 0 and
|
||||
spec.input.len == 0 and
|
||||
spec.nimout.len == 0 and
|
||||
spec.outputCheck != ocSubstr and
|
||||
spec.ccodeCheck.len == 0 and
|
||||
(spec.targets == {} or spec.targets == {targetC})
|
||||
|
||||
proc processCategory(r: var TResults, cat: Category, options: string, runJoinableTests: bool) =
|
||||
case cat.string.normalize
|
||||
|
|
@ -560,191 +609,64 @@ proc processCategory(r: var TResults, cat: Category, options: string, runJoinabl
|
|||
if testsRun == 0:
|
||||
echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run"
|
||||
|
||||
proc norm(s: var string) =
|
||||
while true:
|
||||
let tmp = s.replace("\n\n", "\n")
|
||||
if tmp == s: break
|
||||
s = tmp
|
||||
s = s.strip
|
||||
|
||||
const specialCategories = [
|
||||
"async",
|
||||
"debugger",
|
||||
"dll",
|
||||
"examples",
|
||||
"flags",
|
||||
"gc",
|
||||
"io",
|
||||
"js",
|
||||
"lib",
|
||||
"longgc",
|
||||
"manyloc",
|
||||
"nimble-all",
|
||||
"nimble-core",
|
||||
"nimble-extra",
|
||||
"niminaction",
|
||||
"rodfiles",
|
||||
"threads",
|
||||
"untestable",
|
||||
"stdlib",
|
||||
]
|
||||
|
||||
|
||||
# these tests still have bugs. At some point when the bugs are fixd
|
||||
# this should become empty.
|
||||
|
||||
# exclude for various reasons
|
||||
const specialDisabedTests = [
|
||||
"tests/dir with space/tspace.nim", # can't import dir with spaces.
|
||||
"tests/method/tmultim.nim", # (77, 8) Error: method is not a base
|
||||
"tests/system/talloc2.nim", # too much memory
|
||||
"tests/collections/ttables.nim", # takes too long
|
||||
"tests/system/tparams.nim", # executes itself with parameters
|
||||
"tests/stdlib/tquit.nim", # not testing for obvious reasons
|
||||
"tests/system/trealloc.nim", # out of memory
|
||||
"tests/system/t7894.nim", # causes out of memory in later tests
|
||||
"tests/types/tissues_types.nim", # causes out of memory with --gc:boehm
|
||||
"tests/pragmas/tused.nim", # paths in nimout differ when imported
|
||||
"tests/generics/trtree.nim", # very very ugly test
|
||||
"tests/array/tarray.nim", #
|
||||
"tests/osproc/texecps.nim", # uses getAppFileName() to start itself with arguments
|
||||
"tests/destructor/turn_destroy_into_finalizer.nim", # fails when imported
|
||||
"tests/osproc/texitsignal.nim", # uses getAppFileName() to start itself with arguments
|
||||
]
|
||||
|
||||
proc isJoinableSpec(spec: TSpec): bool =
|
||||
|
||||
if spec.sortoutput:
|
||||
return false
|
||||
|
||||
if spec.action != actionRun:
|
||||
return false
|
||||
|
||||
if spec.file in specialDisabedTests:
|
||||
return false
|
||||
|
||||
if fileExists(spec.file & ".cfg"):
|
||||
return false
|
||||
|
||||
if fileExists(parentDir(spec.file) / "nim.cfg"):
|
||||
return false
|
||||
|
||||
if spec.cmd != cmdTemplate():
|
||||
return false
|
||||
|
||||
if spec.err == reIgnored:
|
||||
return false
|
||||
|
||||
if spec.exitCode != 0:
|
||||
return false
|
||||
|
||||
if spec.input != "":
|
||||
return false
|
||||
|
||||
if spec.targets != {} and spec.targets != {targetC}:
|
||||
return false
|
||||
|
||||
return true
|
||||
|
||||
|
||||
proc runJoinedTest(): bool =
|
||||
proc runJoinedTest(testsDir: string): bool =
|
||||
## returs a list of tests that have problems
|
||||
var specs:seq[TSpec]
|
||||
var specs: seq[TSpec] = @[]
|
||||
|
||||
for file in os.walkFiles("tests/*/t*.nim"):
|
||||
let a = find(file, '/') + 1
|
||||
let b = find(file, '/', a)
|
||||
let cat = file[a ..< b]
|
||||
|
||||
if cat in specialCategories:
|
||||
continue
|
||||
|
||||
let spec = parseSpec(file)
|
||||
|
||||
if isJoinableSpec(spec):
|
||||
specs.add spec
|
||||
for kind, dir in walkDir(testsDir):
|
||||
assert testsDir.startsWith(testsDir)
|
||||
let cat = dir[testsDir.len .. ^1]
|
||||
if kind == pcDir and cat notin specialCategories:
|
||||
for file in os.walkFiles(testsDir / cat / "t*.nim"):
|
||||
let spec = parseSpec(file)
|
||||
if isJoinableSpec(spec):
|
||||
specs.add spec
|
||||
|
||||
echo "joinable specs: ", specs.len
|
||||
|
||||
var megatest: string
|
||||
for runSpec in specs:
|
||||
megatest.add "import \""
|
||||
megatest.add "import r\""
|
||||
megatest.add runSpec.file
|
||||
megatest.add "\"\n"
|
||||
|
||||
writeFile("megatest.nim", megatest)
|
||||
|
||||
let args = ["c", "-d:testing", "--gc:boehm", "megatest.nim"]
|
||||
const args = ["c", "-d:testing", "--listCmd", "megatest.nim"]
|
||||
var (buf, exitCode) = execCmdEx2(command = "nim", args = args, options = {poStdErrToStdOut, poUsePath}, input = "")
|
||||
if exitCode != 0:
|
||||
echo buf
|
||||
quit("megatest compilation failed")
|
||||
|
||||
echo "compilation ok"
|
||||
|
||||
var nimoutOK = true
|
||||
for runSpec in specs:
|
||||
for line in runSpec.nimout.splitLines:
|
||||
if buf.find(line) < 0:
|
||||
echo "could not find: ", line
|
||||
echo runSpec.file
|
||||
nimoutOK = false
|
||||
|
||||
if nimoutOK:
|
||||
echo "nimout OK"
|
||||
else:
|
||||
echo "nimout FAIL"
|
||||
|
||||
(buf, exitCode) = execCmdEx2("./megatest", [], {poStdErrToStdOut}, "")
|
||||
if exitCode != 0:
|
||||
quit("megatest execution failed")
|
||||
|
||||
echo "run ok"
|
||||
|
||||
|
||||
norm buf
|
||||
writeFile("outputGotten.txt", buf)
|
||||
var outputExpected = ""
|
||||
|
||||
var outputErrorCount = 0
|
||||
var currentPos = 0
|
||||
|
||||
var lastLine = ""
|
||||
|
||||
# when a lot of output is skipped, this can be the cause why a later test fails.
|
||||
var warnings = ""
|
||||
|
||||
for i, runSpec in specs:
|
||||
outputExpected.add runSpec.output
|
||||
if outputExpected[^1] != '\n':
|
||||
outputExpected.add '\n'
|
||||
outputExpected.add runSpec.output.strip
|
||||
outputExpected.add '\n'
|
||||
norm outputExpected
|
||||
|
||||
for line in runSpec.output.splitLines:
|
||||
if line != "":
|
||||
#if line == "2":
|
||||
# echo "found the test: ", runSpec.file
|
||||
let newPos = buf.find(line, currentPos)
|
||||
if newPos < 0:
|
||||
if outputErrorCount < 5:
|
||||
echo "could not find: ", line
|
||||
echo "it could be, because the test failed, or too much output is discarded by a previous search in the output."
|
||||
echo warnings
|
||||
warnings.setLen 0
|
||||
|
||||
# don't spam too much of this
|
||||
if outputErrorCount == 0:
|
||||
echo "############"
|
||||
echo buf[currentPos-200 ..< currentPos]
|
||||
echo "| (", current_pos, ")"
|
||||
echo buf[currentPos ..< min(currentPos+200, buf.len)]
|
||||
echo "############"
|
||||
|
||||
inc outputErrorCount
|
||||
else:
|
||||
if currentPos + lastLine.len * 2 < newPos:
|
||||
warnings.addLine "Warning long skip in search for: ", line
|
||||
warnings.addLine "in test: ", runSpec.file
|
||||
currentPos = newPos + line.len
|
||||
|
||||
lastLine = line
|
||||
if outputErrorCount == 0:
|
||||
echo "output OK"
|
||||
if buf != outputExpected:
|
||||
writeFile("outputExpected.txt", outputExpected)
|
||||
discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt")
|
||||
echo "output different!"
|
||||
result = false
|
||||
else:
|
||||
echo "output FAIL (", outputErrorCount, " errors)"
|
||||
|
||||
writeFile("outputExpected.txt", outputExpected)
|
||||
|
||||
# removeFile("megatest.nim")
|
||||
return nimoutOK and outputErrorCount == 0
|
||||
echo "output OK"
|
||||
removeFile("megatest.nim")
|
||||
result = true
|
||||
|
|
|
|||
|
|
@ -14,9 +14,6 @@ var compilerPrefix* = "compiler" / "nim"
|
|||
let isTravis* = existsEnv("TRAVIS")
|
||||
let isAppVeyor* = existsEnv("APPVEYOR")
|
||||
|
||||
proc cmdTemplate*(): string =
|
||||
compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file"
|
||||
|
||||
type
|
||||
TTestAction* = enum
|
||||
actionRun = "run"
|
||||
|
|
@ -70,6 +67,12 @@ type
|
|||
nimout*: string
|
||||
parseErrors*: string # when the spec definition is invalid, this is not empty.
|
||||
|
||||
proc getCmd*(s: TSpec): string =
|
||||
if s.cmd.len == 0:
|
||||
result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file"
|
||||
else:
|
||||
result = s.cmd
|
||||
|
||||
const
|
||||
targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"]
|
||||
targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"]
|
||||
|
|
@ -97,9 +100,6 @@ proc extractSpec(filename: string): string =
|
|||
when not defined(nimhygiene):
|
||||
{.pragma: inject.}
|
||||
|
||||
proc defaultSpec*(): TSpec =
|
||||
result.cmd = cmdTemplate()
|
||||
|
||||
proc parseTargets*(value: string): set[TTarget] =
|
||||
for v in value.normalize.splitWhitespace:
|
||||
case v
|
||||
|
|
@ -119,7 +119,6 @@ proc addLine*(self: var string; a,b: string) =
|
|||
self.add "\n"
|
||||
|
||||
proc parseSpec*(filename: string): TSpec =
|
||||
result = defaultSpec()
|
||||
result.file = filename
|
||||
let specStr = extractSpec(filename)
|
||||
var ss = newStringStream(specStr)
|
||||
|
|
|
|||
|
|
@ -378,13 +378,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
|
|||
|
||||
case expected.action
|
||||
of actionCompile:
|
||||
var given = callCompiler(expected.cmd, test.name, test.options, target,
|
||||
var given = callCompiler(expected.getCmd, test.name, test.options, target,
|
||||
extraOptions=" --stdout --hint[Path]:off --hint[Processing]:off")
|
||||
compilerOutputTests(test, target, given, expected, r)
|
||||
of actionRun:
|
||||
# In this branch of code "early return" pattern is clearer than deep
|
||||
# nested conditionals - the empty rows in between to clarify the "danger"
|
||||
var given = callCompiler(expected.cmd, test.name, test.options,
|
||||
var given = callCompiler(expected.getCmd, test.name, test.options,
|
||||
target)
|
||||
|
||||
if given.err != reSuccess:
|
||||
|
|
@ -438,7 +438,7 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
|
|||
bufB, reExitCodesDiffer)
|
||||
continue
|
||||
|
||||
if (expected.outputCheck == ocEqual and expected.output != bufB) or
|
||||
if (expected.outputCheck == ocEqual and expected.output != bufB) or
|
||||
(expected.outputCheck == ocSubstr and expected.output notin bufB):
|
||||
given.err = reOutputsDiffer
|
||||
r.addResult(test, target, expected.output, bufB, reOutputsDiffer)
|
||||
|
|
@ -448,7 +448,7 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
|
|||
continue
|
||||
|
||||
of actionReject:
|
||||
var given = callCompiler(expected.cmd, test.name, test.options,
|
||||
var given = callCompiler(expected.getCmd, test.name, test.options,
|
||||
target)
|
||||
cmpMsgs(r, expected, given, test, target)
|
||||
continue
|
||||
|
|
@ -458,7 +458,7 @@ proc testC(r: var TResults, test: TTest, action: TTestAction) =
|
|||
let tname = test.name.addFileExt(".c")
|
||||
inc(r.total)
|
||||
maybeStyledEcho "Processing ", fgCyan, extractFilename(tname)
|
||||
var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC)
|
||||
var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC)
|
||||
if given.err != reSuccess:
|
||||
r.addResult(test, targetC, "", given.msg, given.err)
|
||||
elif action == actionRun:
|
||||
|
|
@ -471,7 +471,7 @@ proc testExec(r: var TResults, test: TTest) =
|
|||
# runs executable or script, just goes by exit code
|
||||
inc(r.total)
|
||||
let (outp, errC) = execCmdEx(test.options.strip())
|
||||
var given: TSpec = defaultSpec()
|
||||
var given: TSpec
|
||||
if errC == 0:
|
||||
given.err = reSuccess
|
||||
else:
|
||||
|
|
@ -497,15 +497,9 @@ else:
|
|||
# array of modules disabled from compilation test of stdlib.
|
||||
disabledFiles = ["-"]
|
||||
|
||||
|
||||
|
||||
|
||||
include categories
|
||||
|
||||
# proc runCaasTests(r: var TResults) =
|
||||
# for test, output, status, mode in caasTestsRunner():
|
||||
# r.addResult(test, "", output & "-> " & $mode,
|
||||
# if status: reSuccess else: reOutputsDiffer)
|
||||
const testsDir = "tests" & DirSep
|
||||
|
||||
proc main() =
|
||||
os.putenv "NIMTEST_COLOR", "never"
|
||||
|
|
@ -556,9 +550,8 @@ proc main() =
|
|||
var r = initResults()
|
||||
case action
|
||||
of "all":
|
||||
doAssert runJoinedTest()
|
||||
doAssert runJoinedTest(testsDir)
|
||||
|
||||
let testsDir = "tests" & DirSep
|
||||
var myself = quoteShell(findExe("testament" / "tester"))
|
||||
if targetsStr.len > 0:
|
||||
myself &= " " & quoteShell("--targets:" & targetsStr)
|
||||
|
|
@ -587,7 +580,7 @@ proc main() =
|
|||
of "html":
|
||||
generateHtml(resultsFile, optFailing)
|
||||
of "stats":
|
||||
discard runJoinedTest()
|
||||
discard runJoinedTest(testsDir)
|
||||
else:
|
||||
quit Usage
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue