[Testament] Extend and document message testing aids (#19996)
* [Testament] Extend and document message testing aids * Enable inline msgs when not reject action. Eliminates the pain of changing the line and column numbers in `nimout` or `output` while making changes to the test. * Enable using inline msgs and nimout together. Allows ease of inline msgs for the test as well as testing msgs from other modules. * Add path separator and test filename variable interpolation in msgs. Eases handling path separators in the msgs. * Add some documentation. * Fixed lots of broken tests * Fixed more broken tests * Support multiple inline messages per a line * Fix a broken test * Revert variable substitution in `output` * Remove uneeded params * Update doc/testament.md Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com> * Update testament/specs.nim Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com> * Update testament/specs.nim Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com> * Fix indentation Co-authored-by: quantimnot <quantimnot@users.noreply.github.com> Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>
This commit is contained in:
quantimnot
GitHub
parent
c2cdc752c8
commit
6289b002b6
15 changed files with 247 additions and 149 deletions
|
|
@ -75,6 +75,7 @@ type
|
|||
# xxx make sure `isJoinableSpec` takes into account each field here.
|
||||
action*: TTestAction
|
||||
file*, cmd*: string
|
||||
filename*: string ## Test filename (without path).
|
||||
input*: string
|
||||
outputCheck*: TOutputCheck
|
||||
sortoutput*: bool
|
||||
|
|
@ -127,19 +128,56 @@ when not declared(parseCfgBool):
|
|||
of "n", "no", "false", "0", "off": result = false
|
||||
else: raise newException(ValueError, "cannot interpret as a bool: " & s)
|
||||
|
||||
proc addLine*(self: var string; pieces: varargs[string]) =
|
||||
for piece in pieces:
|
||||
self.add piece
|
||||
self.add "\n"
|
||||
|
||||
|
||||
const
|
||||
inlineErrorMarker = "#[tt."
|
||||
inlineErrorKindMarker = "tt."
|
||||
inlineErrorMarker = "#[" & inlineErrorKindMarker
|
||||
|
||||
proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int =
|
||||
## Extract inline error messages.
|
||||
##
|
||||
## Can parse a single message for a line:
|
||||
##
|
||||
## .. code-block:: nim
|
||||
##
|
||||
## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
|
||||
## ^ 'generic_proc' should be: 'genericProc' [Name] ]#
|
||||
##
|
||||
## Can parse multiple messages for a line when they are separated by ';':
|
||||
##
|
||||
## .. code-block:: nim
|
||||
##
|
||||
## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
|
||||
## ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error
|
||||
## ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error
|
||||
## ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
|
||||
##
|
||||
## .. code-block:: nim
|
||||
##
|
||||
## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
|
||||
## ^ 'generic_proc' should be: 'genericProc' [Name];
|
||||
## tt.Error ^ 'no_destroy' should be: 'nodestroy' [Name];
|
||||
## tt.Error ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
|
||||
##
|
||||
result = i + len(inlineErrorMarker)
|
||||
inc col, len(inlineErrorMarker)
|
||||
let msgLine = line
|
||||
var msgCol = -1
|
||||
var msg = ""
|
||||
var kind = ""
|
||||
while result < s.len and s[result] in IdentChars:
|
||||
kind.add s[result]
|
||||
inc result
|
||||
inc col
|
||||
|
||||
var caret = (line, -1)
|
||||
template parseKind =
|
||||
while result < s.len and s[result] in IdentChars:
|
||||
kind.add s[result]
|
||||
inc result
|
||||
inc col
|
||||
if kind notin ["Hint", "Warning", "Error"]:
|
||||
spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error"
|
||||
|
||||
template skipWhitespace =
|
||||
while result < s.len and s[result] in Whitespace:
|
||||
|
|
@ -150,34 +188,70 @@ proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var T
|
|||
inc col
|
||||
inc result
|
||||
|
||||
skipWhitespace()
|
||||
if result < s.len and s[result] == '^':
|
||||
caret = (line-1, col)
|
||||
inc result
|
||||
inc col
|
||||
skipWhitespace()
|
||||
template parseCaret =
|
||||
if result < s.len and s[result] == '^':
|
||||
msgCol = col
|
||||
inc result
|
||||
inc col
|
||||
skipWhitespace()
|
||||
else:
|
||||
spec.parseErrors.addLine "expected column marker ('^') for inline message"
|
||||
|
||||
template isMsgDelimiter: bool =
|
||||
s[result] == ';' and
|
||||
(block:
|
||||
let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1)
|
||||
if s.len > nextTokenIdx + len(inlineErrorKindMarker) and
|
||||
s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker:
|
||||
true
|
||||
else:
|
||||
false)
|
||||
|
||||
template trimTrailingMsgWhitespace =
|
||||
while msg.len > 0 and msg[^1] in Whitespace:
|
||||
setLen msg, msg.len - 1
|
||||
|
||||
template addInlineError =
|
||||
doAssert msg[^1] notin Whitespace
|
||||
if kind == "Error": spec.action = actionReject
|
||||
spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol)
|
||||
|
||||
parseKind()
|
||||
skipWhitespace()
|
||||
parseCaret()
|
||||
|
||||
var msg = ""
|
||||
while result < s.len-1:
|
||||
if s[result] == '\n':
|
||||
msg.add '\n'
|
||||
inc result
|
||||
inc line
|
||||
col = 1
|
||||
elif s[result] == ']' and s[result+1] == '#':
|
||||
while msg.len > 0 and msg[^1] in Whitespace:
|
||||
setLen msg, msg.len - 1
|
||||
|
||||
elif isMsgDelimiter():
|
||||
trimTrailingMsgWhitespace()
|
||||
inc result
|
||||
skipWhitespace()
|
||||
addInlineError()
|
||||
inc result, len(inlineErrorKindMarker)
|
||||
inc col, 1 + len(inlineErrorKindMarker)
|
||||
kind.setLen 0
|
||||
msg.setLen 0
|
||||
parseKind()
|
||||
skipWhitespace()
|
||||
parseCaret()
|
||||
elif s[result] == ']' and s[result+1] == '#':
|
||||
trimTrailingMsgWhitespace()
|
||||
inc result, 2
|
||||
inc col, 2
|
||||
if kind == "Error": spec.action = actionReject
|
||||
spec.unjoinable = true
|
||||
spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: caret[0], col: caret[1])
|
||||
addInlineError()
|
||||
break
|
||||
else:
|
||||
msg.add s[result]
|
||||
inc result
|
||||
inc col
|
||||
|
||||
if spec.inlineErrors.len > 0:
|
||||
spec.unjoinable = true
|
||||
|
||||
proc extractSpec(filename: string; spec: var TSpec): string =
|
||||
const
|
||||
tripleQuote = "\"\"\""
|
||||
|
|
@ -229,15 +303,6 @@ proc parseTargets*(value: string): set[TTarget] =
|
|||
of "js": result.incl(targetJS)
|
||||
else: raise newException(ValueError, "invalid target: '$#'" % v)
|
||||
|
||||
proc addLine*(self: var string; a: string) =
|
||||
self.add a
|
||||
self.add "\n"
|
||||
|
||||
proc addLine*(self: var string; a, b: string) =
|
||||
self.add a
|
||||
self.add b
|
||||
self.add "\n"
|
||||
|
||||
proc initSpec*(filename: string): TSpec =
|
||||
result.file = filename
|
||||
|
||||
|
|
@ -249,6 +314,7 @@ proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool =
|
|||
|
||||
proc parseSpec*(filename: string): TSpec =
|
||||
result.file = filename
|
||||
result.filename = extractFilename(filename)
|
||||
let specStr = extractSpec(filename, result)
|
||||
var ss = newStringStream(specStr)
|
||||
var p: CfgParser
|
||||
|
|
@ -439,3 +505,15 @@ proc parseSpec*(filename: string): TSpec =
|
|||
result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable
|
||||
if not result.inCurrentBatch:
|
||||
result.err = reDisabled
|
||||
|
||||
# Interpolate variables in msgs:
|
||||
template varSub(msg: string): string =
|
||||
try:
|
||||
msg % ["/", $DirSep, "file", result.filename]
|
||||
except ValueError:
|
||||
result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unitests-output-message-variable-interpolation')"
|
||||
msg
|
||||
result.nimout = result.nimout.varSub
|
||||
result.msg = result.msg.varSub
|
||||
for inlineError in result.inlineErrors.mitems:
|
||||
inlineError.msg = inlineError.msg.varSub
|
||||
|
|
|
|||
|
|
@ -338,46 +338,24 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
|
|||
discard waitForExit(p)
|
||||
close(p)
|
||||
|
||||
proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest,
|
||||
target: TTarget, extraOptions: string) =
|
||||
let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}"
|
||||
var covered = initIntSet()
|
||||
for line in splitLines(given.nimout):
|
||||
proc toString(inlineError: InlineError, filename: string): string =
|
||||
result.add "$file($line, $col) $kind: $msg" % [
|
||||
"file", filename,
|
||||
"line", $inlineError.line,
|
||||
"col", $inlineError.col,
|
||||
"kind", $inlineError.kind,
|
||||
"msg", $inlineError.msg
|
||||
]
|
||||
|
||||
if line =~ pegLine:
|
||||
let file = extractFilename(matches[0])
|
||||
let line = try: parseInt(matches[1]) except: -1
|
||||
let col = try: parseInt(matches[2]) except: -1
|
||||
let kind = matches[3]
|
||||
let msg = matches[4]
|
||||
proc inlineErrorsMsgs(expected: TSpec): string =
|
||||
for inlineError in expected.inlineErrors.items:
|
||||
result.addLine inlineError.toString(expected.filename)
|
||||
|
||||
if file == extractFilename test.name:
|
||||
var i = 0
|
||||
for x in expected.inlineErrors:
|
||||
if x.line == line and (x.col == col or x.col < 0) and
|
||||
x.kind == kind and x.msg in msg:
|
||||
covered.incl i
|
||||
inc i
|
||||
|
||||
block coverCheck:
|
||||
for j in 0..high(expected.inlineErrors):
|
||||
if j notin covered:
|
||||
var e = test.name
|
||||
e.add '('
|
||||
e.addInt expected.inlineErrors[j].line
|
||||
if expected.inlineErrors[j].col > 0:
|
||||
e.add ", "
|
||||
e.addInt expected.inlineErrors[j].col
|
||||
e.add ") "
|
||||
e.add expected.inlineErrors[j].kind
|
||||
e.add ": "
|
||||
e.add expected.inlineErrors[j].msg
|
||||
|
||||
r.addResult(test, target, extraOptions, e, given.nimout, reMsgsDiffer)
|
||||
break coverCheck
|
||||
|
||||
r.addResult(test, target, extraOptions, "", given.msg, reSuccess)
|
||||
inc(r.passed)
|
||||
proc checkForInlineErrors(expected, given: TSpec): bool =
|
||||
for inlineError in expected.inlineErrors:
|
||||
if inlineError.toString(expected.filename) notin given.nimout:
|
||||
return false
|
||||
true
|
||||
|
||||
proc nimoutCheck(expected, given: TSpec): bool =
|
||||
result = true
|
||||
|
|
@ -389,15 +367,16 @@ proc nimoutCheck(expected, given: TSpec): bool =
|
|||
|
||||
proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest,
|
||||
target: TTarget, extraOptions: string) =
|
||||
if expected.inlineErrors.len > 0:
|
||||
checkForInlineErrors(r, expected, given, test, target, extraOptions)
|
||||
if not checkForInlineErrors(expected, given) or
|
||||
(not expected.nimoutFull and not nimoutCheck(expected, given)):
|
||||
r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer)
|
||||
elif strip(expected.msg) notin strip(given.msg):
|
||||
r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer)
|
||||
elif not nimoutCheck(expected, given):
|
||||
r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer)
|
||||
elif extractFilename(expected.file) != extractFilename(given.file) and
|
||||
"internal error:" notin expected.msg:
|
||||
r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer)
|
||||
r.addResult(test, target, extraOptions, expected.filename, given.file, reFilesDiffer)
|
||||
elif expected.line != given.line and expected.line != 0 or
|
||||
expected.column != given.column and expected.column != 0:
|
||||
r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column,
|
||||
|
|
@ -449,9 +428,10 @@ proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string,
|
|||
if expected.needsCodegenCheck:
|
||||
codegenCheck(test, target, expected, expectedmsg, given)
|
||||
givenmsg = given.msg
|
||||
if not nimoutCheck(expected, given):
|
||||
if not nimoutCheck(expected, given) or
|
||||
not checkForInlineErrors(expected, given):
|
||||
given.err = reMsgsDiffer
|
||||
expectedmsg = expected.nimout
|
||||
expectedmsg = expected.nimout & inlineErrorsMsgs(expected)
|
||||
givenmsg = given.nimout.strip
|
||||
else:
|
||||
givenmsg = "$ " & given.cmd & '\n' & given.nimout
|
||||
|
|
@ -480,17 +460,11 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
|
|||
r.addResult(test, target, extraOptions, "", "", test.spec.err)
|
||||
inc(r.skipped)
|
||||
return
|
||||
|
||||
template callNimCompilerImpl(): untyped =
|
||||
# xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently
|
||||
# with other branches
|
||||
callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
|
||||
var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
|
||||
case expected.action
|
||||
of actionCompile:
|
||||
var given = callNimCompilerImpl()
|
||||
compilerOutputTests(test, target, extraOptions, given, expected, r)
|
||||
of actionRun:
|
||||
var given = callNimCompilerImpl()
|
||||
if given.err != reSuccess:
|
||||
r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr)
|
||||
else:
|
||||
|
|
@ -540,10 +514,8 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
|
|||
(expected.outputCheck == ocSubstr and expected.output notin bufB):
|
||||
given.err = reOutputsDiffer
|
||||
r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer)
|
||||
else:
|
||||
compilerOutputTests(test, target, extraOptions, given, expected, r)
|
||||
compilerOutputTests(test, target, extraOptions, given, expected, r)
|
||||
of actionReject:
|
||||
let given = callNimCompilerImpl()
|
||||
cmpMsgs(r, expected, given, test, target, extraOptions)
|
||||
|
||||
proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) =
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue