tester support html generation

This commit is contained in:
Araq 2014-01-14 01:12:56 +01:00
commit c648a5774a
8 changed files with 141 additions and 20 deletions

View file

@ -1,7 +1,7 @@
#
#
# The Nimrod Tester
# (c) Copyright 2013 Andreas Rumpf
# (c) Copyright 2014 Andreas Rumpf
#
# Look at license.txt for more info.
# All rights reserved.
@ -49,10 +49,10 @@ proc createDb() =
# """, [])
type
MachineId = distinct int64
MachineId* = distinct int64
CommitId = distinct int64
proc `$`(id: MachineId): string {.borrow.}
proc `$`*(id: MachineId): string {.borrow.}
proc `$`(id: CommitId): string {.borrow.}
var
@ -61,7 +61,7 @@ var
proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip
proc getMachine: MachineId =
proc getMachine*: MachineId =
var name = "hostname"()
if name.len == 0:
name = when defined(posix): getenv"HOSTNAME".string
@ -73,7 +73,7 @@ proc getMachine: MachineId =
if id.len > 0:
result = id.parseInt.MachineId
else:
result = db.insertId(sql"insert into Machine(name, os, cpu) values (?,?,?)",
result = db.insertId(sql"insert into Machine(name, os, cpu) values (?,?,?)",
name, system.hostOS, system.hostCPU).MachineId
proc getCommit: CommitId =

View file

@ -0,0 +1,184 @@
import osproc, streams, os, strutils, re
## Compiler as a service tester.
##
## Please read docs/idetools.txt for information about this.
type
TRunMode = enum
ProcRun, CaasRun, SymbolProcRun
TNimrodSession* = object
nim: PProcess # Holds the open process for CaasRun sessions, nil otherwise.
mode: TRunMode # Stores the type of run mode the session was started with.
lastOutput: string # Preserves the last output, needed for ProcRun mode.
filename: string # Appended to each command starting with '>'. Also a var.
modname: string # Like filename but without extension.
nimcache: string # Input script based name for the nimcache dir.
const
modes = [CaasRun, ProcRun, SymbolProcRun]
filenameReplaceVar = "$TESTNIM"
moduleReplaceVar = "$MODULE"
silentReplaceVar = "$SILENT"
silentReplaceText = "--verbosity:0 --hints:off"
var
TesterDir = getAppDir()
NimrodBin = TesterDir / "../bin/nimrod"
proc replaceVars(session: var TNimrodSession, text: string): string =
result = text.replace(filenameReplaceVar, session.filename)
result = result.replace(moduleReplaceVar, session.modname)
result = result.replace(silentReplaceVar, silentReplaceText)
proc startNimrodSession(project, script: string, mode: TRunMode):
TNimrodSession =
let (dir, name, ext) = project.splitFile
result.mode = mode
result.lastOutput = ""
result.filename = name & ext
result.modname = name
let (nimcacheDir, nimcacheName, nimcacheExt) = script.splitFile
result.nimcache = "SymbolProcRun." & nimcacheName
if mode == SymbolProcRun:
removeDir(nimcacheDir / result.nimcache)
else:
removeDir(nimcacheDir / "nimcache")
if mode == CaasRun:
result.nim = startProcess(NimrodBin, workingDir = dir,
args = ["serve", "--server.type:stdin", name])
proc doCaasCommand(session: var TNimrodSession, command: string): string =
assert session.mode == CaasRun
session.nim.inputStream.write(session.replaceVars(command) & "\n")
session.nim.inputStream.flush
result = ""
while true:
var line = TaintedString("")
if session.nim.outputStream.readLine(line):
if line.string == "": break
result.add(line.string & "\n")
else:
result = "FAILED TO EXECUTE: " & command & "\n" & result
break
proc doProcCommand(session: var TNimrodSession, command: string): string =
assert session.mode == ProcRun or session.mode == SymbolProcRun
except: result = "FAILED TO EXECUTE: " & command & "\n" & result
var
process = startProcess(NimrodBin, args = session.replaceVars(command).split)
stream = outputStream(process)
line = TaintedString("")
result = ""
while stream.readLine(line):
if result.len > 0: result &= "\n"
result &= line.string
process.close()
proc doCommand(session: var TNimrodSession, command: string) =
if session.mode == CaasRun:
session.lastOutput = doCaasCommand(session,
command & " " & session.filename)
else:
var command = command
# For symbol runs we prepend the necessary parameters to avoid clobbering
# the normal nimcache.
if session.mode == SymbolProcRun:
command = "--symbolFiles:on --nimcache:" & session.nimcache &
" " & command
session.lastOutput = doProcCommand(session,
command & " " & session.filename)
proc close(session: var TNimrodSession) {.destructor.} =
if session.mode == CaasRun:
session.nim.close
proc doScenario(script: string, output: PStream, mode: TRunMode): bool =
result = true
var f = open(script)
var project = TaintedString("")
if f.readLine(project):
var
s = startNimrodSession(script.parentDir / project.string, script, mode)
tline = TaintedString("")
ln = 1
while f.readLine(tline):
var line = tline.string
inc ln
# Filter lines by run mode, removing the prefix if the mode is current.
for testMode in modes:
if line.startsWith($testMode):
if testMode != mode:
line = ""
else:
line = line[len($testMode)..len(line) - 1].strip
break
if line.strip.len == 0: continue
if line.startsWith("#"):
output.writeln line
continue
elif line.startsWith(">"):
s.doCommand(line.substr(1).strip)
output.writeln line, "\n", s.lastOutput
else:
var expectMatch = true
var pattern = s.replaceVars(line)
if line.startsWith("!"):
pattern = pattern.substr(1).strip
expectMatch = false
let actualMatch =
s.lastOutput.find(re(pattern, flags = {reStudy})) != -1
if expectMatch == actualMatch:
output.writeln "SUCCESS ", line
else:
output.writeln "FAILURE ", line
result = false
iterator caasTestsRunner*(filter = ""): tuple[test, output: string,
status: bool, mode: TRunMode] =
for scenario in os.walkFiles(TesterDir / "caas/*.txt"):
if filter.len > 0 and find(scenario, filter) == -1: continue
for mode in modes:
var outStream = newStringStream()
let r = doScenario(scenario, outStream, mode)
yield (scenario, outStream.data, r, mode)
when isMainModule:
var
filter = ""
failures = 0
verbose = false
for i in 0..ParamCount() - 1:
let param = string(paramStr(i + 1))
case param
of "verbose": verbose = true
else: filter = param
if verbose and len(filter) > 0:
echo "Running only test cases matching filter '$1'" % [filter]
for test, output, result, mode in caasTestsRunner(filter):
if not result or verbose:
echo test, "\n", output, "-> ", $mode, ":", $result, "\n-----"
if not result:
failures += 1
quit(failures)

View file

@ -0,0 +1,138 @@
/* ==== Scroll down to find where to put your styles :) ==== */
/* HTML5 ✰ Boilerplate */
html, body, div, span, object, iframe,
h1, h2, h3, h4, h5, h6, p, blockquote, pre,
abbr, address, cite, code, del, dfn, em, img, ins, kbd, q, samp,
small, strong, sub, sup, var, b, i, dl, dt, dd, ol, ul, li,
fieldset, form, label, legend,
table, caption, tbody, tfoot, thead, tr, th, td,
article, aside, canvas, details, figcaption, figure,
footer, header, hgroup, menu, nav, section, summary,
time, mark, audio, video {
margin: 0;
padding: 0;
border: 0;
font-size: 100%;
font: inherit;
vertical-align: baseline;
}
article, aside, details, figcaption, figure,
footer, header, hgroup, menu, nav, section {
display: block;
}
blockquote, q { quotes: none; }
blockquote:before, blockquote:after,
q:before, q:after { content: ''; content: none; }
ins { background-color: #ff9; color: #000; text-decoration: none; }
mark { background-color: #ff9; color: #000; font-style: italic; font-weight: bold; }
del { text-decoration: line-through; }
abbr[title], dfn[title] { border-bottom: 1px dotted; cursor: help; }
table { border-collapse: collapse; border-spacing: 0; }
hr { display: block; height: 1px; border: 0; border-top: 1px solid #ccc; margin: 1em 0; padding: 0; }
input, select { vertical-align: middle; }
body { font:13px/1.231 sans-serif; *font-size:small; }
select, input, textarea, button { font:99% sans-serif; }
pre, code, kbd, samp { font-family: monospace, sans-serif; }
html { overflow-y: scroll; }
a:hover, a:active { outline: none; }
ul, ol { margin-left: 2em; }
ol { list-style-type: decimal; }
nav ul, nav li { margin: 0; list-style:none; list-style-image: none; }
small { font-size: 85%; }
strong, th { font-weight: bold; }
td { vertical-align: top; }
sub, sup { font-size: 75%; line-height: 0; position: relative; }
sup { top: -0.5em; }
sub { bottom: -0.25em; }
pre { white-space: pre; white-space: pre-wrap; word-wrap: break-word; padding: 15px; }
textarea { overflow: auto; }
.ie6 legend, .ie7 legend { margin-left: -7px; }
input[type="radio"] { vertical-align: text-bottom; }
input[type="checkbox"] { vertical-align: bottom; }
.ie7 input[type="checkbox"] { vertical-align: baseline; }
.ie6 input { vertical-align: text-bottom; }
label, input[type="button"], input[type="submit"], input[type="image"], button { cursor: pointer; }
button, input, select, textarea { margin: 0; }
input:valid, textarea:valid { }
input:invalid, textarea:invalid { border-radius: 1px; -moz-box-shadow: 0px 0px 5px red; -webkit-box-shadow: 0px 0px 5px red; box-shadow: 0px 0px 5px red; }
.no-boxshadow input:invalid, .no-boxshadow textarea:invalid { background-color: #f0dddd; }
a:link { -webkit-tap-highlight-color: #FF5E99; }
button { width: auto; overflow: visible; }
.ie7 img { -ms-interpolation-mode: bicubic; }
body, select, input, textarea { color: #444; }
h1, h2, h3, h4, h5, h6 { font-weight: bold; }
a, a:active, a:visited { color: #607890; }
a:hover { color: #036; }
/*
// ========================================== \\
|| ||
|| Your styles ! ||
|| ||
\\ ========================================== //
*/
.ir { display: block; text-indent: -999em; overflow: hidden; background-repeat: no-repeat; text-align: left; direction: ltr; }
.hidden { display: none; visibility: hidden; }
.visuallyhidden { border: 0; clip: rect(0 0 0 0); height: 1px; margin: -1px; overflow: hidden; padding: 0; position: absolute; width: 1px; }
.visuallyhidden.focusable:active,
.visuallyhidden.focusable:focus { clip: auto; height: auto; margin: 0; overflow: visible; position: static; width: auto; }
.invisible { visibility: hidden; }
.clearfix:before, .clearfix:after { content: "\0020"; display: block; height: 0; overflow: hidden; }
.clearfix:after { clear: both; }
.clearfix { zoom: 1; }
@media all and (orientation:portrait) {
}
@media all and (orientation:landscape) {
}
@media screen and (max-device-width: 480px) {
/* html { -webkit-text-size-adjust:none; -ms-text-size-adjust:none; } */
}
@media print {
* { background: transparent !important; color: black !important; text-shadow: none !important; filter:none !important;
-ms-filter: none !important; }
a, a:visited { color: #444 !important; text-decoration: underline; }
a[href]:after { content: " (" attr(href) ")"; }
abbr[title]:after { content: " (" attr(title) ")"; }
.ir a:after, a[href^="javascript:"]:after, a[href^="#"]:after { content: ""; }
pre, blockquote { border: 1px solid #999; page-break-inside: avoid; }
thead { display: table-header-group; }
tr, img { page-break-inside: avoid; }
@page { margin: 0.5cm; }
p, h2, h3 { orphans: 3; widows: 3; }
h2, h3{ page-break-after: avoid; }
}

View file

@ -0,0 +1,114 @@
body {
font-size: medium;
}
div#header {
font-size: 2em;
background-color: #3d3d3d;
border-bottom: solid 2px #000000;
padding: 0.25em;
color: #ffffff;
}
div#content {
margin: 0.5em;
}
table {
text-align: left;
margin-bottom: 0.5em;
}
table td, table th {
padding: 0.15em 0.5em;
}
tr:nth-child(even) {
background-color: #eee;
}
/* Awesome buttons :P */
a.button {
border-radius: 2px 2px 2px 2px;
background: -moz-linear-gradient(top, #f7f7f7, #ebebeb);
background: -webkit-linear-gradient(top, #f7f7f7, #ebebeb);
background: -o-linear-gradient(top, #f7f7f7, #ebebeb);
text-decoration: none;
color: #3d3d3d;
padding: 5px;
border: solid 1px #9d9d9d;
display: inline-block;
position: relative;
text-align: center;
font-size: small;
}
a.button.active {
background: -moz-linear-gradient(top, #00B40C, #03A90E);
background: -webkit-linear-gradient(top, #00B40C, #03A90E);
background: -o-linear-gradient(top, #00B40C, #03A90E);
border: solid 1px #148420;
color: #ffffff;
}
a.button.left {
border-top-right-radius: 0;
border-bottom-right-radius: 0;
}
a.button.middle {
border-radius: 0;
border-left: 0;
}
a.button.right {
border-top-left-radius: 0;
border-bottom-left-radius: 0;
border-left: 0;
}
a.button:hover {
background: -moz-linear-gradient(top, #0099c7, #0294C1);
background: -webkit-linear-gradient(top, #0099c7, #0294C1);
background: -o-linear-gradient(top, #0099c7, #0294C1);
border: solid 1px #077A9C;
color: #ffffff;
}
a.button.middle:hover, a.button.right:hover {
border-left: 0;
}
a.button span.download {
background-image: url("../images/icons.png");
background-repeat: no-repeat;
display: inline-block;
margin: auto 3px auto auto;
height: 15px;
width: 14px;
position: relative;
background-position: 0 -30px;
top: 3px;
}
a.button span.book {
background-image: url("../images/icons.png");
background-repeat: no-repeat;
display: inline-block;
margin: auto 3px auto auto;
height: 15px;
width: 14px;
position: relative;
background-position: 0 0;
top: 3px;
}
a.button.active span.download, a.button:hover span.download {
background-position: 0 -45px;
}
a.button.active span.book, a.button:hover span.book {
background-position: 0 -15px;
}

127
tests/testament/htmlgen.nim Normal file
View file

@ -0,0 +1,127 @@
#
#
# Nimrod Tester
# (c) Copyright 2014 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## HTML generator for the tester.
import db_sqlite, cgi, backend, strutils
const
TableHeader = """<table border="1">
<tr><td>Test</td><td>Category</td><td>Target</td>
<td>Action</td>
<td>Expected</td>
<td>Given</td>
<td>Success</td></tr>"""
TableFooter = "</table>"
HtmlBegin = """<html>
<head>
<title>Test results</title>
<style type="text/css">
<!--""" & slurp("css/boilerplate.css") & "\n" &
slurp("css/style.css") &
"""
ul#tabs { list-style-type: none; margin: 30px 0 0 0; padding: 0 0 0.3em 0; }
ul#tabs li { display: inline; }
ul#tabs li a { color: #42454a; background-color: #dedbde;
border: 1px solid #c9c3ba; border-bottom: none;
padding: 0.3em; text-decoration: none; }
ul#tabs li a:hover { background-color: #f1f0ee; }
ul#tabs li a.selected { color: #000; background-color: #f1f0ee;
font-weight: bold; padding: 0.7em 0.3em 0.38em 0.3em; }
div.tabContent { border: 1px solid #c9c3ba;
padding: 0.5em; background-color: #f1f0ee; }
div.tabContent.hide { display: none; }
-->
</style>
<script>
var tabLinks = new Array();
var contentDivs = new Array();
function init() {
// Grab the tab links and content divs from the page
var tabListItems = document.getElementById('tabs').childNodes;
for ( var i = 0; i < tabListItems.length; i++ ) {
if ( tabListItems[i].nodeName == "LI" ) {
var tabLink = getFirstChildWithTagName( tabListItems[i], 'A' );
var id = getHash( tabLink.getAttribute('href') );
tabLinks[id] = tabLink;
contentDivs[id] = document.getElementById( id );
}
}
// Assign onclick events to the tab links, and
// highlight the first tab
var i = 0;
for ( var id in tabLinks ) {
tabLinks[id].onclick = showTab;
tabLinks[id].onfocus = function() { this.blur() };
if ( i == 0 ) tabLinks[id].className = 'selected';
i++;
}
// Hide all content divs except the first
var i = 0;
for ( var id in contentDivs ) {
if ( i != 0 ) contentDivs[id].className = 'tabContent hide';
i++;
}
}
</script>
</head>
<body onload="init()">"""
HtmlEnd = "</body></html>"
proc td(s: string): string =
result = "<td>" & s.substr(0, 200).XMLEncode & "</td>"
proc generateHtml*(filename: string) =
const selRow = """select name, category, target, action,
expected, given, result
from TestResult
where [commit] = ? and machine = ?
order by category"""
var db = open(connection="testament.db", user="testament", password="",
database="testament")
var outfile = open(filename, fmWrite)
outfile.write(HtmlBegin)
let thisMachine = backend.getMachine()
outfile.write()
let machine = db.getRow(sql"select name, os, cpu from machine where id = ?",
thisMachine)
outfile.write("<p><b>$#</b></p>" % machine.join(" "))
outfile.write("""<ul id="tabs">""")
for lastCommit in db.getRow(sql"select id from [Commit] order by id desc"):
let commit = db.getValue(sql"select hash from [Commit] where id = ?",
lastCommit)
let branch = db.getValue(sql"select branch from [Commit] where id = ?",
lastCommit)
outfile.writeln """<li><a href="#$#">$#: $#</a></li>""" % [
lastCommit, branch, commit]
outfile.write("</ul>")
for lastCommit in db.getRow(sql"select id from [Commit] order by id desc"):
outfile.write("""<div class="tabContent" id="$#">""" % lastCommit)
outfile.write(TableHeader)
for row in db.rows(sql(selRow), lastCommit, thisMachine):
outfile.write("<tr>")
for x in row:
outfile.write(x.td)
outfile.write("</tr>")
outfile.write(TableFooter)
outfile.write("</div>")
outfile.write(HtmlEnd)
close(db)
close(outfile)

View file

@ -10,8 +10,8 @@
## This program verifies Nimrod against the testcases.
import
parseutils, strutils, pegs, os, osproc, streams, parsecfg, browsers, json,
marshal, cgi, backend, parseopt, specs #, caas
parseutils, strutils, pegs, os, osproc, streams, parsecfg, json,
marshal, backend, parseopt, specs, htmlgen, browsers
const
resultsFile = "testresults.html"
@ -249,16 +249,18 @@ proc main() =
processCategory(r, Category(dir), p.cmdLineRest.string)
for a in AdditionalCategories:
processCategory(r, Category(a), p.cmdLineRest.string)
of "c", "category":
of "c", "cat", "category":
var cat = Category(p.key)
p.next
processCategory(r, cat, p.cmdLineRest.string)
of "html":
quit "too implement"
generateHtml(resultsFile)
else:
quit usage
if optPrintResults: echo r, r.data
if optPrintResults:
if action == "html": openDefaultBrowser(resultsFile)
else: echo r, r.data
backend.close()
if paramCount() == 0: