Hey, folks. Today I got a PoC of testcase-stats on top of wikitcms up
and running. It's not complete yet as I haven't written the bits to
provide the CLI and page selection stuff, but all the hard stuff is
there. I'm attaching it. To test it, just stick it in the
stats/testcase-stats directory of a fedora-qa git checkout, symlink an
up-to-date checkout of the wikitcms 'results-types' branch into the same
directory, and run it. It'll output to /tmp/tcstats_test , but you can
easily change that if you like. You can also change the hard-coded list
of pages it runs on.
Tomorrow I'll aim to clean it all up and make it a relval sub-command,
then maybe release wikitcms 2.0. it may get interesting when I look at
feeding in various possible groups of pages, but I'll burn that bridge
when I come to it.
The contortions this goes through to wrangle the data into the correct
forms seem slightly inelegant, but I don't see any easy significant wins
in simplifying them - thoughts welcome. What it does is roughly this:
* Get a list of ResultRow objects for each page in the page set
* make a dict of lists of tuples (!). the keys are tuples for all the
'unique tests' for which at least one ResultRow exists, where a 'unique
test' is the combination of testcase + 'test name' - when you have
something like [[QA:Testcase_default_boot_install|Workstation live]],
QA:Testcase_default_boot_install is the testcase and "Workstation live"
is the 'test name'. the value tuples are the ResultRows for that 'unique
test'; field 0 is a string identifying the compose from which the
ResultRow comes (e.g. 'BetaTC1'), field 1 is the ResultRow object
itself. we need to keep the rows for each compose for each 'unique test'
separate.
* make another dict, using the same keys from the last one, with the
value for each key being a Test() object instantiated from the list of
tuples from the first test case. That class can now produce all the
information the HTML rendering functions need to do their stuff.
if anyone sees a way to simplify that while still making it possible for
the renderers to get all the data they need fairly simply, please do
enlighten me =) one possible thing we could do is let wikitcms provide
milestone / compose properties for resultrow objects, that might help.
I'll have to look at it tomorrow.
If I was writing this from scratch maybe I'd go about it slightly
differently, but I don't think the ugliness is sufficient to merit
burning down all josef's work and starting over. it works fine.
the actual rendering functions are more or less intact from josef's
version, I just adjusted them to consume the data from the Test()
objects. the Test() classes' __init__() is pretty much ripped from
josef's version too.
--
Adam Williamson
Fedora QA Community Monkey
IRC: adamw | Twitter: AdamW_Fedora | XMPP: adamw AT happyassassin . net
http://www.happyassassin.net
import wikitcms.wiki as wk
import wikitcms.event as ev
import wikitcms.page as pg
import wikitcms.release as rl
from collections import defaultdict
import re
import os
import sys
import datetime
import shutil
SANITY_SUB = re.compile(r'[^A-Za-z0-9]')
def get_bitmap_html(bitmap):
out = '<span style="letter-spacing:-0.5em;font-size:1.5em;">'
sym_0 = "⠄"
sym_1 = "⠄"
sym_2 = "⠆"
sym_3 = "⠇"
for compose, cnt, style in bitmap:
out += '<span class="%s">' % style
if cnt == 0:
out+= sym_0
elif cnt < 3:
out+= sym_1
elif cnt < 5:
out+= sym_2
else:
out+= sym_3
out+='</span>'
out += '</span>'
return out
class Test(object):
def __init__(self, name, testcase, resultrows):
self.name = name
self.testcase = testcase
self.milestone = resultrows[len(resultrows)-1][1].milestone
self.passes = defaultdict(list)
self.warns = defaultdict(list)
self.fails = defaultdict(list)
self.composes = set()
self.last_tested = ""
bitmap = []
for rowtup in resultrows:
compose = rowtup[0]
row = rowtup[1]
self.composes.add(compose)
p = 0
f = 0
w = 0
rlists = row.results.values()
for rlist in rlists:
for result in rlist:
if result.status == 'pass':
self.last_tested = compose
self.passes[compose].append(result)
p += 1
if result.status == 'fail':
self.last_tested = compose
self.fails[compose].append(result)
f += 1
if result.status == 'warn':
self.last_tested = compose
self.warns[compose].append(result)
w += 1
bitmap.append([self.last_tested, 0, 'nottested'])
if p+f+w > 0:
b = bitmap[-1]
if f+w == 0: # just passed
b[2] = 'pass'
elif p+w == 0: # just failed
b[2] = 'fail'
elif w > 0:
b[2] = 'warn'
b[1] += p+f+w
self.bitmap = get_bitmap_html(bitmap)
def print_results_html(test_dict, outdir, outfile):
col_names = ['Testcase URL','Release Level', 'Last in', 'Coverage', 'Details']#'Tested', 'Pass', 'Fail', 'Warn', 'Details']
theader = "\n".join(map(lambda x: "<th>%s</th>" % x, col_names))
trs = []
fmt_tc = "<td style=\"padding=0\"><a href=\"https://fedoraproject.org/wiki/%s\";>%s</a></td>"
fmt = '<td style="padding=0">%s</td>\n'* (len(col_names)-2)
fmt_link = '<td style="padding=0"><a href="%s.html">Details</a></td>\n'
# sorted_results = sorted(results_dict.values(), key=attrgetter('release_level', 'last_tested_in'))
for test in tests:
s = fmt_tc % (test.testcase, test.name)
s += fmt % (test.milestone, test.last_tested, test.bitmap)
s += fmt_link % SANITY_SUB.sub('_', test.name)
trs.append("<tr>%s</tr>" % s)
tbody = "\n".join(trs)
# directory with templates for pages
htmldir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), 'html')
# create the Overview page
print "Creating HTML output: %s" % os.path.join(outdir, outfile)
template = open(os.path.join(htmldir, 'index.html')).read()
fout = open(os.path.join(outdir, outfile), 'w')
fout.write(template % {'timestamp': datetime.datetime.utcnow(), 'theader': theader, 'tbody': tbody})
fout.close()
# create detail pages
template = open(os.path.join(htmldir, 'testcase.html')).read()
for test in tests:
body = ""
for page in pages:
body += "<p><strong><a href=\"https://fedoraproject.org/wiki/%s\";>%s</a></strong><br />\n" % (page.wikiname, page.wikiname)
compose = page.event.milestone + page.event.compose
if compose not in test.composes:
body += "<b>No data</b><br />\n"
continue
body += "Passes: %s<br />" % len(test.passes[compose])
body += "Warns: %s<br />" % len(test.warns[compose])
body += "Fails: %s<ul>\n" % len(test.fails[compose])
for fail in test.fails[compose]:
body += "<li>"
try:
body += "%s: " % fail.user
except IndexError:
pass
try:
for bug in fail.bugs:
body += '<a href="https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=%s";>%s</a>, ' % (bug, bug)
except IndexError:
pass
body += "</li>\n"
body += "</ul></p>\n"
fname = os.path.join(outdir, '%s.html' % SANITY_SUB.sub('_', test.name))
fout = open(fname, 'w')
fout.write(template % {'timestamp': datetime.datetime.utcnow(), 'testcase_name': test.testcase, 'body': body})
fout.close()
wiki = wk.Wiki(('https', 'fedoraproject.org'), '/w/')
event1 = ev.ComposeEvent('21', 'Alpha', 'RC1', wiki)
event2 = ev.ComposeEvent('21', 'Beta', 'TC1', wiki)
event3 = ev.ComposeEvent('21', 'Beta', 'TC2', wiki)
pages = list()
pages.append(pg.ComposePage(wiki, event1, 'Installation'))
pages.append(pg.ComposePage(wiki, event2, 'Installation'))
pages.append(pg.ComposePage(wiki, event3, 'Installation'))
row_dict = defaultdict(list)
tests = list()
for page in pages:
print("Processing: " + page.wikiname)
resultrows = page.get_resultrows(statuses=['pass', 'warn', 'fail'], transferred=False)
for row in resultrows:
row_dict[(row.name, row.testcase)].append((page.event.milestone + page.event.compose, row))
for test, rows in row_dict.iteritems():
tests.append(Test(test[0], test[1], rows))
print_results_html(tests, '/tmp/tcstats_test', 'test.html')
# copy javascript
htmldir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), 'html')
shutil.copytree(os.path.join(htmldir, 'js'), os.path.join('/tmp/tcstats_test', 'js'))
"""
def main():
global options
logging.basicConfig(format='%(message)s')
outdir = tempfile.mkdtemp(prefix = 'tc_report_')
parse_args()
# download and parse all pages
for category in (INSTALLATION_NAME, BASE_NAME, DESKTOP_NAME):
pages = list_pages(category)
results_dict = {}
for page in pages:
try:
parse_matrix(page, results_dict)
except MatrixLookupException:
print "Failed to find Results Matrix for %s" % page
filename = SANITY_SUB.sub("_", category)
print_results_html(pages, results_dict, outdir, filename+".html")
# copy javascript
htmldir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), 'html')
shutil.copytree(os.path.join(htmldir, 'js'), os.path.join(outdir, 'js'))
# Create index page
fout = open(os.path.join(outdir, "index.html"), "w")
fout.write("<html><body>\n")
fout.write("<p>Created at: %s (UTC)</p>" % datetime.datetime.utcnow())
for category in (INSTALLATION_NAME, BASE_NAME, DESKTOP_NAME):
fname = SANITY_SUB.sub("_", category) + ".html"
fout.write("<a href=\"%s\">%s</a><br />" % (fname, category))
fout.write("\n</body></html>")
fout.close()
print "HTML Output: %s" % os.path.join(outdir, "index.html")
print outdir
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
sys.exit(1)
"""
_______________________________________________
qa-devel mailing list
[email protected]
https://admin.fedoraproject.org/mailman/listinfo/qa-devel
