Currently this is done in the summary code, and is very complicated. This change has a couple of advantages. First, having it in the TestrunResult object seems a more obvious place for it. Second, this will allows us to store the data in the json, removing the need to calculate the data again and again.
Signed-off-by: Dylan Baker <[email protected]> --- framework/results.py | 45 ++++++++++++++++- framework/tests/results_tests.py | 101 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/framework/results.py b/framework/results.py index 2753fd5..7bfc2fd 100644 --- a/framework/results.py +++ b/framework/results.py @@ -23,7 +23,9 @@ from __future__ import print_function, absolute_import -from framework import status, exceptions +import collections + +from framework import status, exceptions, grouptools __all__ = [ 'TestrunResult', @@ -180,7 +182,24 @@ class TestResult(object): self.subtests.update(dict_['subtest']) +class Totals(dict): + def __init__(self, *args, **kwargs): + super(Totals, self).__init__(*args, **kwargs) + for each in status.ALL: + self[str(each)] = 0 + + def __nonzero__(self): + # Since totals are prepopulated, calling 'if not <Totals instance>' + # will always result in True, this will cause it to return True only if + # one of the values is not zero + for each in self.itervalues(): + if each != 0: + return True + return False + + class TestrunResult(object): + """The result of a single piglit run.""" def __init__(self): self.name = None self.uname = None @@ -190,3 +209,27 @@ class TestrunResult(object): self.lspci = None self.time_elapsed = None self.tests = {} + self.totals = collections.defaultdict(Totals) + + def calculate_group_totals(self): + """Calculate the number of pases, fails, etc at each level.""" + for name, result in self.tests.iteritems(): + # If there are subtests treat the test as if it is a group instead + # of a test. + if result.subtests: + for res in result.subtests.itervalues(): + res = str(res) + temp = name + + self.totals[temp][res] += 1 + while temp: + temp = grouptools.groupname(temp) + self.totals[temp][res] += 1 + self.totals['root'][res] += 1 + else: + res = str(result.result) + while name: + name = grouptools.groupname(name) + self.totals[name][res] += 1 + self.totals['root'][res] += 1 + diff --git a/framework/tests/results_tests.py b/framework/tests/results_tests.py index 143ab09..d2c4206 100644 --- a/framework/tests/results_tests.py +++ b/framework/tests/results_tests.py @@ -25,7 +25,7 @@ from __future__ import print_function, absolute_import import nose.tools as nt -from framework import results, status, exceptions +from framework import results, status, exceptions, grouptools import framework.tests.utils as utils @@ -346,3 +346,102 @@ class TestStringDescriptor(object): def test_delete(self): """results.StringDescriptor.__delete__: raises NotImplementedError""" del self.test.val + + +class TestTestrunResultTotals(object): + """Test the totals generated by TestrunResult.calculate_group_totals().""" + @classmethod + def setup_class(cls): + pass_ = results.TestResult('pass') + fail = results.TestResult('fail') + #warn = results.TestResult('warn') + crash = results.TestResult('crash') + skip = results.TestResult('skip') + tr = results.TestrunResult() + tr.tests = { + 'oink': pass_, + grouptools.join('foo', 'bar'): fail, + grouptools.join('foo', 'foo', 'bar'): crash, + grouptools.join('foo', 'foo', 'oink'): skip, + } + + tr.calculate_group_totals() + cls.test = tr.totals + + def test_root(self): + """results.TestrunResult.totals: The root is correct""" + root = results.Totals() + root['pass'] += 1 + root['fail'] += 1 + root['crash'] += 1 + root['skip'] += 1 + + nt.assert_dict_equal(self.test['root'], root) + + def test_recurse(self): + """results.TestrunResult.totals: Recurses correctly""" + expected = results.Totals() + expected['fail'] += 1 + expected['crash'] += 1 + expected['skip'] += 1 + nt.assert_dict_equal(self.test['foo'], expected) + + def test_two_parents(self): + """results.TestrunResult.totals: Handles multiple parents correctly""" + expected = results.Totals() + expected['crash'] += 1 + expected['skip'] += 1 + nt.assert_dict_equal(self.test[grouptools.join('foo', 'foo')], expected) + + +class TestTestrunResultTotalsSubtests(object): + """results.TestrunResult.totals: Tests with subtests are handled correctly""" + @classmethod + def setup_class(cls): + tr = results.TestResult('incomplete') + tr.subtests['foo'] = status.PASS + tr.subtests['bar'] = status.CRASH + tr.subtests['oink'] = status.FAIL + + run = results.TestrunResult() + run.tests[grouptools.join('sub', 'test')] = tr + run.calculate_group_totals() + + cls.test = run.totals + + def test_root(self): + """results.TestrunResult.totals: The root is correct with subtests""" + expect = results.Totals() + expect['pass'] += 1 + expect['crash'] += 1 + expect['fail'] += 1 + nt.assert_dict_equal(self.test['root'], expect) + + def test_node(self): + """results.TestrunResult.totals: Tests with subtests are treated as groups""" + key = grouptools.join('sub', 'test') + nt.ok_(key in self.test, + msg='Key: {} not found in {}'.format(key, self.test.keys())) + + def test_node_values(self): + """results.TestrunResult.totals: Tests with subtests values are correct""" + expect = results.Totals() + expect['pass'] += 1 + expect['crash'] += 1 + expect['fail'] += 1 + nt.assert_dict_equal(self.test[grouptools.join('sub', 'test')], expect) + + +def test_totals_false(): + """results.Totals: bool() returns False when all values are 0""" + nt.ok_(not bool(results.Totals())) + + +def test_totals_true(): + """results.Totals: bool() returns True when any value is not 0""" + # This might deserve a generator, but it seems so simple that it it's a lot + # of work for no gain + for key in results.Totals().iterkeys(): + test = results.Totals() + test[key] += 1 + nt.ok_(bool(test), msg='Returns false with status {}'.format(key)) -- 2.5.1 _______________________________________________ Piglit mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/piglit
