This sets the backends (both the json and junit) to write out an incomplete status before starting the test. (for junit this is a 'fail' with a message of 'incomplete', junit is a fairly limited format). When the test is completed this incomplete status is overwritten with the final status.
This patch also adds a new mode to the 'summary console' option, '-i', which will list all tests with an incomplete status. This will allow a developer to see what tests failed when a system crashed, or what tests were running when they aborted piglit. Signed-off-by: Dylan Baker <[email protected]> --- framework/backends/abstract.py | 60 ++++++++++++++++++++++++++++--- framework/backends/json.py | 12 +++---- framework/backends/junit.py | 36 ++++++++++--------- framework/profile.py | 5 +-- framework/programs/run.py | 7 ++-- framework/programs/summary.py | 17 ++++++--- framework/status.py | 2 +- framework/summary.py | 40 +++++++++++++-------- framework/tests/json_backend_tests.py | 62 +++++++++++++++++++++++++++------ framework/tests/json_tests.py | 3 +- framework/tests/junit_backends_tests.py | 34 +++++++----------- 11 files changed, 193 insertions(+), 85 deletions(-) diff --git a/framework/backends/abstract.py b/framework/backends/abstract.py index 928ba9a..47186f2 100644 --- a/framework/backends/abstract.py +++ b/framework/backends/abstract.py @@ -28,7 +28,12 @@ This module provides mixins and base classes for backend modules. from __future__ import print_function, absolute_import import os import abc +import shutil import itertools +import contextlib + +from framework.results import TestResult +from framework.status import INCOMPLETE class Backend(object): @@ -100,11 +105,16 @@ class Backend(object): """ @abc.abstractmethod - def write_test(self, name, data): + def write_test(self, name): """ Write a test into the backend store This method writes an actual test into the backend store. + Should be a context manager, used with the with statement. It should + first write an incomplete status value, then yield and object that will + overwrite that value with the final value. That object needs to take a + 'data' paramter whic is a result.TestResult object. + Arguments: name -- the name of the test to be written data -- a TestResult object representing the test data @@ -133,15 +143,55 @@ class FileBackend(Backend): """ def __init__(self, dest, file_start_count=0, file_fsync=False, **kwargs): self._dest = dest - self._counter = itertools.count(file_start_count) - self._file_sync = file_fsync + self.__counter = itertools.count(file_start_count) + self.__file_sync = file_fsync - def _fsync(self, file_): + __INCOMPLETE = TestResult({'result': INCOMPLETE}) + + def __fsync(self, file_): """ Sync the file to disk If self._file_sync is truthy this will sync self._file to disk """ file_.flush() - if self._file_sync: + if self.__file_sync: os.fsync(file_.fileno()) + + @abc.abstractmethod + def _write(self, f, name, data): + """Method that writes a TestResult into a result file.""" + + @abc.abstractproperty + def _file_extension(self): + """The file extension of the backend.""" + + @contextlib.contextmanager + def write_test(self, name): + """Write a test. + + When this context manager is opened it will first write a placeholder + file with the status incomplete. + + When it is called to write the finall result it will create a temporary + file, write to that file, then move that file over the original, + incomplete status file. This helps to make the operation atomic, as + long as the filesystem continues running and the result was valid in + the original file it will be valid at the end + + """ + def finish(val): + tfile = file_ + '.tmp' + with open(tfile, 'w') as f: + self._write(f, name, val) + self.__fsync(f) + shutil.move(tfile, file_) + + file_ = os.path.join(self._dest, 'tests', '{}.{}'.format( + self.__counter.next(), self._file_extension)) + + with open(file_, 'w') as f: + self._write(f, name, self.__INCOMPLETE) + self.__fsync(f) + + yield finish diff --git a/framework/backends/json.py b/framework/backends/json.py index affd64e..9f138a6 100644 --- a/framework/backends/json.py +++ b/framework/backends/json.py @@ -74,6 +74,8 @@ class JSONBackend(FileBackend): a file it just ignores it, making the result atomic. """ + _file_extension = 'json' + def initialize(self, metadata): """ Write boilerplate json code @@ -141,13 +143,9 @@ class JSONBackend(FileBackend): os.unlink(os.path.join(self._dest, 'metadata.json')) shutil.rmtree(os.path.join(self._dest, 'tests')) - def write_test(self, name, data): - """ Write a test into the JSON tests dictionary """ - t = os.path.join(self._dest, 'tests', - '{}.json'.format(self._counter.next())) - with open(t, 'w') as f: - json.dump({name: data}, f, default=piglit_encoder) - self._fsync(f) + @staticmethod + def _write(f, name, data): + json.dump({name: data}, f, default=piglit_encoder) def load_results(filename): diff --git a/framework/backends/junit.py b/framework/backends/junit.py index 3602f9e..21cb991 100644 --- a/framework/backends/junit.py +++ b/framework/backends/junit.py @@ -47,6 +47,7 @@ class JUnitBackend(FileBackend): https://svn.jenkins-ci.org/trunk/hudson/dtkit/dtkit-format/dtkit-junit-model/src/main/resources/com/thalesgroup/dtkit/junit/model/xsd/junit-7.xsd """ + _file_extension = 'xml' def __init__(self, dest, junit_suffix='', **options): super(JUnitBackend, self).__init__(dest, **options) @@ -106,7 +107,7 @@ class JUnitBackend(FileBackend): shutil.rmtree(os.path.join(self._dest, 'tests')) - def write_test(self, name, data): + def _write(self, f, name, data): def calculate_result(): """Set the result.""" @@ -184,27 +185,28 @@ class JUnitBackend(FileBackend): # Create the root element element = etree.Element('testcase', name=full_test_name, classname=classname, - time=str(data['time']), + # Incomplete will not have a time. + time=str(data.get('time')), status=str(data['result'])) - # Add stdout - out = etree.SubElement(element, 'system-out') - out.text = data['out'] + # If this is an incomplete status then none of these values will be + # available, nor + if data['result'] != 'incomplete': + # Add stdout + out = etree.SubElement(element, 'system-out') + out.text = data['out'] - # Prepend command line to stdout - out.text = data['command'] + '\n' + out.text + # Prepend command line to stdout + out.text = data['command'] + '\n' + out.text - # Add stderr - err = etree.SubElement(element, 'system-err') - err.text = data['err'] + # Add stderr + err = etree.SubElement(element, 'system-err') + err.text = data['err'] + calculate_result() + else: + etree.SubElement(element, 'failure', message='Incomplete run.') - calculate_result() - - t = os.path.join(self._dest, 'tests', - '{}.xml'.format(self._counter.next())) - with open(t, 'w') as f: - f.write(etree.tostring(element)) - self._fsync(f) + f.write(etree.tostring(element)) def _load(results_file): diff --git a/framework/profile.py b/framework/profile.py index d5d140c..46a979c 100644 --- a/framework/profile.py +++ b/framework/profile.py @@ -276,8 +276,9 @@ class TestProfile(object): """ name, test = pair - test.execute(name, log.get(), self.dmesg) - backend.write_test(name, test.result) + with backend.write_test(name) as w: + test.execute(name, log.get(), self.dmesg) + w(test.result) def run_threads(pool, testlist): """ Open a pool, close it, and join it """ diff --git a/framework/programs/run.py b/framework/programs/run.py index ca94596..6053074 100644 --- a/framework/programs/run.py +++ b/framework/programs/run.py @@ -339,8 +339,11 @@ def resume(input_): file_start_count=len(results.tests) + 1) # Specifically do not initialize again, everything initialize does is done. - for name in results.tests.iterkeys(): - opts.exclude_tests.add(name) + # Don't re-run tests that have already completed, incomplete status tests + # have obviously not completed. + for name, result in results.tests.iteritems(): + if result['result'] != 'incomplete': + opts.exclude_tests.add(name) profile = framework.profile.merge_test_profiles(results.options['profile']) profile.results_dir = args.results_path diff --git a/framework/programs/summary.py b/framework/programs/summary.py index b012a48..c839930 100644 --- a/framework/programs/summary.py +++ b/framework/programs/summary.py @@ -108,13 +108,22 @@ def console(input_): # and then call for only summary excGroup1 = parser.add_mutually_exclusive_group() excGroup1.add_argument("-d", "--diff", - action="store_true", + action="store_const", + const="diff", + dest='mode', help="Only display the differences between multiple " "result files") excGroup1.add_argument("-s", "--summary", - action="store_true", + action="store_const", + const="summary", + dest='mode', help="Only display the summary, not the individual " "test results") + excGroup1.add_argument("-i", "--incomplete", + action="store_const", + const="incomplete", + dest='mode', + help="Only display tests that are incomplete.") parser.add_argument("-l", "--list", action="store", help="Use test results from a list file") @@ -127,7 +136,7 @@ def console(input_): # Throw an error if -d/--diff is called, but only one results file is # provided - if args.diff and len(args.results) < 2: + if args.mode == 'diff' and len(args.results) < 2: parser.error('-d/--diff cannot be specified unless two or more ' 'results files are specified') @@ -137,7 +146,7 @@ def console(input_): # Generate the output output = summary.Summary(args.results) - output.generate_text(args.diff, args.summary) + output.generate_text(args.mode) def csv(input_): diff --git a/framework/status.py b/framework/status.py index 90f69fb..8d649f2 100644 --- a/framework/status.py +++ b/framework/status.py @@ -183,7 +183,7 @@ class Status(object): raise TypeError("Cannot compare type: {}".format(type(other))) def __ne__(self, other): - return self.fraction != other.fraction or int(self) != int(other) + return not self == other def __ge__(self, other): return self.fraction[1] > other.fraction[1] or ( diff --git a/framework/summary.py b/framework/summary.py index 9b30b5e..66f100a 100644 --- a/framework/summary.py +++ b/framework/summary.py @@ -310,7 +310,7 @@ class Summary: self.totals = {} self.tests = {'all': set(), 'changes': set(), 'problems': set(), 'skipped': set(), 'regressions': set(), 'fixes': set(), - 'enabled': set(), 'disabled': set()} + 'enabled': set(), 'disabled': set(), 'incomplete': set()} def fgh(test, result): """ Helper for updating the fractions and status lists """ @@ -411,6 +411,9 @@ class Summary: if so.SKIP in status: self.tests['skipped'].add(test) + if so.INCOMPLETE in status: + self.tests['incomplete'].add(test) + # find fixes, regressions, and changes for i in xrange(len(status) - 1): first = status[i] @@ -437,7 +440,7 @@ class Summary: """ self.totals = {'pass': 0, 'fail': 0, 'crash': 0, 'skip': 0, 'timeout': 0, 'warn': 0, 'dmesg-warn': 0, - 'dmesg-fail': 0} + 'dmesg-fail': 0, 'incomplete': 0,} for test in results.tests.itervalues(): self.totals[str(test['result'])] += 1 @@ -555,22 +558,30 @@ class Summary: else: out.write(empty_status.render(page=page, pages=pages)) - def generate_text(self, diff, summary): + def generate_text(self, mode): """ Write summary information to the console """ + assert mode in ['summary', 'diff', 'incomplete'] self.__find_totals(self.results[-1]) # Print the name of the test and the status from each test run - if not summary: - if diff: - for test in self.tests['changes']: - print("%(test)s: %(statuses)s" % {'test': test, 'statuses': - ' '.join([str(i.tests.get(test, {'result': so.SKIP}) - ['result']) for i in self.results])}) - else: - for test in self.tests['all']: - print("%(test)s: %(statuses)s" % {'test': test, 'statuses': - ' '.join([str(i.tests.get(test, {'result': so.SKIP}) - ['result']) for i in self.results])}) + if mode == 'diff': + for test in self.tests['changes']: + print("{test}: {statuses}".format( + test=test, + statuses=' '.join(str(i.tests.get(test, {'result': so.SKIP}) + ['result']) for i in self.results))) + elif mode == 'incomplete': + for test in self.tests['incomplete']: + print("{test}: {statuses}".format( + test=test, + statuses=' '.join(str(i.tests.get(test, {'result': so.SKIP}) + ['result']) for i in self.results))) + elif mode != 'summary': + for test in self.tests['all']: + print("{test}: {statuses}".format( + test=test, + statuses=' '.join(str(i.tests.get(test, {'result': so.SKIP}) + ['result']) for i in self.results))) # Print the summary print("summary:\n" @@ -580,6 +591,7 @@ class Summary: " skip: {skip}\n" " timeout: {timeout}\n" " warn: {warn}\n" + " incomplete: {incomplete}\n" " dmesg-warn: {dmesg-warn}\n" " dmesg-fail: {dmesg-fail}".format(**self.totals)) if self.tests['changes']: diff --git a/framework/tests/json_backend_tests.py b/framework/tests/json_backend_tests.py index 45626e8..3c251ae 100644 --- a/framework/tests/json_backend_tests.py +++ b/framework/tests/json_backend_tests.py @@ -87,7 +87,8 @@ class TestJSONTestMethod(utils.StaticDirectory): super(TestJSONTestMethod, cls).setup_class() test = backends.json.JSONBackend(cls.tdir) test.initialize(BACKEND_INITIAL_META) - test.write_test(cls.test_name, cls.result) + with test.write_test(cls.test_name) as t: + t(cls.result) def test_write_test(self): """ JSONBackend.write_test() adds tests to a 'tests' directory """ @@ -122,7 +123,8 @@ class TestJSONTestFinalize(utils.StaticDirectory): super(TestJSONTestFinalize, cls).setup_class() test = backends.json.JSONBackend(cls.tdir) test.initialize(BACKEND_INITIAL_META) - test.write_test(cls.test_name, cls.result) + with test.write_test(cls.test_name) as t: + t(cls.result) test.finalize() def test_remove_metadata(self): @@ -205,9 +207,12 @@ def test_resume_load(): with utils.tempdir() as f: backend = backends.json.JSONBackend(f) backend.initialize(BACKEND_INITIAL_META) - backend.write_test("group1/test1", {'result': 'fail'}) - backend.write_test("group1/test2", {'result': 'pass'}) - backend.write_test("group2/test3", {'result': 'fail'}) + with backend.write_test("group1/test1") as t: + t({'result': 'fail'}) + with backend.write_test("group1/test2") as t: + t({'result': 'pass'}) + with backend.write_test("group2/test3") as t: + t({'result': 'fail'}) backends.json._resume(f) @@ -217,9 +222,12 @@ def test_resume_load_valid(): with utils.tempdir() as f: backend = backends.json.JSONBackend(f) backend.initialize(BACKEND_INITIAL_META) - backend.write_test("group1/test1", {'result': 'fail'}) - backend.write_test("group1/test2", {'result': 'pass'}) - backend.write_test("group2/test3", {'result': 'fail'}) + with backend.write_test("group1/test1") as t: + t({'result': 'fail'}) + with backend.write_test("group1/test2") as t: + t({'result': 'pass'}) + with backend.write_test("group2/test3") as t: + t({'result': 'fail'}) test = backends.json._resume(f) @@ -234,9 +242,12 @@ def test_resume_load_invalid(): with utils.tempdir() as f: backend = backends.json.JSONBackend(f) backend.initialize(BACKEND_INITIAL_META) - backend.write_test("group1/test1", {'result': 'fail'}) - backend.write_test("group1/test2", {'result': 'pass'}) - backend.write_test("group2/test3", {'result': 'fail'}) + with backend.write_test("group1/test1") as t: + t({'result': 'fail'}) + with backend.write_test("group1/test2") as t: + t({'result': 'pass'}) + with backend.write_test("group2/test3") as t: + t({'result': 'fail'}) with open(os.path.join(f, 'tests', 'x.json'), 'w') as w: w.write('foo') @@ -248,6 +259,35 @@ def test_resume_load_invalid(): ) +def test_resume_load_incomplete(): + """backends.json._resume: loads incomplete results. + + Because resume, aggregate, and summary all use the function called _resume + we can't remove incomplete tests here. It's probably worth doing a refactor + to split some code out and allow this to be done in the resume path. + + """ + with utils.tempdir() as f: + backend = backends.json.JSONBackend(f) + backend.initialize(BACKEND_INITIAL_META) + with backend.write_test("group1/test1") as t: + t({'result': 'fail'}) + with backend.write_test("group1/test2") as t: + t({'result': 'pass'}) + with backend.write_test("group2/test3") as t: + t({'result': 'crash'}) + with backend.write_test("group2/test4") as t: + t({'result': 'incomplete'}) + + test = backends.json._resume(f) + + nt.assert_set_equal( + set(test.tests.keys()), + set(['group1/test1', 'group1/test2', 'group2/test3', + 'group2/test4']), + ) + + @utils.no_error def test_load_results_folder_as_main(): """ Test that load_results takes a folder with a file named main in it """ diff --git a/framework/tests/json_tests.py b/framework/tests/json_tests.py index 70a501a..d6ca0bf 100644 --- a/framework/tests/json_tests.py +++ b/framework/tests/json_tests.py @@ -63,7 +63,8 @@ class TestJsonOutput(utils.StaticDirectory): backend = JSONBackend(cls.tdir, file_fsync=True) backend.initialize(_create_metadata(args, 'test', core.Options())) - backend.write_test('result', {'result': 'pass'}) + with backend.write_test('result') as t: + t({'result': 'pass'}) backend.finalize({'time_elapsed': 1.22}) with open(os.path.join(cls.tdir, 'results.json'), 'r') as f: cls.json = json.load(f) diff --git a/framework/tests/junit_backends_tests.py b/framework/tests/junit_backends_tests.py index f42f707..249bf49 100644 --- a/framework/tests/junit_backends_tests.py +++ b/framework/tests/junit_backends_tests.py @@ -83,9 +83,8 @@ class TestJUnitSingleTest(TestJunitNoTests): cls.test_file = os.path.join(cls.tdir, 'results.xml') test = backends.junit.JUnitBackend(cls.tdir) test.initialize(BACKEND_INITIAL_META) - test.write_test( - grouptools.join('a', 'test', 'group', 'test1'), - results.TestResult({ + with test.write_test(grouptools.join('a', 'test', 'group', 'test1')) as t: + t(results.TestResult({ 'time': 1.2345, 'result': 'pass', 'out': 'this is stdout', @@ -115,26 +114,22 @@ class TestJUnitMultiTest(TestJUnitSingleTest): cls.test_file = os.path.join(cls.tdir, 'results.xml') test = backends.junit.JUnitBackend(cls.tdir) test.initialize(BACKEND_INITIAL_META) - test.write_test( - grouptools.join('a', 'test', 'group', 'test1'), - results.TestResult({ + with test.write_test(grouptools.join('a', 'test', 'group', 'test1')) as t: + t(results.TestResult({ 'time': 1.2345, 'result': 'pass', 'out': 'this is stdout', 'err': 'this is stderr', 'command': 'foo', - }) - ) - test.write_test( - 'a/different/test/group/test2', - results.TestResult({ + })) + with test.write_test('a/different/test/group/test2') as t: + t(results.TestResult({ 'time': 1.2345, 'result': 'fail', 'out': 'this is stdout', 'err': 'this is stderr', 'command': 'foo', - }) - ) + })) test.finalize() def test_xml_well_formed(self): @@ -152,16 +147,14 @@ def test_junit_replace(): with utils.tempdir() as tdir: test = backends.junit.JUnitBackend(tdir) test.initialize(BACKEND_INITIAL_META) - test.write_test( - grouptools.join('a', 'test', 'group', 'test1'), - results.TestResult({ + with test.write_test(grouptools.join('a', 'test', 'group', 'test1')) as t: + t(results.TestResult({ 'time': 1.2345, 'result': 'pass', 'out': 'this is stdout', 'err': 'this is stderr', 'command': 'foo', - }) - ) + })) test.finalize() test_value = etree.parse(os.path.join(tdir, 'results.xml')).getroot() @@ -175,9 +168,8 @@ def test_junit_skips_bad_tests(): with utils.tempdir() as tdir: test = backends.junit.JUnitBackend(tdir) test.initialize(BACKEND_INITIAL_META) - test.write_test( - grouptools.join('a', 'test', 'group', 'test1'), - results.TestResult({ + with test.write_test(grouptools.join('a', 'test', 'group', 'test1')) as t: + t(results.TestResult({ 'time': 1.2345, 'result': 'pass', 'out': 'this is stdout', -- 2.3.5 _______________________________________________ Piglit mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/piglit
