On Thu, Jun 5, 2014 at 6:50 PM, Dylan Baker <[email protected]> wrote: > On Thursday, June 05, 2014 06:45:52 PM Ilia Mirkin wrote: >> On Thu, Jun 5, 2014 at 6:26 PM, Dylan Baker <[email protected]> wrote: >> > This patch updates our json to version 1. Changes from version 0 to >> > version 1 are as follows: >> > >> > - renamed 'main' to 'results.json' >> > - dmesg must be a string (It was stored a list in some version 0 >> > >> > results) >> > >> > - subtests are never stored as duplicate entries, a single instance of >> > >> > the test is recorded, (In version 0 both are possible) >> > >> > - there is no info entry in version 1, err, out, and returncode are >> > >> > always split into seperate entries >> > >> > This patch adds support to the results module for handling updates to >> > the results in a sane way. It does this by adding a result_version >> > attribute to the TestrunResult (which is stored as json), and >> > implementing the ability to incrementally update results between >> > versions. >> > >> > It does this automatically on load, non-destructively, moving the old >> > results to results.json.old, but does write the updated results to disk, >> > making the cost of this update a one time cost. >> >> FYI, the way I run piglit-summary right now is: >> >> ~/src/piglit/piglit-summary-html.py -e pass -e skip tmp <(xzcat >> nve7-2014-04-29-mupuf-gs5.xz) <(xzcat nve7-2014-06-05-mupuf-gs5.xz ) >> --overwrite >> >> (or something along those lines.) >> >> Is your approach going to work with this? (Obviously it can't _update_ >> the data, but will it still not break?) > > Honestly I'm not sure, it's not something that I or anyone at Intel does > AFAIK, I can give it a test, but I doubt it, since it relies on being able to > move the old file and write a new one. > > Also, I've got another series after this that just compresses/decompresses the > results for you (although we only get bz2 or gz since we're using python2, > xz/lzma support was added to python3 only)
Could you detect the situation when it's a pipe (or other "weird" file) and just update the results in-memory? > >> >> > Signed-off-by: Dylan Baker <[email protected]> >> > --- >> > >> > framework/programs/run.py | 4 +- >> > framework/results.py | 126 ++++++++++++++++++++++++++++--- >> > framework/tests/results_tests.py | 158 >> > +++++++++++++++++++++++++++++++++++++-- framework/tests/utils.py >> > | 2 + >> > 4 files changed, 273 insertions(+), 17 deletions(-) >> > >> > diff --git a/framework/programs/run.py b/framework/programs/run.py >> > index d61c065..6f352ab 100644 >> > --- a/framework/programs/run.py >> > +++ b/framework/programs/run.py >> > >> > @@ -157,7 +157,7 @@ def run(input_): >> > results.name = path.basename(args.results_path) >> > >> > # Begin json. >> > >> > - result_filepath = path.join(args.results_path, 'main') >> > + result_filepath = path.join(args.results_path, 'results.json') >> > >> > result_file = open(result_filepath, 'w') >> > json_writer = framework.results.JSONWriter(result_file) >> > >> > @@ -216,7 +216,7 @@ def resume(input_): >> > except KeyError: >> > pass >> > >> > - results_path = path.join(args.results_path, "main") >> > + results_path = path.join(args.results_path, 'results.json') >> > >> > json_writer = framework.results.JSONWriter(open(results_path, 'w+')) >> > json_writer.initialize_json(results.options, results.name, >> > >> > env.collectData()) >> > >> > diff --git a/framework/results.py b/framework/results.py >> > index 017c4a4..f26b7cb 100644 >> > --- a/framework/results.py >> > +++ b/framework/results.py >> > @@ -39,6 +39,9 @@ __all__ = [ >> > >> > 'load_results', >> > >> > ] >> > >> > +# The current version of the JSON results >> > +CURRENT_JSON_VERSION = 1 >> > + >> > >> > def _piglit_encoder(obj): >> > """ Encoder for piglit that can transform additional classes into >> > json >> > >> > @@ -119,7 +122,7 @@ class JSONWriter(object): >> > self.__is_collection_empty = [] >> > >> > def initialize_json(self, options, name, env): >> > - """ Write boilerplate json code >> > + """ Write boilerplate json code >> > >> > This writes all of the json except the actuall tests. >> > >> > @@ -132,6 +135,7 @@ class JSONWriter(object): >> > """ >> > self.open_dict() >> > >> > + self.write_dict_item('results_version', CURRENT_JSON_VERSION) >> > >> > self.write_dict_item('name', name) >> > >> > self.write_dict_key('options') >> > >> > @@ -229,6 +233,7 @@ class TestrunResult(object): >> > 'wglinfo', >> > 'glxinfo', >> > 'lspci', >> > >> > + 'results_version', >> > >> > 'time_elapsed'] >> > >> > self.name = None >> > self.uname = None >> > >> > @@ -337,12 +342,113 @@ def load_results(filename): >> > "main" >> > >> > """ >> > >> > - try: >> > - with open(filename, 'r') as resultsfile: >> > - testrun = TestrunResult(resultsfile) >> > - except IOError: >> > - with open(os.path.join(filename, "main"), 'r') as resultsfile: >> > - testrun = TestrunResult(resultsfile) >> > - >> > - assert testrun.name is not None >> > - return testrun >> > + if os.path.isfile(filename): >> > + filepath = filename >> > + else: >> > + # If there are both old and new results in a directory pick the >> > new + # ones first >> > + if os.path.exists(os.path.join(filename, 'results.json')): >> > + filepath = os.path.join(filename, 'results.json') >> > + # Version 0 results are called 'main' >> > + elif os.path.exists(os.path.join(filename, 'main')): >> > + filepath = os.path.join(filename, 'main') >> > + else: >> > + raise Exception("No results found") >> > + >> > + with open(filepath, 'r') as f: >> > + testrun = TestrunResult(f) >> > + >> > + return update_results(testrun, filepath) >> > + >> > + >> > +def update_results(results, filepath): >> > + """ Update results to the lastest version >> > + >> > + This function is a wraper for other update_* functions, providing >> > + incremental updates from one version to another. >> > + >> > + """ >> > + # If the results version is the current version there is no need to >> > + # update, just return the results >> > + if getattr(results, 'results_version', 0) == CURRENT_JSON_VERSION: >> > + return results >> > + >> > + # If there is no version then the results are version 0, and need to >> > be + # updated to version 1 >> > + if getattr(results, 'results_version', False): >> > + results = _update_zero_to_one(results) >> > + >> > + # Move the old results, and write the current results >> > + filedir = os.path.dirname(filepath) >> > + os.rename(filepath, os.path.join(filedir, 'results.json.old')) >> > + results.write(os.path.join(filedir, 'results.json')) >> > + >> > + return results >> > + >> > + >> > +def _update_zero_to_one(results): >> > + """ Update version zero results to version 1 results >> > + >> > + Changes from version 0 to version 1 >> > + >> > + - dmesg is sometimes stored as a list, sometimes stored as a string. >> > In + version 1 it is always stored as a string >> > + - in version 0 subtests are somtimes stored as duplicates, sometimes >> > stored + only with a single entry, in version 1 tests with subtests >> > are only + recorded once, always. >> > + - Version 0 can have an info entry, or returncode, out, and err >> > entries, + Version 1 will only have the latter >> > + - version 0 results are called 'main', while version 1 results are >> > called + 'results.json' (This is not handled internally, it's either >> > handled by + update_results() which will write the file back to >> > disk, or needs to be + handled manually by the user) >> > + >> > + """ >> > + updated_results = {} >> > + remove = set() >> > + >> > + for name, test in results.tests.iteritems(): >> > + # fix dmesg errors if any >> > + if isinstance(test.get('dmesg'), list): >> > + test['dmesg'] = '\n'.join(test['dmesg']) >> > + >> > + # If a test as an info attribute, we want to remove it, if it >> > doesn't + # have a returncode, out, or attribute we'll want to get >> > those out of + # info first >> > + if (None in [test.get('out'), test.get('err'), >> > test.get('returncode')] + and test.get('info')): >> > + code, err, out = test['info'].split('\n\n') >> > + >> > + # returncode can be 0, and 0 is falsy, so ensure it is >> > actually + # None >> > + if test.get('returncode') is None: >> > + test['returncode'] = int(code.split()[1].strip()) >> > + if not test.get('out'): >> > + test['out'] = out.split()[1] >> > + if not test.get('err'): >> > + test['err'] = err.split()[1] >> > + >> > + # Remove the unused info key >> > + if test.get('info'): >> > + del test['info'] >> > + >> > + # Walk through the list of tests, if any of them contain >> > duplicate >> > + # subtests, add those subtests to the remove set, and then write >> > a >> > + # single test result to the update_results dictionary, which will >> > be + # merged into results >> > + # >> > + # this must be the last thing done in this loop, or there will be >> > pain + if test.get('subtest'): >> > + remove.add(name) >> > + testname = os.path.dirname(name) >> > + if testname not in updated_results.iterkeys(): >> > + updated_results[testname] = test >> > + >> > + for name in remove: >> > + del results.tests[name] >> > + results.tests.update(updated_results) >> > + >> > + # set the results version >> > + results.results_version = 1 >> > + >> > + return results >> > diff --git a/framework/tests/results_tests.py >> > b/framework/tests/results_tests.py index 64fb679..afd71f2 100644 >> > --- a/framework/tests/results_tests.py >> > +++ b/framework/tests/results_tests.py >> > @@ -24,6 +24,8 @@ >> > >> > import os >> > import tempfile >> > import json >> > >> > +import copy >> > +import nose.tools as nt >> > >> > import framework.tests.utils as utils >> > import framework.results as results >> > import framework.status as status >> > >> > @@ -66,21 +68,28 @@ def test_initialize_jsonwriter(): >> > assert isinstance(func, results.JSONWriter) >> > >> > -def test_load_results_folder(): >> > >> > +def test_load_results_folder_as_main(): >> > """ Test that load_results takes a folder with a file named main in >> > it """ >> > >> > with utils.tempdir() as tdir: >> > with open(os.path.join(tdir, 'main'), 'w') as tfile: >> > tfile.write(json.dumps(utils.JSON_DATA)) >> > >> > - results_ = results.load_results(tdir) >> > - assert results_ >> > + results.load_results(tdir) >> > + >> > + >> > +def test_load_results_folder(): >> > + """ Test that load_results takes a folder with a file named >> > results.json """ + with utils.tempdir() as tdir: >> > + with open(os.path.join(tdir, 'results.json'), 'w') as tfile: >> > + tfile.write(json.dumps(utils.JSON_DATA)) >> > + >> > + results.load_results(tdir) >> > >> > def test_load_results_file(): >> > """ Test that load_results takes a file """ >> > >> > with utils.resultfile() as tfile: >> > - results_ = results.load_results(tfile.name) >> > - assert results_ >> > + results.load_results(tfile.name) >> > >> > def test_testresult_to_status(): >> > @@ -106,3 +115,142 @@ def test_testrunresult_write(): >> > new = results.load_results(os.path.join(tdir, >> > 'results.json')) >> > >> > assert result.__dict__ == new.__dict__ >> > >> > + >> > + >> > [email protected] >> > +def generate_zero_to_one(): >> > + """ Generate tests for version 0 to version 1 conversion """ >> > + data = copy.deepcopy(utils.JSON_DATA) >> > + data['tests']['sometest']['dmesg'] = ['this', 'is', 'dmesg'] >> > + data['tests']['sometest']['info'] = \ >> > + 'Returncode: 1\n\nErrors: stderr\n\nOutput: stdout\n' >> > + data['tests'].update({ >> > + 'group1/groupA/test/subtest 1': { >> > + 'info': 'Returncode: 1\n\nErrors: stderr\n\nOutput: >> > stdout\n', >> > + 'subtest': { >> > + 'subtest 1': 'pass', >> > + 'subtest 2': 'pass' >> > + }, >> > + 'returncode': 0, >> > + 'command': 'this is a command', >> > + 'result': 'pass', >> > + 'time': 0.1 >> > + }, >> > + 'group1/groupA/test/subtest 2': { >> > + 'info': 'Returncode: 1\n\nErrors: stderr\n\nOutput: >> > stdout\n', >> > + 'subtest': { >> > + 'subtest 1': 'pass', >> > + 'subtest 2': 'pass' >> > + }, >> > + 'returncode': 0, >> > + 'command': 'this is a command', >> > + 'result': 'pass', >> > + 'time': 0.1 >> > + } >> > + }) >> > + >> > + with utils.with_tempfile(json.dumps(data)) as f: >> > + res = results._update_zero_to_one(results.load_results(f)) >> > + >> > + zero_to_one_dmesg.description = \ >> > + "version 1: dmesg is converted from a list to a string" >> > + yield zero_to_one_dmesg, res >> > + >> > + zero_to_one_subtests_remove_duplicates.description = \ >> > + "Version 1: Removes duplicate entrieds" >> > + yield zero_to_one_subtests_remove_duplicates, res >> > + >> > + zero_to_one_subtests_add_test.description = \ >> > + "Version 1: Add an entry for the actual test" >> > + yield zero_to_one_subtests_add_test, res >> > + >> > + zero_to_one_subtests_test_is_testresult.description = \ >> > + "Version 1: The result of the new test is a TestResult Instance" >> > + yield zero_to_one_subtests_test_is_testresult, res >> > + >> > + zero_to_one_info_delete.description = \ >> > + "Version 1: Info should be removed" >> > + yield zero_to_one_info_delete, res >> > + >> > + zero_to_one_returncode_from_info.description = \ >> > + "Version 1: Use the returncode from info if there is no >> > returncode" + yield zero_to_one_returncode_from_info, res >> > + >> > + zero_to_one_returncode_no_override.description = \ >> > + "Version 1: Do not clobber returncode with info" >> > + yield zero_to_one_returncode_no_override, res >> > + >> > + zero_to_one_err_from_info.description = \ >> > + "Version 1: add an err attribute from info" >> > + yield zero_to_one_err_from_info, res >> > + >> > + zero_to_one_out_from_info.description = \ >> > + "Version 1: add an out attribute from info" >> > + yield zero_to_one_out_from_info, res >> > + >> > + zero_to_one_set_version.description = \ >> > + "Version 1: Set results_version to 1" >> > + yield zero_to_one_set_version, res >> > + >> > + >> > +def zero_to_one_dmesg(result): >> > + """ version 1: dmesg is converted from a list to a string """ >> > + assert result.tests['sometest']['dmesg'] == 'this\nis\ndmesg' >> > + >> > + >> > [email protected] >> > +def zero_to_one_subtests_remove_duplicates(result): >> > + """ Version 1: Removes duplicate entrieds""" >> > + assert 'group1/groupA/test/subtest 1' not in result.tests >> > + assert 'group1/groupA/test/subtest 2' not in result.tests >> > + >> > + >> > [email protected] >> > +def zero_to_one_subtests_add_test(result): >> > + """ Add an entry for the actual test """ >> > + assert result.tests.get('group1/groupA/test') >> > + >> > + >> > [email protected] >> > +def zero_to_one_subtests_test_is_testresult(result): >> > + """ The result of the new test is a TestResult Instance """ >> > + assert isinstance( >> > + result.tests['group1/groupA/test'], >> > + results.TestResult) >> > + >> > + >> > +def zero_to_one_info_delete(result): >> > + """ Remove the info name from results """ >> > + for value in result.tests.itervalues(): >> > + assert 'info' not in value >> > + >> > + >> > +def zero_to_one_returncode_from_info(result): >> > + """ Info returncode should be added to returncode if there isn't one >> > """ + assert result.tests['sometest']['returncode'] == 1 >> > + >> > + >> > +def zero_to_one_returncode_no_override(result): >> > + """ The returncode from info should not overwrite an existing >> > returcnode + attribute >> > + >> > + This test only tests that the value in info isn't used when there is >> > a >> > + value in returncode already >> > + >> > + """ >> > + assert result.tests['group1/groupA/test']['returncode'] != 1 >> > + >> > + >> > +def zero_to_one_err_from_info(result): >> > + """ generate err from info """ >> > + assert result.tests['group1/groupA/test']['err'] == 'stderr' >> > + >> > + >> > +def zero_to_one_out_from_info(result): >> > + """ generate out from info """ >> > + assert result.tests['group1/groupA/test']['out'] == 'stdout' >> > + >> > + >> > +def zero_to_one_set_version(result): >> > + """ Set the version to 1 """ >> > + assert result.results_version == 1 >> > diff --git a/framework/tests/utils.py b/framework/tests/utils.py >> > index f337b1e..252ced6 100644 >> > --- a/framework/tests/utils.py >> > +++ b/framework/tests/utils.py >> > >> > @@ -33,6 +33,7 @@ try: >> > import simplejson as json >> > >> > except ImportError: >> > import json >> > >> > +import framework.results >> > >> > __all__ = [ >> > >> > @@ -49,6 +50,7 @@ JSON_DATA = { >> > >> > "filter": [], >> > "exclude_filter": [] >> > >> > }, >> > >> > + "results_version": framework.results.CURRENT_JSON_VERSION, >> > >> > "name": "fake-tests", >> > "lspci": "fake", >> > "glxinfo": "fake", >> > >> > -- >> > 2.0.0 >> > >> > _______________________________________________ >> > Piglit mailing list >> > [email protected] >> > http://lists.freedesktop.org/mailman/listinfo/piglit _______________________________________________ Piglit mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/piglit
