The code currently assumed a DISTRO has one libc and can't handle the case where there are ptest results for both. Add in TCLIBC to the result data to allow them to be differentiated and add support in the reporting to see the difference between them.
We need to carefully handle the case where there is no TCLIBC data in the results as the code currently assumes all data is present. Signed-off-by: Richard Purdie <[email protected]> --- meta/classes-recipe/testimage.bbclass | 1 + .../oeqa/selftest/cases/resulttooltests.py | 2 +- scripts/lib/resulttool/report.py | 50 ++++++++++++------- scripts/lib/resulttool/resultutils.py | 11 +++- .../template/test_report_full_text.txt | 10 ++-- 5 files changed, 48 insertions(+), 26 deletions(-) diff --git a/meta/classes-recipe/testimage.bbclass b/meta/classes-recipe/testimage.bbclass index b89ac2862e3..5f0ec0b3a05 100644 --- a/meta/classes-recipe/testimage.bbclass +++ b/meta/classes-recipe/testimage.bbclass @@ -159,6 +159,7 @@ def get_testimage_configuration(d, test_type, machine): 'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"), 'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"), 'STARTTIME': d.getVar("DATETIME"), + 'TCLIBC': d.getVar("TCLIBC"), 'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'), 'LAYERS': get_layers(d.getVar("BBLAYERS"))} return configuration diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index 2b0e089e959..b03a880b2bb 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -64,7 +64,7 @@ class ResultToolTests(OESelftestTestCase): 'test4': {'status': 'ERROR'}, 'test5': {'status': 'SKIPPED'}}} report = ResultsTextReport() - result_report = report.get_aggregated_test_result(None, result_data, 'DummyMachine') + result_report = report.get_aggregated_test_result(None, result_data, 'DummyMachine', 'glibc') self.assertTrue(result_report['passed'] == 2, msg="Passed count not correct:%s" % result_report['passed']) self.assertTrue(result_report['failed'] == 2, msg="Failed count not correct:%s" % result_report['failed']) self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped']) diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py index 1c100b00ab3..ba9965aa0a9 100644 --- a/scripts/lib/resulttool/report.py +++ b/scripts/lib/resulttool/report.py @@ -24,22 +24,24 @@ class ResultsTextReport(object): 'skipped': ['SKIPPED', 'skipped', 'UNSUPPORTED', 'UNTESTED', 'UNRESOLVED']} - def handle_ptest_result(self, k, status, result, machine): - if machine not in self.ptests: - self.ptests[machine] = {} + def handle_ptest_result(self, k, status, result, machine, libc): + if libc not in self.ptests: + self.ptests[libc] = {} + if machine not in self.ptests[libc]: + self.ptests[libc][machine] = {} if k == 'ptestresult.sections': # Ensure tests without any test results still show up on the report for suite in result['ptestresult.sections']: - if suite not in self.ptests[machine]: - self.ptests[machine][suite] = { + if suite not in self.ptests[libc][machine]: + self.ptests[libc][machine][suite] = { 'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': [], "testcases": set(), } if 'duration' in result['ptestresult.sections'][suite]: - self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration'] + self.ptests[libc][machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration'] if 'timeout' in result['ptestresult.sections'][suite]: - self.ptests[machine][suite]['duration'] += " T" + self.ptests[libc][machine][suite]['duration'] += " T" return True # process test result @@ -57,21 +59,21 @@ class ResultsTextReport(object): except ValueError: pass - if suite not in self.ptests[machine]: - self.ptests[machine][suite] = { + if suite not in self.ptests[libc][machine]: + self.ptests[libc][machine][suite] = { 'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': [], "testcases": set(), } # do not process duplicate results - if test in self.ptests[machine][suite]["testcases"]: + if test in self.ptests[libc][machine][suite]["testcases"]: print("Warning duplicate ptest result '{}.{}' for {}".format(suite, test, machine)) return False for tk in self.result_types: if status in self.result_types[tk]: - self.ptests[machine][suite][tk] += 1 - self.ptests[machine][suite]["testcases"].add(test) + self.ptests[libc][machine][suite][tk] += 1 + self.ptests[libc][machine][suite]["testcases"].add(test) return True def handle_ltptest_result(self, k, status, result, machine): @@ -136,13 +138,13 @@ class ResultsTextReport(object): if status in self.result_types[tk]: self.ltpposixtests[machine][suite][tk] += 1 - def get_aggregated_test_result(self, logger, testresult, machine): + def get_aggregated_test_result(self, logger, testresult, machine, libc): test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []} result = testresult.get('result', []) for k in result: test_status = result[k].get('status', []) if k.startswith("ptestresult."): - if not self.handle_ptest_result(k, test_status, result, machine): + if not self.handle_ptest_result(k, test_status, result, machine, libc): continue elif k.startswith("ltpresult."): self.handle_ltptest_result(k, test_status, result, machine) @@ -166,6 +168,7 @@ class ResultsTextReport(object): havefailed = False reportvalues = [] machines = [] + libcs = set() cols = ['passed', 'failed', 'skipped'] maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' : 0, 'ptest' : 0 ,'ltptest': 0, 'ltpposixtest': 0} for line in test_count_reports: @@ -192,10 +195,13 @@ class ResultsTextReport(object): for k in cols: reporttotalvalues[k] = '%s' % sum([line[k] for line in test_count_reports]) reporttotalvalues['count'] = '%s' % len(test_count_reports) - for (machine, report) in self.ptests.items(): - for ptest in self.ptests[machine]: - if len(ptest) > maxlen['ptest']: - maxlen['ptest'] = len(ptest) + for libc in self.ptests: + libcs.add(libc) + print(str(libc)) + for machine in self.ptests[libc]: + for ptest in self.ptests[libc][machine]: + if len(ptest) > maxlen['ptest']: + maxlen['ptest'] = len(ptest) for (machine, report) in self.ltptests.items(): for ltptest in self.ltptests[machine]: if len(ltptest) > maxlen['ltptest']: @@ -208,6 +214,7 @@ class ResultsTextReport(object): reporttotalvalues=reporttotalvalues, havefailed=havefailed, machines=machines, + libcs=list(libcs), ptests=self.ptests, ltptests=self.ltptests, ltpposixtests=self.ltpposixtests, @@ -268,6 +275,11 @@ class ResultsTextReport(object): skip = False result = testresults[testsuite][resultid] machine = result['configuration']['MACHINE'] + try: + libc = result['configuration']['TCLIBC'] + except KeyError: + # We don't have libc in old test results, not much we can do + libc = 'unknown' # Check to see if there is already results for these kinds of tests for the machine for key in result['result'].keys(): @@ -280,7 +292,7 @@ class ResultsTextReport(object): if skip: break - test_count_report = self.get_aggregated_test_result(logger, result, machine) + test_count_report = self.get_aggregated_test_result(logger, result, machine, libc) test_count_report['machine'] = machine test_count_report['testseries'] = result['configuration']['TESTSERIES'] test_count_report['result_id'] = resultid diff --git a/scripts/lib/resulttool/resultutils.py b/scripts/lib/resulttool/resultutils.py index b8fc79a6acd..53980f8007a 100644 --- a/scripts/lib/resulttool/resultutils.py +++ b/scripts/lib/resulttool/resultutils.py @@ -28,7 +28,7 @@ flatten_map = { } regression_map = { "oeselftest": ['TEST_TYPE', 'MACHINE'], - "runtime": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'IMAGE_PKGTYPE', 'DISTRO'], + "runtime": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'IMAGE_PKGTYPE', 'DISTRO', 'TCLIBC'], "sdk": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'], "sdkext": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'], "manual": ['TEST_TYPE', 'TEST_MODULE', 'IMAGE_BASENAME', 'MACHINE'] @@ -87,7 +87,14 @@ def append_resultsdata(results, f, configmap=store_map, configvars=extra_configv testtype = data[res]["configuration"].get("TEST_TYPE") if testtype not in configmap: raise ValueError("Unknown test type %s" % testtype) - testpath = "/".join(data[res]["configuration"].get(i) for i in configmap[testtype]) + paths = [] + for i in configmap[testtype]: + try: + paths.append(data[res]["configuration"].get(i)) + # Some variables may not exist in older data + except KeyError: + continue + testpath = "/".join(paths) if testpath not in results: results[testpath] = {} results[testpath][res] = data[res] diff --git a/scripts/lib/resulttool/template/test_report_full_text.txt b/scripts/lib/resulttool/template/test_report_full_text.txt index 2efba2ef6f2..a39878a3e11 100644 --- a/scripts/lib/resulttool/template/test_report_full_text.txt +++ b/scripts/lib/resulttool/template/test_report_full_text.txt @@ -11,21 +11,23 @@ Test Result Status Summary (Counts/Percentages sorted by testseries, ID) {{ 'Total'.ljust(maxlen['testseries']) }} | {{ reporttotalvalues['count'].ljust(maxlen['result_id']) }} | {{ reporttotalvalues['passed'].ljust(maxlen['passed']) }} | {{ reporttotalvalues['failed'].ljust(maxlen['failed']) }} | {{ reporttotalvalues['skipped'].ljust(maxlen['skipped']) }} -------------------------------------------------------------------------------------------------------------- +{% for libc in libcs %} {% for machine in machines %} -{% if ptests[machine] %} +{% if ptests[libc][machine] %} ============================================================================================================== -{{ machine }} PTest Result Summary +{{ machine }} PTest Result Summary (Libc: {{ libc }}) ============================================================================================================== -------------------------------------------------------------------------------------------------------------- {{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} -------------------------------------------------------------------------------------------------------------- -{% for ptest in ptests[machine] |sort %} -{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[machine][ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[machine][ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[machine][ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[machine][ptest]['duration']|string) }} +{% for ptest in ptests[libc][machine] |sort %} +{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[libc][machine][ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[libc][machine][ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[libc][machine][ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[libc][machine][ptest]['duration']|string) }} {% endfor %} -------------------------------------------------------------------------------------------------------------- {% endif %} {% endfor %} +{% endfor %} {% for machine in machines %} {% if ltptests[machine] %}
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#241862): https://lists.openembedded.org/g/openembedded-core/message/241862 Mute This Topic: https://lists.openembedded.org/mt/120413104/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
