With --run_isolated, parse_tests() writes the same JSON or JUnit
destination after every kernel. Only the last kernel survives in a file,
so an earlier failing suite can disappear from the saved results.
Stdout similarly receives several independent report documents.
Separate report writing from parsing. Keep printing each kernel's results
as they are parsed, accumulate the top-level tests and counts, and write
one report after execution finishes. Preserve the parsed root for a
single kernel and continue suppressing reports with --raw_output.
Test JSON and JUnit files with an early failure and suite and test
isolation. Also check equivalence between single-kernel execution and
parsing.
Fixes: ff9e09a3762f ("kunit: tool: support running each suite/test separately")
Signed-off-by: Mark Brown <[email protected]>
---
tools/testing/kunit/kunit.py | 35 +++++++++++++++----
tools/testing/kunit/kunit_tool_test.py | 61 ++++++++++++++++++++++++++++++++++
2 files changed, 89 insertions(+), 7 deletions(-)
diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 218ae294ad1b..aea67b6a72c4 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -16,7 +16,7 @@ import time
assert sys.version_info >= (3, 7), "Python version is too old"
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from enum import Enum, auto
from typing import Iterable, List, Optional, Sequence, Tuple
@@ -191,6 +191,10 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitExecRequest) -
metadata = kunit_json.Metadata(arch=linux.arch(),
build_dir=request.build_dir, def_config='kunit_defconfig')
test_counts = kunit_parser.TestCounts()
+ report = kunit_parser.Test()
+ report.name = 'main'
+ report.counts = test_counts
+ parse_request = replace(request, json=None, junit=None)
test_failed = False
exec_time = 0.0
for i, filter_glob in enumerate(filter_globs):
@@ -205,7 +209,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitExecRequest) -
filter_action=request.filter_action,
build_dir=request.build_dir)
- _, test_result = parse_tests(request, metadata, run_result)
+ _, test_result = parse_tests(parse_request, metadata,
run_result)
# run_kernel() doesn't block on the kernel exiting.
# That only happens after we get the last line of output from
`run_result`.
# So exec_time here actually contains parsing + execution time,
which is fine.
@@ -216,6 +220,19 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitExecRequest) -
# A kernel with no results has no failed test counts to
aggregate.
if _map_to_overall_status(test_result.status) !=
KunitStatus.SUCCESS:
test_failed = True
+ if request.json or request.junit:
+ if len(filter_globs) == 1:
+ report = test_result
+ else:
+ report.subtests.extend(test_result.subtests)
+ report.log.extend(test_result.log)
+ if len(filter_globs) != 1:
+ report.expected_count = len(report.subtests)
+ report.status = test_counts.get_status()
+ if test_failed and report.ok_status():
+ report.status =
kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS
+ if not request.raw_output:
+ _write_test_results(request, metadata, report)
if len(filter_globs) == 1 and test_counts.crashed > 0:
bd = request.build_dir
@@ -264,6 +281,15 @@ def parse_tests(request: KunitParseRequest, metadata:
kunit_json.Metadata, input
kunit_parser.print_test(test, request.failed, stdout)
kunit_parser.print_summary_line(test, stdout)
+ _write_test_results(request, metadata, test)
+
+ if test.status != kunit_parser.TestStatus.SUCCESS:
+ return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test
+
+ return KunitResult(KunitStatus.SUCCESS, parse_time), test
+
+def _write_test_results(request: KunitParseRequest, metadata:
kunit_json.Metadata,
+ test: kunit_parser.Test) -> None:
if request.json:
json_str = kunit_json.get_json_result(
test=test,
@@ -283,11 +309,6 @@ def parse_tests(request: KunitParseRequest, metadata:
kunit_json.Metadata, input
kunit_junit.write_junit_result(test=test,filename=request.junit)
stdout.print_with_timestamp(f"Test results stored in
{os.path.abspath(request.junit)}")
- if test.status != kunit_parser.TestStatus.SUCCESS:
- return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test
-
- return KunitResult(KunitStatus.SUCCESS, parse_time), test
-
def run_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitRequest) -> KunitResult:
run_start = time.time()
diff --git a/tools/testing/kunit/kunit_tool_test.py
b/tools/testing/kunit/kunit_tool_test.py
index ca2dac6651aa..4590108e4306 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -19,6 +19,7 @@ import signal
import subprocess
import sys
from typing import Iterable
+import xml.etree.ElementTree as ET
import kunit_config
import kunit_parser
@@ -738,6 +739,7 @@ class KUnitMainTest(unittest.TestCase):
self.mock_linux_init = mock.patch.object(kunit_kernel,
'LinuxSourceTree').start()
self.linux_source_mock = self.mock_linux_init.return_value
+ self.linux_source_mock.arch.return_value = 'um'
self.linux_source_mock.build_reconfig.return_value = True
self.linux_source_mock.build_kernel.return_value = True
self.linux_source_mock.run_kernel.return_value = all_passed_log
@@ -1049,6 +1051,65 @@ class KUnitMainTest(unittest.TestCase):
kunit.main(['exec', '--run_isolated=suite'])
+ @mock.patch.object(kunit, '_list_tests')
+ def test_isolated_reports_include_all_kernels(self, mock_tests):
+ def fake_run_kernel(filter_glob, **kwargs):
+ if '.' in filter_glob:
+ suite, case = filter_glob.split('.')
+ failed = case == 'test1'
+ else:
+ suite, case = filter_glob, 'test'
+ failed = suite == 'suite1'
+ status = 'not ok' if failed else 'ok'
+ return ['KTAP version 1', '1..1', ' # Subtest: ' +
suite,
+ ' 1..1', f' {status} 1 - {case}',
f'{status} 1 - {suite}']
+ self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+ for isolation in ('suite', 'test'):
+ with self.subTest(isolation=isolation), \
+ tempfile.TemporaryDirectory() as output_dir:
+ if isolation == 'suite':
+ mock_tests.return_value =
['suite1.test', 'suite2.test']
+ expected = [('suite1', 'test', 'FAIL'),
('suite2', 'test', 'PASS')]
+ else:
+ mock_tests.return_value =
['suite.test1', 'suite.test2']
+ expected = [('suite', 'test1', 'FAIL'),
('suite', 'test2', 'PASS')]
+ json_path = os.path.join(output_dir,
'results.json')
+ xml_path = os.path.join(output_dir,
'results.xml')
+ with self.assertRaises(SystemExit) as e:
+ kunit.main(['exec', '--run_isolated=' +
isolation,
+ '--json=' + json_path,
'--junit=' + xml_path])
+ self.assertEqual(e.exception.code, 1)
+
+ with open(json_path) as f:
+ report = json.load(f)
+ self.assertEqual(report['misc']['tests'], 2)
+ self.assertEqual(report['misc']['failed'], 1)
+ self.assertEqual(report['arch'], 'um')
+ self.assertEqual([(suite['name'], case['name'],
case['status'])
+ for suite in report['sub_groups'] for
case in suite['test_cases']], expected)
+ root = ET.parse(xml_path).getroot()
+ self.assertEqual(root.get('tests'), '2')
+ self.assertEqual(root.get('failures'), '1')
+ self.assertEqual([(suite.get('name'),
case.get('name'),
+ 'FAIL' if case.find('failure') is not
None else 'PASS')
+ for suite in root for case in suite],
expected)
+
+ def test_single_kernel_reports_match_parse(self):
+ with tempfile.TemporaryDirectory() as output_dir:
+ json_path = os.path.join(output_dir, 'results.json')
+ xml_path = os.path.join(output_dir, 'results.xml')
+ reports = []
+ for args in (['exec'], ['parse',
_test_data_path('test_is_test_passed-all_passed.log')]):
+ kunit.main(args + ['--json=' + json_path,
'--junit=' + xml_path])
+ with open(json_path) as f:
+ report = json.load(f)
+ # Execution adds build metadata, which parse
cannot know.
+ reports.append((report['misc'],
+ [(s['name'], s['test_cases']) for s in
report['sub_groups']],
+
ET.tostring(ET.parse(xml_path).getroot())))
+ self.assertEqual(reports[0], reports[1])
+
@mock.patch.object(kunit, '_list_tests')
@mock.patch.object(sys, 'stdout', new_callable=io.StringIO)
def test_list_suites(self, mock_stdout, mock_tests):
--
2.47.3
