https://github.com/python/cpython/commit/7715c924de820e55b7ad7529fb0354a9a76365f6 commit: 7715c924de820e55b7ad7529fb0354a9a76365f6 branch: main author: Cody Maloney <[email protected]> committer: zware <[email protected]> date: 2026-10-23T20:09:50Z summary:
gh-158895: Teach libregrtest to output github job summaries (GH-158896) Goal is to make it faster to triage individual test failures while keeping the full existing regrtest fidelity. Fairly minimal integration to hopefully save some human eyeball time. Setup so only the test summary process should output the summary, subprocesses have the env var which will cause output unset. Includes a couple of fixes to existing tests that produced false-positives. --------- Co-authored-by: Claude Opus 5.5 <[email protected]> files: A Lib/test/libregrtest/reporter.py A Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst M .github/workflows/reusable-test-lazy-imports-all.yml M Lib/test/libregrtest/main.py M Lib/test/libregrtest/result.py M Lib/test/libregrtest/run_workers.py M Lib/test/libregrtest/runtests.py M Lib/test/libregrtest/setup.py M Lib/test/libregrtest/testresult.py M Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py M Lib/test/test_regrtest.py diff --git a/.github/workflows/reusable-test-lazy-imports-all.yml b/.github/workflows/reusable-test-lazy-imports-all.yml index c0851aeddc59ae..b30a776f1a9858 100644 --- a/.github/workflows/reusable-test-lazy-imports-all.yml +++ b/.github/workflows/reusable-test-lazy-imports-all.yml @@ -64,6 +64,8 @@ jobs: - name: Verify excluded tests still need exclusion run: | set -euo pipefail + # Failures are expected here, don't report them to the job summary + unset GITHUB_STEP_SUMMARY unexpected_passes=() while IFS= read -r test_name; do [ -n "$test_name" ] || continue diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2e8397a8a91d32..032cb94223a7e6 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -15,6 +15,7 @@ from .findtests import findtests, split_test_packages, list_cases, collect_cases from .logger import Logger from .pgo import setup_pgo_tests +from .reporter import get_reporter from .result import TestResult from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak @@ -68,6 +69,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Logger self.logger = Logger(self.results, self.quiet, self.pgo) + self.reporter = get_reporter() # Actions self.want_header: bool = ns.header @@ -284,8 +286,10 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False) + output_on_failure=False, + final_run=True) self.logger.set_tests(runtests) + self.reporter.run_started(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" if not self.single_process: @@ -395,6 +399,7 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) + self.reporter.test_result(result) return result @@ -459,7 +464,8 @@ def get_state(self) -> str: def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from .run_workers import RunWorkers - RunWorkers(num_workers, runtests, self.logger, self.results).run() + RunWorkers(num_workers, runtests, self.logger, self.results, + self.reporter).run() def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.next_single_filename: @@ -480,6 +486,8 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) + self.reporter.summary(self.results, self.get_state()) + def display_summary(self) -> None: if self.first_runtests is None: raise ValueError( @@ -517,6 +525,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + final_run=not self.want_rerun or bool(self.python_cmd), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -568,6 +577,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: runtests = runtests.copy(tests=case_ids, case_groups=case_groups) self.first_runtests = runtests self.logger.set_tests(runtests) + self.reporter.run_started(runtests) if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interferes with refleak check diff --git a/Lib/test/libregrtest/reporter.py b/Lib/test/libregrtest/reporter.py new file mode 100644 index 00000000000000..7d9260b47ec369 --- /dev/null +++ b/Lib/test/libregrtest/reporter.py @@ -0,0 +1,137 @@ +import html +import os +import pathlib +import re +from _colorize import decolor # type: ignore[import-not-found] +from typing import TextIO + +from .findtests import findtestdir +from .result import TestResult +from .results import TestResults +from .runtests import RunTests +from .utils import count + + +class Reporter: + """Report test failures, for example to a CI service. A no-op here.""" + + def run_started(self, runtests: RunTests) -> None: + """Called at the start of each run, and in the test process.""" + + def test_case_failed(self, test_case: str, traceback: str, + stream: TextIO) -> None: + """Called in the test process after a test case failed.""" + + def test_result(self, result: TestResult) -> None: + """Called in the main process after a test file completed.""" + + def summary(self, results: TestResults, state: str) -> None: + """Called in the main process after all runs.""" + + @staticmethod + def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]: + """Return (filename, line, exception) of a failure.""" + # A frame is its "File" line and its source lines + frames = list(re.finditer( + r'^ *File "(.*)", line (\d+).*\n(?: .*\n)*', traceback, + re.MULTILINE)) + filename = line = None + exception = traceback + if frames: + filename, line = frames[0].groups() + exception = traceback[frames[-1].end():] + # Prefer the last frame in a test file, the assertion + for frame in frames: + if pathlib.Path(frame[1]).name.startswith("test_"): + filename, line = frame.groups() + return filename, line, exception.strip() + + +def get_reporter() -> Reporter: + if GitHubActionsReporter.enabled(): + return GitHubActionsReporter() + return Reporter() + + +class GitHubActionsReporter(Reporter): + """Annotate failures in the job log and list them in the job summary.""" + + ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) + ESCAPE_PROPERTY = ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"}) + + @staticmethod + def enabled() -> bool: + return bool(os.environ.get("GITHUB_STEP_SUMMARY")) + + def run_started(self, runtests: RunTests) -> None: + self.runtests = runtests + + def test_case_failed(self, test_case: str, traceback: str, + stream: TextIO) -> None: + if not self.runtests.final_run: + return + filename, line, exception = self.parse_traceback(decolor(traceback)) + self.annotate(test_case, exception, filename, line, stream) + + def test_result(self, result: TestResult) -> None: + if not self.runtests.final_run: + return + if not result.is_failed(self.runtests.fail_env_changed): + return + # Already annotated by test_case_failed() + if result.failed_test_cases: + return + # "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py + name = result.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(self.runtests.test_dir), + *name.split(".")) + filename = None + for candidate in (path.with_suffix(".py"), path / "__init__.py"): + if candidate.exists(): + filename = str(candidate) + message = "\n".join([str(result), *(result.env_changed_reasons or ())]) + self.annotate(result.test_name, decolor(message), filename) + + def annotate(self, title: str, message: str, filename: str | None, + line: str | None = None, + stream: TextIO | None = None) -> None: + if filename: + # Make installed stdlib paths relative to the checkout + path = pathlib.Path(filename) + stdlib_dir = pathlib.Path(os.__file__).parent + if path.is_relative_to(stdlib_dir): + filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() + # GitHub needs a line to show the annotation on the file + line = line or "1" + props = {"file": filename, "line": line, "title": title} + props_text = ",".join(f"{key}={value.translate(self.ESCAPE_PROPERTY)}" + for key, value in props.items() if value) + print(f"::error {props_text}::{message.translate(self.ESCAPE_DATA)}", + file=stream, flush=True) + + def summary(self, results: TestResults, state: str) -> None: + failed = [result for result in results.rerun_results + if result.is_failed(self.runtests.fail_env_changed)] + if not failed: + return + ncases = sum(len(result.failed_test_cases) for result in failed) + blocks = [f"## {decolor(state)}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncases, 'test case')} failed"] + for result in failed: + blocks.append(f"### {decolor(str(result))}") + if result.env_changed_reasons: + blocks.append("\n".join( + f"- {reason}" for reason in result.env_changed_reasons)) + for name, traceback in result.failed_test_cases: + # Expand short tracebacks if there are only a few + if ncases <= 5 and traceback.count("\n") < 30: + details = "<details open>" + else: + details = "<details>" + blocks += [f"{details}<summary>{html.escape(name)}</summary>", + f"```pytb\n{decolor(traceback).rstrip()}\n```", + "</details>"] + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", + encoding="utf-8") as fp: + fp.write("".join(f"{block}\n\n" for block in blocks)) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 5324a46bb8cb1c..f8aa256be1ecfe 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -178,6 +178,10 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) + @property + def failed_test_cases(self) -> list[tuple[str, str]]: + return (self.errors or []) + (self.failures or []) + def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba3..ad02d8dba7c23e 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -16,6 +16,7 @@ from test.support import os_helper, MS_WINDOWS from .logger import Logger +from .reporter import Reporter from .result import TestResult, State from .results import TestResults from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType @@ -499,13 +500,15 @@ def get_running(workers: list[WorkerThread]) -> str | None: class RunWorkers: def __init__(self, num_workers: int, runtests: RunTests, - logger: Logger, results: TestResults) -> None: + logger: Logger, results: TestResults, + reporter: Reporter) -> None: self.num_workers = num_workers self.runtests = runtests self.logger = logger self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results + self.reporter = reporter self.live_worker_count = 0 self.output: queue.Queue[QueueContent] = queue.Queue() @@ -624,6 +627,8 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + # Report after the output: env changed warnings, crash traceback + self.reporter.test_result(result) return result diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index d86b082fb34a60..525f9573ad992d 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,6 +93,8 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + # Failed tests are not re-run after this run + final_run: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e..75a596c1cccc52 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -11,7 +11,9 @@ from _colorize import can_colorize # type: ignore[import-not-found] from .filter import set_match_tests +from .reporter import get_reporter from .runtests import RunTests +from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -112,9 +114,11 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + reporter = get_reporter() + reporter.run_started(runtests) + RegressionTestResult.reporter = reporter if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 605f1f4e6a89fb..3e4243282236cf 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,10 +9,12 @@ import traceback import unittest from test import support +from test.libregrtest.reporter import Reporter from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False + reporter = Reporter() def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -130,6 +132,14 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) + def printErrorList(self, flavour, errors): + for test, err in errors: + super().printErrorList(flavour, [(test, err)]) + self.report_failure(test, err) + + def report_failure(self, test, err): + self.reporter.test_case_failed(str(test), err, self.stream) + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") @@ -146,6 +156,9 @@ def __init__(self, stream, buffer=False): def run(self, test): test(self.result) + # Failures are not printed in quiet mode + for test, err in self.result.errors + self.result.failures: + self.result.report_failure(test, err) return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index ae4ee969f34d8b..dff6c7e3ab2631 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -12,6 +12,7 @@ import unittest from unittest import mock from test.support import requires, requires_remote_subprocess_debugging +from test.support import os_helper from test.support.import_helper import import_module # Only run these tests if curses is available @@ -870,8 +871,11 @@ def test_run_failed_module_live(self): ), mock.patch('curses.wrapper', side_effect=self.mock_curses_wrapper), mock.patch("sys.argv", args), - mock.patch('sys.stderr', new=io.StringIO()) as fake_stderr + mock.patch('sys.stderr', new=io.StringIO()) as fake_stderr, + os_helper.EnvironmentVarGuard() as env, ): + # Don't report the crash to the CI running this test + env.unset('GITHUB_STEP_SUMMARY') main() self.assertIn( 'test test_asdasd crashed -- Traceback (most recent call last):', diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index dc9770fb4e5fbd..e8211526ae847c 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -22,6 +22,7 @@ import tempfile import threading import textwrap +import types import unittest import unittest.mock from xml.etree import ElementTree @@ -37,7 +38,9 @@ from test.libregrtest import setup from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test -from test.libregrtest.result import TestStats +from test.libregrtest.reporter import GitHubActionsReporter +from test.libregrtest.result import State, TestResult, TestStats +from test.libregrtest.results import TestResults from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -628,6 +631,10 @@ def setUp(self): self.tmptestdir = tempfile.mkdtemp() self.addCleanup(os_helper.rmtree, self.tmptestdir) + # Don't report the failures of the tests to the CI running this test + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') + def create_test(self, name=None, code=None): if not name: name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID @@ -1639,6 +1646,35 @@ def test_env_changed(self): success=True), stats=2) + def test_reporter(self): + code = textwrap.dedent(""" + import os, unittest + + class Tests(unittest.TestCase): + def test_fail(self): + self.fail() + """) + fail = self.create_test('reporter_fail', code) + crash = self.create_test('reporter_crash', + code.replace('self.fail()', 'os._exit(1)')) + summary = os.path.join(self.tmptestdir, 'summary.md') + env = dict(os.environ, GITHUB_STEP_SUMMARY=summary) + + def reported(*args, exitcode=EXITCODE_BAD_TEST): + output = self.run_tests('-j1', *args, env=env, exitcode=exitcode) + return re.findall(r'^::error file=.*,title=(\w+)', output, + re.MULTILINE) + + self.assertEqual(reported(self.create_test(), exitcode=0), []) + self.assertFalse(os.path.exists(summary)) + # Quiet mode doesn't print the failure, but reports it + self.assertEqual(reported(fail), ['test_fail']) + # Only the re-run is reported + self.assertEqual(reported('--rerun', fail, crash), + ['test_fail', crash]) + with open(summary, encoding='utf-8') as fp: + self.assertIn('2 test files and 1 test case failed', fp.read()) + def test_rerun_fail(self): # FAILURE then FAILURE code = textwrap.dedent(""" @@ -2946,7 +2982,73 @@ def test_sanitize_xml(self): 'valid t\xe9xt \u20ac') -from test.libregrtest.results import TestResults +class GitHubActionsReporterTestCase(unittest.TestCase): + def setUp(self): + self.reporter = GitHubActionsReporter() + self.reporter.run_started(types.SimpleNamespace( + final_run=True, fail_env_changed=True, test_dir=None)) + + def test_test_case_failed(self): + stdlib_dir = os.path.dirname(os.__file__) + traceback = textwrap.dedent(f"""\ + Traceback (most recent call last): + File "{stdlib_dir}/test/test_x.py", line 3, in test_x + helper() + File "{stdlib_dir}/helper.py", line 9, in helper + raise ValueError('50%') + ValueError: 50% + more + """) + stream = io.StringIO() + self.reporter.test_case_failed('test_x (a: b, c)', traceback, stream) + self.assertEqual( + stream.getvalue(), + '::error file=Lib/test/test_x.py,line=3,' + 'title=test_x (a%3A b%2C c)::ValueError: 50%25%0Amore\n') + + def test_test_result(self): + result = TestResult('test_x', state=State.ENV_CHANGED, + env_changed_reasons=['dangling thread']) + with support.captured_stdout() as stdout: + self.reporter.test_result(result) + self.assertEqual(stdout.getvalue(), + '::error title=test_x::' + 'test_x failed (env changed)%0Adangling thread\n') + + def test_summary(self): + results = TestResults() + results.rerun_results = [ + TestResult('test_x', state=State.FAILED, + failures=[('test_a (<T>)', + 'Traceback\nAssertionError\n')]), + TestResult('test_y', state=State.ENV_CHANGED, + env_changed_reasons=['dangling thread']), + ] + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with os_helper.EnvironmentVarGuard() as env: + env['GITHUB_STEP_SUMMARY'] = os_helper.TESTFN + self.reporter.summary(results, 'FAILURE') + with open(os_helper.TESTFN, encoding='utf-8') as fp: + summary = fp.read() + self.assertEqual(summary, textwrap.dedent("""\ + ## FAILURE: 2 test files and 1 test case failed + + ### test_x failed (1 failure) + + <details open><summary>test_a (<T>)</summary> + + ```pytb + Traceback + AssertionError + ``` + + </details> + + ### test_y failed (env changed) + + - dangling thread + + """)) class TestColorized(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst new file mode 100644 index 00000000000000..a1c9e648e5c744 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -0,0 +1,3 @@ +When run in GitHub Actions, :mod:`test.regrtest` now annotates each test +failure in the job log and writes a summary of the failures to the job +summary. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
