https://github.com/python/cpython/commit/d625ecbb8d36b7dd26c61b13d1c97de117d36642 commit: d625ecbb8d36b7dd26c61b13d1c97de117d36642 branch: 3.15 author: Pablo Galindo Salgado <[email protected]> committer: pablogsal <[email protected]> date: 2026-10-05T16:17:02+01:00 summary:
[3.15] gh-158539: Fix exception mode missing handlers in generators/coroutines (GH-158581) (#158852) * [3.15] gh-158539: Fix exception mode missing handlers in generators/coroutines (GH-158581) Backport of GH-158581. Co-authored-by: LucasZhou <[email protected]> * [3.15] gh-158539: Use portable static assertion messages * [3.15] gh-158539: Keep layout assertions with debug-offset validation * [3.15] gh-158539: Use the platform guard for in-process inspection tests --------- Co-authored-by: LucasZhou <[email protected]> files: A Misc/NEWS.d/next/Library/2026-10-01-14-00-00.gh-issue-158539.q1w2e3.rst M Lib/test/test_external_inspection.py M Modules/_remote_debugging/debug_offsets_validation.h M Modules/_remote_debugging/threads.c diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index a0bedf4910fa2a..fa37629c7a9a1b 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -3219,6 +3219,242 @@ def test_finally_no_exception_no_flag(self): self._check_exception_status(p, thread_tid, expect_exception=False) +@skip_if_not_supported +class TestExceptionDetectionInProcess(RemoteInspectionTestBase): + """gh-158539: HAS_EXCEPTION for handlers running in generators/coroutines. + + ``TestExceptionDetectionScenarios`` samples a child process and therefore + needs subprocess debugging permissions. These tests inspect the current + process with ``RemoteUnwinder`` and only need self-inspection, so they also + run on macOS without special entitlements. + """ + + @classmethod + def setUpClass(cls): + try: + RemoteUnwinder(os.getpid(), all_threads=True).get_stack_trace() + except PermissionError as exc: + raise unittest.SkipTest(f"self-inspection is unavailable: {exc}") + + def _check_running_handler( + self, target, expect_exception, *, mode=PROFILING_MODE_ALL, + skip_non_matching_threads=False, + ): + """Run *target* in a thread and check its HAS_EXCEPTION flag. + + *target* receives ``(ready, stop)`` events and must signal ``ready`` + only once it is executing inside the code region under test, then keep + running until ``stop`` is set. + """ + stop = threading.Event() + ready = threading.Event() + failure = [] + + def runner(): + try: + target(ready, stop) + except BaseException as exc: + failure.append(exc) + ready.set() + + thread = threading.Thread(target=runner, daemon=True) + thread.start() + try: + self.assertTrue(ready.wait(SHORT_TIMEOUT), "handler never started") + self.assertFalse(failure, f"handler raised {failure!r}") + + unwinder = RemoteUnwinder( + os.getpid(), + all_threads=True, + mode=mode, + skip_non_matching_threads=skip_non_matching_threads, + ) + observed = [] + for _ in busy_retry(SHORT_TIMEOUT): + with contextlib.suppress(*TRANSIENT_ERRORS): + statuses = self._get_thread_statuses(unwinder.get_stack_trace()) + status = statuses.get(thread.native_id) + if status is None: + continue + has_exception = bool(status & THREAD_STATUS_HAS_EXCEPTION) + observed.append(has_exception) + if has_exception == expect_exception: + break + self.assertTrue( + observed, "target thread status was never observed" + ) + self.assertIn( + expect_exception, + observed, + f"HAS_EXCEPTION was never {expect_exception} while the " + f"handler was running (observed {observed})", + ) + finally: + stop.set() + thread.join(SHORT_TIMEOUT) + + def _busy_until_stopped(self, ready, stop): + ready.set() + while not stop.is_set(): + time.sleep(0.001) + + def test_handler_in_function(self): + def target(ready, stop): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + + self._check_running_handler(target, expect_exception=True) + + def test_handler_in_generator(self): + def target(ready, stop): + def gen(): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + yield + + for _ in gen(): + pass + + self._check_running_handler(target, expect_exception=True) + + def test_handler_in_genexpr_callee(self): + def target(ready, stop): + def callee(): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + + list(callee() for _ in range(1)) + + self._check_running_handler(target, expect_exception=True) + + def test_handler_in_coroutine(self): + async def coro(ready, stop): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + + def target(ready, stop): + asyncio.run(coro(ready, stop)) + + self._check_running_handler(target, expect_exception=True) + + def test_handler_in_callee_from_coroutine(self): + def callee(ready, stop): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + + async def coro(ready, stop): + callee(ready, stop) + + def target(ready, stop): + asyncio.run(coro(ready, stop)) + + self._check_running_handler(target, expect_exception=True) + + def test_outer_handler_while_generator_runs(self): + """A generator with no handler of its own must not hide the outer one. + + ``exc_info`` points at the generator's empty ``_PyErr_StackItem`` whose + ``previous_item`` is the thread's ``exc_state``, so the profiler has to + walk the chain to find the exception ``sys.exception()`` reports. + """ + def target(ready, stop): + def gen(): + self._busy_until_stopped(ready, stop) + yield + + try: + raise ValueError("outer") + except ValueError: + for _ in gen(): + pass + + self._check_running_handler(target, expect_exception=True) + + def test_generator_without_exception(self): + def target(ready, stop): + def gen(): + self._busy_until_stopped(ready, stop) + yield + + for _ in gen(): + pass + + self._check_running_handler(target, expect_exception=False) + + def test_outer_handler_while_nested_generators_run(self): + def target(ready, stop): + def gen(depth): + if depth: + yield from gen(depth - 1) + else: + self._busy_until_stopped(ready, stop) + yield + + try: + raise ValueError("outer") + except ValueError: + for _ in gen(32): + pass + + self._check_running_handler( + target, + expect_exception=True, + mode=PROFILING_MODE_EXCEPTION, + skip_non_matching_threads=True, + ) + + def test_generator_finally_after_except(self): + """The handled exception is cleared before the generator's finally.""" + def target(ready, stop): + def gen(): + try: + raise ValueError("test") + except ValueError: + pass + finally: + self._busy_until_stopped(ready, stop) + yield + + for _ in gen(): + pass + + self._check_running_handler(target, expect_exception=False) + + def test_exception_mode_filter_keeps_generator_handler(self): + """The exception-mode thread filter must not drop a generator handler. + + This mirrors what ``--mode=exception`` actually does: threads without + HAS_EXCEPTION are skipped before their stack is unwound. + """ + def target(ready, stop): + def gen(): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + yield + + for _ in gen(): + pass + + self._check_running_handler( + target, + expect_exception=True, + mode=PROFILING_MODE_EXCEPTION, + skip_non_matching_threads=True, + ) + + @requires_remote_subprocess_debugging() class TestFrameCaching(RemoteInspectionTestBase): """Test that frame caching produces correct results. diff --git a/Misc/NEWS.d/next/Library/2026-10-01-14-00-00.gh-issue-158539.q1w2e3.rst b/Misc/NEWS.d/next/Library/2026-10-01-14-00-00.gh-issue-158539.q1w2e3.rst new file mode 100644 index 00000000000000..b1ab9c8e017fb2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-01-14-00-00.gh-issue-158539.q1w2e3.rst @@ -0,0 +1,5 @@ +Fix :mod:`profiling.sampling` exception mode discarding samples for code +running inside an ``except`` block in a generator or coroutine. The remote +debugger now follows ``tstate->exc_info`` and its ``previous_item`` chain +instead of only reading the embedded ``exc_state``, matching the exception +that :func:`sys.exception` reports. diff --git a/Modules/_remote_debugging/debug_offsets_validation.h b/Modules/_remote_debugging/debug_offsets_validation.h index c0c01a0a639e19..66c35e5522a9c7 100644 --- a/Modules/_remote_debugging/debug_offsets_validation.h +++ b/Modules/_remote_debugging/debug_offsets_validation.h @@ -48,6 +48,15 @@ static_assert( PY_REMOTE_ASYNC_DEBUG_OFFSETS_TOTAL_SIZE, "Update _remote_debugging validation for _Py_AsyncioModuleDebugOffsets"); +/* Derive unexported offsets from adjacent fields to keep the debug-offset + * table compatible across patch releases. */ +static_assert(offsetof(PyThreadState, exc_info) == + offsetof(PyThreadState, current_exception) + sizeof(uintptr_t), + "exc_info must immediately follow current_exception"); +static_assert(offsetof(_PyErr_StackItem, previous_item) == + offsetof(_PyErr_StackItem, exc_value) + sizeof(uintptr_t), + "previous_item must immediately follow exc_value"); + /* * This logic lives in a private header because it is shared by module.c and * asyncio.c. Keep the helpers static inline so they stay local to those users @@ -249,6 +258,7 @@ validate_fixed_field( #define PY_REMOTE_DEBUG_RUNTIME_STATE_FIELDS(APPLY, buffer_size) \ APPLY(runtime_state, interpreters_head, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size) +/* current_exception also covers the adjacent exc_info pointer. */ #define PY_REMOTE_DEBUG_THREAD_STATE_FIELDS(APPLY, buffer_size) \ APPLY(thread_state, native_thread_id, sizeof(unsigned long), _Alignof(long), buffer_size); \ APPLY(thread_state, interp, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ @@ -256,7 +266,7 @@ validate_fixed_field( APPLY(thread_state, status, FIELD_SIZE(PyThreadState, _status), _Alignof(unsigned int), buffer_size); \ APPLY(thread_state, holds_gil, sizeof(int), _Alignof(int), buffer_size); \ APPLY(thread_state, gil_requested, sizeof(int), _Alignof(int), buffer_size); \ - APPLY(thread_state, current_exception, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(thread_state, current_exception, 2 * sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ APPLY(thread_state, thread_id, sizeof(unsigned long), _Alignof(long), buffer_size); \ APPLY(thread_state, next, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ APPLY(thread_state, current_frame, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ @@ -351,10 +361,11 @@ _PyRemoteDebug_ValidateDebugOffsetsLayout(struct _Py_DebugOffsets *debug_offsets PY_REMOTE_DEBUG_THREAD_STATE_FIELDS( PY_REMOTE_DEBUG_VALIDATE_FIELD, SIZEOF_THREAD_STATE); + /* exc_value also covers the adjacent previous_item pointer. */ PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD( err_stackitem, exc_value, - sizeof(uintptr_t), + 2 * sizeof(uintptr_t), _Alignof(uintptr_t), sizeof(_PyErr_StackItem)); PY_REMOTE_DEBUG_VALIDATE_NESTED_FIELD( diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 666316ceaa28fc..198134fe6cfbea 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -17,6 +17,9 @@ #include <sys/wait.h> #endif +/* Bound traversal of corrupted remote exception chains. */ +#define MAX_EXCEPTION_CHAIN_DEPTH (2 << 15) + /* ============================================================================ * THREAD ITERATION FUNCTIONS * ============================================================================ */ @@ -436,16 +439,49 @@ unwind_stack_for_thread( has_exception = 1; } - // Check exc_state.exc_value (exception being handled in except block) - // exc_state is embedded in PyThreadState, so we read it directly from - // the thread state buffer. This catches most cases; nested exception - // handlers where exc_info points elsewhere are rare. + // Generators and coroutines use their own exception stack items. + // Follow exc_info to find the innermost handler, as sys.exception() does. if (!has_exception) { - uintptr_t exc_value = GET_MEMBER(uintptr_t, ts, - unwinder->debug_offsets.thread_state.exc_state + - unwinder->debug_offsets.err_stackitem.exc_value); - if (exc_value != 0) { - has_exception = 1; + uintptr_t exc_info = GET_MEMBER(uintptr_t, ts, + unwinder->debug_offsets.thread_state.current_exception + + sizeof(uintptr_t)); + uintptr_t exc_state_addr = + *current_tstate + unwinder->debug_offsets.thread_state.exc_state; + uintptr_t exc_value_offset = + unwinder->debug_offsets.err_stackitem.exc_value; + uintptr_t previous_item_offset = + exc_value_offset + sizeof(uintptr_t); + + for (int depth = 0; exc_info != 0 && depth < MAX_EXCEPTION_CHAIN_DEPTH; + depth++) + { + if (exc_info == exc_state_addr) { + // Bottom of the chain: the stack item embedded in the thread + // state, which is already in the local thread state buffer. + uintptr_t exc_value = GET_MEMBER(uintptr_t, ts, + unwinder->debug_offsets.thread_state.exc_state + + exc_value_offset); + if (exc_value != 0) { + has_exception = 1; + } + break; + } + uintptr_t exc_value = 0; + if (read_ptr(unwinder, exc_info + exc_value_offset, &exc_value) < 0) { + PyErr_Clear(); // Best effort: treat as no active exception + break; + } + if (exc_value != 0) { + has_exception = 1; + break; + } + uintptr_t previous_item = 0; + if (read_ptr(unwinder, exc_info + previous_item_offset, + &previous_item) < 0) { + PyErr_Clear(); + break; + } + exc_info = previous_item; } } _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
