https://github.com/python/cpython/commit/923cdee8d0f3cb8a0bbbea935b31c9c576a430da commit: 923cdee8d0f3cb8a0bbbea935b31c9c576a430da branch: 3.15 author: Miss Islington (bot) <[email protected]> committer: pablogsal <[email protected]> date: 2026-11-09T18:36:19+01:00 summary:
[3.15] gh-160267: Keep thread and interpreter state memory alive for remote profilers (GH-161674) (#161687) gh-160267: Keep thread and interpreter state memory alive for remote profilers (GH-161674) (cherry picked from commit e3262d9c3b9492d22b4f7dc4df903299846bf320) Co-authored-by: Pablo Galindo Salgado <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst M Python/pystate.c M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst new file mode 100644 index 000000000000000..c4467c7b64cddf1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst @@ -0,0 +1,4 @@ +Fix a crash when :mod:`profiling.sampling` profiles a process that creates and +destroys threads or subinterpreters quickly. Memory of deleted thread and +interpreter states is now reused for new states instead of being freed until +the runtime is finalized. diff --git a/Python/pystate.c b/Python/pystate.c index 6a7913feafc6341..c389926b493f7bf 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -308,9 +308,25 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime, ""); _Py_COMP_DIAG_POP +/* Remote profilers access interpreter and thread states without synchronizing + with their lifetime. Until the runtime is finalized, keep deleted states in + per-type free lists, so that a stale access only reaches a state of the same + type. */ +typedef struct state_header { + struct state_header *next; // Next entry in the free list + void *mem; // Block to free at finalization +} state_header; + +static struct { + PyMutex mutex; + state_header *tstates; + state_header *interpreters; +} state_freelists; + #define LOCKS_INIT(runtime) \ { \ &(runtime)->interpreters.mutex, \ + &state_freelists.mutex, \ &(runtime)->xi.data_lookup.registry.mutex, \ &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ @@ -389,6 +405,19 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) assert(runtime->object_state.interpreter_leaks == 0); #endif gilstate_clear(); + + state_header **lists[] = {&state_freelists.tstates, &state_freelists.interpreters}; + for (size_t i = 0; i < Py_ARRAY_LENGTH(lists); i++) { + PyMutex_LockFlags(&state_freelists.mutex, _Py_LOCK_DONT_DETACH); + state_header *entry = *lists[i]; + *lists[i] = NULL; + PyMutex_Unlock(&state_freelists.mutex); + while (entry != NULL) { + state_header *next = entry->next; + PyMem_RawFree(entry->mem); + entry = next; + } + } } #ifdef HAVE_FORK @@ -446,24 +475,50 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) return _PyStatus_OK(); } -static PyInterpreterState * -alloc_interpreter(void) +static void * +alloc_state(state_header **freelist, size_t size, size_t alignment) { - // Aligned allocation for PyInterpreterState. - // the first word of the memory block is used to store - // the original pointer to be used later to free the memory. - size_t alignment = _Alignof(PyInterpreterState); - size_t allocsize = sizeof(PyInterpreterState) + sizeof(void *) + alignment - 1; + PyMutex_LockFlags(&state_freelists.mutex, _Py_LOCK_DONT_DETACH); + state_header *entry = *freelist; + if (entry != NULL) { + *freelist = entry->next; + } + PyMutex_Unlock(&state_freelists.mutex); + if (entry != NULL) { + return entry + 1; + } + + size_t allocsize = size + sizeof(state_header) + alignment - 1; void *mem = PyMem_RawCalloc(1, allocsize); if (mem == NULL) { return NULL; } - void *ptr = _Py_ALIGN_UP((char *)mem + sizeof(void *), alignment); - ((void **)ptr)[-1] = mem; + void *ptr = _Py_ALIGN_UP((char *)mem + sizeof(state_header), alignment); + ((state_header *)ptr - 1)->mem = mem; assert(_Py_IS_ALIGNED(ptr, alignment)); return ptr; } +static void +recycle_state(state_header **freelist, void *state, size_t size) +{ + memset(state, 0, size); + state_header *entry = (state_header *)state - 1; + + PyMutex_LockFlags(&state_freelists.mutex, _Py_LOCK_DONT_DETACH); + entry->next = *freelist; + // Keep the list consistent for a child forked by another thread. + _Py_atomic_store_ptr_release(freelist, entry); + PyMutex_Unlock(&state_freelists.mutex); +} + +static PyInterpreterState * +alloc_interpreter(void) +{ + return alloc_state(&state_freelists.interpreters, sizeof(PyInterpreterState), + _Alignof(PyInterpreterState)); +} + static void free_interpreter(PyInterpreterState *interp) { @@ -492,7 +547,13 @@ free_interpreter(PyInterpreterState *interp) interp->obmalloc = NULL; } assert(_Py_IS_ALIGNED(interp, _Alignof(PyInterpreterState))); - PyMem_RawFree(((void **)interp)[-1]); + // A daemon thread may still own the embedded initial thread state. + if (interp->_initialized + && _Py_atomic_load_ptr(&interp->threads.preallocated) == NULL) + { + return; + } + recycle_state(&state_freelists.interpreters, interp, sizeof(*interp)); } } @@ -1488,7 +1549,8 @@ alloc_threadstate(PyInterpreterState *interp) // Fall back to the allocator. if (tstate == NULL) { - tstate = PyMem_RawCalloc(1, sizeof(_PyThreadStateImpl)); + tstate = alloc_state(&state_freelists.tstates, sizeof(*tstate), + _Alignof(_PyThreadStateImpl)); if (tstate == NULL) { return NULL; } @@ -1515,7 +1577,7 @@ free_threadstate(_PyThreadStateImpl *tstate) _Py_atomic_store_ptr(&interp->threads.preallocated, tstate); } else { - PyMem_RawFree(tstate); + recycle_state(&state_freelists.tstates, tstate, sizeof(*tstate)); } } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 6024eb7e093eacf..79edfc53c21a3bc 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -56,6 +56,7 @@ Python/pyhash.c - _Py_HashSecret - ## thread-safe hashtable (internal locks) Python/parking_lot.c - buckets - +Python/pystate.c - state_freelists - ## data needed for introspecting asyncio state from debuggers and profilers Modules/_asynciomodule.c - _Py_AsyncioDebug - _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
