https://github.com/python/cpython/commit/113de8545ffe74a4a1dddb9351fa1cbd3562b621
commit: 113de8545ffe74a4a1dddb9351fa1cbd3562b621
branch: main
author: Neil Schemenauer <nas-git...@arctrix.com>
committer: nascheme <nas-git...@arctrix.com>
date: 2025-06-25T00:06:32-07:00
summary:
GH-133136: Revise QSBR to reduce excess memory held (gh-135473)
The free threading build uses QSBR to delay the freeing of dictionary
keys and list arrays when the objects are accessed by multiple threads
in order to allow concurrent reads to proceed with holding the object
lock. The requests are processed in batches to reduce execution
overhead, but for large memory blocks this can lead to excess memory
usage.
Take into account the size of the memory block when deciding when to
process QSBR requests.
Also track the amount of memory being held by QSBR for mimalloc pages. Advance
the write sequence if this memory exceeds a limit. Advancing the sequence will
allow it to be freed more quickly.
Process the held QSBR items from the "eval breaker", rather than from
`_PyMem_FreeDelayed()`. This gives a higher chance that the global read
sequence has advanced enough so that items can be freed.
Co-authored-by: Sam Gross <colesb...@gmail.com>
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst
M Include/internal/pycore_pymem.h
M Include/internal/pycore_qsbr.h
M Objects/codeobject.c
M Objects/dictobject.c
M Objects/listobject.c
M Objects/obmalloc.c
M Python/ceval_gil.c
M Python/qsbr.c
diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h
index e669a30b072d18..f3f2ae0a140828 100644
--- a/Include/internal/pycore_pymem.h
+++ b/Include/internal/pycore_pymem.h
@@ -88,7 +88,7 @@ extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str);
extern int _PyMem_DebugEnabled(void);
// Enqueue a pointer to be freed possibly after some delay.
-extern void _PyMem_FreeDelayed(void *ptr);
+extern void _PyMem_FreeDelayed(void *ptr, size_t size);
// Periodically process delayed free requests.
extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h
index b835c3abaf5d0b..1f9b3fcf777493 100644
--- a/Include/internal/pycore_qsbr.h
+++ b/Include/internal/pycore_qsbr.h
@@ -48,8 +48,21 @@ struct _qsbr_thread_state {
// Thread state (or NULL)
PyThreadState *tstate;
- // Used to defer advancing write sequence a fixed number of times
- int deferrals;
+ // Number of held items added by this thread since the last write sequence
+ // advance
+ int deferred_count;
+
+ // Estimate for the amount of memory that is held by this thread since
+ // the last write sequence advance
+ size_t deferred_memory;
+
+ // Amount of memory in mimalloc pages deferred from collection. When
+ // deferred, they are prevented from being used for a different size class
+ // and in a different thread.
+ size_t deferred_page_memory;
+
+ // True if the deferred memory frees should be processed.
+ bool should_process;
// Is this thread state allocated?
bool allocated;
@@ -109,11 +122,17 @@ _Py_qbsr_goal_reached(struct _qsbr_thread_state *qsbr,
uint64_t goal)
extern uint64_t
_Py_qsbr_advance(struct _qsbr_shared *shared);
-// Batches requests to advance the write sequence. This advances the write
-// sequence every N calls, which reduces overhead but increases time to
-// reclamation. Returns the new goal.
+// Return the next value for the write sequence (current plus the increment).
extern uint64_t
-_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr);
+_Py_qsbr_shared_next(struct _qsbr_shared *shared);
+
+// Return true if deferred memory frees held by QSBR should be processed to
+// determine if they can be safely freed.
+static inline bool
+_Py_qsbr_should_process(struct _qsbr_thread_state *qsbr)
+{
+ return qsbr->should_process;
+}
// Have the read sequences advanced to the given goal? If this returns true,
// it safe to reclaim any memory tagged with the goal (or earlier goal).
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst
new file mode 100644
index 00000000000000..a9501c13c95b3a
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst
@@ -0,0 +1,2 @@
+Limit excess memory usage in the :term:`free threading` build when a
+large dictionary or list is resized and accessed by multiple threads.
diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index 3f53d4cfeb20ac..91772bc9d19098 100644
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -3369,7 +3369,7 @@ create_tlbc_lock_held(PyCodeObject *co, Py_ssize_t idx)
}
memcpy(new_tlbc->entries, tlbc->entries, tlbc->size * sizeof(void *));
_Py_atomic_store_ptr_release(&co->co_tlbc, new_tlbc);
- _PyMem_FreeDelayed(tlbc);
+ _PyMem_FreeDelayed(tlbc, tlbc->size * sizeof(void *));
tlbc = new_tlbc;
}
char *bc = PyMem_Calloc(1, _PyCode_NBYTES(co));
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 72901377cbb3da..6b7b150f0e2f2c 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -813,7 +813,7 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
{
#ifdef Py_GIL_DISABLED
if (use_qsbr) {
- _PyMem_FreeDelayed(keys);
+ _PyMem_FreeDelayed(keys, _PyDict_KeysSize(keys));
return;
}
#endif
@@ -858,7 +858,7 @@ free_values(PyDictValues *values, bool use_qsbr)
assert(values->embedded == 0);
#ifdef Py_GIL_DISABLED
if (use_qsbr) {
- _PyMem_FreeDelayed(values);
+ _PyMem_FreeDelayed(values, values_size_from_count(values->capacity));
return;
}
#endif
diff --git a/Objects/listobject.c b/Objects/listobject.c
index c5895645a2dd12..23d3472b6d4153 100644
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -61,7 +61,8 @@ free_list_items(PyObject** items, bool use_qsbr)
#ifdef Py_GIL_DISABLED
_PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item);
if (use_qsbr) {
- _PyMem_FreeDelayed(array);
+ size_t size = sizeof(_PyListArray) + array->allocated *
sizeof(PyObject *);
+ _PyMem_FreeDelayed(array, size);
}
else {
PyMem_Free(array);
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index d4b8327cb73b37..deb7fd957e57dd 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -124,6 +124,33 @@ _PyMem_mi_page_is_safe_to_free(mi_page_t *page)
}
+#ifdef Py_GIL_DISABLED
+
+// If we are deferring collection of more than this amount of memory for
+// mimalloc pages, advance the write sequence. Advancing allows these
+// pages to be re-used in a different thread or for a different size class.
+#define QSBR_PAGE_MEM_LIMIT 4096*20
+
+// Return true if the global write sequence should be advanced for a mimalloc
+// page that is deferred from collection.
+static bool
+should_advance_qsbr_for_page(struct _qsbr_thread_state *qsbr, mi_page_t *page)
+{
+ size_t bsize = mi_page_block_size(page);
+ size_t page_size = page->capacity*bsize;
+ if (page_size > QSBR_PAGE_MEM_LIMIT) {
+ qsbr->deferred_page_memory = 0;
+ return true;
+ }
+ qsbr->deferred_page_memory += page_size;
+ if (qsbr->deferred_page_memory > QSBR_PAGE_MEM_LIMIT) {
+ qsbr->deferred_page_memory = 0;
+ return true;
+ }
+ return false;
+}
+#endif
+
static bool
_PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force)
{
@@ -139,7 +166,14 @@ _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t
*pq, bool force)
_PyMem_mi_page_clear_qsbr(page);
page->retire_expire = 0;
- page->qsbr_goal = _Py_qsbr_deferred_advance(tstate->qsbr);
+
+ if (should_advance_qsbr_for_page(tstate->qsbr, page)) {
+ page->qsbr_goal = _Py_qsbr_advance(tstate->qsbr->shared);
+ }
+ else {
+ page->qsbr_goal = _Py_qsbr_shared_next(tstate->qsbr->shared);
+ }
+
llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node);
return false;
}
@@ -1141,8 +1175,44 @@ free_work_item(uintptr_t ptr, delayed_dealloc_cb cb,
void *state)
}
}
+
+#ifdef Py_GIL_DISABLED
+
+// For deferred advance on free: the number of deferred items before advancing
+// the write sequence. This is based on WORK_ITEMS_PER_CHUNK. We ideally
+// want to process a chunk before it overflows.
+#define QSBR_DEFERRED_LIMIT 127
+
+// If the deferred memory exceeds 1 MiB, advance the write sequence. This
+// helps limit memory usage due to QSBR delaying frees too long.
+#define QSBR_FREE_MEM_LIMIT 1024*1024
+
+// Return true if the global write sequence should be advanced for a deferred
+// memory free.
+static bool
+should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size)
+{
+ if (size > QSBR_FREE_MEM_LIMIT) {
+ qsbr->deferred_count = 0;
+ qsbr->deferred_memory = 0;
+ qsbr->should_process = true;
+ return true;
+ }
+ qsbr->deferred_count++;
+ qsbr->deferred_memory += size;
+ if (qsbr->deferred_count > QSBR_DEFERRED_LIMIT ||
+ qsbr->deferred_memory > QSBR_FREE_MEM_LIMIT) {
+ qsbr->deferred_count = 0;
+ qsbr->deferred_memory = 0;
+ qsbr->should_process = true;
+ return true;
+ }
+ return false;
+}
+#endif
+
static void
-free_delayed(uintptr_t ptr)
+free_delayed(uintptr_t ptr, size_t size)
{
#ifndef Py_GIL_DISABLED
free_work_item(ptr, NULL, NULL);
@@ -1200,23 +1270,32 @@ free_delayed(uintptr_t ptr)
}
assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK);
- uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr);
+ uint64_t seq;
+ if (should_advance_qsbr_for_free(tstate->qsbr, size)) {
+ seq = _Py_qsbr_advance(tstate->qsbr->shared);
+ }
+ else {
+ seq = _Py_qsbr_shared_next(tstate->qsbr->shared);
+ }
buf->array[buf->wr_idx].ptr = ptr;
buf->array[buf->wr_idx].qsbr_goal = seq;
buf->wr_idx++;
if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) {
+ // Normally the processing of delayed items is done from the eval
+ // breaker. Processing here is a safety measure to ensure too much
+ // work does not accumulate.
_PyMem_ProcessDelayed((PyThreadState *)tstate);
}
#endif
}
void
-_PyMem_FreeDelayed(void *ptr)
+_PyMem_FreeDelayed(void *ptr, size_t size)
{
assert(!((uintptr_t)ptr & 0x01));
if (ptr != NULL) {
- free_delayed((uintptr_t)ptr);
+ free_delayed((uintptr_t)ptr, size);
}
}
@@ -1226,7 +1305,10 @@ _PyObject_XDecRefDelayed(PyObject *ptr)
{
assert(!((uintptr_t)ptr & 0x01));
if (ptr != NULL) {
- free_delayed(((uintptr_t)ptr)|0x01);
+ // We use 0 as the size since we don't have an easy way to know the
+ // actual size. If we are freeing many objects, the write sequence
+ // will be advanced due to QSBR_DEFERRED_LIMIT.
+ free_delayed(((uintptr_t)ptr)|0x01, 0);
}
}
#endif
@@ -1317,6 +1399,8 @@ _PyMem_ProcessDelayed(PyThreadState *tstate)
PyInterpreterState *interp = tstate->interp;
_PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
+ tstate_impl->qsbr->should_process = false;
+
// Process thread-local work
process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, NULL, NULL);
diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c
index 57d8f68b000b60..aa68371ac8febf 100644
--- a/Python/ceval_gil.c
+++ b/Python/ceval_gil.c
@@ -1387,6 +1387,10 @@ _Py_HandlePending(PyThreadState *tstate)
_Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT);
_Py_brc_merge_refcounts(tstate);
}
+ /* Process deferred memory frees held by QSBR */
+ if (_Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) {
+ _PyMem_ProcessDelayed(tstate);
+ }
#endif
/* GC scheduled to run */
diff --git a/Python/qsbr.c b/Python/qsbr.c
index afa03776c26b6f..c992c285cb13e4 100644
--- a/Python/qsbr.c
+++ b/Python/qsbr.c
@@ -41,10 +41,6 @@
// Starting size of the array of qsbr thread states
#define MIN_ARRAY_SIZE 8
-// For _Py_qsbr_deferred_advance(): the number of deferrals before advancing
-// the write sequence.
-#define QSBR_DEFERRED_LIMIT 10
-
// Allocate a QSBR thread state from the freelist
static struct _qsbr_thread_state *
qsbr_allocate(struct _qsbr_shared *shared)
@@ -117,13 +113,9 @@ _Py_qsbr_advance(struct _qsbr_shared *shared)
}
uint64_t
-_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr)
+_Py_qsbr_shared_next(struct _qsbr_shared *shared)
{
- if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) {
- return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR;
- }
- qsbr->deferrals = 0;
- return _Py_qsbr_advance(qsbr->shared);
+ return _Py_qsbr_shared_current(shared) + QSBR_INCR;
}
static uint64_t
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-le...@python.org
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: arch...@mail-archive.com
