https://github.com/python/cpython/commit/7db871e4fa48eef20ea22414b226998f83facfd6
commit: 7db871e4fa48eef20ea22414b226998f83facfd6
branch: main
author: Dino Viehland <[email protected]>
committer: DinoV <[email protected]>
date: 2024-03-08T09:56:36-08:00
summary:
gh-112075: Support freeing object memory via QSBR (#116344)
Free objects with qsbr if shared
files:
M Include/internal/pycore_gc.h
M Include/internal/pycore_pymem.h
M Objects/obmalloc.c
M Python/gc_free_threading.c
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
index cf0b148c2b6463..4a7191a562cc10 100644
--- a/Include/internal/pycore_gc.h
+++ b/Include/internal/pycore_gc.h
@@ -44,6 +44,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) {
# define _PyGC_BITS_UNREACHABLE (4)
# define _PyGC_BITS_FROZEN (8)
# define _PyGC_BITS_SHARED (16)
+# define _PyGC_BITS_SHARED_INLINE (32)
#endif
/* True if the object is currently tracked by the GC. */
@@ -71,9 +72,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj)
{
#ifdef Py_GIL_DISABLED
-/* True if an object is shared between multiple threads and
- * needs special purpose when freeing to do the possibility
- * of in-flight lock-free reads occurring */
+/* True if memory the object references is shared between
+ * multiple threads and needs special purpose when freeing
+ * those references due to the possibility of in-flight
+ * lock-free reads occurring. The object is responsible
+ * for calling _PyMem_FreeDelayed on the referenced
+ * memory. */
static inline int _PyObject_GC_IS_SHARED(PyObject *op) {
return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0;
}
@@ -84,6 +88,23 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
}
#define _PyObject_GC_SET_SHARED(op)
_PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op))
+/* True if the memory of the object is shared between multiple
+ * threads and needs special purpose when freeing due to
+ * the possibility of in-flight lock-free reads occurring.
+ * Objects with this bit that are GC objects will automatically
+ * delay-freed by PyObject_GC_Del. */
+static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) {
+ return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0;
+}
+#define _PyObject_GC_IS_SHARED_INLINE(op) \
+ _PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op))
+
+static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) {
+ op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE;
+}
+#define _PyObject_GC_SET_SHARED_INLINE(op) \
+ _PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op))
+
#endif
/* Bit flags for _gc_prev */
diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h
index 1aea91abc5d69f..dd6b0762370c92 100644
--- a/Include/internal/pycore_pymem.h
+++ b/Include/internal/pycore_pymem.h
@@ -119,6 +119,9 @@ extern int _PyMem_DebugEnabled(void);
// Enqueue a pointer to be freed possibly after some delay.
extern void _PyMem_FreeDelayed(void *ptr);
+// Enqueue an object to be freed possibly after some delay
+extern void _PyObject_FreeDelayed(void *ptr);
+
// Periodically process delayed free requests.
extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index e7813807674abe..4fe195b63166c1 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1070,7 +1070,7 @@ _PyMem_Strdup(const char *str)
// A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
struct _mem_work_item {
- void *ptr;
+ uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free
uint64_t qsbr_goal;
};
@@ -1084,16 +1084,27 @@ struct _mem_work_chunk {
struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
};
-void
-_PyMem_FreeDelayed(void *ptr)
+static void
+free_work_item(uintptr_t ptr)
+{
+ if (ptr & 0x01) {
+ PyObject_Free((char *)(ptr - 1));
+ }
+ else {
+ PyMem_Free((void *)ptr);
+ }
+}
+
+static void
+free_delayed(uintptr_t ptr)
{
#ifndef Py_GIL_DISABLED
- PyMem_Free(ptr);
+ free_work_item(ptr);
#else
if (_PyRuntime.stoptheworld.world_stopped) {
// Free immediately if the world is stopped, including during
// interpreter shutdown.
- PyMem_Free(ptr);
+ free_work_item(ptr);
return;
}
@@ -1120,7 +1131,7 @@ _PyMem_FreeDelayed(void *ptr)
if (buf == NULL) {
// failed to allocate a buffer, free immediately
_PyEval_StopTheWorld(tstate->base.interp);
- PyMem_Free(ptr);
+ free_work_item(ptr);
_PyEval_StartTheWorld(tstate->base.interp);
return;
}
@@ -1137,6 +1148,20 @@ _PyMem_FreeDelayed(void *ptr)
#endif
}
+void
+_PyMem_FreeDelayed(void *ptr)
+{
+ assert(!((uintptr_t)ptr & 0x01));
+ free_delayed((uintptr_t)ptr);
+}
+
+void
+_PyObject_FreeDelayed(void *ptr)
+{
+ assert(!((uintptr_t)ptr & 0x01));
+ free_delayed(((uintptr_t)ptr)|0x01);
+}
+
static struct _mem_work_chunk *
work_queue_first(struct llist_node *head)
{
@@ -1156,7 +1181,7 @@ process_queue(struct llist_node *head, struct
_qsbr_thread_state *qsbr,
return;
}
- PyMem_Free(item->ptr);
+ free_work_item(item->ptr);
buf->rd_idx++;
}
@@ -1243,7 +1268,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
// Free the remaining items immediately. There should be no other
// threads accessing the memory at this point during shutdown.
struct _mem_work_item *item = &buf->array[buf->rd_idx];
- PyMem_Free(item->ptr);
+ free_work_item(item->ptr);
buf->rd_idx++;
}
diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
index c7883cd38644e1..59e76012f8fc50 100644
--- a/Python/gc_free_threading.c
+++ b/Python/gc_free_threading.c
@@ -1695,6 +1695,7 @@ PyObject_GC_Del(void *op)
{
size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type);
if (_PyObject_GC_IS_TRACKED(op)) {
+ _PyObject_GC_UNTRACK(op);
#ifdef Py_DEBUG
PyObject *exc = PyErr_GetRaisedException();
if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0,
@@ -1707,8 +1708,13 @@ PyObject_GC_Del(void *op)
}
record_deallocation(_PyThreadState_GET());
-
- PyObject_Free(((char *)op)-presize);
+ PyObject *self = (PyObject *)op;
+ if (_PyObject_GC_IS_SHARED_INLINE(self)) {
+ _PyObject_FreeDelayed(((char *)op)-presize);
+ }
+ else {
+ PyObject_Free(((char *)op)-presize);
+ }
}
int
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
