https://github.com/python/cpython/commit/ff6cbb2503a8fe3fceeadd889e34fc9a8f308ecd
commit: ff6cbb2503a8fe3fceeadd889e34fc9a8f308ecd
branch: main
author: Dino Viehland <dinoviehl...@meta.com>
committer: DinoV <dinoviehl...@gmail.com>
date: 2024-05-07T00:22:26Z
summary:
gh-112075: use per-thread dict version pool (#118676)
use thread state set of dict versions
files:
M Include/cpython/pystate.h
M Include/internal/pycore_dict.h
M Lib/test/test_free_threading/test_dict.py
M Modules/_testcapi/dict.c
M Python/pystate.c
diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index 0611e299403031..2df9ecd6d52084 100644
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -188,6 +188,7 @@ struct _ts {
PyObject *previous_executor;
+ uint64_t dict_global_version;
};
#ifdef Py_DEBUG
diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h
index cb7d4c3219a9af..8d8d3748edaea8 100644
--- a/Include/internal/pycore_dict.h
+++ b/Include/internal/pycore_dict.h
@@ -221,8 +221,25 @@ static inline PyDictUnicodeEntry*
DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS +
DICT_WATCHED_MUTATION_BITS)) - 1)
#ifdef Py_GIL_DISABLED
-#define DICT_NEXT_VERSION(INTERP) \
- (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version,
DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
+
+#define THREAD_LOCAL_DICT_VERSION_COUNT 256
+#define THREAD_LOCAL_DICT_VERSION_BATCH THREAD_LOCAL_DICT_VERSION_COUNT *
DICT_VERSION_INCREMENT
+
+static inline uint64_t
+dict_next_version(PyInterpreterState *interp)
+{
+ PyThreadState *tstate = PyThreadState_GET();
+ uint64_t cur_progress = (tstate->dict_global_version &
+ (THREAD_LOCAL_DICT_VERSION_BATCH - 1));
+ if (cur_progress == 0) {
+ uint64_t next =
_Py_atomic_add_uint64(&interp->dict_state.global_version,
+ THREAD_LOCAL_DICT_VERSION_BATCH);
+ tstate->dict_global_version = next;
+ }
+ return tstate->dict_global_version += DICT_VERSION_INCREMENT;
+}
+
+#define DICT_NEXT_VERSION(INTERP) dict_next_version(INTERP)
#else
#define DICT_NEXT_VERSION(INTERP) \
diff --git a/Lib/test/test_free_threading/test_dict.py
b/Lib/test/test_free_threading/test_dict.py
index 6a909dd3ee025f..f877582e6b565c 100644
--- a/Lib/test/test_free_threading/test_dict.py
+++ b/Lib/test/test_free_threading/test_dict.py
@@ -8,6 +8,8 @@
from threading import Thread
from unittest import TestCase
+from _testcapi import dict_version
+
from test.support import threading_helper
@@ -137,5 +139,39 @@ def writer_func(l):
for ref in thread_list:
self.assertIsNone(ref())
+ def test_dict_version(self):
+ THREAD_COUNT = 10
+ DICT_COUNT = 10000
+ lists = []
+ writers = []
+
+ def writer_func(thread_list):
+ for i in range(DICT_COUNT):
+ thread_list.append(dict_version({}))
+
+ for x in range(THREAD_COUNT):
+ thread_list = []
+ lists.append(thread_list)
+ writer = Thread(target=partial(writer_func, thread_list))
+ writers.append(writer)
+
+ for writer in writers:
+ writer.start()
+
+ for writer in writers:
+ writer.join()
+
+ total_len = 0
+ values = set()
+ for thread_list in lists:
+ for v in thread_list:
+ if v in values:
+ print('dup', v, (v/4096)%256)
+ values.add(v)
+ total_len += len(thread_list)
+ versions = set(dict_version for thread_list in lists for dict_version
in thread_list)
+ self.assertEqual(len(versions), THREAD_COUNT*DICT_COUNT)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c
index 4319906dc4fee0..e80d898118daa5 100644
--- a/Modules/_testcapi/dict.c
+++ b/Modules/_testcapi/dict.c
@@ -1,7 +1,6 @@
#include "parts.h"
#include "util.h"
-
static PyObject *
dict_containsstring(PyObject *self, PyObject *args)
{
@@ -182,6 +181,18 @@ dict_popstring_null(PyObject *self, PyObject *args)
RETURN_INT(PyDict_PopString(dict, key, NULL));
}
+static PyObject *
+dict_version(PyObject *self, PyObject *dict)
+{
+ if (!PyDict_Check(dict)) {
+ PyErr_SetString(PyExc_TypeError, "expected dict");
+ return NULL;
+ }
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
+ return PyLong_FromUnsignedLongLong(((PyDictObject *)dict)->ma_version_tag);
+_Py_COMP_DIAG_POP
+}
static PyMethodDef test_methods[] = {
{"dict_containsstring", dict_containsstring, METH_VARARGS},
@@ -193,6 +204,7 @@ static PyMethodDef test_methods[] = {
{"dict_pop_null", dict_pop_null, METH_VARARGS},
{"dict_popstring", dict_popstring, METH_VARARGS},
{"dict_popstring_null", dict_popstring_null, METH_VARARGS},
+ {"dict_version", dict_version, METH_O},
{NULL},
};
diff --git a/Python/pystate.c b/Python/pystate.c
index 7c75263b7e526a..2f1521edd8bf0a 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -1488,6 +1488,7 @@ init_threadstate(_PyThreadStateImpl *_tstate,
tstate->datastack_limit = NULL;
tstate->what_event = -1;
tstate->previous_executor = NULL;
+ tstate->dict_global_version = 0;
tstate->delete_later = NULL;
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-le...@python.org
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: arch...@mail-archive.com
