https://github.com/python/cpython/commit/4497fe9889105b1abc6d633971e84ee9f78bd3b8
commit: 4497fe9889105b1abc6d633971e84ee9f78bd3b8
branch: main
author: Neil Schemenauer <[email protected]>
committer: kumaraditya303 <[email protected]>
date: 2026-11-17T11:13:02+05:30
summary:
gh-161792: Fix crashes in the callable iterator on free-threading (#161793)
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst
M Lib/test/test_free_threading/test_iteration.py
M Lib/test/test_iter.py
M Objects/iterobject.c
diff --git a/Lib/test/test_free_threading/test_iteration.py
b/Lib/test/test_free_threading/test_iteration.py
index 540e8edcc676285..bf42304b41e8b96 100644
--- a/Lib/test/test_free_threading/test_iteration.py
+++ b/Lib/test/test_free_threading/test_iteration.py
@@ -1,3 +1,4 @@
+import itertools
import sys
import threading
import unittest
@@ -170,6 +171,31 @@ def drain(it, barrier, results):
self.assertEqual(sys.getrefcount(seq), refcount_before)
+class ContendedCallIterExhaustionTest(unittest.TestCase):
+ """Test exhausting a shared iter(callable, sentinel) iterator.
+
+ The values seen by each thread are unspecified, but exhausting the
+ iterator from several threads at once must not crash.
+ """
+
+ def test_shared_iterator_exhaustion(self):
+ niters = 2000 if support.check_sanitizer(thread=True) else 20000
+
+ def make_iter():
+ counter = itertools.count().__next__
+ return iter(lambda: counter(), 3)
+
+ shared = [make_iter()]
+
+ def worker():
+ for _ in range(niters):
+ it = shared[0]
+ if next(it, None) is None:
+ shared[0] = make_iter()
+
+ threading_helper.run_concurrently(worker, nthreads=NUMTHREADS)
+
+
class ContendedRangeIterationTest(ContendedTupleIterationTest):
def make_testdata(self, n):
return range(n)
diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py
index fe8617309da29d0..6ebde4848a5a491 100644
--- a/Lib/test/test_iter.py
+++ b/Lib/test/test_iter.py
@@ -532,6 +532,23 @@ def spam():
with self.assertRaises(StopIteration):
next(spam.iterator)
+ def test_iter_function_sentinel_replaced_during_compare(self):
+ # gh-158031: the sentinel's __eq__ replaces the sentinel, which must
+ # stay alive until the comparison is finished.
+ class Sentinel:
+ def __eq__(self, other):
+ it.__setstate__(((), StopIteration))
+ return NotImplemented
+
+ class Result:
+ def __eq__(self, other):
+ self.other_type = type(other)
+ return NotImplemented
+
+ it = iter(Result, Sentinel())
+ result = next(it)
+ self.assertIs(result.other_type, Sentinel)
+
# Test exception propagation through function iterator
def test_exception_function(self):
def spam(state=[0]):
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst
new file mode 100644
index 000000000000000..a4b685d96bc8967
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst
@@ -0,0 +1,4 @@
+Fix crashes in the callable iterator returned by :func:`iter` with a
+sentinel, when :meth:`!__setstate__` is called while the iterator is in use,
+or when the iterator is shared between threads in the :term:`free-threaded
+build`.
diff --git a/Objects/iterobject.c b/Objects/iterobject.c
index fcef1cc9c301434..683683ceaa6600c 100644
--- a/Objects/iterobject.c
+++ b/Objects/iterobject.c
@@ -238,11 +238,33 @@ PyCallIter_New(PyObject *callable, PyObject *sentinel)
return _PyCallIter_NewEx(callable, sentinel, NULL);
}
+// Sharing a callable iterator between threads, or calling __setstate__()
+// on an iterator that is in use, is not supported: the values produced are
+// unspecified, but the interpreter must not crash.
+
+static inline PyObject *
+calliter_swap(PyObject **field, PyObject *value)
+{
+#ifdef Py_GIL_DISABLED
+ return _Py_atomic_exchange_ptr(field, value);
+#else
+ PyObject *old = *field;
+ *field = value;
+ return old;
+#endif
+}
+
+static inline PyObject *
+calliter_load(PyObject **field)
+{
+ return Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(*field));
+}
+
static void
calliter_exhaust(calliterobject *it)
{
- Py_CLEAR(it->it_callable);
- Py_CLEAR(it->it_sentinel);
+ _PyObject_XDecRefDelayed(calliter_swap(&it->it_callable, NULL));
+ _PyObject_XDecRefDelayed(calliter_swap(&it->it_sentinel, NULL));
}
static void
@@ -270,19 +292,26 @@ static PyObject *
calliter_iternext(PyObject *op)
{
calliterobject *it = (calliterobject*)op;
- PyObject *result;
+ PyObject *callable, *sentinel, *stop_exc, *result;
- if (it->it_callable == NULL) {
+ callable = calliter_load(&it->it_callable);
+ if (callable == NULL) {
return NULL;
}
+ result = _PyObject_CallNoArgs(callable);
+ Py_DECREF(callable);
- result = _PyObject_CallNoArgs(it->it_callable);
/* The call can exhaust the iterator re-entrantly. */
- if (result != NULL && it->it_callable != NULL) {
- if (it->it_sentinel == NULL) {
+ if (result != NULL &&
+ FT_ATOMIC_LOAD_PTR_RELAXED(it->it_callable) != NULL)
+ {
+ sentinel = calliter_load(&it->it_sentinel);
+ if (sentinel == NULL) {
return result; /* Common case, fast path */
}
- int ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ);
+ /* The comparison can run code that replaces the sentinel. */
+ int ok = PyObject_RichCompareBool(sentinel, result, Py_EQ);
+ Py_DECREF(sentinel);
if (ok == 0) {
return result; /* Common case, fast path */
}
@@ -291,14 +320,19 @@ calliter_iternext(PyObject *op)
calliter_exhaust(it);
}
}
- else if (PyErr_ExceptionMatches(it->it_stop_exc)) {
- PyErr_Clear();
- calliter_exhaust(it);
- }
- else if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
- /* It would be mistaken for the end of the iteration (see PEP 479). */
- _PyErr_FormatFromCause(PyExc_RuntimeError,
- "callable raised StopIteration");
+ else if (result == NULL) {
+ stop_exc = calliter_load(&it->it_stop_exc);
+ if (PyErr_ExceptionMatches(stop_exc)) {
+ PyErr_Clear();
+ calliter_exhaust(it);
+ }
+ else if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
+ /* It would be mistaken for the end of the iteration (see
+ PEP 479). */
+ _PyErr_FormatFromCause(PyExc_RuntimeError,
+ "callable raised StopIteration");
+ }
+ Py_DECREF(stop_exc);
}
Py_XDECREF(result);
return NULL;
@@ -314,22 +348,29 @@ calliter_reduce(PyObject *op, PyObject
*Py_UNUSED(ignored))
* call must be before access of iterator pointers.
* see issue #101765 */
- if (it->it_callable == NULL) {
- return Py_BuildValue("N(())", iter);
+ PyObject *callable = calliter_load(&it->it_callable);
+ PyObject *sentinel = calliter_load(&it->it_sentinel);
+ PyObject *stop_exc = calliter_load(&it->it_stop_exc);
+ PyObject *res;
+ if (callable == NULL) {
+ res = Py_BuildValue("N(())", iter);
}
/* Only the sentinel can be passed as an argument of iter(), so other
attributes are restored from the state (see calliter_setstate()). */
- if (it->it_sentinel == NULL) {
- return Py_BuildValue("N(OO)(()O)", iter, it->it_callable, Py_None,
- it->it_stop_exc);
+ else if (sentinel == NULL) {
+ res = Py_BuildValue("N(OO)(()O)", iter, callable, Py_None, stop_exc);
}
- else if (it->it_stop_exc == PyExc_StopIteration) {
- return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel);
+ else if (stop_exc == PyExc_StopIteration) {
+ res = Py_BuildValue("N(OO)", iter, callable, sentinel);
}
else {
- return Py_BuildValue("N(OO)((O)O)", iter, it->it_callable, Py_None,
- it->it_sentinel, it->it_stop_exc);
+ res = Py_BuildValue("N(OO)((O)O)", iter, callable, Py_None,
+ sentinel, stop_exc);
}
+ Py_XDECREF(callable);
+ Py_XDECREF(sentinel);
+ Py_DECREF(stop_exc);
+ return res;
}
static PyObject *
@@ -349,11 +390,12 @@ calliter_setstate(PyObject *op, PyObject *state)
if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) {
return NULL;
}
- if (it->it_callable != NULL) {
- Py_XSETREF(it->it_sentinel,
- PyTuple_GET_SIZE(sentinel) ?
- Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL);
- Py_SETREF(it->it_stop_exc, Py_NewRef(stop_exc));
+ if (FT_ATOMIC_LOAD_PTR_RELAXED(it->it_callable) != NULL) {
+ PyObject *value = PyTuple_GET_SIZE(sentinel) ?
+ Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL;
+ _PyObject_XDecRefDelayed(calliter_swap(&it->it_sentinel, value));
+ _PyObject_XDecRefDelayed(calliter_swap(&it->it_stop_exc,
+ Py_NewRef(stop_exc)));
}
Py_RETURN_NONE;
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]