https://github.com/python/cpython/commit/7c5231e1de95b94226be502ba773502ac0f9b9ba commit: 7c5231e1de95b94226be502ba773502ac0f9b9ba branch: main author: Neil Schemenauer <[email protected]> committer: kumaraditya303 <[email protected]> date: 2026-11-05T10:50:54+05:30 summary:
gh-127716: make memoryview thread-safe in free-threaded build (#156248) Co-authored-by: Lu Xiaowei <[email protected]> Co-authored-by: ayaangazali <[email protected]> Co-authored-by: Nathan Goldbaum <[email protected]> Co-authored-by: Kumar Aditya <[email protected]> files: A Lib/test/test_free_threading/test_memoryview.py A Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-14-37-50.gh-issue-127716.KrPth8.rst M Doc/builtins/threadsafety.rst M Include/cpython/memoryobject.h M Lib/test/test_memoryview.py M Modules/_testcapi/buffer.c M Objects/memoryobject.c diff --git a/Doc/builtins/threadsafety.rst b/Doc/builtins/threadsafety.rst index 32600a35ef6eea..e2e605a49db459 100644 --- a/Doc/builtins/threadsafety.rst +++ b/Doc/builtins/threadsafety.rst @@ -604,3 +604,13 @@ underlying object supports it: Resizing or reallocating the underlying object (such as calling :meth:`bytearray.resize`) while a memoryview is exported raises :exc:`BufferError`. This is enforced regardless of threading. + +In the :term:`free-threaded build`, :meth:`~memoryview.release` is +thread-safe. If another thread has an in-flight operation on the memoryview +when it is released, the underlying buffer may remain exported until that +operation completes. Restrictions on the exporter, such as prohibiting +resizing, remain in effect until then. + +For read-heavy workloads that share an underlying object across threads, +creating a separate memoryview of that object in each thread may improve +performance. diff --git a/Include/cpython/memoryobject.h b/Include/cpython/memoryobject.h index 961161b70f2058..419e57d1ad56a4 100644 --- a/Include/cpython/memoryobject.h +++ b/Include/cpython/memoryobject.h @@ -11,7 +11,7 @@ typedef struct { PyObject_HEAD int flags; /* state flags */ - Py_ssize_t exports; /* number of direct memoryview exports */ + Py_ssize_t exports; /* registered views and active operations */ Py_buffer master; /* snapshot buffer obtained from the original exporter */ } _PyManagedBufferObject; diff --git a/Lib/test/test_free_threading/test_memoryview.py b/Lib/test/test_free_threading/test_memoryview.py new file mode 100644 index 00000000000000..3dba4b13ba40e2 --- /dev/null +++ b/Lib/test/test_free_threading/test_memoryview.py @@ -0,0 +1,323 @@ +import pickle +import threading +import unittest + +from test.support import threading_helper + +NTHREADS = 4 +ROUNDS = 4 +ITERS = 10 + +HASH_DATA_TEMPLATE = bytes(range(256)) +HASH_DATA_REPEAT = 256 + +READ_OPS = [ + lambda mv: mv.tobytes(), + lambda mv: mv.hex(), + lambda mv: mv.tolist(), + lambda mv: mv.cast("B"), + lambda mv: mv.toreadonly(), + lambda mv: memoryview(mv), + lambda mv: bytes(mv), + lambda mv: pickle.PickleBuffer(mv).release(), + lambda mv: mv.obj, + lambda mv: mv.format, + lambda mv: mv.c_contiguous, + lambda mv: mv[0], + lambda mv: mv[0:4], + lambda mv: mv.count(0), + lambda mv: mv.index(0), + lambda mv: mv == mv, + lambda mv: list(mv), + lambda mv: len(mv), +] + + +def run_racy(func, *args): + try: + func(*args) + except (ValueError, BufferError): + pass + + +@threading_helper.requires_working_threading() +class TestMemoryViewRaces(unittest.TestCase): + def ensure_exporter_free(self, buf): + # Will raise BufferError if any memoryview still exports it. + buf.append(0) + del buf[-1] + + def test_concurrent_slicing_keeps_export_count(self): + for _ in range(ROUNDS): + mv = memoryview(bytes(64)) + slices = [] + lock = threading.Lock() + + def make_slices(): + local = [mv[0:4] for _ in range(ITERS)] + local += [memoryview(mv) for _ in range(ITERS)] + with lock: + slices.extend(local) + + threading_helper.run_concurrently(make_slices, nthreads=NTHREADS) + del slices + + self.assertEqual(bytes(mv[0:4]), b"\x00" * 4) + + def test_concurrent_release(self): + buf = bytearray(64) + + for _ in range(ROUNDS): + views = [memoryview(buf) for _ in range(NTHREADS)] + + def release(views=views): + for mv in views: + run_racy(mv.release) + + threading_helper.run_concurrently(release, nthreads=NTHREADS) + + self.ensure_exporter_free(buf) + + def test_release_races_with_reads(self): + for _ in range(ROUNDS): + buf = bytearray(64) + cell = [memoryview(buf)] + lock = threading.Lock() + + def releaser(): + for _ in range(ITERS): + mv = cell[0] + run_racy(mv.release) + with lock: + cell[0] = memoryview(buf) + + def reader(): + for _ in range(ITERS): + mv = cell[0] + for op in READ_OPS: + run_racy(op, mv) + + threading_helper.run_concurrently( + [releaser] * (NTHREADS // 2) + [reader] * (NTHREADS // 2), + nthreads=NTHREADS, + ) + + cell[0].release() + self.ensure_exporter_free(buf) + + def test_read_keeps_exporter_alive_after_release(self): + size = 1 << 20 + for _ in range(ROUNDS): + exporter = [bytearray(size)] + view = memoryview(exporter[0]) + stale = [] + + def reader(): + try: + for _ in range(4): + data = view.tobytes() + if data.count(0) != len(data): + stale.append(True) + return + except ValueError: + pass + + def releaser(): + view.release() + exporter.clear() + for _ in range(8): + bytearray(b"\xdb" * size) + + threading_helper.run_concurrently([reader, releaser]) + self.assertFalse(stale) + + def test_release_races_with_writes(self): + for _ in range(ROUNDS): + buf = bytearray(8 * NTHREADS) + cell = [memoryview(buf)] + lock = threading.Lock() + + def releaser(): + for _ in range(ITERS): + mv = cell[0] + run_racy(mv.release) + with lock: + cell[0] = memoryview(buf) + + def writer(slot): + start = slot * 8 + for _ in range(ITERS): + mv = cell[0] + run_racy(mv.__setitem__, start, 1) + run_racy(mv.__setitem__, slice(start, start + 4), b"abcd") + + workers = [releaser] * (NTHREADS // 2) + workers += [lambda s=s: writer(s) for s in range(NTHREADS // 2)] + threading_helper.run_concurrently(workers, nthreads=NTHREADS) + + cell[0].release() + self.ensure_exporter_free(buf) + + def test_release_races_with_buffer_exports(self): + for _ in range(ROUNDS): + buf = bytearray(64) + mv = memoryview(buf) + + def exporter(): + for _ in range(ITERS): + run_racy(lambda: pickle.PickleBuffer(mv).release()) + + def releaser(): + for _ in range(ITERS): + run_racy(mv.release) + + threading_helper.run_concurrently( + [exporter] * (NTHREADS - 1) + [releaser], nthreads=NTHREADS + ) + + mv.release() + self.ensure_exporter_free(buf) + + def test_release_with_live_export(self): + buf = bytearray(64) + + for _ in range(ROUNDS): + mv = memoryview(buf) + held = pickle.PickleBuffer(mv) + + def release(mv=mv): + try: + mv.release() + except BufferError: + pass + + threading_helper.run_concurrently(release, nthreads=NTHREADS) + + self.assertEqual(bytes(mv[0:4]), b"\x00" * 4) + held.release() + mv.release() + + self.ensure_exporter_free(buf) + + def test_compare_two_views_races_with_release(self): + for _ in range(ROUNDS): + buf = bytearray(64) + cell = [memoryview(buf), memoryview(buf)] + lock = threading.Lock() + + def releaser(slot): + for _ in range(ITERS): + mv = cell[slot] + run_racy(mv.release) + with lock: + cell[slot] = memoryview(buf) + + def comparer(): + for _ in range(ITERS): + left, right = cell[0], cell[1] + run_racy(lambda: left == right) + run_racy(lambda: left != right) + + threading_helper.run_concurrently( + [lambda: releaser(0), lambda: releaser(1)] + + [comparer] * (NTHREADS - 2), + nthreads=NTHREADS, + ) + + for mv in cell: + mv.release() + self.ensure_exporter_free(buf) + + def test_release_parent_keeps_child_valid(self): + for _ in range(ROUNDS): + buf = bytearray(range(64)) + parent = memoryview(buf) + child = parent[0:32] + + def use_child(): + for _ in range(ITERS): + self.assertEqual(child[0], 0) + child.tobytes() + + def release_parent(): + parent.release() + + threading_helper.run_concurrently( + [release_parent] + [use_child] * (NTHREADS - 1), + nthreads=NTHREADS, + ) + + self.assertEqual(child.tobytes(), bytes(range(32))) + child.release() + self.ensure_exporter_free(buf) + + def test_concurrent_iteration(self): + for _ in range(ROUNDS): + buf = bytearray(range(64)) + cell = [memoryview(buf)] + lock = threading.Lock() + + def releaser(): + for _ in range(ITERS): + mv = cell[0] + run_racy(mv.release) + with lock: + cell[0] = memoryview(buf) + + def iterator(): + for _ in range(ITERS): + run_racy(list, cell[0]) + + threading_helper.run_concurrently( + [releaser] * (NTHREADS // 2) + [iterator] * (NTHREADS // 2), + nthreads=NTHREADS, + ) + + cell[0].release() + self.ensure_exporter_free(buf) + + def test_iterator_exhaustion_drops_last_reference(self): + for _ in range(ROUNDS): + data = bytearray(range(64)) + iterator = iter(memoryview(data)) + values = [] + lock = threading.Lock() + + def consume(): + local = list(iterator) + with lock: + values.extend(local) + + threading_helper.run_concurrently(consume, nthreads=NTHREADS) + self.assertEqual(sorted(values), list(data)) + data.extend(b"x") + self.assertRaises(StopIteration, next, iterator) + + def test_concurrent_hash(self): + for _ in range(ROUNDS): + data = HASH_DATA_TEMPLATE * HASH_DATA_REPEAT + mv = memoryview(data) + results = [] + lock = threading.Lock() + + def hasher(): + local = {hash(mv) for _ in range(ITERS)} + with lock: + results.append(local) + + threading_helper.run_concurrently(hasher, nthreads=NTHREADS) + self.assertEqual({h for s in results for h in s}, {hash(data)}) + + def test_concurrent_hash_and_release(self): + for _ in range(ROUNDS): + mv = memoryview(HASH_DATA_TEMPLATE * HASH_DATA_REPEAT) + + def work(mv=mv): + run_racy(hash, mv) + run_racy(mv.release) + + threading_helper.run_concurrently(work, nthreads=NTHREADS) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 40d4798efbf18f..283f0077d7ae93 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -91,6 +91,13 @@ def test_iter(self): m = self._view(b) self.assertEqual(list(m), [m[i] for i in range(len(m))]) + def test_released_empty_iterator(self): + for tp in self._types: + m = self._view(tp(b"")) + iterator = iter(m) + m.release() + self.assertRaises(StopIteration, next, iterator) + def test_count(self): for tp in self._types: b = tp(self._source) @@ -554,6 +561,61 @@ def _check_contents(self, tp, obj, contents): class BytesMemoryviewTest(unittest.TestCase, BaseMemoryviewTests, BaseBytesMemoryTests): + def test_comparison_release_during_buffer_callback(self): + # Releasing the view from inside the other operand's __buffer__ + # succeeds, but the buffer stays pinned until the comparison ends: + # the exporter cannot be resized under the running comparison. + data = bytearray(b'abc') + view = memoryview(data) + assert_raises = self.assertRaises + + class Other: + def __buffer__(self, flags): + view.release() + with assert_raises(BufferError): + data.clear() + return memoryview(data) + + view == Other() + with self.assertRaisesRegex(ValueError, "released"): + view.tobytes() + data.clear() + + def test_comparison_releases_buffer_after_access(self): + # __release_buffer__ releasing the very view being compared. + data = bytearray(b'abc') + view = memoryview(data) + + class Other: + def __buffer__(self, flags): + return view + + def __release_buffer__(self, buffer): + buffer.release() + + with support.catch_unraisable_exception() as cm: + self.assertTrue(view == Other()) + self.assertIsNone(cm.unraisable) + data.clear() + + def test_assignment_releases_buffer_after_access(self): + # __release_buffer__ releasing the very view being assigned to. + data = bytearray(b'abc') + view = memoryview(data) + + class Other: + def __buffer__(self, flags): + return view + + def __release_buffer__(self, buffer): + buffer.release() + + with support.catch_unraisable_exception() as cm: + view[:] = Other() + self.assertIsNone(cm.unraisable) + self.assertEqual(data, b'abc') + data.clear() + def test_constructor(self): for tp in self._types: ob = tp(self._source) @@ -689,6 +751,29 @@ class ArrayMemorySliceSliceTest(unittest.TestCase, class OtherTest(unittest.TestCase): + def test_gc_cleared_view(self): + _testcapi = import_helper.import_module("_testcapi") + + view = memoryview(bytearray(b"abc")) + _testcapi.clear_memoryview(view) + with self.assertRaisesRegex(ValueError, "released"): + view.tobytes() + + # Clearing the view while an operation is in progress must not + # free the buffer under that operation, nor crash when it ends. + data = bytearray(b"abc") + view = memoryview(data) + + class Other: + def __buffer__(self, flags): + _testcapi.clear_memoryview(view) + return memoryview(data) + + self.assertTrue(view == Other()) + with self.assertRaisesRegex(ValueError, "released"): + view.tobytes() + data.clear() + def test_cast_order(self): # gh-78959: cast(order=...) selects the result memory layout. b = bytearray(range(6)) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-14-37-50.gh-issue-127716.KrPth8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-14-37-50.gh-issue-127716.KrPth8.rst new file mode 100644 index 00000000000000..ca2e9bf7a892f2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-14-37-50.gh-issue-127716.KrPth8.rst @@ -0,0 +1,10 @@ +On the free-threaded build, :class:`memoryview` lifetime and bookkeeping +operations are now thread safe: +an operation that reads the buffer holds the managed buffer for its duration, +so a concurrent :meth:`~memoryview.release` can no longer free the buffer under +it; slicing a memoryview from several threads no longer loses buffer +registrations and raises ``ValueError: operation forbidden on released +memoryview object``; and :func:`hash` of a memoryview is no longer a data race. +As a result :meth:`~memoryview.release` may now return while an operation +started by another thread is still running, see +:ref:`thread-safety-memoryview`. diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c index 48393a3dd530c6..c086d7250f6774 100644 --- a/Modules/_testcapi/buffer.c +++ b/Modules/_testcapi/buffer.c @@ -114,8 +114,22 @@ buffer_pointer_as_int(PyObject *Py_UNUSED(module), PyObject *obj) return out; } +static PyObject * +clear_memoryview(PyObject *Py_UNUSED(module), PyObject *obj) +{ + if (!PyMemoryView_Check(obj)) { + PyErr_SetString(PyExc_TypeError, "memoryview expected"); + return NULL; + } + if (Py_TYPE(obj)->tp_clear(obj) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + static PyMethodDef test_methods[] = { {"buffer_pointer_as_int", buffer_pointer_as_int, METH_O}, + {"clear_memoryview", clear_memoryview, METH_O}, {NULL}, }; diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index e6d49b6c29a3e9..11a0a7ab04f08b 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -103,13 +103,63 @@ _PyManagedBuffer_FromObject(PyObject *base, int flags) return (PyObject *)mbuf; } +static inline int +flags_set_released(int *flags, int released) +{ +#ifdef Py_GIL_DISABLED + int prev = _Py_atomic_load_int_relaxed(flags); + if (prev & released) { + return 1; + } + if (!_Py_atomic_compare_exchange_int(flags, &prev, prev | released)) { + // RELEASED is the only flag that can change concurrently. + assert(prev & released); + return 1; + } + return 0; +#else + int prev = *flags; + *flags = prev | released; + return (prev & released) != 0; +#endif +} + +static inline int +mv_get_flags(PyMemoryViewObject *mv) +{ + return FT_ATOMIC_LOAD_INT_RELAXED(mv->flags); +} + +static inline int +mv_has_flag(PyMemoryViewObject *mv, int bits) +{ + return (mv_get_flags(mv) & bits) != 0; +} + +static inline int +mv_set_released(PyMemoryViewObject *mv) +{ + return flags_set_released(&mv->flags, _Py_MEMORYVIEW_RELEASED); +} + +static inline int +mbuf_has_flag(_PyManagedBufferObject *mbuf, int bits) +{ + return (FT_ATOMIC_LOAD_INT(mbuf->flags) & bits) != 0; +} + +static inline int +mbuf_set_released(_PyManagedBufferObject *mbuf) +{ + return flags_set_released(&mbuf->flags, _Py_MANAGED_BUFFER_RELEASED); +} + static void mbuf_release(_PyManagedBufferObject *self) { - if (self->flags&_Py_MANAGED_BUFFER_RELEASED) + if (mbuf_set_released(self)) { return; - - self->flags |= _Py_MANAGED_BUFFER_RELEASED; + } /* PyBuffer_Release() decrements master->obj and sets it to NULL. */ _PyObject_GC_UNTRACK(self); @@ -120,9 +170,9 @@ static void mbuf_dealloc(PyObject *_self) { _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; - assert(self->exports == 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); mbuf_release(self); - if (self->flags&_Py_MANAGED_BUFFER_FREE_FORMAT) + if (mbuf_has_flag(self, _Py_MANAGED_BUFFER_FREE_FORMAT)) PyMem_Free(self->master.format); PyObject_GC_Del(self); } @@ -139,7 +189,7 @@ static int mbuf_clear(PyObject *_self) { _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; - assert(self->exports >= 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) >= 0); mbuf_release(self); return 0; } @@ -178,8 +228,8 @@ PyTypeObject _PyManagedBuffer_Type = { /* In the process of breaking reference cycles mbuf_release() can be called before memory_release(). */ #define BASE_INACCESSIBLE(mv) \ - (((PyMemoryViewObject *)mv)->flags&_Py_MEMORYVIEW_RELEASED || \ - ((PyMemoryViewObject *)mv)->mbuf->flags&_Py_MANAGED_BUFFER_RELEASED) + (mv_has_flag(_PyMemoryView_CAST(mv), _Py_MEMORYVIEW_RELEASED) || \ + mbuf_has_flag(_PyMemoryView_CAST(mv)->mbuf, _Py_MANAGED_BUFFER_RELEASED)) #define CHECK_RELEASED(mv) \ if (BASE_INACCESSIBLE(mv)) { \ @@ -196,17 +246,17 @@ PyTypeObject _PyManagedBuffer_Type = { } #define CHECK_RESTRICTED(mv) \ - if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ - PyErr_SetString(PyExc_ValueError, \ - "cannot create new view on restricted memoryview"); \ - return NULL; \ + if (mv_has_flag(_PyMemoryView_CAST(mv), _Py_MEMORYVIEW_RESTRICTED)) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return NULL; \ } #define CHECK_RESTRICTED_INT(mv) \ - if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ - PyErr_SetString(PyExc_ValueError, \ - "cannot create new view on restricted memoryview"); \ - return -1; \ + if (mv_has_flag(_PyMemoryView_CAST(mv), _Py_MEMORYVIEW_RESTRICTED)) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return -1; \ } /* See gh-92888. These macros signal that we need to check the memoryview @@ -699,7 +749,7 @@ mbuf_add_view(_PyManagedBufferObject *mbuf, const Py_buffer *src) init_flags(mv); mv->mbuf = (_PyManagedBufferObject*)Py_NewRef(mbuf); - mbuf->exports++; + FT_ATOMIC_ADD_SSIZE(mbuf->exports, 1); return (PyObject *)mv; } @@ -729,11 +779,79 @@ mbuf_add_incomplete_view(_PyManagedBufferObject *mbuf, const Py_buffer *src, init_shared_values(dest, src); mv->mbuf = (_PyManagedBufferObject*)Py_NewRef(mbuf); - mbuf->exports++; + FT_ATOMIC_ADD_SSIZE(mbuf->exports, 1); return (PyObject *)mv; } +#ifdef Py_GIL_DISABLED +static int +memoryview_add_export(PyMemoryViewObject *self) +{ + int result = 0; + + Py_BEGIN_CRITICAL_SECTION(self); + if (BASE_INACCESSIBLE(self)) { + PyErr_SetString(PyExc_ValueError, + "operation forbidden on released memoryview object"); + result = -1; + } + else { + FT_ATOMIC_ADD_SSIZE(self->exports, 1); + } + Py_END_CRITICAL_SECTION(); + + return result; +} +#endif + +static void +mbuf_drop_export(_PyManagedBufferObject *mbuf) +{ +#ifdef Py_GIL_DISABLED + Py_ssize_t prev_exports = _Py_atomic_add_ssize(&mbuf->exports, -1); +#else + Py_ssize_t prev_exports = mbuf->exports--; +#endif + assert(prev_exports > 0); + if (prev_exports == 1) { + mbuf_release(mbuf); + } +} + +/* Pin the managed buffer for the duration of an operation that reads + self->view. The pin is an export on the managed buffer, so a concurrent + or re-entrant release of this memoryview (or a tp_clear of it) cannot + free the underlying buffer while the operation is running. The pin does + not hold a reference to the managed buffer: the memoryview's own + reference is kept until memory_dealloc(), and the caller holds a + reference to the memoryview through memoryview_unpin(self). Return 0 + on success and -1 with an exception on failure. */ +static inline int +memoryview_pin(PyMemoryViewObject *self) +{ + if (mv_has_flag(self, _Py_MEMORYVIEW_RELEASED)) { + PyErr_SetString(PyExc_ValueError, + "operation forbidden on released memoryview object"); + return -1; + } + _PyManagedBufferObject *mbuf = self->mbuf; + FT_ATOMIC_ADD_SSIZE(mbuf->exports, 1); + if (BASE_INACCESSIBLE(self)) { + mbuf_drop_export(mbuf); + PyErr_SetString(PyExc_ValueError, + "operation forbidden on released memoryview object"); + return -1; + } + return 0; +} + +static inline void +memoryview_unpin(PyMemoryViewObject *self) +{ + mbuf_drop_export(self->mbuf); +} + /* Expose a raw memory area as a view of contiguous bytes. flags can be PyBUF_READ or PyBUF_WRITE. view->format is set to "B" (unsigned bytes). The memoryview has complete buffer information. */ @@ -804,9 +922,20 @@ PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) if (PyMemoryView_Check(v)) { PyMemoryViewObject *mv = (PyMemoryViewObject *)v; - CHECK_RELEASED(mv); - CHECK_RESTRICTED(mv); - return mbuf_add_view(mv->mbuf, &mv->view); + if (memoryview_pin(mv) < 0) { + return NULL; + } + PyObject *result; + if (mv_has_flag(mv, _Py_MEMORYVIEW_RESTRICTED)) { + PyErr_SetString(PyExc_ValueError, + "cannot create new view on restricted memoryview"); + result = NULL; + } + else { + result = mbuf_add_view(mv->mbuf, &mv->view); + } + memoryview_unpin(mv); + return result; } else if (PyObject_CheckBuffer(v)) { PyObject *ret; @@ -1089,11 +1218,7 @@ PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char orde static inline Py_ssize_t get_exports(PyMemoryViewObject *buf) { -#ifdef Py_GIL_DISABLED - return _Py_atomic_load_ssize_relaxed(&buf->exports); -#else - return buf->exports; -#endif + return FT_ATOMIC_LOAD_SSIZE_RELAXED(buf->exports); } @@ -1109,14 +1234,11 @@ static void _memory_release(PyMemoryViewObject *self) { assert(get_exports(self) == 0); - if (self->flags & _Py_MEMORYVIEW_RELEASED) + if (mv_set_released(self)) { return; - - self->flags |= _Py_MEMORYVIEW_RELEASED; - assert(self->mbuf->exports > 0); - if (--self->mbuf->exports == 0) { - mbuf_release(self->mbuf); } + + mbuf_drop_export(self->mbuf); } /*[clinic input] @@ -1129,22 +1251,26 @@ static PyObject * memoryview_release_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=d0b7e3ba95b7fcb9 input=bc71d1d51f4a52f0]*/ { + PyObject *result = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); Py_ssize_t exports = get_exports(self); if (exports == 0) { _memory_release(self); - Py_RETURN_NONE; + result = Py_None; } - - if (exports > 0) { + else if (exports > 0) { PyErr_Format(PyExc_BufferError, "memoryview has %zd exported buffer%s", exports, - exports==1 ? "" : "s"); - return NULL; + exports == 1 ? "" : "s"); + } + else { + PyErr_SetString(PyExc_SystemError, + "memoryview: negative export count"); } + Py_END_CRITICAL_SECTION(); - PyErr_SetString(PyExc_SystemError, - "memoryview: negative export count"); - return NULL; + return result; } static void @@ -1174,8 +1300,11 @@ memory_clear(PyObject *_self) PyMemoryViewObject *self = (PyMemoryViewObject *)_self; if (get_exports(self) == 0) { _memory_release(self); - Py_CLEAR(self->mbuf); } + /* Keep the reference to self->mbuf until memory_dealloc(): an operation + that pinned the managed buffer may still be running (see + memoryview_pin()), and mbuf_clear() already breaks the managed + buffer's own reference cycle through the exporter. */ return 0; } @@ -1479,25 +1608,9 @@ zero_in_shape(PyMemoryViewObject *mv) All casts must result in views that will have the exact byte size of the original input. Otherwise, an error is raised. */ -/*[clinic input] -memoryview.cast - - format: unicode - shape: object = NULL - * - order: int(accept={str}) = 'C' - -Cast a memoryview to a new format or shape. - -With a multidimensional *shape*, *order* selects the result -layout: 'C' for C-contiguous (row-major, the default) or 'F' -for Fortran-contiguous (column-major). -[clinic start generated code]*/ - static PyObject * -memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, +memoryview_cast_pinned(PyMemoryViewObject *self, PyObject *format, PyObject *shape, int order) -/*[clinic end generated code: output=6410d87141f6bb56 input=4a1a2326c59caeb3]*/ { PyMemoryViewObject *mv = NULL; Py_ssize_t ndim = 1; @@ -1510,8 +1623,9 @@ memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, return NULL; } - if (!MV_C_CONTIGUOUS(self->flags)) { - if (shape || !MV_F_CONTIGUOUS(self->flags)) { + int self_flags = mv_get_flags(self); + if (!MV_C_CONTIGUOUS(self_flags)) { + if (shape || !MV_F_CONTIGUOUS(self_flags)) { PyErr_SetString(PyExc_TypeError, "memoryview: casts are restricted to contiguous views"); return NULL; @@ -1556,14 +1670,35 @@ memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, } /*[clinic input] -memoryview.toreadonly +memoryview.cast -Return a readonly version of the memoryview. + format: unicode + shape: object = NULL + * + order: int(accept={str}) = 'C' + +Cast a memoryview to a new format or shape. + +With a multidimensional *shape*, *order* selects the result +layout: 'C' for C-contiguous (row-major, the default) or 'F' +for Fortran-contiguous (column-major). [clinic start generated code]*/ static PyObject * -memoryview_toreadonly_impl(PyMemoryViewObject *self) -/*[clinic end generated code: output=2c7e056f04c99e62 input=dc06d20f19ba236f]*/ +memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, + PyObject *shape, int order) +/*[clinic end generated code: output=6410d87141f6bb56 input=4a1a2326c59caeb3]*/ +{ + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memoryview_cast_pinned(self, format, shape, order); + memoryview_unpin(self); + return result; +} + +static PyObject * +memoryview_toreadonly_pinned(PyMemoryViewObject *self) { CHECK_RELEASED(self); CHECK_RESTRICTED(self); @@ -1577,6 +1712,24 @@ memoryview_toreadonly_impl(PyMemoryViewObject *self) return (PyObject *) self; } +/*[clinic input] +memoryview.toreadonly + +Return a readonly version of the memoryview. +[clinic start generated code]*/ + +static PyObject * +memoryview_toreadonly_impl(PyMemoryViewObject *self) +/*[clinic end generated code: output=2c7e056f04c99e62 input=dc06d20f19ba236f]*/ +{ + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memoryview_toreadonly_pinned(self); + memoryview_unpin(self); + return result; +} + /**************************************************************************/ /* getbuffer */ @@ -1587,7 +1740,7 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *base = &self->view; - int baseflags = self->flags; + int baseflags = mv_get_flags(self); CHECK_RELEASED_INT(self); CHECK_RESTRICTED_INT(self); @@ -1655,10 +1808,16 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->shape = NULL; } +#ifdef Py_GIL_DISABLED + /* memoryview_add_export() keeps the buffer alive or fails. */ + if (memoryview_add_export(self) < 0) { + return -1; + } +#else + self->exports++; +#endif view->obj = Py_NewRef(self); - FT_ATOMIC_ADD_SSIZE(self->exports, 1); - return 0; } @@ -2341,15 +2500,8 @@ tolist_rec(PyMemoryViewObject *self, const char *ptr, Py_ssize_t ndim, const Py_ /* Return a list representation of the memoryview. Currently only buffers with native format strings are supported. */ -/*[clinic input] -memoryview.tolist - -Return the data in the buffer as a list of elements. -[clinic start generated code]*/ - static PyObject * -memoryview_tolist_impl(PyMemoryViewObject *self) -/*[clinic end generated code: output=a6cda89214fd5a1b input=21e7d0c1860b211a]*/ +memoryview_tolist_pinned(PyMemoryViewObject *self) { const Py_buffer *view = &self->view; const char *fmt; @@ -2375,23 +2527,26 @@ memoryview_tolist_impl(PyMemoryViewObject *self) } /*[clinic input] -memoryview.tobytes +memoryview.tolist - order: str(accept={str, NoneType}, c_default="NULL") = 'C' +Return the data in the buffer as a list of elements. +[clinic start generated code]*/ -Return the data in the buffer as a byte string. +static PyObject * +memoryview_tolist_impl(PyMemoryViewObject *self) +/*[clinic end generated code: output=a6cda89214fd5a1b input=21e7d0c1860b211a]*/ +{ + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memoryview_tolist_pinned(self); + memoryview_unpin(self); + return result; +} -Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of -the original array is converted to C or Fortran order. For -contiguous views, 'A' returns an exact copy of the physical memory. -In particular, in-memory Fortran order is preserved. For -non-contiguous views, the data is converted to C first. order=None -is the same as order='C'. -[clinic start generated code]*/ static PyObject * -memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) -/*[clinic end generated code: output=1288b62560a32a23 input=119c70aa91791dc8]*/ +memoryview_tobytes_pinned(PyMemoryViewObject *self, const char *order) { Py_buffer *src = VIEW_ADDR(self); char ord = 'C'; @@ -2427,42 +2582,52 @@ memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) } /*[clinic input] -memoryview.hex +memoryview.tobytes - sep: object = NULL - An optional single character or byte to separate hex bytes. - bytes_per_sep: Py_ssize_t = 1 - How many bytes between separators. Positive values count from - the right, negative values count from the left. + order: str(accept={str, NoneType}, c_default="NULL") = 'C' -Return the data in the buffer as a str of hexadecimal numbers. +Return the data in the buffer as a byte string. -Example: ->>> value = memoryview(b'\xb9\x01\xef') ->>> value.hex() -'b901ef' ->>> value.hex(':') -'b9:01:ef' ->>> value.hex(':', 2) -'b9:01ef' ->>> value.hex(':', -2) -'b901:ef' +Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of +the original array is converted to C or Fortran order. For +contiguous views, 'A' returns an exact copy of the physical memory. +In particular, in-memory Fortran order is preserved. For +non-contiguous views, the data is converted to C first. order=None +is the same as order='C'. [clinic start generated code]*/ static PyObject * -memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, +memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) +/*[clinic end generated code: output=1288b62560a32a23 input=119c70aa91791dc8]*/ +{ + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memoryview_tobytes_pinned(self, order); + memoryview_unpin(self); + return result; +} + + +static PyObject * +memoryview_hex_pinned(PyMemoryViewObject *self, PyObject *sep, Py_ssize_t bytes_per_sep) -/*[clinic end generated code: output=c9bb00c7a8e86056 input=3f1c5d08906e3b70]*/ { Py_buffer *src = VIEW_ADDR(self); CHECK_RELEASED(self); - if (MV_C_CONTIGUOUS(self->flags)) { + if (MV_C_CONTIGUOUS(mv_get_flags(self))) { // Prevent 'self' from being freed if computing len(sep) mutates 'self' // in _Py_strhex_with_sep(). // See: https://github.com/python/cpython/issues/143195. - FT_ATOMIC_ADD_SSIZE(self->exports, 1); +#ifdef Py_GIL_DISABLED + if (memoryview_add_export(self) < 0) { + return NULL; + } +#else + self->exports++; +#endif PyObject *ret = _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); FT_ATOMIC_ADD_SSIZE(self->exports, -1); return ret; @@ -2485,11 +2650,48 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, return ret; } +/*[clinic input] +memoryview.hex + + sep: object = NULL + An optional single character or byte to separate hex bytes. + bytes_per_sep: Py_ssize_t = 1 + How many bytes between separators. Positive values count from + the right, negative values count from the left. + +Return the data in the buffer as a str of hexadecimal numbers. + +Example: +>>> value = memoryview(b'\xb9\x01\xef') +>>> value.hex() +'b901ef' +>>> value.hex(':') +'b9:01:ef' +>>> value.hex(':', 2) +'b9:01ef' +>>> value.hex(':', -2) +'b901:ef' +[clinic start generated code]*/ + +static PyObject * +memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, + Py_ssize_t bytes_per_sep) +/*[clinic end generated code: output=c9bb00c7a8e86056 input=3f1c5d08906e3b70]*/ +{ + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memoryview_hex_pinned(self, sep, bytes_per_sep); + memoryview_unpin(self); + return result; +} + + static PyObject * memory_repr(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - if (self->flags & _Py_MEMORYVIEW_RELEASED) + if (mv_has_flag(self, _Py_MEMORYVIEW_RELEASED)) return PyUnicode_FromFormat("<released memory at %p>", self); else return PyUnicode_FromFormat("<memory at %p>", self); @@ -2562,9 +2764,9 @@ ptr_from_tuple(const Py_buffer *view, PyObject *tup) /* Return the item at index. In a one-dimensional view, this is an object with the type specified by view->format. Otherwise, the item is a sub-view. - The function is used in memory_subscript() and memory_as_sequence. */ + The function is used in memory_subscript_pinned() and by sq_item. */ static PyObject * -memory_item(PyObject *_self, Py_ssize_t index) +memory_item_pinned(PyObject *_self, Py_ssize_t index) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &(self->view); @@ -2690,7 +2892,7 @@ is_multiindex(PyObject *key) 0-d memoryview objects can be referenced using mv[...] or mv[()] but not with anything else. */ static PyObject * -memory_subscript(PyObject *_self, PyObject *key) +memory_subscript_pinned(PyObject *_self, PyObject *key) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view; @@ -2720,7 +2922,7 @@ memory_subscript(PyObject *_self, PyObject *key) index = PyNumber_AsSsize_t(key, PyExc_IndexError); if (index == -1 && PyErr_Occurred()) return NULL; - return memory_item((PyObject *)self, index); + return memory_item_pinned((PyObject *)self, index); } else if (PySlice_Check(key)) { CHECK_RESTRICTED(self); @@ -2753,7 +2955,7 @@ memory_subscript(PyObject *_self, PyObject *key) } static int -memory_ass_sub(PyObject *_self, PyObject *key, PyObject *value) +memory_ass_sub_pinned(PyObject *_self, PyObject *key, PyObject *value) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &(self->view); @@ -2843,7 +3045,7 @@ memory_ass_sub(PyObject *_self, PyObject *key, PyObject *value) return pack_single(self, ptr, value, fmt); } if (PySlice_Check(key) || is_multislice(key)) { - /* Call memory_subscript() to produce a sliced lvalue, then copy + /* Call memory_subscript_pinned() to produce a sliced lvalue, then copy rvalue into lvalue. This is already implemented in _testbuffer.c. */ PyErr_SetString(PyExc_NotImplementedError, "memoryview slice assignments are currently restricted " @@ -2867,6 +3069,42 @@ memory_length(PyObject *_self) return self->view.shape[0]; } +static PyObject * +memory_item(PyObject *_self, Py_ssize_t index) +{ + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memory_item_pinned(_self, index); + memoryview_unpin(self); + return result; +} + +static PyObject * +memory_subscript(PyObject *_self, PyObject *key) +{ + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memory_subscript_pinned(_self, key); + memoryview_unpin(self); + return result; +} + +static int +memory_ass_sub(PyObject *_self, PyObject *key, PyObject *value) +{ + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; + if (memoryview_pin(self) < 0) { + return -1; + } + int result = memory_ass_sub_pinned(_self, key, value); + memoryview_unpin(self); + return result; +} + /* As mapping */ static PyMappingMethods memory_as_mapping = { memory_length, /* mp_length */ @@ -2887,18 +3125,8 @@ static PySequenceMethods memory_as_sequence = { /* Counting */ /****************************************************************************/ -/*[clinic input] -memoryview.count - - value: object - / - -Count the number of occurrences of a value. -[clinic start generated code]*/ - static PyObject * -memoryview_count_impl(PyMemoryViewObject *self, PyObject *value) -/*[clinic end generated code: output=a15cb19311985063 input=e3036ce1ed7d1823]*/ +memoryview_count_pinned(PyMemoryViewObject *self, PyObject *value) { PyObject *iter = PyObject_GetIter(_PyObject_CAST(self)); if (iter == NULL) { @@ -2931,28 +3159,36 @@ memoryview_count_impl(PyMemoryViewObject *self, PyObject *value) return PyLong_FromSsize_t(count); } - -/**************************************************************************/ -/* Lookup */ -/**************************************************************************/ - /*[clinic input] -memoryview.index +memoryview.count value: object - start: slice_index(accept={int}) = 0 - stop: slice_index(accept={int}, c_default="PY_SSIZE_T_MAX") = sys.maxsize / -Return the index of the first occurrence of a value. - -Raises ValueError if the value is not present. +Count the number of occurrences of a value. [clinic start generated code]*/ static PyObject * -memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, - Py_ssize_t start, Py_ssize_t stop) -/*[clinic end generated code: output=e0185e3819e549df input=0697a0165bf90b5a]*/ +memoryview_count_impl(PyMemoryViewObject *self, PyObject *value) +/*[clinic end generated code: output=a15cb19311985063 input=e3036ce1ed7d1823]*/ +{ + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memoryview_count_pinned(self, value); + memoryview_unpin(self); + return result; +} + + + +/**************************************************************************/ +/* Lookup */ +/**************************************************************************/ + +static PyObject * +memoryview_index_pinned(PyMemoryViewObject *self, PyObject *value, + Py_ssize_t start, Py_ssize_t stop) { const Py_buffer *view = &self->view; CHECK_RELEASED(self); @@ -2989,7 +3225,7 @@ memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, // entire loop. assert(index < n); - PyObject *item = memory_item(obj, index); + PyObject *item = memory_item_pinned(obj, index); if (item == NULL) { return NULL; } @@ -3017,6 +3253,32 @@ memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, } +/*[clinic input] +memoryview.index + + value: object + start: slice_index(accept={int}) = 0 + stop: slice_index(accept={int}, c_default="PY_SSIZE_T_MAX") = sys.maxsize + / + +Return the index of the first occurrence of a value. + +Raises ValueError if the value is not present. +[clinic start generated code]*/ + +static PyObject * +memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, + Py_ssize_t start, Py_ssize_t stop) +/*[clinic end generated code: output=e0185e3819e549df input=0697a0165bf90b5a]*/ +{ + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memoryview_index_pinned(self, value, start, stop); + memoryview_unpin(self); + return result; +} + /**************************************************************************/ /* Comparisons */ @@ -3232,7 +3494,7 @@ cmp_rec(const char *p, const char *q, } static PyObject * -memory_richcompare(PyObject *v, PyObject *w, int op) +memory_richcompare_pinned(PyObject *v, PyObject *w, int op) { PyObject *res; Py_buffer wbuf, *vv; @@ -3358,15 +3620,44 @@ memory_richcompare(PyObject *v, PyObject *w, int op) return Py_XNewRef(res); } +static PyObject * +memory_richcompare(PyObject *v, PyObject *w, int op) +{ + PyMemoryViewObject *self = (PyMemoryViewObject *)v; + if (memoryview_pin(self) < 0) { + /* Released memoryviews still compare by identity. */ + PyErr_Clear(); + return memory_richcompare_pinned(v, w, op); + } + + PyMemoryViewObject *other = NULL; + if (PyMemoryView_Check(w) && w != v) { + other = (PyMemoryViewObject *)w; + if (memoryview_pin(other) < 0) { + PyErr_Clear(); + memoryview_unpin(self); + return memory_richcompare_pinned(v, w, op); + } + } + + PyObject *result = memory_richcompare_pinned(v, w, op); + if (other != NULL) { + memoryview_unpin(other); + } + memoryview_unpin(self); + return result; +} + /**************************************************************************/ /* Hash */ /**************************************************************************/ static Py_hash_t -memory_hash(PyObject *_self) +memory_hash_pinned(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - if (self->hash == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hash); + if (hash == -1) { Py_buffer *view = &self->view; char *mem = view->buf; Py_ssize_t ret; @@ -3388,7 +3679,13 @@ memory_hash(PyObject *_self) if (view->obj != NULL) { // Prevent 'self' from being freed when computing the item's hash. // See https://github.com/python/cpython/issues/142664. - FT_ATOMIC_ADD_SSIZE(self->exports, 1); +#ifdef Py_GIL_DISABLED + if (memoryview_add_export(self) < 0) { + return -1; + } +#else + self->exports++; +#endif Py_hash_t h = PyObject_Hash(view->obj); FT_ATOMIC_ADD_SSIZE(self->exports, -1); if (h == -1) { @@ -3397,7 +3694,7 @@ memory_hash(PyObject *_self) } } - if (!MV_C_CONTIGUOUS(self->flags)) { + if (!MV_C_CONTIGUOUS(mv_get_flags(self))) { mem = PyMem_Malloc(view->len); if (mem == NULL) { PyErr_NoMemory(); @@ -3410,13 +3707,30 @@ memory_hash(PyObject *_self) } /* Can't fail */ - self->hash = Py_HashBuffer(mem, view->len); + hash = Py_HashBuffer(mem, view->len); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hash, hash); if (mem != view->buf) PyMem_Free(mem); } - return self->hash; + return hash; +} + +static Py_hash_t +memory_hash(PyObject *_self) +{ + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hash); + if (hash != -1) { + return hash; + } + if (memoryview_pin(self) < 0) { + return -1; + } + Py_hash_t result = memory_hash_pinned(_self); + memoryview_unpin(self); + return result; } @@ -3449,7 +3763,7 @@ _IntTupleFromSsizet(int len, Py_ssize_t *vals) } static PyObject * -memory_obj_get(PyObject *_self, void *Py_UNUSED(ignored)) +memory_obj_get_pinned(PyObject *_self, void *Py_UNUSED(ignored)) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &self->view; @@ -3470,7 +3784,7 @@ memory_nbytes_get(PyObject *_self, void *Py_UNUSED(ignored)) } static PyObject * -memory_format_get(PyObject *_self, void *Py_UNUSED(ignored)) +memory_format_get_pinned(PyObject *_self, void *Py_UNUSED(ignored)) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); @@ -3530,7 +3844,7 @@ memory_c_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); - return PyBool_FromLong(MV_C_CONTIGUOUS(self->flags)); + return PyBool_FromLong(MV_C_CONTIGUOUS(mv_get_flags(self))); } static PyObject * @@ -3538,7 +3852,7 @@ memory_f_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); - return PyBool_FromLong(MV_F_CONTIGUOUS(self->flags)); + return PyBool_FromLong(MV_F_CONTIGUOUS(mv_get_flags(self))); } static PyObject * @@ -3546,9 +3860,27 @@ memory_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); - return PyBool_FromLong(MV_ANY_CONTIGUOUS(self->flags)); + return PyBool_FromLong(MV_ANY_CONTIGUOUS(mv_get_flags(self))); } +#define MEMORYVIEW_PINNED_GETTER(name) \ + static PyObject * \ + name(PyObject *_self, void *ignored) \ + { \ + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; \ + if (memoryview_pin(self) < 0) { \ + return NULL; \ + } \ + PyObject *result = name##_pinned(_self, ignored); \ + memoryview_unpin(self); \ + return result; \ + } + +MEMORYVIEW_PINNED_GETTER(memory_obj_get) +MEMORYVIEW_PINNED_GETTER(memory_format_get) + +#undef MEMORYVIEW_PINNED_GETTER + PyDoc_STRVAR(memory_obj_doc, "The underlying object of the memoryview."); PyDoc_STRVAR(memory_nbytes_doc, @@ -3648,31 +3980,61 @@ memoryiter_next(PyObject *self) { memoryiterobject *it = (memoryiterobject *)self; PyMemoryViewObject *seq; + Py_ssize_t index; + + Py_BEGIN_CRITICAL_SECTION(it); seq = it->it_seq; - if (seq == NULL) { + index = it->it_index; + if (seq != NULL) { + if (index >= it->it_length) { + /* Transfer the iterator's reference to seq before clearing it. */ + it->it_seq = NULL; + } + else { + Py_INCREF(seq); + } + } + Py_END_CRITICAL_SECTION(); + + if (seq == NULL || index >= it->it_length) { + Py_XDECREF(seq); return NULL; } - if (it->it_index < it->it_length) { - CHECK_RELEASED(seq); - Py_buffer *view = &(seq->view); - char *ptr = (char *)seq->view.buf; + /* Pin outside the iterator lock: a failed pin may release the buffer. */ + if (memoryview_pin(seq) < 0) { + Py_DECREF(seq); + return NULL; + } - ptr += view->strides[0] * it->it_index++; - ptr = ADJUST_PTR(ptr, view->suboffsets, 0); - if (ptr == NULL) { - return NULL; - } - return unpack_single(seq, ptr, it->it_fmt); + Py_BEGIN_CRITICAL_SECTION(it); + index = it->it_index; + if (index < it->it_length) { + it->it_index = index + 1; + } + Py_END_CRITICAL_SECTION(); + if (index >= it->it_length) { + memoryview_unpin(seq); + Py_DECREF(seq); + return NULL; } - it->it_seq = NULL; + Py_buffer *view = &seq->view; + char *ptr = (char *)view->buf; + + ptr += view->strides[0] * index; + ptr = ADJUST_PTR(ptr, view->suboffsets, 0); + PyObject *result = NULL; + if (ptr != NULL) { + result = unpack_single(seq, ptr, it->it_fmt); + } + memoryview_unpin(seq); Py_DECREF(seq); - return NULL; + return result; } static PyObject * -memory_iter(PyObject *seq) +memory_iter_pinned(PyObject *seq) { if (!PyMemoryView_Check(seq)) { PyErr_BadInternalCall(); @@ -3709,6 +4071,18 @@ memory_iter(PyObject *seq) return (PyObject *)it; } +static PyObject * +memory_iter(PyObject *seq) +{ + PyMemoryViewObject *self = (PyMemoryViewObject *)seq; + if (memoryview_pin(self) < 0) { + return NULL; + } + PyObject *result = memory_iter_pinned(seq); + memoryview_unpin(self); + return result; +} + PyTypeObject _PyMemoryIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "memory_iterator", _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
