https://github.com/python/cpython/commit/2e9acb457015104ddc033d326c6f1e281ef56dcb commit: 2e9acb457015104ddc033d326c6f1e281ef56dcb branch: 3.14 author: Victor Stinner <[email protected]> committer: vstinner <[email protected]> date: 2026-10-16T13:33:50Z summary:
[3.14] gh-157242: Do not close io.BytesIO on MemoryError (#159529) * gh-157242: Add _PyBytes_ResizeKeepOnError() function (#157243) Add a new internal _PyBytes_ResizeKeepOnError() function similar to _PyBytes_Resize() but leaves the bytes object unchanged on error. Co-authored-by: Maurycy Pawłowski-Wieroński <[email protected]> (cherry picked from commit f5dd52df16e1b3f3f8cc2762f1de783a41d69f4f) * gh-157242: Fix undefined behavior in _PyBytes_ResizeKeepOnError() (#157352) Fix undefined behavior in _PyBytes_ResizeKeepOnError(): raise OverflowError on integer overflow. (cherry picked from commit 63d4398be879e346ef68f899115f86cdb9fa555c) * gh-157242: Do not close io.BytesIO on MemoryError (#157344) Replace _PyBytes_Resize() with _PyBytes_ResizeKeepOnError(). Fix truncate(): on error, restore string_size to its previous value. (cherry picked from commit 23eda007f8d543495ba4935df75d243d76f9ef7f) --------- Co-authored-by: Maurycy Pawłowski-Wieroński <[email protected]> files: A Misc/NEWS.d/next/Library/2026-09-12-03-14-57.gh-issue-157242.ycbtjC.rst M Include/internal/pycore_bytesobject.h M Lib/test/test_memoryio.py M Modules/_io/bytesio.c M Modules/_testcapi/mem.c M Objects/bytesobject.c diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 8ea9b3ebb884541..012110c9fb93373 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -60,6 +60,8 @@ PyAPI_FUNC(void) _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); +extern int _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize); + /* --- _PyBytesWriter ----------------------------------------------------- */ /* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 8780796ce831ee1..ffe66fcf732f295 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -5,6 +5,7 @@ import unittest from test import support +from test.support import import_helper import gc import io @@ -651,6 +652,36 @@ def __buffer__(self, flags): self.assertEqual(memio.getvalue(), b"01AAA56789") self.assertEqual(memio.tell(), 5) + @support.nomemtest + def test_memory_error(self): + # gh-157242: io.BytesIO() must not close the file on MemoryError + _testcapi = import_helper.import_module('_testcapi') + + # write() + stream = self.ioclass() + stream.write(self.buftype('abc')) + with self.assertRaises(MemoryError): + try: + data = self.buftype('def') + _testcapi.set_nomemory(0) + stream.write(data) + finally: + _testcapi.remove_mem_hooks() + stream.write(self.buftype('123')) + self.assertEqual(stream.getvalue(), self.buftype('abc123')) + + # truncate() + data = self.buftype('x' * 100) + stream = self.ioclass() + stream.write(data) + with self.assertRaises(MemoryError): + try: + _testcapi.set_nomemory(0) + stream.truncate(5) + finally: + _testcapi.remove_mem_hooks() + self.assertEqual(stream.getvalue(), data) + class TextIOTestMixin: diff --git a/Misc/NEWS.d/next/Library/2026-09-12-03-14-57.gh-issue-157242.ycbtjC.rst b/Misc/NEWS.d/next/Library/2026-09-12-03-14-57.gh-issue-157242.ycbtjC.rst new file mode 100644 index 000000000000000..54ada2892f76fac --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-12-03-14-57.gh-issue-157242.ycbtjC.rst @@ -0,0 +1,2 @@ +:class:`io.BytesIO` is no longer closed on ``write()`` and ``truncate()`` +failure (:exc:`MemoryError`). Patch by Victor Stinner. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 6ed88357220c944..f920b96a7838cd7 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_bytesobject.h" // _PyBytes_ResizeKeepOnError() #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_object.h" #include "pycore_sysmodule.h" // _PySys_GetSizeOf() @@ -107,7 +108,7 @@ resize_unshared_buffer_lock_held(bytesio *self, Py_ssize_t size) Callers must detach first. */ assert(!self->buf_shared); #endif - int ret = _PyBytes_Resize(&self->buf, size); + int ret = _PyBytes_ResizeKeepOnError(&self->buf, size); if (ret == 0) { clear_shared_buf(self); } @@ -712,9 +713,12 @@ _io_BytesIO_truncate_impl(bytesio *self, PyObject *size) } if (new_size < self->string_size) { + Py_ssize_t old_string_size = self->string_size; self->string_size = new_size; - if (resize_buffer_lock_held(self, new_size) < 0) + if (resize_buffer_lock_held(self, new_size) < 0) { + self->string_size = old_string_size; return NULL; + } } return PyLong_FromSsize_t(new_size); diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index b4896f984510bd6..a6ac34d61bcdada 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -155,18 +155,29 @@ fm_remove_hooks(void) } } +static void +fm_set_nomemory(int start, int stop) +{ + /* Memory allocation fails after 'start' allocation requests, and until + * 'stop' allocation requests except when 'stop' is negative or equal + * to 0 (default) in which case allocation failures never stop. */ + FmData.start = start; + FmData.stop = stop; + FmData.count = 0; + fm_setup_hooks(); +} + static PyObject * set_nomemory(PyObject *self, PyObject *args) { /* Memory allocation fails after 'start' allocation requests, and until * 'stop' allocation requests except when 'stop' is negative or equal * to 0 (default) in which case allocation failures never stop. */ - FmData.count = 0; - FmData.stop = 0; - if (!PyArg_ParseTuple(args, "i|i", &FmData.start, &FmData.stop)) { + int start, stop = 0; + if (!PyArg_ParseTuple(args, "i|i", &start, &stop)) { return NULL; } - fm_setup_hooks(); + fm_set_nomemory(start, stop); Py_RETURN_NONE; } @@ -684,6 +695,129 @@ tracemalloc_track_race(PyObject *self, PyObject *args) } +struct bytes_resize_tracer { + PyObject *create; + PyObject *destroy; +}; + + +static int +bytes_resize_tracer(PyObject *obj, PyRefTracerEvent event, void* data) +{ + if (event != PyRefTracer_CREATE && event != PyRefTracer_DESTROY) { + return 0; + } + + struct bytes_resize_tracer *tracer = (struct bytes_resize_tracer*)data; + if (!PyBytes_Check(obj)) { + return 0; + } + + switch (event) { + case PyRefTracer_CREATE: + tracer->create = obj; + break; + case PyRefTracer_DESTROY: + tracer->destroy = obj; + break; + default: + break; + } + return 0; +} + + +// When _PyBytes_Resize() resizes a bytes object in-place, check that +// PyRefTracer_DESTROY and PyRefTracer_CREATE events are emitted. +// If no_memory is non-zero, inject MemoryError. +static int +check_bytes_resize_tracer(int no_memory) +{ + PyObject *bytes = NULL; + PyRefTracer old_tracer = NULL; + void *old_tracer_data = NULL; + int restore_tracer = 0; + + bytes = PyBytes_FromString("hello"); + if (bytes == NULL) { + goto error; + } + assert(PyUnstable_Object_IsUniquelyReferenced(bytes)); + + old_tracer = PyRefTracer_GetTracer(&old_tracer_data); + restore_tracer = 1; + + struct bytes_resize_tracer tracer = {0}; + if (PyRefTracer_SetTracer(bytes_resize_tracer, &tracer) != 0) { + goto error; + } + + PyObject *old_bytes = bytes; // borrowed reference + if (no_memory) { + fm_set_nomemory(0, 0); + int res = _PyBytes_Resize(&bytes, 100); + assert(res < 0); + assert(bytes == NULL); + fm_remove_hooks(); + + assert(PyErr_ExceptionMatches(PyExc_MemoryError)); + PyErr_Clear(); + } + else { + if (_PyBytes_Resize(&bytes, 100) < 0) { + assert(bytes == NULL); + goto error; + } + } + + if (tracer.destroy != old_bytes) { + PyErr_SetString(PyExc_AssertionError, "PyRefTracer_DESTROY not seen"); + goto error; + } + + int seen_create; + if (no_memory) { + seen_create = (tracer.create == old_bytes); + } + else { + seen_create = (tracer.create == bytes); + } + if (!seen_create) { + PyErr_SetString(PyExc_AssertionError, "PyRefTracer_CREATE not seen"); + goto error; + } + + Py_CLEAR(bytes); + if (PyRefTracer_SetTracer(old_tracer, old_tracer_data) != 0) { + restore_tracer = 0; + goto error; + } + return 0; + +error: + Py_XDECREF(bytes); + if (restore_tracer) { + if (PyRefTracer_SetTracer(old_tracer, old_tracer_data) != 0) { + return -1; + } + } + return -1; +} + + +static PyObject* +test_bytes_resize_tracer(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (check_bytes_resize_tracer(0) < 0) { + return NULL; + } + if (check_bytes_resize_tracer(1) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + static PyMethodDef test_methods[] = { {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, @@ -698,6 +832,7 @@ static PyMethodDef test_methods[] = { {"test_pymem_setrawallocators", test_pymem_setrawallocators, METH_NOARGS}, {"test_pyobject_new", test_pyobject_new, METH_NOARGS}, {"test_pyobject_setallocators", test_pyobject_setallocators, METH_NOARGS}, + {"test_bytes_resize_tracer", test_bytes_resize_tracer, METH_NOARGS}, // Tracemalloc tests {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 9ee378292cc55e2..0b7ddfdf6b00204 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3221,61 +3221,81 @@ PyBytes_ConcatAndDel(PyObject **pv, PyObject *w) does *not* include that), and a trailing \0 byte is stored. */ +// Similar to _PyBytes_Resize(), but leaves the object unchanged on error. int -_PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) +_PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) { - PyObject *v; - PyBytesObject *sv; - v = *pv; + PyObject *v = *pv; + PyObject *result; + if (!PyBytes_Check(v) || newsize < 0) { - *pv = 0; - Py_DECREF(v); PyErr_BadInternalCall(); return -1; } + Py_ssize_t oldsize = PyBytes_GET_SIZE(v); if (oldsize == newsize) { /* return early if newsize equals to v->ob_size */ return 0; } + if (oldsize == 0) { - *pv = _PyBytes_FromSize(newsize, 0); + result = _PyBytes_FromSize(newsize, 0); + if (result == NULL) { + return -1; + } + *pv = result; Py_DECREF(v); - return (*pv == NULL) ? -1 : 0; + return 0; } + if (newsize == 0) { - *pv = bytes_get_empty(); + *pv = bytes_get_empty(); // cannot fail Py_DECREF(v); return 0; } + if (!_PyObject_IsUniquelyReferenced(v)) { // Allocate and then copy so we don't get a shared immortal // one-character singleton! - *pv = _PyBytes_FromSize(newsize, 0); - if (*pv) { - memcpy(PyBytes_AS_STRING(*pv), PyBytes_AS_STRING(v), - Py_MIN(oldsize, newsize)); + result = _PyBytes_FromSize(newsize, 0); + if (!result) { + return -1; } + + memcpy(PyBytes_AS_STRING(result), PyBytes_AS_STRING(v), + Py_MIN(oldsize, newsize)); + *pv = result; Py_DECREF(v); - return (*pv == NULL) ? -1 : 0; + return 0; + } + + if ((size_t)newsize > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { + PyErr_SetString(PyExc_OverflowError, + "byte string is too large"); + return -1; } #ifdef Py_TRACE_REFS _Py_ForgetReference(v); #endif _PyReftracerTrack(v, PyRefTracer_DESTROY); - *pv = (PyObject *) - PyObject_Realloc(v, PyBytesObject_SIZE + newsize); - if (*pv == NULL) { -#ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyThreadState_GET()); + + result = (PyObject *)PyObject_Realloc(v, PyBytesObject_SIZE + newsize); + if (result == NULL) { +#ifdef Py_TRACE_REFS + _Py_AddToAllObjects(v); #endif - PyObject_Free(v); + _PyReftracerTrack(v, PyRefTracer_CREATE); + PyErr_NoMemory(); return -1; } - _Py_NewReferenceNoTotal(*pv); - sv = (PyBytesObject *) *pv; + + *pv = result; + v = result; + _Py_NewReferenceNoTotal(v); + PyBytesObject *sv = (PyBytesObject *)v; Py_SET_SIZE(sv, newsize); sv->ob_sval[newsize] = '\0'; set_ob_shash(sv, -1); /* invalidate cached hash value */ @@ -3283,6 +3303,19 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) } +int +_PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) +{ + int res = _PyBytes_ResizeKeepOnError(pv, newsize); + if (res < 0) { + PyObject *v = *pv; + *pv = NULL; + Py_DECREF(v); + } + return res; +} + + /*********************** Bytes Iterator ****************************/ typedef struct { _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
