https://github.com/python/cpython/commit/73e63011a06ffd792ca94f8fd5117665fff0030a commit: 73e63011a06ffd792ca94f8fd5117665fff0030a branch: 3.15 author: Victor Stinner <[email protected]> committer: vstinner <[email protected]> date: 2026-10-13T00:18:42+02:00 summary:
[3.15] gh-157242: Fix PyBytesWriter_Resize() on MemoryError (#157243) (#159260) gh-157242: Fix PyBytesWriter_Resize() on MemoryError (#157243) If PyBytesWriter_Resize() fails, leave the writer unchanged. Add a new internal _PyBytes_ResizeKeepOnError() function similar to _PyBytes_Resize() but leaves the bytes object unchanged on error. Add BaseWriterTest for BytesWriterTest and ByteArrayWriterTest. Add _testcapi.PyBytesWriter_small_buffer constant. (cherry picked from commit f5dd52df16e1b3f3f8cc2762f1de783a41d69f4f) Co-authored-by: Maurycy Pawłowski-Wieroński <[email protected]> files: A Misc/NEWS.d/next/C_API/2026-09-10-02-51-26.gh-issue-157242.tfXIsf.rst M Include/internal/pycore_bytesobject.h M Lib/test/test_capi/test_bytes.py M Modules/_testcapi/bytes.c M Modules/_testcapi/mem.c M Objects/bytesobject.c diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index be85d9debd8e96..b039ee022a7aad 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -73,6 +73,8 @@ _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, */ #define _PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) +extern int _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize); + // Return the cached hash value, or -1 if not cached yet. static inline Py_hash_t diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 3d8270ddcf6244..630e9f1d0d72c4 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -465,6 +465,24 @@ def test_example_resize(self): def test_example_highlevel(self): self.assertEqual(_testcapi.byteswriter_highlevel(), b'Hello World!') + def test_resize_error(self): + small_buffer = _testcapi.PyBytesWriter_small_buffer + init = b'x' * (small_buffer * 2) + writer = self.create_writer(len(init), init) + size = len(init) + 100 + try: + with self.assertRaises(MemoryError): + _testcapi.set_nomemory(0) + writer.resize(size, b'') + finally: + _testcapi.remove_mem_hooks() + suffix = b'still working' + writer.write_bytes(suffix, -1) + self.bytes_equal(writer.finish(), init + suffix) + + # Note: PyBytesWriter_Resize() leaves the buffer unchanged (no resize) + # if the new size is smaller than the allocated size + class ByteArrayWriterTest(BaseWriterTest, unittest.TestCase): RESULT_TYPE = bytearray diff --git a/Misc/NEWS.d/next/C_API/2026-09-10-02-51-26.gh-issue-157242.tfXIsf.rst b/Misc/NEWS.d/next/C_API/2026-09-10-02-51-26.gh-issue-157242.tfXIsf.rst new file mode 100644 index 00000000000000..d2e48a651e367a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-10-02-51-26.gh-issue-157242.tfXIsf.rst @@ -0,0 +1,3 @@ +Fix :c:func:`PyBytesWriter_Resize` to handle properly memory allocation +failure. Leave the writer unchanged on error, instead of leaving it in an +inconsistent state. Patch by Victor Stinner. diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index 735d8d67ca14c8..984a857a83db11 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -378,6 +378,7 @@ byteswriter_highlevel(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) static size_t pybyteswriter_small_buffer_size(void) { + // PyBytesWriter.obj is the second member, small_buffer is the first member return offsetof(PyBytesWriter, obj); } @@ -503,10 +504,8 @@ _PyTestCapi_Init_Bytes(PyObject *m) } Py_DECREF(writer_type); - // PyBytesWriter.obj is the second member, small_buffer is the first member long size = (long)pybyteswriter_small_buffer_size(); if (PyModule_AddIntConstant(m, "PyBytesWriter_small_buffer", size) < 0) { - Py_DECREF(writer_type); return -1; } diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index 6aaf0c7d4e942d..a4fbf865913c5e 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -155,18 +155,29 @@ fm_remove_hooks(void) } } +static void +fm_set_nomemory(int start, int stop) +{ + /* Memory allocation fails after 'start' allocation requests, and until + * 'stop' allocation requests except when 'stop' is negative or equal + * to 0 (default) in which case allocation failures never stop. */ + FmData.start = start; + FmData.stop = stop; + FmData.count = 0; + fm_setup_hooks(); +} + static PyObject * set_nomemory(PyObject *self, PyObject *args) { /* Memory allocation fails after 'start' allocation requests, and until * 'stop' allocation requests except when 'stop' is negative or equal * to 0 (default) in which case allocation failures never stop. */ - FmData.count = 0; - FmData.stop = 0; - if (!PyArg_ParseTuple(args, "i|i", &FmData.start, &FmData.stop)) { + int start, stop = 0; + if (!PyArg_ParseTuple(args, "i|i", &start, &stop)) { return NULL; } - fm_setup_hooks(); + fm_set_nomemory(start, stop); Py_RETURN_NONE; } @@ -731,6 +742,129 @@ tracemalloc_track_race(PyObject *self, PyObject *args) } +struct bytes_resize_tracer { + PyObject *create; + PyObject *destroy; +}; + + +static int +bytes_resize_tracer(PyObject *obj, PyRefTracerEvent event, void* data) +{ + if (event != PyRefTracer_CREATE && event != PyRefTracer_DESTROY) { + return 0; + } + + struct bytes_resize_tracer *tracer = (struct bytes_resize_tracer*)data; + if (!PyBytes_Check(obj)) { + return 0; + } + + switch (event) { + case PyRefTracer_CREATE: + tracer->create = obj; + break; + case PyRefTracer_DESTROY: + tracer->destroy = obj; + break; + default: + break; + } + return 0; +} + + +// When _PyBytes_Resize() resizes a bytes object in-place, check that +// PyRefTracer_DESTROY and PyRefTracer_CREATE events are emitted. +// If no_memory is non-zero, inject MemoryError. +static int +check_bytes_resize_tracer(int no_memory) +{ + PyObject *bytes = NULL; + PyRefTracer old_tracer = NULL; + void *old_tracer_data = NULL; + int restore_tracer = 0; + + bytes = PyBytes_FromString("hello"); + if (bytes == NULL) { + goto error; + } + assert(PyUnstable_Object_IsUniquelyReferenced(bytes)); + + old_tracer = PyRefTracer_GetTracer(&old_tracer_data); + restore_tracer = 1; + + struct bytes_resize_tracer tracer = {0}; + if (PyRefTracer_SetTracer(bytes_resize_tracer, &tracer) != 0) { + goto error; + } + + PyObject *old_bytes = bytes; // borrowed reference + if (no_memory) { + fm_set_nomemory(0, 0); + int res = _PyBytes_Resize(&bytes, 100); + assert(res < 0); + assert(bytes == NULL); + fm_remove_hooks(); + + assert(PyErr_ExceptionMatches(PyExc_MemoryError)); + PyErr_Clear(); + } + else { + if (_PyBytes_Resize(&bytes, 100) < 0) { + assert(bytes == NULL); + goto error; + } + } + + if (tracer.destroy != old_bytes) { + PyErr_SetString(PyExc_AssertionError, "PyRefTracer_DESTROY not seen"); + goto error; + } + + int seen_create; + if (no_memory) { + seen_create = (tracer.create == old_bytes); + } + else { + seen_create = (tracer.create == bytes); + } + if (!seen_create) { + PyErr_SetString(PyExc_AssertionError, "PyRefTracer_CREATE not seen"); + goto error; + } + + Py_CLEAR(bytes); + if (PyRefTracer_SetTracer(old_tracer, old_tracer_data) != 0) { + restore_tracer = 0; + goto error; + } + return 0; + +error: + Py_XDECREF(bytes); + if (restore_tracer) { + if (PyRefTracer_SetTracer(old_tracer, old_tracer_data) != 0) { + return -1; + } + } + return -1; +} + + +static PyObject* +test_bytes_resize_tracer(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (check_bytes_resize_tracer(0) < 0) { + return NULL; + } + if (check_bytes_resize_tracer(1) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + static PyMethodDef test_methods[] = { {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, @@ -745,6 +879,7 @@ static PyMethodDef test_methods[] = { {"test_pymem_setrawallocators", test_pymem_setrawallocators, METH_NOARGS}, {"test_pyobject_new", test_pyobject_new, METH_NOARGS}, {"test_pyobject_setallocators", test_pyobject_setallocators, METH_NOARGS}, + {"test_bytes_resize_tracer", test_bytes_resize_tracer, METH_NOARGS}, // Tracemalloc tests {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 7d8d22b32843f0..f9285108a83b34 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3333,43 +3333,53 @@ PyBytes_ConcatAndDel(PyObject **pv, PyObject *w) does *not* include that), and a trailing \0 byte is stored. */ +// Similar to _PyBytes_Resize(), but leaves the object unchanged on error. int -_PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) +_PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) { - PyObject *v; - PyBytesObject *sv; - v = *pv; + PyObject *v = *pv; + PyObject *result; + if (!PyBytes_Check(v) || newsize < 0) { - *pv = 0; - Py_DECREF(v); PyErr_BadInternalCall(); return -1; } + Py_ssize_t oldsize = PyBytes_GET_SIZE(v); if (oldsize == newsize) { /* return early if newsize equals to v->ob_size */ return 0; } + if (oldsize == 0) { - *pv = _PyBytes_FromSize(newsize, 0); + result = _PyBytes_FromSize(newsize, 0); + if (result == NULL) { + return -1; + } + *pv = result; Py_DECREF(v); - return (*pv == NULL) ? -1 : 0; + return 0; } + if (newsize == 0) { - *pv = bytes_get_empty(); + *pv = bytes_get_empty(); // cannot fail Py_DECREF(v); return 0; } + if (!_PyObject_IsUniquelyReferenced(v)) { // Allocate and then copy so we don't get a shared immortal // one-character singleton! - *pv = _PyBytes_FromSize(newsize, 0); - if (*pv) { - memcpy(PyBytes_AS_STRING(*pv), PyBytes_AS_STRING(v), - Py_MIN(oldsize, newsize)); + result = _PyBytes_FromSize(newsize, 0); + if (!result) { + return -1; } + + memcpy(PyBytes_AS_STRING(result), PyBytes_AS_STRING(v), + Py_MIN(oldsize, newsize)); + *pv = result; Py_DECREF(v); - return (*pv == NULL) ? -1 : 0; + return 0; } assert(v != bytes_get_empty()); @@ -3377,18 +3387,22 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) _Py_ForgetReference(v); #endif _PyReftracerTrack(v, PyRefTracer_DESTROY); - *pv = (PyObject *) - PyObject_Realloc(v, PyBytesObject_SIZE + newsize); - if (*pv == NULL) { -#ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyThreadState_GET()); + + result = (PyObject *)PyObject_Realloc(v, PyBytesObject_SIZE + newsize); + if (result == NULL) { +#ifdef Py_TRACE_REFS + _Py_AddToAllObjects(v); #endif - PyObject_Free(v); + _PyReftracerTrack(v, PyRefTracer_CREATE); + PyErr_NoMemory(); return -1; } - _Py_NewReferenceNoTotal(*pv); - sv = (PyBytesObject *) *pv; + + *pv = result; + v = result; + _Py_NewReferenceNoTotal(v); + PyBytesObject *sv = (PyBytesObject *)v; Py_SET_SIZE(sv, newsize); sv->ob_sval[newsize] = '\0'; set_ob_shash(sv, -1); /* invalidate cached hash value */ @@ -3396,6 +3410,19 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) } +int +_PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) +{ + int res = _PyBytes_ResizeKeepOnError(pv, newsize); + if (res < 0) { + PyObject *v = *pv; + *pv = NULL; + Py_DECREF(v); + } + return res; +} + + /*********************** Bytes Iterator ****************************/ typedef struct { @@ -3633,7 +3660,8 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) } } else { - if (_PyBytes_Resize(&writer->obj, size)) { + if (_PyBytes_ResizeKeepOnError(&writer->obj, size)) { + assert(writer->obj != NULL); return -1; } } _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
