https://github.com/python/cpython/commit/63d4398be879e346ef68f899115f86cdb9fa555c
commit: 63d4398be879e346ef68f899115f86cdb9fa555c
branch: main
author: Victor Stinner <[email protected]>
committer: vstinner <[email protected]>
date: 2026-09-12T05:30:56Z
summary:
gh-157242: Add more PyBytesWriter tests (#157352)
* Fix PyBytesWriter_Grow() check on the size argument.
* Fix undefined behavior in _PyBytes_ResizeKeepOnError(): raise OverflowError
on integer overflow.
* Add write(), grow() and get_data() methods to the _testcapi wrapper of
PyBytesWriter.
* Remove the string argument from _testcapi.PyBytesWriter() and
resize() method.
* Add tests on PyBytesWriter_Resize() invalid sizes.
files:
M Lib/test/test_capi/test_bytes.py
M Modules/_testcapi/bytes.c
M Objects/bytesobject.c
diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py
index 799a17617be1a3..1500faa7f71344 100644
--- a/Lib/test/test_capi/test_bytes.py
+++ b/Lib/test/test_capi/test_bytes.py
@@ -317,7 +317,11 @@ def test_join(self):
class BaseWriterTest:
- result_type = NotImplementedError
+ RESULT_TYPE = NotImplementedError
+ SMALL_BUFFER = 11 # bytes
+ assert SMALL_BUFFER < _testcapi.PyBytesWriter_small_buffer
+ LARGE_BUFFER = _testcapi.PyBytesWriter_small_buffer + 17 # bytes
+ NEW_BYTE = b'\xff'
def create_writer(self, alloc=0, string=b''):
raise NotImplementedError
@@ -326,85 +330,174 @@ def test_create(self):
# Test PyBytesWriter_Create()
writer = self.create_writer()
self.assertEqual(writer.get_size(), 0)
- self.assertEqual(writer.finish(), self.result_type(b''))
+ self.assertEqual(writer.finish(), b'')
- writer = self.create_writer(3, b'abc')
+ writer = self.create_writer(3)
+ writer.write(0, b'abc')
self.assertEqual(writer.get_size(), 3)
- self.assertEqual(writer.finish(), self.result_type(b'abc'))
+ result = writer.finish()
+ self.assertEqual(result, b'abc')
+ self.assertEqual(type(result), self.RESULT_TYPE)
+
+ @unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
+ def test_get_data(self):
+ # Test PyBytesWriter_GetData()
+ writer = self.create_writer(6)
+ NEW_BYTE = self.NEW_BYTE
+ self.assertEqual(writer.get_data(), NEW_BYTE * 6)
+ writer.write(0, b'abc')
+ self.assertEqual(writer.get_data(), b'abc' + NEW_BYTE * 3)
+ writer.write(3, b'123')
+ self.assertEqual(writer.get_data(), b'abc123')
+
+ # Switch from small buffer to large buffer
+ small, large = self.SMALL_BUFFER, self.LARGE_BUFFER
+ writer = self.create_writer(small)
+ self.assertEqual(writer.get_data(), NEW_BYTE * small)
+ writer.write(0, b's' * small)
+ self.assertEqual(writer.get_data(), b's' * small)
+ writer.resize(large)
+ self.assertEqual(writer.get_data(), b's' * small + NEW_BYTE * (large -
small))
+ writer.write(small, b'L' * (large - small))
+ self.assertEqual(writer.get_data(), b's' * small + b'L' * (large -
small))
+
+ # Resize large buffer
+ small, large = self.SMALL_BUFFER, self.LARGE_BUFFER
+ writer = self.create_writer(large)
+ self.assertEqual(writer.get_data(), NEW_BYTE * large)
+ writer.write(0, b'L' * large)
+ self.assertEqual(writer.get_data(), b'L' * large)
+ writer.resize(large + 10)
+ self.assertEqual(writer.get_data(), b'L' * large + NEW_BYTE * 10)
+ writer.write(large, b'#' * 10)
+ self.assertEqual(writer.get_data(), b'L' * large + b'#' * 10)
def test_finish_with_size(self):
# Test PyBytesWriter_FinishWithSize()
- writer = self.create_writer(10, b'abcdef')
+ writer = self.create_writer(10)
+ writer.write(0, b'abc123')
self.assertEqual(writer.get_size(), 10)
- self.assertEqual(writer.finish_with_size(3), self.result_type(b'abc'))
+ result = writer.finish_with_size(3)
+ self.assertEqual(result, b'abc')
+ self.assertEqual(type(result), self.RESULT_TYPE)
# Error if the size is negative
- writer = self.create_writer(3, b'abc')
+ writer = self.create_writer(3, )
+ writer.write(0, b'abc')
with self.assertRaises(ValueError):
writer.finish_with_size(-3)
# Error if the requested size is larger than the allocated size
- writer = self.create_writer(3, b'abc')
+ writer = self.create_writer(3)
+ writer.write(0, b'abc')
with self.assertRaises(ValueError):
writer.finish_with_size(4)
def test_write_bytes(self):
- # Test PyBytesWriter_WriteBytes()
- writer = self.create_writer()
- writer.write_bytes(b'Hello World!', -1)
- self.assertEqual(writer.finish(), self.result_type(b'Hello World!'))
+ # Test PyBytesWriter_WriteBytes()
+ writer = self.create_writer()
+ writer.write_bytes(b'Hello World!', -1)
+ self.assertEqual(writer.finish(), b'Hello World!')
- writer = self.create_writer()
- writer.write_bytes(b'Hello ', -1)
- writer.write_bytes(b'World! <truncated>', 6)
- self.assertEqual(writer.finish(), self.result_type(b'Hello World!'))
+ writer = self.create_writer()
+ writer.write_bytes(b'Hello ', -1)
+ writer.write_bytes(b'World! <truncated>', 6)
+ self.assertEqual(writer.finish(), b'Hello World!')
def test_resize(self):
# Test PyBytesWriter_Resize()
writer = self.create_writer()
- writer.resize(len(b'number=123456'), b'number=123456')
- writer.resize(len(b'number=123456'), b'')
- self.assertEqual(writer.get_size(), len(b'number=123456'))
- self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
+ writer.resize(len(b'hello'))
+ writer.write(0, b'hello')
+ self.assertEqual(writer.finish(), b'hello')
writer = self.create_writer()
- writer.resize(0, b'')
- writer.resize(len(b'number=123456'), b'number=123456')
- self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
-
+ writer.resize(0) # noop
+ writer.resize(len(b'number'))
+ writer.write(0, b'number')
+ writer.resize(len(b'number='))
+ writer.write(len(b'number'), b'=')
+ writer.resize(len(b'number=123'), )
+ writer.write(len(b'number='), b'123')
+ writer.resize(len(b'number=123')) # noop
+ self.assertEqual(writer.finish(), b'number=123')
+
+ # Switch from small buffer to large buffer
writer = self.create_writer()
- writer.resize(len(b'number='), b'number=')
- writer.resize(len(b'number=123456'), b'123456')
- self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
+ small, large = self.SMALL_BUFFER, self.LARGE_BUFFER
+ writer.resize(small)
+ writer.write(0, b's' * small)
+ writer.resize(large)
+ writer.write(small, b'L' * (large - small))
+ self.assertEqual(writer.finish(),
+ b's' * small + b'L' * (large - small))
+
+ # invalid size
+ for size in (self.SMALL_BUFFER, self.LARGE_BUFFER):
+ with self.subTest(size=size):
+ writer = self.create_writer()
+ writer.write_bytes(b'x' * size, -1)
+ with self.assertRaisesRegex(ValueError, 'size must be >= 0'):
+ writer.resize(-1)
+ with self.assertRaises((MemoryError, OverflowError)):
+ writer.resize(_testcapi.PY_SSIZE_T_MAX)
+ self.assertEqual(writer.finish(), b'x' * size)
+
+ def test_grow(self):
+ # Test PyBytesWriter_Grow()
+ writer = self.create_writer(0)
+ writer.grow(len(b'number=123'))
+ writer.write(0, b'number=123')
+ self.assertEqual(writer.finish(), b'number=123')
writer = self.create_writer()
- writer.resize(len(b'number='), b'number=')
- writer.resize(len(b'number='), b'')
- writer.resize(len(b'number=123456'), b'123456')
- self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
-
+ writer.grow(0) # noop
+ writer.grow(len(b'number'))
+ writer.write(0, b'number')
+ writer.grow(len(b'='))
+ writer.write(len(b'number'), b'=')
+ writer.grow(len(b'123'), )
+ writer.write(len(b'number='), b'123')
+ writer.grow(0) # noop
+ self.assertEqual(writer.finish(), b'number=123')
+
+ # Switch from small buffer to large buffer
writer = self.create_writer()
- writer.resize(len(b'number'), b'number')
- writer.resize(len(b'number='), b'=')
- writer.resize(len(b'number=123'), b'123')
- writer.resize(len(b'number=123456'), b'456')
- self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
+ small, large = self.SMALL_BUFFER, self.LARGE_BUFFER
+ writer.grow(small)
+ writer.write(0, b's' * small)
+ writer.grow(large - small)
+ writer.write(small, b'L' * (large - small))
+ self.assertEqual(writer.finish(),
+ b's' * small + b'L' * (large - small))
+
+ # invalid size
+ for size in (self.SMALL_BUFFER, self.LARGE_BUFFER):
+ with self.subTest(size=size):
+ writer = self.create_writer()
+ writer.write_bytes(b'x' * size, -1)
+ with self.assertRaisesRegex(ValueError, 'size must be >= 0'):
+ writer.grow(-1)
+ with self.assertRaises(MemoryError):
+ writer.grow(_testcapi.PY_SSIZE_T_MAX)
+ self.assertEqual(writer.finish(), b'x' * size)
@support.nomemtest
def test_resize_error(self):
- small_buffer = _testcapi.PyBytesWriter_small_buffer
- init = b'x' * (small_buffer * 2)
- writer = self.create_writer(len(init), init)
+ # Test PyBytesWriter_Resize() error
+ init = b'x' * self.LARGE_BUFFER
+ writer = self.create_writer(len(init))
+ writer.write(0, init)
size = len(init) + 100
try:
with self.assertRaises(MemoryError):
_testcapi.set_nomemory(0)
- writer.resize(size, b'')
+ writer.resize(size)
finally:
_testcapi.remove_mem_hooks()
suffix = b'still working'
writer.write_bytes(suffix, -1)
- self.assertEqual(writer.finish(), self.result_type(init + suffix))
+ self.assertEqual(writer.finish(), init + suffix)
# Note: PyBytesWriter_Resize() leaves the buffer unchanged (no resize)
# if the new size is smaller than the allocated size
@@ -413,48 +506,54 @@ def test_format_i(self):
# Test PyBytesWriter_Format()
writer = self.create_writer()
writer.format_i(b'x=%i', 123456)
- self.assertEqual(writer.finish(), self.result_type(b'x=123456'))
+ self.assertEqual(writer.finish(), b'x=123456')
writer = self.create_writer()
writer.format_i(b'x=%i, ', 123)
writer.format_i(b'y=%i', 456)
- self.assertEqual(writer.finish(), self.result_type(b'x=123, y=456'))
+ self.assertEqual(writer.finish(), b'x=123, y=456')
class BytesWriterTest(BaseWriterTest, unittest.TestCase):
- result_type = bytes
+ RESULT_TYPE = bytes
- def create_writer(self, alloc=0, string=b''):
+ def create_writer(self, size=0):
# Test PyBytesWriter_Create()
- return _testcapi.PyBytesWriter(alloc, string, 0)
+ return _testcapi.PyBytesWriter(size, 0)
# Only PyBytesWriter_Create() returns singletons
def test_singletons(self):
empty = b''
singletons = {ch: bytes((ch,)) for ch in range(256)}
- small_buffer = _testcapi.PyBytesWriter_small_buffer
writer = self.create_writer()
self.assertIs(writer.finish(), empty)
- # Test writer larger than small_buffer
+ # Large buffer
writer = self.create_writer()
- unused_text = b'x' * (small_buffer * 2)
+ unused_text = b'x' * self.LARGE_BUFFER
writer.write_bytes(unused_text, len(unused_text))
self.assertIs(writer.finish_with_size(0), empty)
+ # Large buffer with resize
+ writer = self.create_writer()
+ unused_text = b'x' * self.LARGE_BUFFER
+ writer.write_bytes(unused_text, len(unused_text))
+ writer.resize(0)
+ self.assertIs(writer.finish(), empty)
+
for ch in range(256):
- text = bytes((ch,))
+ byte = bytes((ch,))
writer = self.create_writer()
- writer.write_bytes(text, 1)
+ writer.write_bytes(byte, len(byte))
self.assertIs(writer.finish(), singletons[ch])
- # Test writer larger than small_buffer
+ # Large buffer
writer = self.create_writer()
- writer.write_bytes(text, 1)
- unused_text = b'x' * (small_buffer * 2)
- writer.write_bytes(unused_text, len(unused_text))
+ unused_text = b'x' * self.LARGE_BUFFER
+ writer.write_bytes(byte + unused_text,
+ len(byte) + len(unused_text))
self.assertIs(writer.finish_with_size(1), singletons[ch])
def test_example_abc(self):
@@ -468,11 +567,11 @@ def test_example_highlevel(self):
class ByteArrayWriterTest(BaseWriterTest, unittest.TestCase):
- result_type = bytearray
+ RESULT_TYPE = bytearray
- def create_writer(self, alloc=0, string=b''):
+ def create_writer(self, size=0):
# Test private _PyBytesWriter_CreateByteArray()
- return _testcapi.PyBytesWriter(alloc, string, 1)
+ return _testcapi.PyBytesWriter(size, 1)
if __name__ == "__main__":
diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c
index 4830cc8b54bd83..83249a21c5a3f2 100644
--- a/Modules/_testcapi/bytes.c
+++ b/Modules/_testcapi/bytes.c
@@ -87,34 +87,25 @@ writer_init(PyObject *self_raw, PyObject *args, PyObject
*kwargs)
return -1;
}
- Py_ssize_t alloc;
- char *str;
- Py_ssize_t str_size;
- int use_bytearray;
- if (!PyArg_ParseTuple(args, "ny#i",
- &alloc, &str, &str_size, &use_bytearray)) {
+ Py_ssize_t size;
+ int use_bytearray = 0;
+ if (!PyArg_ParseTuple(args, "n|i", &size, &use_bytearray)) {
return -1;
}
WriterObject *self = (WriterObject *)self_raw;
- if (self->writer) {
- PyBytesWriter_Discard(self->writer);
- }
+ PyBytesWriter_Discard(self->writer);
+
if (use_bytearray) {
- self->writer = _PyBytesWriter_CreateByteArray(alloc);
+ self->writer = _PyBytesWriter_CreateByteArray(size);
}
else {
- self->writer = PyBytesWriter_Create(alloc);
+ self->writer = PyBytesWriter_Create(size);
}
if (self->writer == NULL) {
return -1;
}
- if (str_size) {
- char *buf = PyBytesWriter_GetData(self->writer);
- memcpy(buf, str, str_size);
- }
-
return 0;
}
@@ -143,6 +134,32 @@ writer_check(WriterObject *self)
}
+static PyObject*
+writer_write(PyObject *self_raw, PyObject *args)
+{
+ WriterObject *self = (WriterObject *)self_raw;
+ if (writer_check(self) < 0) {
+ return NULL;
+ }
+
+ Py_ssize_t pos, size;
+ char *str;
+ if (!PyArg_ParseTuple(args, "ny#", &pos, &str, &size)) {
+ return NULL;
+ }
+
+ if (pos < 0 || (pos + size) > PyBytesWriter_GetSize(self->writer)) {
+ PyErr_SetString(PyExc_ValueError, "invalid position or size");
+ return NULL;
+ }
+
+ char *data = PyBytesWriter_GetData(self->writer);
+ memcpy(data + pos, str, size);
+
+ Py_RETURN_NONE;
+}
+
+
static PyObject*
writer_write_bytes(PyObject *self_raw, PyObject *args)
{
@@ -185,6 +202,7 @@ writer_format_i(PyObject *self_raw, PyObject *args)
}
+// PyBytesWriter_Resize
static PyObject*
writer_resize(PyObject *self_raw, PyObject *args)
{
@@ -194,27 +212,52 @@ writer_resize(PyObject *self_raw, PyObject *args)
}
Py_ssize_t size;
- char *str;
- Py_ssize_t str_size;
- if (!PyArg_ParseTuple(args,
- "ny#",
- &size, &str, &str_size)) {
+ if (!PyArg_ParseTuple(args, "n", &size)) {
return NULL;
}
- assert(size >= str_size);
- Py_ssize_t pos = PyBytesWriter_GetSize(self->writer);
if (PyBytesWriter_Resize(self->writer, size) < 0) {
return NULL;
}
+ Py_RETURN_NONE;
+}
+
- char *buf = PyBytesWriter_GetData(self->writer);
- memcpy(buf + pos, str, str_size);
+// Test PyBytesWriter_Grow()
+static PyObject*
+writer_grow(PyObject *self_raw, PyObject *args)
+{
+ WriterObject *self = (WriterObject *)self_raw;
+ if (writer_check(self) < 0) {
+ return NULL;
+ }
+ Py_ssize_t size;
+ if (!PyArg_ParseTuple(args, "n", &size)) {
+ return NULL;
+ }
+
+ if (PyBytesWriter_Grow(self->writer, size) < 0) {
+ return NULL;
+ }
Py_RETURN_NONE;
}
+static PyObject*
+writer_get_data(PyObject *self_raw, PyObject *Py_UNUSED(args))
+{
+ WriterObject *self = (WriterObject *)self_raw;
+ if (writer_check(self) < 0) {
+ return NULL;
+ }
+
+ const char *data = PyBytesWriter_GetData(self->writer);
+ Py_ssize_t size = PyBytesWriter_GetSize(self->writer);
+ return PyBytes_FromStringAndSize(data, size);
+}
+
+
static PyObject*
writer_get_size(PyObject *self_raw, PyObject *Py_UNUSED(args))
{
@@ -223,8 +266,8 @@ writer_get_size(PyObject *self_raw, PyObject
*Py_UNUSED(args))
return NULL;
}
- Py_ssize_t alloc = PyBytesWriter_GetSize(self->writer);
- return PyLong_FromSsize_t(alloc);
+ Py_ssize_t size = PyBytesWriter_GetSize(self->writer);
+ return PyLong_FromSsize_t(size);
}
@@ -262,9 +305,12 @@ writer_finish_with_size(PyObject *self_raw, PyObject *args)
static PyMethodDef writer_methods[] = {
+ {"write", _PyCFunction_CAST(writer_write), METH_VARARGS},
{"write_bytes", _PyCFunction_CAST(writer_write_bytes), METH_VARARGS},
{"format_i", _PyCFunction_CAST(writer_format_i), METH_VARARGS},
{"resize", _PyCFunction_CAST(writer_resize), METH_VARARGS},
+ {"grow", _PyCFunction_CAST(writer_grow), METH_VARARGS},
+ {"get_data", _PyCFunction_CAST(writer_get_data), METH_NOARGS},
{"get_size", _PyCFunction_CAST(writer_get_size), METH_NOARGS},
{"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS},
{"finish_with_size", _PyCFunction_CAST(writer_finish_with_size),
METH_VARARGS},
@@ -313,8 +359,9 @@ byteswriter_resize(PyObject *Py_UNUSED(module), PyObject
*Py_UNUSED(args))
char *buf = PyBytesWriter_GetData(writer);
// Write some bytes
- memcpy(buf, "Hello ", strlen("Hello "));
- buf += strlen("Hello ");
+ const char *hello = "Hello ";
+ memcpy(buf, hello, strlen(hello));
+ buf += strlen(hello);
// Allocate 10 more bytes
buf = PyBytesWriter_GrowAndUpdatePointer(writer, 10, buf);
@@ -324,8 +371,9 @@ byteswriter_resize(PyObject *Py_UNUSED(module), PyObject
*Py_UNUSED(args))
}
// Write more bytes
- memcpy(buf, "World", strlen("World"));
- buf += strlen("World");
+ const char *world = "World";
+ memcpy(buf, world, strlen(world));
+ buf += strlen(world);
// Truncate to the exact size and create a bytes object
return PyBytesWriter_FinishWithPointer(writer, buf);
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index b84fdcd0ecc015..a5ec05764b2b70 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -3356,6 +3356,12 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t
newsize)
}
assert(v != bytes_get_empty());
+ if ((size_t)newsize > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) {
+ PyErr_SetString(PyExc_OverflowError,
+ "byte string is too large");
+ return -1;
+ }
+
#ifdef Py_TRACE_REFS
_Py_ForgetReference(v);
#endif
@@ -3580,6 +3586,8 @@ _PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest,
// --- PyBytesWriter API -----------------------------------------------------
+#define PyBytesWrite_NEW_BYTE 0xff
+
static inline char*
byteswriter_data(PyBytesWriter *writer)
{
@@ -3617,6 +3625,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t
size, int resize)
Py_ssize_t old_allocated = byteswriter_allocated(writer);
if (size <= old_allocated) {
+ // Do not shrink the buffer before PyBytesWriter_FinishWithSize()
return 0;
}
@@ -3633,6 +3642,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t
size, int resize)
}
}
else {
+ // Can raise MemoryError or OverflowError
if (_PyBytes_ResizeKeepOnError(&writer->obj, size)) {
assert(writer->obj != NULL);
return -1;
@@ -3668,7 +3678,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t
size, int resize)
#ifdef Py_DEBUG
Py_ssize_t allocated = byteswriter_allocated(writer);
if (resize && allocated > old_allocated) {
- memset(byteswriter_data(writer) + old_allocated, 0xff,
+ memset(byteswriter_data(writer) + old_allocated, PyBytesWrite_NEW_BYTE,
allocated - old_allocated);
}
#endif
@@ -3706,7 +3716,8 @@ byteswriter_create(Py_ssize_t size, int use_bytearray)
writer->size = size;
}
#ifdef Py_DEBUG
- memset(byteswriter_data(writer), 0xff, byteswriter_allocated(writer));
+ memset(byteswriter_data(writer), PyBytesWrite_NEW_BYTE,
+ byteswriter_allocated(writer));
#endif
return writer;
}
@@ -3857,10 +3868,15 @@ _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter
*writer, Py_ssize_t size,
int
PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t size)
{
- if (size < 0 && writer->size + size < 0) {
- PyErr_SetString(PyExc_ValueError, "invalid size");
+ if (size < 0) {
+ PyErr_SetString(PyExc_ValueError, "size must be >= 0");
return -1;
}
+ if (size == 0) {
+ // Nothing to do
+ return 0;
+ }
+
if (size > PY_SSIZE_T_MAX - writer->size) {
PyErr_NoMemory();
return -1;
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
