https://github.com/python/cpython/commit/854295809ad5a42c9461f34d47b1252a3b4a2027
commit: 854295809ad5a42c9461f34d47b1252a3b4a2027
branch: main
author: Victor Stinner <[email protected]>
committer: vstinner <[email protected]>
date: 2026-09-18T00:32:50Z
summary:
gh-157710: Add _PyUnicodeWriter_CanWrite() function (#157712)
Make sure that a writer can be modified before writing into it.
* Document that PyUnicodeWriter is not thread safe.
* Add singletons tests to test_capi.test_unicode.
* Check PyUnicode_CheckExact() earlier in _PyUnicode_IsModifiable().
files:
M Doc/c-api/unicode.rst
M Include/internal/pycore_unicodeobject.h
M Lib/test/test_capi/test_unicode.py
M Objects/longobject.c
M Objects/unicode_writer.c
M Objects/unicodeobject.c
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index 9bf801ad608c773..8ac4e1377095554 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -1797,6 +1797,9 @@ object.
The instance must be destroyed by :c:func:`PyUnicodeWriter_Finish` on
success, or :c:func:`PyUnicodeWriter_Discard` on error.
+ The API is **not thread safe**. To share a writer with multiple threads, a
+ critical section or a lock is needed.
+
.. c:function:: PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length)
Create a Unicode writer instance.
diff --git a/Include/internal/pycore_unicodeobject.h
b/Include/internal/pycore_unicodeobject.h
index 012f5da2869cd53..acd6bcf0813f9fd 100644
--- a/Include/internal/pycore_unicodeobject.h
+++ b/Include/internal/pycore_unicodeobject.h
@@ -16,7 +16,9 @@ extern "C" {
#define _Py_MAX_UNICODE 0x10ffff
-extern int _PyUnicode_IsModifiable(PyObject *unicode);
+// Export for '_multibytecodec' shared extension. _PyUnicodeWriter_CanWrite()
+// calls this function when assertions are enabled.
+PyAPI_FUNC(int) _PyUnicode_IsModifiable(PyObject *unicode);
extern void _PyUnicodeWriter_InitWithBuffer(
_PyUnicodeWriter *writer,
PyObject *buffer);
@@ -105,12 +107,31 @@ _PyUnicode_EnsureUnicode(PyObject *obj)
return 0;
}
+#ifndef NDEBUG
+static inline int
+_PyUnicodeWriter_CanWrite(_PyUnicodeWriter *writer)
+{
+ // Code adapted from _PyUnicode_IsModifiable()
+ assert(!writer->readonly);
+ PyObject *buffer = writer->buffer;
+ assert(buffer != NULL);
+ // Do not use _PyObject_IsUniquelyReferenced(): the caller can have its own
+ // lock to prevent a writer being used by two theads at the same time.
+ assert(Py_REFCNT(buffer) == 1);
+ assert(PyUnstable_Unicode_GET_CACHED_HASH(buffer) == -1);
+ assert(!PyUnicode_CHECK_INTERNED(buffer));
+ assert(!_Py_IsImmortal(buffer));
+ return 1;
+}
+#endif
+
static inline int
_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch)
{
assert(ch <= _Py_MAX_UNICODE);
if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0)
return -1;
+ assert(_PyUnicodeWriter_CanWrite(writer));
PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch);
writer->pos++;
return 0;
diff --git a/Lib/test/test_capi/test_unicode.py
b/Lib/test/test_capi/test_unicode.py
index 0dcd8a25ad0128d..b74bec15edcd93a 100644
--- a/Lib/test/test_capi/test_unicode.py
+++ b/Lib/test/test_capi/test_unicode.py
@@ -1981,6 +1981,17 @@ def test_substring_empty(self):
writer.write_substring("abc", 1, 1)
self.assertEqual(writer.finish(), '')
+ def test_singletons(self):
+ writer = self.create_writer(5)
+ self.assertIs(writer.finish(), '')
+
+ for ch in range(256):
+ with self.subTest(ch=ch):
+ ch = chr(ch)
+ writer = self.create_writer(0)
+ writer.write_substring(ch + 'xxx', 0, 1)
+ self.assertIs(writer.finish(), ch)
+
@unittest.skipIf(ctypes is None, 'need ctypes')
class PyUnicodeWriterFormatTest(unittest.TestCase):
diff --git a/Objects/longobject.c b/Objects/longobject.c
index e35f938629326ab..1dac70820d142b6 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -2220,6 +2220,7 @@ long_to_decimal_string_internal(PyObject *aa,
Py_DECREF(scratch);
return -1;
}
+ assert(_PyUnicodeWriter_CanWrite(writer));
}
else if (bytes_writer) {
*bytes_str = PyBytesWriter_GrowAndUpdatePointer(bytes_writer, strlen,
@@ -2390,8 +2391,10 @@ long_format_binary(PyObject *aa, int base, int alternate,
}
if (writer) {
- if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1)
+ if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) {
return -1;
+ }
+ assert(_PyUnicodeWriter_CanWrite(writer));
}
else if (bytes_writer) {
*bytes_str = PyBytesWriter_GrowAndUpdatePointer(bytes_writer, sz,
diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c
index a753c9b971c702c..b10d9e94098935a 100644
--- a/Objects/unicode_writer.c
+++ b/Objects/unicode_writer.c
@@ -350,6 +350,8 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer,
PyObject *str)
if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1)
return -1;
}
+
+ assert(_PyUnicodeWriter_CanWrite(writer));
_PyUnicode_FastCopyCharacters(writer->buffer, writer->pos,
str, 0, len);
writer->pos += len;
@@ -428,6 +430,7 @@ _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer,
PyObject *str,
if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) {
return -1;
}
+ assert(_PyUnicodeWriter_CanWrite(writer));
_PyUnicode_FastCopyCharacters(writer->buffer, writer->pos,
str, start, len);
@@ -485,8 +488,10 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer,
return 0;
}
- if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1)
+ if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1) {
return -1;
+ }
+ assert(_PyUnicodeWriter_CanWrite(writer));
switch (writer->kind)
{
@@ -591,6 +596,7 @@ _PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer,
maxchar = ucs1lib_find_max_char((const Py_UCS1*)str, (const Py_UCS1*)str +
len);
if (_PyUnicodeWriter_Prepare(writer, len, maxchar) == -1)
return -1;
+ assert(_PyUnicodeWriter_CanWrite(writer));
unicode_write_cstr(writer->buffer, writer->pos, str, len);
writer->pos += len;
return 0;
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 86b9baadd0d8aa9..1e687e7a36a8818 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -1743,18 +1743,21 @@ unicode_is_singleton(PyObject *unicode)
}
#endif
+// If this function is updated, update also _PyUnicodeWriter_CanWrite().
int
_PyUnicode_IsModifiable(PyObject *unicode)
{
assert(_PyUnicode_CHECK(unicode));
+ if (!PyUnicode_CheckExact(unicode))
+ return 0;
+ // On Free Threading, this test fails if called from a thread other
+ // than the one which created the str object.
if (!_PyObject_IsUniquelyReferenced(unicode))
return 0;
if (PyUnicode_HASH(unicode) != -1)
return 0;
if (PyUnicode_CHECK_INTERNED(unicode))
return 0;
- if (!PyUnicode_CheckExact(unicode))
- return 0;
#ifdef Py_DEBUG
/* singleton refcount is greater than 1 */
assert(!unicode_is_singleton(unicode));
@@ -2008,6 +2011,7 @@ PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *pub_writer,
if (_PyUnicodeWriter_Prepare(writer, size - num_surrogates, maxchar) < 0) {
return -1;
}
+ assert(_PyUnicodeWriter_CanWrite(writer));
int kind = writer->kind;
void *data = (Py_UCS1*)writer->data + writer->pos * kind;
@@ -2266,6 +2270,7 @@ PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *pub_writer,
if (_PyUnicodeWriter_Prepare(writer, size, max_char) < 0) {
return -1;
}
+ assert(_PyUnicodeWriter_CanWrite(writer));
int kind = writer->kind;
void *data = (Py_UCS1*)writer->data + writer->pos * kind;
@@ -2552,8 +2557,10 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer,
PyObject *str,
else
maxchar = writer->maxchar;
- if (_PyUnicodeWriter_Prepare(writer, arglen, maxchar) == -1)
+ if (_PyUnicodeWriter_Prepare(writer, arglen, maxchar) == -1) {
return -1;
+ }
+ assert(_PyUnicodeWriter_CanWrite(writer));
fill = Py_MAX(width - length, 0);
if (fill && !(flags & F_LJUST)) {
@@ -2843,8 +2850,10 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer,
Py_ssize_t spacepad = Py_MAX(width - precision - sign, 0);
Py_ssize_t zeropad = Py_MAX(precision - len, 0);
- if (_PyUnicodeWriter_Prepare(writer, width, 127) == -1)
+ if (_PyUnicodeWriter_Prepare(writer, width, 127) == -1) {
return NULL;
+ }
+ assert(_PyUnicodeWriter_CanWrite(writer));
if (spacepad && !(flags & F_LJUST)) {
if (PyUnicode_Fill(writer->buffer, writer->pos, spacepad, ' ') ==
-1)
@@ -5371,6 +5380,7 @@ _PyUnicode_DecodeUTF8Writer(_PyUnicodeWriter *writer,
if (_PyUnicodeWriter_Prepare(writer, size, 127) < 0) {
return -1;
}
+ assert(_PyUnicodeWriter_CanWrite(writer));
const char *starts = s;
const char *end = s + size;
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
