https://github.com/python/cpython/commit/21223c9ff592d65c362ffa62a9592fe87d07610a
commit: 21223c9ff592d65c362ffa62a9592fe87d07610a
branch: main
author: Victor Stinner <[email protected]>
committer: vstinner <[email protected]>
date: 2026-09-21T10:32:18+02:00
summary:
gh-157710: Enable read-only optimization in PyUnicodeWriter (#157861)
If the first and only write to a PyUnicodeWriter is a Python str
object, PyUnicodeWriter_Finish() returns the object unchanged.
PyUnicodeWriter_WriteChar() uses a singleton if no buffer was
allocated yet.
Move test_unicode_equal() to the correct test case (CAPITest).
files:
M Include/internal/pycore_unicodeobject.h
M Lib/test/test_capi/test_unicode.py
M Objects/unicode_writer.c
M Objects/unicodeobject.c
diff --git a/Include/internal/pycore_unicodeobject.h
b/Include/internal/pycore_unicodeobject.h
index e9a4aed37030e76..ff3fda8583133e2 100644
--- a/Include/internal/pycore_unicodeobject.h
+++ b/Include/internal/pycore_unicodeobject.h
@@ -10,6 +10,7 @@ extern "C" {
#include "pycore_fileutils.h" // _Py_error_handler
#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
+#include "pycore_runtime.h" // _Py_LATIN1_CHR()
// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111).
@@ -111,10 +112,12 @@ _PyUnicode_EnsureUnicode(PyObject *obj)
static inline int
_PyUnicodeWriter_CanWrite(_PyUnicodeWriter *writer)
{
- // Code adapted from _PyUnicode_IsModifiable()
assert(!writer->readonly);
+
PyObject *buffer = writer->buffer;
assert(buffer != NULL);
+
+ // Code adapted from _PyUnicode_IsModifiable().
// Do not use _PyObject_IsUniquelyReferenced(): the caller can have its own
// lock to prevent a writer from being used by two threads at the same
// time.
@@ -126,13 +129,48 @@ _PyUnicodeWriter_CanWrite(_PyUnicodeWriter *writer)
}
#endif
+static inline void
+_PyUnicodeWriter_Update(_PyUnicodeWriter *writer)
+{
+ PyObject *buffer = writer->buffer;
+ writer->maxchar = PyUnicode_MAX_CHAR_VALUE(buffer);
+ writer->data = PyUnicode_DATA(buffer);
+ writer->kind = PyUnicode_KIND(buffer);
+
+ if (!writer->readonly) {
+ writer->size = PyUnicode_GET_LENGTH(buffer);
+ }
+ else {
+ /* Copy-on-write mode: set buffer size to 0 so
+ * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on
+ * next write. */
+ writer->size = 0;
+ }
+}
+
static inline int
_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch)
{
- assert(ch <= _Py_MAX_UNICODE);
- if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0)
- return -1;
+ if (ch > writer->maxchar || 1 > writer->size - writer->pos) {
+ if (writer->buffer == NULL && ch <= 255) {
+ // If the first write is a Latin1 character, use the singleton
+ // as a read-only object
+ PyObject *obj = _Py_LATIN1_CHR(ch);
+ writer->readonly = 1;
+ writer->buffer = obj; // Py_NewRef() is not need on immortal object
+ _PyUnicodeWriter_Update(writer);
+ assert(writer->pos == 0);
+ writer->pos = 1;
+ // The next write will create a new buffer and copy the string
+ return 0;
+ }
+
+ if (_PyUnicodeWriter_PrepareInternal(writer, 1, ch) == -1) {
+ return -1;
+ }
+ }
assert(_PyUnicodeWriter_CanWrite(writer));
+
PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch);
writer->pos++;
return 0;
diff --git a/Lib/test/test_capi/test_unicode.py
b/Lib/test/test_capi/test_unicode.py
index f13ad6f428ec095..9bfb148f87b585d 100644
--- a/Lib/test/test_capi/test_unicode.py
+++ b/Lib/test/test_capi/test_unicode.py
@@ -1817,6 +1817,39 @@ def test_is_compact_ascii(self):
# CRASHES is_compact_ascii(NULL)
+ def test_unicode_equal(self):
+ unicode_equal = _testlimitedcapi.unicode_equal
+
+ def copy(text):
+ return text.encode().decode()
+
+ self.assertTrue(unicode_equal("", ""))
+ self.assertTrue(unicode_equal("abc", "abc"))
+ self.assertTrue(unicode_equal("abc", copy("abc")))
+ self.assertTrue(unicode_equal("\u20ac", copy("\u20ac")))
+ self.assertTrue(unicode_equal("\U0010ffff", copy("\U0010ffff")))
+
+ self.assertFalse(unicode_equal("abc", "abcd"))
+ self.assertFalse(unicode_equal("\u20ac", "\u20ad"))
+ self.assertFalse(unicode_equal("\U0010ffff", "\U0010fffe"))
+
+ # str subclass
+ self.assertTrue(unicode_equal("abc", Str("abc")))
+ self.assertTrue(unicode_equal(Str("abc"), "abc"))
+ self.assertFalse(unicode_equal("abc", Str("abcd")))
+ self.assertFalse(unicode_equal(Str("abc"), "abcd"))
+
+ # invalid type
+ for invalid_type in (b'bytes', 123, ("tuple",)):
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ unicode_equal("abc", invalid_type)
+ with self.assertRaises(TypeError):
+ unicode_equal(invalid_type, "abc")
+
+ # CRASHES unicode_equal("abc", NULL)
+ # CRASHES unicode_equal(NULL, "abc")
+
class PyUnicodeWriterTest(unittest.TestCase):
def create_writer(self, size):
@@ -1865,6 +1898,11 @@ def test_write_char(self):
self.assertEqual(writer.finish(),
"\0$\u20AC\U0010FFFF")
+ writer = self.create_writer(0)
+ for ch in 'hello':
+ writer.write_char(ord(ch))
+ self.assertEqual(writer.finish(), 'hello')
+
def test_utf8(self):
writer = self.create_writer(0)
writer.write_utf8(b"ascii", -1)
@@ -2057,14 +2095,25 @@ def test_singletons(self):
writer.write_substring('text', 0, 0)
self.assertIs(writer.finish(), '')
- for ch in range(256):
- with self.subTest(ch=ch):
- ch = chr(ch)
- writer = self.create_writer(0)
- # Use PyUnicodeWriter_WriteSubstring() to avoid the read-only
- # buffer optimization
- writer.write_substring(ch + 'xxx', 0, 1)
- self.assertIs(writer.finish(), ch)
+ for size in (0, 123):
+ for ch in range(256):
+ with self.subTest(size=size, ch=ch):
+ ch = chr(ch)
+
+ # If the first write is a Latin1 character and no buffer
+ # was allocated yet, use the singleton as the read-only
+ # buffer
+ writer = self.create_writer(size)
+ writer.write_char(ord(ch))
+ self.assertIs(writer.finish(), ch)
+
+ # PyUnicodeWriter_Finish() replaces the buffer
+ # with the singleton
+ writer = self.create_writer(size)
+ # Use PyUnicodeWriter_WriteSubstring() to avoid
+ # the read-only buffer optimization
+ writer.write_substring(ch + 'xxx', 0, 1)
+ self.assertIs(writer.finish(), ch)
@unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)')
def test_detect_overflow(self):
@@ -2115,6 +2164,32 @@ def test_change_kind(self):
self.assertEqual(writer.finish(),
'ascii latin1:\xe9 ucs2:\u20ac ucs4:\U0010ffff')
+ def test_readonly_optim(self):
+ # Read-only optimization: if the first and only write is a Python str
+ # object and no buffer was allocated yet, return the object unchanged
+ unique_string = 'unique string'
+ writer = self.create_writer(0)
+ writer.write_str(unique_string)
+ self.assertIs(writer.finish(), unique_string)
+
+ writer = self.create_writer(0)
+ writer.write_substring(unique_string, 0, len(unique_string))
+ self.assertIs(writer.finish(), unique_string)
+
+ class MyStr:
+ def __str__(self):
+ return unique_string
+ writer = self.create_writer(0)
+ writer.write_str(MyStr())
+ self.assertIs(writer.finish(), unique_string)
+
+ class MyRepr:
+ def __repr__(self):
+ return unique_string
+ writer = self.create_writer(0)
+ writer.write_repr(MyRepr())
+ self.assertIs(writer.finish(), unique_string)
+
# Test PyUnicodeWriter_Format()
@unittest.skipIf(ctypes is None, 'need ctypes')
@@ -2152,112 +2227,106 @@ def test_recover_error(self):
self.assertEqual(writer.finish(), 'Hello World.')
- def test_unicode_equal(self):
- unicode_equal = _testlimitedcapi.unicode_equal
-
- def copy(text):
- return text.encode().decode()
-
- self.assertTrue(unicode_equal("", ""))
- self.assertTrue(unicode_equal("abc", "abc"))
- self.assertTrue(unicode_equal("abc", copy("abc")))
- self.assertTrue(unicode_equal("\u20ac", copy("\u20ac")))
- self.assertTrue(unicode_equal("\U0010ffff", copy("\U0010ffff")))
+ def test_readonly_optim(self):
+ # Read-only optimization: if the first and only write is a Python str
+ # object and no buffer was allocated yet, return the object unchanged
+ from ctypes import py_object
- self.assertFalse(unicode_equal("abc", "abcd"))
- self.assertFalse(unicode_equal("\u20ac", "\u20ad"))
- self.assertFalse(unicode_equal("\U0010ffff", "\U0010fffe"))
-
- # str subclass
- self.assertTrue(unicode_equal("abc", Str("abc")))
- self.assertTrue(unicode_equal(Str("abc"), "abc"))
- self.assertFalse(unicode_equal("abc", Str("abcd")))
- self.assertFalse(unicode_equal(Str("abc"), "abcd"))
-
- # invalid type
- for invalid_type in (b'bytes', 123, ("tuple",)):
- with self.subTest(invalid_type=invalid_type):
- with self.assertRaises(TypeError):
- unicode_equal("abc", invalid_type)
- with self.assertRaises(TypeError):
- unicode_equal(invalid_type, "abc")
+ unique_string = 'unique string'
+ for format in (b'%S', b'%U'):
+ with self.subTest(format=format):
+ writer = self.create_writer(0)
+ self.writer_format(writer, format, py_object(unique_string))
+ self.assertIs(writer.finish(), unique_string)
- # CRASHES unicode_equal("abc", NULL)
- # CRASHES unicode_equal(NULL, "abc")
+ class MyStr:
+ def __str__(self):
+ return unique_string
+ writer = self.create_writer(0)
+ self.writer_format(writer, b'%S', py_object(MyStr()))
+ self.assertIs(writer.finish(), unique_string)
- # TODO: Add tests to the following codec functions:
- # - PyUnicode_AsASCIIString
- # - PyUnicode_AsCharmapString
- # - PyUnicode_AsEncodedString
- # - PyUnicode_AsLatin1String
- # - PyUnicode_AsMBCSString
- # - PyUnicode_AsRawUnicodeEscapeString
- # - PyUnicode_AsUTF16String
- # - PyUnicode_AsUTF32String
- # - PyUnicode_AsUTF8String
- # - PyUnicode_AsUnicodeEscapeString
- # - PyUnicode_BuildEncodingMap
- # - PyUnicode_Decode
- # - PyUnicode_DecodeASCII
- # - PyUnicode_DecodeCharmap
- # - PyUnicode_DecodeCodePageStateful
- # - PyUnicode_DecodeFSDefault
- # - PyUnicode_DecodeFSDefaultAndSize
- # - PyUnicode_DecodeLatin1
- # - PyUnicode_DecodeLocale
- # - PyUnicode_DecodeLocaleAndSize
- # - PyUnicode_DecodeMBCS
- # - PyUnicode_DecodeMBCSStateful
- # - PyUnicode_DecodeRawUnicodeEscape
- # - PyUnicode_DecodeUTF16
- # - PyUnicode_DecodeUTF16Stateful
- # - PyUnicode_DecodeUTF32
- # - PyUnicode_DecodeUTF32Stateful
- # - PyUnicode_DecodeUTF7
- # - PyUnicode_DecodeUTF7Stateful
- # - PyUnicode_DecodeUTF8
- # - PyUnicode_DecodeUTF8Stateful
- # - PyUnicode_DecodeUnicodeEscape
- # - PyUnicode_EncodeCodePage
- # - PyUnicode_EncodeFSDefault
- # - PyUnicode_EncodeLocale
- # - PyUnicode_FSConverter
- # - PyUnicode_FSDecoder
- # - PyUnicode_FromEncodedObject
- # - PyUnicode_Splitlines
-
- # TODO: Add tests to the following character functions:
- # - Py_UNICODE_ISALNUM
- # - Py_UNICODE_ISALPHA
- # - Py_UNICODE_ISDECIMAL
- # - Py_UNICODE_ISDIGIT
- # - Py_UNICODE_ISLINEBREAK
- # - Py_UNICODE_ISLOWER
- # - Py_UNICODE_ISNUMERIC
- # - Py_UNICODE_ISPRINTABLE
- # - Py_UNICODE_ISSPACE
- # - Py_UNICODE_ISTITLE
- # - Py_UNICODE_ISUPPER
- # - Py_UNICODE_TODECIMAL
- # - Py_UNICODE_TODIGIT
- # - Py_UNICODE_TOLOWER
- # - Py_UNICODE_TONUMERIC
- # - Py_UNICODE_TOTITLE
- # - Py_UNICODE_TOUPPER
-
- # TODO: Maybe add tests to the following less important functions:
- # - PyUnicode_1BYTE_DATA
- # - PyUnicode_2BYTE_DATA
- # - PyUnicode_4BYTE_DATA
- # - PyUnicode_DATA
- # - PyUnicode_IS_READY
- # - PyUnicode_READY
- # - Py_UNICODE_HIGH_SURROGATE
- # - Py_UNICODE_IS_HIGH_SURROGATE
- # - Py_UNICODE_IS_LOW_SURROGATE
- # - Py_UNICODE_IS_SURROGATE
- # - Py_UNICODE_JOIN_SURROGATES
- # - Py_UNICODE_LOW_SURROGATE
+ class MyRepr:
+ def __repr__(self):
+ return unique_string
+ writer = self.create_writer(0)
+ self.writer_format(writer, b'%R', py_object(MyRepr()))
+ self.assertIs(writer.finish(), unique_string)
+
+
+# TODO: Add tests to the following codec functions:
+# - PyUnicode_AsASCIIString
+# - PyUnicode_AsCharmapString
+# - PyUnicode_AsEncodedString
+# - PyUnicode_AsLatin1String
+# - PyUnicode_AsMBCSString
+# - PyUnicode_AsRawUnicodeEscapeString
+# - PyUnicode_AsUTF16String
+# - PyUnicode_AsUTF32String
+# - PyUnicode_AsUTF8String
+# - PyUnicode_AsUnicodeEscapeString
+# - PyUnicode_BuildEncodingMap
+# - PyUnicode_Decode
+# - PyUnicode_DecodeASCII
+# - PyUnicode_DecodeCharmap
+# - PyUnicode_DecodeCodePageStateful
+# - PyUnicode_DecodeFSDefault
+# - PyUnicode_DecodeFSDefaultAndSize
+# - PyUnicode_DecodeLatin1
+# - PyUnicode_DecodeLocale
+# - PyUnicode_DecodeLocaleAndSize
+# - PyUnicode_DecodeMBCS
+# - PyUnicode_DecodeMBCSStateful
+# - PyUnicode_DecodeRawUnicodeEscape
+# - PyUnicode_DecodeUTF16
+# - PyUnicode_DecodeUTF16Stateful
+# - PyUnicode_DecodeUTF32
+# - PyUnicode_DecodeUTF32Stateful
+# - PyUnicode_DecodeUTF7
+# - PyUnicode_DecodeUTF7Stateful
+# - PyUnicode_DecodeUTF8
+# - PyUnicode_DecodeUTF8Stateful
+# - PyUnicode_DecodeUnicodeEscape
+# - PyUnicode_EncodeCodePage
+# - PyUnicode_EncodeFSDefault
+# - PyUnicode_EncodeLocale
+# - PyUnicode_FSConverter
+# - PyUnicode_FSDecoder
+# - PyUnicode_FromEncodedObject
+# - PyUnicode_Splitlines
+
+# TODO: Add tests to the following character functions:
+# - Py_UNICODE_ISALNUM
+# - Py_UNICODE_ISALPHA
+# - Py_UNICODE_ISDECIMAL
+# - Py_UNICODE_ISDIGIT
+# - Py_UNICODE_ISLINEBREAK
+# - Py_UNICODE_ISLOWER
+# - Py_UNICODE_ISNUMERIC
+# - Py_UNICODE_ISPRINTABLE
+# - Py_UNICODE_ISSPACE
+# - Py_UNICODE_ISTITLE
+# - Py_UNICODE_ISUPPER
+# - Py_UNICODE_TODECIMAL
+# - Py_UNICODE_TODIGIT
+# - Py_UNICODE_TOLOWER
+# - Py_UNICODE_TONUMERIC
+# - Py_UNICODE_TOTITLE
+# - Py_UNICODE_TOUPPER
+
+# TODO: Maybe add tests to the following less important functions:
+# - PyUnicode_1BYTE_DATA
+# - PyUnicode_2BYTE_DATA
+# - PyUnicode_4BYTE_DATA
+# - PyUnicode_DATA
+# - PyUnicode_IS_READY
+# - PyUnicode_READY
+# - Py_UNICODE_HIGH_SURROGATE
+# - Py_UNICODE_IS_HIGH_SURROGATE
+# - Py_UNICODE_IS_LOW_SURROGATE
+# - Py_UNICODE_IS_SURROGATE
+# - Py_UNICODE_JOIN_SURROGATES
+# - Py_UNICODE_LOW_SURROGATE
if __name__ == "__main__":
diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c
index 751fca9948598ff..fc3a95cd97e4213 100644
--- a/Objects/unicode_writer.c
+++ b/Objects/unicode_writer.c
@@ -115,30 +115,6 @@ unicode_write_cstr(PyObject *unicode, Py_ssize_t index,
}
-static inline void
-_PyUnicodeWriter_Update(_PyUnicodeWriter *writer)
-{
- writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer);
- writer->data = PyUnicode_DATA(writer->buffer);
-
- if (!writer->readonly) {
- writer->kind = PyUnicode_KIND(writer->buffer);
- writer->size = PyUnicode_GET_LENGTH(writer->buffer);
- }
- else {
- /* use a value smaller than PyUnicode_1BYTE_KIND() so
- _PyUnicodeWriter_PrepareKind() will copy the buffer. */
- writer->kind = 0;
- assert(writer->kind <= PyUnicode_1BYTE_KIND);
-
- /* Copy-on-write mode: set buffer size to 0 so
- * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on
- * next write. */
- writer->size = 0;
- }
-}
-
-
void
_PyUnicodeWriter_Init(_PyUnicodeWriter *writer)
{
@@ -342,12 +318,13 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer,
PyObject *str)
return 0;
maxchar = PyUnicode_MAX_CHAR_VALUE(str);
if (maxchar > writer->maxchar || len > writer->size - writer->pos) {
- if (writer->buffer == NULL && !writer->overallocate) {
+ if (writer->buffer == NULL) {
assert(_PyUnicode_CheckConsistency(str, 1));
writer->readonly = 1;
writer->buffer = Py_NewRef(str);
_PyUnicodeWriter_Update(writer);
writer->pos += len;
+ // The next write will create a new buffer and copy the string
return 0;
}
if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1)
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 5d0b818981fc982..da9f8dcc223fc0d 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -5230,6 +5230,7 @@ unicode_decode_utf8_impl(_PyUnicodeWriter *writer,
if (_PyUnicodeWriter_PrepareKind(writer, PyUnicode_2BYTE_KIND) < 0)
goto onError;
+ assert(_PyUnicodeWriter_CanWrite(writer));
for (i=startinpos; i<endinpos; i++) {
ch = (Py_UCS4)(unsigned char)(starts[i]);
PyUnicode_WRITE(writer->kind, writer->data, writer->pos,
@@ -7472,6 +7473,7 @@ PyUnicode_DecodeASCII(const char *s,
but we may switch to UCS2 at the first write */
if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) <
0)
goto onError;
+ assert(_PyUnicodeWriter_CanWrite(&writer));
kind = writer.kind;
data = writer.data;
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
