https://github.com/python/cpython/commit/1e03154546c5c1af5c0c746a2bdedc758acdec93
commit: 1e03154546c5c1af5c0c746a2bdedc758acdec93
branch: main
author: Victor Stinner <[email protected]>
committer: vstinner <[email protected]>
date: 2026-09-17T11:54:26Z
summary:
gh-155742: Use PyMem_Malloc() in decode_unicode_with_escapes() (#157587)
Replace soft deprecated PyBytes_FromStringAndSize(NULL, size) with a
simple PyMem_Malloc(). Avoid also "s = buf" to make the code easier to follow
(and easier to debug).
Add non-ASCII tests to test_string_literals.
files:
M Lib/test/test_string_literals.py
M Parser/string_parser.c
diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py
index 1800b17e1df3d72..bfdb368a6aa532e 100644
--- a/Lib/test/test_string_literals.py
+++ b/Lib/test/test_string_literals.py
@@ -54,6 +54,10 @@
assert ord(h) == 0x1d120
i = r'\U0001d120'
assert list(map(ord, i)) == [92, 85, 48, 48, 48, 49, 100, 49, 50, 48]
+j = 'ä'
+assert list(map(ord, j)) == [228]
+k = '\ä'
+assert list(map(ord, k)) == [92, 228]
"""
@@ -76,7 +80,7 @@ def test_template(self):
# Check that the template doesn't contain any non-printables
# except for \n.
for c in TEMPLATE:
- assert c == '\n' or ' ' <= c <= '~', repr(c)
+ assert c == '\n' or ' ' <= c <= '~' or c == 'ä', repr(c)
def test_eval_str_normal(self):
self.assertEqual(eval(""" 'x' """), 'x')
@@ -88,6 +92,21 @@ def test_eval_str_normal(self):
self.assertEqual(eval(""" '\u1881' """), chr(0x1881))
self.assertEqual(eval(r""" '\U0001d120' """), chr(0x1d120))
self.assertEqual(eval(""" '\U0001d120' """), chr(0x1d120))
+ # Lone "\" character at the end
+ self.assertEqual(eval(r"'abc\\'"), 'abc\\')
+
+ def test_eval_str_unicode(self):
+ for s in (
+ 'ϼўТλФЙ',
+ 'A͏B ﬖ̳AA̝',
+ '\U00100000\U0010ffff\U0010fffd',
+ 'ä',
+ '\\ä',
+ "\\П",
+ "áàäéèęöő.\\n",
+ ):
+ with self.subTest(s=s):
+ self.assertEqual(eval(f"{s!r}"), s)
def test_eval_str_incomplete(self):
self.assertRaises(SyntaxError, eval, r""" '\x' """)
diff --git a/Parser/string_parser.c b/Parser/string_parser.c
index e57460cb1fa6483..20f523a262b225a 100644
--- a/Parser/string_parser.c
+++ b/Parser/string_parser.c
@@ -135,8 +135,6 @@ static PyObject *
decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token
*t)
{
PyObject *v;
- PyObject *u;
- char *buf;
char *p;
const char *end;
@@ -144,22 +142,20 @@ decode_unicode_with_escapes(Parser *parser, const char
*s, size_t len, Token *t)
if (len > (size_t)PY_SSIZE_T_MAX / 6) {
return NULL;
}
- /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5
- "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */
- u = PyBytes_FromStringAndSize((char *)NULL, (Py_ssize_t)len * 6);
- if (u == NULL) {
- return NULL;
- }
- p = buf = PyBytes_AsString(u);
- if (p == NULL) {
+ /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5.
+ * "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6. */
+ Py_ssize_t alloc = (Py_ssize_t)len * 6;
+ char *buf = PyMem_Malloc(alloc);
+ if (buf == NULL) {
return NULL;
}
+ p = buf;
end = s + len;
while (s < end) {
if (*s == '\\') {
*p++ = *s++;
if (s >= end || *s & 0x80) {
- strcpy(p, "u005c");
+ memcpy(p, "u005c", 5);
p += 5;
if (s >= end) {
break;
@@ -174,19 +170,20 @@ decode_unicode_with_escapes(Parser *parser, const char
*s, size_t len, Token *t)
Py_ssize_t i;
w = decode_utf8(&s, end);
if (w == NULL) {
- Py_DECREF(u);
+ PyMem_Free(buf);
return NULL;
}
kind = PyUnicode_KIND(w);
data = PyUnicode_DATA(w);
w_len = PyUnicode_GET_LENGTH(w);
for (i = 0; i < w_len; i++) {
+ // sprintf() writes a null byte: the buffer is large enough
+ // for that thanks to the overallocation.
+ assert((p + 11 - buf) <= alloc);
Py_UCS4 chr = PyUnicode_READ(kind, data, i);
sprintf(p, "\\U%08x", chr);
p += 10;
}
- /* Should be impossible to overflow */
- assert(p - buf <= PyBytes_GET_SIZE(u));
Py_DECREF(w);
}
else {
@@ -194,26 +191,25 @@ decode_unicode_with_escapes(Parser *parser, const char
*s, size_t len, Token *t)
}
}
len = (size_t)(p - buf);
- s = buf;
int first_invalid_escape_char;
const char *first_invalid_escape_ptr;
- v = _PyUnicode_DecodeUnicodeEscapeInternal2(s, (Py_ssize_t)len, NULL, NULL,
+ v = _PyUnicode_DecodeUnicodeEscapeInternal2(buf, (Py_ssize_t)len, NULL,
NULL,
&first_invalid_escape_char,
&first_invalid_escape_ptr);
// HACK: later we can simply pass the line no, since we don't preserve the
tokens
// when we are decoding the string but we preserve the line numbers.
if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL) {
- if (warn_invalid_escape_sequence(parser, s, first_invalid_escape_ptr,
t) < 0) {
- /* We have not decref u before because first_invalid_escape_ptr
- points inside u. */
- Py_XDECREF(u);
+ if (warn_invalid_escape_sequence(parser, buf,
first_invalid_escape_ptr, t) < 0) {
+ /* We have not deallocated the buffer before because
+ * first_invalid_escape_ptr points inside buf. */
+ PyMem_Free(buf);
Py_DECREF(v);
return NULL;
}
}
- Py_XDECREF(u);
+ PyMem_Free(buf);
return v;
}
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
