https://github.com/python/cpython/commit/1a85213940896f180cbcf76921cd2d35a8aa50de
commit: 1a85213940896f180cbcf76921cd2d35a8aa50de
branch: main
author: Victor Stinner <[email protected]>
committer: vstinner <[email protected]>
date: 2026-10-03T00:34:38+02:00
summary:
gh-158585: Adjust allocation in bytes_fromformat() (#158621)
Take in account preallocated bytes when calling
PyBytesWriter_GrowAndUpdatePointer().
Use sprintf() return value instead of calling strlen().
files:
M Lib/test/test_capi/test_bytes.py
M Objects/bytesobject.c
diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py
index f8487a2b8fb085..910f2bedac0bcb 100644
--- a/Lib/test/test_capi/test_bytes.py
+++ b/Lib/test/test_capi/test_bytes.py
@@ -607,15 +607,45 @@ def test_format_i(self):
writer = self.create_writer()
writer.format_i(b'x=%i, ', 123)
- writer.format_i(b'y=%i', 456)
+ writer.format_i(b'y=%d', 456)
self.bytes_equal(writer.finish(), b'x=123, y=456')
+ # Make sure that %i allocates the right number of bytes
+ n_str = '1234567'
+ for i in range(1, len(n_str) + 1):
+ n = int(n_str[:i])
+ writer = self.create_writer(0)
+ writer.format_i(b'n=%i', n)
+ self.bytes_equal(writer.finish(), f'n={n}'.encode())
+
+ # Test invalid format: "%t" is invalid, stop the parser and copy
+ # remaining bytes
+ writer = self.create_writer()
+ writer.write_bytes(b'prefix ', -1)
+ writer.format_i(b'y=%._ t %i', 0)
+ writer.write_bytes(b'.', 1)
+ self.bytes_equal(writer.finish(), b'prefix y=%._ t %i.')
+
def test_format_s(self):
# Test PyBytesWriter_Format()
writer = self.create_writer()
writer.format_s(b's=%s', b'Hello World')
self.bytes_equal(writer.finish(), b's=Hello World')
+ # Test precision
+ writer = self.create_writer()
+ writer.format_s(b'%.6s', b'Monty Python')
+ writer.format_s(b'%.20s', b'3.0')
+ self.bytes_equal(writer.finish(), b'Monty 3.0')
+
+ # Make sure that %s allocates the right number of bytes
+ text = b'1234567'
+ for i in range(1, len(text) + 1):
+ s = text[:i]
+ writer = self.create_writer(0)
+ writer.format_s(b's=%s', s)
+ self.bytes_equal(writer.finish(), b's=' + s)
+
@support.nomemtest
def test_format_s_memory_error(self):
writer = self.create_writer()
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index cb2c2f7bfae634..2339e8bc1d2819 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -194,73 +194,77 @@ static char*
bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos,
const char *format, va_list vargs)
{
- const char *f;
- const char *p;
- Py_ssize_t prec;
- int longflag;
- int size_tflag;
/* Longest 64-bit formatted numbers:
- "18446744073709551615\0" (21 bytes)
- "-9223372036854775808\0" (21 bytes)
Decimal takes the most space (it isn't enough for octal.)
Longest 64-bit pointer representation:
- "0xffffffffffffffff\0" (19 bytes). */
+ "0xffffffffffffffff\0" (19 bytes).
+
+ Longest 64-bit "%p" with "0x" prefix: len(hex(2**64-1)+'\0') = 19.
+ */
char buffer[21];
char *s = (char*)PyBytesWriter_GetData(writer) + writer_pos;
-#define WRITE_BYTES_LEN(str, len_expr) \
+#define WRITE_BYTES(str, len_expr) \
do { \
- size_t len = (len_expr); \
- s = PyBytesWriter_GrowAndUpdatePointer(writer, len, s); \
- if (s == NULL) { \
- return NULL; \
+ size_t _len = (len_expr); \
+ size_t _prealloc = (f - p + 1); \
+ if (_len > _prealloc) { \
+ s = PyBytesWriter_GrowAndUpdatePointer(writer, _len - _prealloc,
s); \
+ if (s == NULL) { \
+ return NULL; \
+ } \
} \
- memcpy(s, (str), len); \
- s += len; \
+ memcpy(s, (str), _len); \
+ s += _len; \
} while (0)
-#define WRITE_BYTES(str) WRITE_BYTES_LEN(str, strlen(str))
- for (f = format; *f; f++) {
+ for (const char *f = format; *f; f++) {
if (*f != '%') {
*s++ = *f;
continue;
}
- p = f++;
+ const char *p = f++;
/* ignore the width (ex: 10 in "%10s") */
while (Py_ISDIGIT(*f))
f++;
/* parse the precision (ex: 10 in "%.10s") */
- prec = 0;
+ Py_ssize_t prec = 0;
if (*f == '.') {
f++;
for (; Py_ISDIGIT(*f); f++) {
prec = (prec * 10) + (*f - '0');
}
}
+ assert(prec >= 0);
- while (*f && *f != '%' && !Py_ISALPHA(*f))
+ while (*f && *f != '%' && !Py_ISALPHA(*f)) {
f++;
+ }
/* handle the long flag ('l'), but only for %ld and %lu.
others can be added when necessary. */
- longflag = 0;
+ int longflag = 0;
if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) {
longflag = 1;
++f;
}
/* handle the size_t flag ('z'). */
- size_tflag = 0;
+ int size_tflag = 0;
if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) {
size_tflag = 1;
++f;
}
+ Py_ssize_t len;
+
switch (*f) {
case 'c':
{
@@ -277,74 +281,77 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t
writer_pos,
case 'd':
if (longflag) {
- sprintf(buffer, "%ld", va_arg(vargs, long));
+ len = sprintf(buffer, "%ld", va_arg(vargs, long));
}
else if (size_tflag) {
- sprintf(buffer, "%zd", va_arg(vargs, Py_ssize_t));
+ len = sprintf(buffer, "%zd", va_arg(vargs, Py_ssize_t));
}
else {
- sprintf(buffer, "%d", va_arg(vargs, int));
+ len = sprintf(buffer, "%d", va_arg(vargs, int));
}
- assert(strlen(buffer) < sizeof(buffer));
- WRITE_BYTES(buffer);
+ assert(1 <= len && len < (Py_ssize_t)sizeof(buffer));
+ WRITE_BYTES(buffer, len);
break;
case 'u':
if (longflag) {
- sprintf(buffer, "%lu", va_arg(vargs, unsigned long));
+ len = sprintf(buffer, "%lu", va_arg(vargs, unsigned long));
}
else if (size_tflag) {
- sprintf(buffer, "%zu", va_arg(vargs, size_t));
+ len = sprintf(buffer, "%zu", va_arg(vargs, size_t));
}
else {
- sprintf(buffer, "%u", va_arg(vargs, unsigned int));
+ len = sprintf(buffer, "%u", va_arg(vargs, unsigned int));
}
- assert(strlen(buffer) < sizeof(buffer));
- WRITE_BYTES(buffer);
+ assert(1 <= len && len < (Py_ssize_t)sizeof(buffer));
+ WRITE_BYTES(buffer, len);
break;
case 'i':
- sprintf(buffer, "%i", va_arg(vargs, int));
- assert(strlen(buffer) < sizeof(buffer));
- WRITE_BYTES(buffer);
+ len = sprintf(buffer, "%i", va_arg(vargs, int));
+ assert(1 <= len && len < (Py_ssize_t)sizeof(buffer));
+ WRITE_BYTES(buffer, len);
break;
case 'x':
- sprintf(buffer, "%x", va_arg(vargs, int));
- assert(strlen(buffer) < sizeof(buffer));
- WRITE_BYTES(buffer);
+ len = sprintf(buffer, "%x", va_arg(vargs, int));
+ assert(1 <= len && len < (Py_ssize_t)sizeof(buffer));
+ WRITE_BYTES(buffer, len);
break;
case 's':
{
- Py_ssize_t i;
-
- p = va_arg(vargs, const char*);
- if (prec <= 0) {
- i = strlen(p);
+ const char *str = va_arg(vargs, const char*);
+ if (prec == 0) {
+ len = strlen(str);
}
else {
- i = 0;
- while (i < prec && p[i]) {
- i++;
+ const char *end = memchr(str, 0, prec);
+ if (end != NULL) {
+ len = (end - str);
+ }
+ else {
+ len = (size_t)prec;
}
}
- WRITE_BYTES_LEN(p, i);
+ WRITE_BYTES(str, len);
break;
}
case 'p':
- sprintf(buffer, "%p", va_arg(vargs, void*));
- assert(strlen(buffer) < sizeof(buffer));
+ len = sprintf(buffer, "%p", va_arg(vargs, void*));
+ assert(1 <= len && len < (Py_ssize_t)sizeof(buffer));
/* %p is ill-defined: ensure leading 0x. */
- if (buffer[1] == 'X')
+ if (buffer[1] == 'X') {
buffer[1] = 'x';
+ }
else if (buffer[1] != 'x') {
- memmove(buffer+2, buffer, strlen(buffer)+1);
+ memmove(buffer + 2, buffer, len + 1);
buffer[0] = '0';
buffer[1] = 'x';
+ len += 2;
}
- WRITE_BYTES(buffer);
+ WRITE_BYTES(buffer, len);
break;
case '%':
@@ -352,16 +359,19 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t
writer_pos,
break;
default:
- /* invalid format string: copy unformatted string and exit */
- WRITE_BYTES(p);
+ // Invalid format string: copy unformatted string and exit.
+ // No need to grow the writer buffer, we already preallocated
+ // enough bytes.
+ len = strlen(p);
+ memcpy(s, p, len);
+ s += len;
return s;
}
}
-#undef WRITE_BYTES
-#undef WRITE_BYTES_LEN
-
return s;
+
+#undef WRITE_BYTES
}
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
