https://github.com/python/cpython/commit/25a614a502e607a333f7aa7f4d69e6f2a47148ad
commit: 25a614a502e607a333f7aa7f4d69e6f2a47148ad
branch: main
author: Bénédikt Tran <[email protected]>
committer: picnixz <[email protected]>
date: 2025-01-23T14:28:33+01:00
summary:
gh-126004: Fix positions handling in `codecs.backslashreplace_errors` (#127676)
This fixes how `PyCodec_BackslashReplaceErrors` handles the `start` and `end`
attributes of `UnicodeError` objects via the `_PyUnicodeError_GetParams` helper.
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst
M Lib/test/test_capi/test_codecs.py
M Python/codecs.c
diff --git a/Lib/test/test_capi/test_codecs.py
b/Lib/test/test_capi/test_codecs.py
index f57191ddcdbeb4..e368f70c196abf 100644
--- a/Lib/test/test_capi/test_codecs.py
+++ b/Lib/test/test_capi/test_codecs.py
@@ -849,7 +849,8 @@ def test_codec_xmlcharrefreplace_errors_handler(self):
def test_codec_backslashreplace_errors_handler(self):
handler = _testcapi.codec_backslashreplace_errors
- self.do_test_codec_errors_handler(handler, self.all_unicode_errors)
+ self.do_test_codec_errors_handler(handler, self.all_unicode_errors,
+ safe=True)
def test_codec_namereplace_errors_handler(self):
handler = _testlimitedcapi.codec_namereplace_errors
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst
new file mode 100644
index 00000000000000..619d73042a9bb8
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst
@@ -0,0 +1,3 @@
+Fix handling of :attr:`UnicodeError.start` and :attr:`UnicodeError.end`
+values in the :func:`codecs.backslashreplace_errors` error handler. Patch by
+Bénédikt Tran.
diff --git a/Python/codecs.c b/Python/codecs.c
index b657dd134a668e..07eaa8ecddcae0 100644
--- a/Python/codecs.c
+++ b/Python/codecs.c
@@ -864,108 +864,107 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc)
{
- PyObject *object;
- Py_ssize_t i;
- Py_ssize_t start;
- Py_ssize_t end;
- PyObject *res;
- Py_UCS1 *outp;
- int ressize;
- Py_UCS4 c;
-
+ PyObject *obj;
+ Py_ssize_t objlen, start, end, slen;
if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {
- const unsigned char *p;
- if (PyUnicodeDecodeError_GetStart(exc, &start))
- return NULL;
- if (PyUnicodeDecodeError_GetEnd(exc, &end))
- return NULL;
- if (!(object = PyUnicodeDecodeError_GetObject(exc)))
+ if (_PyUnicodeError_GetParams(exc,
+ &obj, &objlen,
+ &start, &end, &slen, true) < 0)
+ {
return NULL;
- p = (const unsigned char*)PyBytes_AS_STRING(object);
- res = PyUnicode_New(4 * (end - start), 127);
+ }
+ PyObject *res = PyUnicode_New(4 * slen, 127);
if (res == NULL) {
- Py_DECREF(object);
+ Py_DECREF(obj);
return NULL;
}
- outp = PyUnicode_1BYTE_DATA(res);
- for (i = start; i < end; i++, outp += 4) {
- unsigned char c = p[i];
+ Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res);
+ const unsigned char *p = (const unsigned char *)PyBytes_AS_STRING(obj);
+ for (Py_ssize_t i = start; i < end; i++, outp += 4) {
+ const unsigned char ch = p[i];
outp[0] = '\\';
outp[1] = 'x';
- outp[2] = Py_hexdigits[(c>>4)&0xf];
- outp[3] = Py_hexdigits[c&0xf];
+ outp[2] = Py_hexdigits[(ch >> 4) & 0xf];
+ outp[3] = Py_hexdigits[ch & 0xf];
}
-
assert(_PyUnicode_CheckConsistency(res, 1));
- Py_DECREF(object);
+ Py_DECREF(obj);
return Py_BuildValue("(Nn)", res, end);
}
- if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
- if (PyUnicodeEncodeError_GetStart(exc, &start))
- return NULL;
- if (PyUnicodeEncodeError_GetEnd(exc, &end))
- return NULL;
- if (!(object = PyUnicodeEncodeError_GetObject(exc)))
- return NULL;
- }
- else if (PyObject_TypeCheck(exc, (PyTypeObject
*)PyExc_UnicodeTranslateError)) {
- if (PyUnicodeTranslateError_GetStart(exc, &start))
- return NULL;
- if (PyUnicodeTranslateError_GetEnd(exc, &end))
- return NULL;
- if (!(object = PyUnicodeTranslateError_GetObject(exc)))
+
+ if (
+ PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)
+ || PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)
+ ) {
+ if (_PyUnicodeError_GetParams(exc,
+ &obj, &objlen,
+ &start, &end, &slen, false) < 0)
+ {
return NULL;
+ }
}
else {
wrong_exception_type(exc);
return NULL;
}
- if (end - start > PY_SSIZE_T_MAX / (1+1+8))
- end = start + PY_SSIZE_T_MAX / (1+1+8);
- for (i = start, ressize = 0; i < end; ++i) {
+ // The number of characters that each character 'ch' contributes
+ // in the result is 1 + 1 + k, where k >= min{t >= 1 | 16^t > ch}
+ // and will be formatted as "\\" + ('U'|'u'|'x') + HEXDIGITS,
+ // where the number of hexdigits is either 2, 4, or 8 (not 6).
+ // Since the Unicode range is below 10^7, we choose k = 8 whence
+ // each "block" requires at most 1 + 1 + 8 characters.
+ if (slen > PY_SSIZE_T_MAX / (1 + 1 + 8)) {
+ end = start + PY_SSIZE_T_MAX / (1 + 1 + 8);
+ end = Py_MIN(end, objlen);
+ slen = Py_MAX(0, end - start);
+ }
+
+ Py_ssize_t ressize = 0;
+ for (Py_ssize_t i = start; i < end; ++i) {
/* object is guaranteed to be "ready" */
- c = PyUnicode_READ_CHAR(object, i);
+ Py_UCS4 c = PyUnicode_READ_CHAR(obj, i);
if (c >= 0x10000) {
- ressize += 1+1+8;
+ ressize += 1 + 1 + 8;
}
else if (c >= 0x100) {
- ressize += 1+1+4;
+ ressize += 1 + 1 + 4;
+ }
+ else {
+ ressize += 1 + 1 + 2;
}
- else
- ressize += 1+1+2;
}
- res = PyUnicode_New(ressize, 127);
+ PyObject *res = PyUnicode_New(ressize, 127);
if (res == NULL) {
- Py_DECREF(object);
+ Py_DECREF(obj);
return NULL;
}
- outp = PyUnicode_1BYTE_DATA(res);
- for (i = start; i < end; ++i) {
- c = PyUnicode_READ_CHAR(object, i);
+ Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res);
+ for (Py_ssize_t i = start; i < end; ++i) {
+ Py_UCS4 c = PyUnicode_READ_CHAR(obj, i);
*outp++ = '\\';
if (c >= 0x00010000) {
*outp++ = 'U';
- *outp++ = Py_hexdigits[(c>>28)&0xf];
- *outp++ = Py_hexdigits[(c>>24)&0xf];
- *outp++ = Py_hexdigits[(c>>20)&0xf];
- *outp++ = Py_hexdigits[(c>>16)&0xf];
- *outp++ = Py_hexdigits[(c>>12)&0xf];
- *outp++ = Py_hexdigits[(c>>8)&0xf];
+ *outp++ = Py_hexdigits[(c >> 28) & 0xf];
+ *outp++ = Py_hexdigits[(c >> 24) & 0xf];
+ *outp++ = Py_hexdigits[(c >> 20) & 0xf];
+ *outp++ = Py_hexdigits[(c >> 16) & 0xf];
+ *outp++ = Py_hexdigits[(c >> 12) & 0xf];
+ *outp++ = Py_hexdigits[(c >> 8) & 0xf];
}
else if (c >= 0x100) {
*outp++ = 'u';
- *outp++ = Py_hexdigits[(c>>12)&0xf];
- *outp++ = Py_hexdigits[(c>>8)&0xf];
+ *outp++ = Py_hexdigits[(c >> 12) & 0xf];
+ *outp++ = Py_hexdigits[(c >> 8) & 0xf];
}
- else
+ else {
*outp++ = 'x';
- *outp++ = Py_hexdigits[(c>>4)&0xf];
- *outp++ = Py_hexdigits[c&0xf];
+ }
+ *outp++ = Py_hexdigits[(c >> 4) & 0xf];
+ *outp++ = Py_hexdigits[c & 0xf];
}
-
assert(_PyUnicode_CheckConsistency(res, 1));
- Py_DECREF(object);
+ Py_DECREF(obj);
return Py_BuildValue("(Nn)", res, end);
}
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
