https://github.com/python/cpython/commit/d2ace0c42fa21c4598b7fd3a67718c5da00b66a4
commit: d2ace0c42fa21c4598b7fd3a67718c5da00b66a4
branch: main
author: Vyron Vasileiadis <[email protected]>
committer: serhiy-storchaka <[email protected]>
date: 2026-07-31T10:56:52Z
summary:
gh-154863: Fix iconv encoding of ISO-2022-CN-EXT returning empty bytes
(GH-154899)
The flush that emits the pending shift sequence can report a nonreversible
conversion, which the loop mistook for a substituted character. It then
retried while still flushing, converted nothing and returned the empty
output buffer.
files:
M Lib/test/test_codecs.py
M Objects/unicodeobject.c
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 31704955df3e14e..f6f8089a065059b 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -3679,6 +3679,10 @@ def iconv_encoding_available(name):
# Encodings iconv may provide but for which CPython has no built-in codec
# (cp1047 is EBCDIC, i.e. not ASCII-compatible).
_ICONV_ONLY = ['cp1047', 'cp1133', 'GEORGIAN-PS', 'ARMSCII-8']
+# Encodings that leave a shift state pending, so encoding ends with a flush.
+_ICONV_SHIFT_STATE = [('ISO-2022-CN-EXT', 'ABC\u4e2dDEF'),
+ ('ISO-2022-CN', 'ABC\u4e2dDEF'),
+ ('ISO-2022-JP', 'ABC\u65e5DEF')]
@unittest.skipUnless(hasattr(codecs, 'iconv_encode'),
@@ -3812,6 +3816,33 @@ def test_encode_kinds(self):
with self.subTest(text=text):
self.assertEqual(text.encode('iconv:' + enc), text.encode(enc))
+ def test_encode_shift_state_flush(self):
+ # Encoding ends with a flush that emits the pending shift sequence.
Its
+ # return value counts nonreversible conversions, and some iconv
+ # implementations make it positive for the flush itself (glibc does for
+ # ISO-2022-CN-EXT). That must not be read as a substituted character:
+ # doing so discarded the whole output, ASCII included.
+ #
+ # Only the ASCII around the character is checked, not a full
round-trip.
+ # An iconv that cannot represent the character either rejects it or
+ # substitutes for it silently, as macOS does for ISO-2022-CN.
+ tested = False
+ for enc, text in _ICONV_SHIFT_STATE:
+ if not iconv_encoding_available(enc):
+ continue
+ with self.subTest(encoding=enc):
+ try:
+ data = codecs.iconv_encode(enc, text)[0]
+ except UnicodeEncodeError:
+ continue
+ tested = True
+ self.assertNotEqual(data, b'')
+ decoded = codecs.iconv_decode(enc, data, 'strict', True)[0]
+ self.assertStartsWith(decoded, 'ABC')
+ self.assertEndsWith(decoded, 'DEF')
+ if not tested:
+ self.skipTest('no shift-state iconv encoding is available')
+
def test_encode_surrogateescape(self):
# A lone surrogate lives in the 2-byte kind and round-trips.
enc = self.require('ASCII')
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index a4550c0f5f3363a..eec02f662e79059 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -8462,6 +8462,9 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject
*unicode,
}
if (ret != (size_t)-1) {
+ if (flushing) {
+ break;
+ }
/* A positive result counts nonreversible conversions: iconv()
substituted an unencodable character instead of failing with
EILSEQ (musl and *BSD citrus do this). Treat it as unencodable
@@ -8479,9 +8482,6 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject
*unicode,
out = out_before;
up -= unit;
}
- else if (flushing) {
- break;
- }
else if (careful && up < uend) {
continue;
}
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
