https://github.com/python/cpython/commit/5b2558292978113211c5a720873183b38547f3d0
commit: 5b2558292978113211c5a720873183b38547f3d0
branch: main
author: Gregory P. Smith <[email protected]>
committer: gpshead <[email protected]>
date: 2026-10-31T05:54:09-07:00
summary:
gh-160522: Restore CJK encoder state after a failed incremental encode
(GH-160523)
A failed IncrementalEncoder.encode() or StreamWriter.write() of a
stateful CJK codec kept the shift state reached before the error
although its output was discarded.
files:
A Misc/NEWS.d/next/Library/2026-10-22-08-18-31.gh-issue-160522.cJkSt8.rst
M Lib/test/test_multibytecodec.py
M Modules/cjkcodecs/multibytecodec.c
diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py
index d4923a31aef8356..39d00d8df3c3d3e 100644
--- a/Lib/test/test_multibytecodec.py
+++ b/Lib/test/test_multibytecodec.py
@@ -461,6 +461,48 @@ def test_incrementalencoder_final(self):
self.assertEqual(output, self.expected_reset)
self.assertEqual(encoder.encode('', final=True), b'')
+ def test_incrementalencoder_error(self):
+ # The output of a failed encode() is lost, so the shift state
+ # must not be changed.
+ encoder = codecs.getincrementalencoder(self.encoding)()
+ state = encoder.getstate()
+ for final in False, True:
+ self.assertRaises(UnicodeEncodeError, encoder.encode,
+ self.text + '\U0001f600', final)
+ self.assertEqual(encoder.getstate(), state)
+ self.assertEqual(encoder.encode(self.text), self.expected)
+ state = encoder.getstate()
+ for final in False, True:
+ self.assertRaises(UnicodeEncodeError, encoder.encode,
+ 'a\U0001f600', final)
+ self.assertEqual(encoder.getstate(), state)
+ self.assertEqual(encoder.encode('a'), self.reset + b'a')
+
+ def test_streamwriter_error(self):
+ stream = io.BytesIO()
+ writer = codecs.getwriter(self.encoding)(stream)
+ self.assertRaises(UnicodeEncodeError, writer.write,
+ self.text + '\U0001f600')
+ self.assertEqual(stream.getvalue(), b'')
+ writer.write(self.text)
+ self.assertEqual(stream.getvalue(), self.expected)
+
+ def test_textiowrapper_error(self):
+ raw = io.BytesIO()
+ with io.TextIOWrapper(raw, encoding=self.encoding) as f:
+ self.assertRaises(UnicodeEncodeError, f.write,
+ self.text + '\U0001f600')
+ f.write(self.text)
+ f.flush()
+ self.assertEqual(raw.getvalue(), self.expected)
+
+class TestISO2022KRStateful(TestStateful):
+ text = '\uac00\uac00'
+ encoding = 'iso-2022-kr'
+ expected = b'\x1b$)C\x0e0!0!'
+ reset = b'\x0f'
+ expected_reset = expected + reset
+
class TestHZStateful(TestStateful):
text = '\u804a\u804a'
encoding = 'hz'
diff --git
a/Misc/NEWS.d/next/Library/2026-10-22-08-18-31.gh-issue-160522.cJkSt8.rst
b/Misc/NEWS.d/next/Library/2026-10-22-08-18-31.gh-issue-160522.cJkSt8.rst
new file mode 100644
index 000000000000000..ae93269f8714760
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-10-22-08-18-31.gh-issue-160522.cJkSt8.rst
@@ -0,0 +1,4 @@
+Fix the incremental encoders and stream writers of the stateful CJK codecs
+(``iso2022_*`` and ``hz``) keeping a changed shift state after an
+:meth:`!encode` or :meth:`!write` call failed. The next call could omit the
+escape sequence, writing multibyte characters in ASCII mode.
diff --git a/Modules/cjkcodecs/multibytecodec.c
b/Modules/cjkcodecs/multibytecodec.c
index f513182cb70260a..39ea624a2766aff 100644
--- a/Modules/cjkcodecs/multibytecodec.c
+++ b/Modules/cjkcodecs/multibytecodec.c
@@ -794,6 +794,7 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext
*ctx,
PyObject *inbuf = NULL;
Py_ssize_t inpos, datalen;
PyObject *origpending = NULL;
+ MultibyteCodec_State origstate;
if (PyUnicode_Check(unistr))
ucvt = NULL;
@@ -809,6 +810,7 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext
*ctx,
}
}
+ origstate = ctx->state;
if (ctx->pending) {
PyObject *inbuf_tmp;
@@ -866,6 +868,9 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext
*ctx,
return r;
errorexit:
+ /* recover the original codec state: the discarded output may have
+ * included an escape sequence for a state change */
+ ctx->state = origstate;
Py_XDECREF(r);
Py_XDECREF(ucvt);
Py_XDECREF(origpending);
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
