https://github.com/python/cpython/commit/d67f2a9ec8c248e6526060f49ef147ad4cfe8237 commit: d67f2a9ec8c248e6526060f49ef147ad4cfe8237 branch: 3.14 author: Gregory P. Smith <[email protected]> committer: gpshead <[email protected]> date: 2026-11-01T07:31:17Z summary:
[3.14] gh-159739: Preserve CJK codec state across chunk boundaries and callbacks (GH-160400) (#161070) The HZ encoder can consume a tilde after writing only its first escape byte. A split ISO-2022 escape can exceed the incremental decoder’s pending buffer, and a reset can leave the previous character set selected. Reserve space for both HZ bytes before consuming the tilde, allow the full pending escape, and restore the initial ISO-2022 state on reset. Keep the native codec table and the active encoder error policy alive so replacing their Python owners cannot leave a codec using freed memory. (cherry picked from commit a96fbdca7fe1216f9c6079a570327a283d844aee) Co-authored-by: Jelle Zijlstra <[email protected]> files: A Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst M Lib/test/test_multibytecodec.py M Modules/cjkcodecs/_codecs_cn.c M Modules/cjkcodecs/_codecs_iso2022.c M Modules/cjkcodecs/multibytecodec.c M Modules/cjkcodecs/multibytecodec.h diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 8d5eb78608afc7..39d00d8df3c3d3 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -39,6 +39,10 @@ def test_nullcoding(self): self.assertEqual(str(b'', enc), '') self.assertEqual(''.encode(enc), b'') + def test_hz_tilde_at_buffer_boundary(self): + text = 'a' + '中a' * 6 + '~' + self.assertEqual(text.encode('hz').decode('hz'), text) + def test_str_decode(self): for enc in ALL_CJKENCODINGS: self.assertEqual('abcd'.encode(enc), b'abcd') @@ -91,6 +95,12 @@ def test_decode_unicode(self): class Test_IncrementalEncoder(unittest.TestCase): + def test_iso2022_kr_reset(self): + encoder = codecs.getincrementalencoder('iso2022_kr')() + encoder.encode('가') + encoder.reset() + self.assertEqual(encoder.encode('가', final=True), '가'.encode('iso2022_kr')) + def test_stateless(self): # cp949 encoder isn't stateful at all. encoder = codecs.getincrementalencoder('cp949')() @@ -242,6 +252,19 @@ def test_subinterp(self): class Test_IncrementalDecoder(unittest.TestCase): + def test_iso2022_split_escape(self): + raw = b'\x1b$0000000B' + decoder = codecs.getincrementaldecoder('iso2022_jp')('replace') + decoded = decoder.decode(raw[:-1]) + decoder.decode(raw[-1:], final=True) + self.assertEqual(decoded, raw.decode('iso2022_jp', 'replace')) + + def test_iso2022_reset(self): + decoder = codecs.getincrementaldecoder('iso2022_jp')() + decoder.decode(b'\x1b%') + decoder.reset() + with self.assertRaises(UnicodeDecodeError): + decoder.decode(b'\xff', final=True) + def test_dbcs(self): # cp949 decoder is simple with only 1 or 2 bytes sequences. decoder = codecs.getincrementaldecoder('cp949')() @@ -325,7 +348,7 @@ def test_setstate_validates_input(self): self.assertRaises(TypeError, decoder.setstate, 123) self.assertRaises(TypeError, decoder.setstate, ("invalid", 0)) self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid")) - self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0)) + self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"0" * 17, 0)) def test_setstate_invalid_designation(self): # gh-153603: an unknown charset designation in the state must not crash diff --git a/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst b/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst new file mode 100644 index 00000000000000..2860dc6897ecc0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst @@ -0,0 +1,4 @@ +Fix HZ encoding of a tilde at an output buffer boundary and ISO-2022 decoding +of split escape sequences. Reset ISO-2022 incremental codecs to their initial +state. Keep multibyte codec tables and custom encoder error policies alive +when Python code replaces them during use. diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index dc89f1d899e36b..ede54d09f2875e 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -361,11 +361,13 @@ ENCODER(hz) NEXT_OUT(2); state->c[CN_STATE_OFFSET] = 0; } - WRITEBYTE1((unsigned char)c); - NEXT(1, 1); if (c == '~') { - WRITEBYTE1('~'); - NEXT_OUT(1); + WRITEBYTE2('~', '~'); + NEXT(1, 2); + } + else { + WRITEBYTE1((unsigned char)c); + NEXT(1, 1); } continue; } diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index 662c2e1188e817..38a8a230c7ae45 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -174,9 +174,8 @@ ENCODER_RESET(iso2022) if (STATE_G0 != CHARSET_ASCII) { WRITEBYTE3(ESC, '(', 'B'); NEXT_OUT(3); - STATE_SETG0(CHARSET_ASCII); } - return 0; + return iso2022_encode_init(state, codec); } ENCODER(iso2022) @@ -311,9 +310,7 @@ DECODER_INIT(iso2022) DECODER_RESET(iso2022) { - STATE_SETG0(CHARSET_ASCII); - STATE_CLEARFLAG(F_SHIFTED); - return 0; + return iso2022_decode_init(state, codec); } static Py_ssize_t diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index e106e4d546532e..eda366cd91d9c0 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -521,6 +521,11 @@ multibytecodec_encode(const MultibyteCodec *codec, if (datalen == 0 && !(flags & MBENC_RESET)) return PyBytes_FromStringAndSize(NULL, 0); + /* A callback may change a stateful encoder's errors attribute. */ + if (ERROR_ISCUSTOM(errors)) { + Py_INCREF(errors); + } + buf.excobj = NULL; buf.outobj = NULL; buf.inobj = text; /* borrowed reference */ @@ -581,10 +586,12 @@ multibytecodec_encode(const MultibyteCodec *codec, if (inpos_t) *inpos_t = buf.inpos; Py_XDECREF(buf.excobj); + ERROR_DECREF(errors); return buf.outobj; errorexit: Py_XDECREF(buf.excobj); + ERROR_DECREF(errors); Py_XDECREF(buf.outobj); return NULL; } @@ -1120,7 +1127,7 @@ mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->codec->encinit(&self->state, self->codec) != 0) goto errorexit; - Py_DECREF(codec); + self->codec_owner = codec; return (PyObject *)self; errorexit: @@ -1141,6 +1148,7 @@ mbiencoder_traverse(PyObject *op, visitproc visit, void *arg) MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op); if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); + Py_VISIT(self->codec_owner); return 0; } @@ -1152,6 +1160,7 @@ mbiencoder_dealloc(PyObject *op) PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); Py_CLEAR(self->pending); + Py_XDECREF(self->codec_owner); tp->tp_free(self); Py_DECREF(tp); } @@ -1403,7 +1412,7 @@ mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->codec->decinit(&self->state, self->codec) != 0) goto errorexit; - Py_DECREF(codec); + self->codec_owner = codec; return (PyObject *)self; errorexit: @@ -1424,6 +1433,7 @@ mbidecoder_traverse(PyObject *op, visitproc visit, void *arg) MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op); if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); + Py_VISIT(self->codec_owner); return 0; } @@ -1434,6 +1444,7 @@ mbidecoder_dealloc(PyObject *op) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); + Py_XDECREF(self->codec_owner); tp->tp_free(self); Py_DECREF(tp); } @@ -1720,7 +1731,7 @@ mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->codec->decinit(&self->state, self->codec) != 0) goto errorexit; - Py_DECREF(codec); + self->codec_owner = codec; return (PyObject *)self; errorexit: @@ -1742,6 +1753,7 @@ mbstreamreader_traverse(PyObject *op, visitproc visit, void *arg) if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); Py_VISIT(self->stream); + Py_VISIT(self->codec_owner); return 0; } @@ -1753,6 +1765,7 @@ mbstreamreader_dealloc(PyObject *op) PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); Py_XDECREF(self->stream); + Py_XDECREF(self->codec_owner); tp->tp_free(self); Py_DECREF(tp); } @@ -1944,7 +1957,7 @@ mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->codec->encinit(&self->state, self->codec) != 0) goto errorexit; - Py_DECREF(codec); + self->codec_owner = codec; return (PyObject *)self; errorexit: @@ -1966,6 +1979,7 @@ mbstreamwriter_traverse(PyObject *op, visitproc visit, void *arg) if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); Py_VISIT(self->stream); + Py_VISIT(self->codec_owner); return 0; } @@ -1977,6 +1991,7 @@ mbstreamwriter_dealloc(PyObject *op) PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); Py_XDECREF(self->stream); + Py_XDECREF(self->codec_owner); tp->tp_free(self); Py_DECREF(tp); } diff --git a/Modules/cjkcodecs/multibytecodec.h b/Modules/cjkcodecs/multibytecodec.h index 5b85e2e3de316d..bacd6e1078b685 100644 --- a/Modules/cjkcodecs/multibytecodec.h +++ b/Modules/cjkcodecs/multibytecodec.h @@ -77,6 +77,7 @@ typedef struct { #define _MultibyteStatefulCodec_HEAD \ PyObject_HEAD \ const MultibyteCodec *codec; \ + PyObject *codec_owner; \ MultibyteCodec_State state; \ PyObject *errors; typedef struct { @@ -91,7 +92,8 @@ typedef struct { _MultibyteStatefulEncoder_HEAD } MultibyteStatefulEncoderContext; -#define MAXDECPENDING 8 +/* ISO-2022 may retain an incomplete escape sequence up to 15 bytes. */ +#define MAXDECPENDING 16 #define _MultibyteStatefulDecoder_HEAD \ _MultibyteStatefulCodec_HEAD \ unsigned char pending[MAXDECPENDING]; \ _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
