https://github.com/python/cpython/commit/6d18fc954489441fec68b67a9540206f4ec844e0 commit: 6d18fc954489441fec68b67a9540206f4ec844e0 branch: 3.15 author: Miss Islington (bot) <[email protected]> committer: gpshead <[email protected]> date: 2026-11-01T09:23:22-08:00 summary:
[3.15] gh-160185: Fix CJK StreamWriter.reset() not returning to ASCII (GH-160186) (#161067) gh-160185: Fix CJK StreamWriter.reset() not returning to ASCII (GH-160186) reset() returned early when nothing was pending, so the shift-back sequence of hz and iso2022_* was never written (regression from gh-67436). Also fix a reference leak of the stream.write() result. (cherry picked from commit aa4f541a9742fa1de1008d13cbf23b2eb3799e3a) Co-authored-by: Gregory P. Smith <[email protected]> files: A Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst M Lib/codecs.py M Lib/test/test_multibytecodec.py M Modules/cjkcodecs/multibytecodec.c diff --git a/Lib/codecs.py b/Lib/codecs.py index e4a8010aba90a5..a43421e1ea0508 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -400,9 +400,10 @@ def reset(self): pass def seek(self, offset, whence=0): - self.stream.seek(offset, whence) + # reset() may write to the stream: call it at the old position. if whence == 0 and offset == 0: self.reset() + self.stream.seek(offset, whence) def __getattr__(self, name, getattr=getattr): @@ -739,10 +740,11 @@ def reset(self): self.writer.reset() def seek(self, offset, whence=0): - self.stream.seek(offset, whence) - self.reader.reset() + # writer.reset() may write to the stream: call it at the old position. if whence == 0 and offset == 0: self.writer.reset() + self.stream.seek(offset, whence) + self.reader.reset() def __getattr__(self, name, getattr=getattr): @@ -862,8 +864,8 @@ def reset(self): def seek(self, offset, whence=0): # Seeks must be propagated to both the readers and writers # as they might need to reset their internal buffers. - self.reader.seek(offset, whence) self.writer.seek(offset, whence) + self.reader.seek(offset, whence) def __getattr__(self, name, getattr=getattr): diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 39d00d8df3c3d3..b863e0f1edeba6 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -461,6 +461,42 @@ def test_incrementalencoder_final(self): self.assertEqual(output, self.expected_reset) self.assertEqual(encoder.encode('', final=True), b'') + def test_streamwriter_reset(self): + stream = io.BytesIO() + writer = codecs.getwriter(self.encoding)(stream) + writer.reset() + self.assertEqual(stream.getvalue(), b'') + writer.write(self.text) + self.assertEqual(stream.getvalue(), self.expected) + writer.reset() + self.assertEqual(stream.getvalue(), self.expected_reset) + writer.reset() + self.assertEqual(stream.getvalue(), self.expected_reset) + writer.write(self.text) + writer.reset() + self.assertEqual(stream.getvalue(), self.expected_reset * 2) + + def test_streamwriter_seek0(self): + # The reset sequence is written before the stream is rewound. + info = codecs.lookup(self.encoding) + for wrap in ( + info.streamwriter, + lambda stream: codecs.StreamReaderWriter( + stream, info.streamreader, info.streamwriter), + lambda stream: codecs.EncodedFile(stream, 'utf-8', self.encoding), + ): + stream = io.BytesIO() + writer = wrap(stream) + text = self.text + if isinstance(writer, codecs.StreamRecoder): + text = text.encode() + writer.write(text) + writer.seek(0) + self.assertEqual(stream.tell(), 0) + self.assertEqual(stream.getvalue(), self.expected_reset) + writer.write(text) + self.assertEqual(stream.getvalue(), self.expected_reset) + def test_incrementalencoder_error(self): # The output of a failed encode() is lost, so the shift state # must not be changed. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst new file mode 100644 index 00000000000000..a9d7aef03214a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst @@ -0,0 +1,5 @@ +Fix :meth:`codecs.StreamWriter.reset` of the stateful CJK codecs (``hz`` and +the ``iso2022_*`` family) not writing the sequence that returns the output to +ASCII. ``seek(0)`` of :class:`codecs.StreamWriter`, +:class:`codecs.StreamReaderWriter` and :class:`codecs.StreamRecoder` now +resets the writer before moving the stream position. diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 7af02e5a9fba1d..1c5e1f4813aa41 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -1879,13 +1879,17 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se PyTypeObject *cls) /*[clinic end generated code: output=32ef224c2a38aa3d input=28af6a9cd38d1979]*/ { - PyObject *pwrt; + PyObject *pwrt, *pending; - if (!self->pending) - Py_RETURN_NONE; + /* even without pending characters, a stateful codec may have to + * write a sequence that returns the output to its initial state */ + pending = self->pending; + if (pending == NULL) { + pending = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_STR); + } pwrt = multibytecodec_encode(self->codec, &self->state, - self->pending, NULL, self->errors, + pending, NULL, self->errors, MBENC_FLUSH | MBENC_RESET); /* some pending buffer can be truncated when UnicodeEncodeError is * raised on 'strict' mode. but, 'reset' method is designed to @@ -1908,6 +1912,7 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se Py_DECREF(pwrt); return NULL; } + Py_DECREF(wr); } Py_DECREF(pwrt); _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
