https://github.com/python/cpython/commit/e2311cfb3dd518f008f312fe0631f4f7490d237a commit: e2311cfb3dd518f008f312fe0631f4f7490d237a branch: main author: rasmusfaber <[email protected]> committer: encukou <[email protected]> date: 2026-09-10T15:39:59+02:00 summary:
gh-156002: Keep reading through monkey-patched zipfile decompressors (GH-157180) GH-156003 made ZipExtFile._read1() call decompress(data, max_length) on non-deflate decompressors and consult needs_input before reading more. A decompressor installed by monkey-patching _get_decompressor() (as projects like zipfile-zstd, zipfile-deflate64, ... do) may support neither, and every read through it then failed with AttributeError. - Make LZMADecompressor.needs_input public to simplify implementation. - Make the needs_input attribute optional. - If `decompress()` fails with TypeError, try again with one argument. - Since the fallback to one-argument call is a maintenance burden, raise DeprecationWarning. - Add tests for future changes, so we can make informed decisions about breaking monkey-patchers. Co-authored-by: Petr Viktorin <[email protected]> files: A Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst M Lib/_py_warnings.py M Lib/test/test_zipfile/test_core.py M Lib/zipfile/__init__.py diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index ab09913de6812dd..c82d3a21981d0f0 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -873,7 +873,8 @@ def wrapper(*args, **kwargs): _DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" -def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): +def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info, + stacklevel=3): """Warn that *name* is deprecated or should be removed. RuntimeError is raised if *remove* specifies a major/minor tuple older than @@ -889,7 +890,7 @@ def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_i raise RuntimeError(msg) else: msg = message.format(name=name, remove=remove_formatted) - _wm.warn(msg, DeprecationWarning, stacklevel=3) + _wm.warn(msg, DeprecationWarning, stacklevel=stacklevel) # Private utility function called by _PyErr_WarnUnawaitedCoroutine diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index fdf2cd26f8c7c64..708db4d4387df5e 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -33,7 +33,9 @@ with_source_date_epoch, without_source_date_epoch, ) from test.support.import_helper import ensure_lazy_imports -from test.support.warnings_helper import check_no_resource_warning +from test.support.warnings_helper import ( + check_no_resource_warning, ignore_warnings, +) TESTFN2 = TESTFN + "2" @@ -4916,6 +4918,75 @@ class ZstdBoundedDecompressTests(AbstractBoundedDecompressTests, compression = zipfile.ZIP_ZSTANDARD +class MonkeypatchedDecompressorTests(unittest.TestCase): + # Some third-party projects monkey-patch _get_decompressor() to add + # additional compression schemes. This can break at any time as the + # internal compressor objects change. + # To protect users, we try to keep this case working. + # See also: GH-156002 and GH-113767. + COMPRESSION = 99 + + class Compressor: + """Compressor with only the original BZ2Compressor API""" + def compress(self, data): + return data.swapcase() + + def flush(self): + return b'' + + class Decompressor: + """Decompressor with only the 3.3+ BZ2Decompressor API""" + eof = False + + def decompress(self, data): + return data.swapcase() + + def setUp(self): + orig_check_compression = zipfile._check_compression + orig_get_compressor = zipfile._get_compressor + orig_get_decompressor = zipfile._get_decompressor + + def check_compression(compression): + if compression != self.COMPRESSION: + orig_check_compression(compression) + + def get_compressor(compress_type, compresslevel=None): + if compress_type == self.COMPRESSION: + return self.Compressor() + return orig_get_compressor(compress_type, compresslevel) + + def get_decompressor(compress_type): + if compress_type == self.COMPRESSION: + return self.Decompressor() + return orig_get_decompressor(compress_type) + + self.enterContext(mock.patch.object( + zipfile, '_check_compression', check_compression)) + self.enterContext(mock.patch.object( + zipfile, '_get_compressor', get_compressor)) + self.enterContext(mock.patch.object( + zipfile, '_get_decompressor', get_decompressor)) + + def test_roundtrip_monkeypatched_decompressor(self): + data = bytes(range(256)) * 8 + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=self.COMPRESSION) as zf: + zf.writestr("member", data) + self.assertIn(data.swapcase(), buf.getvalue()) + with (ignore_warnings(category=DeprecationWarning, + message='.*two arguments.*'), + zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf): + self.assertEqual(zf.read("member"), data) + with zf.open("member") as f: + self.assertEqual(f.read(100), data[:100]) + self.assertEqual(f.read1(100), data[100:200]) + f.seek(-100, os.SEEK_END) + self.assertEqual(f.read(), data[-100:]) + # Rewinding past the read buffer re-creates the decompressor. + f.seek(0) + self.assertEqual(f.read(), data) + + class AbstractBadCrcTests: def test_testzip_with_bad_crc(self): """Tests that files with bad CRCs return their name from testzip.""" diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 0accf324c90e3fd..d817bdd8769e7d7 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -802,7 +802,7 @@ def unused_data(self): return b'' @property - def _needs_input(self): + def needs_input(self): # While the LZMA properties header is still being buffered, more input # is required; afterwards defer to the wrapped decompressor so a bounded # decompress() call can be drained across reads. @@ -893,13 +893,6 @@ def _get_compressor(compress_type, compresslevel=None): return None -def _decompressor_needs_input(decompressor): - # bz2/zstd expose the stdlib decompressor's public needs_input; the LZMA - # wrapper keeps it private (_needs_input) to avoid adding public API. - needs_input = getattr(decompressor, "needs_input", None) - return decompressor._needs_input if needs_input is None else needs_input - - def _get_decompressor(compress_type): _check_compression(compress_type) if compress_type == ZIP_STORED: @@ -1207,7 +1200,7 @@ def _read1(self, n): else: # bzip2/lzma/zstd: a bounded decompress() call may leave input # buffered inside the decompressor; drain that before reading more. - if _decompressor_needs_input(self._decompressor): + if getattr(self._decompressor, "needs_input", True): data = self._read2(n) else: data = b'' @@ -1226,10 +1219,23 @@ def _read1(self, n): # Bound the output of a single decompress() call (mirroring the # DEFLATE path above) so that a small compressed member cannot # expand into one unbounded read. - data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + try: + data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + except TypeError: + # See MonkeypatchedDecompressorTests in test_core.py + warnings._deprecated( + 'one-argument decompress()', + 'The decompress() method of ' + + type(self._decompressor).__name__ + + ' should take two arguments, data and max_length.' + + ' One-argument calls will stop working before' + + ' Python 3.21.', + remove=(3, 21), + stacklevel=4) + data = self._decompressor.decompress(data) self._eof = (self._decompressor.eof or self._compress_left <= 0 and - _decompressor_needs_input(self._decompressor)) + getattr(self._decompressor, "needs_input", True)) data = data[:self._left] self._left -= len(data) diff --git a/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst new file mode 100644 index 000000000000000..3fc3b6d4c2279be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst @@ -0,0 +1,6 @@ +:mod:`zipfile` again reads members through a third-party decompressor +installed by monkey-patching the private ``_get_decompressor()`` to return an +object that only implements old BZ2Decompressor API from Python 3.3. +Calling decompress() with one argument is deprecated. +Note that decompressors without ``needs_input`` and two-argument +``decompress()`` are vulnerable to :cve:`2026-15310`. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
