https://github.com/python/cpython/commit/6257029de42ef89b67f3d20137de87d43e197530 commit: 6257029de42ef89b67f3d20137de87d43e197530 branch: 3.11 author: Miss Islington (bot) <[email protected]> committer: pablogsal <[email protected]> date: 2026-10-01T01:30:19+01:00 summary:
[3.11] gh-156002: Bound zipfile decompression for bzip2/LZMA/Zstandard (GH-156003) (GH-156362) (#156740) * [3.15] gh-156002: Bound zipfile decompression for bzip2/LZMA/Zstandard (GH-156003) (GH-156362) Patch by @tonghuaroot. zipfile.ZipExtFile._read1() bounds the output of each decompress() call for DEFLATE members by passing a max_length to zlib, but for bzip2, LZMA, and Zstandard members it called decompress() with no bound. A whole compressed chunk was therefore expanded into a single allocation before the data[:self._left] clip ran, so a consumer that deliberately reads in small chunks to limit memory (for example zf.open(name).read(8192)) was silently unprotected for non-DEFLATE members. A small, spec-conformant archive member declaring a large uncompressed size could drive multi-GB peak memory. _read1() now passes a per-call bound to the non-DEFLATE decompress() (mirroring the DEFLATE branch) and drains the decompressor's internal buffer across calls by checking needs_input before reading more compressed input. zipfile's LZMADecompressor wrapper forwards max_length and exposes needs_input so the bound also holds for LZMA members. (cherry picked from commit f897dbf2f36a5935700b7c2d94d4681d2136b7d4) (cherry picked from commit 1b424c0178a01e155fd0267dc28a8fc1159b33a8) Co-authored-by: Petr Viktorin <[email protected]> Co-authored-by: tonghuaroot <[email protected]> * Remove zstd test (3.14+) * gh-156002: Keep reading through monkey-patched zipfile decompressors (GH-157180) (GH-157557) (cherry picked from commit f507e6946a3194e83e1d7b8ee6e14567175e46de) Co-authored-by: Petr Viktorin <[email protected]> Co-authored-by: rasmusfaber <[email protected]> * Don't use the :cve: RST role, it doesn't exist yet --------- Co-authored-by: Petr Viktorin <[email protected]> Co-authored-by: tonghuaroot <[email protected]> Co-authored-by: rasmusfaber <[email protected]> files: A Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst A Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst M Lib/test/test_zipfile.py M Lib/zipfile.py diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index a0bca62ab88ac34..5726feb44fe7acc 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2418,6 +2418,110 @@ def tearDown(self): unlink(TESTFN2) +class AbstractBoundedDecompressTests: + # ZipExtFile._read1() bounds the output of each decompress() call so that a + # small member declaring a large uncompressed size cannot expand into one + # unbounded read. + def test_read1_output_is_bounded(self): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=self.compression) as zf: + zf.writestr("big", b"\0" * (4 * 1024 * 1024)) + with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: + with zf.open("big") as f: + self.assertLessEqual(len(f._read1(100)), f.MIN_READ_SIZE) + + +class StoredBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_STORED + + +@requires_zlib() +class DeflateBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + + +@requires_bz2() +class Bzip2BoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + + +@requires_lzma() +class LzmaBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_LZMA + + + +class MonkeypatchedDecompressorTests(unittest.TestCase): + # Some third-party projects monkey-patch _get_decompressor() to add + # additional compression schemes. This can break at any time as the + # internal compressor objects change. + # To protect users, we try to keep this case working. + # See also: GH-156002 and GH-113767. + COMPRESSION = 99 + + class Compressor: + """Compressor with only the original BZ2Compressor API""" + def compress(self, data): + return data.swapcase() + + def flush(self): + return b'' + + class Decompressor: + """Decompressor with only the 3.3+ BZ2Decompressor API""" + eof = False + + def decompress(self, data): + return data.swapcase() + + def setUp(self): + orig_check_compression = zipfile._check_compression + orig_get_compressor = zipfile._get_compressor + orig_get_decompressor = zipfile._get_decompressor + + def check_compression(compression): + if compression != self.COMPRESSION: + orig_check_compression(compression) + + def get_compressor(compress_type, compresslevel=None): + if compress_type == self.COMPRESSION: + return self.Compressor() + return orig_get_compressor(compress_type, compresslevel) + + def get_decompressor(compress_type): + if compress_type == self.COMPRESSION: + return self.Decompressor() + return orig_get_decompressor(compress_type) + + self.enterContext(mock.patch.object( + zipfile, '_check_compression', check_compression)) + self.enterContext(mock.patch.object( + zipfile, '_get_compressor', get_compressor)) + self.enterContext(mock.patch.object( + zipfile, '_get_decompressor', get_decompressor)) + + def test_roundtrip_monkeypatched_decompressor(self): + data = bytes(range(256)) * 8 + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=self.COMPRESSION) as zf: + zf.writestr("member", data) + self.assertIn(data.swapcase(), buf.getvalue()) + with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: + self.assertEqual(zf.read("member"), data) + with zf.open("member") as f: + self.assertEqual(f.read(100), data[:100]) + self.assertEqual(f.read1(100), data[100:200]) + f.seek(-100, os.SEEK_END) + self.assertEqual(f.read(), data[-100:]) + # Rewinding past the read buffer re-creates the decompressor. + f.seek(0) + self.assertEqual(f.read(), data) + + class AbstractBadCrcTests: def test_testzip_with_bad_crc(self): """Tests that files with bad CRCs return their name from testzip.""" diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 9789b06e021a65c..f06dd2e3a4065cb 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -676,7 +676,16 @@ def __init__(self): self._unconsumed = b'' self.eof = False - def decompress(self, data): + @property + def needs_input(self): + # While the LZMA properties header is still being buffered, more input + # is required; afterwards defer to the wrapped decompressor so a bounded + # decompress() call can be drained across reads. + if self._decomp is None: + return True + return self._decomp.needs_input + + def decompress(self, data, max_length=-1): if self._decomp is None: self._unconsumed += data if len(self._unconsumed) <= 4: @@ -692,7 +701,7 @@ def decompress(self, data): data = self._unconsumed[4 + psize:] del self._unconsumed - result = self._decomp.decompress(data) + result = self._decomp.decompress(data, max_length) self.eof = self._decomp.eof return result @@ -1048,8 +1057,15 @@ def _read1(self, n): data = self._decompressor.unconsumed_tail if n > len(data): data += self._read2(n - len(data)) - else: + elif self._compress_type == ZIP_STORED: data = self._read2(n) + else: + # bzip2/lzma/zstd: a bounded decompress() call may leave input + # buffered inside the decompressor; drain that before reading more. + if getattr(self._decompressor, "needs_input", True): + data = self._read2(n) + else: + data = b'' if self._compress_type == ZIP_STORED: self._eof = self._compress_left <= 0 @@ -1062,8 +1078,17 @@ def _read1(self, n): if self._eof: data += self._decompressor.flush() else: - data = self._decompressor.decompress(data) - self._eof = self._decompressor.eof or self._compress_left <= 0 + # Bound the output of a single decompress() call (mirroring the + # DEFLATE path above) so that a small compressed member cannot + # expand into one unbounded read. + try: + data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + except TypeError: + # See MonkeypatchedDecompressorTests in test_core.py + data = self._decompressor.decompress(data) + self._eof = (self._decompressor.eof or + self._compress_left <= 0 and + getattr(self._decompressor, "needs_input", True)) data = data[:self._left] self._left -= len(data) diff --git a/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst new file mode 100644 index 000000000000000..689967bff72fe7c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst @@ -0,0 +1,5 @@ +:mod:`zipfile` again reads members through a third-party decompressor +installed by monkey-patching the private ``_get_decompressor()`` to return an +object that only implements old BZ2Decompressor API from Python 3.3. +Note that decompressors without ``needs_input`` and two-argument +``decompress()`` are vulnerable to CVE 2026-15310. diff --git a/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst b/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst new file mode 100644 index 000000000000000..4e49ad5ce8fa00a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst @@ -0,0 +1,4 @@ +Bound the amount of data :mod:`zipfile` decompresses per read for members +compressed with bzip2, LZMA, or Zstandard, matching the existing limit for +deflate. A small archive member could previously expand into an unbounded +allocation even when read in small chunks. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
