https://github.com/python/cpython/commit/a4f80477b02d53a592e6d3891686921c111ae771
commit: a4f80477b02d53a592e6d3891686921c111ae771
branch: main
author: Gregory P. Smith <[email protected]>
committer: gpshead <[email protected]>
date: 2026-10-31T20:13:35Z
summary:
gh-160314: Raise for a damaged later stream in lzma and bz2 (GH-160315)
A damaged .xz or bzip2 stream that followed another stream was taken
for trailing data and silently ignored. Leftover data that starts with
the stream magic is now treated as a stream, as gzip does.
ANT-2026-3TVAVL5A
files:
A Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst
M Doc/library/bz2.rst
M Doc/library/lzma.rst
M Lib/bz2.py
M Lib/compression/_common/_streams.py
M Lib/lzma.py
M Lib/test/test_bz2.py
M Lib/test/test_lzma.py
diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst
index e52565dd4bb0a77..bb7028526acf972 100644
--- a/Doc/library/bz2.rst
+++ b/Doc/library/bz2.rst
@@ -191,6 +191,10 @@ The :mod:`!bz2` module contains:
readers or writers, just like its equivalent classes in :mod:`gzip` and
:mod:`lzma` have always been.
+ .. versionchanged:: next
+ A damaged bzip2 stream that follows another stream now raises
+ :exc:`OSError` instead of being ignored as trailing data.
+
Incremental (de)compression
---------------------------
@@ -307,6 +311,10 @@ One-shot (de)compression
.. versionchanged:: 3.3
Support for multi-stream inputs was added.
+ .. versionchanged:: next
+ A damaged bzip2 stream that follows another stream now raises
+ :exc:`OSError` instead of being ignored as trailing data.
+
Miscellaneous
-------------
diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst
index ae3dcd811fbec0f..ef6e931fb11f4fc 100644
--- a/Doc/library/lzma.rst
+++ b/Doc/library/lzma.rst
@@ -140,6 +140,10 @@ Reading and writing compressed files
.. versionchanged:: 3.6
Accepts a :term:`path-like object`.
+ .. versionchanged:: next
+ A damaged ``.xz`` stream that follows another stream now raises
+ :exc:`LZMAError` instead of being ignored as trailing data.
+
Compressing and decompressing data in memory
--------------------------------------------
@@ -300,6 +304,10 @@ Compressing and decompressing data in memory
See :class:`LZMADecompressor` above for a description of the *format*,
*memlimit* and *filters* arguments.
+ .. versionchanged:: next
+ A damaged ``.xz`` stream that follows another stream now raises
+ :exc:`LZMAError` instead of being ignored as trailing data.
+
Miscellaneous
-------------
diff --git a/Lib/bz2.py b/Lib/bz2.py
index bcfc5ceb6dc3458..3f575088571fa90 100644
--- a/Lib/bz2.py
+++ b/Lib/bz2.py
@@ -25,6 +25,9 @@
# Value 2 no longer used
_MODE_WRITE = 3
+# Every bzip2 stream starts with one of these, "BZh" and the block size.
+_STREAM_MAGIC = (b'BZh1', b'BZh2', b'BZh3', b'BZh4', b'BZh5', b'BZh6',
b'BZh7', b'BZh8', b'BZh9')
+
class BZ2File(_streams.BaseStream):
@@ -92,7 +95,8 @@ def __init__(self, filename, mode="r", *, compresslevel=9):
if self._mode == _MODE_READ:
raw = _streams.DecompressReader(self._fp,
- BZ2Decompressor, trailing_error=OSError)
+ BZ2Decompressor, trailing_error=OSError,
+ stream_magic=_STREAM_MAGIC)
self._buffer = io.BufferedReader(raw)
else:
self._pos = 0
@@ -343,10 +347,11 @@ def decompress(data):
try:
res = decomp.decompress(data)
except OSError:
- if results:
- break # Leftover data is not a valid bzip2 stream; ignore it.
- else:
+ if not results:
raise # Error on the first iteration; bail out.
+ if data.startswith(_STREAM_MAGIC):
+ raise # Leftover data is a damaged bzip2 stream; bail out.
+ break # Leftover data is not a valid bzip2 stream; ignore it.
results.append(res)
if not decomp.eof:
raise ValueError("Compressed data ended before the "
diff --git a/Lib/compression/_common/_streams.py
b/Lib/compression/_common/_streams.py
index 9f367d4e30440f7..09c65507297886b 100644
--- a/Lib/compression/_common/_streams.py
+++ b/Lib/compression/_common/_streams.py
@@ -36,7 +36,8 @@ class DecompressReader(io.RawIOBase):
def readable(self):
return True
- def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
+ def __init__(self, fp, decomp_factory, trailing_error=(), *,
+ stream_magic=(), **decomp_args):
self._fp = fp
self._eof = False
self._pos = 0 # Current offset in decompressed stream
@@ -56,6 +57,11 @@ def __init__(self, fp, decomp_factory, trailing_error=(),
**decomp_args):
# trailing data to ignore
self._trailing_error = trailing_error
+ # Prefix (or tuple of prefixes) that every compressed stream starts
+ # with, if the format has one. Data starting with it is a damaged
+ # stream rather than trailing data, so its error is not ignored.
+ self._stream_magic = stream_magic
+
def close(self):
self._decompressor = None
return super().close()
@@ -90,6 +96,9 @@ def read(self, size=-1):
try:
data = self._decompressor.decompress(rawblock, size)
except self._trailing_error:
+ if (self._stream_magic
+ and rawblock.startswith(self._stream_magic)):
+ raise # Damaged stream, not trailing data.
# Trailing data isn't a valid compressed stream; ignore it.
break
else:
diff --git a/Lib/lzma.py b/Lib/lzma.py
index 73a7c702e037112..ac2c2f7a76a8c44 100644
--- a/Lib/lzma.py
+++ b/Lib/lzma.py
@@ -37,6 +37,13 @@
_MODE_WRITE = 3
+def _stream_magic(format):
+ # Every stream of an .xz file starts with these bytes. The other
+ # formats, including an .lzma stream read with FORMAT_AUTO, have
+ # nothing that tells a damaged stream from trailing data.
+ return b"\xfd7zXZ\x00" if format in (FORMAT_AUTO, FORMAT_XZ) else b""
+
+
class LZMAFile(_streams.BaseStream):
"""A file object providing transparent LZMA (de)compression.
@@ -130,7 +137,8 @@ def __init__(self, filename=None, mode="r", *,
if self._mode == _MODE_READ:
raw = _streams.DecompressReader(self._fp, LZMADecompressor,
- trailing_error=LZMAError, format=format, filters=filters)
+ trailing_error=LZMAError, stream_magic=_stream_magic(format),
+ format=format, filters=filters)
self._buffer = io.BufferedReader(raw)
def close(self):
@@ -347,15 +355,17 @@ def decompress(data, format=FORMAT_AUTO, memlimit=None,
filters=None):
For incremental decompression, use an LZMADecompressor instead.
"""
results = []
+ magic = _stream_magic(format)
while True:
decomp = LZMADecompressor(format, memlimit, filters)
try:
res = decomp.decompress(data)
except LZMAError:
- if results:
- break # Leftover data is not a valid LZMA/XZ stream; ignore
it.
- else:
+ if not results:
raise # Error on the first iteration; bail out.
+ if magic and data.startswith(magic):
+ raise # Leftover data is a damaged XZ stream; bail out.
+ break # Leftover data is not a valid LZMA/XZ stream; ignore it.
results.append(res)
if not decomp.eof:
raise LZMAError("Compressed data ended before the "
diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py
index 2f59f337abbe320..7c6c3d4d6372c0a 100644
--- a/Lib/test/test_bz2.py
+++ b/Lib/test/test_bz2.py
@@ -156,6 +156,22 @@ def testReadMultiStreamTrailingJunk(self):
with BZ2File(self.filename) as bz2f:
self.assertEqual(bz2f.read(), self.TEXT * 5)
+ def testReadMultiStreamDamagedStream(self):
+ # A damaged bzip2 stream following a valid one is not trailing junk,
+ # whatever the size of the reads.
+ for pos in 4, 100, -3:
+ with self.subTest(pos=pos):
+ damaged = bytearray(self.DATA)
+ damaged[pos] ^= 1
+ self.createTempFile(suffix=damaged)
+ with BZ2File(self.filename) as bz2f:
+ self.assertRaises(OSError, bz2f.read)
+ with BZ2File(self.filename) as bz2f:
+ self.assertEqual(bz2f.read(len(self.TEXT)), self.TEXT)
+ with self.assertRaises(OSError):
+ while bz2f.read(10):
+ pass
+
def testRead0(self):
self.createTempFile()
with BZ2File(self.filename) as bz2f:
@@ -1100,6 +1116,18 @@ def testDecompressMultiStreamTrailingJunk(self):
text = bz2.decompress(self.DATA * 5 + self.BAD_DATA)
self.assertEqual(text, self.TEXT * 5)
+ def testDecompressMultiStreamDamagedStream(self):
+ # A damaged bzip2 stream following a valid one is not trailing junk.
+ for pos in 4, 100, -3:
+ with self.subTest(pos=pos):
+ damaged = bytearray(self.DATA)
+ damaged[pos] ^= 1
+ self.assertRaises(OSError, bz2.decompress,
+ self.DATA + damaged)
+ # Not a bzip2 header: the block size digit must be 1 to 9.
+ text = bz2.decompress(self.DATA + b"BZh0" + self.BAD_DATA)
+ self.assertEqual(text, self.TEXT)
+
class OpenTest(BaseTest):
"Test the open function."
diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py
index df0b66039d77a1a..d5431ed6df49d52 100644
--- a/Lib/test/test_lzma.py
+++ b/Lib/test/test_lzma.py
@@ -524,6 +524,16 @@ def test_decompress_multistream_trailing_junk(self):
ddata = lzma.decompress(COMPRESSED_XZ * 3 + COMPRESSED_BOGUS)
self.assertEqual(ddata, INPUT * 3)
+ def test_decompress_multistream_damaged_stream(self):
+ # A damaged .xz stream following a valid one is not trailing junk.
+ for pos in 6, 100, -20, -1:
+ damaged = bytearray(COMPRESSED_XZ)
+ damaged[pos] ^= 1
+ for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ:
+ with self.subTest(pos=pos, format=format):
+ self.assertRaises(LZMAError, lzma.decompress,
+ COMPRESSED_XZ + damaged, format)
+
class TempFile:
"""Context manager - creates a file, and deletes it on __exit__."""
@@ -885,6 +895,25 @@ def test_read_multistream_trailing_junk(self):
with LZMAFile(BytesIO(COMPRESSED_XZ * 5 + COMPRESSED_BOGUS)) as f:
self.assertEqual(f.read(), INPUT * 5)
+ def test_read_multistream_damaged_stream(self):
+ # A damaged .xz stream following a valid one is not trailing junk,
+ # whatever the size of the reads.
+ for pos in 6, 100, -20, -1:
+ damaged = bytearray(COMPRESSED_XZ)
+ damaged[pos] ^= 1
+ cdata = COMPRESSED_XZ + damaged
+ for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ:
+ with self.subTest(pos=pos, format=format):
+ with LZMAFile(BytesIO(cdata), format=format) as f:
+ self.assertRaises(LZMAError, f.read)
+ with LZMAFile(BytesIO(cdata), format=format) as f:
+ self.assertEqual(f.read(len(INPUT)), INPUT)
+ with self.assertRaises(LZMAError):
+ while f.read(10):
+ pass
+ with LZMAFile(BytesIO(cdata), format=format) as f:
+ self.assertRaises(LZMAError, f.seek, 0, 2)
+
def test_read_from_file(self):
with TempFile(TESTFN, COMPRESSED_XZ):
with LZMAFile(TESTFN) as f:
diff --git
a/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst
b/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst
new file mode 100644
index 000000000000000..3694e255546f68a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst
@@ -0,0 +1,4 @@
+Raise an error in :class:`lzma.LZMAFile`, :func:`lzma.decompress`,
+:class:`bz2.BZ2File` and :func:`bz2.decompress` for a damaged ``.xz`` or bzip2
+stream that follows another stream, instead of silently ignoring it and
+everything after it as trailing data.
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
