https://github.com/python/cpython/commit/1a52eaedce6f1d32cdb5ee18ecec74cfd82d5550 commit: 1a52eaedce6f1d32cdb5ee18ecec74cfd82d5550 branch: main author: Valentin Samir <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-08-15T22:51:09+03:00 summary:
gh-83869: tarfile: compute next header offset using pax size for sparse file (GH-18562) In case of a sparse file, the tarinfo.size attribute is set to the sparse file expanded size (pax attribute GNU.sparse.size or GNU.sparse.size) and do not correspond to the actual size of the data block. The size of the data block is specified by the size pax header if present or by the ustar size header. Moreover, for GNU sparse 1.0 files, the data block start at the beginning of the sparse mapping and not after the sparse mapping and so the offset should be computed from here. Co-authored-by: Serhiy Storchaka <[email protected]> files: A Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 592c4638c52c9b..cee21bfc6fe5aa 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1613,17 +1613,22 @@ def _proc_pax(self, tarfile): if self.type in (XHDTYPE, SOLARIS_XHDTYPE): # Patch the TarInfo object with the extended header info. next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) - next.offset = self.offset if "size" in pax_headers: # If the extended header replaces the size field, # we need to recalculate the offset where the next # header starts. - offset = next.offset_data + offset = next.offset + BLOCKSIZE if next.isreg() or next.type not in SUPPORTED_TYPES: - offset += next._block(next.size) + try: + size = PAX_NUMBER_FIELDS["size"](pax_headers["size"]) + except ValueError: + size = 0 + offset += next._block(size) tarfile.offset = offset + next.offset = self.offset + return next def _proc_gnusparse_00(self, next, raw_headers): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index bd544dfea51da3..c8c9b49fb6fc61 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1400,6 +1400,37 @@ def test_sparse_file_01(self): def test_sparse_file_10(self): self._test_sparse_file("gnu/sparse-1.0") + def test_sparse_file_10_pax_size(self): + # gh-83869: when the pax header replaces the size field, the offset + # of the next header must be computed from the size of the data in + # the archive, not from the apparent size of the sparse file. + data = b"payload!" * 4 + realsize = 1 << 20 + smap = b"1\n%d\n%d\n" % (realsize - len(data), len(data)) + smap += b"\0" * (-len(smap) % tarfile.BLOCKSIZE) + + sparse = tarfile.TarInfo("sparse") + sparse.size = len(smap) + len(data) + sparse.pax_headers = { + "GNU.sparse.major": "1", + "GNU.sparse.minor": "0", + "GNU.sparse.name": "sparse", + "GNU.sparse.realsize": str(realsize), + "size": str(sparse.size), + } + buf = sparse.tobuf(tarfile.PAX_FORMAT) + buf += smap + data + b"\0" * (-len(data) % tarfile.BLOCKSIZE) + + last = tarfile.TarInfo("last") + last.size = len(data) + buf += last.tobuf(tarfile.PAX_FORMAT) + buf += data + b"\0" * (-len(data) % tarfile.BLOCKSIZE) + buf += b"\0" * (tarfile.BLOCKSIZE * 2) + + with tarfile.open(fileobj=io.BytesIO(buf)) as tar: + self.assertEqual(tar.getnames(), ["sparse", "last"]) + self.assertEqual(tar.extractfile("last").read(), data) + @staticmethod def _fs_supports_holes(): # Return True if the platform knows the st_blocks stat attribute and @@ -1473,6 +1504,31 @@ def test_pax_global_headers(self): finally: tar.close() + def test_offset_after_global_header(self): + # gh-83869: a global header is a member of its own, the member which + # follows it keeps the offset of its own header. + rec = b"30 comment=global header here\n" + glob = tarfile.TarInfo("././@PaxHeader") + glob.type = tarfile.XGLTYPE + glob.size = len(rec) + buf = glob.tobuf(tarfile.USTAR_FORMAT) + buf += rec + b"\0" * (-len(rec) % tarfile.BLOCKSIZE) + + member = tarfile.TarInfo("member") + data = b"hello\n" + member.size = len(data) + offset = len(buf) + buf += member.tobuf(tarfile.USTAR_FORMAT) + buf += data + b"\0" * (-len(data) % tarfile.BLOCKSIZE) + buf += b"\0" * (tarfile.BLOCKSIZE * 2) + + with tarfile.open(fileobj=io.BytesIO(buf)) as tar: + tarinfo = tar.getmember("member") + self.assertEqual(tarinfo.offset, offset) + self.assertEqual(tarinfo.pax_headers.get("comment"), + "global header here") + self.assertEqual(tar.extractfile(tarinfo).read(), data) + def test_pax_number_fields(self): # All following number fields are read from the pax header. tar = tarfile.open(tarname, encoding="iso8859-1") diff --git a/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst b/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst new file mode 100644 index 00000000000000..efadbb289c3498 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst @@ -0,0 +1,6 @@ +Fix :mod:`tarfile` reading an archive with a GNU sparse 1.0 member whose +size is set in the pax extended header. +The offset of the next header was computed from the offset of the data, +which is already past the sparse map, and from the size of the member, +which can be the apparent size of the sparse file. +All following members were unreachable. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
