https://github.com/python/cpython/commit/fa1a212903cf2540b3d4b7970e2cd1bfb9df858d commit: fa1a212903cf2540b3d4b7970e2cd1bfb9df858d branch: 3.15 author: Miss Islington (bot) <[email protected]> committer: gpshead <[email protected]> date: 2026-10-25T06:13:47Z summary:
[3.15] gh-160535: Keep the ZstdDict alive until the zstd context is freed (GH-160536) (#160635) gh-160535: Keep the ZstdDict alive until the zstd context is freed (GH-160536) The garbage collector could release a ZstdCompressor's dictionary, or the digested dictionaries cached by the ZstdDict, while worker threads of the compression context were still reading them. They are now only released on deallocation, after the context has been freed. ANT-2026-CT3P1CSJ + Visit the type in the compressor and decompressor traverse; move NEWS to Library (cherry picked from commit b949033ae8b22ff77e798a08f4ecc6eaf4cf0d8a) Co-authored-by: Gregory P. Smith <[email protected]> files: A Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst M Lib/test/test_zstd.py M Modules/_zstd/compressor.c M Modules/_zstd/decompressor.c M Modules/_zstd/zstddict.c diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 9225c42e0e711aa..b133597bcd532c3 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -7,9 +7,12 @@ import os import unittest import tempfile +import textwrap import threading +from test import support from test.support.import_helper import import_module +from test.support.script_helper import assert_python_ok from test.support import threading_helper from test.support import _1M @@ -1548,6 +1551,42 @@ def test_len(self): self.assertEqual(len(TRAINED_DICT), len(TRAINED_DICT.dict_content)) self.assertIn(str(len(TRAINED_DICT)), str(TRAINED_DICT)) + @unittest.skipUnless(SUPPORT_MULTITHREADING, + "zstd build doesn't support multi-threaded compression") + @support.requires_subprocess() + def test_collect_cycle_with_busy_compressor(self): + # The dictionary must outlive the compression context when both are + # collected by the garbage collector while a worker thread is still + # using the dictionary. Whether the failure is detected depends on + # the memory allocator, so use a fresh process for each case. + code = textwrap.dedent(""" + import gc, random, sys + from compression.zstd import (CompressionParameter, + ZstdCompressor, ZstdDict) + + class Cycle: + pass + + job_size = 1 << 20 + content = random.randbytes(1 << 10) * (1 << 8) + zd = ZstdDict(content, is_raw=True) + options = {CompressionParameter.compression_level: 10, + CompressionParameter.nb_workers: 1, + CompressionParameter.job_size: job_size} + c = ZstdCompressor(options=options, + zstd_dict=getattr(zd, sys.argv[1])) + cycle = Cycle() + cycle.cycle = cycle + cycle.compressor = c + # Returns while the worker thread is compressing the first job. + c.compress(random.randbytes(1 << 10) * ((job_size >> 10) + 64)) + del zd, c, cycle + gc.collect() + """) + for attr in 'as_digested_dict', 'as_prefix': + with self.subTest(attr=attr): + assert_python_ok('-c', code, attr) + class FileTestCase(unittest.TestCase): def setUp(self): self.DECOMPRESSED_42 = b'a'*42 diff --git a/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst new file mode 100644 index 000000000000000..ea2082716d64ace --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst @@ -0,0 +1,4 @@ +Fix a crash in :mod:`compression.zstd` when the garbage collector frees a +:class:`~compression.zstd.ZstdCompressor` that uses worker threads together +with its :class:`~compression.zstd.ZstdDict` before the compression of +already submitted data has finished. diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 894568fce28d0d5..19e12ed4af7479a 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -766,17 +766,13 @@ static int ZstdCompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdCompressor *self = ZstdCompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; } -static int -ZstdCompressor_clear(PyObject *ob) -{ - ZstdCompressor *self = ZstdCompressor_CAST(ob); - Py_CLEAR(self->dict); - return 0; -} +/* No tp_clear: the compression context (and its worker threads) uses the + dictionary until ZstdCompressor_dealloc() has freed the context. */ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_new, _zstd_ZstdCompressor_new}, @@ -785,7 +781,6 @@ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_members, ZstdCompressor_members}, {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, - {Py_tp_clear, ZstdCompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index ca9ea7e3b8f99bb..454736d687a4305 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -689,18 +689,13 @@ static int ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdDecompressor *self = ZstdDecompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; } -static int -ZstdDecompressor_clear(PyObject *ob) -{ - ZstdDecompressor *self = ZstdDecompressor_CAST(ob); - Py_CLEAR(self->dict); - Py_CLEAR(self->unused_data); - return 0; -} +/* No tp_clear: the decompression context references the dictionary until + ZstdDecompressor_dealloc() has freed the context. */ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_new, _zstd_ZstdDecompressor_new}, @@ -710,7 +705,6 @@ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_getset, ZstdDecompressor_getset}, {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, - {Py_tp_clear, ZstdDecompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index e1b9d998e697fb8..11ea7619976bf92 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -240,16 +240,11 @@ ZstdDict_length(PyObject *ob) static int ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg) { - ZstdDict *self = ZstdDict_CAST(ob); - Py_VISIT(self->c_dicts); - return 0; -} - -static int -ZstdDict_clear(PyObject *ob) -{ - ZstdDict *self = ZstdDict_CAST(ob); - Py_CLEAR(self->c_dicts); + /* c_dicts is not visited and there is no tp_clear: it only maps ints + to capsules, and the ZSTD_CDict instances they own must stay alive + until ZstdDict_dealloc() as a compression context being collected + together with this object may still use them. */ + Py_VISIT(Py_TYPE(ob)); return 0; } @@ -262,7 +257,6 @@ static PyType_Slot zstddict_slots[] = { {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, - {Py_tp_clear, ZstdDict_clear}, {0, 0} }; _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
