https://github.com/python/cpython/commit/14918f76324dc3f233f40070057f82ec521617c7 commit: 14918f76324dc3f233f40070057f82ec521617c7 branch: 3.14 author: Miss Islington (bot) <[email protected]> committer: gpshead <[email protected]> date: 2026-10-25T06:12:31Z summary:
[3.14] gh-160535: Keep the ZstdDict alive until the zstd context is freed (GH-160536) (#160636) gh-160535: Keep the ZstdDict alive until the zstd context is freed (GH-160536) The garbage collector could release a ZstdCompressor's dictionary, or the digested dictionaries cached by the ZstdDict, while worker threads of the compression context were still reading them. They are now only released on deallocation, after the context has been freed. ANT-2026-CT3P1CSJ + Visit the type in the compressor and decompressor traverse; move NEWS to Library (cherry picked from commit b949033ae8b22ff77e798a08f4ecc6eaf4cf0d8a) Co-authored-by: Gregory P. Smith <[email protected]> files: A Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst M Lib/test/test_zstd.py M Modules/_zstd/compressor.c M Modules/_zstd/decompressor.c M Modules/_zstd/zstddict.c diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 6de63d840898ac7..b62b125aaecde38 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -7,9 +7,12 @@ import os import unittest import tempfile +import textwrap import threading +from test import support from test.support.import_helper import import_module +from test.support.script_helper import assert_python_ok from test.support import threading_helper from test.support import _1M from test.support import Py_GIL_DISABLED @@ -1549,6 +1552,42 @@ def test_len(self): self.assertEqual(len(TRAINED_DICT), len(TRAINED_DICT.dict_content)) self.assertIn(str(len(TRAINED_DICT)), str(TRAINED_DICT)) + @unittest.skipUnless(SUPPORT_MULTITHREADING, + "zstd build doesn't support multi-threaded compression") + @support.requires_subprocess() + def test_collect_cycle_with_busy_compressor(self): + # The dictionary must outlive the compression context when both are + # collected by the garbage collector while a worker thread is still + # using the dictionary. Whether the failure is detected depends on + # the memory allocator, so use a fresh process for each case. + code = textwrap.dedent(""" + import gc, random, sys + from compression.zstd import (CompressionParameter, + ZstdCompressor, ZstdDict) + + class Cycle: + pass + + job_size = 1 << 20 + content = random.randbytes(1 << 10) * (1 << 8) + zd = ZstdDict(content, is_raw=True) + options = {CompressionParameter.compression_level: 10, + CompressionParameter.nb_workers: 1, + CompressionParameter.job_size: job_size} + c = ZstdCompressor(options=options, + zstd_dict=getattr(zd, sys.argv[1])) + cycle = Cycle() + cycle.cycle = cycle + cycle.compressor = c + # Returns while the worker thread is compressing the first job. + c.compress(random.randbytes(1 << 10) * ((job_size >> 10) + 64)) + del zd, c, cycle + gc.collect() + """) + for attr in 'as_digested_dict', 'as_prefix': + with self.subTest(attr=attr): + assert_python_ok('-c', code, attr) + class FileTestCase(unittest.TestCase): def setUp(self): self.DECOMPRESSED_42 = b'a'*42 diff --git a/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst new file mode 100644 index 000000000000000..ea2082716d64ace --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst @@ -0,0 +1,4 @@ +Fix a crash in :mod:`compression.zstd` when the garbage collector frees a +:class:`~compression.zstd.ZstdCompressor` that uses worker threads together +with its :class:`~compression.zstd.ZstdDict` before the compression of +already submitted data has finished. diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index fa31d3a0e90abde..02162c5c9a6f581 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -765,17 +765,13 @@ static int ZstdCompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdCompressor *self = ZstdCompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; } -static int -ZstdCompressor_clear(PyObject *ob) -{ - ZstdCompressor *self = ZstdCompressor_CAST(ob); - Py_CLEAR(self->dict); - return 0; -} +/* No tp_clear: the compression context (and its worker threads) uses the + dictionary until ZstdCompressor_dealloc() has freed the context. */ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_new, _zstd_ZstdCompressor_new}, @@ -784,7 +780,6 @@ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_members, ZstdCompressor_members}, {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, - {Py_tp_clear, ZstdCompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 85b67d468864890..08918629446cd65 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -688,18 +688,13 @@ static int ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdDecompressor *self = ZstdDecompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; } -static int -ZstdDecompressor_clear(PyObject *ob) -{ - ZstdDecompressor *self = ZstdDecompressor_CAST(ob); - Py_CLEAR(self->dict); - Py_CLEAR(self->unused_data); - return 0; -} +/* No tp_clear: the decompression context references the dictionary until + ZstdDecompressor_dealloc() has freed the context. */ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_new, _zstd_ZstdDecompressor_new}, @@ -709,7 +704,6 @@ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_getset, ZstdDecompressor_getset}, {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, - {Py_tp_clear, ZstdDecompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index e1b9d998e697fb8..11ea7619976bf92 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -240,16 +240,11 @@ ZstdDict_length(PyObject *ob) static int ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg) { - ZstdDict *self = ZstdDict_CAST(ob); - Py_VISIT(self->c_dicts); - return 0; -} - -static int -ZstdDict_clear(PyObject *ob) -{ - ZstdDict *self = ZstdDict_CAST(ob); - Py_CLEAR(self->c_dicts); + /* c_dicts is not visited and there is no tp_clear: it only maps ints + to capsules, and the ZSTD_CDict instances they own must stay alive + until ZstdDict_dealloc() as a compression context being collected + together with this object may still use them. */ + Py_VISIT(Py_TYPE(ob)); return 0; } @@ -262,7 +257,6 @@ static PyType_Slot zstddict_slots[] = { {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, - {Py_tp_clear, ZstdDict_clear}, {0, 0} }; _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
