https://github.com/python/cpython/commit/165b15209aee920c6a8e821d4295e59b91030458 commit: 165b15209aee920c6a8e821d4295e59b91030458 branch: 3.15 author: Miss Islington (bot) <[email protected]> committer: JelleZijlstra <[email protected]> date: 2026-10-22T15:22:04-07:00 summary:
[3.15] gh-159739: Fully clear the unpickler memo (GH-160173) (#160328) Clearing an Unpickler memo left its sequential index and sparse entries intact, so reusing the unpickler for an independent record could raise UnpicklingError or retrieve an object from a previous record. Replace the memo and reset all of its state before releasing the old entries. Backport to 3.15; manually backport only the counter reset to 3.14, which has no sparse memo. (cherry picked from commit cc36604f7b308b68fb1b39cc7774c3e49def74e5) Co-authored-by: Jelle Zijlstra <[email protected]> files: A Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst M Lib/test/pickletester.py M Modules/_pickle.c diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 83a71b7efedd37..805c010abaeed8 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -5186,6 +5186,22 @@ def test_clear_pickler_memo(self): self.assertNotEqual(first_pickled, second_pickled) self.assertEqual(first_pickled, third_pickled) + def test_clear_unpickler_memo(self): + value = [b'repeated'] * 2 + data = pickle.dumps(value, protocol=4) + unpickler = self.unpickler_class(io.BytesIO(data * 2)) + self.assertEqual(unpickler.load(), value) + unpickler.memo.clear() + self.assertEqual(unpickler.load(), value) + + def test_clear_unpickler_memo_sparse(self): + # Store None at index 1000, then retrieve it in a second record. + unpickler = self.unpickler_class(io.BytesIO(b'Np1000\n.g1000\n.')) + self.assertIsNone(unpickler.load()) + unpickler.memo.clear() + with self.assertRaises(pickle.UnpicklingError): + unpickler.load() + def test_priming_pickler_memo(self): # Verify that we can set the Pickler's memo attribute. data = ["abcdefg", "abcdefg", 44] diff --git a/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst b/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst new file mode 100644 index 00000000000000..7052cdb6bd9682 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst @@ -0,0 +1,2 @@ +Fix clearing a :class:`pickle.Unpickler` memo so that the unpickler can be +reused for independent records. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index c5c9405f35aa98..869daf2efeae80 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1747,16 +1747,16 @@ _Unpickler_NewMemo(Py_ssize_t new_size) return memo; } -/* Free the unpickler's memo, taking care to decref any items left in it. */ +/* Replace the unpickler's memo and release the old entries. */ static void -_Unpickler_MemoCleanup(UnpicklerObject *self) +_Unpickler_MemoCleanup(UnpicklerObject *self, PyObject **replacement) { Py_ssize_t i; PyObject **memo = self->memo; - if (self->memo == NULL) + self->memo = replacement; + if (memo == NULL) return; - self->memo = NULL; i = self->memo_size; while (--i >= 0) { Py_XDECREF(memo[i]); @@ -7577,7 +7577,7 @@ Unpickler_clear(PyObject *op) } Py_CLEAR(self->memo_dict); - _Unpickler_MemoCleanup(self); + _Unpickler_MemoCleanup(self, NULL); PyMem_Free(self->marks); self->marks = NULL; PyMem_Free(self->input_line); @@ -7724,10 +7724,16 @@ static PyObject * _pickle_UnpicklerMemoProxy_clear_impl(UnpicklerMemoProxyObject *self) /*[clinic end generated code: output=d20cd43f4ba1fb1f input=b1df7c52e7afd9bd]*/ { - _Unpickler_MemoCleanup(self->unpickler); - self->unpickler->memo = _Unpickler_NewMemo(self->unpickler->memo_size); - if (self->unpickler->memo == NULL) + UnpicklerObject *unpickler = self->unpickler; + PyObject **memo = _Unpickler_NewMemo(unpickler->memo_size); + if (memo == NULL) { return NULL; + } + PyObject *memo_dict = unpickler->memo_dict; + unpickler->memo_dict = NULL; + unpickler->memo_len = 0; + _Unpickler_MemoCleanup(unpickler, memo); + Py_XDECREF(memo_dict); Py_RETURN_NONE; } @@ -7932,7 +7938,7 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure)) return -1; } - _Unpickler_MemoCleanup(self); + _Unpickler_MemoCleanup(self, NULL); self->memo_size = new_memo_size; self->memo = new_memo; _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
