https://github.com/python/cpython/commit/cc36604f7b308b68fb1b39cc7774c3e49def74e5
commit: cc36604f7b308b68fb1b39cc7774c3e49def74e5
branch: main
author: Jelle Zijlstra <[email protected]>
committer: JelleZijlstra <[email protected]>
date: 2026-10-22T19:29:51Z
summary:
gh-159739: Fully clear the unpickler memo (#160173)
Clearing an Unpickler memo left its sequential index and sparse entries intact,
so reusing the unpickler for an independent record could raise UnpicklingError
or retrieve an object from a previous record. Replace the memo and reset all of
its state before releasing the old entries. Backport to 3.15; manually backport
only the counter reset to 3.14, which has no sparse memo.
files:
A Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst
M Lib/test/pickletester.py
M Modules/_pickle.c
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index 9c0679d1d13d23..e9d367f9ea95aa 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -5299,6 +5299,22 @@ def test_clear_pickler_memo(self):
self.assertNotEqual(first_pickled, second_pickled)
self.assertEqual(first_pickled, third_pickled)
+ def test_clear_unpickler_memo(self):
+ value = [b'repeated'] * 2
+ data = pickle.dumps(value, protocol=4)
+ unpickler = self.unpickler_class(io.BytesIO(data * 2))
+ self.assertEqual(unpickler.load(), value)
+ unpickler.memo.clear()
+ self.assertEqual(unpickler.load(), value)
+
+ def test_clear_unpickler_memo_sparse(self):
+ # Store None at index 1000, then retrieve it in a second record.
+ unpickler = self.unpickler_class(io.BytesIO(b'Np1000\n.g1000\n.'))
+ self.assertIsNone(unpickler.load())
+ unpickler.memo.clear()
+ with self.assertRaises(pickle.UnpicklingError):
+ unpickler.load()
+
def test_priming_pickler_memo(self):
# Verify that we can set the Pickler's memo attribute.
data = ["abcdefg", "abcdefg", 44]
diff --git
a/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst
b/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst
new file mode 100644
index 00000000000000..7052cdb6bd9682
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst
@@ -0,0 +1,2 @@
+Fix clearing a :class:`pickle.Unpickler` memo so that the unpickler can be
+reused for independent records.
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index 114534dce1b3d4..b8f9b1c92f2072 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -1762,16 +1762,16 @@ _Unpickler_NewMemo(Py_ssize_t new_size)
return memo;
}
-/* Free the unpickler's memo, taking care to decref any items left in it. */
+/* Replace the unpickler's memo and release the old entries. */
static void
-_Unpickler_MemoCleanup(UnpicklerObject *self)
+_Unpickler_MemoCleanup(UnpicklerObject *self, PyObject **replacement)
{
Py_ssize_t i;
PyObject **memo = self->memo;
- if (self->memo == NULL)
+ self->memo = replacement;
+ if (memo == NULL)
return;
- self->memo = NULL;
i = self->memo_size;
while (--i >= 0) {
Py_XDECREF(memo[i]);
@@ -7769,7 +7769,7 @@ Unpickler_clear(PyObject *op)
}
Py_CLEAR(self->memo_dict);
- _Unpickler_MemoCleanup(self);
+ _Unpickler_MemoCleanup(self, NULL);
PyMem_Free(self->marks);
self->marks = NULL;
PyMem_Free(self->input_line);
@@ -7916,10 +7916,16 @@ static PyObject *
_pickle_UnpicklerMemoProxy_clear_impl(UnpicklerMemoProxyObject *self)
/*[clinic end generated code: output=d20cd43f4ba1fb1f input=b1df7c52e7afd9bd]*/
{
- _Unpickler_MemoCleanup(self->unpickler);
- self->unpickler->memo = _Unpickler_NewMemo(self->unpickler->memo_size);
- if (self->unpickler->memo == NULL)
+ UnpicklerObject *unpickler = self->unpickler;
+ PyObject **memo = _Unpickler_NewMemo(unpickler->memo_size);
+ if (memo == NULL) {
return NULL;
+ }
+ PyObject *memo_dict = unpickler->memo_dict;
+ unpickler->memo_dict = NULL;
+ unpickler->memo_len = 0;
+ _Unpickler_MemoCleanup(unpickler, memo);
+ Py_XDECREF(memo_dict);
Py_RETURN_NONE;
}
@@ -8124,7 +8130,7 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void
*Py_UNUSED(closure))
return -1;
}
- _Unpickler_MemoCleanup(self);
+ _Unpickler_MemoCleanup(self, NULL);
self->memo_size = new_memo_size;
self->memo = new_memo;
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
