https://github.com/python/cpython/commit/01a1dd283b2d39af822f38f005233d1f5cadc927 commit: 01a1dd283b2d39af822f38f005233d1f5cadc927 branch: main author: Zackery Spytz <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-02-05T19:50:51Z summary:
gh-77188: Add support for pickling private methods and nested classes (GH-21480) Co-authored-by: Serhiy Storchaka <[email protected]> files: A Misc/NEWS.d/next/Library/2020-07-14-23-54-18.gh-issue-77188.TyI3_Q.rst M Doc/whatsnew/3.15.rst M Include/internal/pycore_symtable.h M Lib/pickle.py M Lib/test/picklecommon.py M Lib/test/pickletester.py M Modules/_pickle.c M Objects/classobject.c M Python/symtable.c diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 05cd7404066167..20250003dca34e 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -704,6 +704,13 @@ os.path (Contributed by Petr Viktorin for :cve:`2025-4517`.) +pickle +------ + +* Add support for pickling private methods and nested classes. + (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) + + resource -------- diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 9dbfa913219afa..c0164507ea033e 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -151,7 +151,12 @@ extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject extern void _PySymtable_Free(struct symtable *); extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name); -extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); + +// Export for '_pickle' shared extension +PyAPI_FUNC(PyObject *) +_Py_Mangle(PyObject *, PyObject *); +PyAPI_FUNC(int) +_Py_IsPrivateName(PyObject *); /* Flags for def-use information */ diff --git a/Lib/pickle.py b/Lib/pickle.py index 71c12c50f7f035..3e7cf25cb05337 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1175,6 +1175,17 @@ def save_global(self, obj, name=None): if name is None: name = obj.__name__ + if '.__' in name: + # Mangle names of private attributes. + dotted_path = name.split('.') + for i, subpath in enumerate(dotted_path): + if i and subpath.startswith('__') and not subpath.endswith('__'): + prev = prev.lstrip('_') + if prev: + dotted_path[i] = f"_{prev.lstrip('_')}{subpath}" + prev = subpath + name = '.'.join(dotted_path) + module_name = whichmodule(obj, name) if self.proto >= 2: code = _extension_registry.get((module_name, name), _NoValue) diff --git a/Lib/test/picklecommon.py b/Lib/test/picklecommon.py index 4c19b6c421fc61..b749ee09f564bf 100644 --- a/Lib/test/picklecommon.py +++ b/Lib/test/picklecommon.py @@ -388,3 +388,48 @@ def pie(self): class Subclass(tuple): class Nested(str): pass + +# For test_private_methods +class PrivateMethods: + def __init__(self, value): + self.value = value + + def __private_method(self): + return self.value + + def get_method(self): + return self.__private_method + + @classmethod + def get_unbound_method(cls): + return cls.__private_method + + @classmethod + def __private_classmethod(cls): + return 43 + + @classmethod + def get_classmethod(cls): + return cls.__private_classmethod + + @staticmethod + def __private_staticmethod(): + return 44 + + @classmethod + def get_staticmethod(cls): + return cls.__private_staticmethod + +# For test_private_nested_classes +class PrivateNestedClasses: + @classmethod + def get_nested(cls): + return cls.__Nested + + class __Nested: + @classmethod + def get_nested2(cls): + return cls.__Nested2 + + class __Nested2: + pass diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index d2b8d036bfd9e7..7b1b117d6d3e32 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -4118,6 +4118,33 @@ def test_c_methods(self): with self.subTest(proto=proto, descr=descr): self.assertRaises(TypeError, self.dumps, descr, proto) + def test_private_methods(self): + if self.py_version < (3, 15): + self.skipTest('not supported in Python < 3.15') + obj = PrivateMethods(42) + for proto in protocols: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(obj.get_method(), proto)) + self.assertEqual(unpickled(), 42) + unpickled = self.loads(self.dumps(obj.get_unbound_method(), proto)) + self.assertEqual(unpickled(obj), 42) + unpickled = self.loads(self.dumps(obj.get_classmethod(), proto)) + self.assertEqual(unpickled(), 43) + unpickled = self.loads(self.dumps(obj.get_staticmethod(), proto)) + self.assertEqual(unpickled(), 44) + + def test_private_nested_classes(self): + if self.py_version < (3, 15): + self.skipTest('not supported in Python < 3.15') + cls1 = PrivateNestedClasses.get_nested() + cls2 = cls1.get_nested2() + for proto in protocols: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(cls1, proto)) + self.assertIs(unpickled, cls1) + unpickled = self.loads(self.dumps(cls2, proto)) + self.assertIs(unpickled, cls2) + def test_object_with_attrs(self): obj = Object() obj.a = 1 diff --git a/Misc/NEWS.d/next/Library/2020-07-14-23-54-18.gh-issue-77188.TyI3_Q.rst b/Misc/NEWS.d/next/Library/2020-07-14-23-54-18.gh-issue-77188.TyI3_Q.rst new file mode 100644 index 00000000000000..3e956409d52a58 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-07-14-23-54-18.gh-issue-77188.TyI3_Q.rst @@ -0,0 +1 @@ +The :mod:`pickle` module now properly handles name-mangled private methods. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 063547c9a4d020..a897e45f00fab6 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -19,6 +19,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime.h" // _Py_ID() #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_symtable.h" // _Py_Mangle() #include "pycore_sysmodule.h" // _PySys_GetSizeOf() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() @@ -1928,6 +1929,37 @@ get_dotted_path(PyObject *name) return PyUnicode_Split(name, _Py_LATIN1_CHR('.'), -1); } +static PyObject * +join_dotted_path(PyObject *dotted_path) +{ + return PyUnicode_Join(_Py_LATIN1_CHR('.'), dotted_path); +} + +/* Returns -1 (with an exception set) on error, 0 if there were no changes, + * 1 if some names were mangled. */ +static int +mangle_dotted_path(PyObject *dotted_path) +{ + int rc = 0; + Py_ssize_t n = PyList_GET_SIZE(dotted_path); + for (Py_ssize_t i = n-1; i > 0; i--) { + PyObject *subpath = PyList_GET_ITEM(dotted_path, i); + if (_Py_IsPrivateName(subpath)) { + PyObject *parent = PyList_GET_ITEM(dotted_path, i-1); + PyObject *mangled = _Py_Mangle(parent, subpath); + if (mangled == NULL) { + return -1; + } + if (mangled != subpath) { + rc = 1; + } + PyList_SET_ITEM(dotted_path, i, mangled); + Py_DECREF(subpath); + } + } + return rc; +} + static int check_dotted_path(PickleState *st, PyObject *obj, PyObject *dotted_path) { @@ -3809,6 +3841,15 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, dotted_path = get_dotted_path(global_name); if (dotted_path == NULL) goto error; + switch (mangle_dotted_path(dotted_path)) { + case -1: + goto error; + case 1: + Py_SETREF(global_name, join_dotted_path(dotted_path)); + if (global_name == NULL) { + goto error; + } + } module_name = whichmodule(st, obj, global_name, dotted_path); if (module_name == NULL) goto error; diff --git a/Objects/classobject.c b/Objects/classobject.c index e71f301f2efd77..4c99c194df53a5 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -7,6 +7,7 @@ #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_symtable.h" // _Py_Mangle() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -143,6 +144,20 @@ method___reduce___impl(PyMethodObject *self) if (funcname == NULL) { return NULL; } + if (_Py_IsPrivateName(funcname)) { + PyObject *classname = PyType_Check(funcself) + ? PyType_GetName((PyTypeObject *)funcself) + : PyType_GetName(Py_TYPE(funcself)); + if (classname == NULL) { + Py_DECREF(funcname); + return NULL; + } + Py_SETREF(funcname, _Py_Mangle(classname, funcname)); + Py_DECREF(classname); + if (funcname == NULL) { + return NULL; + } + } return Py_BuildValue( "N(ON)", _PyEval_GetBuiltin(&_Py_ID(getattr)), funcself, funcname); } diff --git a/Python/symtable.c b/Python/symtable.c index 29cf9190a4e95b..29ac8f6880c575 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -3183,6 +3183,27 @@ _Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name) return _Py_Mangle(privateobj, name); } +int +_Py_IsPrivateName(PyObject *ident) +{ + if (!PyUnicode_Check(ident)) { + return 0; + } + Py_ssize_t nlen = PyUnicode_GET_LENGTH(ident); + if (nlen < 3 || + PyUnicode_READ_CHAR(ident, 0) != '_' || + PyUnicode_READ_CHAR(ident, 1) != '_') + { + return 0; + } + if (PyUnicode_READ_CHAR(ident, nlen-1) == '_' && + PyUnicode_READ_CHAR(ident, nlen-2) == '_') + { + return 0; /* Don't mangle __whatever__ */ + } + return 1; +} + PyObject * _Py_Mangle(PyObject *privateobj, PyObject *ident) { _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
