Github user BryanCutler commented on a diff in the pull request:
https://github.com/apache/spark/pull/20373#discussion_r163419329
--- Diff: python/pyspark/cloudpickle.py ---
@@ -181,6 +180,32 @@ def _builtin_type(name):
return getattr(types, name)
+def _make__new__factory(type_):
+ def _factory():
+ return type_.__new__
+ return _factory
+
+
+# NOTE: These need to be module globals so that they're pickleable as
globals.
+_get_dict_new = _make__new__factory(dict)
+_get_frozenset_new = _make__new__factory(frozenset)
+_get_list_new = _make__new__factory(list)
+_get_set_new = _make__new__factory(set)
+_get_tuple_new = _make__new__factory(tuple)
+_get_object_new = _make__new__factory(object)
+
+# Pre-defined set of builtin_function_or_method instances that can be
+# serialized.
+_BUILTIN_TYPE_CONSTRUCTORS = {
+ dict.__new__: _get_dict_new,
+ frozenset.__new__: _get_frozenset_new,
+ set.__new__: _get_set_new,
+ list.__new__: _get_list_new,
+ tuple.__new__: _get_tuple_new,
+ object.__new__: _get_object_new,
+}
+
+
--- End diff --
MAINT: Handle builtin type __new__ attrs:
https://github.com/cloudpipe/cloudpickle/commit/f0d2011f9fc88105c174b7c861f2c2f56e870350
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
