Github user BryanCutler commented on a diff in the pull request:
https://github.com/apache/spark/pull/20373#discussion_r163421163
--- Diff: python/pyspark/cloudpickle.py ---
@@ -867,23 +797,21 @@ def save_not_implemented(self, obj):
dispatch[type(Ellipsis)] = save_ellipsis
dispatch[type(NotImplemented)] = save_not_implemented
- # WeakSet was added in 2.7.
- if hasattr(weakref, 'WeakSet'):
- def save_weakset(self, obj):
- self.save_reduce(weakref.WeakSet, (list(obj),))
-
- dispatch[weakref.WeakSet] = save_weakset
+ def save_weakset(self, obj):
+ self.save_reduce(weakref.WeakSet, (list(obj),))
- """Special functions for Add-on libraries"""
- def inject_addons(self):
- """Plug in system. Register additional pickling functions if
modules already loaded"""
- pass
+ dispatch[weakref.WeakSet] = save_weakset
def save_logger(self, obj):
self.save_reduce(logging.getLogger, (obj.name,), obj=obj)
dispatch[logging.Logger] = save_logger
+ """Special functions for Add-on libraries"""
+ def inject_addons(self):
+ """Plug in system. Register additional pickling functions if
modules already loaded"""
+ pass
+
--- End diff --
Further cleanups
https://github.com/cloudpipe/cloudpickle/commit/c91aaf110441991307f5097f950764079d0f9652
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
