Github user BryanCutler commented on a diff in the pull request:
https://github.com/apache/spark/pull/20373#discussion_r163421517
--- Diff: python/pyspark/cloudpickle.py ---
@@ -1019,18 +948,40 @@ def __reduce__(cls):
return cls.__name__
-def _fill_function(func, globals, defaults, dict, module, closure_values):
- """ Fills in the rest of function data into the skeleton function
object
- that were created via _make_skel_func().
+def _fill_function(*args):
+ """Fills in the rest of function data into the skeleton function object
+
+ The skeleton itself is create by _make_skel_func().
"""
- func.__globals__.update(globals)
- func.__defaults__ = defaults
- func.__dict__ = dict
- func.__module__ = module
+ if len(args) == 2:
+ func = args[0]
+ state = args[1]
+ elif len(args) == 5:
+ # Backwards compat for cloudpickle v0.4.0, after which the `module`
+ # argument was introduced
+ func = args[0]
+ keys = ['globals', 'defaults', 'dict', 'closure_values']
+ state = dict(zip(keys, args[1:]))
+ elif len(args) == 6:
+ # Backwards compat for cloudpickle v0.4.1, after which the function
+ # state was passed as a dict to the _fill_function it-self.
+ func = args[0]
+ keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']
+ state = dict(zip(keys, args[1:]))
+ else:
+ raise ValueError('Unexpected _fill_value arguments: %r' % (args,))
+
+ func.__globals__.update(state['globals'])
+ func.__defaults__ = state['defaults']
+ func.__dict__ = state['dict']
+ if 'module' in state:
+ func.__module__ = state['module']
+ if 'qualname' in state:
+ func.__qualname__ = state['qualname']
--- End diff --
Preserve func.__qualname__ when defined
https://github.com/cloudpipe/cloudpickle/commit/14b38a3ab5970d96cce1492c790494932285f845
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
