Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/21467#discussion_r192284083
--- Diff: python/pyspark/worker.py ---
@@ -140,15 +139,18 @@ def read_single_udf(pickleSer, infile, eval_type):
else:
row_func = chain(row_func, f)
+ func = fail_on_stopiteration(row_func)
+
# the last returnType will be the return type of UDF
if eval_type == PythonEvalType.SQL_SCALAR_PANDAS_UDF:
- return arg_offsets, wrap_scalar_pandas_udf(row_func, return_type)
+ return arg_offsets, wrap_scalar_pandas_udf(func, return_type)
elif eval_type == PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF:
- return arg_offsets, wrap_grouped_map_pandas_udf(row_func,
return_type)
+ argspec = _get_argspec(row_func) # fails on func
--- End diff --
hmm, can we have more meaningful comment than `fails on func`?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
