allisonwang-db commented on code in PR #41321:
URL: https://github.com/apache/spark/pull/41321#discussion_r1218626596
##########
python/pyspark/sql/udf.py:
##########
@@ -129,18 +127,12 @@ def _create_py_udf(
else useArrow
)
regular_udf = _create_udf(f, returnType, PythonEvalType.SQL_BATCHED_UDF)
- return_type = regular_udf.returnType
try:
is_func_with_args = len(getfullargspec(f).args) > 0
except TypeError:
is_func_with_args = False
- is_output_atomic_type = (
- not isinstance(return_type, StructType)
- and not isinstance(return_type, MapType)
- and not isinstance(return_type, ArrayType)
- )
if is_arrow_enabled:
- if is_output_atomic_type and is_func_with_args:
+ if is_func_with_args:
Review Comment:
Just curious, why can't we enable arrow optimization for UDFs that do not
have input arguments?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
