xinrong-meng commented on code in PR #41603:
URL: https://github.com/apache/spark/pull/41603#discussion_r1231552983
##########
python/pyspark/sql/pandas/types.py:
##########
@@ -578,29 +578,44 @@ def _converter(
if isinstance(dt, ArrayType):
_element_conv = _converter(dt.elementType, _struct_in_pandas,
_ndarray_as_list)
- def convert_array(value: Any) -> Any:
- if value is None:
- return None
- elif isinstance(value, np.ndarray):
- element_list = (
- [_element_conv(v) for v in value]
- if _element_conv is not None
- else [v for v in value]
- )
-
- if _ndarray_as_list:
+ if _ndarray_as_list:
+
+ def convert_array_ndarray_as_list(value: Any) -> Any:
+ if value is None:
+ return None
+ elif isinstance(value, np.ndarray):
# In Arrow Python UDF, ArrayType is converted to
`np.ndarray`
# whereas a list is expected.
- return element_list
+ return (
+ [_element_conv(v) for v in value]
+ if _element_conv is not None
+ else [v for v in value]
+ )
else:
+ assert isinstance(value, list)
+ # otherwise, `list` should be used.
+ return [_element_conv(v) for v in value] # type:
ignore[misc]
+
+ return convert_array_ndarray_as_list
+ else:
+
+ def convert_array_ndarray_as_ndarray(value: Any) -> Any:
Review Comment:
Good point!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
