xinrong-meng commented on code in PR #40864:
URL: https://github.com/apache/spark/pull/40864#discussion_r1173143012
##########
python/pyspark/sql/pandas/serializers.py:
##########
@@ -167,26 +174,23 @@ def __init__(self, timezone, safecheck,
assign_cols_by_name):
self._safecheck = safecheck
self._assign_cols_by_name = assign_cols_by_name
- def arrow_to_pandas(self, arrow_column):
- from pyspark.sql.pandas.types import (
- _check_series_localize_timestamps,
- _convert_map_items_to_dict,
- )
- import pyarrow
-
- # If the given column is a date type column, creates a series of
datetime.date directly
- # instead of creating datetime64[ns] as intermediate data to avoid
overflow caused by
- # datetime64[ns] type handling.
- s = arrow_column.to_pandas(date_as_object=True)
+ def arrow_to_pandas(self, arrow_column) -> pd.Series:
+ """Convert an Arrow column to a pandas Seires."""
+ pser = arrow_column.to_pandas(date_as_object=True)
+ if type_require_conversion(arrow_column.type):
+ # We have to do per-value conversion for nested types.
+ return pser.map(lambda v: recursive_convert_inputs(v,
arrow_column.type))
Review Comment:
Thanks for sharing! We expect a pandas Series so `pser.map` would be more
appropriate considering the return type.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
