xinrong-meng commented on code in PR #41147:
URL: https://github.com/apache/spark/pull/41147#discussion_r1198097720
##########
python/pyspark/sql/pandas/serializers.py:
##########
@@ -168,23 +173,21 @@ def __init__(self, timezone, safecheck):
self._safecheck = safecheck
def arrow_to_pandas(self, arrow_column):
- from pyspark.sql.pandas.types import (
- _check_series_localize_timestamps,
- _convert_map_items_to_dict,
- )
- import pyarrow
-
# If the given column is a date type column, creates a series of
datetime.date directly
# instead of creating datetime64[ns] as intermediate data to avoid
overflow caused by
# datetime64[ns] type handling.
+ # Cast dates to objects instead of datetime64[ns] dtype to avoid
overflow.
s = arrow_column.to_pandas(date_as_object=True)
- if pyarrow.types.is_timestamp(arrow_column.type) and
arrow_column.type.tz is not None:
- return _check_series_localize_timestamps(s, self._timezone)
- elif pyarrow.types.is_map(arrow_column.type):
- return _convert_map_items_to_dict(s)
- else:
- return s
+ # TODO: cache the converter for reuse
Review Comment:
Certainly, done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
