HyukjinKwon commented on code in PR #46529:
URL: https://github.com/apache/spark/pull/46529#discussion_r1618112723
##########
python/pyspark/sql/pandas/types.py:
##########
@@ -86,30 +110,58 @@ def to_arrow_type(dt: DataType) -> "pa.DataType":
arrow_type = pa.binary()
elif type(dt) == DateType:
arrow_type = pa.date32()
- elif type(dt) == TimestampType:
+ elif type(dt) == TimestampType and timestamp_utc:
# Timestamps should be in UTC, JVM Arrow timestamps require a timezone
to be read
arrow_type = pa.timestamp("us", tz="UTC")
+ elif type(dt) == TimestampType:
+ arrow_type = pa.timestamp("us", tz=None)
elif type(dt) == TimestampNTZType:
arrow_type = pa.timestamp("us", tz=None)
elif type(dt) == DayTimeIntervalType:
arrow_type = pa.duration("us")
elif type(dt) == ArrayType:
- field = pa.field("element", to_arrow_type(dt.elementType),
nullable=dt.containsNull)
+ field = pa.field(
+ "element",
+ to_arrow_type(dt.elementType,
error_on_duplicated_field_names_in_struct, timestamp_utc),
+ nullable=dt.containsNull,
+ )
arrow_type = pa.list_(field)
elif type(dt) == MapType:
- key_field = pa.field("key", to_arrow_type(dt.keyType), nullable=False)
- value_field = pa.field("value", to_arrow_type(dt.valueType),
nullable=dt.valueContainsNull)
+ key_field = pa.field(
+ "key",
+ to_arrow_type(dt.keyType,
error_on_duplicated_field_names_in_struct, timestamp_utc),
+ nullable=False,
+ )
+ value_field = pa.field(
+ "value",
+ to_arrow_type(dt.valueType,
error_on_duplicated_field_names_in_struct, timestamp_utc),
+ nullable=dt.valueContainsNull,
+ )
arrow_type = pa.map_(key_field, value_field)
elif type(dt) == StructType:
+ field_names = dt.names
+ if error_on_duplicated_field_names_in_struct and len(set(field_names))
!= len(field_names):
Review Comment:
Quick question. Why do we enforce this? Did it not work before?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
