canyang25 commented on code in PR #59298:
URL: https://github.com/apache/spark/pull/59298#discussion_r4236644651
##########
python/pyspark/sql/tests/test_udf.py:
##########
@@ -809,6 +811,45 @@ def noop(x):
self.assertEqual(df.schema[0].dataType.typeName(), "timestamp_ntz")
self.assertEqual(df.first()[0], datetime.datetime(1970, 1, 1, 0,
0))
+ def test_udf_timestamp_dst_fold_round_trip(self):
+ # SPARK-60081: preserve datetime.fold in the repeated DST hour for
Python UDFs.
+ tz = "America/Los_Angeles"
+ tz_prev = os.environ.get("TZ", None)
+ try:
+ os.environ["TZ"] = tz
+ time.tzset()
+ with self.sql_conf(
+ {
+ "spark.sql.session.timeZone": tz,
+ "spark.sql.execution.pythonUDF.arrow.enabled": "true",
+ }
+ ):
+ second_fold = self.spark.sql("SELECT timestamp'2021-11-07
09:30:00Z' AS t")
+ first_fold = self.spark.sql("SELECT timestamp'2021-11-07
08:30:00Z' AS t")
+
+ for kwargs in [{}, {"useArrow": False}]:
+ to_epoch = udf(lambda x: int(x.timestamp()), LongType(),
**kwargs)
+ identity = udf(lambda x: x, TimestampType(), **kwargs)
+
+ self.assertEqual(1636277400,
second_fold.select(to_epoch("t")).first()[0])
Review Comment:
Added driver-side .timestamp() assertions for both folds.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
