wombatu-kun commented on code in PR #20519:
URL: https://github.com/apache/hudi/pull/20519#discussion_r4478635404
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala:
##########
@@ -117,6 +117,23 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
assertEquals(Some(dateAttr), matched(DateDiff(dateAttr,
Literal(java.sql.Date.valueOf("2022-03-07")))))
}
+ @Test
+ def testUtcTimestampConversionsRequireFixedOffsetZone(): Unit = {
+ // A zone with DST runs local time backwards across a fall-back and shifts
the spring-forward gap.
+ assertEquals(None, matched(FromUTCTimestamp(tsAttr,
Literal("America/New_York"))))
+ assertEquals(None, matched(ToUTCTimestamp(tsAttr,
Literal("America/New_York"))))
+ // A fixed-offset zone shifts every timestamp by the same amount.
+ assertEquals(Some(tsAttr), matched(FromUTCTimestamp(tsAttr,
Literal("+08:00"))))
+ assertEquals(Some(tsAttr), matched(ToUTCTimestamp(tsAttr, Literal("UTC"))))
+ // Zones are resolved the way Spark evaluates them: the short ID EST is a
fixed -05:00, and +8:00 is +08:00.
+ assertEquals(Some(tsAttr), matched(FromUTCTimestamp(tsAttr,
Literal("EST"))))
Review Comment:
Done e86de12c33d7
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala:
##########
@@ -914,7 +914,46 @@ object TestDataSkippingUtils {
IndexRow("file_1", valueCount = 2, C_minValue = new
Timestamp(1000L), C_maxValue = new Timestamp(50000L), C_nullCount = 0),
IndexRow("file_2", valueCount = 2, C_minValue = new
Timestamp(101000L), C_maxValue = new Timestamp(200000L), C_nullCount = 0)
),
- Seq("file_2"))
+ Seq("file_2")),
+ // New York local time runs backwards across the DST fall-back: 05:30Z
is 01:30 EDT, but the later
+ // 06:10Z is 01:10 EST. Re-applying from_utc_timestamp over min/max
stats would wrongly prune
+ // file_1 even though its 05:30Z row matches; must fall back to
always-true, keeping the file
+ arguments(
+ "from_utc_timestamp(C, 'America/New_York') > TIMESTAMP '2026-11-01
01:20:00'",
Review Comment:
Done e86de12c33d7. `TestDataSkippingQuery` now runs the #20518 repro over a
partitioned table; the `to_utc_timestamp` rows sit in separate files, so
without the fix only the partition stats index prunes the matching row.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
