This is an automated email from the ASF dual-hosted git repository.

voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 94677a7c1e10 fix(spark): require a fixed-offset zone for 
from/to_utc_timestamp data skipping (#20519)
94677a7c1e10 is described below

commit 94677a7c1e10c78eae20e757fa149fefedf7a233
Author: Vova Kolmakov <[email protected]>
AuthorDate: Thu Nov 5 17:55:51 2026 +0700

    fix(spark): require a fixed-offset zone for from/to_utc_timestamp data 
skipping (#20519)
    
    * fix(spark): require a fixed-offset zone for from/to_utc_timestamp data 
skipping
    
    * test(spark): update data skipping golden filters after rebase onto master
    
    * addressed review comments: pin JDK-stable zones, cover partition stats 
with a SQL test and share the fall-back timestamps
---
 .../dataskipping/rules/TimeZoneShiftRule.scala     | 27 +++++++++---
 .../data-skipping/golden-index-filters.txt         | 18 ++++----
 .../org/apache/hudi/TestDataSkippingUtils.scala    | 50 ++++++++++++++++++++--
 .../hudi/TestHoodieCatalystExpressionUtils.scala   | 22 +++++++++-
 .../sql/hudi/feature/TestDataSkippingQuery.scala   | 50 ++++++++++++++++++++++
 5 files changed, 149 insertions(+), 18 deletions(-)

diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
index b118ee0e1719..fc2a4fce0fee 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
@@ -17,22 +17,39 @@
 
 package org.apache.spark.sql.hudi.dataskipping.rules
 
-import org.apache.spark.sql.catalyst.expressions.{Expression, 
FromUTCTimestamp, ToUTCTimestamp}
+import org.apache.spark.sql.catalyst.expressions.{Expression, 
FromUTCTimestamp, Literal, ToUTCTimestamp}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step}
 
+import scala.util.Try
+
 /**
  * Time zone shifts: `FromUTCTimestamp`, `ToUTCTimestamp`.
  *
- * Shared guard: none; any time zone operand is accepted today.
+ * Shared guard: [[isFixedOffsetTimeZone]], a literal time zone whose offset 
never changes.
  *
- * NOTE: The offset jumps at DST transitions, so the shift is not monotonic 
across a fall-back (#20449)
+ * NOTE: Both shift by the zone's offset at that time, so they only keep the 
ordering for a zone whose offset never
+ *       changes, DST or historical. Local time runs backwards when the offset 
drops (for ex, 05:30Z is 01:30 EDT but
+ *       the later 06:10Z is 01:10 EST), and a local time in the gap of a 
rising offset shifts past later ones (for ex,
+ *       02:30 maps to 07:30Z but the later 03:00 to 07:00Z)
  */
 object TimeZoneShiftRule extends RangeRule {
 
   override def steps(term: Expression): Seq[Step] = term match {
-    case FromUTCTimestamp(timestamp, _) => Seq(Step(term, timestamp, 
Increasing, fileGuard = None, partial = false))
-    case ToUTCTimestamp(timestamp, _) => Seq(Step(term, timestamp, Increasing, 
fileGuard = None, partial = false))
+    case FromUTCTimestamp(timestamp, timeZone) if 
isFixedOffsetTimeZone(timeZone) =>
+      Seq(Step(term, timestamp, Increasing, fileGuard = None, partial = false))
+    case ToUTCTimestamp(timestamp, timeZone) if 
isFixedOffsetTimeZone(timeZone) =>
+      Seq(Step(term, timestamp, Increasing, fileGuard = None, partial = false))
 
     case _ => Nil
   }
+
+  // Only a literal zone is matched, since the optimizer folds a constant zone 
to a literal before data
+  // skipping runs. It is resolved the way Spark evaluates it, so for ex +8:00 
is +08:00 and the short
+  // ID PST is America/Los_Angeles; an unknown zone is not matched
+  private def isFixedOffsetTimeZone(expr: Expression): Boolean = expr match {
+    case Literal(timeZone, _) if timeZone != null =>
+      
Try(DateTimeUtils.getZoneId(timeZone.toString).getRules.isFixedOffset).getOrElse(false)
+    case _ => false
+  }
 }
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
 
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
index 0c4745b09939..6c12c1b81d91 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
@@ -876,17 +876,17 @@ cs        startswith(from_utc_timestamp(ts, 'UTC'), 'ab') 
true:boolean
 ei     startswith(from_utc_timestamp(ts, 'UTC'), 'ab') 
((from_utc_timestamp(ts_minValue, 'UTC') <= 'ab') AND 
(from_utc_timestamp(ts_maxValue, 'UTC') >= 'ab'))
 cs     NOT startswith(from_utc_timestamp(ts, 'UTC'), 'ab')     true:boolean
 ei     NOT startswith(from_utc_timestamp(ts, 'UTC'), 'ab')     NOT 
(startswith(from_utc_timestamp(ts_minValue, 'UTC'), 'ab') AND 
startswith(from_utc_timestamp(ts_maxValue, 'UTC'), 'ab'))
-cs     (to_utc_timestamp(ts, 'America/New_York') = 0:timestamp)        
((to_utc_timestamp(ts_minValue, 'America/New_York') <= 0:timestamp) AND 
(to_utc_timestamp(ts_maxValue, 'America/New_York') >= 0:timestamp))
-cs     (0:timestamp < to_utc_timestamp(ts, 'America/New_York'))        
(to_utc_timestamp(ts_maxValue, 'America/New_York') > 0:timestamp)
-cs     (to_utc_timestamp(ts, 'America/New_York') >= 0:timestamp)       
(to_utc_timestamp(ts_maxValue, 'America/New_York') >= 0:timestamp)
-cs     NOT (to_utc_timestamp(ts, 'America/New_York') = 0:timestamp)    NOT 
((to_utc_timestamp(ts_minValue, 'America/New_York') = 0:timestamp) AND 
(to_utc_timestamp(ts_maxValue, 'America/New_York') = 0:timestamp))
-cs     (to_utc_timestamp(ts, 'America/New_York') = cast(0:timestamp as 
timestamp))     ((to_utc_timestamp(ts_minValue, 'America/New_York') <= 
cast(0:timestamp as timestamp)) AND (to_utc_timestamp(ts_maxValue, 
'America/New_York') >= cast(0:timestamp as timestamp)))
-cs     to_utc_timestamp(ts, 'America/New_York') IN (0:timestamp, 
null:timestamp)       (((to_utc_timestamp(ts_minValue, 'America/New_York') <= 
0:timestamp) AND (to_utc_timestamp(ts_maxValue, 'America/New_York') >= 
0:timestamp)) OR ((to_utc_timestamp(ts_minValue, 'America/New_York') <= 
null:timestamp) AND (to_utc_timestamp(ts_maxValue, 'America/New_York') >= 
null:timestamp)))
-cs     NOT to_utc_timestamp(ts, 'America/New_York') IN (0:timestamp)   NOT 
((to_utc_timestamp(ts_minValue, 'America/New_York') = 0:timestamp) AND 
(to_utc_timestamp(ts_maxValue, 'America/New_York') = 0:timestamp))
+cs     (to_utc_timestamp(ts, 'America/New_York') = 0:timestamp)        
true:boolean
+cs     (0:timestamp < to_utc_timestamp(ts, 'America/New_York'))        
true:boolean
+cs     (to_utc_timestamp(ts, 'America/New_York') >= 0:timestamp)       
true:boolean
+cs     NOT (to_utc_timestamp(ts, 'America/New_York') = 0:timestamp)    
true:boolean
+cs     (to_utc_timestamp(ts, 'America/New_York') = cast(0:timestamp as 
timestamp))     true:boolean
+cs     to_utc_timestamp(ts, 'America/New_York') IN (0:timestamp, 
null:timestamp)       true:boolean
+cs     NOT to_utc_timestamp(ts, 'America/New_York') IN (0:timestamp)   
true:boolean
 cs     startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')      
true:boolean
-ei     startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')      
((to_utc_timestamp(ts_minValue, 'America/New_York') <= 'ab') AND 
(to_utc_timestamp(ts_maxValue, 'America/New_York') >= 'ab'))
+ei     startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')      
true:boolean
 cs     NOT startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')  
true:boolean
-ei     NOT startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')  NOT 
(startswith(to_utc_timestamp(ts_minValue, 'America/New_York'), 'ab') AND 
startswith(to_utc_timestamp(ts_maxValue, 'America/New_York'), 'ab'))
+ei     NOT startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')  
true:boolean
 cs     (left(s, 2:int) = '')   ((left(s_minValue, 2:int) <= '') AND 
(left(s_maxValue, 2:int) >= ''))
 cs     ('' < left(s, 2:int))   (left(s_maxValue, 2:int) > '')
 cs     (left(s, 2:int) >= '')  (left(s_maxValue, 2:int) >= '')
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index eb275eb1e539..7aba61718b4a 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -146,8 +146,8 @@ class TestDataSkippingUtils extends 
HoodieSparkClientTestBase with SparkAdapterS
       // file_1 even though its 01:30 row matches; must fall back to 
always-true, keeping the file
       val input = Seq(
         IndexRow("file_1", valueCount = 2,
-          C_minValue = new Timestamp(1793511000000L), // 2026-11-01 05:30Z
-          C_maxValue = new Timestamp(1793513400000L), // 2026-11-01 06:10Z
+          C_minValue = TestDataSkippingUtils.NyFallBackEdt0130,
+          C_maxValue = TestDataSkippingUtils.NyFallBackEst0110,
           C_nullCount = 0))
       val resolvedFilterExpr = resolveExpr(spark, "CAST(C AS STRING) > 
'2026-11-01 01:20:00'", sourceTableSchema)
 
@@ -212,6 +212,11 @@ class TestDataSkippingUtils extends 
HoodieSparkClientTestBase with SparkAdapterS
 }
 
 object TestDataSkippingUtils {
+  // New York local time runs backwards across the DST fall-back: 05:30Z is 
01:30 EDT, but the later
+  // 06:10Z is 01:10 EST
+  private val NyFallBackEdt0130 = new Timestamp(1793511000000L) // 2026-11-01 
05:30Z
+  private val NyFallBackEst0110 = new Timestamp(1793513400000L) // 2026-11-01 
06:10Z
+
   def testStringsLookupFilterExpressionsSource(): 
java.util.stream.Stream[Arguments] = {
     java.util.stream.Stream.of(
       arguments(
@@ -922,7 +927,46 @@ object TestDataSkippingUtils {
           IndexRow("file_1", valueCount = 2, C_minValue = new 
Timestamp(1000L), C_maxValue = new Timestamp(50000L), C_nullCount = 0),
           IndexRow("file_2", valueCount = 2, C_minValue = new 
Timestamp(101000L), C_maxValue = new Timestamp(200000L), C_nullCount = 0)
         ),
-        Seq("file_2"))
+        Seq("file_2")),
+      // New York local time runs backwards across the DST fall-back: 05:30Z 
is 01:30 EDT, but the later
+      // 06:10Z is 01:10 EST. Re-applying from_utc_timestamp over min/max 
stats would wrongly prune
+      // file_1 even though its 05:30Z row matches; must fall back to 
always-true, keeping the file
+      arguments(
+        "from_utc_timestamp(C, 'America/New_York') > TIMESTAMP '2026-11-01 
01:20:00'",
+        Seq(
+          IndexRow("file_1", valueCount = 2,
+            C_minValue = NyFallBackEdt0130,
+            C_maxValue = NyFallBackEst0110,
+            C_nullCount = 0)
+        ),
+        Seq("file_1")),
+      // to_utc_timestamp reads 02:30Z as New York wall-clock time, which 
falls in the DST spring-forward
+      // gap, so it shifts to 07:30Z, past the 07:00Z of the later 03:00Z. 
Re-applying it over min/max stats
+      // would wrongly prune file_1 even though its 02:30Z row matches; must 
fall back to always-true,
+      // keeping the file
+      arguments(
+        "to_utc_timestamp(C, 'America/New_York') > TIMESTAMP '2026-03-08 
07:15:00'",
+        Seq(
+          IndexRow("file_1", valueCount = 2,
+            C_minValue = new Timestamp(1772937000000L), // 2026-03-08 02:30Z
+            C_maxValue = new Timestamp(1772938800000L), // 2026-03-08 03:00Z
+            C_nullCount = 0)
+        ),
+        Seq("file_1")),
+      // A fixed-offset zone shifts every timestamp by the same amount, so it 
still translates and prunes
+      arguments(
+        "from_utc_timestamp(C, '+08:00') > TIMESTAMP '2026-11-01 13:20:00'",
+        Seq(
+          IndexRow("file_1", valueCount = 2,
+            C_minValue = NyFallBackEdt0130,
+            C_maxValue = NyFallBackEst0110,
+            C_nullCount = 0),
+          IndexRow("file_2", valueCount = 2,
+            C_minValue = new Timestamp(1793491200000L), // 2026-11-01 00:00Z
+            C_maxValue = new Timestamp(1793494800000L), // 2026-11-01 01:00Z
+            C_nullCount = 0)
+        ),
+        Seq("file_1"))
     )
   }
 }
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 33d0eb7bb040..5481a76c6fa7 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -17,7 +17,7 @@
 
 package org.apache.hudi
 
-import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference, 
BitwiseOr, Cast, DateAdd, DateDiff, DateSub, Divide, Exp, Expression, Literal, 
Log, Lower, Multiply, ParseToDate, Rand, ShiftLeft, Sqrt, Upper}
+import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference, 
BitwiseOr, Cast, DateAdd, DateDiff, DateSub, Divide, Exp, Expression, 
FromUTCTimestamp, Literal, Log, Lower, Multiply, ParseToDate, Rand, ShiftLeft, 
Sqrt, ToUTCTimestamp, Upper}
 import org.apache.spark.sql.hudi.dataskipping.{ColumnStatsSource, TermRange}
 import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType, 
DataType, DateType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, 
LongType, ShortType, StringType, TimestampType}
 import org.junit.jupiter.api.Assertions.assertEquals
@@ -120,6 +120,26 @@ class TestHoodieCatalystExpressionUtils extends 
SparkAdapterSupport {
     assertEquals(Some(dateAttr), matched(DateDiff(dateAttr, 
Literal(java.sql.Date.valueOf("2022-03-07")))))
   }
 
+  @Test
+  def testUtcTimestampConversionsRequireFixedOffsetZone(): Unit = {
+    // A zone with DST runs local time backwards across a fall-back and shifts 
the spring-forward gap.
+    assertEquals(None, matched(FromUTCTimestamp(tsAttr, 
Literal("America/New_York"))))
+    assertEquals(None, matched(ToUTCTimestamp(tsAttr, 
Literal("America/New_York"))))
+    // A fixed-offset zone shifts every timestamp by the same amount.
+    assertEquals(Some(tsAttr), matched(FromUTCTimestamp(tsAttr, 
Literal("+08:00"))))
+    assertEquals(Some(tsAttr), matched(ToUTCTimestamp(tsAttr, Literal("UTC"))))
+    // Zones are resolved the way Spark evaluates them: Etc/GMT+5 is a fixed 
-05:00, and +8:00 is +08:00.
+    assertEquals(Some(tsAttr), matched(FromUTCTimestamp(tsAttr, 
Literal("Etc/GMT+5"))))
+    assertEquals(Some(tsAttr), matched(ToUTCTimestamp(tsAttr, 
Literal("+8:00"))))
+    // Non-literal, null and unknown zones are not matched.
+    assertEquals(None, matched(FromUTCTimestamp(tsAttr, strAttr)))
+    assertEquals(None, matched(FromUTCTimestamp(tsAttr, Literal(null, 
StringType))))
+    assertEquals(None, matched(ToUTCTimestamp(tsAttr, Literal("Not/AZone"))))
+    // A zone with historical offset changes but no DST is not fixed-offset 
either; GMT+8 is.
+    assertEquals(None, matched(FromUTCTimestamp(tsAttr, 
Literal("Asia/Kolkata"))))
+    assertEquals(Some(tsAttr), matched(ToUTCTimestamp(tsAttr, 
Literal("GMT+8"))))
+  }
+
   @Test
   def testDateParsingTransformationsPreserveOrdering(): Unit = {
     // ParseToDate's case-class arity differs across Spark versions, so 
DateTimeFormatRule matches it
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala
index 38cc323f58c6..69f45fb0018f 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala
@@ -270,4 +270,54 @@ class TestDataSkippingQuery extends HoodieSparkSqlTestBase 
{
       }
     }
   }
+
+  test("Test data skipping keeps rows across DST transitions of 
from_utc_timestamp and to_utc_timestamp") {
+    withTempDir { tmp =>
+      val tableName = generateTableName
+      withSQLConf(
+        "hoodie.metadata.enable" -> "true",
+        "hoodie.metadata.index.column.stats.enable" -> "true",
+        "hoodie.metadata.index.partition.stats.enable" -> "true",
+        "hoodie.enable.data.skipping" -> "true",
+        // keep every insert in its own base file
+        "hoodie.parquet.small.file.limit" -> "0"
+      ) {
+        spark.sql(
+          s"""
+             |create table $tableName (
+             |  id int,
+             |  ts timestamp,
+             |  dt string
+             |) using hudi
+             | tblproperties (primaryKey = 'id', type = 'cow')
+             | partitioned by (dt)
+             | location '${tmp.getCanonicalPath}'
+                """.stripMargin)
+        // 05:30Z is 01:30 EDT, but the later 06:10Z is 01:10 EST (#20518). 
One insert writes both rows to one
+        // file, so the file's and the partition's ts range both span the New 
York fall-back
+        spark.sql(s"insert into $tableName values " +
+          "(1, timestamp'2026-11-01 05:30:00', 'fall'), (2, 
timestamp'2026-11-01 06:10:00', 'fall')")
+        // Read as New York time, 02:30 falls in the spring-forward gap and 
maps to 07:30Z, past the 07:00Z of
+        // the later 03:00. Two inserts write one file each, so only the 
partition's ts range spans the gap
+        spark.sql(s"insert into $tableName values (3, timestamp'2026-03-08 
02:30:00', 'spring')")
+        spark.sql(s"insert into $tableName values (4, timestamp'2026-03-08 
03:00:00', 'spring')")
+        checkAnswer(s"select dt, count(distinct _hoodie_file_name) from 
$tableName group by dt order by dt")(
+          Seq("fall", 1L),
+          Seq("spring", 2L)
+        )
+
+        // The column stats of row 3's single-row file keep it, so only the 
partition stats index could
+        // wrongly prune it
+        checkAnswer(s"select id from $tableName where to_utc_timestamp(ts, 
'America/New_York') " +
+          "between timestamp'2026-03-08 07:15:00' and timestamp'2026-03-08 
08:00:00'")(
+          Seq(3)
+        )
+        // Both the column stats and the partition stats index could wrongly 
prune row 1's file
+        checkAnswer(s"select id from $tableName where from_utc_timestamp(ts, 
'America/New_York') " +
+          "> timestamp'2026-11-01 01:20:00'")(
+          Seq(1)
+        )
+      }
+    }
+  }
 }

Reply via email to