This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 94677a7c1e10 fix(spark): require a fixed-offset zone for
from/to_utc_timestamp data skipping (#20519)
94677a7c1e10 is described below
commit 94677a7c1e10c78eae20e757fa149fefedf7a233
Author: Vova Kolmakov <[email protected]>
AuthorDate: Thu Nov 5 17:55:51 2026 +0700
fix(spark): require a fixed-offset zone for from/to_utc_timestamp data
skipping (#20519)
* fix(spark): require a fixed-offset zone for from/to_utc_timestamp data
skipping
* test(spark): update data skipping golden filters after rebase onto master
* addressed review comments: pin JDK-stable zones, cover partition stats
with a SQL test and share the fall-back timestamps
---
.../dataskipping/rules/TimeZoneShiftRule.scala | 27 +++++++++---
.../data-skipping/golden-index-filters.txt | 18 ++++----
.../org/apache/hudi/TestDataSkippingUtils.scala | 50 ++++++++++++++++++++--
.../hudi/TestHoodieCatalystExpressionUtils.scala | 22 +++++++++-
.../sql/hudi/feature/TestDataSkippingQuery.scala | 50 ++++++++++++++++++++++
5 files changed, 149 insertions(+), 18 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
index b118ee0e1719..fc2a4fce0fee 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
@@ -17,22 +17,39 @@
package org.apache.spark.sql.hudi.dataskipping.rules
-import org.apache.spark.sql.catalyst.expressions.{Expression,
FromUTCTimestamp, ToUTCTimestamp}
+import org.apache.spark.sql.catalyst.expressions.{Expression,
FromUTCTimestamp, Literal, ToUTCTimestamp}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step}
+import scala.util.Try
+
/**
* Time zone shifts: `FromUTCTimestamp`, `ToUTCTimestamp`.
*
- * Shared guard: none; any time zone operand is accepted today.
+ * Shared guard: [[isFixedOffsetTimeZone]], a literal time zone whose offset
never changes.
*
- * NOTE: The offset jumps at DST transitions, so the shift is not monotonic
across a fall-back (#20449)
+ * NOTE: Both shift by the zone's offset at that time, so they only keep the
ordering for a zone whose offset never
+ * changes, DST or historical. Local time runs backwards when the offset
drops (for ex, 05:30Z is 01:30 EDT but
+ * the later 06:10Z is 01:10 EST), and a local time in the gap of a
rising offset shifts past later ones (for ex,
+ * 02:30 maps to 07:30Z but the later 03:00 to 07:00Z)
*/
object TimeZoneShiftRule extends RangeRule {
override def steps(term: Expression): Seq[Step] = term match {
- case FromUTCTimestamp(timestamp, _) => Seq(Step(term, timestamp,
Increasing, fileGuard = None, partial = false))
- case ToUTCTimestamp(timestamp, _) => Seq(Step(term, timestamp, Increasing,
fileGuard = None, partial = false))
+ case FromUTCTimestamp(timestamp, timeZone) if
isFixedOffsetTimeZone(timeZone) =>
+ Seq(Step(term, timestamp, Increasing, fileGuard = None, partial = false))
+ case ToUTCTimestamp(timestamp, timeZone) if
isFixedOffsetTimeZone(timeZone) =>
+ Seq(Step(term, timestamp, Increasing, fileGuard = None, partial = false))
case _ => Nil
}
+
+ // Only a literal zone is matched, since the optimizer folds a constant zone
to a literal before data
+ // skipping runs. It is resolved the way Spark evaluates it, so for ex +8:00
is +08:00 and the short
+ // ID PST is America/Los_Angeles; an unknown zone is not matched
+ private def isFixedOffsetTimeZone(expr: Expression): Boolean = expr match {
+ case Literal(timeZone, _) if timeZone != null =>
+
Try(DateTimeUtils.getZoneId(timeZone.toString).getRules.isFixedOffset).getOrElse(false)
+ case _ => false
+ }
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
index 0c4745b09939..6c12c1b81d91 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
+++
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
@@ -876,17 +876,17 @@ cs startswith(from_utc_timestamp(ts, 'UTC'), 'ab')
true:boolean
ei startswith(from_utc_timestamp(ts, 'UTC'), 'ab')
((from_utc_timestamp(ts_minValue, 'UTC') <= 'ab') AND
(from_utc_timestamp(ts_maxValue, 'UTC') >= 'ab'))
cs NOT startswith(from_utc_timestamp(ts, 'UTC'), 'ab') true:boolean
ei NOT startswith(from_utc_timestamp(ts, 'UTC'), 'ab') NOT
(startswith(from_utc_timestamp(ts_minValue, 'UTC'), 'ab') AND
startswith(from_utc_timestamp(ts_maxValue, 'UTC'), 'ab'))
-cs (to_utc_timestamp(ts, 'America/New_York') = 0:timestamp)
((to_utc_timestamp(ts_minValue, 'America/New_York') <= 0:timestamp) AND
(to_utc_timestamp(ts_maxValue, 'America/New_York') >= 0:timestamp))
-cs (0:timestamp < to_utc_timestamp(ts, 'America/New_York'))
(to_utc_timestamp(ts_maxValue, 'America/New_York') > 0:timestamp)
-cs (to_utc_timestamp(ts, 'America/New_York') >= 0:timestamp)
(to_utc_timestamp(ts_maxValue, 'America/New_York') >= 0:timestamp)
-cs NOT (to_utc_timestamp(ts, 'America/New_York') = 0:timestamp) NOT
((to_utc_timestamp(ts_minValue, 'America/New_York') = 0:timestamp) AND
(to_utc_timestamp(ts_maxValue, 'America/New_York') = 0:timestamp))
-cs (to_utc_timestamp(ts, 'America/New_York') = cast(0:timestamp as
timestamp)) ((to_utc_timestamp(ts_minValue, 'America/New_York') <=
cast(0:timestamp as timestamp)) AND (to_utc_timestamp(ts_maxValue,
'America/New_York') >= cast(0:timestamp as timestamp)))
-cs to_utc_timestamp(ts, 'America/New_York') IN (0:timestamp,
null:timestamp) (((to_utc_timestamp(ts_minValue, 'America/New_York') <=
0:timestamp) AND (to_utc_timestamp(ts_maxValue, 'America/New_York') >=
0:timestamp)) OR ((to_utc_timestamp(ts_minValue, 'America/New_York') <=
null:timestamp) AND (to_utc_timestamp(ts_maxValue, 'America/New_York') >=
null:timestamp)))
-cs NOT to_utc_timestamp(ts, 'America/New_York') IN (0:timestamp) NOT
((to_utc_timestamp(ts_minValue, 'America/New_York') = 0:timestamp) AND
(to_utc_timestamp(ts_maxValue, 'America/New_York') = 0:timestamp))
+cs (to_utc_timestamp(ts, 'America/New_York') = 0:timestamp)
true:boolean
+cs (0:timestamp < to_utc_timestamp(ts, 'America/New_York'))
true:boolean
+cs (to_utc_timestamp(ts, 'America/New_York') >= 0:timestamp)
true:boolean
+cs NOT (to_utc_timestamp(ts, 'America/New_York') = 0:timestamp)
true:boolean
+cs (to_utc_timestamp(ts, 'America/New_York') = cast(0:timestamp as
timestamp)) true:boolean
+cs to_utc_timestamp(ts, 'America/New_York') IN (0:timestamp,
null:timestamp) true:boolean
+cs NOT to_utc_timestamp(ts, 'America/New_York') IN (0:timestamp)
true:boolean
cs startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')
true:boolean
-ei startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')
((to_utc_timestamp(ts_minValue, 'America/New_York') <= 'ab') AND
(to_utc_timestamp(ts_maxValue, 'America/New_York') >= 'ab'))
+ei startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')
true:boolean
cs NOT startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')
true:boolean
-ei NOT startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab') NOT
(startswith(to_utc_timestamp(ts_minValue, 'America/New_York'), 'ab') AND
startswith(to_utc_timestamp(ts_maxValue, 'America/New_York'), 'ab'))
+ei NOT startswith(to_utc_timestamp(ts, 'America/New_York'), 'ab')
true:boolean
cs (left(s, 2:int) = '') ((left(s_minValue, 2:int) <= '') AND
(left(s_maxValue, 2:int) >= ''))
cs ('' < left(s, 2:int)) (left(s_maxValue, 2:int) > '')
cs (left(s, 2:int) >= '') (left(s_maxValue, 2:int) >= '')
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index eb275eb1e539..7aba61718b4a 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -146,8 +146,8 @@ class TestDataSkippingUtils extends
HoodieSparkClientTestBase with SparkAdapterS
// file_1 even though its 01:30 row matches; must fall back to
always-true, keeping the file
val input = Seq(
IndexRow("file_1", valueCount = 2,
- C_minValue = new Timestamp(1793511000000L), // 2026-11-01 05:30Z
- C_maxValue = new Timestamp(1793513400000L), // 2026-11-01 06:10Z
+ C_minValue = TestDataSkippingUtils.NyFallBackEdt0130,
+ C_maxValue = TestDataSkippingUtils.NyFallBackEst0110,
C_nullCount = 0))
val resolvedFilterExpr = resolveExpr(spark, "CAST(C AS STRING) >
'2026-11-01 01:20:00'", sourceTableSchema)
@@ -212,6 +212,11 @@ class TestDataSkippingUtils extends
HoodieSparkClientTestBase with SparkAdapterS
}
object TestDataSkippingUtils {
+ // New York local time runs backwards across the DST fall-back: 05:30Z is
01:30 EDT, but the later
+ // 06:10Z is 01:10 EST
+ private val NyFallBackEdt0130 = new Timestamp(1793511000000L) // 2026-11-01
05:30Z
+ private val NyFallBackEst0110 = new Timestamp(1793513400000L) // 2026-11-01
06:10Z
+
def testStringsLookupFilterExpressionsSource():
java.util.stream.Stream[Arguments] = {
java.util.stream.Stream.of(
arguments(
@@ -922,7 +927,46 @@ object TestDataSkippingUtils {
IndexRow("file_1", valueCount = 2, C_minValue = new
Timestamp(1000L), C_maxValue = new Timestamp(50000L), C_nullCount = 0),
IndexRow("file_2", valueCount = 2, C_minValue = new
Timestamp(101000L), C_maxValue = new Timestamp(200000L), C_nullCount = 0)
),
- Seq("file_2"))
+ Seq("file_2")),
+ // New York local time runs backwards across the DST fall-back: 05:30Z
is 01:30 EDT, but the later
+ // 06:10Z is 01:10 EST. Re-applying from_utc_timestamp over min/max
stats would wrongly prune
+ // file_1 even though its 05:30Z row matches; must fall back to
always-true, keeping the file
+ arguments(
+ "from_utc_timestamp(C, 'America/New_York') > TIMESTAMP '2026-11-01
01:20:00'",
+ Seq(
+ IndexRow("file_1", valueCount = 2,
+ C_minValue = NyFallBackEdt0130,
+ C_maxValue = NyFallBackEst0110,
+ C_nullCount = 0)
+ ),
+ Seq("file_1")),
+ // to_utc_timestamp reads 02:30Z as New York wall-clock time, which
falls in the DST spring-forward
+ // gap, so it shifts to 07:30Z, past the 07:00Z of the later 03:00Z.
Re-applying it over min/max stats
+ // would wrongly prune file_1 even though its 02:30Z row matches; must
fall back to always-true,
+ // keeping the file
+ arguments(
+ "to_utc_timestamp(C, 'America/New_York') > TIMESTAMP '2026-03-08
07:15:00'",
+ Seq(
+ IndexRow("file_1", valueCount = 2,
+ C_minValue = new Timestamp(1772937000000L), // 2026-03-08 02:30Z
+ C_maxValue = new Timestamp(1772938800000L), // 2026-03-08 03:00Z
+ C_nullCount = 0)
+ ),
+ Seq("file_1")),
+ // A fixed-offset zone shifts every timestamp by the same amount, so it
still translates and prunes
+ arguments(
+ "from_utc_timestamp(C, '+08:00') > TIMESTAMP '2026-11-01 13:20:00'",
+ Seq(
+ IndexRow("file_1", valueCount = 2,
+ C_minValue = NyFallBackEdt0130,
+ C_maxValue = NyFallBackEst0110,
+ C_nullCount = 0),
+ IndexRow("file_2", valueCount = 2,
+ C_minValue = new Timestamp(1793491200000L), // 2026-11-01 00:00Z
+ C_maxValue = new Timestamp(1793494800000L), // 2026-11-01 01:00Z
+ C_nullCount = 0)
+ ),
+ Seq("file_1"))
)
}
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 33d0eb7bb040..5481a76c6fa7 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -17,7 +17,7 @@
package org.apache.hudi
-import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference,
BitwiseOr, Cast, DateAdd, DateDiff, DateSub, Divide, Exp, Expression, Literal,
Log, Lower, Multiply, ParseToDate, Rand, ShiftLeft, Sqrt, Upper}
+import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference,
BitwiseOr, Cast, DateAdd, DateDiff, DateSub, Divide, Exp, Expression,
FromUTCTimestamp, Literal, Log, Lower, Multiply, ParseToDate, Rand, ShiftLeft,
Sqrt, ToUTCTimestamp, Upper}
import org.apache.spark.sql.hudi.dataskipping.{ColumnStatsSource, TermRange}
import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType,
DataType, DateType, Decimal, DecimalType, DoubleType, FloatType, IntegerType,
LongType, ShortType, StringType, TimestampType}
import org.junit.jupiter.api.Assertions.assertEquals
@@ -120,6 +120,26 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
assertEquals(Some(dateAttr), matched(DateDiff(dateAttr,
Literal(java.sql.Date.valueOf("2022-03-07")))))
}
+ @Test
+ def testUtcTimestampConversionsRequireFixedOffsetZone(): Unit = {
+ // A zone with DST runs local time backwards across a fall-back and shifts
the spring-forward gap.
+ assertEquals(None, matched(FromUTCTimestamp(tsAttr,
Literal("America/New_York"))))
+ assertEquals(None, matched(ToUTCTimestamp(tsAttr,
Literal("America/New_York"))))
+ // A fixed-offset zone shifts every timestamp by the same amount.
+ assertEquals(Some(tsAttr), matched(FromUTCTimestamp(tsAttr,
Literal("+08:00"))))
+ assertEquals(Some(tsAttr), matched(ToUTCTimestamp(tsAttr, Literal("UTC"))))
+ // Zones are resolved the way Spark evaluates them: Etc/GMT+5 is a fixed
-05:00, and +8:00 is +08:00.
+ assertEquals(Some(tsAttr), matched(FromUTCTimestamp(tsAttr,
Literal("Etc/GMT+5"))))
+ assertEquals(Some(tsAttr), matched(ToUTCTimestamp(tsAttr,
Literal("+8:00"))))
+ // Non-literal, null and unknown zones are not matched.
+ assertEquals(None, matched(FromUTCTimestamp(tsAttr, strAttr)))
+ assertEquals(None, matched(FromUTCTimestamp(tsAttr, Literal(null,
StringType))))
+ assertEquals(None, matched(ToUTCTimestamp(tsAttr, Literal("Not/AZone"))))
+ // A zone with historical offset changes but no DST is not fixed-offset
either; GMT+8 is.
+ assertEquals(None, matched(FromUTCTimestamp(tsAttr,
Literal("Asia/Kolkata"))))
+ assertEquals(Some(tsAttr), matched(ToUTCTimestamp(tsAttr,
Literal("GMT+8"))))
+ }
+
@Test
def testDateParsingTransformationsPreserveOrdering(): Unit = {
// ParseToDate's case-class arity differs across Spark versions, so
DateTimeFormatRule matches it
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala
index 38cc323f58c6..69f45fb0018f 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestDataSkippingQuery.scala
@@ -270,4 +270,54 @@ class TestDataSkippingQuery extends HoodieSparkSqlTestBase
{
}
}
}
+
+ test("Test data skipping keeps rows across DST transitions of
from_utc_timestamp and to_utc_timestamp") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ withSQLConf(
+ "hoodie.metadata.enable" -> "true",
+ "hoodie.metadata.index.column.stats.enable" -> "true",
+ "hoodie.metadata.index.partition.stats.enable" -> "true",
+ "hoodie.enable.data.skipping" -> "true",
+ // keep every insert in its own base file
+ "hoodie.parquet.small.file.limit" -> "0"
+ ) {
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | ts timestamp,
+ | dt string
+ |) using hudi
+ | tblproperties (primaryKey = 'id', type = 'cow')
+ | partitioned by (dt)
+ | location '${tmp.getCanonicalPath}'
+ """.stripMargin)
+ // 05:30Z is 01:30 EDT, but the later 06:10Z is 01:10 EST (#20518).
One insert writes both rows to one
+ // file, so the file's and the partition's ts range both span the New
York fall-back
+ spark.sql(s"insert into $tableName values " +
+ "(1, timestamp'2026-11-01 05:30:00', 'fall'), (2,
timestamp'2026-11-01 06:10:00', 'fall')")
+ // Read as New York time, 02:30 falls in the spring-forward gap and
maps to 07:30Z, past the 07:00Z of
+ // the later 03:00. Two inserts write one file each, so only the
partition's ts range spans the gap
+ spark.sql(s"insert into $tableName values (3, timestamp'2026-03-08
02:30:00', 'spring')")
+ spark.sql(s"insert into $tableName values (4, timestamp'2026-03-08
03:00:00', 'spring')")
+ checkAnswer(s"select dt, count(distinct _hoodie_file_name) from
$tableName group by dt order by dt")(
+ Seq("fall", 1L),
+ Seq("spring", 2L)
+ )
+
+ // The column stats of row 3's single-row file keep it, so only the
partition stats index could
+ // wrongly prune it
+ checkAnswer(s"select id from $tableName where to_utc_timestamp(ts,
'America/New_York') " +
+ "between timestamp'2026-03-08 07:15:00' and timestamp'2026-03-08
08:00:00'")(
+ Seq(3)
+ )
+ // Both the column stats and the partition stats index could wrongly
prune row 1's file
+ checkAnswer(s"select id from $tableName where from_utc_timestamp(ts,
'America/New_York') " +
+ "> timestamp'2026-11-01 01:20:00'")(
+ Seq(1)
+ )
+ }
+ }
+ }
}