This is an automated email from the ASF dual-hosted git repository.
wombatu-kun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 1568cebd32dc fix(spark): stop treating the datediff start date as
order-preserving in data skipping (#20230)
1568cebd32dc is described below
commit 1568cebd32dc2164e1087e83658f97311e9e5678
Author: Vova Kolmakov <[email protected]>
AuthorDate: Fri Oct 16 16:17:54 2026 +0700
fix(spark): stop treating the datediff start date as order-preserving in
data skipping (#20230)
Co-authored-by: Vova Kolmakov <[email protected]>
---
.../sql/BaseHoodieCatalystExpressionUtils.scala | 3 ++-
.../org/apache/hudi/TestDataSkippingUtils.scala | 25 ++++++++++++++++++++++
.../hudi/TestHoodieCatalystExpressionUtils.scala | 6 +++++-
3 files changed, 32 insertions(+), 2 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystExpressionUtils.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystExpressionUtils.scala
index 821eebcce230..e2a39d7c29a5 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystExpressionUtils.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/BaseHoodieCatalystExpressionUtils.scala
@@ -64,8 +64,9 @@ abstract class BaseHoodieCatalystExpressionUtils extends
HoodieCatalystExpressio
case DateFormatClass(OrderPreservingTransformation(attrRef), _, _) =>
Some(attrRef)
case DateAdd(OrderPreservingTransformation(attrRef), _) =>
Some(attrRef)
case DateSub(OrderPreservingTransformation(attrRef), _) =>
Some(attrRef)
+ // NOTE: DateDiff(end, start) is end - start, so it reverses the
ordering of its start date,
+ // which hence must not be matched
case DateDiff(OrderPreservingTransformation(attrRef), _) =>
Some(attrRef)
- case DateDiff(_, OrderPreservingTransformation(attrRef)) =>
Some(attrRef)
case FromUnixTime(OrderPreservingTransformation(attrRef), _, _) =>
Some(attrRef)
case FromUTCTimestamp(OrderPreservingTransformation(attrRef), _) =>
Some(attrRef)
case ToUnixTimestamp(OrderPreservingTransformation(attrRef), _, _, _)
=> Some(attrRef)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index a86778645e13..51e5aae98be6 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -714,6 +714,31 @@ object TestDataSkippingUtils {
B_maxValue = "2022-03-06", // 03/06/2022
B_nullCount = 0)
),
+ Seq("file_1")),
+ arguments(
+ // datediff(end, start) preserves the ordering of its end date, hence
is translated
+ "datediff(CAST(C AS DATE), DATE '2022-03-07') >= 2",
+ Seq(
+ IndexRow("file_1", valueCount = 1,
+ C_minValue = new Timestamp(1646711448000L), // 03/08/2022
+ C_maxValue = new Timestamp(1646797848000L), // 03/09/2022
+ C_nullCount = 0),
+ IndexRow("file_2", valueCount = 1,
+ C_minValue = new Timestamp(1646625048000L), // 03/07/2022
+ C_maxValue = new Timestamp(1646711448000L), // 03/08/2022
+ C_nullCount = 0)
+ ),
+ Seq("file_1")),
+ arguments(
+ // NOTE: datediff(end, start) reverses the ordering of its start date,
so it isn't translated:
+ // file_1 holds 03/08/2022 (1 day before the end date) though
its min is 2 days before it
+ "datediff(DATE '2022-03-09', CAST(C AS DATE)) <= 1",
+ Seq(
+ IndexRow("file_1", valueCount = 1,
+ C_minValue = new Timestamp(1646625048000L), // 03/07/2022
+ C_maxValue = new Timestamp(1646711448000L), // 03/08/2022
+ C_nullCount = 0)
+ ),
Seq("file_1"))
)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 41638d4c4242..a7bd52dcefd7 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -17,7 +17,7 @@
package org.apache.hudi
-import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference,
BitwiseOr, Cast, DateAdd, DateSub, Divide, Exp, Expression, Literal, Log,
Lower, Multiply, ParseToDate, ShiftLeft, Sqrt, Upper}
+import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference,
BitwiseOr, Cast, DateAdd, DateDiff, DateSub, Divide, Exp, Expression, Literal,
Log, Lower, Multiply, ParseToDate, ShiftLeft, Sqrt, Upper}
import org.apache.spark.sql.types.{DateType, DoubleType, IntegerType,
LongType, StringType}
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
@@ -68,6 +68,8 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
def testDateTransformationsPreserveOrdering(): Unit = {
assertEquals(Some(dateAttr), matched(DateAdd(dateAttr, Literal(1))))
assertEquals(Some(dateAttr), matched(DateSub(dateAttr, Literal(1))))
+ // DateDiff(end, start) = end - start preserves the ordering of its end
date.
+ assertEquals(Some(dateAttr), matched(DateDiff(dateAttr,
Literal(java.sql.Date.valueOf("2022-03-07")))))
}
@Test
@@ -97,6 +99,8 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
@Test
def testNonOrderPreservingExpressionsDoNotMatch(): Unit = {
+ // DateDiff(end, start) = end - start reverses the ordering of its start
date.
+ assertEquals(None,
matched(DateDiff(Literal(java.sql.Date.valueOf("2022-03-09")), dateAttr)))
// A bare literal carries no attribute.
assertEquals(None, matched(Literal(5)))
// No attribute on either operand.
