This is an automated email from the ASF dual-hosted git repository.

voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new c4f0c4c2a80d fix(spark): make the data skipping cast allowlist 
deny-by-default (#20513)
c4f0c4c2a80d is described below

commit c4f0c4c2a80d8ac2ef11939a3301ef38b7415567
Author: voonhous <[email protected]>
AuthorDate: Thu Nov 5 17:09:33 2026 +0800

    fix(spark): make the data skipping cast allowlist deny-by-default (#20513)
    
    * fix(spark): make the data skipping cast allowlist deny-by-default
    
    CastRule.preservesOrder rejected a list of known-bad casts and
    returned true for everything else, so every unlisted pair was
    re-applied to min/max stats. Each soundness bug found so far came
    through that fallthrough, for ex timestamp to timestamp_ntz, which
    takes the local wall-clock time and re-orders across a DST fall-back.
    
    It now accepts only an allowlist: identity, numeric up-casts, decimal
    to float/double, boolean to integral/float/double, timestamp to
    bigint/float/double, date <-> timestamp(_ntz) truncation, and default
    string <-> binary. Every other cast falls back to no pruning.
    
    String to date/timestamp stays allowed, though unpadded strings parse
    out of order: the optimizer rewrites to_date/to_timestamp without a
    format into that cast, and expression indexes on them would stop
    pruning. Tracked in #19481.
    
    Refs #19445
    
    * fix(spark): drop string to date/timestamp from the cast allowlist
    
    The arm was kept so that expression indexes on to_date/to_timestamp
    keep pruning, but they never reach it: ExpressionIndexSupport strips
    the indexed function and rebuilds the query over the bare column, so
    the stored f(col) bounds are compared as is. Only plain column stats
    on a string column used it, where unpadded strings sort differently
    from what they parse to ("2022-1-5" > "2022-01-10" as strings, but
    parses to an earlier date).
    
    * test(spark): update golden filters for the cast allowlist
    
    Dropping string to date/timestamp from the allowlist changes 9
    golden lines, all cast(s as date), which now compile to TRUE.
---
 .../sql/hudi/dataskipping/rules/CastRule.scala     | 76 ++++++++++++----------
 .../data-skipping/golden-index-filters.txt         | 18 ++---
 .../org/apache/hudi/TestDataSkippingUtils.scala    | 10 ++-
 .../hudi/TestHoodieCatalystExpressionUtils.scala   | 31 +++++++++
 4 files changed, 91 insertions(+), 44 deletions(-)

diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
index cde92d492f77..7f6f86d5411b 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
@@ -19,52 +19,45 @@ package org.apache.spark.sql.hudi.dataskipping.rules
 
 import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
 import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step}
-import org.apache.spark.sql.types.{BooleanType, DataType, DecimalType, 
DoubleType, FloatType, LongType, NumericType, StringType, TimestampType}
+import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType, 
DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, 
NumericType, ShortType, StringType, TimestampNTZType, TimestampType}
 
 import scala.util.Try
 
 /**
  * `Cast`.
  *
- * Shared guard: [[preservesOrder]] of the child type to the target type 
(rejects, for ex, string to
- * numeric, timestamp to string, narrowing numeric casts, and collation 
changes). The (from, to)
- * allowlist lives here, at the top of this object.
- *
- * NOTE: The trailing arm still accepts every pair not listed; making the 
allowlist deny-by-default
- *       (trailing `case _ => false`) is pending in #20513
+ * Shared guard: [[preservesOrder]] of the child type to the target type. The 
(from, to) allowlist
+ * lives here, at the top of this object, and is deny-by-default: a cast pair 
not listed has no range.
  */
 object CastRule extends RangeRule {
 
   /**
    * Checks whether casting expression of [[from]] [[DataType]] to [[to]] 
[[DataType]] will
-   * preserve ordering of the elements
+   * preserve ordering of the elements.
+   *
+   * Only casts known to preserve ordering are accepted; anything else is 
rejected. A rejection only
+   * costs pruning opportunity, never correctness, and makes new types (for 
ex, Spark 4 collations or
+   * TimeType) safe by construction. Notable rejected classes:
+   *  - string to/from numeric re-orders values (for ex, "10" < "9" 
lexicographically)
+   *  - timestamp to string formats in the session time zone, so local time 
repeats across a DST
+   *    fall-back (for ex, 01:30 EDT is followed by 01:10 EST)
+   *  - collation-changing string casts (for ex, "B" < "a" in UTF8_BINARY but 
"a" < "B" in UTF8_LCASE)
+   *  - casts to boolean (for ex, min -5 and max 5 both cast to true while 0 
casts to false)
+   *  - narrowing numeric casts, which overflow and wrap around in non-ANSI 
mode (for ex, bigint to int)
+   *  - timestamp to/from timestamp_ntz, which shifts across DST (for ex, a 
local time in the
+   *    spring-forward gap shifts forward, so 02:30 maps after 03:00)
+   *  - string to date/timestamp, since unpadded strings sort differently from 
what they parse to (for
+   *    ex, "2022-1-5" > "2022-01-10" as strings, but parses to an earlier 
date). Expression indexes on
+   *    to_date/to_timestamp do not need it: `ExpressionIndexSupport` strips 
the function first
    */
   private[dataskipping] def preservesOrder(from: DataType, to: DataType): 
Boolean =
     (from, to) match {
-      // NOTE: Casting between String and Numeric types re-orders elements 
(for ex, "10" < "9"
-      //       lexicographically). These match by type, so that collated 
strings on Spark 4 are
-      //       rejected as well
-      case (_: StringType, _: NumericType) => false
-      case (_: NumericType, _: StringType) => false
-      // NOTE: Timestamps format in the session time zone, so local time 
repeats across a DST
-      //       fall-back (for ex, 01:30 EDT is followed by 01:10 EST) and the 
strings re-order
-      case (TimestampType, _: StringType) => false
-      // NOTE: On Spark 4 StringType carries a collation, and casting to a 
different collation changes
-      //       the sort order (for ex, "B" < "a" in UTF8_BINARY but "a" < "B" 
in UTF8_LCASE). Equality
-      //       also compares the char/varchar length constraint, so those 
casts are conservatively
-      //       rejected too. On Spark 3 StringType is a singleton, making this 
arm trivially true
-      case (fromStr: StringType, toStr: StringType) => fromStr == toStr
-      // NOTE: Every type casts to a string, so collated targets are rejected 
whatever the source
-      //       (for ex, binary bounds are in byte order, which UTF8_LCASE does 
not follow)
-      case (_, toStr: StringType) => toStr == StringType
-      // NOTE: Casting to boolean maps every non-zero (or "true"-like) value 
to true, so it is not
-      //       monotonic (for ex, min -5 and max 5 both cast to true while 0 
casts to false)
-      case (BooleanType, BooleanType) => true
-      case (_, BooleanType) => false
-      // NOTE: A collated string does not sort by its bytes, so casting it to 
any non-string type
-      //       (for ex, binary, which compares bytes) can re-order values. 
String targets are
-      //       handled above, so this only rejects collated sources
-      case (fromStr: StringType, _) => fromStr == StringType
+      // NOTE: On Spark 4 StringType equality includes the collation and the 
char/varchar length
+      //       constraint, so a collation-changing cast is not an identity 
(for ex, "B" < "a" in
+      //       UTF8_BINARY but "a" < "B" in UTF8_LCASE). On Spark 3 StringType 
is a singleton
+      case _ if from == to => true
+
+      // Numeric
       // NOTE: Cast.canUpCast rejects decimal to float/double as lossy, but 
rounding is monotonic
       //       and every decimal fits within float range. Spark's type 
coercion inserts this cast
       //       when a decimal column is compared with a float/double (for ex, 
`price > 99.5D`)
@@ -74,13 +67,28 @@ object CastRule extends RangeRule {
       //       This is conservative (for ex, double to float rounds 
monotonically but is
       //       rejected), which only costs pruning opportunity, never 
correctness
       case (fromNum: NumericType, toNum: NumericType) => 
Cast.canUpCast(fromNum, toNum)
+      // NOTE: false < true maps to 0 < 1. Decimal targets are excluded since, 
for ex,
+      //       DECIMAL(1, 1) cannot hold 1 and overflows into NULL
+      case (BooleanType, ByteType | ShortType | IntegerType | LongType | 
FloatType | DoubleType) => true
+
+      // Date and time
       // NOTE: Timestamps cast to numbers through their epoch seconds, which 
only fit bigint and
       //       floating-point targets; narrower integral targets wrap around 
and decimals overflow
       //       into NULL in non-ANSI mode
       case (TimestampType, LongType | FloatType | DoubleType) => true
-      case (TimestampType, _: NumericType) => false
+      // NOTE: A date maps to the start of that day, which increases with the 
date
+      case (DateType, TimestampType | TimestampNTZType) => true
+      // NOTE: Truncating to the (local) date is non-decreasing, also across a 
DST fall-back since
+      //       the date does not repeat
+      case (TimestampType | TimestampNTZType, DateType) => true
+
+      // Binary and strings
+      // NOTE: The default (UTF8_BINARY) string compares by its bytes, same as 
binary. Matching the
+      //       StringType companion excludes collated strings on Spark 4
+      case (StringType, BinaryType) => true
+      case (BinaryType, StringType) => true
 
-      case _ => true
+      case _ => false
     }
 
   override def steps(term: Expression): Seq[Step] = term match {
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
 
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
index b51e4170e117..0c4745b09939 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
@@ -626,17 +626,17 @@ cs        startswith(cast(dec as double), 'ab')   
true:boolean
 ei     startswith(cast(dec as double), 'ab')   ((cast(dec_minValue as double) 
<= 'ab') AND (cast(dec_maxValue as double) >= 'ab'))
 cs     NOT startswith(cast(dec as double), 'ab')       true:boolean
 ei     NOT startswith(cast(dec as double), 'ab')       NOT 
(startswith(cast(dec_minValue as double), 'ab') AND 
startswith(cast(dec_maxValue as double), 'ab'))
-cs     (cast(s as date) = 0:date)      ((cast(s_minValue as date) <= 0:date) 
AND (cast(s_maxValue as date) >= 0:date))
-cs     (0:date < cast(s as date))      (cast(s_maxValue as date) > 0:date)
-cs     (cast(s as date) >= 0:date)     (cast(s_maxValue as date) >= 0:date)
-cs     NOT (cast(s as date) = 0:date)  NOT ((cast(s_minValue as date) = 
0:date) AND (cast(s_maxValue as date) = 0:date))
-cs     (cast(s as date) = cast(0:date as date))        ((cast(s_minValue as 
date) <= cast(0:date as date)) AND (cast(s_maxValue as date) >= cast(0:date as 
date)))
-cs     cast(s as date) IN (0:date, null:date)  (((cast(s_minValue as date) <= 
0:date) AND (cast(s_maxValue as date) >= 0:date)) OR ((cast(s_minValue as date) 
<= null:date) AND (cast(s_maxValue as date) >= null:date)))
-cs     NOT cast(s as date) IN (0:date) NOT ((cast(s_minValue as date) = 
0:date) AND (cast(s_maxValue as date) = 0:date))
+cs     (cast(s as date) = 0:date)      true:boolean
+cs     (0:date < cast(s as date))      true:boolean
+cs     (cast(s as date) >= 0:date)     true:boolean
+cs     NOT (cast(s as date) = 0:date)  true:boolean
+cs     (cast(s as date) = cast(0:date as date))        true:boolean
+cs     cast(s as date) IN (0:date, null:date)  true:boolean
+cs     NOT cast(s as date) IN (0:date) true:boolean
 cs     startswith(cast(s as date), 'ab')       true:boolean
-ei     startswith(cast(s as date), 'ab')       ((cast(s_minValue as date) <= 
'ab') AND (cast(s_maxValue as date) >= 'ab'))
+ei     startswith(cast(s as date), 'ab')       true:boolean
 cs     NOT startswith(cast(s as date), 'ab')   true:boolean
-ei     NOT startswith(cast(s as date), 'ab')   NOT (startswith(cast(s_minValue 
as date), 'ab') AND startswith(cast(s_maxValue as date), 'ab'))
+ei     NOT startswith(cast(s as date), 'ab')   true:boolean
 cs     (cast(d as timestamp) = 0:timestamp)    ((cast(d_minValue as timestamp) 
<= 0:timestamp) AND (cast(d_maxValue as timestamp) >= 0:timestamp))
 cs     (0:timestamp < cast(d as timestamp))    (cast(d_maxValue as timestamp) 
> 0:timestamp)
 cs     (cast(d as timestamp) >= 0:timestamp)   (cast(d_maxValue as timestamp) 
>= 0:timestamp)
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index 4ac23be2174b..eb275eb1e539 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -138,7 +138,7 @@ class TestDataSkippingUtils extends 
HoodieSparkClientTestBase with SparkAdapterS
   }
 
   @Test
-  def testTimestampToStringCastAcrossDstFallBack(): Unit = {
+  def testTimestampCastsAcrossDstFallBack(): Unit = {
     spark.sqlContext.setConf(SESSION_LOCAL_TIMEZONE.key, "America/New_York")
     try {
       // 05:30Z formats as 01:30 EDT but the later 06:10Z formats as 01:10 
EST, so the string max
@@ -152,6 +152,14 @@ class TestDataSkippingUtils extends 
HoodieSparkClientTestBase with SparkAdapterS
       val resolvedFilterExpr = resolveExpr(spark, "CAST(C AS STRING) > 
'2026-11-01 01:20:00'", sourceTableSchema)
 
       assertEquals(Seq("file_1"), applyFilterExpr(resolvedFilterExpr, input))
+
+      // Converting to timestamp_ntz takes the same local wall-clock time, so 
it re-orders the same way
+      if (HoodieSparkUtils.gteqSpark3_4) {
+        val ntzFilterExpr = resolveExpr(spark,
+          "CAST(C AS TIMESTAMP_NTZ) > TIMESTAMP_NTZ '2026-11-01 01:20:00'", 
sourceTableSchema)
+
+        assertEquals(Seq("file_1"), applyFilterExpr(ntzFilterExpr, input))
+      }
     } finally {
       spark.sqlContext.setConf(SESSION_LOCAL_TIMEZONE.key, "UTC")
     }
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 324d01e8c78b..33d0eb7bb040 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -41,6 +41,9 @@ class TestHoodieCatalystExpressionUtils extends 
SparkAdapterSupport {
   private val tsAttr = AttributeReference("ts", TimestampType)()
   private val boolAttr = AttributeReference("b", BooleanType)()
   private val binAttr = AttributeReference("bin", BinaryType)()
+  // TimestampNTZType is private[spark] on Spark 3.3, where TIMESTAMP_NTZ is 
unsupported, so it is
+  // parsed from DDL and only built on Spark 3.4+.
+  private lazy val ntzAttr = AttributeReference("ntz", 
DataType.fromDDL("timestamp_ntz"))()
 
   /**
    * The column `expr` is ranged through, found by the production range walk 
([[TermRange.derive]])
@@ -204,6 +207,34 @@ class TestHoodieCatalystExpressionUtils extends 
SparkAdapterSupport {
     assertEquals(Some(collatedAttr), matched(Cast(collatedAttr, lcase)))
   }
 
+  @Test
+  def testCastsOutsideTheAllowlistDoNotPreserveOrdering(): Unit = {
+    // Dates and booleans only cast to the explicitly allowed targets; 
DECIMAL(1, 1) cannot hold true.
+    assertEquals(None, matched(Cast(dateAttr, StringType)))
+    assertEquals(None, matched(Cast(dateAttr, IntegerType)))
+    assertEquals(None, matched(Cast(boolAttr, DecimalType(1, 1))))
+    // A date maps to the start of its day, and truncating a timestamp to its 
date is non-decreasing.
+    assertEquals(Some(dateAttr), matched(Cast(dateAttr, TimestampType)))
+    assertEquals(Some(tsAttr), matched(Cast(tsAttr, DateType)))
+    // The default string and binary both compare by bytes.
+    assertEquals(Some(binAttr), matched(Cast(binAttr, StringType)))
+    // Boolean to bigint keeps false < true as 0 < 1, and an identity cast is 
always allowed.
+    assertEquals(Some(boolAttr), matched(Cast(boolAttr, LongType)))
+    assertEquals(Some(intAttr), matched(Cast(intAttr, IntegerType)))
+    // Unpadded strings sort differently from what they parse to ("2022-1-5" > 
"2022-01-10").
+    assertEquals(None, matched(Cast(strAttr, DateType)))
+    assertEquals(None, matched(Cast(strAttr, TimestampType)))
+    if (HoodieSparkUtils.gteqSpark3_4) {
+      // Converting between timestamp and timestamp_ntz shifts local times in 
the DST spring-forward gap.
+      assertEquals(None, matched(Cast(tsAttr, ntzAttr.dataType)))
+      assertEquals(None, matched(Cast(ntzAttr, TimestampType)))
+      assertEquals(None, matched(Cast(strAttr, ntzAttr.dataType)))
+      // Dates map to the start of the day and back without a time zone shift.
+      assertEquals(Some(dateAttr), matched(Cast(dateAttr, ntzAttr.dataType)))
+      assertEquals(Some(ntzAttr), matched(Cast(ntzAttr, DateType)))
+    }
+  }
+
   @Test
   def testNestedComposition(): Unit = {
     assertEquals(Some(intAttr), matched(Add(Multiply(intAttr, Literal(2)), 
Literal(3))))

Reply via email to