This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a9dcd6807105 fix(spark): reject collation-changing casts in data
skipping (#20458)
a9dcd6807105 is described below
commit a9dcd6807105cbcc739f993b51c507d58334c4ed
Author: voonhous <[email protected]>
AuthorDate: Wed Nov 4 10:01:23 2026 +0800
fix(spark): reject collation-changing casts in data skipping (#20458)
* fix(spark): reject collation-changing casts in data skipping
On Spark 4 StringType carries a collation. isCastPreservingOrdering
matched String <-> Numeric with the StringType companion, which only
equals the default collation, so every cast involving a collated
string fell through to `case _ => true`. A cast to another collation
changes the sort order ("B" < "a" in UTF8_BINARY, "a" < "B" in
UTF8_LCASE), so re-applying it to min/max stats could prune files
holding matching rows.
- String <-> Numeric arms now match by type, covering collated strings
- string -> string casts require an identical StringType; on Spark 3
StringType is a singleton, so nothing changes there
Refs #19445
* fix(spark): reject casts from any type to a collated string
Every type casts to a string, so a cast into a collated target is
rejected whatever its source, not only from another string. A binary
column's byte-ordered bounds would otherwise be compared in
UTF8_LCASE order.
* fix(spark): reject casts from a collated string to non-string types
A collated string does not sort by its bytes, so a cast from it to a
type that does, such as binary, can re-order values. The earlier arms
only covered string targets and numeric targets; a collated source cast
to binary or date still fell through to the order-preserving default.
Reject every cast from a collated string to a non-string type. Default
strings are unchanged, so Spark 3 behaves as before.
---
.../apache/spark/sql/HoodieSparkTypeUtils.scala | 21 ++++++++++++---
.../org/apache/hudi/TestDataSkippingUtils.scala | 13 ++++++++++
.../hudi/TestHoodieCatalystExpressionUtils.scala | 30 +++++++++++++++++++++-
3 files changed, 59 insertions(+), 5 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSparkTypeUtils.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSparkTypeUtils.scala
index 355bbcd1881e..ca4c73330454 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSparkTypeUtils.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSparkTypeUtils.scala
@@ -37,16 +37,29 @@ object HoodieSparkTypeUtils {
def isCastPreservingOrdering(from: DataType, to: DataType): Boolean =
(from, to) match {
// NOTE: Casting between String and Numeric types re-orders elements
(for ex, "10" < "9"
- // lexicographically)
- case (StringType, _: NumericType) => false
- case (_: NumericType, StringType) => false
+ // lexicographically). These match by type, so that collated
strings on Spark 4 are
+ // rejected as well
+ case (_: StringType, _: NumericType) => false
+ case (_: NumericType, _: StringType) => false
// NOTE: Timestamps format in the session time zone, so local time
repeats across a DST
// fall-back (for ex, 01:30 EDT is followed by 01:10 EST) and the
strings re-order
- case (TimestampType, StringType) => false
+ case (TimestampType, _: StringType) => false
+ // NOTE: On Spark 4 StringType carries a collation, and casting to a
different collation changes
+ // the sort order (for ex, "B" < "a" in UTF8_BINARY but "a" < "B"
in UTF8_LCASE). Equality
+ // also compares the char/varchar length constraint, so those
casts are conservatively
+ // rejected too. On Spark 3 StringType is a singleton, making this
arm trivially true
+ case (fromStr: StringType, toStr: StringType) => fromStr == toStr
+ // NOTE: Every type casts to a string, so collated targets are rejected
whatever the source
+ // (for ex, binary bounds are in byte order, which UTF8_LCASE does
not follow)
+ case (_, toStr: StringType) => toStr == StringType
// NOTE: Casting to boolean maps every non-zero (or "true"-like) value
to true, so it is not
// monotonic (for ex, min -5 and max 5 both cast to true while 0
casts to false)
case (BooleanType, BooleanType) => true
case (_, BooleanType) => false
+ // NOTE: A collated string does not sort by its bytes, so casting it to
any non-string type
+ // (for ex, binary, which compares bytes) can re-order values.
String targets are
+ // handled above, so this only rejects collated sources
+ case (fromStr: StringType, _) => fromStr == StringType
// NOTE: Cast.canUpCast rejects decimal to float/double as lossy, but
rounding is monotonic
// and every decimal fits within float range. Spark's type
coercion inserts this cast
// when a decimal column is compared with a float/double (for ex,
`price > 99.5D`)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index a09b91a54ed7..41c04681fd54 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -35,6 +35,7 @@ import
org.apache.spark.sql.internal.SQLConf.SESSION_LOCAL_TIMEZONE
import org.apache.spark.sql.types._
import org.junit.jupiter.api.{BeforeEach, Test}
import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assumptions.assumeTrue
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.{Arguments, MethodSource}
import org.junit.jupiter.params.provider.Arguments.arguments
@@ -108,6 +109,18 @@ class TestDataSkippingUtils extends
HoodieSparkClientTestBase with SparkAdapterS
assertEquals(expectedOutput, rows)
}
+ @Test
+ def testCollationChangingCastDoesNotPrune(): Unit = {
+ assumeTrue(HoodieSparkUtils.gteqSpark4_0, "String collations only exist on
Spark 4.x")
+ // Stats are in UTF8_BINARY order ("B" < "a"), but the filter compares in
UTF8_LCASE order
+ // ("a" < "B"): re-applying the cast to the bounds gives "B" <= "a" =
false and would wrongly
+ // prune file_1, which holds "a". Must fall back to always-true, keeping
the file
+ testLookupFilterExpressions(
+ "CAST(B AS STRING COLLATE UTF8_LCASE) = 'a'",
+ Seq(IndexRow("file_1", valueCount = 2, B_minValue = "B", B_maxValue =
"a", B_nullCount = 0)),
+ Seq("file_1"))
+ }
+
@ParameterizedTest
@MethodSource(Array(
"testMiscLookupFilterExpressionsSource"
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 9b79c24a6340..d937ed51f7ca 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -18,8 +18,9 @@
package org.apache.hudi
import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference,
BitwiseOr, Cast, DateAdd, DateDiff, DateSub, Divide, Exp, Expression, Literal,
Log, Lower, Multiply, ParseToDate, ShiftLeft, Sqrt, Upper}
-import org.apache.spark.sql.types.{BooleanType, ByteType, DateType,
DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType,
StringType, TimestampType}
+import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType,
DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType,
ShortType, StringType, TimestampType}
import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assumptions.assumeTrue
import org.junit.jupiter.api.Test
/**
@@ -38,6 +39,7 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
private val decAttr = AttributeReference("dec", DecimalType(10, 2))()
private val tsAttr = AttributeReference("ts", TimestampType)()
private val boolAttr = AttributeReference("b", BooleanType)()
+ private val binAttr = AttributeReference("bin", BinaryType)()
private def matched(expr: Expression): Option[AttributeReference] =
sparkAdapter.getCatalystExpressionUtils.tryMatchAttributeOrderingPreservingTransformation(expr)
@@ -134,6 +136,32 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
assertEquals(Some(boolAttr), matched(Cast(boolAttr, IntegerType)))
}
+ @Test
+ def testCollationChangingStringCastsDoNotPreserveOrdering(): Unit = {
+ assumeTrue(HoodieSparkUtils.gteqSpark4_0, "String collations only exist on
Spark 4.x")
+ // StringType("UTF8_LCASE") is a Spark 4 API, so it is obtained
reflectively to keep this
+ // file compiling against the Spark 3 profiles.
+ val lcase = StringType.getClass.getMethod("apply", classOf[String])
+ .invoke(StringType, "UTF8_LCASE").asInstanceOf[DataType]
+ val collatedAttr = AttributeReference("cs", lcase)()
+ // Changing collation changes the sort order, in either direction.
+ assertEquals(None, matched(Cast(collatedAttr, StringType)))
+ assertEquals(None, matched(Cast(strAttr, lcase)))
+ // Collated strings and numbers re-order each other like default strings
do.
+ assertEquals(None, matched(Cast(collatedAttr, IntegerType)))
+ assertEquals(None, matched(Cast(intAttr, lcase)))
+ // Any other source cast to a collated string is rejected too, binary
bounds being in byte order.
+ assertEquals(None, matched(Cast(binAttr, lcase)))
+ assertEquals(None, matched(Cast(dateAttr, lcase)))
+ // A collated string cast to any non-string type is rejected, for ex
binary compares bytes.
+ assertEquals(None, matched(Cast(collatedAttr, BinaryType)))
+ assertEquals(None, matched(Cast(collatedAttr, DateType)))
+ // The default string still casts to binary, which keeps its byte order.
+ assertEquals(Some(strAttr), matched(Cast(strAttr, BinaryType)))
+ // A collation-preserving cast keeps the ordering.
+ assertEquals(Some(collatedAttr), matched(Cast(collatedAttr, lcase)))
+ }
+
@Test
def testNestedComposition(): Unit = {
assertEquals(Some(intAttr), matched(Add(Multiply(intAttr, Literal(2)),
Literal(3))))