This is an automated email from the ASF dual-hosted git repository.
LuciferYang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3071f767a0 [GLUTEN-12859][TEST] Restore unhex null assertions across
Spark modules (#13319)
3071f767a0 is described below
commit 3071f767a0f260804c6904bba74f51d9ce9dd94e
Author: sabbasani <[email protected]>
AuthorDate: Fri Oct 30 07:58:09 2026 +0530
[GLUTEN-12859][TEST] Restore unhex null assertions across Spark modules
(#13319)
## What changes were proposed in this pull request?
In `GlutenMathExpressionsSuite`, assertions verifying that `unhex` returns
`null` for invalid hex inputs (e.g., `"GG"`, `"三重的"`) were previously commented
out because the ClickHouse backend does not return `null` on invalid hex inputs
(see #7232). Commenting out these lines disabled test coverage for backends
like Velox that conform to Spark's null-returning behavior.
This PR:
1. Splits the invalid-input assertions out of `testGluten("unhex")` into a
dedicated test: `testGluten("unhex returns null for invalid hex input")`.
2. Restores the null assertions in `GlutenMathExpressionsSuite` across all
Spark versions (`spark34`, `spark35`, `spark40`, `spark41`, `spark42`).
3. Excludes only `unhex returns null for invalid hex input` in
`ClickHouseTestSettings.scala` for each Spark version (referencing #7232),
ensuring ClickHouse still validates valid-input `unhex` behavior while other
backends test both valid and invalid input handling.
## Why are the changes needed?
- Fixes [GLUTEN-12859](https://github.com/apache/gluten/issues/12859).
- Re-enables full test coverage for Spark-compliant `unhex` null behavior
on invalid input for Velox backend without breaking ClickHouse CI.
- Allows ClickHouse to keep running valid-input `unhex` tests rather than
leaving invalid test lines commented out in common test suites.
## How was this patch tested?
- Tested `GlutenMathExpressionsSuite` across Spark modules:
```bash
./build/mvn test -pl gluten-ut -Pspark-ut -Pbackends-velox -Pspark-3.5
-Pscala-2.12 -Dtest=GlutenMathExpressionsSuite
./build/mvn test -pl gluten-ut -Pspark-ut -Pbackends-clickhouse
-Pspark-3.5 -Pscala-2.12 -Dtest=GlutenMathExpressionsSuite
## Was this patch authored or co-authored using generative AI tooling?
No
---
.../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../catalyst/expressions/GlutenMathExpressionsSuite.scala | 12 ++++++++++--
.../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../catalyst/expressions/GlutenMathExpressionsSuite.scala | 12 ++++++++++--
.../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../catalyst/expressions/GlutenMathExpressionsSuite.scala | 12 ++++++++++--
.../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../catalyst/expressions/GlutenMathExpressionsSuite.scala | 12 ++++++++++--
.../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../catalyst/expressions/GlutenMathExpressionsSuite.scala | 12 ++++++++++--
10 files changed, 60 insertions(+), 10 deletions(-)
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 5203a075f9..944f312d75 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -741,6 +741,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("binary log")
.exclude("round/bround/floor/ceil")
.excludeGlutenTest("round/bround/floor/ceil")
+ // See https://github.com/apache/gluten/issues/7232
+ .excludeGlutenTest("unhex returns null for invalid hex input")
.exclude("SPARK-36922: Support ANSI intervals for SIGN/SIGNUM")
.exclude("SPARK-35926: Support YearMonthIntervalType in width-bucket
function")
.exclude("SPARK-35925: Support DayTimeIntervalType in width-bucket
function")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
index b793896de2..56f6befc69 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
@@ -284,14 +284,22 @@ class GlutenMathExpressionsSuite extends
MathExpressionsSuite with GlutenTestsTr
checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
-// checkEvaluation(Unhex(Literal("GG")), null)
checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
// scalastyle:off
// Turn off scala style for non-ascii chars
checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")),
"三重的".getBytes(StandardCharsets.UTF_8))
-// checkEvaluation(Unhex(Literal("三重的")), null)
// scalastyle:on
checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Unhex(e),
StringType)
}
+
+ // Split out of "unhex" so that ClickHouse, which does not return null for
invalid hex
+ // input (see https://github.com/apache/gluten/issues/7232), can exclude
only this case.
+ testGluten("unhex returns null for invalid hex input") {
+ checkEvaluation(Unhex(Literal("GG")), null)
+ // scalastyle:off
+ // Turn off scala style for non-ascii chars
+ checkEvaluation(Unhex(Literal("三重的")), null)
+ // scalastyle:on
+ }
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 64bf921217..788a5c82ef 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -1201,6 +1201,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeCH("atan2")
.excludeCH("SPARK-42045: integer overflow in round/bround")
.excludeCH("Gluten - round/bround/floor/ceil")
+ // See https://github.com/apache/gluten/issues/7232
+ .excludeGlutenTest("unhex returns null for invalid hex input")
enableSuite[GlutenMathFunctionsSuite]
enableSuite[GlutenMergedOrcReadSchemaSuite]
.includeCH("append column into middle")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
index 2e329264d8..3c5d3543bd 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
@@ -287,14 +287,22 @@ class GlutenMathExpressionsSuite extends
MathExpressionsSuite with GlutenTestsTr
checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
-// checkEvaluation(Unhex(Literal("GG")), null)
checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
// scalastyle:off
// Turn off scala style for non-ascii chars
checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")),
"三重的".getBytes(StandardCharsets.UTF_8))
-// checkEvaluation(Unhex(Literal("三重的")), null)
// scalastyle:on
checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Unhex(e),
StringType)
}
+
+ // Split out of "unhex" so that ClickHouse, which does not return null for
invalid hex
+ // input (see https://github.com/apache/gluten/issues/7232), can exclude
only this case.
+ testGluten("unhex returns null for invalid hex input") {
+ checkEvaluation(Unhex(Literal("GG")), null)
+ // scalastyle:off
+ // Turn off scala style for non-ascii chars
+ checkEvaluation(Unhex(Literal("三重的")), null)
+ // scalastyle:on
+ }
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index decdc1990d..b9afc2225c 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -1160,6 +1160,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeCH("atan2")
.excludeCH("SPARK-42045: integer overflow in round/bround")
.excludeCH("Gluten - round/bround/floor/ceil")
+ // See https://github.com/apache/gluten/issues/7232
+ .excludeGlutenTest("unhex returns null for invalid hex input")
enableSuite[GlutenMathFunctionsSuite]
enableSuite[GlutenMergedOrcReadSchemaSuite]
.includeCH("append column into middle")
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
index db1be819ad..e4631d9f4d 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
@@ -288,14 +288,22 @@ class GlutenMathExpressionsSuite extends
MathExpressionsSuite with GlutenExpress
checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
-// checkEvaluation(Unhex(Literal("GG")), null)
checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
// scalastyle:off
// Turn off scala style for non-ascii chars
checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")),
"三重的".getBytes(StandardCharsets.UTF_8))
-// checkEvaluation(Unhex(Literal("三重的")), null)
// scalastyle:on
checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Unhex(e),
StringType)
}
+
+ // Split out of "unhex" so that ClickHouse, which does not return null for
invalid hex
+ // input (see https://github.com/apache/gluten/issues/7232), can exclude
only this case.
+ testGluten("unhex returns null for invalid hex input") {
+ checkEvaluation(Unhex(Literal("GG")), null)
+ // scalastyle:off
+ // Turn off scala style for non-ascii chars
+ checkEvaluation(Unhex(Literal("三重的")), null)
+ // scalastyle:on
+ }
}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index decdc1990d..b9afc2225c 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -1160,6 +1160,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeCH("atan2")
.excludeCH("SPARK-42045: integer overflow in round/bround")
.excludeCH("Gluten - round/bround/floor/ceil")
+ // See https://github.com/apache/gluten/issues/7232
+ .excludeGlutenTest("unhex returns null for invalid hex input")
enableSuite[GlutenMathFunctionsSuite]
enableSuite[GlutenMergedOrcReadSchemaSuite]
.includeCH("append column into middle")
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
index 9d4d1fe848..4b73cbff73 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
@@ -292,14 +292,22 @@ class GlutenMathExpressionsSuite
checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
-// checkEvaluation(Unhex(Literal("GG")), null)
checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
// scalastyle:off
// Turn off scala style for non-ascii chars
checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")),
"三重的".getBytes(StandardCharsets.UTF_8))
-// checkEvaluation(Unhex(Literal("三重的")), null)
// scalastyle:on
checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Unhex(e),
StringType)
}
+
+ // Split out of "unhex" so that ClickHouse, which does not return null for
invalid hex
+ // input (see https://github.com/apache/gluten/issues/7232), can exclude
only this case.
+ testGluten("unhex returns null for invalid hex input") {
+ checkEvaluation(Unhex(Literal("GG")), null)
+ // scalastyle:off
+ // Turn off scala style for non-ascii chars
+ checkEvaluation(Unhex(Literal("三重的")), null)
+ // scalastyle:on
+ }
}
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index decdc1990d..b9afc2225c 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -1160,6 +1160,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeCH("atan2")
.excludeCH("SPARK-42045: integer overflow in round/bround")
.excludeCH("Gluten - round/bround/floor/ceil")
+ // See https://github.com/apache/gluten/issues/7232
+ .excludeGlutenTest("unhex returns null for invalid hex input")
enableSuite[GlutenMathFunctionsSuite]
enableSuite[GlutenMergedOrcReadSchemaSuite]
.includeCH("append column into middle")
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
index 9d4d1fe848..4b73cbff73 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
@@ -292,14 +292,22 @@ class GlutenMathExpressionsSuite
checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
-// checkEvaluation(Unhex(Literal("GG")), null)
checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
// scalastyle:off
// Turn off scala style for non-ascii chars
checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")),
"三重的".getBytes(StandardCharsets.UTF_8))
-// checkEvaluation(Unhex(Literal("三重的")), null)
// scalastyle:on
checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Unhex(e),
StringType)
}
+
+ // Split out of "unhex" so that ClickHouse, which does not return null for
invalid hex
+ // input (see https://github.com/apache/gluten/issues/7232), can exclude
only this case.
+ testGluten("unhex returns null for invalid hex input") {
+ checkEvaluation(Unhex(Literal("GG")), null)
+ // scalastyle:off
+ // Turn off scala style for non-ascii chars
+ checkEvaluation(Unhex(Literal("三重的")), null)
+ // scalastyle:on
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
