This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new c6201a2cd5b9 [SPARK-58431][SQL] Preserve ANSI array index errors
during optimization
c6201a2cd5b9 is described below
commit c6201a2cd5b953a3a36eac1b0763600286e64282
Author: Marcus Lin <[email protected]>
AuthorDate: Fri Jul 31 22:00:04 2026 +0800
[SPARK-58431][SQL] Preserve ANSI array index errors during optimization
### What changes were proposed in this pull request?
This PR updates `SimplifyExtractValueOps` so it does not fold out-of-bounds
`GetArrayItem(CreateArray(...), literalIndex)` expressions to `NULL` when ANSI
error behavior is enabled.
Valid literal indexes are still simplified to the selected element, and
non-ANSI out-of-bounds access still folds to `NULL` as before.
This PR also adds regression coverage for both optimizer behavior and the
end-to-end SQL error path.
### Why are the changes needed?
Under ANSI mode, out-of-bounds array access should raise
`INVALID_ARRAY_INDEX`. The optimizer previously simplified
`array(...)[out_of_bounds]` to a null literal even when the expression had
`failOnError = true`, which skipped the runtime error and changed query
semantics.
### Does this PR introduce _any_ user-facing change?
Yes. This fixes a bug where ANSI out-of-bounds array access could
incorrectly return `NULL` after optimization instead of raising
`INVALID_ARRAY_INDEX`.
For example, `select array(id, 2, 3)[5] from range(1)` now preserves the
expected ANSI error behavior.
### How was this patch tested?
Added regression tests and ran:
`build/sbt 'sql/testOnly
org.apache.spark.sql.errors.QueryExecutionAnsiErrorsSuite -- -z
"INVALID_ARRAY_INDEX: get element from array"'`
`build/sbt 'catalyst/testOnly
org.apache.spark.sql.catalyst.optimizer.ComplexTypesSuite -- -z "SPARK-58431"'`
Also ran `git diff --check`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (OpenAI GPT-5) was used for code assistance and review
support; the patch was guided by the contributor.
Closes #57640 from marcuslin123/SPARK-58431-ansi-array-index.
Authored-by: Marcus Lin <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit a027e552eb9b28b6cd1b48c4d4177d3722332ae1)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../spark/sql/catalyst/optimizer/ComplexTypes.scala | 3 +++
.../sql/catalyst/optimizer/complexTypesSuite.scala | 18 ++++++++++++++++++
.../sql/errors/QueryExecutionAnsiErrorsSuite.scala | 8 ++++++++
3 files changed, 29 insertions(+)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
index 5c1967c094ff..34c7aa442d4e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
@@ -59,6 +59,9 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
if (idx >= 0 && idx < elems.size) {
// valid index
elems(idx)
+ } else if (ga.failOnError) {
+ // Keep ANSI runtime behavior, which raises on out-of-bounds access.
+ ga
} else {
// out of bounds, mimic the runtime behavior and return null
Literal(null, ga.dataType)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
index 71acbdfdd2fc..9d4cf9dff953 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
@@ -409,6 +409,24 @@ class ComplexTypesSuite extends PlanTest with
ExpressionEvalHelper {
checkRule(mapRel, mapExpected)
}
+ test("SPARK-58431: don't simplify ANSI out-of-bounds array access to null") {
+ val ansiQuery = relation
+ .select(
+ GetArrayItem(
+ CreateArray(Seq($"nullable_id", $"nullable_id" + 1L)),
+ 5,
+ failOnError = true) as "a1")
+ val nonAnsiQuery = relation
+ .select(
+ GetArrayItem(
+ CreateArray(Seq($"nullable_id", $"nullable_id" + 1L)),
+ 5,
+ failOnError = false) as "a1")
+
+ checkRule(ansiQuery, ansiQuery)
+ checkRule(nonAnsiQuery, relation.select(Literal.create(null, LongType) as
"a1"))
+ }
+
test("SPARK-23500: Ensure that aggregation expressions are not simplified") {
// Make sure that aggregation exprs are correctly ignored. Maps can't be
used in
// grouping exprs so aren't tested here.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
index d5185289e021..1ce04c8064ab 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
@@ -195,6 +195,14 @@ class QueryExecutionAnsiErrorsSuite extends
SharedSparkSession {
context = ExpectedContext(
fragment = "apply",
callSitePattern = getCurrentClassCallSitePattern))
+
+ checkError(
+ exception = intercept[SparkArrayIndexOutOfBoundsException] {
+ sql("select array(id, 2, 3)[5] from range(1)").collect()
+ },
+ condition = "INVALID_ARRAY_INDEX",
+ parameters = Map("indexValue" -> "5", "arraySize" -> "3"),
+ context = ExpectedContext(fragment = "array(id, 2, 3)[5]", start = 7,
stop = 24))
}
test("INVALID_ARRAY_INDEX_IN_ELEMENT_AT: element_at from array") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]