This is an automated email from the ASF dual-hosted git repository.

cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new a027e552eb9b [SPARK-58431][SQL] Preserve ANSI array index errors 
during optimization
a027e552eb9b is described below

commit a027e552eb9b28b6cd1b48c4d4177d3722332ae1
Author: Marcus Lin <[email protected]>
AuthorDate: Fri Jul 31 22:00:04 2026 +0800

    [SPARK-58431][SQL] Preserve ANSI array index errors during optimization
    
    ### What changes were proposed in this pull request?
    This PR updates `SimplifyExtractValueOps` so it does not fold out-of-bounds 
`GetArrayItem(CreateArray(...), literalIndex)` expressions to `NULL` when ANSI 
error behavior is enabled.
    
    Valid literal indexes are still simplified to the selected element, and 
non-ANSI out-of-bounds access still folds to `NULL` as before.
    
    This PR also adds regression coverage for both optimizer behavior and the 
end-to-end SQL error path.
    
    ### Why are the changes needed?
    Under ANSI mode, out-of-bounds array access should raise 
`INVALID_ARRAY_INDEX`. The optimizer previously simplified 
`array(...)[out_of_bounds]` to a null literal even when the expression had 
`failOnError = true`, which skipped the runtime error and changed query 
semantics.
    
    ### Does this PR introduce _any_ user-facing change?
    Yes. This fixes a bug where ANSI out-of-bounds array access could 
incorrectly return `NULL` after optimization instead of raising 
`INVALID_ARRAY_INDEX`.
    
    For example, `select array(id, 2, 3)[5] from range(1)` now preserves the 
expected ANSI error behavior.
    
    ### How was this patch tested?
    Added regression tests and ran:
    
    `build/sbt 'sql/testOnly 
org.apache.spark.sql.errors.QueryExecutionAnsiErrorsSuite -- -z 
"INVALID_ARRAY_INDEX: get element from array"'`
    
    `build/sbt 'catalyst/testOnly 
org.apache.spark.sql.catalyst.optimizer.ComplexTypesSuite -- -z "SPARK-58431"'`
    
    Also ran `git diff --check`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Generated-by: Codex (OpenAI GPT-5) was used for code assistance and review 
support; the patch was guided by the contributor.
    
    Closes #57640 from marcuslin123/SPARK-58431-ansi-array-index.
    
    Authored-by: Marcus Lin <[email protected]>
    Signed-off-by: Wenchen Fan <[email protected]>
---
 .../spark/sql/catalyst/optimizer/ComplexTypes.scala    |  3 +++
 .../sql/catalyst/optimizer/complexTypesSuite.scala     | 18 ++++++++++++++++++
 .../sql/errors/QueryExecutionAnsiErrorsSuite.scala     |  8 ++++++++
 3 files changed, 29 insertions(+)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
index 5c1967c094ff..34c7aa442d4e 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
@@ -59,6 +59,9 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
         if (idx >= 0 && idx < elems.size) {
           // valid index
           elems(idx)
+        } else if (ga.failOnError) {
+          // Keep ANSI runtime behavior, which raises on out-of-bounds access.
+          ga
         } else {
           // out of bounds, mimic the runtime behavior and return null
           Literal(null, ga.dataType)
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
index 71acbdfdd2fc..9d4cf9dff953 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
@@ -409,6 +409,24 @@ class ComplexTypesSuite extends PlanTest with 
ExpressionEvalHelper {
     checkRule(mapRel, mapExpected)
   }
 
+  test("SPARK-58431: don't simplify ANSI out-of-bounds array access to null") {
+    val ansiQuery = relation
+      .select(
+        GetArrayItem(
+          CreateArray(Seq($"nullable_id", $"nullable_id" + 1L)),
+          5,
+          failOnError = true) as "a1")
+    val nonAnsiQuery = relation
+      .select(
+        GetArrayItem(
+          CreateArray(Seq($"nullable_id", $"nullable_id" + 1L)),
+          5,
+          failOnError = false) as "a1")
+
+    checkRule(ansiQuery, ansiQuery)
+    checkRule(nonAnsiQuery, relation.select(Literal.create(null, LongType) as 
"a1"))
+  }
+
   test("SPARK-23500: Ensure that aggregation expressions are not simplified") {
     // Make sure that aggregation exprs are correctly ignored. Maps can't be 
used in
     // grouping exprs so aren't tested here.
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
index d5185289e021..1ce04c8064ab 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
@@ -195,6 +195,14 @@ class QueryExecutionAnsiErrorsSuite extends 
SharedSparkSession {
       context = ExpectedContext(
         fragment = "apply",
         callSitePattern = getCurrentClassCallSitePattern))
+
+    checkError(
+      exception = intercept[SparkArrayIndexOutOfBoundsException] {
+        sql("select array(id, 2, 3)[5] from range(1)").collect()
+      },
+      condition = "INVALID_ARRAY_INDEX",
+      parameters = Map("indexValue" -> "5", "arraySize" -> "3"),
+      context = ExpectedContext(fragment = "array(id, 2, 3)[5]", start = 7, 
stop = 24))
   }
 
   test("INVALID_ARRAY_INDEX_IN_ELEMENT_AT: element_at from array") {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to