This is an automated email from the ASF dual-hosted git repository.
MaxGekk pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new b8458cc27309 [SPARK-57335][SQL] Fix element_at internal error on
Int.MinValue index
b8458cc27309 is described below
commit b8458cc27309ef8ee4e070a02e24428da8e480b1
Author: Eric Yang <[email protected]>
AuthorDate: Mon Jun 22 16:19:24 2026 +0200
[SPARK-57335][SQL] Fix element_at internal error on Int.MinValue index
### What changes were proposed in this pull request?
Widen the index to `Long` in `ElementAt`'s bounds check (interpreted eval,
codegen, and the `nullability` helper) and in
`ArrayExpressionUtils.resolveArrayIndex` (ANSI path), so an `Int.MinValue`
index is correctly detected as out of bounds.
### Why are the changes needed?
`element_at(arr, Int.MinValue)` leaks an internal
`IndexOutOfBoundsException` instead of returning `NULL` (non-ANSI) or raising
`INVALID_ARRAY_INDEX_IN_ELEMENT_AT` (ANSI), and `try_element_at` throws despite
its no-throw contract. - see the stack trace in
https://issues.apache.org/jira/browse/SPARK-57335
```SQL
SELECT element_at(array(1,2,3), -2147483648); -- expected NULL, but
throws internal IndexOutOfBoundsException
SELECT try_element_at(array(1,2,3), -2147483648); -- must never throw,
but actually throws an exception
```
**Root cause**: `Math.abs(Int.MinValue)` wraps back to a negative value,
bypassing the out-of-bounds guard.
### Does this PR introduce _any_ user-facing change?
Yes. element_at/try_element_at with an Int.MinValue index now return NULL
(non-ANSI) or raise the documented INVALID_ARRAY_INDEX_IN_ELEMENT_AT error
(ANSI) instead of leaking an internal exception.
### How was this patch tested?
New test case added
### Was this patch authored or co-authored using generative AI tooling?
Yes. Claude Code
Closes #56388 from jiwen624/spark-element-at-minvalue.
Authored-by: Eric Yang <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
(cherry picked from commit aca29287e5555b2bf485b2ffdec1af642fea1613)
Signed-off-by: Max Gekk <[email protected]>
---
.../catalyst/expressions/ArrayExpressionUtils.java | 4 +-
.../expressions/collectionOperations.scala | 22 +++++++----
.../expressions/CollectionExpressionsSuite.scala | 24 ++++++++++-
.../sql-tests/analyzer-results/array.sql.out | 15 +++++++
.../analyzer-results/nonansi/array.sql.out | 15 +++++++
.../nonansi/try_element_at.sql.out | 15 +++++++
.../analyzer-results/try_element_at.sql.out | 15 +++++++
.../src/test/resources/sql-tests/inputs/array.sql | 3 ++
.../resources/sql-tests/inputs/try_element_at.sql | 3 ++
.../test/resources/sql-tests/results/array.sql.out | 46 ++++++++++++++++++++++
.../sql-tests/results/nonansi/array.sql.out | 16 ++++++++
.../results/nonansi/try_element_at.sql.out | 16 ++++++++
.../sql-tests/results/try_element_at.sql.out | 16 ++++++++
13 files changed, 201 insertions(+), 9 deletions(-)
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ArrayExpressionUtils.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ArrayExpressionUtils.java
index 07a7f98ea1b8..8f5f360e3da3 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ArrayExpressionUtils.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ArrayExpressionUtils.java
@@ -39,7 +39,9 @@ public class ArrayExpressionUtils {
* @return the resolved 0-based position
*/
public static int resolveArrayIndex(int length, int index, QueryContext
context) {
- if (length < Math.abs(index)) {
+ // Widen the index to long before Math.abs to avoid overflow:
Math.abs(Int.MinValue)
+ // wraps back to a negative value and would bypass this out-of-bounds
guard.
+ if (length < Math.abs((long) index)) {
throw QueryExecutionErrors.invalidElementAtIndexError(index, length,
context);
}
if (index == 0) {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 09d438060378..ab56386869a0 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -2701,9 +2701,11 @@ case class ElementAt(
}
private def nullability(elements: Seq[Expression], ordinal: Int): Boolean = {
+ // Widen `ordinal` to Long before `abs` to avoid overflow:
`math.abs(Int.MinValue)`
+ // wraps back to a negative value and would bypass the out-of-bounds guard
below.
if (ordinal == 0) {
false
- } else if (elements.length < math.abs(ordinal)) {
+ } else if (elements.length < math.abs(ordinal.toLong)) {
!failOnError
} else {
if (ordinal < 0) {
@@ -2735,7 +2737,9 @@ case class ElementAt(
case _: ArrayType =>
(value, ordinal) => {
val array = value.asInstanceOf[ArrayData]
- val index = ordinal.asInstanceOf[Int]
+ // Widen the index to Long before `abs` to avoid overflow:
`math.abs(Int.MinValue)`
+ // wraps back to a negative value and would bypass this out-of-bounds
guard.
+ val index = ordinal.asInstanceOf[Int].toLong
if (array.numElements() < math.abs(index)) {
defaultValueOutOfBound match {
case Some(value) => value.eval()
@@ -2745,9 +2749,9 @@ case class ElementAt(
val idx = if (index == 0) {
throw
QueryExecutionErrors.invalidIndexOfZeroError(getContextOrNull())
} else if (index > 0) {
- index - 1
+ (index - 1).toInt
} else {
- array.numElements() + index
+ (array.numElements() + index).toInt
}
if (arrayElementNullable && array.isNullAt(idx)) {
null
@@ -2790,9 +2794,10 @@ case class ElementAt(
case _: ArrayType =>
nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
val index = ctx.freshName("elementAtIndex")
+ val intIndex = ctx.freshName("elementAtIntIndex")
val nullCheck = if (arrayElementNullable) {
s"""
- |if ($eval1.isNullAt($index)) {
+ |if ($eval1.isNullAt($intIndex)) {
| ${ev.isNull} = true;
|} else
""".stripMargin
@@ -2811,8 +2816,10 @@ case class ElementAt(
case None => s"${ev.isNull} = true;"
}
+ // Widen the index to long before Math.abs to avoid overflow:
Math.abs(Int.MinValue)
+ // wraps back to a negative value and would bypass this
out-of-bounds guard.
s"""
- |int $index = (int) $eval2;
+ |long $index = (long) $eval2;
|if ($eval1.numElements() < Math.abs($index)) {
| $indexOutOfBoundBranch
|} else {
@@ -2823,9 +2830,10 @@ case class ElementAt(
| } else {
| $index += $eval1.numElements();
| }
+ | int $intIndex = (int) $index;
| $nullCheck
| {
- | ${ev.value} = ${CodeGenerator.getValue(eval1, dataType,
index)};
+ | ${ev.value} = ${CodeGenerator.getValue(eval1, dataType,
intIndex)};
| }
|}
""".stripMargin
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 0cf269b8360e..37e0b53dd46d 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -24,7 +24,7 @@ import java.util.TimeZone
import scala.language.implicitConversions
import scala.util.Random
-import org.apache.spark.{SparkFunSuite, SparkRuntimeException}
+import org.apache.spark.{SparkArrayIndexOutOfBoundsException, SparkFunSuite,
SparkRuntimeException}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
@@ -3324,4 +3324,26 @@ class CollectionExpressionsSuite
"maxRoundedArrayLength" ->
ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH.toString,
"functionName" -> "`array_insert`"))
}
+
+ test("SPARK-57335: element_at with index = Int.MinValue") {
+ val a = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType))
+ // Non-ANSI: an out-of-bounds index must return null.
`Math.abs(Int.MinValue)`
+ // overflows back to a negative value, which previously bypassed the
bounds check
+ // and leaked an internal IndexOutOfBoundsException.
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ checkEvaluation(ElementAt(a, Literal(Int.MinValue)), null)
+ }
+ // ANSI: the documented error is raised instead of an internal exception.
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+ checkExceptionInExpression[SparkArrayIndexOutOfBoundsException](
+ ElementAt(a, Literal(Int.MinValue)),
+ "INVALID_ARRAY_INDEX_IN_ELEMENT_AT")
+ }
+ // try_element_at must not throw even under ANSI, since it hardcodes
+ // failOnError = false. Without the widened bounds check this leaked
+ // an internal IndexOutOfBoundsException.
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+ checkEvaluation(ElementAt(a, Literal(Int.MinValue), None, failOnError =
false), null)
+ }
+ }
}
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/array.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/array.sql.out
index af5b4f9b129e..ede479100dcb 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/array.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/array.sql.out
@@ -242,6 +242,21 @@ Project [element_at(array(1, 2, 3), 0, None, true) AS
element_at(array(1, 2, 3),
+- OneRowRelation
+-- !query
+select element_at(array(1, 2, 3), -2147483648)
+-- !query analysis
+Project [element_at(array(1, 2, 3), -2147483648, None, true) AS
element_at(array(1, 2, 3), -2147483648)#x]
++- OneRowRelation
+
+
+-- !query
+select element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx)
+-- !query analysis
+Project [element_at(array(1, 2, 3), idx#x, None, true) AS element_at(array(1,
2, 3), idx)#x]
++- SubqueryAlias t
+ +- LocalRelation [idx#x]
+
+
-- !query
select elt(4, '123', '456')
-- !query analysis
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/array.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/array.sql.out
index c60e2c3737b4..5f2ea9b475ea 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/array.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/array.sql.out
@@ -242,6 +242,21 @@ Project [element_at(array(1, 2, 3), 0, None, false) AS
element_at(array(1, 2, 3)
+- OneRowRelation
+-- !query
+select element_at(array(1, 2, 3), -2147483648)
+-- !query analysis
+Project [element_at(array(1, 2, 3), -2147483648, None, false) AS
element_at(array(1, 2, 3), -2147483648)#x]
++- OneRowRelation
+
+
+-- !query
+select element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx)
+-- !query analysis
+Project [element_at(array(1, 2, 3), idx#x, None, false) AS element_at(array(1,
2, 3), idx)#x]
++- SubqueryAlias t
+ +- LocalRelation [idx#x]
+
+
-- !query
select elt(4, '123', '456')
-- !query analysis
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/try_element_at.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/try_element_at.sql.out
index 2475e315884f..5b07ddf04fef 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/try_element_at.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/try_element_at.sql.out
@@ -41,6 +41,21 @@ Project [try_element_at(array(1, 2, 3), -4) AS
try_element_at(array(1, 2, 3), -4
+- OneRowRelation
+-- !query
+SELECT try_element_at(array(1, 2, 3), -2147483648)
+-- !query analysis
+Project [try_element_at(array(1, 2, 3), -2147483648) AS
try_element_at(array(1, 2, 3), -2147483648)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT try_element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx)
+-- !query analysis
+Project [try_element_at(array(1, 2, 3), idx#x) AS try_element_at(array(1, 2,
3), idx)#x]
++- SubqueryAlias t
+ +- LocalRelation [idx#x]
+
+
-- !query
SELECT try_element_at(map('a','b'), 'a')
-- !query analysis
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/try_element_at.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/try_element_at.sql.out
index 2475e315884f..5b07ddf04fef 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/try_element_at.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/try_element_at.sql.out
@@ -41,6 +41,21 @@ Project [try_element_at(array(1, 2, 3), -4) AS
try_element_at(array(1, 2, 3), -4
+- OneRowRelation
+-- !query
+SELECT try_element_at(array(1, 2, 3), -2147483648)
+-- !query analysis
+Project [try_element_at(array(1, 2, 3), -2147483648) AS
try_element_at(array(1, 2, 3), -2147483648)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT try_element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx)
+-- !query analysis
+Project [try_element_at(array(1, 2, 3), idx#x) AS try_element_at(array(1, 2,
3), idx)#x]
++- SubqueryAlias t
+ +- LocalRelation [idx#x]
+
+
-- !query
SELECT try_element_at(map('a','b'), 'a')
-- !query analysis
diff --git a/sql/core/src/test/resources/sql-tests/inputs/array.sql
b/sql/core/src/test/resources/sql-tests/inputs/array.sql
index e923c3cdc360..27fb3b3d0777 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/array.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/array.sql
@@ -95,6 +95,9 @@ from primitive_arrays;
select element_at(array(1, 2, 3), 5);
select element_at(array(1, 2, 3), -5);
select element_at(array(1, 2, 3), 0);
+select element_at(array(1, 2, 3), -2147483648);
+-- non-literal index: avoids constant folding so the codegen path is exercised
end-to-end
+select element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx);
select elt(4, '123', '456');
select elt(0, '123', '456');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/try_element_at.sql
b/sql/core/src/test/resources/sql-tests/inputs/try_element_at.sql
index c02c2dcb34fd..51bc92fdde11 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/try_element_at.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/try_element_at.sql
@@ -5,6 +5,9 @@ SELECT try_element_at(array(1, 2, 3), 3);
SELECT try_element_at(array(1, 2, 3), 4);
SELECT try_element_at(array(1, 2, 3), -1);
SELECT try_element_at(array(1, 2, 3), -4);
+SELECT try_element_at(array(1, 2, 3), -2147483648);
+-- non-literal index: avoids constant folding so the codegen path is exercised
end-to-end
+SELECT try_element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx);
-- map input
SELECT try_element_at(map('a','b'), 'a');
diff --git a/sql/core/src/test/resources/sql-tests/results/array.sql.out
b/sql/core/src/test/resources/sql-tests/results/array.sql.out
index 90a605734c1a..2113ff5ee491 100644
--- a/sql/core/src/test/resources/sql-tests/results/array.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/array.sql.out
@@ -240,6 +240,52 @@ org.apache.spark.SparkRuntimeException
}
+-- !query
+select element_at(array(1, 2, 3), -2147483648)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkArrayIndexOutOfBoundsException
+{
+ "errorClass" : "INVALID_ARRAY_INDEX_IN_ELEMENT_AT",
+ "sqlState" : "22003",
+ "messageParameters" : {
+ "arraySize" : "3",
+ "indexValue" : "-2147483648"
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 46,
+ "fragment" : "element_at(array(1, 2, 3), -2147483648)"
+ } ]
+}
+
+
+-- !query
+select element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkArrayIndexOutOfBoundsException
+{
+ "errorClass" : "INVALID_ARRAY_INDEX_IN_ELEMENT_AT",
+ "sqlState" : "22003",
+ "messageParameters" : {
+ "arraySize" : "3",
+ "indexValue" : "-2147483648"
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 38,
+ "fragment" : "element_at(array(1, 2, 3), idx)"
+ } ]
+}
+
+
-- !query
select elt(4, '123', '456')
-- !query schema
diff --git
a/sql/core/src/test/resources/sql-tests/results/nonansi/array.sql.out
b/sql/core/src/test/resources/sql-tests/results/nonansi/array.sql.out
index 460cb89113ab..3bd7671ae386 100644
--- a/sql/core/src/test/resources/sql-tests/results/nonansi/array.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/nonansi/array.sql.out
@@ -203,6 +203,22 @@ org.apache.spark.SparkRuntimeException
}
+-- !query
+select element_at(array(1, 2, 3), -2147483648)
+-- !query schema
+struct<element_at(array(1, 2, 3), -2147483648):int>
+-- !query output
+NULL
+
+
+-- !query
+select element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx)
+-- !query schema
+struct<element_at(array(1, 2, 3), idx):int>
+-- !query output
+NULL
+
+
-- !query
select elt(4, '123', '456')
-- !query schema
diff --git
a/sql/core/src/test/resources/sql-tests/results/nonansi/try_element_at.sql.out
b/sql/core/src/test/resources/sql-tests/results/nonansi/try_element_at.sql.out
index 0437f9d6dd9e..63ccc115e66c 100644
---
a/sql/core/src/test/resources/sql-tests/results/nonansi/try_element_at.sql.out
+++
b/sql/core/src/test/resources/sql-tests/results/nonansi/try_element_at.sql.out
@@ -51,6 +51,22 @@ struct<try_element_at(array(1, 2, 3), -4):int>
NULL
+-- !query
+SELECT try_element_at(array(1, 2, 3), -2147483648)
+-- !query schema
+struct<try_element_at(array(1, 2, 3), -2147483648):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT try_element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx)
+-- !query schema
+struct<try_element_at(array(1, 2, 3), idx):int>
+-- !query output
+NULL
+
+
-- !query
SELECT try_element_at(map('a','b'), 'a')
-- !query schema
diff --git
a/sql/core/src/test/resources/sql-tests/results/try_element_at.sql.out
b/sql/core/src/test/resources/sql-tests/results/try_element_at.sql.out
index 0437f9d6dd9e..63ccc115e66c 100644
--- a/sql/core/src/test/resources/sql-tests/results/try_element_at.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/try_element_at.sql.out
@@ -51,6 +51,22 @@ struct<try_element_at(array(1, 2, 3), -4):int>
NULL
+-- !query
+SELECT try_element_at(array(1, 2, 3), -2147483648)
+-- !query schema
+struct<try_element_at(array(1, 2, 3), -2147483648):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT try_element_at(array(1, 2, 3), idx) from values (-2147483648) as t(idx)
+-- !query schema
+struct<try_element_at(array(1, 2, 3), idx):int>
+-- !query output
+NULL
+
+
-- !query
SELECT try_element_at(map('a','b'), 'a')
-- !query schema
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
