This is an automated email from the ASF dual-hosted git repository.
peter-toth pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 14c5af36a641 [SPARK-55749][SQL][DOC] Clarify `array_contains`
null-handling in the docs and example
14c5af36a641 is described below
commit 14c5af36a641fa2dd388daca10e9fd4f99001eab
Author: Oleks V <[email protected]>
AuthorDate: Tue Jul 28 09:34:56 2026 +0200
[SPARK-55749][SQL][DOC] Clarify `array_contains` null-handling in the docs
and example
### What changes were proposed in this pull request?
Clarify the `array_contains` documentation:
- `sql/api/.../functions.scala`: rewrote the Scaladoc summary to cover
all three return cases (true / false / null).
- `sql/catalyst/.../collectionOperations.scala`: added a SQL example
showing the null result when the value is absent but the array contains a null
element.
### Why are the changes needed?
The old docs said `array_contains` returns "false otherwise", omitting
the null-propagation case:
```
scala> spark.sql("select array_contains(array(1, null, 3),
2)").show(false)
+------------------------------------+
|array_contains(array(1, NULL, 3), 2)|
+------------------------------------+
|NULL |
+------------------------------------+
```
### Does this PR introduce _any_ user-facing change?
No. Documentation-only.
### How was this patch tested?
N/A — docs only.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #57413 from comphead/arr_contains_doc.
Lead-authored-by: Oleks V <[email protected]>
Co-authored-by: ovoievodin <[email protected]>
Signed-off-by: Peter Toth <[email protected]>
(cherry picked from commit 9ccb1ddab42aef78e6eb2d3b260fdedbb60e0002)
Signed-off-by: Peter Toth <[email protected]>
---
python/pyspark/sql/functions/builtin.py | 17 ++++++++++++++---
.../src/main/scala/org/apache/spark/sql/functions.scala | 3 ++-
.../sql/catalyst/expressions/collectionOperations.scala | 6 +++++-
3 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/python/pyspark/sql/functions/builtin.py
b/python/pyspark/sql/functions/builtin.py
index 5d6a23b0b486..671887dd9d7e 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -19892,9 +19892,9 @@ def array(
@_try_remote_functions
def array_contains(col: "ColumnOrName", value: Any) -> Column:
"""
- Collection function: This function returns a boolean indicating whether
the array
- contains the given value, returning null if the array is null, true if the
array
- contains the given value, and false otherwise.
+ Collection function: Returns true if the array contains the value, false
if not. Returns
+ null if the array or value is null, or if the value is not found and the
array contains a
+ null element.
.. versionadded:: 1.5.0
@@ -19972,6 +19972,17 @@ def array_contains(col: "ColumnOrName", value: Any) ->
Column:
+-----------------------+
| true|
+-----------------------+
+
+ Example 5: Value absent from an array that contains a null element returns
NULL.
+
+ >>> from pyspark.sql import functions as sf
+ >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data'])
+ >>> df.select(sf.array_contains(df.data, "b")).show()
+ +-----------------------+
+ |array_contains(data, b)|
+ +-----------------------+
+ | NULL|
+ +-----------------------+
"""
return _invoke_function_over_columns("array_contains", col, lit(value))
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index 3d75d49a1785..85b13c691919 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -12967,7 +12967,8 @@ object functions {
//////////////////////////////////////////////////////////////////////////////////////////////
/**
- * Returns null if the array is null, true if the array contains `value`,
and false otherwise.
+ * Returns true if the array contains `value`, false if not. Returns null if
the array or
+ * `value` is null, or if `value` is not found and the array contains a null
element.
* @param column
* the target column containing the arrays. A column that evaluates to an
array.
* @param value
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 2ea95d2c46f6..f9dabd38664a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -1483,8 +1483,9 @@ case class Reverse(child: Expression)
/**
* Checks if the array (left) has the element (right)
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(array, value) - Returns true if the array contains the
value.",
+ usage = "_FUNC_(array, value) - Returns true if the array contains the
value, false if not. Returns null if the array or value is null, or if the
value is not found and the array contains a null element.",
arguments = """
Arguments:
* array - The array to search.
@@ -1496,9 +1497,12 @@ case class Reverse(child: Expression)
Examples:
> SELECT _FUNC_(array(1, 2, 3), 2);
true
+ > SELECT _FUNC_(array(1, NULL, 3), 2);
+ NULL
""",
group = "array_funcs",
since = "1.5.0")
+// scalastyle:on line.size.limit
case class ArrayContains(left: Expression, right: Expression)
extends BinaryExpression with ImplicitCastInputTypes with Predicate
with QueryErrorsBase {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]