This is an automated email from the ASF dual-hosted git repository.
peter-toth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9ccb1ddab42a [SPARK-55749][SQL][DOC] Clarify `array_contains`
null-handling in the docs and example
9ccb1ddab42a is described below
commit 9ccb1ddab42aef78e6eb2d3b260fdedbb60e0002
Author: Oleks V <[email protected]>
AuthorDate: Tue Jul 28 09:34:56 2026 +0200
[SPARK-55749][SQL][DOC] Clarify `array_contains` null-handling in the docs
and example
### What changes were proposed in this pull request?
Clarify the `array_contains` documentation:
- `sql/api/.../functions.scala`: rewrote the Scaladoc summary to cover
all three return cases (true / false / null).
- `sql/catalyst/.../collectionOperations.scala`: added a SQL example
showing the null result when the value is absent but the array contains a null
element.
### Why are the changes needed?
The old docs said `array_contains` returns "false otherwise", omitting
the null-propagation case:
```
scala> spark.sql("select array_contains(array(1, null, 3),
2)").show(false)
+------------------------------------+
|array_contains(array(1, NULL, 3), 2)|
+------------------------------------+
|NULL |
+------------------------------------+
```
### Does this PR introduce _any_ user-facing change?
No. Documentation-only.
### How was this patch tested?
N/A — docs only.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #57413 from comphead/arr_contains_doc.
Lead-authored-by: Oleks V <[email protected]>
Co-authored-by: ovoievodin <[email protected]>
Signed-off-by: Peter Toth <[email protected]>
---
python/pyspark/sql/functions/builtin.py | 17 ++++++++++++++---
.../src/main/scala/org/apache/spark/sql/functions.scala | 3 ++-
.../sql/catalyst/expressions/collectionOperations.scala | 6 +++++-
3 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/python/pyspark/sql/functions/builtin.py
b/python/pyspark/sql/functions/builtin.py
index a9f3769b0831..319f1a2f2b02 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -19892,9 +19892,9 @@ def array(
@_try_remote_functions
def array_contains(col: "ColumnOrName", value: Any) -> Column:
"""
- Collection function: This function returns a boolean indicating whether
the array
- contains the given value, returning null if the array is null, true if the
array
- contains the given value, and false otherwise.
+ Collection function: Returns true if the array contains the value, false
if not. Returns
+ null if the array or value is null, or if the value is not found and the
array contains a
+ null element.
.. versionadded:: 1.5.0
@@ -19972,6 +19972,17 @@ def array_contains(col: "ColumnOrName", value: Any) ->
Column:
+-----------------------+
| true|
+-----------------------+
+
+ Example 5: Value absent from an array that contains a null element returns
NULL.
+
+ >>> from pyspark.sql import functions as sf
+ >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data'])
+ >>> df.select(sf.array_contains(df.data, "b")).show()
+ +-----------------------+
+ |array_contains(data, b)|
+ +-----------------------+
+ | NULL|
+ +-----------------------+
"""
return _invoke_function_over_columns("array_contains", col, lit(value))
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index 3961a2d9de1d..5f9e02ae9b05 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -12967,7 +12967,8 @@ object functions {
//////////////////////////////////////////////////////////////////////////////////////////////
/**
- * Returns null if the array is null, true if the array contains `value`,
and false otherwise.
+ * Returns true if the array contains `value`, false if not. Returns null if
the array or
+ * `value` is null, or if `value` is not found and the array contains a null
element.
* @param column
* the target column containing the arrays. A column that evaluates to an
array.
* @param value
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 2ea95d2c46f6..f9dabd38664a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -1483,8 +1483,9 @@ case class Reverse(child: Expression)
/**
* Checks if the array (left) has the element (right)
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(array, value) - Returns true if the array contains the
value.",
+ usage = "_FUNC_(array, value) - Returns true if the array contains the
value, false if not. Returns null if the array or value is null, or if the
value is not found and the array contains a null element.",
arguments = """
Arguments:
* array - The array to search.
@@ -1496,9 +1497,12 @@ case class Reverse(child: Expression)
Examples:
> SELECT _FUNC_(array(1, 2, 3), 2);
true
+ > SELECT _FUNC_(array(1, NULL, 3), 2);
+ NULL
""",
group = "array_funcs",
since = "1.5.0")
+// scalastyle:on line.size.limit
case class ArrayContains(left: Expression, right: Expression)
extends BinaryExpression with ImplicitCastInputTypes with Predicate
with QueryErrorsBase {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]