This is an automated email from the ASF dual-hosted git repository.

peter-toth pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 14c5af36a641 [SPARK-55749][SQL][DOC] Clarify `array_contains` 
null-handling in the docs and example
14c5af36a641 is described below

commit 14c5af36a641fa2dd388daca10e9fd4f99001eab
Author: Oleks V <[email protected]>
AuthorDate: Tue Jul 28 09:34:56 2026 +0200

    [SPARK-55749][SQL][DOC] Clarify `array_contains` null-handling in the docs 
and example
    
    ### What changes were proposed in this pull request?
    
    Clarify the `array_contains` documentation:
    
      - `sql/api/.../functions.scala`: rewrote the Scaladoc summary to cover 
all three return cases (true / false / null).
      - `sql/catalyst/.../collectionOperations.scala`: added a SQL example 
showing the null result when the value is absent but the array contains a null 
element.
    
    ### Why are the changes needed?
    
      The old docs said `array_contains` returns "false otherwise", omitting 
the null-propagation case:
    
      ```
      scala> spark.sql("select array_contains(array(1, null, 3), 
2)").show(false)
      +------------------------------------+
      |array_contains(array(1, NULL, 3), 2)|
      +------------------------------------+
      |NULL                                |
      +------------------------------------+
      ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. Documentation-only.
    
    ### How was this patch tested?
    
    N/A — docs only.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No
    
    Closes #57413 from comphead/arr_contains_doc.
    
    Lead-authored-by: Oleks V <[email protected]>
    Co-authored-by: ovoievodin <[email protected]>
    Signed-off-by: Peter Toth <[email protected]>
    (cherry picked from commit 9ccb1ddab42aef78e6eb2d3b260fdedbb60e0002)
    Signed-off-by: Peter Toth <[email protected]>
---
 python/pyspark/sql/functions/builtin.py                 | 17 ++++++++++++++---
 .../src/main/scala/org/apache/spark/sql/functions.scala |  3 ++-
 .../sql/catalyst/expressions/collectionOperations.scala |  6 +++++-
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/python/pyspark/sql/functions/builtin.py 
b/python/pyspark/sql/functions/builtin.py
index 5d6a23b0b486..671887dd9d7e 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -19892,9 +19892,9 @@ def array(
 @_try_remote_functions
 def array_contains(col: "ColumnOrName", value: Any) -> Column:
     """
-    Collection function: This function returns a boolean indicating whether 
the array
-    contains the given value, returning null if the array is null, true if the 
array
-    contains the given value, and false otherwise.
+    Collection function: Returns true if the array contains the value, false 
if not. Returns
+    null if the array or value is null, or if the value is not found and the 
array contains a
+    null element.
 
     .. versionadded:: 1.5.0
 
@@ -19972,6 +19972,17 @@ def array_contains(col: "ColumnOrName", value: Any) -> 
Column:
     +-----------------------+
     |                   true|
     +-----------------------+
+
+    Example 5: Value absent from an array that contains a null element returns 
NULL.
+
+    >>> from pyspark.sql import functions as sf
+    >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data'])
+    >>> df.select(sf.array_contains(df.data, "b")).show()
+    +-----------------------+
+    |array_contains(data, b)|
+    +-----------------------+
+    |                   NULL|
+    +-----------------------+
     """
     return _invoke_function_over_columns("array_contains", col, lit(value))
 
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala 
b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index 3d75d49a1785..85b13c691919 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -12967,7 +12967,8 @@ object functions {
   
//////////////////////////////////////////////////////////////////////////////////////////////
 
   /**
-   * Returns null if the array is null, true if the array contains `value`, 
and false otherwise.
+   * Returns true if the array contains `value`, false if not. Returns null if 
the array or
+   * `value` is null, or if `value` is not found and the array contains a null 
element.
    * @param column
    *   the target column containing the arrays. A column that evaluates to an 
array.
    * @param value
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 2ea95d2c46f6..f9dabd38664a 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -1483,8 +1483,9 @@ case class Reverse(child: Expression)
 /**
  * Checks if the array (left) has the element (right)
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(array, value) - Returns true if the array contains the 
value.",
+  usage = "_FUNC_(array, value) - Returns true if the array contains the 
value, false if not. Returns null if the array or value is null, or if the 
value is not found and the array contains a null element.",
   arguments = """
     Arguments:
       * array - The array to search.
@@ -1496,9 +1497,12 @@ case class Reverse(child: Expression)
     Examples:
       > SELECT _FUNC_(array(1, 2, 3), 2);
        true
+      > SELECT _FUNC_(array(1, NULL, 3), 2);
+       NULL
   """,
   group = "array_funcs",
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class ArrayContains(left: Expression, right: Expression)
   extends BinaryExpression with ImplicitCastInputTypes with Predicate
   with QueryErrorsBase {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to