This is an automated email from the ASF dual-hosted git repository.

cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new d5ec6583c9ce [SPARK-58399][SQL][PYTHON] Add `collect_union` aggregate 
function
d5ec6583c9ce is described below

commit d5ec6583c9ce7d9bde1a2f9948ff4a563cd79f51
Author: ChuckLin2025 <[email protected]>
AuthorDate: Thu Jul 30 19:49:52 2026 +0800

    [SPARK-58399][SQL][PYTHON] Add `collect_union` aggregate function
    
    ### What changes were proposed in this pull request?
    
    This PR adds a new aggregate function `collect_union` that takes an
    array-typed column and returns the distinct union of the elements of the
    arrays across rows.
    
    `collect_union(col: array<T>) : array<T>`
    
    It is equivalent to `array_distinct(flatten(collect_list(col)))`, but the
    aggregation buffer holds only the distinct elements (a `HashSet`), so its
    size is bounded by the element universe rather than by the number of input
    rows. This avoids buffering every row's whole array, which for a hot
    grouping key can grow without bound.
    
    The function is implemented as a `Collect[mutable.HashSet[Any]]`
    (sibling of `collect_set`); the only material difference is that `update`
    iterates the input array and adds each non-null element, and the result
    element type is the input array's element type. NULL input arrays and NULL
    elements are skipped, following `collect_set` semantics.
    
    Added across the usual surfaces: Catalyst expression + registry, the Scala
    DataFrame API, and PySpark (classic + Spark Connect). Spark Connect needs
    no protocol change: the function travels as a generic `UnresolvedFunction`
    resolved against the registry.
    
    ### Why are the changes needed?
    
    There is no built-in aggregate that unions the elements of an array column
    across rows into a single distinct array. The workaround
    `array_distinct(flatten(collect_list(arr)))` buffers every row's whole
    array before de-duplicating, which can OOM on skewed keys. `collect_union`
    de-duplicates during aggregation, keeping the buffer bounded by the
    distinct-element universe.
    
    Note that `collect_set` cannot replace this. `collect_set(element)` over
    `explode(col)` does bound the buffer, but it stops being a plain aggregate:
    each array column must be exploded and grouped on its own and then joined
    back on the grouping keys. A query needing the distinct union of N array
    columns therefore pays N explodes + N joins purely to work around the
    missing array-input aggregate. `collect_union` keeps the bounded buffer
    while staying an ordinary aggregate, so multiple array columns aggregate
    together in one GROUP BY with no join.
    
    Industry precedent: BigQuery (GoogleSQL) already supports this style of
    array-input aggregate (`ARRAY_CONCAT_AGG`, which concatenates arrays across
    rows; distinct is then applied), whereas PostgreSQL has no dedicated
    function and users fall back to `array_agg(DISTINCT ...)` over `unnest(...)`
    (the analogue of the explode + `collect_set` workaround above). 
`collect_union`
    gives Spark a first-class, bounded-buffer form of this operation.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. It adds a new SQL function `collect_union` and the corresponding
    `functions.collect_union` in the Scala and Python DataFrame APIs.
    
    ### How was this patch tested?
    
    - New `collect_union function` case in `DataFrameAggregateSuite` (distinct
      union, NULL array, NULL element, per-group, empty result). Full suite:
      170 tests, all pass.
    - New `test_collect_union` in `python/pyspark/sql/tests/test_functions.py`
      covering `array<int>`, `array<string>`, `array<double>`, NULL elements,
      and `array<struct>` (passes end-to-end through the PySpark runtime).
    - Spark Connect parity check in `test_connect_function.py`.
    - `ExpressionsSchemaSuite` regenerated `sql-expression-schema.md`.
    
    Closes #57592 from ChuckLin2025/collect_union-oss.
    
    Lead-authored-by: ChuckLin2025 <[email protected]>
    Co-authored-by: Zequn Lin <[email protected]>
    Signed-off-by: Wenchen Fan <[email protected]>
---
 python/pyspark/sql/connect/functions/builtin.py    |   7 +
 python/pyspark/sql/functions/__init__.py           |   1 +
 python/pyspark/sql/functions/builtin.py            |  68 +++++++++
 .../sql/tests/connect/test_connect_function.py     |   7 +
 python/pyspark/sql/tests/test_functions.py         |  50 ++++++
 .../scala/org/apache/spark/sql/functions.scala     |  41 +++++
 .../sql/catalyst/analysis/FunctionRegistry.scala   |   1 +
 .../sql/catalyst/analysis/FunctionResolution.scala |   1 +
 .../catalyst/expressions/aggregate/collect.scala   | 169 +++++++++++++++++++++
 .../sql-functions/sql-expression-schema.md         |   1 +
 .../apache/spark/sql/DataFrameAggregateSuite.scala |  66 ++++++++
 11 files changed, 412 insertions(+)

diff --git a/python/pyspark/sql/connect/functions/builtin.py 
b/python/pyspark/sql/connect/functions/builtin.py
index f3e1a0c058f2..e88d584ea34f 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -1082,6 +1082,13 @@ def collect_set(col: "ColumnOrName") -> Column:
 collect_set.__doc__ = pysparkfuncs.collect_set.__doc__
 
 
+def collect_union(col: "ColumnOrName") -> Column:
+    return _invoke_function_over_columns("collect_union", col)
+
+
+collect_union.__doc__ = pysparkfuncs.collect_union.__doc__
+
+
 def listagg(col: "ColumnOrName", delimiter: Optional[Union[Column, str, 
bytes]] = None) -> Column:
     if delimiter is None:
         return _invoke_function_over_columns("listagg", col)
diff --git a/python/pyspark/sql/functions/__init__.py 
b/python/pyspark/sql/functions/__init__.py
index 50f0fd440dd9..d4762a56da9b 100644
--- a/python/pyspark/sql/functions/__init__.py
+++ b/python/pyspark/sql/functions/__init__.py
@@ -367,6 +367,7 @@ __all__ = [  # noqa: F405
     "bool_or",
     "collect_list",
     "collect_set",
+    "collect_union",
     "corr",
     "count",
     "count_distinct",
diff --git a/python/pyspark/sql/functions/builtin.py 
b/python/pyspark/sql/functions/builtin.py
index 319f1a2f2b02..ebc8a4447876 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -6284,6 +6284,74 @@ def collect_set(col: "ColumnOrName") -> Column:
     return _invoke_function_over_columns("collect_set", col)
 
 
+@_try_remote_functions
+def collect_union(col: "ColumnOrName") -> Column:
+    """
+    Aggregate function: given an array-typed column, collects the distinct 
union of the
+    elements of the arrays across rows and returns it as an array.
+
+    The aggregation buffer holds only the distinct elements, so its size is 
bounded by the
+    element universe rather than by the number of input rows. Null elements 
are dropped by
+    default (``IGNORE NULLS``), matching :func:`collect_set`. With ``RESPECT 
NULLS`` a single
+    null element is kept, in which case this is equivalent to
+    ``array_distinct(flatten(collect_list(col)))``. The ``RESPECT NULLS`` 
clause is only
+    available through SQL, e.g. ``expr("collect_union(col) RESPECT NULLS")``.
+
+    .. versionadded:: 4.3.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or column name
+        The target array column on which the function is computed.
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        A new Column object representing the distinct union of the array 
elements.
+
+    See Also
+    --------
+    :meth:`pyspark.sql.functions.collect_set`
+    :meth:`pyspark.sql.functions.collect_list`
+    :meth:`pyspark.sql.functions.array_distinct`
+    :meth:`pyspark.sql.functions.flatten`
+
+    Notes
+    -----
+    This function is non-deterministic as the order of collected results 
depends
+    on the order of the rows, which may be non-deterministic after any shuffle 
operations.
+
+    Examples
+    --------
+    Example 1: Union the elements of array columns across rows
+
+    >>> from pyspark.sql import functions as sf
+    >>> df = spark.createDataFrame(
+    ...     [([1, 2],), ([2, 3],), ([1],)], ('value',))
+    >>> df.select(sf.sort_array(sf.collect_union('value')).alias('u')).show()
+    +---------+
+    |        u|
+    +---------+
+    |[1, 2, 3]|
+    +---------+
+
+    Example 2: Union per group
+
+    >>> from pyspark.sql import functions as sf
+    >>> df = spark.createDataFrame(
+    ...     [("a", [1, 2]), ("a", [2, 3]), ("b", [4])], ("k", "value"))
+    >>> df = 
df.groupBy("k").agg(sf.sort_array(sf.collect_union('value')).alias('u'))
+    >>> df.orderBy("k").show()
+    +---+---------+
+    |  k|        u|
+    +---+---------+
+    |  a|[1, 2, 3]|
+    |  b|      [4]|
+    +---+---------+
+    """
+    return _invoke_function_over_columns("collect_union", col)
+
+
 @_try_remote_functions
 def degrees(col: "ColumnOrName") -> Column:
     """
diff --git a/python/pyspark/sql/tests/connect/test_connect_function.py 
b/python/pyspark/sql/tests/connect/test_connect_function.py
index 5c19cb895c7c..a452ee851eb7 100644
--- a/python/pyspark/sql/tests/connect/test_connect_function.py
+++ b/python/pyspark/sql/tests/connect/test_connect_function.py
@@ -615,6 +615,13 @@ class SparkConnectFunctionTests(ReusedMixedTestCase, 
PandasOnSparkTestUtils):
                 check_exact=False,
             )
 
+        # collect_union takes an array-typed column; build one via array(b, c).
+        self.assert_eq(
+            cdf.groupBy("a").agg(CF.sort_array(CF.collect_union(CF.array("b", 
"c")))).toPandas(),
+            sdf.groupBy("a").agg(SF.sort_array(SF.collect_union(SF.array("b", 
"c")))).toPandas(),
+            check_exact=False,
+        )
+
         for cfunc, sfunc in [
             (CF.corr, SF.corr),
             (CF.covar_pop, SF.covar_pop),
diff --git a/python/pyspark/sql/tests/test_functions.py 
b/python/pyspark/sql/tests/test_functions.py
index d3a4072c707e..41764083c3c8 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -2148,6 +2148,56 @@ class FunctionsTestsMixin:
             ["1", "2", "2", "2"],
         )
 
+    def test_collect_union(self):
+        # array<int>: distinct union across rows; NULL arrays ignored.
+        df = self.spark.createDataFrame([([1, 2],), ([2, 3],), ([1],), 
(None,)], ["value"])
+        self.assertEqual(
+            
sorted(df.select(F.collect_union(df.value).alias("r")).collect()[0].r),
+            [1, 2, 3],
+        )
+
+        # array<string>
+        sdf = self.spark.createDataFrame([(["a", "b"],), (["b", "c"],), 
(["a"],)], ["value"])
+        self.assertEqual(
+            
sorted(sdf.select(F.collect_union("value").alias("r")).collect()[0].r),
+            ["a", "b", "c"],
+        )
+
+        # array<double> (buffer keyed by bit pattern; values round-trip)
+        ddf = self.spark.createDataFrame([([1.5, 2.5],), ([2.5, 3.5],)], 
["value"])
+        self.assertEqual(
+            
sorted(ddf.select(F.collect_union("value").alias("r")).collect()[0].r),
+            [1.5, 2.5, 3.5],
+        )
+
+        # NULL elements inside a non-null array are dropped by default (IGNORE 
NULLS) ...
+        ndf = self.spark.createDataFrame([([1, None],), ([2],)], "value: 
array<int>")
+        self.assertEqual(
+            
sorted(ndf.select(F.collect_union("value").alias("r")).collect()[0].r),
+            [1, 2],
+        )
+        # ... and kept (a single null) with RESPECT NULLS (SQL clause via 
expr).
+        respect = ndf.select(F.expr("collect_union(value) RESPECT 
NULLS").alias("r")).collect()[0].r
+        self.assertEqual(sorted(respect, key=lambda x: (x is not None, x)), 
[None, 1, 2])
+
+        # array<struct>: the motivating case (dedups whole structs, stays 
element-wise).
+        struct_data = [
+            ([{"id": 1, "flag": True}, {"id": 2, "flag": False}],),
+            ([{"id": 2, "flag": False}, {"id": 3, "flag": True}],),
+        ]
+        stdf = self.spark.createDataFrame(struct_data, "value: 
array<struct<id:int,flag:boolean>>")
+        struct_rows = 
stdf.select(F.collect_union("value").alias("r")).collect()[0].r
+        self.assertEqual(
+            sorted((row.id, row.flag) for row in struct_rows),
+            [(1, True), (2, False), (3, True)],
+        )
+
+        # Per-group union.
+        gdf = self.spark.createDataFrame([("a", [1, 2]), ("a", [2, 3]), ("b", 
[4])], ["k", "value"])
+        rows = 
gdf.groupBy("k").agg(F.collect_union("value").alias("r")).orderBy("k").collect()
+        self.assertEqual(sorted(rows[0].r), [1, 2, 3])
+        self.assertEqual(sorted(rows[1].r), [4])
+
     def test_listagg_functions(self):
         df = self.spark.createDataFrame(
             [(1, "1"), (2, "2"), (None, None), (1, "2")], ["key", "value"]
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala 
b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index 5f9e02ae9b05..183c5379355d 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -432,6 +432,47 @@ object functions {
    */
   def collect_set(columnName: String): Column = collect_set(Column(columnName))
 
+  /**
+   * Aggregate function: returns the distinct union of the elements of an 
array-typed column
+   * across rows.
+   *
+   * The aggregation buffer holds only the distinct elements, so its size is 
bounded by the
+   * element universe rather than by the number of input rows. Null elements 
are dropped by
+   * default (IGNORE NULLS), matching `collect_set`. With `RESPECT NULLS`, a 
single null element
+   * is kept, in which case this is equivalent to 
`array_distinct(flatten(collect_list(e)))`. The
+   * `RESPECT NULLS` clause is only available through SQL (e.g.
+   * `expr("collect_union(col) RESPECT NULLS")`).
+   *
+   * @param e
+   *   The array column to collect the union of. A column of type array.
+   * @note
+   *   The function is non-deterministic because the order of collected 
results depends on the
+   *   order of the rows which may be non-deterministic after a shuffle.
+   *
+   * @group agg_funcs
+   * @since 4.3.0
+   * @return
+   *   Returns a column that evaluates to an array.
+   */
+  def collect_union(e: Column): Column = Column.fn("collect_union", e)
+
+  /**
+   * Aggregate function: returns the distinct union of the elements of an 
array-typed column
+   * across rows.
+   *
+   * @param columnName
+   *   The name of the array column to collect the union of. A column of type 
array.
+   * @note
+   *   The function is non-deterministic because the order of collected 
results depends on the
+   *   order of the rows which may be non-deterministic after a shuffle.
+   *
+   * @group agg_funcs
+   * @since 4.3.0
+   * @return
+   *   Returns a column that evaluates to an array.
+   */
+  def collect_union(columnName: String): Column = 
collect_union(Column(columnName))
+
   /**
    * Returns a count-min sketch of a column with the given esp, confidence and 
seed. The result is
    * an array of bytes, which can be deserialized to a `CountMinSketch` before 
usage. Count-min
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 965330db5828..862324a83c31 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -551,6 +551,7 @@ object FunctionRegistry {
     expression[CollectList]("collect_list"),
     expression[CollectList]("array_agg", true, Some("3.3.0")),
     expression[CollectSet]("collect_set"),
+    expression[CollectUnion]("collect_union"),
     expression[ListAgg]("listagg"),
     expression[ListAgg]("string_agg", setAlias = true),
     expressionBuilder("count_min_sketch", CountMinSketchAggExpressionBuilder),
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala
index 543e9da670a6..40cf4471c12a 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala
@@ -622,6 +622,7 @@ class FunctionResolution(
       case anyValue: AnyValue => anyValue.copy(ignoreNulls = ignoreNulls)
       case collectList: CollectList => collectList.copy(ignoreNulls = 
ignoreNulls)
       case collectSet: CollectSet => collectSet.copy(ignoreNulls = ignoreNulls)
+      case collectUnion: CollectUnion => collectUnion.copy(ignoreNulls = 
ignoreNulls)
       case _ if ignoreNulls =>
         // Only fail for IGNORE NULLS; RESPECT NULLS is the default behavior
         throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
index bdc5248ffa73..c3385bc39026 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
@@ -316,6 +316,175 @@ case class CollectSet(
     copy(child = newChild)
 }
 
+/**
+ * Collect the distinct union of the elements of an array-typed input across 
rows.
+ *
+ * Unlike collect_set, whose input is a scalar and whose output is the set of 
those scalars,
+ * collect_union's input is itself an array and its output is the set of the 
array's
+ * *elements* unioned across all rows. The aggregation buffer holds only the 
distinct
+ * elements (a set), so its size is bounded by the element universe rather 
than by the
+ * number of input rows.
+ *
+ * Null handling mirrors collect_set: by default (IGNORE NULLS) null elements 
are dropped.
+ * With RESPECT NULLS, one null element is kept, in which case collect_union 
is equivalent to
+ * `array_distinct(flatten(collect_list(arr)))`.
+ *
+ * @param ignoreNulls when true (IGNORE NULLS, the default), null elements are 
excluded from
+ *                    the result array. When false (RESPECT NULLS), a single 
null element is
+ *                    kept.
+ */
+@ExpressionDescription(
+  usage =
+    "_FUNC_(expr) - Collects and returns the distinct union of the elements of 
array `expr`.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(col) FROM VALUES (array(1, 2)), (array(2, 3)), 
(array(1)) AS tab(col);
+       [1,2,3]
+  """,
+  note = """
+    The function is non-deterministic because the order of collected results 
depends
+    on the order of the rows which may be non-deterministic after a shuffle.
+  """,
+  group = "agg_funcs",
+  since = "4.3.0")
+case class CollectUnion(
+    child: Expression,
+    mutableAggBufferOffset: Int = 0,
+    inputAggBufferOffset: Int = 0,
+    ignoreNulls: Boolean = true)
+  extends Collect[mutable.HashSet[Any]] with QueryErrorsBase with 
UnaryLike[Expression] {
+
+  def this(child: Expression) = this(child, 0, 0, true)
+
+  // The input is guarded by checkInputDataTypes to be an ArrayType; this is 
its element type.
+  private lazy val elementType: DataType = child.dataType match {
+    case ArrayType(et, _) => et
+    case other => other
+  }
+
+  // The result array contains a null only when null elements are respected 
(RESPECT NULLS).
+  override protected def bufferContainsNull: Boolean = !ignoreNulls
+
+  // Result is array<elementType>; containsNull is true iff null elements are 
kept.
+  override def dataType: DataType = ArrayType(elementType, containsNull = 
bufferContainsNull)
+
+  // The buffer stores distinct elements. Mirror CollectSet's keying so 
equality is correct
+  // for float/double (bit pattern) and binary (byte array) element types.
+  override lazy val bufferElementType: DataType = elementType match {
+    case BinaryType => ArrayType(ByteType)
+    case DoubleType => LongType
+    case FloatType => IntegerType
+    case other => other
+  }
+
+  @transient private lazy val complexNormalizer: Any => Any = {
+    val ref = BoundReference(0, elementType, nullable = true)
+    val proj = UnsafeProjection.create(NormalizeFloatingNumbers.normalize(ref))
+    (value: Any) => InternalRow.copyValue(proj(InternalRow(value)).get(0, 
elementType))
+  }
+
+  override def convertToBufferElement(value: Any): Any = elementType match {
+    // See CollectSet.convertToBufferElement for why binary/float/double are 
keyed specially.
+    case BinaryType => 
UnsafeArrayData.fromPrimitiveArray(value.asInstanceOf[Array[Byte]])
+    case DoubleType =>
+      java.lang.Double.doubleToLongBits(
+        NormalizeFloatingNumbers.DOUBLE_NORMALIZER(value).asInstanceOf[Double])
+    case FloatType =>
+      java.lang.Float.floatToIntBits(
+        NormalizeFloatingNumbers.FLOAT_NORMALIZER(value).asInstanceOf[Float])
+    case dt if NormalizeFloatingNumbers.needNormalize(dt) => 
complexNormalizer(value)
+    case _ => InternalRow.copyValue(value)
+  }
+
+  // Iterate the input array and add each element to the set. NULL input 
arrays are skipped;
+  // a NULL element is dropped when ignoreNulls (IGNORE NULLS) and kept 
otherwise (RESPECT
+  // NULLS), where the HashSet naturally dedups it to a single null.
+  override def update(
+      buffer: mutable.HashSet[Any],
+      input: InternalRow): mutable.HashSet[Any] = {
+    val arr = child.eval(input)
+    if (arr != null) {
+      arr.asInstanceOf[ArrayData].foreach(elementType, (_, element: Any) =>
+        if (element != null) {
+          buffer += convertToBufferElement(element)
+        } else if (!ignoreNulls) {
+          buffer += null
+        })
+    }
+    buffer
+  }
+
+  override def eval(buffer: mutable.HashSet[Any]): Any = {
+    val array = elementType match {
+      case BinaryType =>
+        buffer.iterator.map {
+          case null => null
+          case v => v.asInstanceOf[ArrayData].toByteArray()
+        }.toArray[Any]
+      case DoubleType =>
+        buffer.iterator.map {
+          case null => null
+          case v => java.lang.Double.longBitsToDouble(v.asInstanceOf[Long])
+        }.toArray[Any]
+      case FloatType =>
+        buffer.iterator.map {
+          case null => null
+          case v => java.lang.Float.intBitsToFloat(v.asInstanceOf[Int])
+        }.toArray[Any]
+      case _ => buffer.toArray
+    }
+    new GenericArrayData(array)
+  }
+
+  override def checkInputDataTypes(): TypeCheckResult = child.dataType match {
+    case ArrayType(et, _)
+        if !et.existsRecursively(_.isInstanceOf[MapType]) && 
UnsafeRowUtils.isBinaryStable(et) =>
+      TypeCheckResult.TypeCheckSuccess
+    case ArrayType(_, _) =>
+      DataTypeMismatch(
+        errorSubClass = "UNSUPPORTED_INPUT_TYPE",
+        messageParameters = Map(
+          "functionName" -> toSQLId(prettyName),
+          "dataType" -> (s"${toSQLType(MapType)} " + "or \"COLLATED STRING\"")
+        )
+      )
+    case _ =>
+      DataTypeMismatch(
+        errorSubClass = "UNEXPECTED_INPUT_TYPE",
+        messageParameters = Map(
+          "paramIndex" -> ordinalNumber(0),
+          "requiredType" -> toSQLType(ArrayType),
+          "inputSql" -> toSQLExpr(child),
+          "inputType" -> toSQLType(child.dataType)
+        )
+      )
+  }
+
+  override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): 
ImperativeAggregate =
+    copy(mutableAggBufferOffset = newMutableAggBufferOffset)
+
+  override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): 
ImperativeAggregate =
+    copy(inputAggBufferOffset = newInputAggBufferOffset)
+
+  override def prettyName: String = "collect_union"
+
+  override def createAggregationBuffer(): mutable.HashSet[Any] = 
mutable.HashSet.empty
+
+  override def toString: String = {
+    val ignoreNullsStr = if (ignoreNulls) "" else " respect nulls"
+    s"$prettyName($child)$ignoreNullsStr"
+  }
+
+  override def sql(isDistinct: Boolean): String = {
+    val distinct = if (isDistinct) "DISTINCT " else ""
+    val nullsStr = if (ignoreNulls) "" else " RESPECT NULLS"
+    s"$prettyName($distinct${child.sql})$nullsStr"
+  }
+
+  override protected def withNewChildInternal(newChild: Expression): 
CollectUnion =
+    copy(child = newChild)
+}
+
 /**
  * Collect the top-k elements. This expression is dedicated only for Spark-ML.
  * @param reverse when true, returns the smallest k elements.
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md 
b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 2637fbc6f077..03505b37c4c2 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -482,6 +482,7 @@
 | org.apache.spark.sql.catalyst.expressions.aggregate.CollectList | array_agg 
| SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col) | 
struct<collect_list(col):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.CollectList | 
collect_list | SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col) | 
struct<collect_list(col):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.CollectSet | collect_set 
| SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col) | 
struct<collect_set(col):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CollectUnion | 
collect_union | SELECT collect_union(col) FROM VALUES (array(1, 2)), (array(2, 
3)), (array(1)) AS tab(col) | struct<collect_union(col):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Corr | corr | SELECT 
corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2) | 
struct<corr(c1, c2):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Count | count | SELECT 
count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col) | 
struct<count(1):bigint> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.CountIf | count_if | 
SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col) 
| struct<count_if(((col % 2) = 0)):bigint> |
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index 32517af68c91..3bc766860372 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -676,6 +676,72 @@ class DataFrameAggregateSuite extends SharedSparkSession
     )
   }
 
+  test("collect_union function") {
+    // Distinct union of array elements across rows.
+    val df = Seq(Seq(1, 2), Seq(2, 3), Seq(1)).toDF("arr")
+    checkDataset(
+      df.select(collect_union($"arr").as("u")).as[Set[Int]],
+      Set(1, 2, 3))
+    checkAnswer(
+      df.select(sort_array(collect_union($"arr"))),
+      Seq(Row(Seq(1, 2, 3))))
+    checkAnswer(
+      df.selectExpr("sort_array(collect_union(arr))"),
+      Seq(Row(Seq(1, 2, 3))))
+
+    // NULL array inputs are always skipped.
+    val dfNulls = Seq(Seq(1, 2), null, Seq(2, 3)).toDF("arr")
+    checkAnswer(
+      dfNulls.select(sort_array(collect_union($"arr"))),
+      Seq(Row(Seq(1, 2, 3))))
+
+    // NULL elements: dropped by default (IGNORE NULLS, matching collect_set) 
...
+    val dfNullElem = Seq(Seq(Integer.valueOf(1), null), 
Seq(Integer.valueOf(2))).toDF("arr")
+    checkAnswer(
+      dfNullElem.select(sort_array(collect_union($"arr"))),
+      Seq(Row(Seq(1, 2))))
+    // ... and kept (a single null) with RESPECT NULLS, matching
+    // array_distinct(flatten(collect_list(...))). sort_array puts null first 
in asc order.
+    checkAnswer(
+      dfNullElem.selectExpr("sort_array(collect_union(arr) RESPECT NULLS)"),
+      Seq(Row(Seq(null, 1, 2))))
+    // Equivalent to array_distinct(flatten(collect_list(...))) under RESPECT 
NULLS. Compare
+    // order-insensitively via sort_array, since element order in either 
result is unspecified.
+    checkAnswer(
+      dfNullElem.selectExpr("sort_array(collect_union(arr) RESPECT NULLS)"),
+      
dfNullElem.selectExpr("sort_array(array_distinct(flatten(collect_list(arr))))"))
+
+    // Per-group union.
+    val g = Seq(("a", Seq(1, 2)), ("a", Seq(2, 3)), ("b", Seq(4))).toDF("k", 
"arr")
+    checkAnswer(
+      g.groupBy("k").agg(sort_array(collect_union($"arr"))).orderBy("k"),
+      Seq(Row("a", Seq(1, 2, 3)), Row("b", Seq(4))))
+
+    // Empty result: only-NULL arrays produce an empty array, not null.
+    val dfEmpty = Seq[Seq[Int]](null, null).toDF("arr")
+    checkAnswer(
+      dfEmpty.select(collect_union($"arr")),
+      Seq(Row(Seq.empty[Int])))
+  }
+
+  test("collect_union requires an array input") {
+    // A non-array (scalar) input is rejected at analysis time.
+    val df = Seq(1, 2, 3).toDF("a")
+    checkError(
+      exception = intercept[AnalysisException] {
+        df.select(collect_union($"a")).collect()
+      },
+      condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+      parameters = Map(
+        "sqlExpr" -> "\"collect_union(a)\"",
+        "paramIndex" -> "first",
+        "inputSql" -> "\"a\"",
+        "inputType" -> "\"INT\"",
+        "requiredType" -> "\"ARRAY\""),
+      context = ExpectedContext(
+        fragment = "collect_union", callSitePattern = 
getCurrentClassCallSitePattern))
+  }
+
   test("SPARK-55256: array_agg and collect_list skip nulls by default") {
     val df = Seq((1, Some(2)), (2, None), (3, Some(4))).toDF("a", "b")
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to