ChuckLin2025 commented on code in PR #57592:
URL: https://github.com/apache/spark/pull/57592#discussion_r3671170450
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala:
##########
@@ -316,6 +316,150 @@ case class CollectSet(
copy(child = newChild)
}
+/**
+ * Collect the distinct union of the elements of an array-typed input across
rows.
+ *
+ * Unlike collect_set, whose input is a scalar and whose output is the set of
those scalars,
+ * collect_union's input is itself an array and its output is the set of the
array's
+ * *elements* unioned across all rows. It is equivalent to
+ * `array_distinct(flatten(collect_list(arr)))`, but the aggregation buffer
holds only the
+ * distinct elements (a set), so its size is bounded by the element universe
rather than by
+ * the number of input rows.
+ */
+@ExpressionDescription(
+ usage =
+ "_FUNC_(expr) - Collects and returns the distinct union of the elements of
array `expr`.",
+ examples = """
+ Examples:
+ > SELECT _FUNC_(col) FROM VALUES (array(1, 2)), (array(2, 3)),
(array(1)) AS tab(col);
+ [1,2,3]
+ """,
+ note = """
+ The function is non-deterministic because the order of collected results
depends
+ on the order of the rows which may be non-deterministic after a shuffle.
+ """,
+ group = "agg_funcs",
+ since = "4.3.0")
+case class CollectUnion(
+ child: Expression,
+ mutableAggBufferOffset: Int = 0,
+ inputAggBufferOffset: Int = 0)
+ extends Collect[mutable.HashSet[Any]] with QueryErrorsBase with
UnaryLike[Expression] {
+
+ def this(child: Expression) = this(child, 0, 0)
+
+ // The element type of the input array. Guarded by checkInputDataTypes to be
an ArrayType.
Review Comment:
Applied, thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
