uros-db commented on code in PR #56891:
URL: https://github.com/apache/spark/pull/56891#discussion_r3528836050
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/GroupingAnalyticsTransformer.scala:
##########
@@ -81,34 +88,117 @@ object GroupingAnalyticsTransformer extends SQLConfHelper
with AliasHelper {
child: LogicalPlan,
aggregationExpressions: Seq[NamedExpression]): Aggregate = {
- val groupByAliases = constructGroupByAlias(newAlias, groupByExpressions)
+ if (shouldLowerToGrandTotalAggregate(groupByExpressions,
selectedGroupByExpressions)) {
+ constructGrandTotalAggregate(newAlias, aggregationExpressions, child)
+ } else {
+ val groupByAliases = constructGroupByAlias(newAlias, groupByExpressions)
- val gid = AttributeReference(VirtualColumn.groupingIdName,
GroupingID.dataType, false)()
- val expand = constructExpand(
- selectedGroupByExpressions = selectedGroupByExpressions,
- child = child,
- groupByAliases = groupByAliases,
- gid = gid,
- childOutput = childOutput
- )
- val groupingAttributes = expand.output.drop(childOutput.length)
+ val gid = AttributeReference(VirtualColumn.groupingIdName,
GroupingID.dataType, false)()
+ val expand = constructExpand(
+ selectedGroupByExpressions = selectedGroupByExpressions,
+ child = child,
+ groupByAliases = groupByAliases,
+ gid = gid,
+ childOutput = childOutput
+ )
+ val groupingAttributes = expand.output.drop(childOutput.length)
- val aggregations = constructAggregateExpressions(
- newAlias = newAlias,
- groupByExpressions = groupByExpressions,
- aggregations = aggregationExpressions,
- groupByAliases = groupByAliases,
- groupingAttributes = groupingAttributes,
- gid = gid
- )
+ val aggregations = constructAggregateExpressions(
+ newAlias = newAlias,
+ groupByExpressions = groupByExpressions,
+ aggregations = aggregationExpressions,
+ groupByAliases = groupByAliases,
+ groupingAttributes = groupingAttributes,
+ gid = gid
+ )
+
+ Aggregate(
+ groupingExpressions = groupingAttributes,
+ aggregateExpressions = aggregations,
+ child = expand
+ )
+ }
+ }
- val aggregate = Aggregate(
- groupingExpressions = groupingAttributes,
+ /**
+ * Whether a grouping-set spec is the grand-total-only case `GROUP BY
GROUPING SETS (())` (and
+ * the equivalent empty `CUBE()`/`ROLLUP()`) that [[apply]] lowers to a
global [[Aggregate]]:
+ * no leading group-by expressions and a single empty grouping set, with
+ * [[SQLConf.LOWER_EMPTY_GROUPING_SET_TO_GLOBAL_AGGREGATE]] enabled. This is
the
+ * pre-lowering decision; [[isLoweredToGrandTotalAggregate]] detects the
same case post-lowering.
+ */
+ def shouldLowerToGrandTotalAggregate(
+ groupByExpressions: Seq[Expression],
+ selectedGroupByExpressions: Seq[Seq[Expression]]): Boolean = {
+ conf.getConf(SQLConf.LOWER_EMPTY_GROUPING_SET_TO_GLOBAL_AGGREGATE) &&
+ groupByExpressions.isEmpty &&
+ selectedGroupByExpressions.length == 1 &&
+ selectedGroupByExpressions.head.isEmpty
+ }
+
+ /**
+ * Whether a lowered [[Aggregate]] is the grand total produced by
+ * [[shouldLowerToGrandTotalAggregate]]: its resolved grouping expressions
(without the
+ * `spark_grouping_id` key, as returned by [[collectGroupingExpressions]])
are empty, with the
+ * flag enabled. The flag is part of the check because with it off the same
grand total is
+ * lowered via [[Expand]], whose `spark_grouping_id` key must still be
referenced.
+ *
+ * This keys purely on the collected grouping expressions being empty, so it
also matches a
+ * value-equivalent all-empty multi-set [[Expand]] aggregate (e.g. `GROUP BY
GROUPING SETS ((),
Review Comment:
Added (SELECT/HAVING/ORDER BY, flag-on/off). Note: ((), ()) is a duplicated
set → gets _gen_grouping_pos, stays on Expand, so it's not lowered and
HAVING/ORDER BY errors already on master. Corrected the scaladoc/comments
accordingly. (14544d66880)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
