Github user hvanhovell commented on a diff in the pull request:
https://github.com/apache/spark/pull/19175#discussion_r140718061
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -1287,3 +1288,33 @@ object RemoveRepetitionFromGroupExpressions extends
Rule[LogicalPlan] {
a.copy(groupingExpressions = newGrouping)
}
}
+
+/**
+ * Splits [[Aggregate]] on [[Expand]], which has large number of
projections,
+ * into various [[Aggregate]]s.
+ */
+object SplitAggregateWithExpand extends Rule[LogicalPlan] {
+ /**
+ * Split [[Expand]] operator to a number of [[Expand]] operators
+ */
+ private def splitExpand(expand: Expand): Seq[Expand] = {
+ val len = expand.projections.length
+ val allProjections = expand.projections
+ Seq.tabulate(len)(
+ i => Expand(Seq(allProjections(i)), expand.output, expand.child)
+ )
+ }
+
+ def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+ case a @ Aggregate(_, _, e @ Expand(projections, _, _)) =>
+ if (SQLConf.get.groupingWithUnion && projections.length > 1) {
+ val expands = splitExpand(e)
+ val aggregates: Seq[Aggregate] = Seq.tabulate(expands.length)(
--- End diff --
You can just map over the `expands`
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
