Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/21432#discussion_r191015080
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala ---
@@ -63,17 +63,17 @@ class RelationalGroupedDataset protected[sql](
groupType match {
case RelationalGroupedDataset.GroupByType =>
Dataset.ofRows(
- df.sparkSession, Aggregate(groupingExprs, aliasedAgg,
df.logicalPlan))
+ df.sparkSession, Aggregate(groupingExprs, aliasedAgg,
df.planWithBarrier))
case RelationalGroupedDataset.RollupType =>
Dataset.ofRows(
- df.sparkSession, Aggregate(Seq(Rollup(groupingExprs)),
aliasedAgg, df.logicalPlan))
+ df.sparkSession, Aggregate(Seq(Rollup(groupingExprs)),
aliasedAgg, df.planWithBarrier))
case RelationalGroupedDataset.CubeType =>
Dataset.ofRows(
- df.sparkSession, Aggregate(Seq(Cube(groupingExprs)), aliasedAgg,
df.logicalPlan))
+ df.sparkSession, Aggregate(Seq(Cube(groupingExprs)), aliasedAgg,
df.planWithBarrier))
case RelationalGroupedDataset.PivotType(pivotCol, values) =>
val aliasedGrps = groupingExprs.map(alias)
Dataset.ofRows(
- df.sparkSession, Pivot(Some(aliasedGrps), pivotCol, values,
aggExprs, df.logicalPlan))
+ df.sparkSession, Pivot(Some(aliasedGrps), pivotCol, values,
aggExprs, df.planWithBarrier))
--- End diff --
The fix is not complete. For example, `flatMapGroupsInPandas` in
RelationalGroupedDataset.scala
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
