beliefer commented on code in PR #37320:
URL: https://github.com/apache/spark/pull/37320#discussion_r939479595
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala:
##########
@@ -408,16 +411,27 @@ object V2ScanRelationPushDown extends Rule[LogicalPlan]
with PredicateHelper {
}
(operation, isPushed && !isPartiallyPushed)
case s @ Sort(order, _, operation @ ScanOperation(project, filter,
sHolder: ScanBuilderHolder))
- // Without building the Scan, we do not know the resulting column
names after aggregate
- // push-down, and thus can't push down Top-N which needs to know the
ordering column names.
- // TODO: we can support simple cases like GROUP BY columns directly
and ORDER BY the same
- // columns, which we know the resulting column names: the
original table columns.
- if sHolder.pushedAggregate.isEmpty && filter.isEmpty &&
- CollapseProject.canCollapseExpressions(order, project, alwaysInline
= true) =>
+ if filter.isEmpty &&
+ CollapseProject.canCollapseExpressions(order, project, alwaysInline =
true) =>
val aliasMap = getAliasMap(project)
- val newOrder = order.map(replaceAlias(_,
aliasMap)).asInstanceOf[Seq[SortOrder]]
+ val aliasReplacedOrder = order.map(replaceAlias(_, aliasMap))
+ val newOrder = if (sHolder.pushedAggregate.isDefined) {
+ // Without building the Scan, Aggregate push-down give the expected
output starts with
+ // `group_col_` or `agg_func_`. When Aggregate push-down working with
Sort for top n
+ // push-down, we need replace these expected output with the origin
expressions.
+ aliasReplacedOrder.map {
+ _.transform {
+ case a: Attribute => sHolder.pushedAggOutputMap.getOrElse(a, a)
+ }.asInstanceOf[SortOrder]
+ }
+ } else {
+ aliasReplacedOrder.asInstanceOf[Seq[SortOrder]]
+ }
val normalizedOrders = DataSourceStrategy.normalizeExprs(
newOrder, sHolder.relation.output).asInstanceOf[Seq[SortOrder]]
+ // Because V2ExpressionBuilder can't translate aggregate functions, so
we can't
Review Comment:
OK
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
