cloud-fan commented on a change in pull request #32049:
URL: https://github.com/apache/spark/pull/32049#discussion_r667102394
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
##########
@@ -17,61 +17,179 @@
package org.apache.spark.sql.execution.datasources.v2
-import org.apache.spark.sql.catalyst.expressions.{And, Expression,
NamedExpression, ProjectionOverSchema, SubqueryExpression}
-import org.apache.spark.sql.catalyst.planning.ScanOperation
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan,
Project}
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+import org.apache.spark.sql.catalyst.planning.{OperationHelper, ScanOperation}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter,
LeafNode, LogicalPlan, Project}
import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.read.{Scan, V1Scan}
+import org.apache.spark.sql.catalyst.util.toPrettySQL
+import org.apache.spark.sql.connector.expressions.Aggregation
+import org.apache.spark.sql.connector.read.{Scan, ScanBuilder,
SupportsPushDownAggregates, SupportsPushDownFilters, V1Scan}
import org.apache.spark.sql.execution.datasources.DataSourceStrategy
import org.apache.spark.sql.sources
import org.apache.spark.sql.types.StructType
-object V2ScanRelationPushDown extends Rule[LogicalPlan] {
+object V2ScanRelationPushDown extends Rule[LogicalPlan] with AliasHelper
+ with OperationHelper with PredicateHelper {
import DataSourceV2Implicits._
- override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
- case ScanOperation(project, filters, relation: DataSourceV2Relation) =>
- val scanBuilder =
relation.table.asReadable.newScanBuilder(relation.options)
+ def apply(plan: LogicalPlan): LogicalPlan = {
+
applyColumnPruning(pushdownAggregate(pushDownFilters(createScanBuilder(plan))))
+ }
+
+ private def createScanBuilder(plan: LogicalPlan) = plan.transform {
+ case r: DataSourceV2Relation =>
+ ScanBuilderHolder(r.output, r,
r.table.asReadable.newScanBuilder(r.options))
+ }
- val normalizedFilters = DataSourceStrategy.normalizeExprs(filters,
relation.output)
+ private def pushDownFilters(plan: LogicalPlan) = plan.transform {
+ // update the scan builder with filter push down and return a new plan
with filter pushed
+ case filter @ Filter(_, sHolder: ScanBuilderHolder) =>
+ val (filters, _, _) = collectFilters(filter).get
+
+ val normalizedFilters =
+ DataSourceStrategy.normalizeExprs(filters, sHolder.relation.output)
val (normalizedFiltersWithSubquery, normalizedFiltersWithoutSubquery) =
normalizedFilters.partition(SubqueryExpression.hasSubquery)
// `pushedFilters` will be pushed down and evaluated in the underlying
data sources.
// `postScanFilters` need to be evaluated after the scan.
// `postScanFilters` and `pushedFilters` can overlap, e.g. the parquet
row group filter.
val (pushedFilters, postScanFiltersWithoutSubquery) =
PushDownUtils.pushFilters(
- scanBuilder, normalizedFiltersWithoutSubquery)
+ sHolder.builder, normalizedFiltersWithoutSubquery)
val postScanFilters = postScanFiltersWithoutSubquery ++
normalizedFiltersWithSubquery
+ logInfo(
+ s"""
+ |Pushing operators to ${sHolder.relation.name}
+ |Pushed Filters: ${pushedFilters.mkString(", ")}
+ |Post-Scan Filters: ${postScanFilters.mkString(",")}
+ """.stripMargin)
+
+ val filterCondition = postScanFilters.reduceLeftOption(And)
+ filterCondition.map(Filter(_, sHolder)).getOrElse(sHolder)
+ }
+
+ def pushdownAggregate(plan: LogicalPlan): LogicalPlan = plan.transform {
+ // update the scan builder with agg pushdown and return a new plan with
agg pushed
+ case aggNode@Aggregate(groupingExpressions, resultExpressions, child) =>
+ child match {
+ case ScanOperation(project, filters, sHolder: ScanBuilderHolder) =>
+ sHolder.builder match {
+ case r: SupportsPushDownAggregates =>
+ if (filters.length == 0) { // can't push down aggregate if
postScanFilters exist
+ val aggregates = getAggregateExpression(resultExpressions,
project, sHolder)
+ val pushedAggregates = PushDownUtils
+ .pushAggregates(sHolder.builder, aggregates,
groupingExpressions)
+ if (pushedAggregates.aggregateExpressions.isEmpty) {
+ aggNode // return original plan node
+ } else {
+ // use the aggregate columns as the output columns
+ // e.g. TABLE t (c1 INT, c2 INT, c3 INT)
+ // SELECT min(c1), max(c1) FROM t;
+ // Use min(c1), max(c1) as output for
DataSourceV2ScanRelation
+ // We want to have the following logical plan:
+ // == Optimized Logical Plan ==
+ // Aggregate [min(min(c1)#21) AS min(c1)#17, max(max(c1)#22)
AS max(c1)#18]
+ // +- RelationV2[min(c1)#21, max(c1)#22] parquet file ...
+ val output = aggregates.map {
+ case agg: AggregateExpression =>
+ AttributeReference(toPrettySQL(agg), agg.dataType)()
+ }
+
+ // No need to do column pruning because only the aggregate
columns are used as
+ // DataSourceV2ScanRelation output columns. All the other
columns are not
+ // included in the output. Since PushDownUtils.pruneColumns
is not called,
+ // ScanBuilder.requiredSchema is not pruned, but
ScanBuilder.requiredSchema is
+ // not used anyways. The schema for aggregate columns will
be built in Scan.
+ val scan = sHolder.builder.build()
+
+ logInfo(
+ s"""
+ |Pushing operators to ${sHolder.relation.name}
+ |Pushed Aggregate Functions:
+ | ${pushedAggregates.aggregateExpressions.mkString(",
")}
+ |Output: ${output.mkString(", ")}
+ """.stripMargin)
+
+ val scanRelation =
DataSourceV2ScanRelation(sHolder.relation, scan, output)
+ val plan = Aggregate(groupingExpressions, resultExpressions,
scanRelation)
+
+ // Change the optimized logical plan to reflect the pushed
down aggregate
+ // e.g. TABLE t (c1 INT, c2 INT, c3 INT)
+ // SELECT min(c1), max(c1) FROM t;
+ // The original logical plan is
+ // Aggregate [min(c1#9) AS min(c1)#17, max(c1#9) AS
max(c1)#18]
+ // +- RelationV2[c1#9] parquet ...
+ //
+ // After change the V2ScanRelation output to [min(_1)#21,
max(_1)#22]
+ // we have the following
+ // !Aggregate [min(_1#9) AS min(_1)#17, max(_1#9) AS
max(_1)#18]
+ // +- RelationV2[min(_1)#21, max(_1)#22] parquet ...
+ //
+ // We want to change it to
+ // == Optimized Logical Plan ==
+ // Aggregate [min(min(c1)#21) AS min(c1)#17, max(max(c1)#22)
AS max(c1)#18]
+ // +- RelationV2[min(c1)#21, max(c1)#22] parquet file ...
+ var i = 0
+ plan.transformExpressions {
Review comment:
one thing we should take care of is the changing of expr ID. For example,
```
Sort(a#1, sum_b#3)
Aggregate(groupBy=a#1, result=a#1, sum(b#2) as sum_b#3)
V2Relation(a#1, b#2)
```
After pushdown
```
!Sort(a#1, sum_b#3)
Aggregate(groupBy=a#4, result=a#4, sum(sum(b)#5) as sum_b#3)
V2Scan(a#4, sum(b)#5)
```
The sort operator becomes invalid because `a#1` is missing now. We should
find a way to keep the original expr ID, via `Alias(...)(exprId = originalId)`
```
Sort(a#1, sum_b#3)
Aggregate(groupBy=a#4, result=a#4 as a#1, sum(sum(b)#5) as sum_b#3)
V2Scan(a#4, sum(b)#5)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
