maryannxue commented on a change in pull request #33541:
URL: https://github.com/apache/spark/pull/33541#discussion_r677640114
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
##########
@@ -113,29 +164,17 @@ case class AdaptiveSparkPlanExec(
CollapseCodegenStages()
)
- // The partitioning of the query output depends on the shuffle(s) in the
final stage. If the
- // original plan contains a repartition operator, we need to preserve the
specified partitioning,
- // whether or not the repartition-introduced shuffle is optimized out
because of an underlying
- // shuffle of the same partitioning. Thus, we need to exclude some
`AQEShuffleReadRule`s
- // from the final stage, depending on the presence and properties of
repartition operators.
- private def finalStageOptimizerRules: Seq[Rule[SparkPlan]] = {
- val origins = inputPlan.collect {
- case s: ShuffleExchangeLike => s.shuffleOrigin
- }
- val allRules = queryStageOptimizerRules ++ postStageCreationRules
- allRules.filter {
- case c: AQEShuffleReadRule =>
- origins.forall(c.supportedShuffleOrigins.contains)
- case _ => true
- }
- }
-
- private def optimizeQueryStage(plan: SparkPlan, rules:
Seq[Rule[SparkPlan]]): SparkPlan = {
- val optimized = rules.foldLeft(plan) { case (latestPlan, rule) =>
+ private def optimizeQueryStage(plan: SparkPlan, isFinalStage: Boolean):
SparkPlan = {
+ val optimized = queryStageOptimizerRules.foldLeft(plan) { case
(latestPlan, rule) =>
val applied = rule.apply(latestPlan)
val result = rule match {
- case c: AQEShuffleReadRule if c.mayAddExtraShuffles =>
- if (ValidateRequirements.validate(applied)) {
+ case _: AQEShuffleReadRule =>
Review comment:
if plan has changed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
