This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 57bf23c [SPARK-31037][SQL][FOLLOW-UP] Replace legacy
ReduceNumShufflePartitions with CoalesceShufflePartitions in comment
57bf23c is described below
commit 57bf23c01b2cffe5011a9d15eb68eff5c28519f4
Author: yi.wu <[email protected]>
AuthorDate: Tue Mar 10 11:09:36 2020 -0700
[SPARK-31037][SQL][FOLLOW-UP] Replace legacy ReduceNumShufflePartitions
with CoalesceShufflePartitions in comment
### What changes were proposed in this pull request?
Replace legacy `ReduceNumShufflePartitions` with
`CoalesceShufflePartitions` in comment.
### Why are the changes needed?
Rule `ReduceNumShufflePartitions` has renamed to
`CoalesceShufflePartitions`, we should update related comment as well.
### Does this PR introduce any user-facing change?
No.
### How was this patch tested?
N/A.
Closes #27865 from Ngone51/spark_31037_followup.
Authored-by: yi.wu <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 34be83e08b6f5313bdd9d165d3e203d06eff677b)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala | 6 +++---
.../apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index fc88a7f..c1486aa 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -97,12 +97,12 @@ case class AdaptiveSparkPlanExec(
@transient private val queryStageOptimizerRules: Seq[Rule[SparkPlan]] = Seq(
ReuseAdaptiveSubquery(conf, context.subqueryCache),
// Here the 'OptimizeSkewedJoin' rule should be executed
- // before 'ReduceNumShufflePartitions', as the skewed partition handled
- // in 'OptimizeSkewedJoin' rule, should be omitted in
'ReduceNumShufflePartitions'.
+ // before 'CoalesceShufflePartitions', as the skewed partition handled
+ // in 'OptimizeSkewedJoin' rule, should be omitted in
'CoalesceShufflePartitions'.
OptimizeSkewedJoin(conf),
CoalesceShufflePartitions(conf),
// The rule of 'OptimizeLocalShuffleReader' need to make use of the
'partitionStartIndices'
- // in 'ReduceNumShufflePartitions' rule. So it must be after
'ReduceNumShufflePartitions' rule.
+ // in 'CoalesceShufflePartitions' rule. So it must be after
'CoalesceShufflePartitions' rule.
OptimizeLocalShuffleReader(conf),
ApplyColumnarRulesAndInsertTransitions(conf,
context.session.sessionState.columnarRules),
CollapseCodegenStages(conf)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index c3bcce4..4387409 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -52,7 +52,7 @@ import org.apache.spark.sql.internal.SQLConf
* (L4-1, R4-1), (L4-2, R4-1), (L4-1, R4-2), (L4-2, R4-2)
*
* Note that, when this rule is enabled, it also coalesces non-skewed
partitions like
- * `ReduceNumShufflePartitions` does.
+ * `CoalesceShufflePartitions` does.
*/
case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
@@ -191,7 +191,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends
Rule[SparkPlan] {
val leftSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
val rightSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
// This is used to delay the creation of non-skew partitions so that we
can potentially
- // coalesce them like `ReduceNumShufflePartitions` does.
+ // coalesce them like `CoalesceShufflePartitions` does.
val nonSkewPartitionIndices = mutable.ArrayBuffer.empty[Int]
val leftSkewDesc = new SkewDesc
val rightSkewDesc = new SkewDesc
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
