maryannxue commented on a change in pull request #27669: [SPARK-30918][SQL]
improve the splitting of skewed partitions
URL: https://github.com/apache/spark/pull/27669#discussion_r382668795
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
##########
@@ -72,14 +108,16 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends
Rule[SparkPlan] {
/**
* Split the skewed partition based on the map size and the max split number.
*/
- private def getMapStartIndices(stage: ShuffleQueryStageExec, partitionId:
Int): Array[Int] = {
+ private def getMapStartIndices(
+ stage: ShuffleQueryStageExec,
+ partitionId: Int,
+ targetSize: Long): Array[Int] = {
val shuffleId = stage.shuffle.shuffleDependency.shuffleHandle.shuffleId
val mapPartitionSizes = getMapSizesForReduceId(shuffleId, partitionId)
val maxSplits = math.min(conf.getConf(
SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_MAX_SPLITS),
mapPartitionSizes.length)
val avgPartitionSize = mapPartitionSizes.sum / maxSplits
- val advisoryPartitionSize = math.max(avgPartitionSize,
Review comment:
Can we remove this maxSplits and avgPartitionSize in this PR?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
