Ngone51 commented on a change in pull request #30164:
URL: https://github.com/apache/spark/pull/30164#discussion_r520302849
##########
File path: core/src/main/scala/org/apache/spark/internal/config/package.scala
##########
@@ -1938,4 +1938,38 @@ package object config {
.version("3.0.1")
.booleanConf
.createWithDefault(false)
+
+ private[spark] val PUSH_BASED_SHUFFLE_ENABLED =
+ ConfigBuilder("spark.shuffle.push.enabled")
+ .doc("Set to 'true' to enable push based shuffle on the client side and
this works in" +
+ "conjunction with the server side flag
spark.shuffle.server.mergedShuffleFileManagerImpl" +
+ "which needs to be set with the appropriate" +
+ "org.apache.spark.network.shuffle.MergedShuffleFileManager
implementation for push-based" +
+ "shuffle to be enabled")
+ .booleanConf
+ .createWithDefault(false)
+
+ private[spark] val MAX_MERGER_LOCATIONS_CACHED =
+ ConfigBuilder("spark.shuffle.push.retainedMergerLocations")
+ .doc("Maximum number of shuffle push mergers locations cached for push
based shuffle." +
+ "Currently Shuffle push merger locations are nothing but shuffle
services where an" +
+ "executor is launched in the case of Push based shuffle.")
+ .intConf
+ .createWithDefault(500)
+
+ private[spark] val MERGER_LOCATIONS_MIN_THRESHOLD_RATIO =
+ ConfigBuilder("spark.shuffle.push.mergersMinThresholdRatio")
+ .doc("Minimum percentage of shuffle push mergers locations required to
enable push based" +
+ "shuffle for the stage with respect to number of partitions of the
child stage. This is" +
+ " the number of unique Node Manager locations needed to enable push
based shuffle.")
+ .doubleConf
+ .createWithDefault(0.05)
+
+ private[spark] val MERGER_LOCATIONS_MIN_STATIC_THRESHOLD =
+ ConfigBuilder("spark.shuffle.push.mergersMinStaticThreshold")
+ .doc("Minimum static number of of shuffle push mergers locations should
be available in" +
+ " order to enable push based shuffle for a stage. Note this config
works in" +
+ " conjunction with spark.shuffle.push.mergersMinThresholdRatio")
+ .doubleConf
+ .createWithDefault(5)
Review comment:
Please add the version (3.1.0) info for these newly added confs.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
