turboFei commented on code in PR #3158:
URL: https://github.com/apache/celeborn/pull/3158#discussion_r2004742371
##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -5208,6 +5212,27 @@ object CelebornConf extends Logging {
.intConf
.createWithDefault(2000)
+ val CLIENT_SHUFFLE_GET_REDUCER_FILE_GROUP_BROADCAST_ENABLED =
+
buildConf("celeborn.client.spark.shuffle.getReducerFileGroup.broadcast.enabled")
+ .categories("client")
+ .doc(
+ "Whether to leverage Spark broadcast mechanism to send the
GetReducerFileGroupResponse. " +
+ "If the response size is large and Spark executor number is large,
the Spark driver network " +
+ "may be exhausted because each executor will pull the response from
the driver. With broadcasting " +
+ "GetReducerFileGroupResponse, it prevents the driver from being the
bottleneck in sending out multiple " +
+ "copies of the GetReducerFileGroupResponse (one per executor).")
+ .version("0.6.0")
+ .booleanConf
+ .createWithDefault(false)
+
+ val CLIENT_SHUFFLE_GET_REDUCER_FILE_GROUP_BROADCAST_MINI_SIZE =
Review Comment:
same with
https://github.com/apache/spark/blob/8d260084b8a50ff59a127c7292c0cdb6737981b0/core/src/main/scala/org/apache/spark/internal/config/package.scala#L1718
```
private[spark] val SHUFFLE_MAPOUTPUT_MIN_SIZE_FOR_BROADCAST =
ConfigBuilder("spark.shuffle.mapOutput.minSizeForBroadcast")
.doc("The size at which we use Broadcast to send the map output
statuses to the executors.")
.version("2.0.0")
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("512k")
```
##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -5208,6 +5212,27 @@ object CelebornConf extends Logging {
.intConf
.createWithDefault(2000)
+ val CLIENT_SHUFFLE_GET_REDUCER_FILE_GROUP_BROADCAST_ENABLED =
+
buildConf("celeborn.client.spark.shuffle.getReducerFileGroup.broadcast.enabled")
+ .categories("client")
+ .doc(
+ "Whether to leverage Spark broadcast mechanism to send the
GetReducerFileGroupResponse. " +
+ "If the response size is large and Spark executor number is large,
the Spark driver network " +
+ "may be exhausted because each executor will pull the response from
the driver. With broadcasting " +
+ "GetReducerFileGroupResponse, it prevents the driver from being the
bottleneck in sending out multiple " +
+ "copies of the GetReducerFileGroupResponse (one per executor).")
+ .version("0.6.0")
+ .booleanConf
+ .createWithDefault(false)
+
+ val CLIENT_SHUFFLE_GET_REDUCER_FILE_GROUP_BROADCAST_MINI_SIZE =
Review Comment:
same default value with
https://github.com/apache/spark/blob/8d260084b8a50ff59a127c7292c0cdb6737981b0/core/src/main/scala/org/apache/spark/internal/config/package.scala#L1718
```
private[spark] val SHUFFLE_MAPOUTPUT_MIN_SIZE_FOR_BROADCAST =
ConfigBuilder("spark.shuffle.mapOutput.minSizeForBroadcast")
.doc("The size at which we use Broadcast to send the map output
statuses to the executors.")
.version("2.0.0")
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("512k")
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
