CodingCat commented on code in PR #2358:
URL:
https://github.com/apache/incubator-celeborn/pull/2358#discussion_r1520864481
##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -4018,6 +4022,32 @@ object CelebornConf extends Logging {
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("64m")
+ val CLIENT_PUSH_SORT_USE_ADAPTIVE_MEMORY_THRESHOLD: ConfigEntry[Boolean] =
+ buildConf("celeborn.client.spark.push.sort.memory.useAdaptiveThreshold")
+ .withAlternative("celeborn.push.sortMemory.useAdaptiveThreshold")
+ .categories("client")
+ .doc("adaptively adjust threshold for sort shuffle writer's memory
threshold")
+ .version("0.5.0")
+ .booleanConf
+ .createWithDefault(false)
+
+ val CLIENT_PUSH_SORT_SMALL_PUSH_TOLERATE_FACTOR: ConfigEntry[Double] =
+ buildConf("celeborn.client.spark.push.sort.smallPushTolerateFactor")
+ .withAlternative("celeborn.push.sortMemory.adaptiveThreshold")
+ .categories("client")
+ .doc("only be in effect when
celeborn.client.spark.push.sort.memory.adaptiveThreshold is" +
Review Comment:
updated
##########
client-spark/common/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedPusher.java:
##########
@@ -44,6 +44,65 @@
public class SortBasedPusher extends MemoryConsumer {
+ class MemoryThresholdManager {
+
+ private long maxMemoryThresholdInBytes;
Review Comment:
updated
##########
client-spark/common/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedPusher.java:
##########
@@ -44,6 +44,65 @@
public class SortBasedPusher extends MemoryConsumer {
+ class MemoryThresholdManager {
+
+ private long maxMemoryThresholdInBytes;
+ private double smallPushTolerateFactor;
Review Comment:
updated
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
