CodingCat commented on code in PR #3109:
URL: https://github.com/apache/celeborn/pull/3109#discussion_r2062847650
##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -4750,6 +4753,23 @@ object CelebornConf extends Logging {
.booleanConf
.createWithDefault(true)
+ val CLIENT_FETCH_CLEAN_FAILED_SHUFFLE: ConfigEntry[Boolean] =
+ buildConf("celeborn.client.spark.fetch.cleanFailedShuffle")
+ .categories("client")
+ .version("0.6.0")
+ .doc("whether to clean those disk space occupied by shuffles which
cannot be fetched")
+ .booleanConf
+ .createWithDefault(false)
+
+ val CLIENT_FETCH_CLEAN_FAILED_SHUFFLE_INTERVAL: ConfigEntry[Long] =
+ buildConf("celeborn.client.spark.fetch.cleanFailedShuffleIntervalMs")
+ .categories("client")
+ .version("0.6.0")
+ .doc("the interval to clean the failed-to-fetch shuffle files, only
valid when" +
+ s" ${CLIENT_FETCH_CLEAN_FAILED_SHUFFLE.key} is enabled")
+ .longConf
Review Comment:
sure, updated
##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -4750,6 +4753,23 @@ object CelebornConf extends Logging {
.booleanConf
.createWithDefault(true)
+ val CLIENT_FETCH_CLEAN_FAILED_SHUFFLE: ConfigEntry[Boolean] =
+ buildConf("celeborn.client.spark.fetch.cleanFailedShuffle")
+ .categories("client")
+ .version("0.6.0")
+ .doc("whether to clean those disk space occupied by shuffles which
cannot be fetched")
+ .booleanConf
+ .createWithDefault(false)
+
+ val CLIENT_FETCH_CLEAN_FAILED_SHUFFLE_INTERVAL: ConfigEntry[Long] =
+ buildConf("celeborn.client.spark.fetch.cleanFailedShuffleIntervalMs")
Review Comment:
updated
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
