DDDominik commented on code in PR #3248:
URL: https://github.com/apache/celeborn/pull/3248#discussion_r2221407065
##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -4647,6 +4687,19 @@ object CelebornConf extends Logging {
.intConf
.createWithDefault(32)
+ val CLIENT_PUSH_MAX_BYTES_SIZE_IN_FLIGHT_PERWORKER:
OptionalConfigEntry[Long] =
+ buildConf("celeborn.client.push.maxBytesSizeInFlight.perWorker")
+ .categories("client")
+ .version("0.6.1")
+ .doc(
+ "Bytes size of Netty in-flight requests per worker. Default max memory
of in flight requests " +
+ " per worker is `celeborn.client.push.maxReqsInFlight.perWorker` *
`celeborn.client.push.buffer.max.size` " +
+ "* compression ratio(1 in worst case): 64KiB * 32 = 2MiB. " +
+ "This is an alternative to
`celeborn.client.push.maxReqsInFlight.perWorker` " +
+ "in cases where records are huge and exceed the maximum memory.")
+ .bytesConf(ByteUnit.BYTE)
+ .createOptional
+
Review Comment:
ditto
##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -4635,6 +4662,19 @@ object CelebornConf extends Logging {
.intConf
.createWithDefault(256)
+ val CLIENT_PUSH_MAX_BYTES_SIZE_IN_FLIGHT_TOTAL: OptionalConfigEntry[Long] =
+ buildConf("celeborn.client.push.maxBytesSizeInFlight.total")
+ .categories("client")
+ .version("0.6.1")
+ .doc(
+ "Bytes size of total Netty in-flight requests. The maximum memory is "
+
+ "`celeborn.client.push.maxReqsInFlight.total` *
`celeborn.client.push.buffer.max.size` " +
+ "* compression ratio(1 in worst case): 64KiB * 256 = 16MiB. " +
+ "This is an addition to `celeborn.client.push.maxReqsInFlight.total`
" +
+ "in cases where records are huge and exceed the maximum memory.")
+ .bytesConf(ByteUnit.BYTE)
+ .createOptional
Review Comment:
An initial value is required here, .createWithDefault(0L)
##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -4647,6 +4687,19 @@ object CelebornConf extends Logging {
.intConf
.createWithDefault(32)
+ val CLIENT_PUSH_MAX_BYTES_SIZE_IN_FLIGHT_PERWORKER:
OptionalConfigEntry[Long] =
+ buildConf("celeborn.client.push.maxBytesSizeInFlight.perWorker")
+ .categories("client")
+ .version("0.6.1")
+ .doc(
+ "Bytes size of Netty in-flight requests per worker. Default max memory
of in flight requests " +
+ " per worker is `celeborn.client.push.maxReqsInFlight.perWorker` *
`celeborn.client.push.buffer.max.size` " +
+ "* compression ratio(1 in worst case): 64KiB * 32 = 2MiB. " +
+ "This is an alternative to
`celeborn.client.push.maxReqsInFlight.perWorker` " +
+ "in cases where records are huge and exceed the maximum memory.")
+ .bytesConf(ByteUnit.BYTE)
+ .createOptional
+
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
