This is an automated email from the ASF dual-hosted git repository.
rexxiong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new ff2bc9206 [CELEBORN-1317][FOLLOWUP] Update default value of
celeborn.master.http.maxWorkerThreads and celeborn.worker.http.maxWorkerThreads
via QueuedThreadPool
ff2bc9206 is described below
commit ff2bc92067b5785376900a025079d9f96674cae2
Author: SteNicholas <[email protected]>
AuthorDate: Fri Mar 29 11:56:04 2024 +0800
[CELEBORN-1317][FOLLOWUP] Update default value of
celeborn.master.http.maxWorkerThreads and celeborn.worker.http.maxWorkerThreads
via QueuedThreadPool
### What changes were proposed in this pull request?
Update default value of `celeborn.master.http.maxWorkerThreads` and
`celeborn.worker.http.maxWorkerThreads` via `QueuedThreadPool`, of which
default value is 200.
### Why are the changes needed?
`QueuedThreadPool` determines that the default minimum threads is 8, and
the default maximum threads is 200 in
[QueuedThreadPool#L121](https://github.com/jetty/jetty.project/blob/48f6ab7289e6bf359db92686a3b81d154bc6c074/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/thread/QueuedThreadPool.java#L1210)
and
[QueuedThreadPool#L125](https://github.com/jetty/jetty.project/blob/48f6ab7289e6bf359db92686a3b81d154bc6c074/jetty-core/jetty-util/src/main/java/org/eclipse/jetty/util/th
[...]
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
No.
Closes #2428 from SteNicholas/CELEBORN-1317.
Authored-by: SteNicholas <[email protected]>
Signed-off-by: Shuang <[email protected]>
---
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala | 4 ++--
docs/configuration/master.md | 2 +-
docs/configuration/worker.md | 2 +-
.../scala/org/apache/celeborn/server/common/http/HttpServer.scala | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index 2e4bd2d5a..8e849a4b0 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -1989,7 +1989,7 @@ object CelebornConf extends Logging {
.doc("Maximum number of threads in the master http worker thread pool.")
.intConf
.checkValue(_ > 0, "Must be positive.")
- .createWithDefault(999)
+ .createWithDefault(200)
val MASTER_HTTP_STOP_TIMEOUT: ConfigEntry[Long] =
buildConf("celeborn.master.http.stopTimeout")
@@ -2564,7 +2564,7 @@ object CelebornConf extends Logging {
.doc("Maximum number of threads in the worker http worker thread pool.")
.intConf
.checkValue(_ > 0, "Must be positive.")
- .createWithDefault(999)
+ .createWithDefault(200)
val WORKER_HTTP_STOP_TIMEOUT: ConfigEntry[Long] =
buildConf("celeborn.worker.http.stopTimeout")
diff --git a/docs/configuration/master.md b/docs/configuration/master.md
index 13f1d5c40..e2217868b 100644
--- a/docs/configuration/master.md
+++ b/docs/configuration/master.md
@@ -43,7 +43,7 @@ license: |
| celeborn.master.heartbeat.worker.timeout | 120s | false | Worker heartbeat
timeout. | 0.3.0 | celeborn.worker.heartbeat.timeout |
| celeborn.master.host | <localhost> | false | Hostname for master to
bind. | 0.2.0 | |
| celeborn.master.http.host | <localhost> | false | Master's http host.
| 0.4.0 |
celeborn.metrics.master.prometheus.host,celeborn.master.metrics.prometheus.host
|
-| celeborn.master.http.maxWorkerThreads | 999 | false | Maximum number of
threads in the master http worker thread pool. | 0.5.0 | |
+| celeborn.master.http.maxWorkerThreads | 200 | false | Maximum number of
threads in the master http worker thread pool. | 0.5.0 | |
| celeborn.master.http.port | 9098 | false | Master's http port. | 0.4.0 |
celeborn.metrics.master.prometheus.port,celeborn.master.metrics.prometheus.port
|
| celeborn.master.http.stopTimeout | 5s | false | Master http server stop
timeout. | 0.5.0 | |
| celeborn.master.internal.port | 8097 | false | Internal port on the master
where both workers and other master nodes connect. | 0.5.0 | |
diff --git a/docs/configuration/worker.md b/docs/configuration/worker.md
index 484cf1b6d..675ec7064 100644
--- a/docs/configuration/worker.md
+++ b/docs/configuration/worker.md
@@ -83,7 +83,7 @@ license: |
| celeborn.worker.graceful.shutdown.saveCommittedFileInfo.sync | false | false
| Whether to call sync method to save committed file infos into Level DB to
handle OS crash. | 0.3.1 | |
| celeborn.worker.graceful.shutdown.timeout | 600s | false | The worker's
graceful shutdown timeout time. | 0.2.0 | |
| celeborn.worker.http.host | <localhost> | false | Worker's http host.
| 0.4.0 |
celeborn.metrics.worker.prometheus.host,celeborn.worker.metrics.prometheus.host
|
-| celeborn.worker.http.maxWorkerThreads | 999 | false | Maximum number of
threads in the worker http worker thread pool. | 0.5.0 | |
+| celeborn.worker.http.maxWorkerThreads | 200 | false | Maximum number of
threads in the worker http worker thread pool. | 0.5.0 | |
| celeborn.worker.http.port | 9096 | false | Worker's http port. | 0.4.0 |
celeborn.metrics.worker.prometheus.port,celeborn.worker.metrics.prometheus.port
|
| celeborn.worker.http.stopTimeout | 5s | false | Worker http server stop
timeout. | 0.5.0 | |
| celeborn.worker.internal.port | 0 | false | Internal server port on the
Worker where the master nodes connect. | 0.5.0 | |
diff --git
a/service/src/main/scala/org/apache/celeborn/server/common/http/HttpServer.scala
b/service/src/main/scala/org/apache/celeborn/server/common/http/HttpServer.scala
index cbb61136e..6f62f5ced 100644
---
a/service/src/main/scala/org/apache/celeborn/server/common/http/HttpServer.scala
+++
b/service/src/main/scala/org/apache/celeborn/server/common/http/HttpServer.scala
@@ -97,7 +97,7 @@ private[celeborn] case class HttpServer(
object HttpServer {
def apply(role: String, host: String, port: Int, poolSize: Int, stopTimeout:
Long): HttpServer = {
- val pool = new QueuedThreadPool(poolSize)
+ val pool = new QueuedThreadPool(math.max(poolSize, 8))
pool.setName(s"$role-JettyThreadPool")
pool.setDaemon(true)
val server = new Server(pool)
