This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c25b9071ad [opt](conf) Modify brpc work pool conf default value #22406
c25b9071ad is described below
commit c25b9071ad778a14535db44269da1aa3fdec1071
Author: Xinyi Zou <[email protected]>
AuthorDate: Mon Jul 31 20:38:34 2023 +0800
[opt](conf) Modify brpc work pool conf default value #22406
Default, if less than or equal 32 core, the following are 128, 128, 10240,
10240 in turn.
if greater than 32 core, the following are core num * 4, core num * 4, core
num * 320, core num * 320 in turn
brpc_heavy_work_pool_threads
brpc_light_work_pool_threads
brpc_heavy_work_pool_max_queue_size
brpc_light_work_pool_max_queue_size
---
be/src/common/config.cpp | 12 ++++--------
be/src/common/config.h | 2 ++
be/src/service/internal_service.cpp | 18 ++++++++++++++----
3 files changed, 20 insertions(+), 12 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 74af2f754a..d7248e0f0f 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -454,14 +454,10 @@ DEFINE_Int64(load_data_reserve_hours, "4");
// log error log will be removed after this time
DEFINE_mInt64(load_error_log_reserve_hours, "48");
-// be brpc interface is classified into two categories: light and heavy
-// each category has diffrent thread number
-// threads to handle heavy api interface, such as transmit_data/transmit_block
etc
-DEFINE_Int32(brpc_heavy_work_pool_threads, "128");
-// threads to handle light api interface, such as
exec_plan_fragment_prepare/exec_plan_fragment_start
-DEFINE_Int32(brpc_light_work_pool_threads, "128");
-DEFINE_Int32(brpc_heavy_work_pool_max_queue_size, "10240");
-DEFINE_Int32(brpc_light_work_pool_max_queue_size, "10240");
+DEFINE_Int32(brpc_heavy_work_pool_threads, "-1");
+DEFINE_Int32(brpc_light_work_pool_threads, "-1");
+DEFINE_Int32(brpc_heavy_work_pool_max_queue_size, "-1");
+DEFINE_Int32(brpc_light_work_pool_max_queue_size, "-1");
// The maximum amount of data that can be processed by a stream load
DEFINE_mInt64(streaming_load_max_mb, "10240");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index c6e54d45b7..d60d816a99 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -498,6 +498,8 @@ DECLARE_mInt64(load_error_log_reserve_hours);
// be brpc interface is classified into two categories: light and heavy
// each category has diffrent thread number
// threads to handle heavy api interface, such as transmit_data/transmit_block
etc
+// Default, if less than or equal 32 core, the following are 128, 128, 10240,
10240 in turn.
+// if greater than 32 core, the following are core num * 4, core num
* 4, core num * 320, core num * 320 in turn
DECLARE_Int32(brpc_heavy_work_pool_threads);
// threads to handle light api interface, such as
exec_plan_fragment_prepare/exec_plan_fragment_start
DECLARE_Int32(brpc_light_work_pool_threads);
diff --git a/be/src/service/internal_service.cpp
b/be/src/service/internal_service.cpp
index 3a22919ea2..528331ee12 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -170,10 +170,20 @@ private:
PInternalServiceImpl::PInternalServiceImpl(ExecEnv* exec_env)
: _exec_env(exec_env),
- _heavy_work_pool(config::brpc_heavy_work_pool_threads,
- config::brpc_heavy_work_pool_max_queue_size,
"brpc_heavy"),
- _light_work_pool(config::brpc_light_work_pool_threads,
- config::brpc_light_work_pool_max_queue_size,
"brpc_light") {
+ _heavy_work_pool(config::brpc_heavy_work_pool_threads != -1
+ ? config::brpc_heavy_work_pool_threads
+ : std::max(128, CpuInfo::num_cores() * 4),
+ config::brpc_heavy_work_pool_max_queue_size != -1
+ ?
config::brpc_heavy_work_pool_max_queue_size
+ : std::max(10240, CpuInfo::num_cores() *
320),
+ "brpc_heavy"),
+ _light_work_pool(config::brpc_light_work_pool_threads != -1
+ ? config::brpc_light_work_pool_threads
+ : std::max(128, CpuInfo::num_cores() * 4),
+ config::brpc_light_work_pool_max_queue_size != -1
+ ?
config::brpc_light_work_pool_max_queue_size
+ : std::max(10240, CpuInfo::num_cores() *
320),
+ "brpc_light") {
REGISTER_HOOK_METRIC(heavy_work_pool_queue_size,
[this]() { return _heavy_work_pool.get_queue_size();
});
REGISTER_HOOK_METRIC(light_work_pool_queue_size,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
