This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new bc87002028 [opt](conf) remote scanner thread num is changed to core
num * 10 (#22427)
bc87002028 is described below
commit bc870020287719b9f35f6e031603492b03cd5100
Author: Xinyi Zou <[email protected]>
AuthorDate: Tue Aug 1 23:09:49 2023 +0800
[opt](conf) remote scanner thread num is changed to core num * 10 (#22427)
---
be/src/common/config.cpp | 3 +--
be/src/common/config.h | 1 +
be/src/vec/exec/scan/scanner_scheduler.cpp | 7 +++++--
bin/start_be.sh | 2 +-
conf/be.conf | 2 +-
5 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 8cf6d75c4e..b3abc7fdfb 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -231,8 +231,7 @@
DEFINE_mInt64(doris_blocking_priority_queue_wait_timeout_ms, "500");
// number of scanner thread pool size for olap table
// and the min thread num of remote scanner thread pool
DEFINE_Int32(doris_scanner_thread_pool_thread_num, "48");
-// max number of remote scanner thread pool size
-DEFINE_Int32(doris_max_remote_scanner_thread_pool_thread_num, "512");
+DEFINE_Int32(doris_max_remote_scanner_thread_pool_thread_num, "-1");
// number of olap scanner thread pool queue size
DEFINE_Int32(doris_scanner_thread_pool_queue_size, "102400");
// default thrift client connect timeout(in seconds)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 097c1f02bc..f3527c2947 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -275,6 +275,7 @@
DECLARE_mInt64(doris_blocking_priority_queue_wait_timeout_ms);
// and the min thread num of remote scanner thread pool
DECLARE_Int32(doris_scanner_thread_pool_thread_num);
// max number of remote scanner thread pool size
+// if equal to -1, value is std::max(512, CpuInfo::num_cores() * 10)
DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num);
// number of olap scanner thread pool queue size
DECLARE_Int32(doris_scanner_thread_pool_queue_size);
diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp
b/be/src/vec/exec/scan/scanner_scheduler.cpp
index 1f8e2051bf..e72a01c952 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -39,6 +39,7 @@
#include "scan_task_queue.h"
#include "util/async_io.h" // IWYU pragma: keep
#include "util/blocking_queue.hpp"
+#include "util/cpu_info.h"
#include "util/defer_op.h"
#include "util/priority_thread_pool.hpp"
#include "util/priority_work_stealing_thread_pool.hpp"
@@ -106,8 +107,10 @@ Status ScannerScheduler::init(ExecEnv* env) {
// 3. remote scan thread pool
ThreadPoolBuilder("RemoteScanThreadPool")
- .set_min_threads(config::doris_scanner_thread_pool_thread_num)
// 48 default
-
.set_max_threads(config::doris_max_remote_scanner_thread_pool_thread_num) //
512 default
+ .set_min_threads(config::doris_scanner_thread_pool_thread_num) //
48 default
+
.set_max_threads(config::doris_max_remote_scanner_thread_pool_thread_num != -1
+ ?
config::doris_max_remote_scanner_thread_pool_thread_num
+ : std::max(512, CpuInfo::num_cores() *
10))
.set_max_queue_size(config::doris_scanner_thread_pool_queue_size)
.build(&_remote_scan_thread_pool);
diff --git a/bin/start_be.sh b/bin/start_be.sh
index 07e99f59ff..6c321dd0f4 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -304,7 +304,7 @@ export LIBHDFS_OPTS="${final_java_opt}"
#echo "LIBHDFS_OPTS: ${LIBHDFS_OPTS}"
if [[ -z ${JEMALLOC_CONF} ]]; then
-
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false,confirm_conf:true"
+
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
fi
if [[ -z ${JEMALLOC_PROF_PRFIX} ]]; then
diff --git a/conf/be.conf b/conf/be.conf
index 5041b901f2..71dc62fd0f 100644
--- a/conf/be.conf
+++ b/conf/be.conf
@@ -29,7 +29,7 @@ JAVA_OPTS_FOR_JDK_9="-Xmx1024m
-DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS
#
https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
# https://jemalloc.net/jemalloc.3.html
-JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false,confirm_conf:true"
+JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
JEMALLOC_PROF_PRFIX=""
# INFO, WARNING, ERROR, FATAL
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
