This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit f96616af86bb2a6daf1f8aefa39ec0ce666870ef Author: Xinyi Zou <[email protected]> AuthorDate: Tue Aug 1 23:09:49 2023 +0800 [opt](conf) remote scanner thread num is changed to core num * 10 (#22427) --- be/src/common/config.cpp | 3 +-- be/src/common/config.h | 1 + be/src/vec/exec/scan/scanner_scheduler.cpp | 8 ++++++-- bin/start_be.sh | 2 +- conf/be.conf | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 912fae1b5c..10373e42ee 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -227,8 +227,7 @@ DEFINE_mInt64(doris_blocking_priority_queue_wait_timeout_ms, "500"); // number of scanner thread pool size for olap table // and the min thread num of remote scanner thread pool DEFINE_Int32(doris_scanner_thread_pool_thread_num, "48"); -// max number of remote scanner thread pool size -DEFINE_Int32(doris_max_remote_scanner_thread_pool_thread_num, "512"); +DEFINE_Int32(doris_max_remote_scanner_thread_pool_thread_num, "-1"); // number of olap scanner thread pool queue size DEFINE_Int32(doris_scanner_thread_pool_queue_size, "102400"); // default thrift client connect timeout(in seconds) diff --git a/be/src/common/config.h b/be/src/common/config.h index c38badffe7..e5f3e59c52 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -270,6 +270,7 @@ DECLARE_mInt64(doris_blocking_priority_queue_wait_timeout_ms); // and the min thread num of remote scanner thread pool DECLARE_Int32(doris_scanner_thread_pool_thread_num); // max number of remote scanner thread pool size +// if equal to -1, value is std::max(512, CpuInfo::num_cores() * 10) DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num); // number of olap scanner thread pool queue size DECLARE_Int32(doris_scanner_thread_pool_queue_size); diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp index aaa7e1e69c..2cc228bedb 100644 --- a/be/src/vec/exec/scan/scanner_scheduler.cpp +++ b/be/src/vec/exec/scan/scanner_scheduler.cpp @@ -38,6 +38,8 @@ #include "runtime/thread_context.h" #include "util/async_io.h" // IWYU pragma: keep #include "util/blocking_queue.hpp" +#include "util/cpu_info.h" +#include "util/defer_op.h" #include "util/priority_thread_pool.hpp" #include "util/priority_work_stealing_thread_pool.hpp" #include "util/thread.h" @@ -97,8 +99,10 @@ Status ScannerScheduler::init(ExecEnv* env) { // 3. remote scan thread pool ThreadPoolBuilder("RemoteScanThreadPool") - .set_min_threads(config::doris_scanner_thread_pool_thread_num) // 48 default - .set_max_threads(config::doris_max_remote_scanner_thread_pool_thread_num) // 512 default + .set_min_threads(config::doris_scanner_thread_pool_thread_num) // 48 default + .set_max_threads(config::doris_max_remote_scanner_thread_pool_thread_num != -1 + ? config::doris_max_remote_scanner_thread_pool_thread_num + : std::max(512, CpuInfo::num_cores() * 10)) .set_max_queue_size(config::doris_scanner_thread_pool_queue_size) .build(&_remote_scan_thread_pool); diff --git a/bin/start_be.sh b/bin/start_be.sh index 07e99f59ff..6c321dd0f4 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -304,7 +304,7 @@ export LIBHDFS_OPTS="${final_java_opt}" #echo "LIBHDFS_OPTS: ${LIBHDFS_OPTS}" if [[ -z ${JEMALLOC_CONF} ]]; then - JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false,confirm_conf:true" + JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false" fi if [[ -z ${JEMALLOC_PROF_PRFIX} ]]; then diff --git a/conf/be.conf b/conf/be.conf index d5be2042e2..3f560deda8 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -29,7 +29,7 @@ JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS # https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile # https://jemalloc.net/jemalloc.3.html -JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false,confirm_conf:true" +JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false" JEMALLOC_PROF_PRFIX="" # INFO, WARNING, ERROR, FATAL --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
