This is an automated email from the ASF dual-hosted git repository. panxiaolei pushed a commit to branch dev_syxj_2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 88aa7928bc17e0904c1ff61df9b0bef5c23a4d94 Author: morningman <[email protected]> AuthorDate: Thu Aug 17 19:46:43 2023 +0800 [tmp] add config max_scanner_for_file_scan_node to limit the concurrency of file scanner --- be/src/common/config.cpp | 2 ++ be/src/common/config.h | 2 ++ be/src/vec/exec/scan/new_file_scan_node.cpp | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index b535273ae3..4b33ab5db3 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1037,6 +1037,8 @@ DEFINE_Bool(enable_hdfs_hedged_read, "false"); DEFINE_Int32(hdfs_hedged_read_thread_num, "128"); DEFINE_Int32(hdfs_hedged_read_threshold_time, "500"); +DEFINE_mInt32(max_scanner_for_file_scan_node, "16"); + #ifdef BE_TEST // test s3 DEFINE_String(test_s3_resource, "resource"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 61df537188..a3e6b5522e 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1085,6 +1085,8 @@ DECLARE_Int32(hdfs_hedged_read_thread_num); // Maybe overwritten by the value specified when creating catalog DECLARE_Int32(hdfs_hedged_read_threshold_time); +DECLARE_mInt32(max_scanner_for_file_scan_node); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/vec/exec/scan/new_file_scan_node.cpp b/be/src/vec/exec/scan/new_file_scan_node.cpp index 5eb8d7e1c6..8f0fcbacd4 100644 --- a/be/src/vec/exec/scan/new_file_scan_node.cpp +++ b/be/src/vec/exec/scan/new_file_scan_node.cpp @@ -60,7 +60,7 @@ Status NewFileScanNode::prepare(RuntimeState* state) { } void NewFileScanNode::set_scan_ranges(const std::vector<TScanRangeParams>& scan_ranges) { - int max_scanners = config::doris_scanner_thread_pool_thread_num; + int max_scanners = config::max_scanner_for_file_scan_node; if (scan_ranges.size() <= max_scanners) { _scan_ranges = scan_ranges; } else { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
