This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch opt_memtable_speed
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/opt_memtable_speed by this
push:
new f1fcba3162 [cherry-pick](conf) add be conf for partition topn
partitions threshold (#23233)
f1fcba3162 is described below
commit f1fcba3162a8ca1af8bae7b3fe13bc991c3dcd54
Author: zhangstar333 <[email protected]>
AuthorDate: Mon Aug 21 11:25:58 2023 +0800
[cherry-pick](conf) add be conf for partition topn partitions threshold
(#23233)
[cherry-pick](conf) add be conf for partition topn partitions threshold
(#23233)
pick from master #23220
---
be/src/common/config.cpp | 1 +
be/src/common/config.h | 3 +++
be/src/vec/exec/vpartition_sort_node.cpp | 3 ++-
3 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index b27b8932f2..28e7464a2a 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1067,6 +1067,7 @@ DEFINE_mBool(enable_merge_on_write_correctness_check,
"true");
DEFINE_mString(user_files_secure_path, "${DORIS_HOME}");
DEFINE_mBool(sync_dirty_data_on_close, "true");
+DEFINE_Int32(partition_topn_partition_threshold, "1024");
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index daa6519477..7e498896aa 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1124,6 +1124,9 @@ DECLARE_mBool(enable_merge_on_write_correctness_check);
DECLARE_mString(user_files_secure_path);
DECLARE_mBool(sync_dirty_data_on_close);
+// This threshold determines how many partitions will be allocated for window
function get topn.
+// and if this threshold is exceeded, the remaining data will be pass through
to other node directly.
+DECLARE_Int32(partition_topn_partition_threshold);
#ifdef BE_TEST
// test s3
diff --git a/be/src/vec/exec/vpartition_sort_node.cpp
b/be/src/vec/exec/vpartition_sort_node.cpp
index eb807ad4b4..3815d8c47c 100644
--- a/be/src/vec/exec/vpartition_sort_node.cpp
+++ b/be/src/vec/exec/vpartition_sort_node.cpp
@@ -181,7 +181,8 @@ Status VPartitionSortNode::sink(RuntimeState* state,
vectorized::Block* input_bl
_value_places[0]->append_whole_block(input_block,
child(0)->row_desc());
} else {
//just simply use partition num to check
- if (_num_partition > 512 && child_input_rows < 10000 *
_num_partition) {
+ if (_num_partition > config::partition_topn_partition_threshold &&
+ child_input_rows < 10000 * _num_partition) {
{
std::lock_guard<std::mutex> lock(_buffer_mutex);
_blocks_buffer.push(std::move(*input_block));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
