This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 86d97213200 [improvement](segmentcache) limit segment cache by fd
limit or memory (#39658)
86d97213200 is described below
commit 86d97213200549addf08f60e6f3e2ccaf68c0d61
Author: Yongqiang YANG <[email protected]>
AuthorDate: Wed Aug 21 12:15:41 2024 +0800
[improvement](segmentcache) limit segment cache by fd limit or memory
(#39658)
remove a useless config.
---
be/src/common/config.cpp | 7 ++++---
be/src/common/config.h | 6 +++---
be/src/runtime/exec_env_init.cpp | 15 ++++++---------
3 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 78805a58ac6..458c124c304 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -40,6 +40,7 @@
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
+#include "config.h"
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"
#include "util/cpu_info.h"
@@ -1067,10 +1068,10 @@ DEFINE_mInt32(schema_cache_capacity, "1024");
DEFINE_mInt32(schema_cache_sweep_time_sec, "100");
// max number of segment cache, default -1 for backward compatibility
fd_number*2/5
-DEFINE_mInt32(segment_cache_capacity, "-1");
-DEFINE_mInt32(estimated_num_columns_per_segment, "200");
+DEFINE_Int32(segment_cache_capacity, "-1");
+DEFINE_Int32(segment_cache_fd_percentage, "40");
DEFINE_mInt32(estimated_mem_per_column_reader, "1024");
-DEFINE_mInt32(segment_cache_memory_percentage, "2");
+DEFINE_Int32(segment_cache_memory_percentage, "2");
// enable feature binlog, default false
DEFINE_Bool(enable_feature_binlog, "false");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index aeaeea30ae8..2449826936f 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1120,10 +1120,10 @@ DECLARE_mInt32(schema_cache_capacity);
DECLARE_mInt32(schema_cache_sweep_time_sec);
// max number of segment cache
-DECLARE_mInt32(segment_cache_capacity);
-DECLARE_mInt32(estimated_num_columns_per_segment);
-DECLARE_mInt32(estimated_mem_per_column_reader);
+DECLARE_Int32(segment_cache_capacity);
+DECLARE_Int32(segment_cache_fd_percentage);
DECLARE_Int32(segment_cache_memory_percentage);
+DECLARE_mInt32(estimated_mem_per_column_reader);
// enable binlog
DECLARE_Bool(enable_feature_binlog);
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 37ac346198a..53fe1993139 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -518,21 +518,18 @@ Status ExecEnv::_init_mem_env() {
// SegmentLoader caches segments in rowset granularity. So the size of
// opened files will greater than segment_cache_capacity.
int64_t segment_cache_capacity = config::segment_cache_capacity;
- if (segment_cache_capacity < 0 || segment_cache_capacity > fd_number * 1 /
5) {
- segment_cache_capacity = fd_number * 1 / 5;
+ int64_t segment_cache_fd_limit = fd_number / 100 *
config::segment_cache_fd_percentage;
+ if (segment_cache_capacity < 0 || segment_cache_capacity >
segment_cache_fd_limit) {
+ segment_cache_capacity = segment_cache_fd_limit;
}
int64_t segment_cache_mem_limit =
MemInfo::mem_limit() / 100 *
config::segment_cache_memory_percentage;
- // config::segment_cache_memory_percentage;
- int64_t min_segment_cache_mem_limit =
- min(segment_cache_mem_limit, segment_cache_capacity *
-
config::estimated_num_columns_per_segment *
-
config::estimated_mem_per_column_reader);
- _segment_loader = new SegmentLoader(min_segment_cache_mem_limit,
segment_cache_capacity);
+
+ _segment_loader = new SegmentLoader(segment_cache_mem_limit,
segment_cache_capacity);
LOG(INFO) << "segment_cache_capacity <= fd_number * 1 / 5, fd_number: " <<
fd_number
<< " segment_cache_capacity: " << segment_cache_capacity
- << " min_segment_cache_mem_limit " <<
min_segment_cache_mem_limit;
+ << " min_segment_cache_mem_limit " << segment_cache_mem_limit;
_schema_cache = new SchemaCache(config::schema_cache_capacity);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
