This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1253ed006e1 [fix](memtable-limiter) do not block write if load mem
usage is low (#28602)
1253ed006e1 is described below
commit 1253ed006e182c6e6ae4c27a1fd9acdd1a07286c
Author: Kaijie Chen <[email protected]>
AuthorDate: Tue Dec 19 13:28:17 2023 +0800
[fix](memtable-limiter) do not block write if load mem usage is low (#28602)
Co-authored-by: Yongqiang YANG
<[email protected]>
---
be/src/common/config.cpp | 4 ++++
be/src/common/config.h | 4 ++++
be/src/olap/memtable_memory_limiter.cpp | 10 ++++++++--
be/src/olap/memtable_memory_limiter.h | 2 ++
4 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 549404e80ff..e75ac2c1c75 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -602,6 +602,10 @@ DEFINE_Int32(load_process_max_memory_limit_percent, "50");
// 50%
// might avoid all load jobs hang at the same time.
DEFINE_Int32(load_process_soft_mem_limit_percent, "80");
+// If load memory consumption is within load_process_safe_mem_permit_percent,
+// memtable memory limiter will do nothing.
+DEFINE_Int32(load_process_safe_mem_permit_percent, "5");
+
// result buffer cancelled time (unit: second)
DEFINE_mInt32(result_buffer_cancelled_interval_time, "300");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index fb5943ec9fd..c911963a87f 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -661,6 +661,10 @@ DECLARE_Int32(load_process_max_memory_limit_percent); //
50%
// might avoid all load jobs hang at the same time.
DECLARE_Int32(load_process_soft_mem_limit_percent);
+// If load memory consumption is within load_process_safe_mem_permit_percent,
+// memtable memory limiter will do nothing.
+DECLARE_Int32(load_process_safe_mem_permit_percent);
+
// result buffer cancelled time (unit: second)
DECLARE_mInt32(result_buffer_cancelled_interval_time);
diff --git a/be/src/olap/memtable_memory_limiter.cpp
b/be/src/olap/memtable_memory_limiter.cpp
index 6d5ee2f3f76..4b4298853a3 100644
--- a/be/src/olap/memtable_memory_limiter.cpp
+++ b/be/src/olap/memtable_memory_limiter.cpp
@@ -59,6 +59,8 @@ MemTableMemoryLimiter::~MemTableMemoryLimiter() {
Status MemTableMemoryLimiter::init(int64_t process_mem_limit) {
_load_hard_mem_limit = calc_process_max_load_memory(process_mem_limit);
_load_soft_mem_limit = _load_hard_mem_limit *
config::load_process_soft_mem_limit_percent / 100;
+ _load_safe_mem_permit =
+ _load_hard_mem_limit *
config::load_process_safe_mem_permit_percent / 100;
g_load_hard_mem_limit.set_value(_load_hard_mem_limit);
g_load_soft_mem_limit.set_value(_load_soft_mem_limit);
_mem_tracker =
std::make_unique<MemTrackerLimiter>(MemTrackerLimiter::Type::LOAD,
@@ -97,10 +99,14 @@ bool MemTableMemoryLimiter::_hard_limit_reached() {
_proc_mem_extra() >= 0;
}
+bool MemTableMemoryLimiter::_load_usage_low() {
+ return _mem_tracker->consumption() <= _load_safe_mem_permit;
+}
+
void MemTableMemoryLimiter::handle_memtable_flush() {
// Check the soft limit.
DCHECK(_load_soft_mem_limit > 0);
- if (!_soft_limit_reached()) {
+ if (!_soft_limit_reached() || _load_usage_low()) {
return;
}
MonotonicStopWatch timer;
@@ -243,4 +249,4 @@ void MemTableMemoryLimiter::_refresh_mem_tracker() {
}
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/memtable_memory_limiter.h
b/be/src/olap/memtable_memory_limiter.h
index f1ed6418128..7b91c9a9089 100644
--- a/be/src/olap/memtable_memory_limiter.h
+++ b/be/src/olap/memtable_memory_limiter.h
@@ -54,6 +54,7 @@ private:
bool _soft_limit_reached();
bool _hard_limit_reached();
+ bool _load_usage_low();
void _flush_active_memtables(int64_t need_flush);
int64_t _flush_memtable(std::weak_ptr<MemTableWriter> writer_to_flush,
int64_t threshold);
void _refresh_mem_tracker();
@@ -68,6 +69,7 @@ private:
std::unique_ptr<MemTrackerLimiter> _mem_tracker;
int64_t _load_hard_mem_limit = -1;
int64_t _load_soft_mem_limit = -1;
+ int64_t _load_safe_mem_permit = -1;
std::vector<std::weak_ptr<MemTableWriter>> _writers;
std::vector<std::weak_ptr<MemTableWriter>> _active_writers;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
