This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 02802432e91 branch-3.1: [opt](load) avoid over-flushing active
memtables in memory limiter #52906 (#53910)
02802432e91 is described below
commit 02802432e91593a52343de1cca61d0ad9f2f2637
Author: Kaijie Chen <[email protected]>
AuthorDate: Mon Jul 28 14:13:38 2025 +0800
branch-3.1: [opt](load) avoid over-flushing active memtables in memory
limiter #52906 (#53910)
backport #52906
---
be/src/olap/memtable_memory_limiter.cpp | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/memtable_memory_limiter.cpp
b/be/src/olap/memtable_memory_limiter.cpp
index 1f59f8fc341..81a0fdd316b 100644
--- a/be/src/olap/memtable_memory_limiter.cpp
+++ b/be/src/olap/memtable_memory_limiter.cpp
@@ -39,6 +39,8 @@ bvar::Status<int64_t>
g_memtable_flush_memory("mm_limiter_mem_flush", 0);
bvar::Status<int64_t> g_memtable_load_memory("mm_limiter_mem_load", 0);
bvar::Status<int64_t> g_load_hard_mem_limit("mm_limiter_limit_hard", 0);
bvar::Status<int64_t> g_load_soft_mem_limit("mm_limiter_limit_soft", 0);
+bvar::Adder<int>
g_memtable_memory_limit_flush_memtable_count("mm_limiter_flush_memtable_count");
+bvar::LatencyRecorder
g_memtable_memory_limit_flush_size_bytes("mm_limiter_flush_size_bytes");
// Calculate the total memory limit of all load tasks on this BE
static int64_t calc_process_max_load_memory(int64_t process_mem_limit) {
@@ -113,7 +115,7 @@ int64_t MemTableMemoryLimiter::_need_flush() {
int64_t limit2 = _sys_avail_mem_less_than_warning_water_mark();
int64_t limit3 = _process_used_mem_more_than_soft_mem_limit();
int64_t need_flush = std::max(limit1, std::max(limit2, limit3));
- return need_flush - _queue_mem_usage;
+ return need_flush - _queue_mem_usage - _flush_mem_usage;
}
void MemTableMemoryLimiter::handle_memtable_flush() {
@@ -150,7 +152,8 @@ void MemTableMemoryLimiter::handle_memtable_flush() {
<< ", memtable writers num: " << _writers.size()
<< ", active: " <<
PrettyPrinter::print_bytes(_active_mem_usage)
<< ", queue: " <<
PrettyPrinter::print_bytes(_queue_mem_usage)
- << ", flush: " <<
PrettyPrinter::print_bytes(_flush_mem_usage);
+ << ", flush: " <<
PrettyPrinter::print_bytes(_flush_mem_usage)
+ << ", need flush: " <<
PrettyPrinter::print_bytes(need_flush);
_flush_active_memtables(need_flush);
}
} while (_hard_limit_reached());
@@ -209,6 +212,8 @@ void MemTableMemoryLimiter::_flush_active_memtables(int64_t
need_flush) {
}
mem_flushed += mem;
num_flushed += (mem > 0);
+ g_memtable_memory_limit_flush_memtable_count << 1;
+ g_memtable_memory_limit_flush_size_bytes << mem;
}
LOG(INFO) << "flushed " << num_flushed << " out of " <<
_active_writers.size()
<< " active writers, flushed size: " <<
PrettyPrinter::print_bytes(mem_flushed);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
