This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new bd45f01a8af [opt](load) avoid over-flushing active memtables in memory
limiter (#52906) (#53909)
bd45f01a8af is described below
commit bd45f01a8af94a3d072086f29b0c930c4508a24a
Author: Kaijie Chen <[email protected]>
AuthorDate: Wed Aug 13 11:02:30 2025 +0800
[opt](load) avoid over-flushing active memtables in memory limiter (#52906)
(#53909)
backport #52906
---
be/src/olap/memtable_memory_limiter.cpp | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/memtable_memory_limiter.cpp
b/be/src/olap/memtable_memory_limiter.cpp
index dba30d042d7..35700aa36d6 100644
--- a/be/src/olap/memtable_memory_limiter.cpp
+++ b/be/src/olap/memtable_memory_limiter.cpp
@@ -39,6 +39,8 @@ bvar::Status<int64_t>
g_memtable_flush_memory("mm_limiter_mem_flush", 0);
bvar::Status<int64_t> g_memtable_load_memory("mm_limiter_mem_load", 0);
bvar::Status<int64_t> g_load_hard_mem_limit("mm_limiter_limit_hard", 0);
bvar::Status<int64_t> g_load_soft_mem_limit("mm_limiter_limit_soft", 0);
+bvar::Adder<int>
g_memtable_memory_limit_flush_memtable_count("mm_limiter_flush_memtable_count");
+bvar::LatencyRecorder
g_memtable_memory_limit_flush_size_bytes("mm_limiter_flush_size_bytes");
// Calculate the total memory limit of all load tasks on this BE
static int64_t calc_process_max_load_memory(int64_t process_mem_limit) {
@@ -113,7 +115,7 @@ int64_t MemTableMemoryLimiter::_need_flush() {
int64_t limit2 = _sys_avail_mem_less_than_warning_water_mark();
int64_t limit3 = _process_used_mem_more_than_soft_mem_limit();
int64_t need_flush = std::max(limit1, std::max(limit2, limit3));
- return need_flush - _queue_mem_usage;
+ return need_flush - _queue_mem_usage - _flush_mem_usage;
}
void MemTableMemoryLimiter::handle_memtable_flush() {
@@ -150,7 +152,8 @@ void MemTableMemoryLimiter::handle_memtable_flush() {
<< ", memtable writers num: " << _writers.size()
<< ", active: " <<
PrettyPrinter::print_bytes(_active_mem_usage)
<< ", queue: " <<
PrettyPrinter::print_bytes(_queue_mem_usage)
- << ", flush: " <<
PrettyPrinter::print_bytes(_flush_mem_usage);
+ << ", flush: " <<
PrettyPrinter::print_bytes(_flush_mem_usage)
+ << ", need flush: " <<
PrettyPrinter::print_bytes(need_flush);
_flush_active_memtables(need_flush);
}
} while (_hard_limit_reached() && !_load_usage_low());
@@ -219,6 +222,8 @@ void MemTableMemoryLimiter::_flush_active_memtables(int64_t
need_flush) {
}
mem_flushed += mem;
num_flushed += (mem > 0);
+ g_memtable_memory_limit_flush_memtable_count << 1;
+ g_memtable_memory_limit_flush_size_bytes << mem;
}
LOG(INFO) << "flushed " << num_flushed << " out of " <<
_active_writers.size()
<< " active writers, flushed size: " <<
PrettyPrinter::print_bytes(mem_flushed);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
