This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 22157077e9 [fix](memtracker) Optimize the return msg of process memory
limit exceed #12086
22157077e9 is described below
commit 22157077e97e95e8c3dd879919a103f2e8ee8369
Author: Xinyi Zou <[email protected]>
AuthorDate: Fri Aug 26 14:28:46 2022 +0800
[fix](memtracker) Optimize the return msg of process memory limit exceed
#12086
Return the real process memory information when the process exceeds mem
limit
Optimize the memory exceed limit log printing logic
process tracker does not participate in process memory limit.
---
be/src/runtime/exec_env_init.cpp | 3 +-
be/src/runtime/memory/mem_tracker_limiter.cpp | 82 +++++++++++++++++++--------
be/src/runtime/memory/mem_tracker_limiter.h | 34 ++++++-----
be/src/runtime/runtime_filter_mgr.cpp | 2 +-
4 files changed, 81 insertions(+), 40 deletions(-)
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 22495aeaf3..0fab11f3ee 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -203,7 +203,8 @@ Status ExecEnv::_init_mem_tracker() {
<< ". Using physical memory instead";
global_memory_limit_bytes = MemInfo::physical_mem();
}
- _process_mem_tracker = std::make_shared<MemTrackerLimiter>(-1, "Process");
+ _process_mem_tracker =
+ std::make_shared<MemTrackerLimiter>(global_memory_limit_bytes,
"Process");
thread_context()->_thread_mem_tracker_mgr->init();
thread_context()->_thread_mem_tracker_mgr->set_check_attach(false);
#if defined(USE_MEM_TRACKER) && !defined(__SANITIZE_ADDRESS__) &&
!defined(ADDRESS_SANITIZER) && \
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp
b/be/src/runtime/memory/mem_tracker_limiter.cpp
index 410ff12d19..32061c8ad9 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -149,8 +149,9 @@ bool MemTrackerLimiter::gc_memory(int64_t max_consumption) {
Status MemTrackerLimiter::try_gc_memory(int64_t bytes) {
if (UNLIKELY(gc_memory(_limit - bytes))) {
return Status::MemoryLimitExceeded(fmt::format(
- "need_size={}, exceeded_tracker={}, limit={}, peak_used={},
current_used={}", bytes,
- label(), _limit, _consumption->value(),
_consumption->current_value()));
+ "failed_alloc_size={}Bytes, exceeded_tracker={}, limit={}B,
peak_used={}B, "
+ "current_used={}B",
+ bytes, label(), _limit, _consumption->value(),
_consumption->current_value()));
}
VLOG_NOTICE << "GC succeeded, TryConsume bytes=" << bytes
<< " consumption=" << _consumption->current_value() << "
limit=" << _limit;
@@ -223,22 +224,26 @@ std::string MemTrackerLimiter::log_usage(int
max_recursive_depth,
return join(usage_strings, "\n");
}
-Status MemTrackerLimiter::mem_limit_exceeded_log(const std::string& msg) {
- DCHECK(_limit != -1);
+Status MemTrackerLimiter::mem_limit_exceeded_construct(const std::string& msg)
{
std::string detail = fmt::format(
- "{}, backend={} memory used={}, free memory left={}. If is query,
can change the limit "
+ "{}, backend {} process memory used {}Bytes, process limit {}B. If
is query, can "
+ "change the limit "
"by `set exec_mem_limit=xxx`, details mem usage see be.INFO.",
msg, BackendOptions::get_localhost(),
PrettyPrinter::print(PerfCounters::get_vm_rss(), TUnit::BYTES),
- PrettyPrinter::print(MemInfo::mem_limit() -
PerfCounters::get_vm_rss(), TUnit::BYTES));
- Status status = Status::MemoryLimitExceeded(detail);
+ PrettyPrinter::print(MemInfo::mem_limit(), TUnit::BYTES));
+ return Status::MemoryLimitExceeded(detail);
+}
+void MemTrackerLimiter::print_log_usage(const std::string& msg) {
+ DCHECK(_limit != -1);
// only print the tracker log_usage in be log.
+ std::string detail = msg;
if (_print_log_usage) {
if (_label == "Process") {
// Dumping the process MemTracker is expensive. Limiting the
recursive depth to two
// levels limits the level of detail to a one-line summary for
each query MemTracker.
- detail += "\n" +
ExecEnv::GetInstance()->process_mem_tracker()->log_usage(2);
+ detail += "\n" + log_usage(2);
} else {
detail += "\n" + log_usage();
}
@@ -246,37 +251,66 @@ Status MemTrackerLimiter::mem_limit_exceeded_log(const
std::string& msg) {
LOG(WARNING) << detail;
_print_log_usage = false;
}
- return status;
}
Status MemTrackerLimiter::mem_limit_exceeded(const std::string& msg,
int64_t failed_allocation_size) {
STOP_CHECK_THREAD_MEM_TRACKER_LIMIT();
DCHECK(!_limited_ancestors.empty());
- std::string detail = fmt::format("memory limit
exceeded:<consumed_tracker={}, ", _label);
+ std::string detail = fmt::format("Memory limit exceeded,
<consuming_tracker={}, ", _label);
if (failed_allocation_size != 0)
- detail += fmt::format("need_size={}, ",
+ detail += fmt::format("failed_alloc_size={}Bytes, ",
PrettyPrinter::print(failed_allocation_size,
TUnit::BYTES));
- MemTrackerLimiter* exceeded_tracker = this;
- int64_t free_size = INT_MAX;
+ MemTrackerLimiter* exceeded_tracker = nullptr;
+ MemTrackerLimiter* max_consumption_tracker = nullptr;
+ int64_t free_size = INT64_MAX;
+ // Find the tracker that exceed limit and has the least free.
for (const auto& tracker : _limited_ancestors) {
+ if (tracker->label() == "Process") continue;
int64_t max_consumption = tracker->peak_consumption() >
tracker->consumption()
? tracker->peak_consumption()
: tracker->consumption();
- if (tracker->has_limit() && tracker->limit() < max_consumption +
failed_allocation_size) {
+ if (tracker->limit() < max_consumption + failed_allocation_size) {
exceeded_tracker = tracker;
break;
}
- if (tracker->has_limit() && tracker->limit() - max_consumption <
free_size) {
+ if (tracker->limit() - max_consumption < free_size) {
free_size = tracker->limit() - max_consumption;
- exceeded_tracker = tracker;
+ max_consumption_tracker = tracker;
}
}
- detail += fmt::format(
- "exceeded_tracker={}, limit={}, peak_used={}, current_used={}>,
executing_msg:<{}>",
- exceeded_tracker->label(), exceeded_tracker->limit(),
- exceeded_tracker->peak_consumption(),
exceeded_tracker->consumption(), msg);
- return exceeded_tracker->mem_limit_exceeded_log(detail);
+
+ auto sys_exceed_st = check_sys_mem_info(failed_allocation_size);
+ MemTrackerLimiter* print_log_usage_tracker = nullptr;
+ if (exceeded_tracker != nullptr) {
+ detail += fmt::format(
+ "exceeded_tracker={}, limit={}B, peak_used={}B,
current_used={B}>, "
+ "executing_msg:<{}>",
+ exceeded_tracker->label(), exceeded_tracker->limit(),
+ exceeded_tracker->peak_consumption(),
exceeded_tracker->consumption(), msg);
+ print_log_usage_tracker = exceeded_tracker;
+ } else if (!sys_exceed_st) {
+ detail = fmt::format("Memory limit exceeded, {}, executing_msg:<{}>",
+ sys_exceed_st.get_error_msg(), msg);
+ } else if (max_consumption_tracker != nullptr) {
+ // must after check_sys_mem_info false
+ detail += fmt::format(
+ "max_consumption_tracker={}, limit={}B, peak_used={}B,
current_used={}B>, "
+ "executing_msg:<{}>",
+ max_consumption_tracker->label(),
max_consumption_tracker->limit(),
+ max_consumption_tracker->peak_consumption(),
max_consumption_tracker->consumption(),
+ msg);
+ print_log_usage_tracker = max_consumption_tracker;
+ } else {
+ // The limit of the current tracker and parents is less than 0, the
consume will not fail,
+ // and the current process memory has no excess limit.
+ detail += fmt::format("unknown exceed reason, executing_msg:<{}>",
msg);
+ print_log_usage_tracker =
ExecEnv::GetInstance()->process_mem_tracker().get();
+ }
+ auto st = MemTrackerLimiter::mem_limit_exceeded_construct(detail);
+ if (print_log_usage_tracker != nullptr)
+ print_log_usage_tracker->print_log_usage(st.get_error_msg());
+ return st;
}
Status MemTrackerLimiter::mem_limit_exceeded(const std::string& msg,
@@ -284,9 +318,11 @@ Status MemTrackerLimiter::mem_limit_exceeded(const
std::string& msg,
Status failed_try_consume_st) {
STOP_CHECK_THREAD_MEM_TRACKER_LIMIT();
std::string detail =
- fmt::format("memory limit exceeded:<consumed_tracker={}, {}>,
executing_msg:<{}>",
+ fmt::format("memory limit exceeded:<consuming_tracker={}, {}>,
executing_msg:<{}>",
_label, failed_try_consume_st.get_error_msg(), msg);
- return failed_tracker->mem_limit_exceeded_log(detail);
+ auto st = MemTrackerLimiter::mem_limit_exceeded_construct(detail);
+ failed_tracker->print_log_usage(st.get_error_msg());
+ return st;
}
Status MemTrackerLimiter::mem_limit_exceeded(RuntimeState* state, const
std::string& msg,
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h
b/be/src/runtime/memory/mem_tracker_limiter.h
index 0e6bbdfd8d..3aaa9d973f 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -70,11 +70,12 @@ public:
// of the process malloc.
// for fast, expect MemInfo::initialized() to be true.
if (PerfCounters::get_vm_rss() + bytes >= MemInfo::mem_limit()) {
- return Status::MemoryLimitExceeded(
- "{}: TryConsume failed, bytes={} process physical memory
consumption={} and "
- "virtual memory consumption={}, mem limit={}",
- _label, bytes, PerfCounters::get_vm_rss(),
MemInfo::current_mem(),
- MemInfo::mem_limit());
+ auto st = Status::MemoryLimitExceeded(
+ "Memory limit exceeded, process memory used {}Bytes exceed
limit {}B, "
+ "consuming_tracker={}, failed_alloc_size={}",
+ PerfCounters::get_vm_rss(), MemInfo::mem_limit(), _label,
bytes);
+
ExecEnv::GetInstance()->process_mem_tracker()->print_log_usage(st.get_error_msg());
+ return st;
}
return Status::OK();
}
@@ -197,7 +198,8 @@ private:
const std::list<MemTrackerLimiter*>& trackers,
int64_t* logged_consumption);
- Status mem_limit_exceeded_log(const std::string& msg);
+ static Status mem_limit_exceeded_construct(const std::string& msg);
+ void print_log_usage(const std::string& msg);
private:
// Limit on memory consumption, in bytes. If limit_ == -1, there is no
consumption limit. Used in log_usage。
@@ -280,7 +282,9 @@ inline Status MemTrackerLimiter::try_consume(int64_t bytes)
{
// Walk the tracker tree top-down.
for (i = _all_ancestors.size() - 1; i >= 0; --i) {
MemTrackerLimiter* tracker = _all_ancestors[i];
- if (tracker->limit() < 0) {
+ // Process tracker does not participate in the process memory limit,
process tracker consumption is virtual memory,
+ // and there is a diff between the real physical memory value of the
process. It is replaced by check_sys_mem_info.
+ if (tracker->limit() < 0 || _label == "Process") {
tracker->_consumption->add(bytes); // No limit at this tracker.
} else {
// If TryConsume fails, we can try to GC, but we may need to try
several times if
@@ -309,14 +313,14 @@ inline Status MemTrackerLimiter::check_limit(int64_t
bytes) {
RETURN_IF_ERROR(check_sys_mem_info(bytes));
int i;
// Walk the tracker tree top-down.
- for (i = _all_ancestors.size() - 1; i >= 0; --i) {
- MemTrackerLimiter* tracker = _all_ancestors[i];
- if (tracker->limit() > 0) {
- while (true) {
- if (LIKELY(tracker->_consumption->current_value() + bytes <
tracker->limit()))
- break;
- RETURN_IF_ERROR(tracker->try_gc_memory(bytes));
- }
+ for (i = _limited_ancestors.size() - 1; i >= 0; --i) {
+ MemTrackerLimiter* tracker = _limited_ancestors[i];
+ // Process tracker does not participate in the process memory limit,
process tracker consumption is virtual memory,
+ // and there is a diff between the real physical memory value of the
process. It is replaced by check_sys_mem_info.
+ if (tracker->label() == "Process") continue;
+ while (true) {
+ if (LIKELY(tracker->_consumption->current_value() + bytes <
tracker->limit())) break;
+ RETURN_IF_ERROR(tracker->try_gc_memory(bytes));
}
}
return Status::OK();
diff --git a/be/src/runtime/runtime_filter_mgr.cpp
b/be/src/runtime/runtime_filter_mgr.cpp
index 85d0fab248..0134886779 100644
--- a/be/src/runtime/runtime_filter_mgr.cpp
+++ b/be/src/runtime/runtime_filter_mgr.cpp
@@ -185,7 +185,7 @@ Status RuntimeFilterMergeControllerEntity::init(UniqueId
query_id, UniqueId frag
// merge data
Status RuntimeFilterMergeControllerEntity::merge(const PMergeFilterRequest*
request,
const char* data) {
- SCOPED_CONSUME_MEM_TRACKER(_mem_tracker.get());
+ // SCOPED_CONSUME_MEM_TRACKER(_mem_tracker.get());
std::shared_ptr<RuntimeFilterCntlVal> cntVal;
int merged_size = 0;
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
