This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 7ab71ca3bc [fix](taskgroup) Fix task group overcommit memory GC
profile (#22764)
7ab71ca3bc is described below
commit 7ab71ca3bc5d08381eca9981149da8c1d401458a
Author: Xinyi Zou <[email protected]>
AuthorDate: Wed Aug 9 18:29:46 2023 +0800
[fix](taskgroup) Fix task group overcommit memory GC profile (#22764)
---
be/src/util/mem_info.cpp | 30 ++++++++++++++++++++++++------
1 file changed, 24 insertions(+), 6 deletions(-)
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index b28a0b43fd..701abab029 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -120,8 +120,9 @@ bool MemInfo::process_minor_gc() {
je_purge_all_arena_dirty_pages();
std::stringstream ss;
profile->pretty_print(&ss);
- LOG(INFO) << fmt::format("End Minor GC, Free Memory {} Bytes.
cost(us): {}, details: {}",
- freed_mem, watch.elapsed_time() / 1000,
ss.str());
+ LOG(INFO) << fmt::format("End Minor GC, Free Memory {}. cost(us): {},
details: {}",
+ PrettyPrinter::print(freed_mem, TUnit::BYTES),
+ watch.elapsed_time() / 1000, ss.str());
}};
freed_mem +=
CacheManager::instance()->for_each_cache_prune_stale(profile.get());
@@ -169,8 +170,9 @@ bool MemInfo::process_full_gc() {
je_purge_all_arena_dirty_pages();
std::stringstream ss;
profile->pretty_print(&ss);
- LOG(INFO) << fmt::format("End Full GC Free, Memory {} Bytes. cost(us):
{}, details: {}",
- freed_mem, watch.elapsed_time() / 1000,
ss.str());
+ LOG(INFO) << fmt::format("End Full GC, Free Memory {}. cost(us): {},
details: {}",
+ PrettyPrinter::print(freed_mem, TUnit::BYTES),
+ watch.elapsed_time() / 1000, ss.str());
}};
freed_mem +=
CacheManager::instance()->for_each_cache_prune_all(profile.get());
@@ -219,21 +221,37 @@ bool MemInfo::process_full_gc() {
}
int64_t MemInfo::tg_hard_memory_limit_gc() {
+ MonotonicStopWatch watch;
+ watch.start();
std::vector<taskgroup::TaskGroupPtr> task_groups;
+ std::unique_ptr<RuntimeProfile> tg_profile =
std::make_unique<RuntimeProfile>("WorkloadGroup");
+ int64_t total_free_memory = 0;
+
+ Defer defer {[&]() {
+ if (total_free_memory > 0) {
+ std::stringstream ss;
+ tg_profile->pretty_print(&ss);
+ LOG(INFO) << fmt::format(
+ "End Task Group Overcommit Memory GC, Free Memory {}.
cost(us): {}, "
+ "details: {}",
+ PrettyPrinter::print(total_free_memory, TUnit::BYTES),
+ watch.elapsed_time() / 1000, ss.str());
+ }
+ }};
+
taskgroup::TaskGroupManager::instance()->get_resource_groups(
[](const taskgroup::TaskGroupPtr& task_group) {
return !task_group->enable_memory_overcommit();
},
&task_groups);
- int64_t total_free_memory = 0;
for (const auto& task_group : task_groups) {
taskgroup::TaskGroupInfo tg_info;
task_group->task_group_info(&tg_info);
auto used = task_group->memory_used();
total_free_memory += MemTrackerLimiter::tg_memory_limit_gc(
used - tg_info.memory_limit, used, tg_info.id, tg_info.name,
tg_info.memory_limit,
- task_group->mem_tracker_limiter_pool(), nullptr);
+ task_group->mem_tracker_limiter_pool(), tg_profile.get());
}
return total_free_memory;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
