This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6ec299ef1c0 [opt](memory) Modify memory gc conf and add
`crash_in_alloc_large_memory_bytes` (#39611)
6ec299ef1c0 is described below
commit 6ec299ef1c0aece1503707108410b4c666b186bd
Author: Xinyi Zou <[email protected]>
AuthorDate: Wed Aug 21 14:32:44 2024 +0800
[opt](memory) Modify memory gc conf and add
`crash_in_alloc_large_memory_bytes` (#39611)
1. faster frequency and small batch memory GC, which helps query
stability.
2. when alloc memory larger than crash_in_alloc_large_memory_bytes will
crash, default -1 means disabled. if you need a core dump to analyze
large memory allocation, modify this parameter to crash when large
memory allocation occur will help
---
be/src/common/config.cpp | 8 +++---
be/src/common/config.h | 4 +++
be/src/runtime/memory/thread_mem_tracker_mgr.h | 34 +++++++++++++++++---------
3 files changed, 32 insertions(+), 14 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 458c124c304..1b11f3dc667 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -125,8 +125,8 @@ DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes,
"6871947673");
DEFINE_Int64(memtable_limiter_reserved_memory_bytes, "838860800");
// The size of the memory that gc wants to release each time, as a percentage
of the mem limit.
-DEFINE_mString(process_minor_gc_size, "10%");
-DEFINE_mString(process_full_gc_size, "20%");
+DEFINE_mString(process_minor_gc_size, "5%");
+DEFINE_mString(process_full_gc_size, "10%");
// If true, when the process does not exceed the soft mem limit, the query
memory will not be limited;
// when the process memory exceeds the soft mem limit, the query with the
largest ratio between the currently
@@ -140,6 +140,8 @@ DEFINE_mBool(enable_stacktrace, "true");
DEFINE_mInt64(stacktrace_in_alloc_large_memory_bytes, "2147483648");
+DEFINE_mInt64(crash_in_alloc_large_memory_bytes, "-1");
+
DEFINE_mBool(enable_memory_orphan_check, "false");
// The maximum time a thread waits for full GC. Currently only query will wait
for full gc.
@@ -588,7 +590,7 @@ DEFINE_mInt32(memory_maintenance_sleep_time_ms, "100");
// After full gc, no longer full gc and minor gc during sleep.
// After minor gc, no minor gc during sleep, but full gc is possible.
-DEFINE_mInt32(memory_gc_sleep_time_ms, "1000");
+DEFINE_mInt32(memory_gc_sleep_time_ms, "500");
// Sleep time in milliseconds between memtbale flush mgr refresh iterations
DEFINE_mInt64(memtable_mem_tracker_refresh_interval_ms, "5");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 2449826936f..8fb7acd36d8 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -191,6 +191,10 @@ DECLARE_mBool(enable_stacktrace);
// if alloc failed using Doris Allocator, will print stacktrace in error log.
// if is -1, disable print stacktrace when alloc large memory.
DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes);
+// when alloc memory larger than crash_in_alloc_large_memory_bytes will crash,
default -1 means disabled.
+// if you need a core dump to analyze large memory allocation,
+// modify this parameter to crash when large memory allocation occur will help
+DECLARE_mInt64(crash_in_alloc_large_memory_bytes);
// default is true. if any memory tracking in Orphan mem tracker will report
error.
DECLARE_mBool(enable_memory_orphan_check);
diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h
b/be/src/runtime/memory/thread_mem_tracker_mgr.h
index d9c4e093a4a..73cdd3243da 100644
--- a/be/src/runtime/memory/thread_mem_tracker_mgr.h
+++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h
@@ -242,17 +242,29 @@ inline void ThreadMemTrackerMgr::consume(int64_t size,
int skip_large_memory_che
flush_untracked_mem();
}
- if (skip_large_memory_check == 0 &&
doris::config::stacktrace_in_alloc_large_memory_bytes > 0 &&
- size > doris::config::stacktrace_in_alloc_large_memory_bytes) {
- _stop_consume = true;
- LOG(WARNING) << fmt::format(
- "malloc or new large memory: {}, {}, this is just a warning,
not prevent memory "
- "alloc, stacktrace:\n{}",
- size,
- is_attach_query() ? "in query or load: " + print_id(_query_id)
- : "not in query or load",
- get_stack_trace());
- _stop_consume = false;
+ if (skip_large_memory_check == 0) {
+ if (doris::config::stacktrace_in_alloc_large_memory_bytes > 0 &&
+ size > doris::config::stacktrace_in_alloc_large_memory_bytes) {
+ _stop_consume = true;
+ LOG(WARNING) << fmt::format(
+ "alloc large memory: {}, {}, this is just a warning, not
prevent memory alloc, "
+ "stacktrace:\n{}",
+ size,
+ is_attach_query() ? "in query or load: " +
print_id(_query_id)
+ : "not in query or load",
+ get_stack_trace());
+ _stop_consume = false;
+ }
+ if (doris::config::crash_in_alloc_large_memory_bytes > 0 &&
+ size > doris::config::crash_in_alloc_large_memory_bytes) {
+ LOG(FATAL) << fmt::format(
+ "alloc large memory: {}, {}, crash generate core dumpsto
help analyze, "
+ "stacktrace:\n{}",
+ size,
+ is_attach_query() ? "in query or load: " +
print_id(_query_id)
+ : "not in query or load",
+ get_stack_trace());
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
