This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 747172237ac [branch-2.1](memory) Pick some memory GC patch (#37725)
747172237ac is described below
commit 747172237ac36ff0bdb60acf59c5b0b92b02d9a2
Author: Xinyi Zou <[email protected]>
AuthorDate: Sun Jul 14 15:19:40 2024 +0800
[branch-2.1](memory) Pick some memory GC patch (#37725)
pick
#36768
#37164
#37174
#37525
---
be/src/common/config.cpp | 8 ++------
be/src/common/config.h | 10 ++++++----
be/src/common/daemon.cpp | 27 +++++++++++++++------------
be/src/olap/page_cache.cpp | 25 +++++++++----------------
be/src/olap/page_cache.h | 3 +--
be/src/util/mem_info.cpp | 8 +++++++-
be/src/util/mem_info.h | 8 ++++++++
7 files changed, 48 insertions(+), 41 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index c747391abe0..7c687923803 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -113,12 +113,7 @@ DEFINE_mInt32(max_fill_rate, "2");
DEFINE_mInt32(double_resize_threshold, "23");
-// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit
byte, default 1.6G,
-// actual low water mark=min(1.6G, MemTotal * 10%), avoid wasting too much
memory on machines
-// with large memory larger than 16G.
-// Turn up max. On machines with more than 16G memory, more memory buffers
will be reserved for Full GC.
-// Turn down max. will use as much memory as possible.
-DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "1717986918");
+DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "6871947673");
// The size of the memory that gc wants to release each time, as a percentage
of the mem limit.
DEFINE_mString(process_minor_gc_size, "10%");
@@ -556,6 +551,7 @@ DEFINE_String(pprof_profile_dir, "${DORIS_HOME}/log");
// for jeprofile in jemalloc
DEFINE_mString(jeprofile_dir, "${DORIS_HOME}/log");
DEFINE_mBool(enable_je_purge_dirty_pages, "true");
+DEFINE_mString(je_dirty_pages_mem_limit_percent, "5%");
// to forward compatibility, will be removed later
DEFINE_mBool(enable_token_check, "true");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 75cb0e2e272..945454079bf 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -155,10 +155,10 @@ DECLARE_mInt32(max_fill_rate);
DECLARE_mInt32(double_resize_threshold);
-// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit
byte, default 1.6G,
-// actual low water mark=min(1.6G, MemTotal * 10%), avoid wasting too much
memory on machines
-// with large memory larger than 16G.
-// Turn up max. On machines with more than 16G memory, more memory buffers
will be reserved for Full GC.
+// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit
byte, default 6.4G,
+// actual low water mark=min(6.4G, MemTotal * 5%), avoid wasting too much
memory on machines
+// with large memory larger than 128G.
+// Turn up max. On machines with more than 128G memory, more memory buffers
will be reserved for Full GC.
// Turn down max. will use as much memory as possible.
DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes);
@@ -609,6 +609,8 @@ DECLARE_String(pprof_profile_dir);
DECLARE_mString(jeprofile_dir);
// Purge all unused dirty pages for all arenas.
DECLARE_mBool(enable_je_purge_dirty_pages);
+// Purge all unused Jemalloc dirty pages for all arenas when exceed
je_dirty_pages_mem_limit and process exceed soft limit.
+DECLARE_mString(je_dirty_pages_mem_limit_percent);
// to forward compatibility, will be removed later
DECLARE_mBool(enable_token_check);
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index d54189bce23..00d9caa4155 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -194,26 +194,29 @@ void Daemon::memory_maintenance_thread() {
doris::PerfCounters::refresh_proc_status();
doris::MemInfo::refresh_proc_meminfo();
doris::GlobalMemoryArbitrator::reset_refresh_interval_memory_growth();
+
ExecEnv::GetInstance()->brpc_iobuf_block_memory_tracker()->set_consumption(
+ butil::IOBuf::block_memory());
+ // Refresh allocator memory metrics.
+#if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) &&
!defined(THREAD_SANITIZER)
+ doris::MemInfo::refresh_allocator_mem();
+#ifdef USE_JEMALLOC
+ if (doris::MemInfo::je_dirty_pages_mem() >
doris::MemInfo::je_dirty_pages_mem_limit() &&
+ GlobalMemoryArbitrator::is_exceed_soft_mem_limit()) {
+ doris::MemInfo::notify_je_purge_dirty_pages();
+ }
+#endif
+ if (config::enable_system_metrics) {
+
DorisMetrics::instance()->system_metrics()->update_allocator_metrics();
+ }
+#endif
// Update and print memory stat when the memory changes by 256M.
if (abs(last_print_proc_mem - PerfCounters::get_vm_rss()) > 268435456)
{
last_print_proc_mem = PerfCounters::get_vm_rss();
doris::MemTrackerLimiter::clean_tracker_limiter_group();
doris::MemTrackerLimiter::enable_print_log_process_usage();
-
// Refresh mem tracker each type counter.
doris::MemTrackerLimiter::refresh_global_counter();
-
- // Refresh allocator memory metrics.
-#if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) &&
!defined(THREAD_SANITIZER)
- doris::MemInfo::refresh_allocator_mem();
- if (config::enable_system_metrics) {
-
DorisMetrics::instance()->system_metrics()->update_allocator_metrics();
- }
-#endif
-
-
ExecEnv::GetInstance()->brpc_iobuf_block_memory_tracker()->set_consumption(
- butil::IOBuf::block_memory());
LOG(INFO) << doris::GlobalMemoryArbitrator::
process_mem_log_str(); // print mem log when
memory state by 256M
}
diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp
index fe0a99af34f..1f0556f4642 100644
--- a/be/src/olap/page_cache.cpp
+++ b/be/src/olap/page_cache.cpp
@@ -26,16 +26,14 @@
namespace doris {
template <typename TAllocator>
PageBase<TAllocator>::PageBase(size_t b, bool use_cache,
segment_v2::PageTypePB page_type)
- : LRUCacheValueBase(),
- _size(b),
- _capacity(b),
- _use_cache(use_cache),
- _page_type(page_type) {
- if (_use_cache) {
- SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
- StoragePageCache::instance()->mem_tracker(_page_type));
- _data = reinterpret_cast<char*>(TAllocator::alloc(_capacity,
ALLOCATOR_ALIGNMENT_16));
+ : LRUCacheValueBase(), _size(b), _capacity(b) {
+ if (use_cache) {
+ _mem_tracker_by_allocator =
StoragePageCache::instance()->mem_tracker(page_type);
} else {
+ _mem_tracker_by_allocator =
thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker();
+ }
+ {
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
_data = reinterpret_cast<char*>(TAllocator::alloc(_capacity,
ALLOCATOR_ALIGNMENT_16));
}
}
@@ -44,13 +42,8 @@ template <typename TAllocator>
PageBase<TAllocator>::~PageBase() {
if (_data != nullptr) {
DCHECK(_capacity != 0 && _size != 0);
- if (_use_cache) {
- SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
- StoragePageCache::instance()->mem_tracker(_page_type));
- TAllocator::free(_data, _capacity);
- } else {
- TAllocator::free(_data, _capacity);
- }
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
+ TAllocator::free(_data, _capacity);
}
}
diff --git a/be/src/olap/page_cache.h b/be/src/olap/page_cache.h
index 23b3574a10a..09fc689959c 100644
--- a/be/src/olap/page_cache.h
+++ b/be/src/olap/page_cache.h
@@ -60,8 +60,7 @@ private:
// Effective size, smaller than capacity, such as data page remove
checksum suffix.
size_t _size = 0;
size_t _capacity = 0;
- bool _use_cache;
- segment_v2::PageTypePB _page_type;
+ std::shared_ptr<MemTrackerLimiter> _mem_tracker_by_allocator;
};
using DataPage = PageBase<Allocator<false>>;
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index 45e609d7100..b6f35df5d9c 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -52,6 +52,8 @@ std::atomic<int64_t> MemInfo::_s_mem_limit =
std::numeric_limits<int64_t>::max()
std::atomic<int64_t> MemInfo::_s_soft_mem_limit =
std::numeric_limits<int64_t>::max();
std::atomic<int64_t> MemInfo::_s_allocator_cache_mem = 0;
+std::atomic<int64_t> MemInfo::_s_je_dirty_pages_mem =
std::numeric_limits<int64_t>::min();
+std::atomic<int64_t> MemInfo::_s_je_dirty_pages_mem_limit =
std::numeric_limits<int64_t>::max();
std::atomic<int64_t> MemInfo::_s_virtual_memory_used = 0;
int64_t MemInfo::_s_cgroup_mem_limit = std::numeric_limits<int64_t>::max();
@@ -86,6 +88,8 @@ void MemInfo::refresh_allocator_mem() {
get_je_metrics("stats.metadata") +
get_je_all_arena_metrics("pdirty") *
get_page_size(),
std::memory_order_relaxed);
+ _s_je_dirty_pages_mem.store(get_je_all_arena_metrics("pdirty") *
get_page_size(),
+ std::memory_order_relaxed);
_s_virtual_memory_used.store(get_je_metrics("stats.mapped"),
std::memory_order_relaxed);
#else
_s_allocator_cache_mem.store(get_tc_metrics("tcmalloc.pageheap_free_bytes") +
@@ -244,6 +248,8 @@ void MemInfo::refresh_proc_meminfo() {
_s_mem_limit,
&is_percent));
_s_process_full_gc_size.store(ParseUtil::parse_mem_spec(config::process_full_gc_size,
-1,
_s_mem_limit,
&is_percent));
+ _s_je_dirty_pages_mem_limit.store(ParseUtil::parse_mem_spec(
+ config::je_dirty_pages_mem_limit_percent, -1, _s_mem_limit,
&is_percent));
}
// 3. refresh process available memory
@@ -298,7 +304,7 @@ void MemInfo::init() {
// upper sys_mem_available_low_water_mark, avoid wasting too much
memory.
_s_sys_mem_available_low_water_mark = std::max<int64_t>(
std::min<int64_t>(std::min<int64_t>(_s_physical_mem -
_s_mem_limit,
- int64_t(_s_physical_mem *
0.1)),
+ int64_t(_s_physical_mem *
0.05)),
config::max_sys_mem_available_low_water_mark_bytes),
0);
_s_sys_mem_available_warning_water_mark =
_s_sys_mem_available_low_water_mark * 2;
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 1b92d0eb9f0..6b66d05033f 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -144,6 +144,12 @@ public:
static inline size_t allocator_cache_mem() {
return _s_allocator_cache_mem.load(std::memory_order_relaxed);
}
+ static inline int64_t je_dirty_pages_mem() {
+ return _s_je_dirty_pages_mem.load(std::memory_order_relaxed);
+ }
+ static inline int64_t je_dirty_pages_mem_limit() {
+ return _s_je_dirty_pages_mem_limit.load(std::memory_order_relaxed);
+ }
// Tcmalloc property `generic.total_physical_bytes` records the total
length of the virtual memory
// obtained by the process malloc, not the physical memory actually used
by the process in the OS.
@@ -178,6 +184,8 @@ private:
static std::atomic<int64_t> _s_soft_mem_limit;
static std::atomic<int64_t> _s_allocator_cache_mem;
+ static std::atomic<int64_t> _s_je_dirty_pages_mem;
+ static std::atomic<int64_t> _s_je_dirty_pages_mem_limit;
static std::atomic<int64_t> _s_virtual_memory_used;
static int64_t _s_cgroup_mem_limit;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
