This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 63d57b83f3 [fix](memory) Fix request jemallloc metrics wait lock
je_malloc_mutex_lock_slow #16381
63d57b83f3 is described below
commit 63d57b83f30e223b2ae16c3554c29f9b9f24db80
Author: Xinyi Zou <[email protected]>
AuthorDate: Sat Feb 4 22:49:22 2023 +0800
[fix](memory) Fix request jemallloc metrics wait lock
je_malloc_mutex_lock_slow #16381
MetricRegistry::trigger_all_hooks holds the metrics lock and is stuck in
get_je_metrics, to_prometheus is waiting for MetricRegistry::trigger_all_hooks
to release the lock, so get_je_metrics is no longer called in
MetricRegistry::trigger_all_hooks.
---
be/src/common/daemon.cpp | 1 +
be/src/runtime/memory/jemalloc_hook.cpp | 3 +++
be/src/util/system_metrics.cpp | 3 +++
be/src/util/system_metrics.h | 1 +
4 files changed, 8 insertions(+)
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index 2c132ceb13..deadffc423 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -181,6 +181,7 @@ void Daemon::memory_maintenance_thread() {
// Refresh allocator memory metrics.
#if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) &&
!defined(THREAD_SANITIZER)
doris::MemInfo::refresh_allocator_mem();
+ DorisMetrics::instance()->system_metrics()->update_allocator_metrics();
#endif
doris::MemInfo::refresh_proc_mem_no_allocator_cache();
diff --git a/be/src/runtime/memory/jemalloc_hook.cpp
b/be/src/runtime/memory/jemalloc_hook.cpp
index bf0c3e20bc..1558776331 100644
--- a/be/src/runtime/memory/jemalloc_hook.cpp
+++ b/be/src/runtime/memory/jemalloc_hook.cpp
@@ -28,6 +28,9 @@
extern "C" {
void* doris_malloc(size_t size) __THROW {
+ // Both je_nallocx and je_malloc will use the lock
je_malloc_mutex_lock_slow,
+ // so enabling the jemalloc hook will double the lock usage.
+ // In extreme cases this will affect performance, consider turning off mem
hook
TRY_CONSUME_MEM_TRACKER(je_nallocx(size, 0), nullptr);
void* ptr = je_malloc(size);
if (UNLIKELY(ptr == nullptr)) {
diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp
index 11703f2187..5e94b7ce79 100644
--- a/be/src/util/system_metrics.cpp
+++ b/be/src/util/system_metrics.cpp
@@ -430,6 +430,9 @@ void SystemMetrics::_install_memory_metrics(MetricEntity*
entity) {
void SystemMetrics::_update_memory_metrics() {
_memory_metrics->memory_allocated_bytes->set_value(PerfCounters::get_vm_rss());
get_metrics_from_proc_vmstat();
+}
+
+void SystemMetrics::update_allocator_metrics() {
#if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) ||
defined(THREAD_SANITIZER)
LOG(INFO) << "Memory tracking is not available with address sanitizer
builds.";
#elif defined(USE_JEMALLOC)
diff --git a/be/src/util/system_metrics.h b/be/src/util/system_metrics.h
index 5354f494de..8f54aa9440 100644
--- a/be/src/util/system_metrics.h
+++ b/be/src/util/system_metrics.h
@@ -55,6 +55,7 @@ public:
int64_t interval_sec);
void update_max_network_send_bytes_rate(int64_t max_send_bytes_rate);
void update_max_network_receive_bytes_rate(int64_t max_receive_bytes_rate);
+ void update_allocator_metrics();
private:
void _install_cpu_metrics();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
