This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 711ccf596775303f94f040df90e2e468376fa8f8 Author: Xinyi Zou <[email protected]> AuthorDate: Sat Feb 4 22:49:22 2023 +0800 [fix](memory) Fix request jemallloc metrics wait lock je_malloc_mutex_lock_slow #16381 MetricRegistry::trigger_all_hooks holds the metrics lock and is stuck in get_je_metrics, to_prometheus is waiting for MetricRegistry::trigger_all_hooks to release the lock, so get_je_metrics is no longer called in MetricRegistry::trigger_all_hooks. --- be/src/common/daemon.cpp | 1 + be/src/runtime/memory/jemalloc_hook.cpp | 3 +++ be/src/util/system_metrics.cpp | 3 +++ be/src/util/system_metrics.h | 1 + 4 files changed, 8 insertions(+) diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 2f6afb99b7..b80787cb4e 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -218,6 +218,7 @@ void Daemon::memory_maintenance_thread() { // Refresh allocator memory metrics. #if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && !defined(THREAD_SANITIZER) doris::MemInfo::refresh_allocator_mem(); + DorisMetrics::instance()->system_metrics()->update_allocator_metrics(); #endif doris::MemInfo::refresh_proc_mem_no_allocator_cache(); diff --git a/be/src/runtime/memory/jemalloc_hook.cpp b/be/src/runtime/memory/jemalloc_hook.cpp index bf0c3e20bc..1558776331 100644 --- a/be/src/runtime/memory/jemalloc_hook.cpp +++ b/be/src/runtime/memory/jemalloc_hook.cpp @@ -28,6 +28,9 @@ extern "C" { void* doris_malloc(size_t size) __THROW { + // Both je_nallocx and je_malloc will use the lock je_malloc_mutex_lock_slow, + // so enabling the jemalloc hook will double the lock usage. + // In extreme cases this will affect performance, consider turning off mem hook TRY_CONSUME_MEM_TRACKER(je_nallocx(size, 0), nullptr); void* ptr = je_malloc(size); if (UNLIKELY(ptr == nullptr)) { diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp index 11703f2187..5e94b7ce79 100644 --- a/be/src/util/system_metrics.cpp +++ b/be/src/util/system_metrics.cpp @@ -430,6 +430,9 @@ void SystemMetrics::_install_memory_metrics(MetricEntity* entity) { void SystemMetrics::_update_memory_metrics() { _memory_metrics->memory_allocated_bytes->set_value(PerfCounters::get_vm_rss()); get_metrics_from_proc_vmstat(); +} + +void SystemMetrics::update_allocator_metrics() { #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER) LOG(INFO) << "Memory tracking is not available with address sanitizer builds."; #elif defined(USE_JEMALLOC) diff --git a/be/src/util/system_metrics.h b/be/src/util/system_metrics.h index 5354f494de..8f54aa9440 100644 --- a/be/src/util/system_metrics.h +++ b/be/src/util/system_metrics.h @@ -55,6 +55,7 @@ public: int64_t interval_sec); void update_max_network_send_bytes_rate(int64_t max_send_bytes_rate); void update_max_network_receive_bytes_rate(int64_t max_receive_bytes_rate); + void update_allocator_metrics(); private: void _install_cpu_metrics(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
