This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1847e440b2 [fix](memory) enable Jemalloc arena dirty pages (#22639)
1847e440b2 is described below
commit 1847e440b21e49b53d3654d705e7aade557b55b7
Author: Xinyi Zou <[email protected]>
AuthorDate: Sun Aug 6 19:18:44 2023 +0800
[fix](memory) enable Jemalloc arena dirty pages (#22639)
If there is a core dump here, it may cover up the real stack, if stack
trace indicates heap corruption
(which led to invalid jemalloc metadata), like double free or
use-after-free in the application.
Try sanitizers such as ASAN, or build jemalloc with --enable-debug to
investigate further.
---
be/src/common/config.cpp | 1 +
be/src/common/config.h | 2 ++
be/src/util/mem_info.h | 21 +++++++++++++--------
3 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 12eb3caaa2..8d32774a99 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -516,6 +516,7 @@ DEFINE_Bool(enable_quadratic_probing, "false");
DEFINE_String(pprof_profile_dir, "${DORIS_HOME}/log");
// for jeprofile in jemalloc
DEFINE_mString(jeprofile_dir, "${DORIS_HOME}/log");
+DEFINE_mBool(enable_je_purge_dirty_pages, "true");
// to forward compatibility, will be removed later
DEFINE_mBool(enable_token_check, "true");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 752d008ed4..2cbc1914d5 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -565,6 +565,8 @@ DECLARE_Bool(enable_quadratic_probing);
DECLARE_String(pprof_profile_dir);
// for jeprofile in jemalloc
DECLARE_mString(jeprofile_dir);
+// Purge all unused dirty pages for all arenas.
+DECLARE_mBool(enable_je_purge_dirty_pages);
// to forward compatibility, will be removed later
DECLARE_mBool(enable_token_check);
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 98179dda2c..1ccd918c4a 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -38,6 +38,7 @@
#else
#include <gperftools/malloc_extension.h>
#endif
+#include "common/config.h"
#include "util/perf_counters.h"
namespace doris {
@@ -111,14 +112,18 @@ public:
static inline void je_purge_all_arena_dirty_pages() {
#ifdef USE_JEMALLOC
// https://github.com/jemalloc/jemalloc/issues/2470
- // Occasional core dump during stress test, purge should be turned on
after the heap corruption is resolved.
- // try {
- // // Purge all unused dirty pages for arena <i>, or for all
arenas if <i> equals MALLCTL_ARENAS_ALL.
- // jemallctl(fmt::format("arena.{}.purge",
MALLCTL_ARENAS_ALL).c_str(), nullptr, nullptr,
- // nullptr, 0);
- // } catch (...) {
- // LOG(WARNING) << "Purge all unused dirty pages for all arenas
failed";
- // }
+ // If there is a core dump here, it may cover up the real stack, if
stack trace indicates heap corruption
+ // (which led to invalid jemalloc metadata), like double free or
use-after-free in the application.
+ // Try sanitizers such as ASAN, or build jemalloc with --enable-debug
to investigate further.
+ if (config::enable_je_purge_dirty_pages) {
+ try {
+ // Purge all unused dirty pages for arena <i>, or for all
arenas if <i> equals MALLCTL_ARENAS_ALL.
+ jemallctl(fmt::format("arena.{}.purge",
MALLCTL_ARENAS_ALL).c_str(), nullptr,
+ nullptr, nullptr, 0);
+ } catch (...) {
+ LOG(WARNING) << "Purge all unused dirty pages for all arenas
failed";
+ }
+ }
#endif
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
