This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new e56216211e8 [pick](branch-2.1) pick #40667 #40714 (#41905)
e56216211e8 is described below
commit e56216211e89e092519f6189758dace36cc68702
Author: Xinyi Zou <[email protected]>
AuthorDate: Wed Oct 16 14:09:03 2024 +0800
[pick](branch-2.1) pick #40667 #40714 (#41905)
pick
#40667
#40714
---------
Co-authored-by: wangbo <[email protected]>
---
be/src/common/config.cpp | 3 +++
be/src/common/config.h | 4 ++++
be/src/olap/task/engine_alter_tablet_task.cpp | 2 +-
be/src/runtime/memory/mem_tracker_limiter.cpp | 22 +++++++++++-----------
be/src/runtime/memory/mem_tracker_limiter.h | 7 ++-----
regression-test/pipeline/external/conf/be.conf | 1 +
regression-test/pipeline/p0/conf/be.conf | 1 +
regression-test/pipeline/p1/conf/be.conf | 2 +-
8 files changed, 24 insertions(+), 18 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 134ebbb2741..94418cc8357 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -140,6 +140,9 @@ DEFINE_mInt64(stacktrace_in_alloc_large_memory_bytes,
"2147483648");
DEFINE_mInt64(crash_in_alloc_large_memory_bytes, "-1");
+// If memory tracker value is inaccurate, BE will crash. usually used in test
environments, default value is false.
+DEFINE_mBool(crash_in_memory_tracker_inaccurate, "false");
+
// default is true. if any memory tracking in Orphan mem tracker will report
error.
// !! not modify the default value of this conf!! otherwise memory errors
cannot be detected in time.
// allocator free memory not need to check, because when the thread memory
tracker label is Orphan,
diff --git a/be/src/common/config.h b/be/src/common/config.h
index ecc34f0db7d..384b1bd95ab 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -188,11 +188,15 @@ DECLARE_mBool(disable_memory_gc);
// if alloc failed using Doris Allocator, will print stacktrace in error log.
// if is -1, disable print stacktrace when alloc large memory.
DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes);
+
// when alloc memory larger than crash_in_alloc_large_memory_bytes will crash,
default -1 means disabled.
// if you need a core dump to analyze large memory allocation,
// modify this parameter to crash when large memory allocation occur will help
DECLARE_mInt64(crash_in_alloc_large_memory_bytes);
+// If memory tracker value is inaccurate, BE will crash. usually used in test
environments, default value is false.
+DECLARE_mBool(crash_in_memory_tracker_inaccurate);
+
// default is true. if any memory tracking in Orphan mem tracker will report
error.
// !! not modify the default value of this conf!! otherwise memory errors
cannot be detected in time.
// allocator free memory not need to check, because when the thread memory
tracker label is Orphan,
diff --git a/be/src/olap/task/engine_alter_tablet_task.cpp
b/be/src/olap/task/engine_alter_tablet_task.cpp
index 8a8d27b7657..f73a4f8a82d 100644
--- a/be/src/olap/task/engine_alter_tablet_task.cpp
+++ b/be/src/olap/task/engine_alter_tablet_task.cpp
@@ -39,7 +39,7 @@ EngineAlterTabletTask::EngineAlterTabletTask(const
TAlterTabletReqV2& request)
->get_storage_engine()
->memory_limitation_bytes_per_thread_for_schema_change();
_mem_tracker = MemTrackerLimiter::create_shared(
- MemTrackerLimiter::Type::OTHER,
+ MemTrackerLimiter::Type::SCHEMA_CHANGE,
fmt::format("EngineAlterTabletTask#baseTabletId={}:newTabletId={}",
std::to_string(_alter_tablet_req.base_tablet_id),
std::to_string(_alter_tablet_req.new_tablet_id)),
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp
b/be/src/runtime/memory/mem_tracker_limiter.cpp
index bfe3a5441da..e6144aa05fe 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -108,6 +108,12 @@ std::shared_ptr<MemTrackerLimiter>
MemTrackerLimiter::create_shared(MemTrackerLi
return tracker;
}
+bool MemTrackerLimiter::open_memory_tracker_inaccurate_detect() {
+ return doris::config::crash_in_memory_tracker_inaccurate &&
+ (_type == Type::COMPACTION || _type == Type::SCHEMA_CHANGE || _type
== Type::QUERY ||
+ (_type == Type::LOAD && !is_group_commit_load));
+}
+
MemTrackerLimiter::~MemTrackerLimiter() {
consume(_untracked_mem);
static std::string mem_tracker_inaccurate_msg =
@@ -127,34 +133,29 @@ MemTrackerLimiter::~MemTrackerLimiter() {
"4. If you need to "
"transfer memory tracking value between two trackers, can use
transfer_to.";
if (_consumption->current_value() != 0) {
- // TODO, expect mem tracker equal to 0 at the load/compaction/etc.
task end.
-#ifndef NDEBUG
- if (_type == Type::QUERY || (_type == Type::LOAD &&
!is_group_commit_load)) {
+ if (open_memory_tracker_inaccurate_detect()) {
std::string err_msg =
fmt::format("mem tracker label: {}, consumption: {}, peak
consumption: {}, {}.",
label(), _consumption->current_value(),
_consumption->peak_value(),
mem_tracker_inaccurate_msg);
LOG(FATAL) << err_msg << print_address_sanitizers();
}
-#endif
if (ExecEnv::tracking_memory()) {
ExecEnv::GetInstance()->orphan_mem_tracker()->consume(_consumption->current_value());
}
_consumption->set(0);
-#ifndef NDEBUG
- } else if (!_address_sanitizers.empty() && !is_group_commit_load) {
+ } else if (doris::config::crash_in_memory_tracker_inaccurate &&
!_address_sanitizers.empty() &&
+ !is_group_commit_load) {
LOG(FATAL) << "[Address Sanitizer] consumption is 0, but address
sanitizers not empty. "
<< ", mem tracker label: " << _label
<< ", peak consumption: " << _consumption->peak_value()
<< print_address_sanitizers();
-#endif
}
memory_memtrackerlimiter_cnt << -1;
}
-#ifndef NDEBUG
void MemTrackerLimiter::add_address_sanitizers(void* buf, size_t size) {
- if (_type == Type::QUERY || (_type == Type::LOAD &&
!is_group_commit_load)) {
+ if (open_memory_tracker_inaccurate_detect()) {
std::lock_guard<std::mutex> l(_address_sanitizers_mtx);
auto it = _address_sanitizers.find(buf);
if (it != _address_sanitizers.end()) {
@@ -176,7 +177,7 @@ void MemTrackerLimiter::add_address_sanitizers(void* buf,
size_t size) {
}
void MemTrackerLimiter::remove_address_sanitizers(void* buf, size_t size) {
- if (_type == Type::QUERY || (_type == Type::LOAD &&
!is_group_commit_load)) {
+ if (open_memory_tracker_inaccurate_detect()) {
std::lock_guard<std::mutex> l(_address_sanitizers_mtx);
auto it = _address_sanitizers.find(buf);
if (it != _address_sanitizers.end()) {
@@ -220,7 +221,6 @@ std::string MemTrackerLimiter::print_address_sanitizers() {
}
return detail;
}
-#endif
MemTracker::Snapshot MemTrackerLimiter::make_snapshot() const {
Snapshot snapshot;
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h
b/be/src/runtime/memory/mem_tracker_limiter.h
index 344f3dc92b6..c8a8c845793 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -205,12 +205,9 @@ public:
// Log the memory usage when memory limit is exceeded.
std::string tracker_limit_exceeded_str();
-#ifndef NDEBUG
void add_address_sanitizers(void* buf, size_t size);
void remove_address_sanitizers(void* buf, size_t size);
- std::string print_address_sanitizers();
bool is_group_commit_load {false};
-#endif
std::string debug_string() override {
std::stringstream msg;
@@ -253,16 +250,16 @@ private:
bool _enable_print_log_usage = false;
static std::atomic<bool> _enable_print_log_process_usage;
-#ifndef NDEBUG
struct AddressSanitizer {
size_t size;
std::string stack_trace;
};
+ std::string print_address_sanitizers();
+ bool open_memory_tracker_inaccurate_detect();
std::mutex _address_sanitizers_mtx;
std::unordered_map<void*, AddressSanitizer> _address_sanitizers;
std::vector<std::string> _error_address_sanitizers;
-#endif
};
inline int64_t MemTrackerLimiter::add_untracked_mem(int64_t bytes) {
diff --git a/regression-test/pipeline/external/conf/be.conf
b/regression-test/pipeline/external/conf/be.conf
index e2350638264..4725881f751 100644
--- a/regression-test/pipeline/external/conf/be.conf
+++ b/regression-test/pipeline/external/conf/be.conf
@@ -91,3 +91,4 @@ enable_missing_rows_correctness_check=true
KRB5_CONFIG=/keytabs/krb5.conf
kerberos_krb5_conf_path=/keytabs/krb5.conf
+crash_in_memory_tracker_inaccurate = true
diff --git a/regression-test/pipeline/p0/conf/be.conf
b/regression-test/pipeline/p0/conf/be.conf
index bdd0e3f0676..3aaf8777b37 100644
--- a/regression-test/pipeline/p0/conf/be.conf
+++ b/regression-test/pipeline/p0/conf/be.conf
@@ -89,3 +89,4 @@ enable_missing_rows_correctness_check=true
#enable_jvm_monitor = true
+crash_in_memory_tracker_inaccurate = true
diff --git a/regression-test/pipeline/p1/conf/be.conf
b/regression-test/pipeline/p1/conf/be.conf
index 225b0ad84d6..e9bf1dbdd88 100644
--- a/regression-test/pipeline/p1/conf/be.conf
+++ b/regression-test/pipeline/p1/conf/be.conf
@@ -81,4 +81,4 @@ max_sys_mem_available_low_water_mark_bytes=69206016
enable_missing_rows_correctness_check=true
enable_jvm_monitor = true
-
+crash_in_memory_tracker_inaccurate = true
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
