This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new a8cc13f66cc [pick](branch-3.0) pick #40714 (#41906)
a8cc13f66cc is described below
commit a8cc13f66cc6b8a483733e480e190170aa2f11db
Author: Xinyi Zou <[email protected]>
AuthorDate: Wed Oct 16 14:13:45 2024 +0800
[pick](branch-3.0) pick #40714 (#41906)
pick #40714
---
be/src/common/config.cpp | 3 +++
be/src/common/config.h | 4 ++++
be/src/runtime/memory/mem_tracker_limiter.cpp | 23 +++++++++++-----------
be/src/runtime/memory/mem_tracker_limiter.h | 7 ++-----
.../pipeline/cloud_p0/conf/be_custom.conf | 1 +
.../pipeline/cloud_p1/conf/be_custom.conf | 2 ++
regression-test/pipeline/external/conf/be.conf | 1 +
regression-test/pipeline/p0/conf/be.conf | 2 +-
regression-test/pipeline/p1/conf/be.conf | 2 +-
9 files changed, 26 insertions(+), 19 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 70d56a923ea..6e459b3638b 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -151,6 +151,9 @@ DEFINE_mInt64(stacktrace_in_alloc_large_memory_bytes,
"2147483648");
DEFINE_mInt64(crash_in_alloc_large_memory_bytes, "-1");
+// If memory tracker value is inaccurate, BE will crash. usually used in test
environments, default value is false.
+DEFINE_mBool(crash_in_memory_tracker_inaccurate, "false");
+
// default is true. if any memory tracking in Orphan mem tracker will report
error.
// !! not modify the default value of this conf!! otherwise memory errors
cannot be detected in time.
// allocator free memory not need to check, because when the thread memory
tracker label is Orphan,
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 00d217aa6ba..dc1f60b69f8 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -194,11 +194,15 @@ DECLARE_mBool(enable_stacktrace);
// if alloc failed using Doris Allocator, will print stacktrace in error log.
// if is -1, disable print stacktrace when alloc large memory.
DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes);
+
// when alloc memory larger than crash_in_alloc_large_memory_bytes will crash,
default -1 means disabled.
// if you need a core dump to analyze large memory allocation,
// modify this parameter to crash when large memory allocation occur will help
DECLARE_mInt64(crash_in_alloc_large_memory_bytes);
+// If memory tracker value is inaccurate, BE will crash. usually used in test
environments, default value is false.
+DECLARE_mBool(crash_in_memory_tracker_inaccurate);
+
// default is true. if any memory tracking in Orphan mem tracker will report
error.
// !! not modify the default value of this conf!! otherwise memory errors
cannot be detected in time.
// allocator free memory not need to check, because when the thread memory
tracker label is Orphan,
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp
b/be/src/runtime/memory/mem_tracker_limiter.cpp
index fb513509d4e..a1eb2ed67d3 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -108,6 +108,12 @@ std::shared_ptr<MemTrackerLimiter>
MemTrackerLimiter::create_shared(MemTrackerLi
return tracker;
}
+bool MemTrackerLimiter::open_memory_tracker_inaccurate_detect() {
+ return doris::config::crash_in_memory_tracker_inaccurate &&
+ (_type == Type::COMPACTION || _type == Type::SCHEMA_CHANGE || _type
== Type::QUERY ||
+ (_type == Type::LOAD && !is_group_commit_load));
+}
+
MemTrackerLimiter::~MemTrackerLimiter() {
consume(_untracked_mem);
static std::string mem_tracker_inaccurate_msg =
@@ -127,35 +133,29 @@ MemTrackerLimiter::~MemTrackerLimiter() {
"4. If you need to "
"transfer memory tracking value between two trackers, can use
transfer_to.";
if (_consumption->current_value() != 0) {
- // TODO, expect mem tracker equal to 0 at the load/compaction/etc.
task end.
-#ifndef NDEBUG
- if (_type == Type::COMPACTION || _type == Type::SCHEMA_CHANGE || _type
== Type::QUERY ||
- (_type == Type::LOAD && !is_group_commit_load)) {
+ if (open_memory_tracker_inaccurate_detect()) {
std::string err_msg =
fmt::format("mem tracker label: {}, consumption: {}, peak
consumption: {}, {}.",
label(), _consumption->current_value(),
_consumption->peak_value(),
mem_tracker_inaccurate_msg);
LOG(FATAL) << err_msg << print_address_sanitizers();
}
-#endif
if (ExecEnv::tracking_memory()) {
ExecEnv::GetInstance()->orphan_mem_tracker()->consume(_consumption->current_value());
}
_consumption->set(0);
-#ifndef NDEBUG
- } else if (!_address_sanitizers.empty() && !is_group_commit_load) {
+ } else if (doris::config::crash_in_memory_tracker_inaccurate &&
!_address_sanitizers.empty() &&
+ !is_group_commit_load) {
LOG(FATAL) << "[Address Sanitizer] consumption is 0, but address
sanitizers not empty. "
<< ", mem tracker label: " << _label
<< ", peak consumption: " << _consumption->peak_value()
<< print_address_sanitizers();
-#endif
}
memory_memtrackerlimiter_cnt << -1;
}
-#ifndef NDEBUG
void MemTrackerLimiter::add_address_sanitizers(void* buf, size_t size) {
- if (_type == Type::QUERY || (_type == Type::LOAD &&
!is_group_commit_load)) {
+ if (open_memory_tracker_inaccurate_detect()) {
std::lock_guard<std::mutex> l(_address_sanitizers_mtx);
auto it = _address_sanitizers.find(buf);
if (it != _address_sanitizers.end()) {
@@ -177,7 +177,7 @@ void MemTrackerLimiter::add_address_sanitizers(void* buf,
size_t size) {
}
void MemTrackerLimiter::remove_address_sanitizers(void* buf, size_t size) {
- if (_type == Type::QUERY || (_type == Type::LOAD &&
!is_group_commit_load)) {
+ if (open_memory_tracker_inaccurate_detect()) {
std::lock_guard<std::mutex> l(_address_sanitizers_mtx);
auto it = _address_sanitizers.find(buf);
if (it != _address_sanitizers.end()) {
@@ -221,7 +221,6 @@ std::string MemTrackerLimiter::print_address_sanitizers() {
}
return detail;
}
-#endif
MemTracker::Snapshot MemTrackerLimiter::make_snapshot() const {
Snapshot snapshot;
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h
b/be/src/runtime/memory/mem_tracker_limiter.h
index 344f3dc92b6..c8a8c845793 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -205,12 +205,9 @@ public:
// Log the memory usage when memory limit is exceeded.
std::string tracker_limit_exceeded_str();
-#ifndef NDEBUG
void add_address_sanitizers(void* buf, size_t size);
void remove_address_sanitizers(void* buf, size_t size);
- std::string print_address_sanitizers();
bool is_group_commit_load {false};
-#endif
std::string debug_string() override {
std::stringstream msg;
@@ -253,16 +250,16 @@ private:
bool _enable_print_log_usage = false;
static std::atomic<bool> _enable_print_log_process_usage;
-#ifndef NDEBUG
struct AddressSanitizer {
size_t size;
std::string stack_trace;
};
+ std::string print_address_sanitizers();
+ bool open_memory_tracker_inaccurate_detect();
std::mutex _address_sanitizers_mtx;
std::unordered_map<void*, AddressSanitizer> _address_sanitizers;
std::vector<std::string> _error_address_sanitizers;
-#endif
};
inline int64_t MemTrackerLimiter::add_untracked_mem(int64_t bytes) {
diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf
b/regression-test/pipeline/cloud_p0/conf/be_custom.conf
index e258abbbb94..377a02536c6 100644
--- a/regression-test/pipeline/cloud_p0/conf/be_custom.conf
+++ b/regression-test/pipeline/cloud_p0/conf/be_custom.conf
@@ -36,3 +36,4 @@ webserver_num_workers = 128
enable_new_tablet_do_compaction = true
arrow_flight_sql_port = 8181
pipeline_task_leakage_detect_period_sec=1
+crash_in_memory_tracker_inaccurate = true
diff --git a/regression-test/pipeline/cloud_p1/conf/be_custom.conf
b/regression-test/pipeline/cloud_p1/conf/be_custom.conf
index a5715ac7a7a..401cf64775f 100644
--- a/regression-test/pipeline/cloud_p1/conf/be_custom.conf
+++ b/regression-test/pipeline/cloud_p1/conf/be_custom.conf
@@ -29,3 +29,5 @@ enable_file_cache = true
file_cache_path =
[{"path":"/data/doris_cloud/file_cache","total_size":104857600,"query_limit":104857600}]
tmp_file_dirs =
[{"path":"/data/doris_cloud/tmp","max_cache_bytes":104857600,"max_upload_bytes":104857600}]
save_load_error_log_to_s3 = true
+arrow_flight_sql_port = 8181
+crash_in_memory_tracker_inaccurate = true
diff --git a/regression-test/pipeline/external/conf/be.conf
b/regression-test/pipeline/external/conf/be.conf
index da464b5b824..fb5bbead921 100644
--- a/regression-test/pipeline/external/conf/be.conf
+++ b/regression-test/pipeline/external/conf/be.conf
@@ -66,3 +66,4 @@ enable_jvm_monitor = true
KRB5_CONFIG=/keytabs/krb5.conf
kerberos_krb5_conf_path=/keytabs/krb5.conf
+crash_in_memory_tracker_inaccurate = true
diff --git a/regression-test/pipeline/p0/conf/be.conf
b/regression-test/pipeline/p0/conf/be.conf
index b4d3152e6c7..f984fe6723a 100644
--- a/regression-test/pipeline/p0/conf/be.conf
+++ b/regression-test/pipeline/p0/conf/be.conf
@@ -68,4 +68,4 @@ enable_jvm_monitor = true
enable_be_proc_monitor = true
be_proc_monitor_interval_ms = 30000
webserver_num_workers = 128
-
+crash_in_memory_tracker_inaccurate = true
diff --git a/regression-test/pipeline/p1/conf/be.conf
b/regression-test/pipeline/p1/conf/be.conf
index 6844d674020..1ca87eb073a 100644
--- a/regression-test/pipeline/p1/conf/be.conf
+++ b/regression-test/pipeline/p1/conf/be.conf
@@ -61,4 +61,4 @@ enable_debug_log_timeout_secs=0
enable_missing_rows_correctness_check=true
enable_jvm_monitor = true
-
+crash_in_memory_tracker_inaccurate = true
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
