This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 58cb404661 [fix](memory) Allocator throws Exception instead of
std::bad_alloc (#19285)
58cb404661 is described below
commit 58cb4046614c0ecc8fd452a3c1debf83fa202294
Author: Xinyi Zou <[email protected]>
AuthorDate: Fri May 5 18:01:48 2023 +0800
[fix](memory) Allocator throws Exception instead of std::bad_alloc (#19285)
W0505 01:31:25.840227 1727715 scanner_scheduler.cpp:340] Scan thread read
VScanner failed: [MEM_LIMIT_EXCEEDED]PreCatch error code:11, [E11] Allocator
sys memory check failed: Cannot alloc:16384, consuming tracker:<Orphan>, exec
node:<>, process memory used 5.87 GB exceed limit 5.64 GB or sys mem available
252.17 GB less than low water mark 1.60 GB, failed alloc size 16.00 KB.
@ 0x555c19e0cca8 doris::Exception::Exception()
@ 0x555c1c3e0c3f Allocator<>::sys_memory_check()
@ 0x555c1c3e1052 Allocator<>::memory_check()
@ 0x555c19e0a645 Allocator<>::alloc()
@ 0x555c1c34508b COWHelper<>::create<>()
@ 0x555c1e23f574
doris::vectorized::ConvertThroughParsing<>::execute<>()
@ 0x555c1e23f209
doris::vectorized::FunctionConvertFromString<>::execute_impl()
@ 0x555c1e23f4aa
doris::vectorized::FunctionConvertFromString<>::execute_impl()
@ 0x555c1e15ac29
doris::vectorized::PreparedFunctionImpl::execute_without_low_cardinality_columns()
@ 0x555c1e15ac56 doris::vectorized::PreparedFunctionImpl::execute()
@ 0x555c1e245276
_ZNSt17_Function_handlerIFN5doris6StatusEPNS0_15FunctionContextERNS0_10vectorized5BlockERKSt6vectorImSaImEEmmEZNKS4_12FunctionCast14create_wrapperINS4_14DataTypeNumberIiEEEESt8functionISC_ERKSt10shared_ptrIKNS4_9IDataTypeEEPKT_bEUlS3_S6_SB_mmE_E9_M_invokeERKSt9_Any_dataOS3_S6_SB_OmSY_
@ 0x555c1e2a9341
_ZZNK5doris10vectorized12FunctionCast23prepare_remove_nullableEPNS_15FunctionContextERKSt10shared_ptrIKNS0_9IDataTypeEES9_bENKUlS3_RNS0_5BlockERKSt6vectorImSaImEEmmE_clES3_SB_SG_mm
@ 0x555c1e2a8d42
_ZNSt17_Function_handlerIFN5doris6StatusEPNS0_15FunctionContextERNS0_10vectorized5BlockERKSt6vectorImSaImEEmmEZNKS4_12FunctionCast23prepare_remove_nullableES3_RKSt10shared_ptrIKNS4_9IDataTypeEESJ_bEUlS3_S6_SB_mmE_E9_M_invokeERKSt9_Any_dataOS3_S6_SB_OmSQ_
@ 0x555c1e20e42b
doris::vectorized::PreparedFunctionCast::execute_impl()
@ 0x555c1e15ac29
doris::vectorized::PreparedFunctionImpl::execute_without_low_cardinality_columns()
@ 0x555c1e15ac56 doris::vectorized::PreparedFunctionImpl::execute()
@ 0x555c1d63e960 doris::vectorized::IFunctionBase::execute()
@ 0x555c1d628700 doris::vectorized::VCastExpr::execute()
@ 0x555c1d6163e5 doris::vectorized::VExprContext::execute()
@ 0x555c20a83fe1
doris::vectorized::VFileScanner::_convert_to_output_block()
@ 0x555c20a809af doris::vectorized::VFileScanner::_get_block_impl()
@ 0x555c209b9bc4 doris::vectorized::VScanner::get_block()
@ 0x555c209b1a50
doris::vectorized::ScannerScheduler::_scanner_scan()
@ 0x555c209b2ac1
_ZNSt17_Function_handlerIFvvEZZN5doris10vectorized16ScannerScheduler18_schedule_scannersEPNS2_14ScannerContextEENK3$_0clEvEUlvE1_E9_M_invokeERKSt9_Any_data
@ 0x555c1a8378cf doris::ThreadPool::dispatch_thread()
@ 0x555c1a830fac doris::Thread::supervise_thread()
@ 0x7f461faa117a start_thread
@ 0x7f462033bdf3 __GI___clone
@ (nil) (unknown)
---
be/src/common/config.h | 3 --
be/src/runtime/exec_env_init.cpp | 4 +--
be/src/runtime/memory/mem_tracker.cpp | 2 +-
be/src/runtime/memory/thread_mem_tracker_mgr.h | 11 ------
be/src/runtime/thread_context.h | 46 +++++++++++++++-----------
be/src/vec/common/allocator.cpp | 21 ++++++------
6 files changed, 38 insertions(+), 49 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 9186e5881e..330721b7e7 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -686,9 +686,6 @@ CONF_Int32(aws_log_level, "3");
// the buffer size when read data from remote storage like s3
CONF_mInt32(remote_storage_read_buffer_mb, "16");
-// Whether Hook TCmalloc new/delete, currently consume/release tls mem tracker
in Hook.
-CONF_Bool(enable_tcmalloc_hook, "true");
-
// Print more detailed logs, more detailed records, etc.
CONF_mBool(memory_debug, "false");
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 9655c62a40..8d72a4d54f 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -212,9 +212,7 @@ Status ExecEnv::_init_mem_env() {
thread_context()->thread_mem_tracker_mgr->init();
#if defined(USE_MEM_TRACKER) && !defined(__SANITIZE_ADDRESS__) &&
!defined(ADDRESS_SANITIZER) && \
!defined(LEAK_SANITIZER) && !defined(THREAD_SANITIZER) &&
!defined(USE_JEMALLOC)
- if (doris::config::enable_tcmalloc_hook) {
- init_hook();
- }
+ init_hook();
#endif
// 2. init buffer pool
diff --git a/be/src/runtime/memory/mem_tracker.cpp
b/be/src/runtime/memory/mem_tracker.cpp
index 75e59385c2..bf3902821d 100644
--- a/be/src/runtime/memory/mem_tracker.cpp
+++ b/be/src/runtime/memory/mem_tracker.cpp
@@ -113,7 +113,7 @@ void
MemTracker::make_group_snapshot(std::vector<MemTracker::Snapshot>* snapshot
int64_t group_num, std::string
parent_label) {
std::lock_guard<std::mutex> l(mem_tracker_pool[group_num].group_lock);
for (auto tracker : mem_tracker_pool[group_num].trackers) {
- if (tracker->parent_label() == parent_label && tracker->consumption()
!= 0) {
+ if (tracker->parent_label() == parent_label &&
tracker->peak_consumption() != 0) {
snapshots->push_back(tracker->make_snapshot());
}
}
diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h
b/be/src/runtime/memory/thread_mem_tracker_mgr.h
index 1192bdb2a1..a0bef7a76b 100644
--- a/be/src/runtime/memory/thread_mem_tracker_mgr.h
+++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h
@@ -89,9 +89,6 @@ public:
return _limiter_tracker_raw;
}
- std::string exceed_mem_limit_msg() { return _exceed_mem_limit_msg; }
- void save_exceed_mem_limit_msg(const std::string& msg) {
_exceed_mem_limit_msg = msg; }
- void clear_exceed_mem_limit_msg() { _exceed_mem_limit_msg = ""; }
void disable_wait_gc() { _wait_gc = false; }
bool wait_gc() { return _wait_gc; }
void cancel_fragment(const std::string& exceed_msg);
@@ -119,7 +116,6 @@ private:
int64_t _scope_mem = 0;
std::string _failed_consume_msg = std::string();
- std::string _exceed_mem_limit_msg = std::string();
// If true, the Allocator will wait for the GC to free memory if it finds
that the memory exceed limit.
// A thread of query/load will only wait once during execution.
bool _wait_gc = false;
@@ -191,13 +187,6 @@ inline bool ThreadMemTrackerMgr::flush_untracked_mem() {
_stop_consume = true;
init();
DCHECK(_limiter_tracker_raw);
- if (doris::MemTrackerLimiter::sys_mem_exceed_limit_check(_untracked_mem)) {
- LOG(WARNING) << fmt::format(
- "MemHook alloc:{} failed, not enough system memory, consuming
tracker:<{}>, exec "
- "node:<{}>, {}.",
- _untracked_mem, _limiter_tracker_raw->label(),
last_consumer_tracker(),
-
doris::MemTrackerLimiter::process_limit_exceeded_errmsg_str(_untracked_mem));
- }
old_untracked_mem = _untracked_mem;
if (_count_scope_mem) _scope_mem += _untracked_mem;
diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h
index 983ad2fd0c..6c0fd3d996 100644
--- a/be/src/runtime/thread_context.h
+++ b/be/src/runtime/thread_context.h
@@ -35,6 +35,24 @@
#include "runtime/threadlocal.h"
#include "util/defer_op.h" // IWYU pragma: keep
+#define RETURN_IF_CATCH_EXCEPTION(stmt)
\
+ do {
\
+ try {
\
+ doris::enable_thread_catch_bad_alloc++;
\
+ Defer defer {[&]() { doris::enable_thread_catch_bad_alloc--; }};
\
+ { stmt; }
\
+ } catch (std::bad_alloc const& e) {
\
+ return Status::MemoryLimitExceeded(fmt::format("PreCatch {}",
e.what())); \
+ } catch (const doris::Exception& e) {
\
+ if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) {
\
+ return Status::MemoryLimitExceeded(
\
+ fmt::format("PreCatch error code:{}, {}", e.code(),
e.to_string())); \
+ } else {
\
+ return Status::Error(e.code(), e.to_string());
\
+ }
\
+ }
\
+ } while (0)
+
// Used to observe the memory usage of the specified code segment
#ifdef USE_MEM_TRACKER
// Count a code segment memory (memory malloc - memory free) to int64_t
@@ -74,6 +92,13 @@
#define SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(mem_tracker_limiter) (void)0
#endif
+#define SKIP_MEMORY_CHECK(...) \
+ do { \
+ doris::skip_memory_check++; \
+ DEFER({ doris::skip_memory_check--; }); \
+ __VA_ARGS__; \
+ } while (0)
+
namespace doris {
class ThreadContext;
@@ -118,6 +143,7 @@ public:
inline thread_local ThreadContextPtr thread_context_ptr;
inline thread_local int enable_thread_catch_bad_alloc = 0;
+inline thread_local int skip_memory_check = 0;
// To avoid performance problems caused by frequently calling
`bthread_getspecific` to obtain bthread TLS
// in tcmalloc hook, cache the key and value of bthread TLS in pthread TLS.
@@ -299,24 +325,6 @@ private:
tracker->transfer_to( \
size,
doris::thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker_raw())
-#define RETURN_IF_CATCH_EXCEPTION(stmt)
\
- do {
\
- try {
\
-
doris::thread_context()->thread_mem_tracker_mgr->clear_exceed_mem_limit_msg();
\
- doris::enable_thread_catch_bad_alloc++;
\
- Defer defer {[&]() { doris::enable_thread_catch_bad_alloc--; }};
\
- { stmt; }
\
- } catch (std::bad_alloc const& e) {
\
- doris::thread_context()->thread_mem_tracker()->print_log_usage(
\
-
doris::thread_context()->thread_mem_tracker_mgr->exceed_mem_limit_msg()); \
- return Status::MemoryLimitExceeded(fmt::format(
\
- "PreCatch {}, {}", e.what(),
\
-
doris::thread_context()->thread_mem_tracker_mgr->exceed_mem_limit_msg())); \
- } catch (const doris::Exception& e) {
\
- return Status::Error(e.code(), e.to_string());
\
- }
\
- } while (0)
-
// Mem Hook to consume thread mem tracker
// TODO: In the original design, the MemTracker consume method is called
before the memory is allocated.
// If the consume succeeds, the memory is actually allocated, otherwise an
exception is thrown.
@@ -346,7 +354,5 @@ private:
#define THREAD_MEM_TRACKER_TRANSFER_FROM(size, tracker) (void)0
#define CONSUME_MEM_TRACKER(size) (void)0
#define RELEASE_MEM_TRACKER(size) (void)0
-#define RETURN_IF_CATCH_EXCEPTION(stmt) \
- { stmt; }
#endif
} // namespace doris
diff --git a/be/src/vec/common/allocator.cpp b/be/src/vec/common/allocator.cpp
index e6b163734b..fed24bf539 100644
--- a/be/src/vec/common/allocator.cpp
+++ b/be/src/vec/common/allocator.cpp
@@ -34,6 +34,7 @@
template <bool clear_memory_, bool mmap_populate>
void Allocator<clear_memory_, mmap_populate>::sys_memory_check(size_t size)
const {
+ if (doris::skip_memory_check) return;
if (doris::MemTrackerLimiter::sys_mem_exceed_limit_check(size)) {
// Only thread attach query, and has not completely waited for
thread_wait_gc_max_milliseconds,
// will wait for gc, asynchronous cancel or throw bad::alloc.
@@ -58,13 +59,12 @@ void Allocator<clear_memory_,
mmap_populate>::sys_memory_check(size_t size) cons
size,
doris::thread_context()->thread_mem_tracker()->label(),
doris::thread_context()->thread_mem_tracker_mgr->last_consumer_tracker(),
doris::MemTrackerLimiter::process_limit_exceeded_errmsg_str(size));
+ doris::MemTrackerLimiter::print_log_process_usage(err_msg);
// If the external catch, throw bad::alloc first, let the
query actively cancel. Otherwise asynchronous cancel.
if (!doris::enable_thread_catch_bad_alloc) {
doris::thread_context()->thread_mem_tracker_mgr->cancel_fragment(err_msg);
} else {
-
doris::thread_context()->thread_mem_tracker_mgr->save_exceed_mem_limit_msg(
- err_msg);
- throw std::bad_alloc {};
+ throw doris::Exception(doris::ErrorCode::MEM_ALLOC_FAILED,
err_msg);
}
}
} else if (doris::enable_thread_catch_bad_alloc) {
@@ -74,14 +74,15 @@ void Allocator<clear_memory_,
mmap_populate>::sys_memory_check(size_t size) cons
size,
doris::thread_context()->thread_mem_tracker()->label(),
doris::thread_context()->thread_mem_tracker_mgr->last_consumer_tracker(),
doris::MemTrackerLimiter::process_limit_exceeded_errmsg_str(size));
-
doris::thread_context()->thread_mem_tracker_mgr->save_exceed_mem_limit_msg(err_msg);
- throw std::bad_alloc {};
+ doris::MemTrackerLimiter::print_log_process_usage(err_msg);
+ throw doris::Exception(doris::ErrorCode::MEM_ALLOC_FAILED,
err_msg);
}
}
}
template <bool clear_memory_, bool mmap_populate>
void Allocator<clear_memory_, mmap_populate>::memory_tracker_check(size_t
size) const {
+ if (doris::skip_memory_check) return;
auto st = doris::thread_context()->thread_mem_tracker()->check_limit(size);
if (!st) {
doris::thread_context()->thread_mem_tracker_mgr->disable_wait_gc();
@@ -90,13 +91,12 @@ void Allocator<clear_memory_,
mmap_populate>::memory_tracker_check(size_t size)
st.to_string(),
doris::thread_context()->thread_mem_tracker_mgr->last_consumer_tracker(),
"Allocator mem tracker check failed");
+
doris::thread_context()->thread_mem_tracker()->print_log_usage(err_msg);
// If the external catch, throw bad::alloc first, let the query
actively cancel. Otherwise asynchronous cancel.
if (!doris::enable_thread_catch_bad_alloc) {
doris::thread_context()->thread_mem_tracker_mgr->cancel_fragment(err_msg);
-
doris::thread_context()->thread_mem_tracker()->print_log_usage(err_msg);
} else {
-
doris::thread_context()->thread_mem_tracker_mgr->save_exceed_mem_limit_msg(err_msg);
- throw std::bad_alloc {};
+ throw doris::Exception(doris::ErrorCode::MEM_ALLOC_FAILED,
err_msg);
}
}
}
@@ -120,9 +120,8 @@ void Allocator<clear_memory_,
mmap_populate>::release_memory(size_t size) const
template <bool clear_memory_, bool mmap_populate>
void Allocator<clear_memory_, mmap_populate>::throw_bad_alloc(const
std::string& err) const {
LOG(WARNING) << err;
- if (!doris::enable_thread_catch_bad_alloc)
- doris::MemTrackerLimiter::print_log_process_usage(err);
- throw std::bad_alloc {};
+ doris::MemTrackerLimiter::print_log_process_usage(err);
+ throw doris::Exception(doris::ErrorCode::MEM_ALLOC_FAILED, err);
}
template class Allocator<true, true>;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
