This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch spill_and_reserve
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/spill_and_reserve by this push:
new 0545a93cd1e avoid scanner run exceed memory limit (#41209)
0545a93cd1e is described below
commit 0545a93cd1e0b530cc9a7a48e59183fd864d0eb4
Author: yiguolei <[email protected]>
AuthorDate: Tue Sep 24 15:44:35 2024 +0800
avoid scanner run exceed memory limit (#41209)
fix query limit reserve bug
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
Co-authored-by: yiguolei <[email protected]>
---
be/src/common/status.h | 6 ++---
be/src/pipeline/pipeline_task.cpp | 16 +------------
be/src/runtime/memory/mem_tracker_limiter.h | 26 +++++++---------------
be/src/runtime/memory/thread_mem_tracker_mgr.h | 22 ++++++++++--------
be/src/runtime/query_context.cpp | 5 +++--
be/src/runtime/query_context.h | 14 +++++++-----
be/src/runtime/workload_group/workload_group.cpp | 6 +++--
be/src/runtime/workload_group/workload_group.h | 1 +
.../workload_group/workload_group_manager.cpp | 8 +++----
be/src/vec/exec/scan/scanner_scheduler.cpp | 4 +++-
10 files changed, 49 insertions(+), 59 deletions(-)
diff --git a/be/src/common/status.h b/be/src/common/status.h
index c6e2712dc9c..64feaf12fc2 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -130,9 +130,9 @@ namespace ErrorCode {
E(BAD_CAST, -254, true); \
E(ARITHMETIC_OVERFLOW_ERRROR, -255, false); \
E(PERMISSION_DENIED, -256, false); \
- E(QUERY_MEMORY_EXCEED, -257, false); \
- E(WORKLOAD_GROUP_MEMORY_EXCEED, -257, false); \
- E(PROCESS_MEMORY_EXCEED, -257, false); \
+ E(QUERY_MEMORY_EXCEEDED, -257, false); \
+ E(WORKLOAD_GROUP_MEMORY_EXCEEDED, -258, false); \
+ E(PROCESS_MEMORY_EXCEEDED, -259, false); \
E(CE_CMD_PARAMS_ERROR, -300, true); \
E(CE_BUFFER_TOO_SMALL, -301, true); \
E(CE_CMD_NOT_VALID, -302, true); \
diff --git a/be/src/pipeline/pipeline_task.cpp
b/be/src/pipeline/pipeline_task.cpp
index 28ca578757d..356feea9c36 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -398,21 +398,7 @@ Status PipelineTask::execute(bool* eos) {
bool is_high_wartermark = false;
bool is_low_wartermark = false;
workload_group->check_mem_used(&is_low_wartermark,
&is_high_wartermark);
-
- /// The larger reserved memory size is likely due to a
larger available revocable size.
- /// If the available memory for revoking is large enough,
here trigger revoking proactively.
- bool need_to_pause = false;
- const auto revocable_mem_size =
_sink->revocable_mem_size(_state);
- if (revocable_mem_size > 100L * 1024 * 1024) {
- LOG(INFO) << "query: " << print_id(query_id)
- << ", task id: " << _state->task_id()
- << " has big memory to revoke: " <<
revocable_mem_size;
- RETURN_IF_ERROR(_sink->revoke_memory(_state));
- need_to_pause = true;
- } else {
- need_to_pause = is_low_wartermark ||
is_high_wartermark;
- }
- if (need_to_pause) {
+ if (is_low_wartermark || is_high_wartermark) {
_memory_sufficient_dependency->block();
ExecEnv::GetInstance()->workload_group_mgr()->add_paused_query(
_state->get_query_ctx()->shared_from_this());
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h
b/be/src/runtime/memory/mem_tracker_limiter.h
index bc7e966e27f..80ddde30351 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -150,23 +150,6 @@ public:
void release(int64_t bytes) { _mem_counter.sub(bytes); }
- bool try_consume(int64_t bytes) {
- if (UNLIKELY(bytes == 0)) {
- return true;
- }
- bool rt = true;
- if (is_overcommit_tracker() &&
!config::enable_query_memory_overcommit) {
- rt = _mem_counter.try_add(bytes, _limit);
- } else {
- _mem_counter.add(bytes);
- }
- if (rt && _query_statistics) {
- _query_statistics->set_max_peak_memory_bytes(peak_consumption());
- _query_statistics->set_current_used_memory_bytes(consumption());
- }
- return rt;
- }
-
void set_consumption(int64_t bytes) { _mem_counter.set(bytes); }
// Transfer 'bytes' of consumption from this tracker to 'dst'.
@@ -189,7 +172,14 @@ public:
int64_t reserved_peak_consumption() const { return
_reserved_counter.peak_value(); }
bool try_reserve(int64_t bytes) {
- bool rt = try_consume(bytes);
+ if (UNLIKELY(bytes == 0)) {
+ return true;
+ }
+ bool rt = _mem_counter.try_add(bytes, _limit);
+ if (rt && _query_statistics) {
+ _query_statistics->set_max_peak_memory_bytes(peak_consumption());
+ _query_statistics->set_current_used_memory_bytes(consumption());
+ }
if (rt) {
_reserved_counter.add(bytes);
}
diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h
b/be/src/runtime/memory/thread_mem_tracker_mgr.h
index 01df735fff1..80b00d32a0b 100644
--- a/be/src/runtime/memory/thread_mem_tracker_mgr.h
+++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h
@@ -285,21 +285,25 @@ inline doris::Status
ThreadMemTrackerMgr::try_reserve(int64_t size) {
// if _reserved_mem not equal to 0, repeat reserve,
// _untracked_mem store bytes that not synchronized to process reserved
memory.
flush_untracked_mem();
- if (!_limiter_tracker->try_reserve(size)) {
- auto err_msg = fmt::format(
- "reserve memory failed, size: {}, because memory tracker
consumption: {}, limit: "
- "{}",
- size, _limiter_tracker->consumption(),
_limiter_tracker->limit());
- return doris::Status::Error<ErrorCode::QUERY_MEMORY_EXCEED>(err_msg);
- }
auto wg_ptr = _wg_wptr.lock();
+ // If wg not exist or wg enable overcommit, then query's memlimit is not
considered.
+ if (wg_ptr != nullptr && !wg_ptr->enable_memory_overcommit()) {
+ if (!_limiter_tracker->try_reserve(size)) {
+ auto err_msg = fmt::format(
+ "reserve memory failed, size: {}, because memory tracker
consumption: {}, "
+ "limit: "
+ "{}",
+ size, _limiter_tracker->consumption(),
_limiter_tracker->limit());
+ return
doris::Status::Error<ErrorCode::QUERY_MEMORY_EXCEEDED>(err_msg);
+ }
+ }
if (wg_ptr) {
if (!wg_ptr->add_wg_refresh_interval_memory_growth(size)) {
auto err_msg = fmt::format("reserve memory failed, size: {},
because {}", size,
wg_ptr->memory_debug_string());
_limiter_tracker->release(size); // rollback
_limiter_tracker->release_reserved(size); // rollback
- return
doris::Status::Error<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEED>(err_msg);
+ return
doris::Status::Error<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>(err_msg);
}
}
if (!doris::GlobalMemoryArbitrator::try_reserve_process_memory(size)) {
@@ -310,7 +314,7 @@ inline doris::Status
ThreadMemTrackerMgr::try_reserve(int64_t size) {
if (wg_ptr) {
wg_ptr->sub_wg_refresh_interval_memory_growth(size); // rollback
}
- return doris::Status::Error<ErrorCode::PROCESS_MEMORY_EXCEED>(err_msg);
+ return
doris::Status::Error<ErrorCode::PROCESS_MEMORY_EXCEEDED>(err_msg);
}
_reserved_mem += size;
return doris::Status::OK();
diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp
index 2932bb708ec..6a0cf20a4dd 100644
--- a/be/src/runtime/query_context.cpp
+++ b/be/src/runtime/query_context.cpp
@@ -118,11 +118,11 @@ QueryContext::QueryContext(TUniqueId query_id, ExecEnv*
exec_env,
void QueryContext::_init_query_mem_tracker() {
bool has_query_mem_limit = _query_options.__isset.mem_limit &&
(_query_options.mem_limit > 0);
int64_t bytes_limit = has_query_mem_limit ? _query_options.mem_limit : -1;
- if (bytes_limit > MemInfo::mem_limit()) {
+ if (bytes_limit > MemInfo::mem_limit() || bytes_limit == -1) {
VLOG_NOTICE << "Query memory limit " <<
PrettyPrinter::print(bytes_limit, TUnit::BYTES)
<< " exceeds process memory limit of "
<< PrettyPrinter::print(MemInfo::mem_limit(), TUnit::BYTES)
- << ". Using process memory limit instead";
+ << " OR is -1. Using process memory limit instead.";
bytes_limit = MemInfo::mem_limit();
}
if (_query_options.query_type == TQueryType::SELECT) {
@@ -141,6 +141,7 @@ void QueryContext::_init_query_mem_tracker() {
if (_query_options.__isset.is_report_success &&
_query_options.is_report_success) {
query_mem_tracker->enable_print_log_usage();
}
+ _user_set_mem_limit = bytes_limit;
}
QueryContext::~QueryContext() {
diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h
index 471f82ba807..96b18eedfb9 100644
--- a/be/src/runtime/query_context.h
+++ b/be/src/runtime/query_context.h
@@ -236,7 +236,11 @@ public:
bool* has_running_task) const;
size_t get_revocable_size() const;
- void set_mem_limit(int64_t new_mem_limit) {
query_mem_tracker->set_limit(new_mem_limit); }
+ // This method is called by workload group manager to set query's memlimit
using slot
+ // If user set query limit explicitly, then should use less one
+ void set_mem_limit(int64_t new_mem_limit) {
+ query_mem_tracker->set_limit(std::min<int64_t>(new_mem_limit,
_user_set_mem_limit));
+ }
std::shared_ptr<MemTrackerLimiter>& get_mem_tracker() { return
query_mem_tracker; }
@@ -323,10 +327,10 @@ public:
void update_paused_reason(const Status& st) {
std::lock_guard l(_paused_mutex);
- if (_paused_reason.is<ErrorCode::QUERY_MEMORY_EXCEED>()) {
+ if (_paused_reason.is<ErrorCode::QUERY_MEMORY_EXCEEDED>()) {
return;
- } else if
(_paused_reason.is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEED>()) {
- if (st.is<ErrorCode::QUERY_MEMORY_EXCEED>()) {
+ } else if
(_paused_reason.is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>()) {
+ if (st.is<ErrorCode::QUERY_MEMORY_EXCEEDED>()) {
_paused_reason = st;
return;
} else {
@@ -389,8 +393,8 @@ private:
std::map<int, std::weak_ptr<pipeline::PipelineFragmentContext>>
_fragment_id_to_pipeline_ctx;
std::mutex _pipeline_map_write_lock;
- std::atomic<int64_t> _spill_threshold {0};
std::atomic<bool> _low_memory_mode = false;
+ int64_t _user_set_mem_limit = 0;
std::mutex _profile_mutex;
timespec _query_arrival_timestamp;
diff --git a/be/src/runtime/workload_group/workload_group.cpp
b/be/src/runtime/workload_group/workload_group.cpp
index b4afe462696..e0438a9d729 100644
--- a/be/src/runtime/workload_group/workload_group.cpp
+++ b/be/src/runtime/workload_group/workload_group.cpp
@@ -84,6 +84,8 @@ WorkloadGroup::WorkloadGroup(const WorkloadGroupInfo& tg_info)
_total_local_scan_io_per_second =
std::make_unique<bvar::PerSecond<bvar::Adder<size_t>>>(
_name, "total_local_read_bytes_per_second",
_total_local_scan_io_adder.get(), 1);
_load_buffer_limit = (int64_t)(_memory_limit * 0.2);
+ // Its initial value should equal to memory limit, or it will be 0 and all
reserve memory request will failed.
+ _weighted_memory_limit = _memory_limit;
}
std::string WorkloadGroup::debug_string() const {
@@ -114,9 +116,9 @@ std::string WorkloadGroup::debug_string() const {
std::string WorkloadGroup::memory_debug_string() const {
auto realtime_total_mem_used = _total_mem_used +
_wg_refresh_interval_memory_growth.load();
- auto mem_used_ratio = realtime_total_mem_used /
(double)_weighted_memory_limit;
+ auto mem_used_ratio = realtime_total_mem_used /
((double)_weighted_memory_limit + 1);
return fmt::format(
- "TG[id = {}, name = {}, memory_limit = {},
enable_memory_overcommit = "
+ "WorkloadGroup[id = {}, name = {}, memory_limit = {},
enable_memory_overcommit = "
"{}, weighted_memory_limit = {}, total_mem_used = {},"
"wg_refresh_interval_memory_growth = {}, mem_used_ratio = {},
spill_low_watermark = "
"{}, "
diff --git a/be/src/runtime/workload_group/workload_group.h
b/be/src/runtime/workload_group/workload_group.h
index 86e5f494f77..1d640afee1d 100644
--- a/be/src/runtime/workload_group/workload_group.h
+++ b/be/src/runtime/workload_group/workload_group.h
@@ -121,6 +121,7 @@ public:
return true;
}
}
+
void sub_wg_refresh_interval_memory_growth(int64_t size) {
_wg_refresh_interval_memory_growth.fetch_sub(size);
}
diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp
b/be/src/runtime/workload_group/workload_group_manager.cpp
index b9a7989d59e..0dceaac7b3d 100644
--- a/be/src/runtime/workload_group/workload_group_manager.cpp
+++ b/be/src/runtime/workload_group/workload_group_manager.cpp
@@ -381,7 +381,7 @@ void WorkloadGroupMgr::handle_paused_queries() {
query_it = queries_list.erase(query_it);
continue;
}
- if
(query_ctx->paused_reason().is<ErrorCode::QUERY_MEMORY_EXCEED>()) {
+ if
(query_ctx->paused_reason().is<ErrorCode::QUERY_MEMORY_EXCEEDED>()) {
bool spill_res = spill_or_cancel_query(query_ctx,
query_ctx->paused_reason());
if (!spill_res) {
++query_it;
@@ -390,7 +390,7 @@ void WorkloadGroupMgr::handle_paused_queries() {
query_it = queries_list.erase(query_it);
continue;
}
- } else if
(query_ctx->paused_reason().is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEED>()) {
+ } else if
(query_ctx->paused_reason().is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>()) {
if (wg->memory_sufficent()) {
wg->update_memory_sufficent(false);
LOG(INFO) << "query: " << print_id(query_ctx->query_id())
@@ -459,13 +459,13 @@ bool
WorkloadGroupMgr::spill_or_cancel_query(std::shared_ptr<QueryContext> query
query_ctx->get_revocable_info(&revocable_size, &memory_usage,
&has_running_task);
if (has_running_task) {
LOG(INFO) << "query: " << print_id(query_ctx->query_id())
- << "is paused, but still has running task, skip it.";
+ << " is paused, but still has running task, skip it.";
return false;
}
auto revocable_tasks = query_ctx->get_revocable_tasks();
if (revocable_tasks.empty()) {
- if (paused_reason.is<ErrorCode::QUERY_MEMORY_EXCEED>()) {
+ if (paused_reason.is<ErrorCode::QUERY_MEMORY_EXCEEDED>()) {
// Use MEM_LIMIT_EXCEEDED so that FE could parse the error code
and do try logic
query_ctx->cancel(doris::Status::Error<ErrorCode::MEM_LIMIT_EXCEEDED>(
"query reserve memory failed, but could not find memory
that "
diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp
b/be/src/vec/exec/scan/scanner_scheduler.cpp
index 87eda999b41..b3c23750138 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -276,7 +276,9 @@ void
ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
// During low memory mode, every scan task will return at most 2
block to reduce memory usage.
while (!eos && raw_bytes_read < raw_bytes_threshold &&
- !(ctx->low_memory_mode() && has_first_full_block)) {
+ !(ctx->low_memory_mode() && has_first_full_block) &&
+ !(has_first_full_block &&
+
doris::thread_context()->thread_mem_tracker()->limit_exceeded())) {
if (UNLIKELY(ctx->done())) {
eos = true;
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]