This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch spill_and_reserve
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/spill_and_reserve by this push:
     new 0545a93cd1e avoid scanner run exceed memory limit (#41209)
0545a93cd1e is described below

commit 0545a93cd1e0b530cc9a7a48e59183fd864d0eb4
Author: yiguolei <[email protected]>
AuthorDate: Tue Sep 24 15:44:35 2024 +0800

    avoid scanner run exceed memory limit (#41209)
    
    fix query limit reserve bug
    
    ## Proposed changes
    
    Issue Number: close #xxx
    
    <!--Describe your changes.-->
    
    Co-authored-by: yiguolei <[email protected]>
---
 be/src/common/status.h                             |  6 ++---
 be/src/pipeline/pipeline_task.cpp                  | 16 +------------
 be/src/runtime/memory/mem_tracker_limiter.h        | 26 +++++++---------------
 be/src/runtime/memory/thread_mem_tracker_mgr.h     | 22 ++++++++++--------
 be/src/runtime/query_context.cpp                   |  5 +++--
 be/src/runtime/query_context.h                     | 14 +++++++-----
 be/src/runtime/workload_group/workload_group.cpp   |  6 +++--
 be/src/runtime/workload_group/workload_group.h     |  1 +
 .../workload_group/workload_group_manager.cpp      |  8 +++----
 be/src/vec/exec/scan/scanner_scheduler.cpp         |  4 +++-
 10 files changed, 49 insertions(+), 59 deletions(-)

diff --git a/be/src/common/status.h b/be/src/common/status.h
index c6e2712dc9c..64feaf12fc2 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -130,9 +130,9 @@ namespace ErrorCode {
     E(BAD_CAST, -254, true);                                 \
     E(ARITHMETIC_OVERFLOW_ERRROR, -255, false);              \
     E(PERMISSION_DENIED, -256, false);                       \
-    E(QUERY_MEMORY_EXCEED, -257, false);                     \
-    E(WORKLOAD_GROUP_MEMORY_EXCEED, -257, false);            \
-    E(PROCESS_MEMORY_EXCEED, -257, false);                   \
+    E(QUERY_MEMORY_EXCEEDED, -257, false);                   \
+    E(WORKLOAD_GROUP_MEMORY_EXCEEDED, -258, false);          \
+    E(PROCESS_MEMORY_EXCEEDED, -259, false);                 \
     E(CE_CMD_PARAMS_ERROR, -300, true);                      \
     E(CE_BUFFER_TOO_SMALL, -301, true);                      \
     E(CE_CMD_NOT_VALID, -302, true);                         \
diff --git a/be/src/pipeline/pipeline_task.cpp 
b/be/src/pipeline/pipeline_task.cpp
index 28ca578757d..356feea9c36 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -398,21 +398,7 @@ Status PipelineTask::execute(bool* eos) {
                     bool is_high_wartermark = false;
                     bool is_low_wartermark = false;
                     workload_group->check_mem_used(&is_low_wartermark, 
&is_high_wartermark);
-
-                    /// The larger reserved memory size is likely due to a 
larger available revocable size.
-                    /// If the available memory for revoking is large enough, 
here trigger revoking proactively.
-                    bool need_to_pause = false;
-                    const auto revocable_mem_size = 
_sink->revocable_mem_size(_state);
-                    if (revocable_mem_size > 100L * 1024 * 1024) {
-                        LOG(INFO) << "query: " << print_id(query_id)
-                                  << ", task id: " << _state->task_id()
-                                  << " has big memory to revoke: " << 
revocable_mem_size;
-                        RETURN_IF_ERROR(_sink->revoke_memory(_state));
-                        need_to_pause = true;
-                    } else {
-                        need_to_pause = is_low_wartermark || 
is_high_wartermark;
-                    }
-                    if (need_to_pause) {
+                    if (is_low_wartermark || is_high_wartermark) {
                         _memory_sufficient_dependency->block();
                         
ExecEnv::GetInstance()->workload_group_mgr()->add_paused_query(
                                 _state->get_query_ctx()->shared_from_this());
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h 
b/be/src/runtime/memory/mem_tracker_limiter.h
index bc7e966e27f..80ddde30351 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -150,23 +150,6 @@ public:
 
     void release(int64_t bytes) { _mem_counter.sub(bytes); }
 
-    bool try_consume(int64_t bytes) {
-        if (UNLIKELY(bytes == 0)) {
-            return true;
-        }
-        bool rt = true;
-        if (is_overcommit_tracker() && 
!config::enable_query_memory_overcommit) {
-            rt = _mem_counter.try_add(bytes, _limit);
-        } else {
-            _mem_counter.add(bytes);
-        }
-        if (rt && _query_statistics) {
-            _query_statistics->set_max_peak_memory_bytes(peak_consumption());
-            _query_statistics->set_current_used_memory_bytes(consumption());
-        }
-        return rt;
-    }
-
     void set_consumption(int64_t bytes) { _mem_counter.set(bytes); }
 
     // Transfer 'bytes' of consumption from this tracker to 'dst'.
@@ -189,7 +172,14 @@ public:
     int64_t reserved_peak_consumption() const { return 
_reserved_counter.peak_value(); }
 
     bool try_reserve(int64_t bytes) {
-        bool rt = try_consume(bytes);
+        if (UNLIKELY(bytes == 0)) {
+            return true;
+        }
+        bool rt = _mem_counter.try_add(bytes, _limit);
+        if (rt && _query_statistics) {
+            _query_statistics->set_max_peak_memory_bytes(peak_consumption());
+            _query_statistics->set_current_used_memory_bytes(consumption());
+        }
         if (rt) {
             _reserved_counter.add(bytes);
         }
diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h 
b/be/src/runtime/memory/thread_mem_tracker_mgr.h
index 01df735fff1..80b00d32a0b 100644
--- a/be/src/runtime/memory/thread_mem_tracker_mgr.h
+++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h
@@ -285,21 +285,25 @@ inline doris::Status 
ThreadMemTrackerMgr::try_reserve(int64_t size) {
     // if _reserved_mem not equal to 0, repeat reserve,
     // _untracked_mem store bytes that not synchronized to process reserved 
memory.
     flush_untracked_mem();
-    if (!_limiter_tracker->try_reserve(size)) {
-        auto err_msg = fmt::format(
-                "reserve memory failed, size: {}, because memory tracker 
consumption: {}, limit: "
-                "{}",
-                size, _limiter_tracker->consumption(), 
_limiter_tracker->limit());
-        return doris::Status::Error<ErrorCode::QUERY_MEMORY_EXCEED>(err_msg);
-    }
     auto wg_ptr = _wg_wptr.lock();
+    // If wg not exist or wg enable overcommit, then query's memlimit is not 
considered.
+    if (wg_ptr != nullptr && !wg_ptr->enable_memory_overcommit()) {
+        if (!_limiter_tracker->try_reserve(size)) {
+            auto err_msg = fmt::format(
+                    "reserve memory failed, size: {}, because memory tracker 
consumption: {}, "
+                    "limit: "
+                    "{}",
+                    size, _limiter_tracker->consumption(), 
_limiter_tracker->limit());
+            return 
doris::Status::Error<ErrorCode::QUERY_MEMORY_EXCEEDED>(err_msg);
+        }
+    }
     if (wg_ptr) {
         if (!wg_ptr->add_wg_refresh_interval_memory_growth(size)) {
             auto err_msg = fmt::format("reserve memory failed, size: {}, 
because {}", size,
                                        wg_ptr->memory_debug_string());
             _limiter_tracker->release(size);          // rollback
             _limiter_tracker->release_reserved(size); // rollback
-            return 
doris::Status::Error<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEED>(err_msg);
+            return 
doris::Status::Error<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>(err_msg);
         }
     }
     if (!doris::GlobalMemoryArbitrator::try_reserve_process_memory(size)) {
@@ -310,7 +314,7 @@ inline doris::Status 
ThreadMemTrackerMgr::try_reserve(int64_t size) {
         if (wg_ptr) {
             wg_ptr->sub_wg_refresh_interval_memory_growth(size); // rollback
         }
-        return doris::Status::Error<ErrorCode::PROCESS_MEMORY_EXCEED>(err_msg);
+        return 
doris::Status::Error<ErrorCode::PROCESS_MEMORY_EXCEEDED>(err_msg);
     }
     _reserved_mem += size;
     return doris::Status::OK();
diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp
index 2932bb708ec..6a0cf20a4dd 100644
--- a/be/src/runtime/query_context.cpp
+++ b/be/src/runtime/query_context.cpp
@@ -118,11 +118,11 @@ QueryContext::QueryContext(TUniqueId query_id, ExecEnv* 
exec_env,
 void QueryContext::_init_query_mem_tracker() {
     bool has_query_mem_limit = _query_options.__isset.mem_limit && 
(_query_options.mem_limit > 0);
     int64_t bytes_limit = has_query_mem_limit ? _query_options.mem_limit : -1;
-    if (bytes_limit > MemInfo::mem_limit()) {
+    if (bytes_limit > MemInfo::mem_limit() || bytes_limit == -1) {
         VLOG_NOTICE << "Query memory limit " << 
PrettyPrinter::print(bytes_limit, TUnit::BYTES)
                     << " exceeds process memory limit of "
                     << PrettyPrinter::print(MemInfo::mem_limit(), TUnit::BYTES)
-                    << ". Using process memory limit instead";
+                    << " OR is -1. Using process memory limit instead.";
         bytes_limit = MemInfo::mem_limit();
     }
     if (_query_options.query_type == TQueryType::SELECT) {
@@ -141,6 +141,7 @@ void QueryContext::_init_query_mem_tracker() {
     if (_query_options.__isset.is_report_success && 
_query_options.is_report_success) {
         query_mem_tracker->enable_print_log_usage();
     }
+    _user_set_mem_limit = bytes_limit;
 }
 
 QueryContext::~QueryContext() {
diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h
index 471f82ba807..96b18eedfb9 100644
--- a/be/src/runtime/query_context.h
+++ b/be/src/runtime/query_context.h
@@ -236,7 +236,11 @@ public:
                             bool* has_running_task) const;
     size_t get_revocable_size() const;
 
-    void set_mem_limit(int64_t new_mem_limit) { 
query_mem_tracker->set_limit(new_mem_limit); }
+    // This method is called by workload group manager to set query's memlimit 
using slot
+    // If user set query limit explicitly, then should use less one
+    void set_mem_limit(int64_t new_mem_limit) {
+        query_mem_tracker->set_limit(std::min<int64_t>(new_mem_limit, 
_user_set_mem_limit));
+    }
 
     std::shared_ptr<MemTrackerLimiter>& get_mem_tracker() { return 
query_mem_tracker; }
 
@@ -323,10 +327,10 @@ public:
 
     void update_paused_reason(const Status& st) {
         std::lock_guard l(_paused_mutex);
-        if (_paused_reason.is<ErrorCode::QUERY_MEMORY_EXCEED>()) {
+        if (_paused_reason.is<ErrorCode::QUERY_MEMORY_EXCEEDED>()) {
             return;
-        } else if 
(_paused_reason.is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEED>()) {
-            if (st.is<ErrorCode::QUERY_MEMORY_EXCEED>()) {
+        } else if 
(_paused_reason.is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>()) {
+            if (st.is<ErrorCode::QUERY_MEMORY_EXCEEDED>()) {
                 _paused_reason = st;
                 return;
             } else {
@@ -389,8 +393,8 @@ private:
     std::map<int, std::weak_ptr<pipeline::PipelineFragmentContext>> 
_fragment_id_to_pipeline_ctx;
     std::mutex _pipeline_map_write_lock;
 
-    std::atomic<int64_t> _spill_threshold {0};
     std::atomic<bool> _low_memory_mode = false;
+    int64_t _user_set_mem_limit = 0;
 
     std::mutex _profile_mutex;
     timespec _query_arrival_timestamp;
diff --git a/be/src/runtime/workload_group/workload_group.cpp 
b/be/src/runtime/workload_group/workload_group.cpp
index b4afe462696..e0438a9d729 100644
--- a/be/src/runtime/workload_group/workload_group.cpp
+++ b/be/src/runtime/workload_group/workload_group.cpp
@@ -84,6 +84,8 @@ WorkloadGroup::WorkloadGroup(const WorkloadGroupInfo& tg_info)
     _total_local_scan_io_per_second = 
std::make_unique<bvar::PerSecond<bvar::Adder<size_t>>>(
             _name, "total_local_read_bytes_per_second", 
_total_local_scan_io_adder.get(), 1);
     _load_buffer_limit = (int64_t)(_memory_limit * 0.2);
+    // Its initial value should equal to memory limit, or it will be 0 and all 
reserve memory request will failed.
+    _weighted_memory_limit = _memory_limit;
 }
 
 std::string WorkloadGroup::debug_string() const {
@@ -114,9 +116,9 @@ std::string WorkloadGroup::debug_string() const {
 
 std::string WorkloadGroup::memory_debug_string() const {
     auto realtime_total_mem_used = _total_mem_used + 
_wg_refresh_interval_memory_growth.load();
-    auto mem_used_ratio = realtime_total_mem_used / 
(double)_weighted_memory_limit;
+    auto mem_used_ratio = realtime_total_mem_used / 
((double)_weighted_memory_limit + 1);
     return fmt::format(
-            "TG[id = {}, name = {}, memory_limit = {}, 
enable_memory_overcommit = "
+            "WorkloadGroup[id = {}, name = {}, memory_limit = {}, 
enable_memory_overcommit = "
             "{}, weighted_memory_limit = {}, total_mem_used = {},"
             "wg_refresh_interval_memory_growth = {},  mem_used_ratio = {}, 
spill_low_watermark = "
             "{}, "
diff --git a/be/src/runtime/workload_group/workload_group.h 
b/be/src/runtime/workload_group/workload_group.h
index 86e5f494f77..1d640afee1d 100644
--- a/be/src/runtime/workload_group/workload_group.h
+++ b/be/src/runtime/workload_group/workload_group.h
@@ -121,6 +121,7 @@ public:
             return true;
         }
     }
+
     void sub_wg_refresh_interval_memory_growth(int64_t size) {
         _wg_refresh_interval_memory_growth.fetch_sub(size);
     }
diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp 
b/be/src/runtime/workload_group/workload_group_manager.cpp
index b9a7989d59e..0dceaac7b3d 100644
--- a/be/src/runtime/workload_group/workload_group_manager.cpp
+++ b/be/src/runtime/workload_group/workload_group_manager.cpp
@@ -381,7 +381,7 @@ void WorkloadGroupMgr::handle_paused_queries() {
                 query_it = queries_list.erase(query_it);
                 continue;
             }
-            if 
(query_ctx->paused_reason().is<ErrorCode::QUERY_MEMORY_EXCEED>()) {
+            if 
(query_ctx->paused_reason().is<ErrorCode::QUERY_MEMORY_EXCEEDED>()) {
                 bool spill_res = spill_or_cancel_query(query_ctx, 
query_ctx->paused_reason());
                 if (!spill_res) {
                     ++query_it;
@@ -390,7 +390,7 @@ void WorkloadGroupMgr::handle_paused_queries() {
                     query_it = queries_list.erase(query_it);
                     continue;
                 }
-            } else if 
(query_ctx->paused_reason().is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEED>()) {
+            } else if 
(query_ctx->paused_reason().is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>()) {
                 if (wg->memory_sufficent()) {
                     wg->update_memory_sufficent(false);
                     LOG(INFO) << "query: " << print_id(query_ctx->query_id())
@@ -459,13 +459,13 @@ bool 
WorkloadGroupMgr::spill_or_cancel_query(std::shared_ptr<QueryContext> query
     query_ctx->get_revocable_info(&revocable_size, &memory_usage, 
&has_running_task);
     if (has_running_task) {
         LOG(INFO) << "query: " << print_id(query_ctx->query_id())
-                  << "is paused, but still has running task, skip it.";
+                  << " is paused, but still has running task, skip it.";
         return false;
     }
 
     auto revocable_tasks = query_ctx->get_revocable_tasks();
     if (revocable_tasks.empty()) {
-        if (paused_reason.is<ErrorCode::QUERY_MEMORY_EXCEED>()) {
+        if (paused_reason.is<ErrorCode::QUERY_MEMORY_EXCEEDED>()) {
             // Use MEM_LIMIT_EXCEEDED so that FE could parse the error code 
and do try logic
             
query_ctx->cancel(doris::Status::Error<ErrorCode::MEM_LIMIT_EXCEEDED>(
                     "query reserve memory failed, but could not find  memory 
that "
diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp 
b/be/src/vec/exec/scan/scanner_scheduler.cpp
index 87eda999b41..b3c23750138 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -276,7 +276,9 @@ void 
ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
 
             // During low memory mode, every scan task will return at most 2 
block to reduce memory usage.
             while (!eos && raw_bytes_read < raw_bytes_threshold &&
-                   !(ctx->low_memory_mode() && has_first_full_block)) {
+                   !(ctx->low_memory_mode() && has_first_full_block) &&
+                   !(has_first_full_block &&
+                     
doris::thread_context()->thread_mem_tracker()->limit_exceeded())) {
                 if (UNLIKELY(ctx->done())) {
                     eos = true;
                     break;


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to