This is an automated email from the ASF dual-hosted git repository.

lihaopeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 8be43857ef [feature](executor) Add memory limit for 
pip_scanner_context (#18238)
8be43857ef is described below

commit 8be43857ef6d5d5c83ddba997118945d2873c326
Author: HappenLee <[email protected]>
AuthorDate: Fri Mar 31 09:36:57 2023 +0800

    [feature](executor) Add memory limit for pip_scanner_context (#18238)
    
    Co-authored-by: wangbo <[email protected]>
---
 be/src/pipeline/exec/scan_operator.cpp     |  5 +++-
 be/src/vec/exec/scan/pip_scanner_context.h | 18 ++++++++++--
 be/src/vec/exec/scan/scanner_context.cpp   | 47 +++++++++++++++---------------
 be/src/vec/exec/scan/scanner_context.h     | 11 ++++---
 4 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/be/src/pipeline/exec/scan_operator.cpp 
b/be/src/pipeline/exec/scan_operator.cpp
index 6451927f77..60864c6dd9 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -32,12 +32,15 @@ bool ScanOperator::can_read() {
             return false;
         }
     } else {
-        if (_node->_eos || _node->_scanner_ctx->done() || 
_node->_scanner_ctx->no_schedule()) {
+        if (_node->_eos || _node->_scanner_ctx->done()) {
             // _eos: need eos
             // _scanner_ctx->done(): need finish
             // _scanner_ctx->no_schedule(): should schedule _scanner_ctx
             return true;
         } else {
+            if (_node->_scanner_ctx->has_enough_space_in_blocks_queue()) {
+                _node->_scanner_ctx->reschedule_scanner_ctx();
+            }
             return _node->ready_to_read(); // there are some blocks to process
         }
     }
diff --git a/be/src/vec/exec/scan/pip_scanner_context.h 
b/be/src/vec/exec/scan/pip_scanner_context.h
index 6a38617a9d..cacc61df1d 100644
--- a/be/src/vec/exec/scan/pip_scanner_context.h
+++ b/be/src/vec/exec/scan/pip_scanner_context.h
@@ -51,11 +51,12 @@ public:
             if (!_blocks_queues[id].empty()) {
                 *block = std::move(_blocks_queues[id].front());
                 _blocks_queues[id].pop_front();
-                return Status::OK();
             } else {
                 *eos = _is_finished || _should_stop;
+                return Status::OK();
             }
         }
+        _current_used_bytes -= (*block)->allocated_bytes();
         return Status::OK();
     }
 
@@ -65,6 +66,11 @@ public:
     void append_blocks_to_queue(std::vector<vectorized::BlockUPtr>& blocks) 
override {
         const int queue_size = _queue_mutexs.size();
         const int block_size = blocks.size();
+        int64_t local_bytes = 0;
+        for (const auto& block : blocks) {
+            local_bytes += block->allocated_bytes();
+        }
+
         for (int i = 0; i < queue_size && i < block_size; ++i) {
             int queue = _next_queue_to_feed;
             {
@@ -75,6 +81,7 @@ public:
             }
             _next_queue_to_feed = queue + 1 < queue_size ? queue + 1 : 0;
         }
+        _current_used_bytes += local_bytes;
     }
 
     bool empty_in_queue(int id) override {
@@ -82,17 +89,24 @@ public:
         return _blocks_queues[id].empty();
     }
 
-    void set_max_queue_size(int max_queue_size) override {
+    void set_max_queue_size(const int max_queue_size) override {
+        _max_queue_size = max_queue_size;
         for (int i = 0; i < max_queue_size; ++i) {
             _queue_mutexs.emplace_back(new std::mutex);
             _blocks_queues.emplace_back(std::list<vectorized::BlockUPtr>());
         }
     }
 
+    bool has_enough_space_in_blocks_queue() const override {
+        return _current_used_bytes < _max_bytes_in_queue / 2 * _max_queue_size;
+    }
+
 private:
+    int _max_queue_size = 1;
     int _next_queue_to_feed = 0;
     std::vector<std::unique_ptr<std::mutex>> _queue_mutexs;
     std::vector<std::list<vectorized::BlockUPtr>> _blocks_queues;
+    std::atomic_int64_t _current_used_bytes = 0;
 };
 } // namespace pipeline
 } // namespace doris
diff --git a/be/src/vec/exec/scan/scanner_context.cpp 
b/be/src/vec/exec/scan/scanner_context.cpp
index 16d8971227..0b3cbcd3d7 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -164,7 +164,7 @@ Status ScannerContext::get_block_from_queue(RuntimeState* 
state, vectorized::Blo
     // (if the scheduler continues to schedule, it will cause a lot of busy 
running).
     // At this point, consumers are required to trigger new scheduling to 
ensure that
     // data can be continuously fetched.
-    if (_has_enough_space_in_blocks_queue() && _num_running_scanners == 0) {
+    if (has_enough_space_in_blocks_queue() && _num_running_scanners == 0) {
         _num_scheduling_ctx++;
         _scanner_scheduler->submit(this);
     }
@@ -288,6 +288,15 @@ std::string ScannerContext::debug_string() {
             _max_thread_num, _block_per_scanner, _cur_bytes_in_queue, 
_max_bytes_in_queue);
 }
 
+void ScannerContext::reschedule_scanner_ctx() {
+    std::lock_guard l(_transfer_lock);
+    auto submit_st = _scanner_scheduler->submit(this);
+    //todo(wb) rethinking is it better to mark current scan_context failed 
when submit failed many times?
+    if (submit_st.ok()) {
+        _num_scheduling_ctx++;
+    }
+}
+
 void ScannerContext::push_back_scanner_and_reschedule(VScanner* scanner) {
     {
         std::unique_lock l(_scanners_lock);
@@ -295,10 +304,12 @@ void 
ScannerContext::push_back_scanner_and_reschedule(VScanner* scanner) {
     }
 
     std::lock_guard l(_transfer_lock);
-    _num_scheduling_ctx++;
-    auto submit_st = _scanner_scheduler->submit(this);
-    if (!submit_st.ok()) {
-        _num_scheduling_ctx--;
+    if (has_enough_space_in_blocks_queue()) {
+        _num_scheduling_ctx++;
+        auto submit_st = _scanner_scheduler->submit(this);
+        if (!submit_st.ok()) {
+            _num_scheduling_ctx--;
+        }
     }
 
     // Notice that after calling "_scanners.push_front(scanner)", there may be 
other ctx in scheduler
@@ -320,24 +331,14 @@ void 
ScannerContext::get_next_batch_of_scanners(std::list<VScanner*>* current_ru
     // 1. Calculate how many scanners should be scheduled at this run.
     int thread_slot_num = 0;
     {
-        std::unique_lock l(_transfer_lock);
-        if (_has_enough_space_in_blocks_queue()) {
-            // If there are enough space in blocks queue,
-            // the scanner number depends on the _free_blocks numbers
-            std::lock_guard f(_free_blocks_lock);
-            thread_slot_num = _free_blocks.size() / _block_per_scanner;
-            thread_slot_num += (_free_blocks.size() % _block_per_scanner != 0);
-            thread_slot_num = std::min(thread_slot_num, _max_thread_num - 
_num_running_scanners);
-            if (thread_slot_num <= 0) {
-                thread_slot_num = 1;
-            }
-        } else {
-            // The blocks queue reaches limit, just return to stop scheduling
-            // There will be two cases:
-            // 1. There are running scanners, these scanner will continue 
scheduler the ctx.
-            // 2. No running scanners, the consumer(ScanNode.get_next()) will 
continue scheduling the ctx.
-            // In both cases, we do not need to continue to schedule ctx here. 
So just return
-            return;
+        // If there are enough space in blocks queue,
+        // the scanner number depends on the _free_blocks numbers
+        std::lock_guard f(_free_blocks_lock);
+        thread_slot_num = _free_blocks.size() / _block_per_scanner;
+        thread_slot_num += (_free_blocks.size() % _block_per_scanner != 0);
+        thread_slot_num = std::min(thread_slot_num, _max_thread_num - 
_num_running_scanners);
+        if (thread_slot_num <= 0) {
+            thread_slot_num = 1;
         }
     }
 
diff --git a/be/src/vec/exec/scan/scanner_context.h 
b/be/src/vec/exec/scan/scanner_context.h
index 1285467a5d..1d937c5fb8 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -123,6 +123,13 @@ public:
 
     virtual void set_max_queue_size(int max_queue_size) {};
 
+    // todo(wb) rethinking how to calculate ```_max_bytes_in_queue``` when 
executing shared scan
+    virtual inline bool has_enough_space_in_blocks_queue() const {
+        return _cur_bytes_in_queue < _max_bytes_in_queue / 2;
+    }
+
+    void reschedule_scanner_ctx();
+
     // the unique id of this context
     std::string ctx_id;
     int32_t queue_idx = -1;
@@ -132,10 +139,6 @@ public:
 private:
     Status _close_and_clear_scanners(VScanNode* node, RuntimeState* state);
 
-    inline bool _has_enough_space_in_blocks_queue() const {
-        return _cur_bytes_in_queue < _max_bytes_in_queue / 2;
-    }
-
 protected:
     RuntimeState* _state;
     VScanNode* _parent;


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to