This is an automated email from the ASF dual-hosted git repository.
lihaopeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8be43857ef [feature](executor) Add memory limit for
pip_scanner_context (#18238)
8be43857ef is described below
commit 8be43857ef6d5d5c83ddba997118945d2873c326
Author: HappenLee <[email protected]>
AuthorDate: Fri Mar 31 09:36:57 2023 +0800
[feature](executor) Add memory limit for pip_scanner_context (#18238)
Co-authored-by: wangbo <[email protected]>
---
be/src/pipeline/exec/scan_operator.cpp | 5 +++-
be/src/vec/exec/scan/pip_scanner_context.h | 18 ++++++++++--
be/src/vec/exec/scan/scanner_context.cpp | 47 +++++++++++++++---------------
be/src/vec/exec/scan/scanner_context.h | 11 ++++---
4 files changed, 51 insertions(+), 30 deletions(-)
diff --git a/be/src/pipeline/exec/scan_operator.cpp
b/be/src/pipeline/exec/scan_operator.cpp
index 6451927f77..60864c6dd9 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -32,12 +32,15 @@ bool ScanOperator::can_read() {
return false;
}
} else {
- if (_node->_eos || _node->_scanner_ctx->done() ||
_node->_scanner_ctx->no_schedule()) {
+ if (_node->_eos || _node->_scanner_ctx->done()) {
// _eos: need eos
// _scanner_ctx->done(): need finish
// _scanner_ctx->no_schedule(): should schedule _scanner_ctx
return true;
} else {
+ if (_node->_scanner_ctx->has_enough_space_in_blocks_queue()) {
+ _node->_scanner_ctx->reschedule_scanner_ctx();
+ }
return _node->ready_to_read(); // there are some blocks to process
}
}
diff --git a/be/src/vec/exec/scan/pip_scanner_context.h
b/be/src/vec/exec/scan/pip_scanner_context.h
index 6a38617a9d..cacc61df1d 100644
--- a/be/src/vec/exec/scan/pip_scanner_context.h
+++ b/be/src/vec/exec/scan/pip_scanner_context.h
@@ -51,11 +51,12 @@ public:
if (!_blocks_queues[id].empty()) {
*block = std::move(_blocks_queues[id].front());
_blocks_queues[id].pop_front();
- return Status::OK();
} else {
*eos = _is_finished || _should_stop;
+ return Status::OK();
}
}
+ _current_used_bytes -= (*block)->allocated_bytes();
return Status::OK();
}
@@ -65,6 +66,11 @@ public:
void append_blocks_to_queue(std::vector<vectorized::BlockUPtr>& blocks)
override {
const int queue_size = _queue_mutexs.size();
const int block_size = blocks.size();
+ int64_t local_bytes = 0;
+ for (const auto& block : blocks) {
+ local_bytes += block->allocated_bytes();
+ }
+
for (int i = 0; i < queue_size && i < block_size; ++i) {
int queue = _next_queue_to_feed;
{
@@ -75,6 +81,7 @@ public:
}
_next_queue_to_feed = queue + 1 < queue_size ? queue + 1 : 0;
}
+ _current_used_bytes += local_bytes;
}
bool empty_in_queue(int id) override {
@@ -82,17 +89,24 @@ public:
return _blocks_queues[id].empty();
}
- void set_max_queue_size(int max_queue_size) override {
+ void set_max_queue_size(const int max_queue_size) override {
+ _max_queue_size = max_queue_size;
for (int i = 0; i < max_queue_size; ++i) {
_queue_mutexs.emplace_back(new std::mutex);
_blocks_queues.emplace_back(std::list<vectorized::BlockUPtr>());
}
}
+ bool has_enough_space_in_blocks_queue() const override {
+ return _current_used_bytes < _max_bytes_in_queue / 2 * _max_queue_size;
+ }
+
private:
+ int _max_queue_size = 1;
int _next_queue_to_feed = 0;
std::vector<std::unique_ptr<std::mutex>> _queue_mutexs;
std::vector<std::list<vectorized::BlockUPtr>> _blocks_queues;
+ std::atomic_int64_t _current_used_bytes = 0;
};
} // namespace pipeline
} // namespace doris
diff --git a/be/src/vec/exec/scan/scanner_context.cpp
b/be/src/vec/exec/scan/scanner_context.cpp
index 16d8971227..0b3cbcd3d7 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -164,7 +164,7 @@ Status ScannerContext::get_block_from_queue(RuntimeState*
state, vectorized::Blo
// (if the scheduler continues to schedule, it will cause a lot of busy
running).
// At this point, consumers are required to trigger new scheduling to
ensure that
// data can be continuously fetched.
- if (_has_enough_space_in_blocks_queue() && _num_running_scanners == 0) {
+ if (has_enough_space_in_blocks_queue() && _num_running_scanners == 0) {
_num_scheduling_ctx++;
_scanner_scheduler->submit(this);
}
@@ -288,6 +288,15 @@ std::string ScannerContext::debug_string() {
_max_thread_num, _block_per_scanner, _cur_bytes_in_queue,
_max_bytes_in_queue);
}
+void ScannerContext::reschedule_scanner_ctx() {
+ std::lock_guard l(_transfer_lock);
+ auto submit_st = _scanner_scheduler->submit(this);
+ //todo(wb) rethinking is it better to mark current scan_context failed
when submit failed many times?
+ if (submit_st.ok()) {
+ _num_scheduling_ctx++;
+ }
+}
+
void ScannerContext::push_back_scanner_and_reschedule(VScanner* scanner) {
{
std::unique_lock l(_scanners_lock);
@@ -295,10 +304,12 @@ void
ScannerContext::push_back_scanner_and_reschedule(VScanner* scanner) {
}
std::lock_guard l(_transfer_lock);
- _num_scheduling_ctx++;
- auto submit_st = _scanner_scheduler->submit(this);
- if (!submit_st.ok()) {
- _num_scheduling_ctx--;
+ if (has_enough_space_in_blocks_queue()) {
+ _num_scheduling_ctx++;
+ auto submit_st = _scanner_scheduler->submit(this);
+ if (!submit_st.ok()) {
+ _num_scheduling_ctx--;
+ }
}
// Notice that after calling "_scanners.push_front(scanner)", there may be
other ctx in scheduler
@@ -320,24 +331,14 @@ void
ScannerContext::get_next_batch_of_scanners(std::list<VScanner*>* current_ru
// 1. Calculate how many scanners should be scheduled at this run.
int thread_slot_num = 0;
{
- std::unique_lock l(_transfer_lock);
- if (_has_enough_space_in_blocks_queue()) {
- // If there are enough space in blocks queue,
- // the scanner number depends on the _free_blocks numbers
- std::lock_guard f(_free_blocks_lock);
- thread_slot_num = _free_blocks.size() / _block_per_scanner;
- thread_slot_num += (_free_blocks.size() % _block_per_scanner != 0);
- thread_slot_num = std::min(thread_slot_num, _max_thread_num -
_num_running_scanners);
- if (thread_slot_num <= 0) {
- thread_slot_num = 1;
- }
- } else {
- // The blocks queue reaches limit, just return to stop scheduling
- // There will be two cases:
- // 1. There are running scanners, these scanner will continue
scheduler the ctx.
- // 2. No running scanners, the consumer(ScanNode.get_next()) will
continue scheduling the ctx.
- // In both cases, we do not need to continue to schedule ctx here.
So just return
- return;
+ // If there are enough space in blocks queue,
+ // the scanner number depends on the _free_blocks numbers
+ std::lock_guard f(_free_blocks_lock);
+ thread_slot_num = _free_blocks.size() / _block_per_scanner;
+ thread_slot_num += (_free_blocks.size() % _block_per_scanner != 0);
+ thread_slot_num = std::min(thread_slot_num, _max_thread_num -
_num_running_scanners);
+ if (thread_slot_num <= 0) {
+ thread_slot_num = 1;
}
}
diff --git a/be/src/vec/exec/scan/scanner_context.h
b/be/src/vec/exec/scan/scanner_context.h
index 1285467a5d..1d937c5fb8 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -123,6 +123,13 @@ public:
virtual void set_max_queue_size(int max_queue_size) {};
+ // todo(wb) rethinking how to calculate ```_max_bytes_in_queue``` when
executing shared scan
+ virtual inline bool has_enough_space_in_blocks_queue() const {
+ return _cur_bytes_in_queue < _max_bytes_in_queue / 2;
+ }
+
+ void reschedule_scanner_ctx();
+
// the unique id of this context
std::string ctx_id;
int32_t queue_idx = -1;
@@ -132,10 +139,6 @@ public:
private:
Status _close_and_clear_scanners(VScanNode* node, RuntimeState* state);
- inline bool _has_enough_space_in_blocks_queue() const {
- return _cur_bytes_in_queue < _max_bytes_in_queue / 2;
- }
-
protected:
RuntimeState* _state;
VScanNode* _parent;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]