This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new 88646dcf8e [enhancement](olap scanner) Scanner row bytes buffer is too
small bug (#13924)
88646dcf8e is described below
commit 88646dcf8e9d8e1c87b4abcab32314222100fae3
Author: yiguolei <[email protected]>
AuthorDate: Thu Nov 3 11:51:59 2022 +0800
[enhancement](olap scanner) Scanner row bytes buffer is too small bug
(#13924)
Co-authored-by: yiguolei <[email protected]>
---
be/src/exec/olap_scanner.cpp | 39 +++++++++++++++++----------------------
1 file changed, 17 insertions(+), 22 deletions(-)
diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
index a5f29970ea..47802cc6c0 100644
--- a/be/src/exec/olap_scanner.cpp
+++ b/be/src/exec/olap_scanner.cpp
@@ -50,8 +50,7 @@ OlapScanner::OlapScanner(RuntimeState* runtime_state,
OlapScanNode* parent, bool
_version(-1),
_mem_tracker(MemTracker::CreateTracker(
runtime_state->fragment_mem_tracker()->limit(),
"OlapScanner",
- runtime_state->fragment_mem_tracker(), true, true,
MemTrackerLevel::VERBOSE)) {
-}
+ runtime_state->fragment_mem_tracker(), true, true,
MemTrackerLevel::VERBOSE)) {}
Status OlapScanner::prepare(
const TPaloScanRange& scan_range, const std::vector<OlapScanRange*>&
key_ranges,
@@ -91,8 +90,8 @@ Status OlapScanner::prepare(
// to prevent this case: when there are lots of olap scanners to
run for example 10000
// the rowsets maybe compacted when the last olap scanner starts
Version rd_version(0, _version);
- OLAPStatus acquire_reader_st =
- _tablet->capture_rs_readers(rd_version,
&_tablet_reader_params.rs_readers, _mem_tracker);
+ OLAPStatus acquire_reader_st = _tablet->capture_rs_readers(
+ rd_version, &_tablet_reader_params.rs_readers,
_mem_tracker);
if (acquire_reader_st != OLAP_SUCCESS) {
LOG(WARNING) << "fail to init reader.res=" <<
acquire_reader_st;
std::stringstream ss;
@@ -125,8 +124,9 @@ Status OlapScanner::open() {
if (res != OLAP_SUCCESS) {
OLAP_LOG_WARNING("fail to init reader.[res=%d]", res);
std::stringstream ss;
- ss << "failed to initialize storage reader. tablet=" <<
_tablet_reader_params.tablet->full_name()
- << ", res=" << res << ", backend=" <<
BackendOptions::get_localhost();
+ ss << "failed to initialize storage reader. tablet="
+ << _tablet_reader_params.tablet->full_name() << ", res=" << res
+ << ", backend=" << BackendOptions::get_localhost();
return Status::InternalError(ss.str().c_str());
}
return Status::OK();
@@ -167,7 +167,8 @@ Status OlapScanner::_init_tablet_reader_params(
_tablet_reader_params.conditions.push_back(filter);
}
std::copy(bloom_filters.cbegin(), bloom_filters.cend(),
- std::inserter(_tablet_reader_params.bloom_filters,
_tablet_reader_params.bloom_filters.begin()));
+ std::inserter(_tablet_reader_params.bloom_filters,
+ _tablet_reader_params.bloom_filters.begin()));
// Range
for (auto key_range : key_ranges) {
@@ -206,7 +207,8 @@ Status OlapScanner::_init_tablet_reader_params(
}
// use _tablet_reader_params.return_columns, because reader use this to
merge sort
- OLAPStatus res = _read_row_cursor.init(_tablet->tablet_schema(),
_tablet_reader_params.return_columns);
+ OLAPStatus res =
+ _read_row_cursor.init(_tablet->tablet_schema(),
_tablet_reader_params.return_columns);
if (res != OLAP_SUCCESS) {
OLAP_LOG_WARNING("fail to init row cursor.[res=%d]", res);
return Status::InternalError("failed to initialize storage read row
cursor");
@@ -286,23 +288,19 @@ Status OlapScanner::get_batch(RuntimeState* state,
RowBatch* batch, bool* eof) {
ObjectPool tmp_object_pool;
// release the memory of the object which can't pass the conjuncts.
ObjectPool unused_object_pool;
- if (batch->tuple_data_pool()->total_reserved_bytes() >=
raw_bytes_threshold) {
- return Status::RuntimeError(
- "Scanner row bytes buffer is too small, please try to
increase be config "
- "'doris_scanner_row_bytes'.");
- }
while (true) {
// Batch is full or reach raw_rows_threshold or
raw_bytes_threshold, break
- if (batch->is_full() ||
- (batch->tuple_data_pool()->total_allocated_bytes() >=
raw_bytes_threshold &&
- batch->num_rows() > 0) ||
+ // Use total_byte_size here, not tuple_pool's allocated bytes,
because we preallocated tuple pool at beginning
+ // its size maybe larger than threshold, so that scanner will
break here and may dead loop.
+ // Not need check num_rows > 0, because total_byte_size() == 0 if
num_rows == 0.
+ if (batch->is_full() || batch->total_byte_size() >=
raw_bytes_threshold ||
raw_rows_read() >= raw_rows_threshold) {
_update_realtime_counter();
break;
}
if (tmp_object_pool.size() > 0) {
- unused_object_pool.acquire_data(&tmp_object_pool);
+ unused_object_pool.acquire_data(&tmp_object_pool);
}
if (unused_object_pool.size() >= config::object_pool_buffer_size) {
@@ -456,7 +454,6 @@ Status OlapScanner::get_batch(RuntimeState* state,
RowBatch* batch, bool* eof) {
}
}
} while (false);
-
}
}
@@ -574,8 +571,7 @@ void OlapScanner::update_counter() {
COUNTER_UPDATE(_parent->_del_filtered_counter, stats.rows_del_filtered);
COUNTER_UPDATE(_parent->_del_filtered_counter,
stats.rows_vec_del_cond_filtered);
- COUNTER_UPDATE(_parent->_conditions_filtered_counter,
- stats.rows_conditions_filtered);
+ COUNTER_UPDATE(_parent->_conditions_filtered_counter,
stats.rows_conditions_filtered);
COUNTER_UPDATE(_parent->_key_range_filtered_counter,
stats.rows_key_range_filtered);
COUNTER_UPDATE(_parent->_index_load_timer, stats.index_load_ns);
@@ -588,8 +584,7 @@ void OlapScanner::update_counter() {
COUNTER_UPDATE(_parent->_total_pages_num_counter, stats.total_pages_num);
COUNTER_UPDATE(_parent->_cached_pages_num_counter, stats.cached_pages_num);
- COUNTER_UPDATE(_parent->_bitmap_index_filter_counter,
- stats.rows_bitmap_index_filtered);
+ COUNTER_UPDATE(_parent->_bitmap_index_filter_counter,
stats.rows_bitmap_index_filtered);
COUNTER_UPDATE(_parent->_bitmap_index_filter_timer,
stats.bitmap_index_filter_timer);
COUNTER_UPDATE(_parent->_block_seek_counter, stats.block_seek_num);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
