This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch dev-1.0.1 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 9cfed506d38c85867e8a9c5def4369235987309a Author: SleepyBear <[email protected]> AuthorDate: Tue Apr 26 10:25:56 2022 +0800 [fix](OlapScanner)fix bitmap or hll's OOM when loading too many unqualified data (#9205) --- be/src/common/config.h | 4 ++++ be/src/common/object_pool.h | 5 +++++ be/src/exec/olap_scanner.cpp | 17 ++++++++++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index 2a5f03de8e..b37fbc048a 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -711,6 +711,10 @@ CONF_mInt32(string_type_length_soft_limit_bytes, "1048576"); CONF_Validator(string_type_length_soft_limit_bytes, [](const int config) -> bool { return config > 0 && config <= 2147483643; }); +// used for olap scanner to save memory, when the size of unused_object_pool +// is greater than object_pool_buffer_size, release the object in the unused_object_pool. +CONF_Int32(object_pool_buffer_size, "100"); + } // namespace config } // namespace doris diff --git a/be/src/common/object_pool.h b/be/src/common/object_pool.h index b2eb4e3024..50001c1737 100644 --- a/be/src/common/object_pool.h +++ b/be/src/common/object_pool.h @@ -60,6 +60,11 @@ public: src->_objects.clear(); } + uint64_t size() { + std::lock_guard<SpinLock> l(_lock); + return _objects.size(); + } + private: DISALLOW_COPY_AND_ASSIGN(ObjectPool); diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index 04dfa81daf..9baef00001 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -272,6 +272,11 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) { int64_t raw_bytes_threshold = config::doris_scanner_row_bytes; { SCOPED_TIMER(_parent->_scan_timer); + // store the object which may can't pass the conjuncts temporarily. + // otherwise, pushed all objects into agg_object_pool directly may lead to OOM. + ObjectPool tmp_object_pool; + // release the memory of the object which can't pass the conjuncts. + ObjectPool unused_object_pool; while (true) { // Batch is full or reach raw_rows_threshold or raw_bytes_threshold, break if (batch->is_full() || @@ -280,9 +285,18 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) { _update_realtime_counter(); break; } + + if (tmp_object_pool.size() > 0) { + unused_object_pool.acquire_data(&tmp_object_pool); + } + + if (unused_object_pool.size() >= config::object_pool_buffer_size) { + unused_object_pool.clear(); + } + // Read one row from reader auto res = _tablet_reader->next_row_with_aggregation(&_read_row_cursor, mem_pool.get(), - batch->agg_object_pool(), eof); + &tmp_object_pool, eof); if (res != OLAP_SUCCESS) { std::stringstream ss; ss << "Internal Error: read storage fail. res=" << res @@ -404,6 +418,7 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) { // check direct && pushdown conjuncts success then commit tuple batch->commit_last_row(); + batch->agg_object_pool()->acquire_data(&tmp_object_pool); char* new_tuple = reinterpret_cast<char*>(tuple); new_tuple += _tuple_desc->byte_size(); tuple = reinterpret_cast<Tuple*>(new_tuple); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
