This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4bb46a8d580 [fix](memory safe) Fix block-reader memory alloc failed
make be core dump. (#36299)
4bb46a8d580 is described below
commit 4bb46a8d580cbcd35200af5e4f23830ceb9c3ef2
Author: zhiqiang <[email protected]>
AuthorDate: Fri Jun 14 20:29:18 2024 +0800
[fix](memory safe) Fix block-reader memory alloc failed make be core dump.
(#36299)
We should leave the construction of Arena of BlockReader to its init
function to avoid problem like below:
```text
*** Query id: 221bb894cfc945fb-ba8f584e7668a36c ***
*** is nereids: 1 ***
*** tablet id: 0 ***
*** Aborted at 1718335148 (unix time) try "date -d @1718335148" if you are
using GNU date ***
*** Current BE git commitID: ac0568acff ***
*** SIGSEGV address not mapped to object (@0x0) received by PID 1330723
(TID 1332580 OR 0x7fc6c7af5700) from PID 0; stack trace: ***
0# doris::signal::(anonymous namespace)::FailureSignalHandler(int,
siginfo_t*, void*) at
/mnt/disk1/hezhiqiang/doris/be/src/common/signal_handler.h:421
1# 0x00007FCF485C3B50 in /lib64/libc.so.6
2# doris::vectorized::NewOlapScanner::open(doris::RuntimeState*) at
/mnt/disk1/hezhiqiang/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:218
3#
doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>) at
/mnt/disk1/hezhiqiang/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:249
4#
doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)::$_0::operator()()
const::{lambda()#1}::operator()() const at
/mnt/disk1/hezhiqiang/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:141
5#
doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)::$_0::operator()() const at
/mnt/disk1/hezhiqiang/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:140
6# void std::__invoke_impl<void,
doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)::$_0&>(std::__invoke_other,
doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)::$_0&) at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61
7# std::enable_if<is_invocable_r_v<void,
doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)::$_0&>, void>::type
std::__invoke_r<void,
doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)::$_0&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<do [...]
8# std::_Function_handler<void (),
doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)::$_0>::_M_invoke(std::_Any_data
const&) at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291
9# std::function<void ()>::operator()() const at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560
10# doris::FunctionRunnable::run() at
/mnt/disk1/hezhiqiang/doris/be/src/util/threadpool.cpp:48
11# doris::ThreadPool::dispatch_thread() at
/mnt/disk1/hezhiqiang/doris/be/src/util/threadpool.cpp:543
12# void std::__invoke_impl<void, void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>(std::__invoke_memfun_deref, void
(doris::ThreadPool::*&)(), doris::ThreadPool*&) at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74
13# std::__invoke_result<void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>::type std::__invoke<void (doris::ThreadPool::*&)(),
doris::ThreadPool*&>(void (doris::ThreadPool::*&)(), doris::ThreadPool*&) at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96
14# void std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>::__call<void, ,
0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/functional:420
15# void std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>::operator()<, void>() at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/functional:503
16# void std::__invoke_impl<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::__invoke_other,
std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>&) at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61
17# std::enable_if<is_invocable_r_v<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>, void>::type
std::__invoke_r<void, std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()>&) at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:117
18# std::_Function_handler<void (), std::_Bind<void
(doris::ThreadPool::*(doris::ThreadPool*))()> >::_M_invoke(std::_Any_data
const&) at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291
19# std::function<void ()>::operator()() const at
/mnt/disk1/hezhiqiang/toolchains/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560
20# doris::Thread::supervise_thread(void*) at
/mnt/disk1/hezhiqiang/doris/be/src/util/thread.cpp:498
21# start_thread in /lib64/libpthread.so.0
22# __clone in /lib64/libc.so.6
```
---
be/src/vec/olap/block_reader.cpp | 11 +++++++++--
be/src/vec/olap/block_reader.h | 3 ++-
be/src/vec/olap/vertical_block_reader.cpp | 9 +++++++--
be/src/vec/olap/vertical_block_reader.h | 4 +++-
4 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/be/src/vec/olap/block_reader.cpp b/be/src/vec/olap/block_reader.cpp
index ec6bb5cc92b..fef95303a3e 100644
--- a/be/src/vec/olap/block_reader.cpp
+++ b/be/src/vec/olap/block_reader.cpp
@@ -209,6 +209,8 @@ Status BlockReader::_init_agg_state(const ReaderParams&
read_params) {
Status BlockReader::init(const ReaderParams& read_params) {
RETURN_IF_ERROR(TabletReader::init(read_params));
+ _arena = std::make_unique<Arena>();
+
int32_t return_column_size = read_params.origin_return_columns->size();
_return_columns_loc.resize(read_params.return_columns.size());
for (int i = 0; i < return_column_size; ++i) {
@@ -511,6 +513,10 @@ size_t BlockReader::_copy_agg_data() {
}
void BlockReader::_update_agg_value(MutableColumns& columns, int begin, int
end, bool is_close) {
+ if (!_arena) [[unlikely]] {
+ return;
+ }
+
for (int i = 0; i < _agg_columns_idx.size(); i++) {
auto idx = _agg_columns_idx[i];
@@ -520,7 +526,7 @@ void BlockReader::_update_agg_value(MutableColumns&
columns, int begin, int end,
if (begin <= end) {
function->add_batch_range(begin, end, place, const_cast<const
IColumn**>(&column_ptr),
- &_arena, _stored_has_null_tag[idx]);
+ _arena.get(), _stored_has_null_tag[idx]);
}
if (is_close) {
@@ -529,8 +535,9 @@ void BlockReader::_update_agg_value(MutableColumns&
columns, int begin, int end,
function->reset(place);
}
}
+
if (is_close) {
- _arena.clear();
+ _arena->clear();
}
}
diff --git a/be/src/vec/olap/block_reader.h b/be/src/vec/olap/block_reader.h
index 6f9792929db..b60a5bc2373 100644
--- a/be/src/vec/olap/block_reader.h
+++ b/be/src/vec/olap/block_reader.h
@@ -119,7 +119,8 @@ private:
bool _is_rowsets_overlapping = true;
- Arena _arena;
+ // Use pointer to avoid allocing memory during construction
+ std::unique_ptr<Arena> _arena;
};
} // namespace vectorized
diff --git a/be/src/vec/olap/vertical_block_reader.cpp
b/be/src/vec/olap/vertical_block_reader.cpp
index 0092a020a2e..c4dda20f40f 100644
--- a/be/src/vec/olap/vertical_block_reader.cpp
+++ b/be/src/vec/olap/vertical_block_reader.cpp
@@ -208,6 +208,8 @@ Status VerticalBlockReader::init(const ReaderParams&
read_params) {
_reader_context.batch_size = opts.block_row_max;
RETURN_IF_ERROR(TabletReader::init(read_params));
+ _arena = std::make_unique<Arena>();
+
auto status = _init_collect_iter(read_params);
if (!status.ok()) [[unlikely]] {
if (!config::is_cloud_mode()) {
@@ -298,6 +300,9 @@ void VerticalBlockReader::_update_agg_data(MutableColumns&
columns) {
void VerticalBlockReader::_update_agg_value(MutableColumns& columns, int
begin, int end,
bool is_close) {
+ if (!_arena) [[unlikely]] {
+ return;
+ }
for (size_t idx = 0; idx < _return_columns.size(); ++idx) {
AggregateFunctionPtr function = _agg_functions[idx];
AggregateDataPtr place = _agg_places[idx];
@@ -305,7 +310,7 @@ void VerticalBlockReader::_update_agg_value(MutableColumns&
columns, int begin,
if (begin <= end) {
function->add_batch_range(begin, end, place, const_cast<const
IColumn**>(&column_ptr),
- &_arena, _stored_has_null_tag[idx]);
+ _arena.get(), _stored_has_null_tag[idx]);
}
if (is_close) {
@@ -315,7 +320,7 @@ void VerticalBlockReader::_update_agg_value(MutableColumns&
columns, int begin,
}
}
if (is_close) {
- _arena.clear();
+ _arena->clear();
}
}
diff --git a/be/src/vec/olap/vertical_block_reader.h
b/be/src/vec/olap/vertical_block_reader.h
index 77a01587b58..81ef8d79100 100644
--- a/be/src/vec/olap/vertical_block_reader.h
+++ b/be/src/vec/olap/vertical_block_reader.h
@@ -107,7 +107,9 @@ private:
// for agg mode
std::vector<AggregateFunctionPtr> _agg_functions;
std::vector<AggregateDataPtr> _agg_places;
- Arena _arena;
+
+ // Use pointer to avoid memory allocation during construction
+ std::unique_ptr<Arena> _arena;
std::vector<int> _normal_columns_idx;
std::vector<int> _agg_columns_idx;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
