gohalo opened a new pull request, #53088:
URL: https://github.com/apache/doris/pull/53088
### What problem does this PR solve?
Issue Number: close #52767
Problem Summary:
When try to load some invalid bitmap value, the related be nodes will core
dumped.
```
*** Aborted at 1749536908 (unix time) try "date -d @1749536908" if you are
using GNU date ***
*** Current BE git commitID: d5a02e095d ***
*** SIGABRT unknown detail explain (@0x55d8000bf626) received by PID 783910
(TID 1564162 OR 0x7f2bd4e16700) from PID 783910; stack trace: ***
0# doris::signal::(anonymous namespace)::FailureSignalHandler(int,
siginfo_t*, void*) at
/home/zcp/repo_center/doris_release/doris/be/src/common/signal_handler.h:421
1# 0x00007F4DB9021400 in /lib64/libc.so.6
2# gsignal in /lib64/libc.so.6
3# abort in /lib64/libc.so.6
4# __gnu_cxx::__verbose_terminate_handler() [clone .cold] at
../../../../libstdc++-v3/libsupc++/vterminate.cc:75
5# __cxxabiv1::__terminate(void (*)()) at
../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48
6# 0x000055C8818B4A31 in
/home/olap/music-doris-gy/doris-be-log/default-be/approot/be/lib/doris_be
7# 0x000055C8818B4B84 in
/home/olap/music-doris-gy/doris-be-log/default-be/approot/be/lib/doris_be
8# roaring::Roaring::read(char const*, bool) in
/home/olap/music-doris-gy/doris-be-log/default-be/approot/be/lib/doris_be
9# doris::detail::Roaring64Map::read(char const*) in
/home/olap/music-doris-gy/doris-be-log/default-be/approot/be/lib/doris_be
10# doris::BitmapValue::deserialize(char const*) at
/home/zcp/repo_center/doris_release/doris/be/src/util/bitmap_value.h:1911
11#
doris::vectorized::BitmapFromBase64::vector(doris::vectorized::PODArray<unsigned
char, 4096ul, Allocator<false, false, false>, 15ul, 16ul> const&,
doris::vectorized::PODArray<unsigned int, 4096ul, Allocator<false, false,
false>, 15ul, 16ul> const&, std::vector<doris::BitmapValue,
std::allocator<doris::BitmapValue> >&, doris::vectorized::PODArray<unsigned
char, 4096ul, Allocator<false, false, false>, 15ul, 16ul>&, unsigned long) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/functions/function_bitmap.cpp:291
12#
doris::vectorized::FunctionBitmapAlwaysNull<doris::vectorized::BitmapFromBase64>::execute_impl(doris::FunctionContext*,
doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned
long> > const&, unsigned long, unsigned long) const at
/home/zcp/repo_center/doris_release/doris/be/src/vec/functions/function_bitmap.cpp:366
13#
doris::vectorized::DefaultExecutable::execute_impl(doris::FunctionContext*,
doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned
long> > const&, unsigned long, unsigned long) const at
/home/zcp/repo_center/doris_release/doris/be/src/vec/functions/function.h:429
14#
doris::vectorized::PreparedFunctionImpl::_execute_skipped_constant_deal(doris::FunctionContext*,
doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned
long> > const&, unsigned long, unsigned long, bool) const at
/home/zcp/repo_center/doris_release/doris/be/src/vec/functions/function.cpp:122
15#
doris::vectorized::PreparedFunctionImpl::default_implementation_for_nulls(doris::FunctionContext*,
doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned
long> > const&, unsigned long, unsigned long, bool, bool*) const at
/home/zcp/repo_center/doris_release/doris/be/src/vec/functions/function.cpp:217
16#
doris::vectorized::PreparedFunctionImpl::execute_without_low_cardinality_columns(doris::FunctionContext*,
doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned
long> > const&, unsigned long, unsigned long, bool) const at
/home/zcp/repo_center/doris_release/doris/be/src/vec/functions/function.cpp:244
17#
doris::vectorized::PreparedFunctionImpl::execute(doris::FunctionContext*,
doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned
long> > const&, unsigned long, unsigned long, bool) const at
/home/zcp/repo_center/doris_release/doris/be/src/vec/functions/function.cpp:250
18# doris::vectorized::IFunctionBase::execute(doris::FunctionContext*,
doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned
long> > const&, unsigned long, unsigned long, bool) const at
/home/zcp/repo_center/doris_release/doris/be/src/vec/functions/function.h:179
19#
doris::vectorized::VectorizedFnCall::_do_execute(doris::vectorized::VExprContext*,
doris::vectorized::Block*, int*, std::vector<unsigned long,
std::allocator<unsigned long> >&) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/exprs/vectorized_fn_call.cpp:169
20#
doris::vectorized::VectorizedFnCall::execute(doris::vectorized::VExprContext*,
doris::vectorized::Block*, int*) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/exprs/vectorized_fn_call.cpp:185
21#
doris::vectorized::VectorizedFnCall::_do_execute(doris::vectorized::VExprContext*,
doris::vectorized::Block*, int*, std::vector<unsigned long,
std::allocator<unsigned long> >&) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/exprs/vectorized_fn_call.cpp:152
22#
doris::vectorized::VectorizedFnCall::execute(doris::vectorized::VExprContext*,
doris::vectorized::Block*, int*) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/exprs/vectorized_fn_call.cpp:185
23# doris::vectorized::VExprContext::execute(doris::vectorized::Block*,
int*) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/exprs/vexpr_context.cpp:54
24#
doris::vectorized::VFileScanner::_convert_to_output_block(doris::vectorized::Block*)
in /home/olap/music-doris-gy/doris-be-log/default-be/approot/be/lib/doris_be
25#
doris::vectorized::VFileScanner::_get_block_wrapped(doris::RuntimeState*,
doris::vectorized::Block*, bool*) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/vfile_scanner.cpp:380
26# doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*,
doris::vectorized::Block*, bool*) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/vfile_scanner.cpp:302
27# doris::vectorized::VScanner::get_block(doris::RuntimeState*,
doris::vectorized::Block*, bool*) in
/home/olap/music-doris-gy/doris-be-log/default-be/approot/be/lib/doris_be
28#
doris::vectorized::VScanner::get_block_after_projects(doris::RuntimeState*,
doris::vectorized::Block*, bool*) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/vscanner.cpp:100
29#
doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>) at
/home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:269
30# std::_Function_handler<void (),
doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>,
std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()()
const::{lambda()#1}>::_M_invoke(std::_Any_data const&) at
/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291
31# doris::ThreadPool::dispatch_thread() in
/home/olap/music-doris-gy/doris-be-log/default-be/approot/be/lib/doris_be
32# doris::Thread::supervise_thread(void*) at
/home/zcp/repo_center/doris_release/doris/be/src/util/thread.cpp:499
33# start_thread in /lib64/libpthread.so.0
34# clone in /lib64/libc.so.6
```
Could reproduce with the following steps, which copied from #52767
Create a simple table with bitmap value.
```
CREATE TABLE test_bitmap(
id BIGINT NOT NULL,
arr BITMAP BITMAP_UNION NOT NULL
)
AGGREGATE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 1
properties(
"replication_num" = "1"
);
```
and `test.csv` file, only one simple line
```
1|Av8DCg==
```
then load it, and the be will core dump
```
curl --location-trusted -u root: \
-H "Expect: 100-continue" \
-H "column_separator:|" \
-H "columns:id,arr,arr=bitmap_from_base64(arr)" \
-T test.csv \
-XPUT http://127.0.0.1:8030/api/test/test_bitmap/_stream_load
```
https://github.com/RoaringBitmap/CRoaring/blob/835884b2939f02337313c2abef2992568d9b2e9d/cpp/roaring.hh#L647-L655
``` c
static Roaring read(const char *buf, bool portable = true) {
roaring_bitmap_t *r =
portable ? api::roaring_bitmap_portable_deserialize(buf)
: api::roaring_bitmap_deserialize(buf);
if (r == NULL) {
ROARING_TERMINATE("failed alloc while reading");
}
return Roaring(r);
}
```
When the buffer is invalid, in this case, it's has invalid leading bytes and
which is not equal to `SERIAL_COOKIE`, and then `NULL` returned. And the
default action of `ROARING_EXCEPTIONS` will be `throw std::runtime_error(_s)`.
``` c
#ifndef ROARING_TERMINATE
#if ROARING_EXCEPTIONS
#define ROARING_TERMINATE(_s) throw std::runtime_error(_s)
#else
#define ROARING_TERMINATE(_s) std::terminate()
#endif
#endif
```
If this hasn't been caught, then bingo!!!
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [x] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR should
merge into -->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
