This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d9e01a418a0 [fix](be) fix BE core due to explode_bitmap size exceeding
INT_MAX (#66034)
d9e01a418a0 is described below
commit d9e01a418a0c3b52ad38d38a3f849b7bf3b82ab3
Author: York Cao <[email protected]>
AuthorDate: Mon Aug 24 18:17:04 2026 +0800
[fix](be) fix BE core due to explode_bitmap size exceeding INT_MAX (#66034)
### What problem does this PR solve?
Issue Number: close #66033
Problem Summary:
`VExplodeBitmapTableFunction::get_value` computed the number of rows to
emit with `max_step = std::min(max_step, (int)(_cur_size -
_cur_offset));`. Both `_cur_size` (the bitmap cardinality) and
`_cur_offset` are `int64_t`. When a bitmap holds more than `INT_MAX`
(~2.1 billion) elements, the subtraction `_cur_size - _cur_offset`
exceeds the range of `int`, so the C-style `(int)` cast overflows and
yields a negative `max_step`. That negative value is later used in
`target->resize(origin_size + max_step)`, which underflows the size
computation and crashes the BE (core dump).
The fix performs the `std::min` in `int64_t` space so the comparison is
done without truncation, then casts the result — now provably within
`[0, max_step]` and therefore in `int` range — back to `int`. A
`DCHECK_GE(max_step, 0)` guard documents and asserts the post-condition,
matching the glog `DCHECK` idiom already used by the sibling
table_function sources.
### Release note
Fix BE crash when exploding a bitmap whose cardinality exceeds INT_MAX.
### Check List (For Author)
- Test
- [x] Unit Test: added `be/test/exprs/function/table_function_test.cpp`:
`vexplode_bitmap_cardinality_exceeds_int_max`. It cheaply builds a
bitmap whose cardinality exceeds INT_MAX from a Roaring range (a few KB
— the overflow occurs on the first `get_value` call, before any element
is materialized, so no large allocation is needed) and asserts
`get_value` returns a positive batch. On the pre-fix code the `(int)`
cast overflows to a negative `max_step` and crashes; the test reproduces
this and passes on the fix (verified under ASAN).
- Behavior changed:
- [x] No (crash avoidance only; correct results are unchanged).
- Does this need documentation?
- [x] No.
---
be/src/exprs/table_function/vexplode_bitmap.cpp | 4 +-
be/test/exprs/function/table_function_test.cpp | 64 +++++++++++++++++++++++++
2 files changed, 67 insertions(+), 1 deletion(-)
diff --git a/be/src/exprs/table_function/vexplode_bitmap.cpp
b/be/src/exprs/table_function/vexplode_bitmap.cpp
index b21f099dde7..d9f4f8ded05 100644
--- a/be/src/exprs/table_function/vexplode_bitmap.cpp
+++ b/be/src/exprs/table_function/vexplode_bitmap.cpp
@@ -104,7 +104,9 @@ void VExplodeBitmapTableFunction::process_close() {
}
int VExplodeBitmapTableFunction::get_value(MutableColumnPtr& column, int
max_step) {
- max_step = std::min(max_step, (int)(_cur_size - _cur_offset));
+ max_step =
+ static_cast<int>(std::min(static_cast<int64_t>(max_step),
(_cur_size - _cur_offset)));
+ DCHECK_GE(max_step, 0);
// should dispose the empty status, forward one step
if (current_empty()) {
column->insert_default();
diff --git a/be/test/exprs/function/table_function_test.cpp
b/be/test/exprs/function/table_function_test.cpp
index 06ae08af713..9231adce8d1 100644
--- a/be/test/exprs/function/table_function_test.cpp
+++ b/be/test/exprs/function/table_function_test.cpp
@@ -28,6 +28,7 @@
#include "exprs/function/function_test_util.h"
#include "exprs/mock_vexpr.h"
#include "exprs/table_function/vexplode.h"
+#include "exprs/table_function/vexplode_bitmap.h"
#include "exprs/table_function/vexplode_numbers.h"
#include "exprs/table_function/vexplode_v2.h"
#include "exprs/table_function/vjson_each.h"
@@ -1407,4 +1408,67 @@ TEST_F(TableFunctionTest,
vjson_each_get_same_many_values_non_nullable) {
fn.process_close();
}
+// Regression test for a BE core crash when exploding a bitmap whose
cardinality
+// exceeds INT_MAX. VExplodeBitmapTableFunction::get_value used to compute the
+// batch size as `std::min(max_step, (int)(_cur_size - _cur_offset))`. Both
+// `_cur_size` (the bitmap cardinality) and `_cur_offset` are int64_t, so when
+// the cardinality is above INT_MAX the C-style `(int)` cast overflows to a
+// NEGATIVE value, which is then fed into `target->resize(origin_size +
max_step)`
+// and underflows -> crash. The overflow happens on the FIRST get_value call
+// (_cur_offset == 0), before any element is materialized, so we only need a
+// bitmap whose *cardinality* exceeds INT_MAX -- built cheaply here from a
+// Roaring range (a few KB, microseconds), not billions of individual inserts.
+TEST_F(TableFunctionTest, vexplode_bitmap_cardinality_exceeds_int_max) {
+ // Build a bitmap holding [0, 3'000'000'000) via a Roaring range. All
values
+ // are < 2^32 so a plain 32-bit Roaring suffices; the range is stored as
run
+ // containers (a few KB). Cardinality 3e9 > INT_MAX (2^31 - 1). Serialize
to
+ // the Doris BITMAP wire format and deserialize into a BitmapValue, since
+ // BitmapValue has no public addRange.
+ roaring::Roaring inner;
+ inner.addRange(0, 3000000000ULL); // [0, 3e9)
+ detail::Roaring64Map r64(inner);
+ const int serialize_version = config::bitmap_serialize_version;
+ const size_t nbytes = r64.getSizeInBytes(serialize_version);
+ std::string buffer;
+ buffer.resize(nbytes);
+ r64.write(buffer.data(), serialize_version);
+ BitmapValue bv(buffer.data());
+ // Fail fast if the cheap build path did not actually exceed INT_MAX.
+ ASSERT_GT(bv.cardinality(),
static_cast<uint64_t>(std::numeric_limits<int>::max()));
+
+ // One-column bitmap input block; the MockVExpr child returns column at
pos 0.
+ init_expr_context(1);
+ auto bitmap_col = ColumnBitmap::create();
+ bitmap_col->insert_value(std::move(bv));
+ auto block = Block::create_unique();
+ block->insert({std::move(bitmap_col), std::make_shared<DataTypeBitMap>(),
"bm"});
+
+ VExplodeBitmapTableFunction fn;
+ fn.set_expr_context(_ctx);
+
+ TQueryOptions q_opts;
+ TQueryGlobals q_globals;
+ RuntimeState rs(q_opts, q_globals);
+ ASSERT_TRUE(fn.process_init(block.get(), &rs).ok());
+ fn.process_row(0);
+ ASSERT_FALSE(fn.current_empty());
+
+ // Non-nullable path (_is_nullable == false): request exactly ONE batch.
+ // Pre-fix: (int)(_cur_size - _cur_offset) overflows to a negative max_step
+ // -> target->resize underflow -> crash (caught by ASAN).
+ // Fixed: the std::min is done in int64_t then cast back -> positive
batch.
+ MutableColumnPtr out = ColumnInt64::create();
+ int ret = fn.get_value(out, 4096);
+ EXPECT_EQ(ret, 4096);
+ EXPECT_EQ(out->size(), 4096);
+
+ // The first batch materializes the smallest elements 0,1,2,... in order.
+ const auto& data = assert_cast<const ColumnInt64&>(*out).get_data();
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(data[i], static_cast<int64_t>(i));
+ }
+
+ fn.process_close();
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
