This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 27357c32063 [fix](be) Track block bloom filter memory (#65578)
27357c32063 is described below
commit 27357c32063c0ec46f5cb1678d080a421b3aec2b
Author: TengJianPing <[email protected]>
AuthorDate: Thu Jul 16 12:00:48 2026 +0800
[fix](be) Track block bloom filter memory (#65578)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary: BlockBloomFilter allocated its directory with
posix_memalign, bypassing Doris Allocator and MemTracker. Use the
tracked allocator for aligned allocation and release, preserving the
previous directory size across reinitialization so accounting remains
exact.
### Release note
None
### Check List (For Author)
- Test: Unit Test
- BloomFilterFuncTest.TrackBlockBloomFilterMemory
- Behavior changed: Yes (Block Bloom Filter directory memory is now
recorded by MemTracker)
- Does this need documentation: No
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/exprs/block_bloom_filter_impl.cc | 23 +++++++++++++----------
be/test/exprs/bloom_filter_func_test.cpp | 27 +++++++++++++++++++++++++++
2 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/be/src/exprs/block_bloom_filter_impl.cc
b/be/src/exprs/block_bloom_filter_impl.cc
index d0d8d4245e9..84098e716ca 100644
--- a/be/src/exprs/block_bloom_filter_impl.cc
+++ b/be/src/exprs/block_bloom_filter_impl.cc
@@ -27,10 +27,12 @@
#include <cmath> // IWYU pragma: keep
#include <cstddef>
#include <cstdint>
-#include <cstdlib>
#include <cstring>
+#include "common/exception.h"
#include "common/status.h"
+#include "core/allocator.h"
+#include "core/allocator_fwd.h"
#include "exprs/block_bloom_filter.hpp"
// IWYU pragma: no_include <emmintrin.h>
#include "util/sse_util.hpp"
@@ -56,24 +58,25 @@ BlockBloomFilter::~BlockBloomFilter() {
Status BlockBloomFilter::init_internal(const int log_space_bytes, uint32_t
hash_seed) {
// Since log_space_bytes is in bytes, we need to convert it to the number
of tiny
// Bloom filters we will use.
- _log_num_buckets = std::max(1, log_space_bytes - kLogBucketByteSize);
+ const int log_num_buckets = std::max(1, log_space_bytes -
kLogBucketByteSize);
// Since we use 32 bits in the arguments of Insert() and Find(),
_log_num_buckets
// must be limited.
- if (_log_num_buckets > 32) {
+ if (log_num_buckets > 32) {
return Status::InvalidArgument("Bloom filter too large.
log_space_bytes: {}",
log_space_bytes);
}
+
+ close(); // Ensure that any previously allocated memory for directory_ is
released.
+ DCHECK(_directory == nullptr);
+
+ _log_num_buckets = log_num_buckets;
// Don't use _log_num_buckets if it will lead to undefined behavior by a
shift
// that is too large.
_directory_mask = (1 << _log_num_buckets) - 1;
const size_t alloc_size = directory_size();
- close(); // Ensure that any previously allocated memory for directory_ is
released.
- DCHECK(_directory == nullptr);
- int rc = posix_memalign((void**)&_directory, 32, alloc_size);
- if (rc != 0) {
- return Status::InternalError("block_bloom_filter alloc fail");
- }
+ RETURN_IF_CATCH_EXCEPTION(_directory = reinterpret_cast<Bucket*>(
+ Allocator<false> {}.alloc(alloc_size,
kBucketByteSize)));
_hash_seed = hash_seed;
return Status::OK();
@@ -113,7 +116,7 @@ Status BlockBloomFilter::init_from_directory(int
log_space_bytes,
void BlockBloomFilter::close() {
if (_directory != nullptr) {
- free(_directory);
+ Allocator<false> {}.free(_directory, directory_size());
_directory = nullptr;
}
}
diff --git a/be/test/exprs/bloom_filter_func_test.cpp
b/be/test/exprs/bloom_filter_func_test.cpp
index bc0ce276a37..5e0c02479be 100644
--- a/be/test/exprs/bloom_filter_func_test.cpp
+++ b/be/test/exprs/bloom_filter_func_test.cpp
@@ -29,9 +29,12 @@
#include "core/data_type/define_primitive_type.h"
#include "core/data_type/primitive_type.h"
#include "core/value/vdatetime_value.h"
+#include "exprs/block_bloom_filter.hpp"
#include "exprs/create_predicate_function.h"
#include "exprs/function/cast/cast_to_datev2_impl.hpp"
#include "gtest/gtest.h"
+#include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/thread_context.h"
#include "testutil/column_helper.h"
#include "util/url_coding.h"
@@ -79,6 +82,30 @@ TEST_F(BloomFilterFuncTest, Init) {
bloom_filter_func2.light_copy(&bloom_filter_func);
}
+TEST_F(BloomFilterFuncTest, TrackBlockBloomFilterMemory) {
+ constexpr int initial_log_space_bytes = 22;
+ constexpr int resized_log_space_bytes = 23;
+ auto mem_tracker =
MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
+
"BlockBloomFilterMemoryTest");
+ auto switch_mem_tracker = SwitchThreadMemTrackerLimiter(mem_tracker);
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ const int64_t initial_consumption = mem_tracker->consumption();
+
+ BlockBloomFilter bloom_filter;
+ ASSERT_TRUE(bloom_filter.init(initial_log_space_bytes, 0).ok());
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ EXPECT_EQ(mem_tracker->consumption(), initial_consumption + (1ULL <<
initial_log_space_bytes));
+ EXPECT_EQ(reinterpret_cast<uintptr_t>(bloom_filter.directory().data) % 32,
0);
+
+ ASSERT_TRUE(bloom_filter.init(resized_log_space_bytes, 0).ok());
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ EXPECT_EQ(mem_tracker->consumption(), initial_consumption + (1ULL <<
resized_log_space_bytes));
+
+ bloom_filter.close();
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ EXPECT_EQ(mem_tracker->consumption(), initial_consumption);
+}
+
TEST_F(BloomFilterFuncTest, FixedLenToUInt32) {
fixed_len_to_uint32_v2 fixed_lenv2;
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
