This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 08b68a2d8da [fix](inverted index) AcceptNullPredicate should include
null rows in result bitmap (#59959)
08b68a2d8da is described below
commit 08b68a2d8da11b0eb985fe9f1bdc85c66229adaf
Author: zzzxl <[email protected]>
AuthorDate: Sun Jan 18 16:20:25 2026 +0800
[fix](inverted index) AcceptNullPredicate should include null rows in
result bitmap (#59959)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
```
---
be/src/olap/accept_null_predicate.h | 15 +-
be/src/runtime/memory/lru_cache_policy.h | 5 +-
be/test/olap/accept_null_predicate_test.cpp | 217 ++++++++++++++++++++++++++++
3 files changed, 234 insertions(+), 3 deletions(-)
diff --git a/be/src/olap/accept_null_predicate.h
b/be/src/olap/accept_null_predicate.h
index b223cd3a401..ef6f1a88dc7 100644
--- a/be/src/olap/accept_null_predicate.h
+++ b/be/src/olap/accept_null_predicate.h
@@ -23,6 +23,7 @@
#include "common/factory_creator.h"
#include "olap/column_predicate.h"
#include "olap/rowset/segment_v2/bloom_filter.h"
+#include "olap/rowset/segment_v2/inverted_index_cache.h"
#include "olap/rowset/segment_v2/inverted_index_reader.h"
#include "olap/wrapper_field.h"
#include "vec/columns/column_dictionary.h"
@@ -68,7 +69,19 @@ public:
Status evaluate(const vectorized::IndexFieldNameAndTypePair&
name_with_type,
IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const override {
- return _nested->evaluate(name_with_type, iterator, num_rows, bitmap);
+ RETURN_IF_ERROR(_nested->evaluate(name_with_type, iterator, num_rows,
bitmap));
+ if (iterator != nullptr) {
+ bool has_null = DORIS_TRY(iterator->has_null());
+ if (has_null) {
+ InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
+
RETURN_IF_ERROR(iterator->read_null_bitmap(&null_bitmap_cache_handle));
+ auto null_bitmap = null_bitmap_cache_handle.get_bitmap();
+ if (null_bitmap) {
+ *bitmap |= *null_bitmap;
+ }
+ }
+ }
+ return Status::OK();
}
void evaluate_and(const vectorized::IColumn& column, const uint16_t* sel,
uint16_t size,
diff --git a/be/src/runtime/memory/lru_cache_policy.h
b/be/src/runtime/memory/lru_cache_policy.h
index dd2ebcabbb0..6977cc39549 100644
--- a/be/src/runtime/memory/lru_cache_policy.h
+++ b/be/src/runtime/memory/lru_cache_policy.h
@@ -21,6 +21,7 @@
#include <memory>
+#include "common/be_mock_util.h"
#include "olap/lru_cache.h"
#include "runtime/memory/cache_policy.h"
#include "runtime/memory/lru_cache_value_base.h"
@@ -135,9 +136,9 @@ public:
Cache::Handle* lookup(const CacheKey& key) { return _cache->lookup(key); }
- void release(Cache::Handle* handle) { _cache->release(handle); }
+ MOCK_FUNCTION void release(Cache::Handle* handle) {
_cache->release(handle); }
- void* value(Cache::Handle* handle) { return _cache->value(handle); }
+ MOCK_FUNCTION void* value(Cache::Handle* handle) { return
_cache->value(handle); }
void erase(const CacheKey& key) { _cache->erase(key); }
diff --git a/be/test/olap/accept_null_predicate_test.cpp
b/be/test/olap/accept_null_predicate_test.cpp
new file mode 100644
index 00000000000..edffd80b0d4
--- /dev/null
+++ b/be/test/olap/accept_null_predicate_test.cpp
@@ -0,0 +1,217 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/accept_null_predicate.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <roaring/roaring.hh>
+
+#include "olap/comparison_predicate.h"
+#include "olap/rowset/segment_v2/index_iterator.h"
+#include "olap/rowset/segment_v2/inverted_index_cache.h"
+#include "runtime/define_primitive_type.h"
+#include "runtime/memory/lru_cache_policy.h"
+
+namespace doris {
+
+class MockLRUCachePolicy : public LRUCachePolicy {
+public:
+ MockLRUCachePolicy(std::shared_ptr<roaring::Roaring> bitmap)
+ :
LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_QUERY_CACHE, 1024,
+ LRUCacheType::SIZE, 3600, 1, 0, true, false) {
+ _cache_value.bitmap = bitmap;
+ }
+
+ void* value(Cache::Handle* handle) override { return &_cache_value; }
+
+ void release(Cache::Handle* handle) override {}
+
+ segment_v2::InvertedIndexQueryCache::CacheValue _cache_value;
+};
+
+class MockIndexIterator : public segment_v2::IndexIterator {
+public:
+ MockIndexIterator(bool has_null, std::shared_ptr<roaring::Roaring>
null_bitmap)
+ : _has_null_value(has_null), _null_bitmap(std::move(null_bitmap)) {
+ if (_null_bitmap) {
+ _mock_cache = std::make_unique<MockLRUCachePolicy>(_null_bitmap);
+ }
+ }
+
+ Result<bool> has_null() override { return _has_null_value; }
+
+ Status read_null_bitmap(segment_v2::InvertedIndexQueryCacheHandle*
cache_handle) override {
+ if (_mock_cache && _null_bitmap) {
+ *cache_handle = segment_v2::InvertedIndexQueryCacheHandle(
+ _mock_cache.get(), reinterpret_cast<Cache::Handle*>(1));
+ }
+ return Status::OK();
+ }
+
+ segment_v2::IndexReaderPtr get_reader(segment_v2::IndexReaderType) const
override {
+ return nullptr;
+ }
+
+ Status read_from_index(const segment_v2::IndexParam&) override { return
Status::OK(); }
+
+private:
+ bool _has_null_value;
+ std::shared_ptr<roaring::Roaring> _null_bitmap;
+ std::unique_ptr<MockLRUCachePolicy> _mock_cache;
+};
+
+class MockNestedPredicate : public ColumnPredicate {
+public:
+ MockNestedPredicate(uint32_t column_id, std::shared_ptr<roaring::Roaring>
result_bitmap)
+ : ColumnPredicate(column_id, "mock_col", PrimitiveType::TYPE_INT,
false),
+ _result_bitmap(std::move(result_bitmap)) {}
+
+ PredicateType type() const override { return PredicateType::EQ; }
+
+ Status evaluate(const vectorized::IndexFieldNameAndTypePair&
name_with_type,
+ segment_v2::IndexIterator* iterator, uint32_t num_rows,
+ roaring::Roaring* bitmap) const override {
+ *bitmap = *_result_bitmap;
+ return Status::OK();
+ }
+
+ std::shared_ptr<ColumnPredicate> clone(uint32_t col_id) const override {
+ return std::make_shared<MockNestedPredicate>(col_id, _result_bitmap);
+ }
+
+private:
+ uint16_t _evaluate_inner(const vectorized::IColumn& column, uint16_t* sel,
+ uint16_t size) const override {
+ return size;
+ }
+
+ std::shared_ptr<roaring::Roaring> _result_bitmap;
+};
+
+class AcceptNullPredicateTest : public testing::Test {
+public:
+ AcceptNullPredicateTest() = default;
+ ~AcceptNullPredicateTest() = default;
+};
+
+TEST_F(AcceptNullPredicateTest, EvaluateWithNullIterator) {
+ auto nested_result = std::make_shared<roaring::Roaring>();
+ nested_result->add(2);
+ nested_result->add(5);
+ nested_result->add(8);
+
+ auto nested_pred = std::make_shared<MockNestedPredicate>(0, nested_result);
+ AcceptNullPredicate predicate(nested_pred);
+
+ roaring::Roaring bitmap;
+ vectorized::IndexFieldNameAndTypePair name_with_type = {"test_col",
nullptr};
+
+ Status status = predicate.evaluate(name_with_type, nullptr, 10, &bitmap);
+
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(bitmap.cardinality(), 3);
+ EXPECT_TRUE(bitmap.contains(2));
+ EXPECT_TRUE(bitmap.contains(5));
+ EXPECT_TRUE(bitmap.contains(8));
+}
+
+TEST_F(AcceptNullPredicateTest, EvaluateWithNoNull) {
+ auto nested_result = std::make_shared<roaring::Roaring>();
+ nested_result->add(1);
+ nested_result->add(3);
+ nested_result->add(7);
+
+ auto nested_pred = std::make_shared<MockNestedPredicate>(0, nested_result);
+ AcceptNullPredicate predicate(nested_pred);
+
+ auto null_bitmap = std::make_shared<roaring::Roaring>();
+ null_bitmap->add(0);
+ MockIndexIterator iterator(false, null_bitmap);
+
+ roaring::Roaring bitmap;
+ vectorized::IndexFieldNameAndTypePair name_with_type = {"test_col",
nullptr};
+
+ Status status = predicate.evaluate(name_with_type, &iterator, 10, &bitmap);
+
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(bitmap.cardinality(), 3);
+ EXPECT_TRUE(bitmap.contains(1));
+ EXPECT_TRUE(bitmap.contains(3));
+ EXPECT_TRUE(bitmap.contains(7));
+ EXPECT_FALSE(bitmap.contains(0));
+}
+
+TEST_F(AcceptNullPredicateTest, EvaluateWithNullRows) {
+ auto nested_result = std::make_shared<roaring::Roaring>();
+ nested_result->add(2);
+ nested_result->add(5);
+ nested_result->add(8);
+
+ auto nested_pred = std::make_shared<MockNestedPredicate>(0, nested_result);
+ AcceptNullPredicate predicate(nested_pred);
+
+ auto null_bitmap = std::make_shared<roaring::Roaring>();
+ null_bitmap->add(0);
+ null_bitmap->add(3);
+ null_bitmap->add(9);
+ MockIndexIterator iterator(true, null_bitmap);
+
+ roaring::Roaring bitmap;
+ vectorized::IndexFieldNameAndTypePair name_with_type = {"test_col",
nullptr};
+
+ Status status = predicate.evaluate(name_with_type, &iterator, 10, &bitmap);
+
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(bitmap.cardinality(), 6);
+ EXPECT_TRUE(bitmap.contains(0));
+ EXPECT_TRUE(bitmap.contains(2));
+ EXPECT_TRUE(bitmap.contains(3));
+ EXPECT_TRUE(bitmap.contains(5));
+ EXPECT_TRUE(bitmap.contains(8));
+ EXPECT_TRUE(bitmap.contains(9));
+}
+
+TEST_F(AcceptNullPredicateTest, DebugString) {
+ auto nested_result = std::make_shared<roaring::Roaring>();
+ auto nested_pred = std::make_shared<MockNestedPredicate>(0, nested_result);
+ AcceptNullPredicate predicate(nested_pred);
+
+ std::string debug_str = predicate.debug_string();
+ EXPECT_TRUE(debug_str.find("AcceptNullPredicate") != std::string::npos);
+}
+
+TEST_F(AcceptNullPredicateTest, TypeDelegation) {
+ auto nested_result = std::make_shared<roaring::Roaring>();
+ auto nested_pred = std::make_shared<MockNestedPredicate>(0, nested_result);
+ AcceptNullPredicate predicate(nested_pred);
+
+ EXPECT_EQ(predicate.type(), PredicateType::EQ);
+}
+
+TEST_F(AcceptNullPredicateTest, Clone) {
+ auto nested_result = std::make_shared<roaring::Roaring>();
+ auto nested_pred = std::make_shared<MockNestedPredicate>(0, nested_result);
+ AcceptNullPredicate predicate(nested_pred);
+
+ auto cloned = predicate.clone(1);
+ EXPECT_NE(cloned, nullptr);
+ EXPECT_EQ(cloned->column_id(), 1);
+}
+
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
