This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new c7f95adaaae branch-4.1: [fix](be) Prevent score leakage across Boolean
search branches #69053 (#69160)
c7f95adaaae is described below
commit c7f95adaaaec4bc0caeab9ee831a9009745d56d0
Author: Jack <[email protected]>
AuthorDate: Mon Oct 19 14:35:39 2026 +0800
branch-4.1: [fix](be) Prevent score leakage across Boolean search branches
#69053 (#69160)
### What problem does this PR solve?
Related PR: #69053
Problem Summary: Backport the Boolean SEARCH score-isolation fix to
`branch-4.1`. Scores from failing AND branches must not leak into rows
matched by another OR branch; NOT contributes no positive score.
Incomplete scored evaluation returns an explicit error.
#### Branch adaptations
Preserve branch-4.1 compaction paths and compile-check guards, add
move-based score handoff, and adapt tests to the branch-4.1
IndexExecContext constructor and expression interface. No master merge
or storage-directory refactor is included.
#### Nested error propagation
A nested scored child releases its temporary score map before reporting
unsupported index evaluation. Its parent AND/OR previously swallowed
this error when no sibling contributed scores, returning matching rows
with score zero. Scored parents now propagate this error independently
of remaining map contents. Unscored evaluation keeps its existing
fallback behavior.
The SQL suite explicitly enables SEARCH's required expression-pushdown
and inverted-index settings under CI session fuzzing.
#### Score-map memory accounting
Compound score tables now use the existing Doris tracked allocator.
Allocations count toward query memory limits; move handoff preserves the
charge, and merging retains charges for both maps until their storage is
released.
---
be/src/exprs/vcompound_pred.h | 45 +++-
.../storage/compaction/collection_similarity.cpp | 11 +-
be/src/storage/compaction/collection_similarity.h | 18 +-
be/test/exprs/vcompound_pred_score_test.cpp | 276 +++++++++++++++++++++
.../compaction/collection_similarity_test.cpp | 66 ++++-
.../search/test_search_score_boolean_branches.out | 59 +++++
.../test_search_score_boolean_branches.groovy | 133 ++++++++++
7 files changed, 603 insertions(+), 5 deletions(-)
diff --git a/be/src/exprs/vcompound_pred.h b/be/src/exprs/vcompound_pred.h
index 44e0ef1fbc9..bfe76d61496 100644
--- a/be/src/exprs/vcompound_pred.h
+++ b/be/src/exprs/vcompound_pred.h
@@ -20,6 +20,7 @@
#include <algorithm>
#include <cstdint>
+#include <memory>
#include "common/logging.h"
#include "common/status.h"
@@ -29,7 +30,9 @@
#include "exprs/vectorized_fn_call.h"
#include "exprs/vexpr_context.h"
#include "exprs/vexpr_fwd.h"
+#include "storage/compaction/collection_similarity.h"
#include "storage/index/zone_map/zonemap_eval_context.h"
+#include "util/defer_op.h"
#include "util/simd/bits.h"
namespace doris {
@@ -262,9 +265,24 @@ public:
}
}
+ //
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity):
Keep each operator's bitmap and score updates together.
Status evaluate_inverted_index(VExprContext* context, uint32_t
segment_num_rows) override {
segment_v2::InvertedIndexResultBitmap res;
bool all_pass = true;
+ auto index_context = context->get_index_context();
+ auto query_context = index_context->get_index_query_context();
+ auto parent_similarity =
+ query_context != nullptr ?
query_context->collection_similarity : nullptr;
+ CollectionSimilarityPtr branch_similarity;
+ if (parent_similarity != nullptr) {
+ branch_similarity = std::make_shared<CollectionSimilarity>();
+ query_context->collection_similarity = branch_similarity;
+ }
+ Defer restore_similarity {[&]() {
+ if (branch_similarity != nullptr) {
+ query_context->collection_similarity = parent_similarity;
+ }
+ }};
switch (_op) {
case TExprOpcode::COMPOUND_OR: {
@@ -273,6 +291,12 @@ public:
!st.ok()) {
LOG(ERROR) << "expr:" << child->expr_name()
<< " evaluate_inverted_index error:" <<
st.to_string();
+ // A scored child may have discarded its map before
reporting this error.
+ // Preserve the failure even when no sibling contributes
scores.
+ if (branch_similarity != nullptr &&
+ st.code() == ErrorCode::INVERTED_INDEX_NOT_SUPPORTED) {
+ return st;
+ }
all_pass = false;
continue;
}
@@ -285,7 +309,7 @@ public:
} else {
res |= *index_result;
}
- if (inverted_index_context->get_score_runtime() ==
nullptr) {
+ if (branch_similarity == nullptr) {
if (res.get_data_bitmap()->cardinality() ==
segment_num_rows) {
break; // Early exit if result is full
}
@@ -302,6 +326,12 @@ public:
!st.ok()) {
LOG(ERROR) << "expr:" << child->expr_name()
<< " evaluate_inverted_index error:" <<
st.to_string();
+ // A scored child may have discarded its map before
reporting this error.
+ // Preserve the failure even when no sibling contributes
scores.
+ if (branch_similarity != nullptr &&
+ st.code() == ErrorCode::INVERTED_INDEX_NOT_SUPPORTED) {
+ return st;
+ }
all_pass = false;
continue;
}
@@ -348,8 +378,19 @@ public:
"Compound operator must be AND, OR, or NOT to execute with
inverted index.");
}
+ if (branch_similarity != nullptr && _op != TExprOpcode::COMPOUND_NOT) {
+ auto scores = branch_similarity->release_scores();
+ if (!scores.empty() && (!all_pass || res.is_empty())) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "Cannot score a compound predicate with incomplete
index evaluation");
+ }
+ if (all_pass && !res.is_empty()) {
+ DORIS_CHECK(res.get_data_bitmap() != nullptr);
+ parent_similarity->merge_scores(scores,
*res.get_data_bitmap());
+ }
+ }
if (all_pass && !res.is_empty()) {
- context->get_index_context()->set_index_result_for_expr(this, res);
+ index_context->set_index_result_for_expr(this, res);
}
return Status::OK();
}
diff --git a/be/src/storage/compaction/collection_similarity.cpp
b/be/src/storage/compaction/collection_similarity.cpp
index a4e5e64a807..0cce8016d37 100644
--- a/be/src/storage/compaction/collection_similarity.cpp
+++ b/be/src/storage/compaction/collection_similarity.cpp
@@ -27,6 +27,15 @@ void CollectionSimilarity::collect(segment_v2::rowid_t
row_id, float score) {
_bm25_scores[row_id] += score;
}
+void CollectionSimilarity::merge_scores(const ScoreMap& scores,
+ const roaring::Roaring& matched_rows) {
+ for (const auto& [row_id, score] : scores) {
+ if (matched_rows.contains(row_id)) {
+ collect(row_id, score);
+ }
+ }
+}
+
void CollectionSimilarity::get_bm25_scores(roaring::Roaring* row_bitmap,
IColumn::MutablePtr& scores,
std::unique_ptr<std::vector<uint64_t>>& row_ids,
@@ -167,4 +176,4 @@ void CollectionSimilarity::find_top_k_scores(const
roaring::Roaring* row_bitmap,
}
#include "common/compile_check_end.h"
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/storage/compaction/collection_similarity.h
b/be/src/storage/compaction/collection_similarity.h
index 97256be13c8..51ef8cd108e 100644
--- a/be/src/storage/compaction/collection_similarity.h
+++ b/be/src/storage/compaction/collection_similarity.h
@@ -20,12 +20,13 @@
#include <gen_cpp/Opcodes_types.h>
#include "core/column/column.h"
+#include "exec/common/hash_table/phmap_fwd_decl.h"
#include "storage/segment/common.h"
namespace doris {
#include "common/compile_check_begin.h"
-using ScoreMap = phmap::flat_hash_map<segment_v2::rowid_t, float>;
+using ScoreMap = doris::flat_hash_map<segment_v2::rowid_t, float>;
using ScoreMapIterator = ScoreMap::const_iterator;
enum class OrderType {
@@ -50,6 +51,21 @@ public:
void collect(segment_v2::rowid_t row_id, float score);
+ void merge_scores(const ScoreMap& scores, const roaring::Roaring&
matched_rows);
+
+ // Hands the collected scores over to the caller and leaves this instance
empty.
+ // A reader that computes per-document scores inside its own query()
cannot return them
+ // through the query API, so it publishes them into a throwaway
CollectionSimilarity that the
+ // caller then relocates into a scorer. Moving instead of copying keeps
that hand-off free of
+ // a full rehash of a map that can hold one entry per matched row.
+ ScoreMap release_scores() {
+ ScoreMap released = std::move(_bm25_scores);
+ // A moved-from flat_hash_map is valid but unspecified, not guaranteed
empty, so make the
+ // "leaves this instance empty" half of the contract true rather than
merely likely.
+ _bm25_scores.clear();
+ return released;
+ }
+
void get_bm25_scores(roaring::Roaring* row_bitmap, IColumn::MutablePtr&
scores,
std::unique_ptr<std::vector<uint64_t>>& row_ids,
const ScoreRangeFilterPtr& filter = nullptr) const;
diff --git a/be/test/exprs/vcompound_pred_score_test.cpp
b/be/test/exprs/vcompound_pred_score_test.cpp
new file mode 100644
index 00000000000..35ee59dddf6
--- /dev/null
+++ b/be/test/exprs/vcompound_pred_score_test.cpp
@@ -0,0 +1,276 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <initializer_list>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "core/data_type/primitive_type.h"
+#include "exprs/score_runtime.h"
+#include "exprs/vcompound_pred.h"
+#include "exprs/vexpr_context.h"
+#include "storage/compaction/collection_similarity.h"
+#include "storage/index/index_iterator.h"
+#include "storage/index/index_query_context.h"
+
+namespace doris {
+namespace {
+
+class ScoredBitmapExpr final : public VExpr {
+public:
+ ScoredBitmapExpr(std::initializer_list<uint32_t> rows, float score) :
_score(score) {
+ set_node_type(TExprNodeType::MATCH_PRED);
+ for (uint32_t row : rows) {
+ _rows.add(row);
+ }
+ }
+
+ const std::string& expr_name() const override {
+ static const std::string name = "ScoredBitmapExpr";
+ return name;
+ }
+
+ Status execute(VExprContext*, Block*, int*) const override { return
Status::OK(); }
+
+ Status execute_column(VExprContext*, const Block*, Selector*, size_t,
+ ColumnPtr&) const override {
+ return Status::OK();
+ }
+
+ Status evaluate_inverted_index(VExprContext* context, uint32_t) override {
+ auto index_context = context->get_index_context();
+ auto query_context = index_context->get_index_query_context();
+ for (uint32_t row : _rows) {
+ query_context->collection_similarity->collect(row, _score);
+ }
+ index_context->set_index_result_for_expr(
+ this,
+
segment_v2::InvertedIndexResultBitmap(std::make_shared<roaring::Roaring>(_rows),
+
std::make_shared<roaring::Roaring>()));
+ return Status::OK();
+ }
+
+private:
+ roaring::Roaring _rows;
+ float _score;
+};
+
+class UnindexedExpr final : public VExpr {
+public:
+ const std::string& expr_name() const override {
+ static const std::string name = "UnindexedExpr";
+ return name;
+ }
+
+ Status execute(VExprContext*, Block*, int*) const override { return
Status::OK(); }
+
+ Status execute_column(VExprContext*, const Block*, Selector*, size_t,
+ ColumnPtr&) const override {
+ return Status::OK();
+ }
+
+ Status evaluate_inverted_index(VExprContext*, uint32_t) override { return
Status::OK(); }
+};
+
+std::shared_ptr<VCompoundPred> make_compound(TExprOpcode::type op, VExprSPtr
left,
+ VExprSPtr right) {
+ TExprNode node;
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_node_type(TExprNodeType::COMPOUND_PRED);
+ node.__set_opcode(op);
+ node.__set_num_children(2);
+ node.__set_is_nullable(false);
+ auto expr = VCompoundPred::create_shared(node);
+ expr->add_child(left);
+ expr->add_child(right);
+ return expr;
+}
+
+std::shared_ptr<VCompoundPred> make_not(VExprSPtr child) {
+ TExprNode node;
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_node_type(TExprNodeType::COMPOUND_PRED);
+ node.__set_opcode(TExprOpcode::COMPOUND_NOT);
+ node.__set_num_children(1);
+ node.__set_is_nullable(false);
+ auto expr = VCompoundPred::create_shared(node);
+ expr->add_child(child);
+ return expr;
+}
+
+TEST(VCompoundPredScoreTest, FailedAndBranchDoesNotContributeToOrScore) {
+ auto first = make_compound(
+ TExprOpcode::COMPOUND_AND,
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 2, 4}, 1.0F),
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 2, 5}, 2.0F));
+ auto second = make_compound(
+ TExprOpcode::COMPOUND_AND,
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 2, 3, 4, 5},
+ 3.0F),
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 3, 4, 5}, 4.0F));
+ auto root = make_compound(TExprOpcode::COMPOUND_OR, first, second);
+
+ std::vector<ColumnId> col_ids;
+ std::vector<std::unique_ptr<segment_v2::IndexIterator>> iterators;
+ std::vector<IndexFieldNameAndTypePair> storage_types;
+ std::unordered_map<ColumnId, std::unordered_map<const VExpr*, bool>>
status_map;
+ auto score_runtime = std::make_shared<ScoreRuntime>(nullptr, false, 0);
+ auto index_context = std::make_shared<IndexExecContext>(col_ids,
iterators, storage_types,
+ status_map,
score_runtime);
+ auto query_context = std::make_shared<segment_v2::IndexQueryContext>();
+ query_context->collection_similarity =
std::make_shared<CollectionSimilarity>();
+ auto root_similarity = query_context->collection_similarity;
+ index_context->set_index_query_context(query_context);
+ auto expr_context = VExprContext::create_shared(root);
+ expr_context->set_index_context(index_context);
+
+ ASSERT_TRUE(root->evaluate_inverted_index(expr_context.get(), 6).ok());
+ const auto* result = index_context->get_index_result_for_expr(root.get());
+ ASSERT_NE(result, nullptr);
+ EXPECT_EQ(result->get_data_bitmap()->cardinality(), 5);
+ EXPECT_EQ(query_context->collection_similarity, root_similarity);
+
+ auto scores = root_similarity->release_scores();
+ ASSERT_EQ(scores.size(), 5);
+ EXPECT_FLOAT_EQ(scores.at(1), 10.0F);
+ EXPECT_FLOAT_EQ(scores.at(2), 3.0F);
+ EXPECT_FLOAT_EQ(scores.at(3), 7.0F);
+ EXPECT_FLOAT_EQ(scores.at(4), 7.0F);
+ EXPECT_FLOAT_EQ(scores.at(5), 7.0F);
+}
+
+TEST(VCompoundPredScoreTest, NotBranchDoesNotContributeToOrScore) {
+ auto negative = make_not(
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 2}, 1.0F));
+ auto positive =
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 3}, 5.0F);
+ auto root = make_compound(TExprOpcode::COMPOUND_OR, negative, positive);
+
+ std::vector<ColumnId> col_ids;
+ std::vector<std::unique_ptr<segment_v2::IndexIterator>> iterators;
+ std::vector<IndexFieldNameAndTypePair> storage_types;
+ std::unordered_map<ColumnId, std::unordered_map<const VExpr*, bool>>
status_map;
+ auto score_runtime = std::make_shared<ScoreRuntime>(nullptr, false, 0);
+ auto index_context = std::make_shared<IndexExecContext>(col_ids,
iterators, storage_types,
+ status_map,
score_runtime);
+ auto query_context = std::make_shared<segment_v2::IndexQueryContext>();
+ query_context->collection_similarity =
std::make_shared<CollectionSimilarity>();
+ index_context->set_index_query_context(query_context);
+ auto expr_context = VExprContext::create_shared(root);
+ expr_context->set_index_context(index_context);
+
+ ASSERT_TRUE(root->evaluate_inverted_index(expr_context.get(), 4).ok());
+ const auto* result = index_context->get_index_result_for_expr(root.get());
+ ASSERT_NE(result, nullptr);
+ EXPECT_EQ(result->get_data_bitmap()->cardinality(), 3);
+ EXPECT_TRUE(result->get_data_bitmap()->contains(0));
+ EXPECT_TRUE(result->get_data_bitmap()->contains(1));
+ EXPECT_TRUE(result->get_data_bitmap()->contains(3));
+
+ auto scores = query_context->collection_similarity->release_scores();
+ ASSERT_EQ(scores.size(), 2);
+ EXPECT_FLOAT_EQ(scores.at(1), 5.0F);
+ EXPECT_FLOAT_EQ(scores.at(3), 5.0F);
+}
+
+TEST(VCompoundPredScoreTest,
IncompleteScoredBranchReturnsErrorWithoutPublishingScores) {
+ auto root = make_compound(
+ TExprOpcode::COMPOUND_AND,
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1}, 1.0F),
+ std::make_shared<UnindexedExpr>());
+
+ std::vector<ColumnId> col_ids;
+ std::vector<std::unique_ptr<segment_v2::IndexIterator>> iterators;
+ std::vector<IndexFieldNameAndTypePair> storage_types;
+ std::unordered_map<ColumnId, std::unordered_map<const VExpr*, bool>>
status_map;
+ auto score_runtime = std::make_shared<ScoreRuntime>(nullptr, false, 0);
+ auto index_context = std::make_shared<IndexExecContext>(col_ids,
iterators, storage_types,
+ status_map,
score_runtime);
+ auto query_context = std::make_shared<segment_v2::IndexQueryContext>();
+ query_context->collection_similarity =
std::make_shared<CollectionSimilarity>();
+ auto parent_similarity = query_context->collection_similarity;
+ index_context->set_index_query_context(query_context);
+ auto expr_context = VExprContext::create_shared(root);
+ expr_context->set_index_context(index_context);
+
+ auto status = root->evaluate_inverted_index(expr_context.get(), 2);
+ EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
+ EXPECT_EQ(query_context->collection_similarity, parent_similarity);
+ EXPECT_TRUE(parent_similarity->release_scores().empty());
+}
+
+TEST(VCompoundPredScoreTest,
NestedIncompleteScoringErrorSurvivesEmptyOrUnindexedSiblings) {
+ for (auto op : {TExprOpcode::COMPOUND_OR, TExprOpcode::COMPOUND_AND}) {
+ for (bool unindexed_sibling : {false, true}) {
+ for (bool reverse_children : {false, true}) {
+ SCOPED_TRACE(::testing::Message()
+ << "op=" << op << " unindexed=" <<
unindexed_sibling
+ << " reversed=" << reverse_children);
+ auto incomplete = make_compound(TExprOpcode::COMPOUND_AND,
+
std::make_shared<ScoredBitmapExpr>(
+
std::initializer_list<uint32_t> {1}, 1.0F),
+
std::make_shared<UnindexedExpr>());
+ VExprSPtr sibling;
+ if (unindexed_sibling) {
+ sibling = std::make_shared<UnindexedExpr>();
+ } else if (op == TExprOpcode::COMPOUND_AND) {
+ // An empty AND may short-circuit before the incomplete
child is reached.
+ sibling = std::make_shared<ScoredBitmapExpr>(
+ std::initializer_list<uint32_t> {1}, 0.0F);
+ } else {
+ sibling =
std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t> {},
+ 0.0F);
+ }
+ auto nested = reverse_children ? make_compound(op, sibling,
incomplete)
+ : make_compound(op, incomplete,
sibling);
+ // Exercise propagation across multiple compound levels,
including NOT.
+ auto root = make_not(make_not(nested));
+
+ std::vector<ColumnId> col_ids;
+ std::vector<std::unique_ptr<segment_v2::IndexIterator>>
iterators;
+ std::vector<IndexFieldNameAndTypePair> storage_types;
+ std::unordered_map<ColumnId, std::unordered_map<const VExpr*,
bool>> status_map;
+ auto score_runtime = std::make_shared<ScoreRuntime>(nullptr,
false, 0);
+ auto index_context = std::make_shared<IndexExecContext>(
+ col_ids, iterators, storage_types, status_map,
score_runtime);
+ auto query_context =
std::make_shared<segment_v2::IndexQueryContext>();
+ auto parent_similarity =
std::make_shared<CollectionSimilarity>();
+ parent_similarity->collect(0, 7.0F);
+ query_context->collection_similarity = parent_similarity;
+ index_context->set_index_query_context(query_context);
+ auto expr_context = VExprContext::create_shared(root);
+ expr_context->set_index_context(index_context);
+
+ auto status =
root->evaluate_inverted_index(expr_context.get(), 2);
+ EXPECT_EQ(status.code(),
ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
+ EXPECT_EQ(query_context->collection_similarity,
parent_similarity);
+
EXPECT_FALSE(index_context->has_index_result_for_expr(root.get()));
+ auto scores = parent_similarity->release_scores();
+ ASSERT_EQ(scores.size(), 1);
+ EXPECT_FLOAT_EQ(scores.at(0), 7.0F);
+ }
+ }
+ }
+}
+
+} // namespace
+} // namespace doris
diff --git a/be/test/storage/compaction/collection_similarity_test.cpp
b/be/test/storage/compaction/collection_similarity_test.cpp
index 7b5bf796353..ba76d3418c4 100644
--- a/be/test/storage/compaction/collection_similarity_test.cpp
+++ b/be/test/storage/compaction/collection_similarity_test.cpp
@@ -22,8 +22,12 @@
#include <memory>
#include <vector>
+#include "common/exception.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type_number.h"
+#include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/thread_context.h"
+#include "util/defer_op.h"
namespace doris {
@@ -763,4 +767,64 @@ TEST_F(CollectionSimilarityTest,
LargeDataBoundaryThresholdTest) {
}
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
+namespace doris {
+
+TEST(CollectionSimilarityMemoryTest, TracksCollectedMovedAndMergedScores) {
+ auto tracker =
+ MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
"compound-score-map");
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(tracker);
+ const auto consumption = [&] {
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ return tracker->consumption();
+ };
+ const auto baseline = consumption();
+ constexpr uint32_t row_count = 32768;
+ roaring::Roaring matched_rows;
+ matched_rows.addRange(0, row_count);
+ {
+ CollectionSimilarity child;
+ EXPECT_GT(consumption(), baseline);
+ for (uint32_t row = 0; row < row_count; ++row) {
+ child.collect(row, 1.0F);
+ }
+ const auto child_bytes = consumption() - baseline;
+ EXPECT_GE(child_bytes, row_count * sizeof(ScoreMap::value_type));
+ {
+ auto scores = child.release_scores();
+ EXPECT_EQ(scores.size(), row_count);
+ EXPECT_EQ(consumption(), baseline + child_bytes);
+ {
+ CollectionSimilarity parent;
+ parent.merge_scores(scores, matched_rows);
+ EXPECT_GE(consumption(), baseline + 2 * child_bytes);
+ auto merged = parent.release_scores();
+ ASSERT_EQ(merged.size(), row_count);
+ EXPECT_FLOAT_EQ(merged.at(row_count - 1), 1.0F);
+ }
+ EXPECT_EQ(consumption(), baseline + child_bytes);
+ }
+ EXPECT_EQ(consumption(), baseline);
+ child.collect(7, 2.0F);
+ EXPECT_GT(consumption(), baseline);
+ }
+ EXPECT_EQ(consumption(), baseline);
+}
+
+TEST(CollectionSimilarityMemoryTest, RejectsAllocationsAboveTrackerLimit) {
+ auto tracker =
MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
+
"compound-score-map-limit", 1024);
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(tracker);
+ ++enable_thread_catch_bad_alloc;
+ Defer restore_catch_bad_alloc {[] { --enable_thread_catch_bad_alloc; }};
+ try {
+ CollectionSimilarity collector;
+ FAIL() << "Expected the score map to respect the memory limit";
+ } catch (const Exception& e) {
+ EXPECT_EQ(e.code(), ErrorCode::MEM_ALLOC_FAILED);
+ }
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ EXPECT_EQ(tracker->consumption(), 0);
+}
+
+} // namespace doris
diff --git a/regression-test/data/search/test_search_score_boolean_branches.out
b/regression-test/data/search/test_search_score_boolean_branches.out
new file mode 100644
index 00000000000..d68e69ed252
--- /dev/null
+++ b/regression-test/data/search/test_search_score_boolean_branches.out
@@ -0,0 +1,59 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !search --
+1 1.3714
+10 1.2833
+2 0.8781
+4 0.4932
+5 0.4932
+6 1.5112
+7 0.4932
+9 1.5553
+
+-- !match --
+1 1.3714
+10 1.2833
+2 0.8781
+4 0.4932
+5 0.4932
+6 1.5112
+7 0.4932
+9 1.5553
+
+-- !match_warm --
+1 1.3714
+10 1.2833
+2 0.8781
+4 0.4932
+5 0.4932
+6 1.5112
+7 0.4932
+9 1.5553
+
+-- !failed_and --
+1 0.4932
+10 0.4052
+4 0.4932
+5 0.4932
+6 0.4932
+7 0.4932
+9 0.4932
+
+-- !failed_and_reversed --
+1 0.4932
+10 0.4052
+4 0.4932
+5 0.4932
+6 0.4932
+7 0.4932
+9 0.4932
+
+-- !not_branch --
+1 0.4932
+10 0.4052
+3 0
+4 0.4932
+5 0.4932
+6 0.4932
+7 0.4932
+8 0
+9 0.4932
diff --git
a/regression-test/suites/search/test_search_score_boolean_branches.groovy
b/regression-test/suites/search/test_search_score_boolean_branches.groovy
new file mode 100644
index 00000000000..39bd56cf107
--- /dev/null
+++ b/regression-test/suites/search/test_search_score_boolean_branches.groovy
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_search_score_boolean_branches", "p0") {
+ // SEARCH has no row-scan fallback. Keep its index path enabled under CI
session fuzzing.
+ sql "set enable_common_expr_pushdown = true"
+ sql "set enable_inverted_index_query = true"
+
+ sql "DROP TABLE IF EXISTS test_search_score_multi_column_mixed"
+ sql """
+ CREATE TABLE test_search_score_multi_column_mixed (
+ id INT NOT NULL,
+ title STRING NULL,
+ content STRING NULL,
+ tags STRING NULL,
+ INDEX idx_title(title) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_content(content) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_tags(tags) USING INVERTED PROPERTIES("parser" =
"english")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default:1")
+ """
+ sql """
+ INSERT INTO test_search_score_multi_column_mixed VALUES
+ (1, 'apple banana', 'search engine', 'database analytics'),
+ (2, 'apple guide', 'search basics', 'database'),
+ (3, 'banana manual', 'engine tuning', 'analytics'),
+ (4, 'apple banana', 'storage engine', 'database analytics'),
+ (5, 'pear guide', 'search engine', 'database analytics'),
+ (6, 'apple', 'engine search', 'analytics database'),
+ (7, 'banana', 'search', 'database analytics'),
+ (8, 'grape', 'storage', 'archive'),
+ (9, 'apple banana apple', 'search engine search', 'database
analytics'),
+ (10, 'apple docs', 'search engine', 'database analytics archive')
+ """
+
+ order_qt_search """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE search('(title:apple AND content:search) OR (tags:database AND
tags:analytics)',
+
'{"default_operator":"AND","mode":"lucene","minimum_should_match":0}')
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_match """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (title MATCH_ANY 'apple' AND content MATCH_ANY 'search')
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_match_warm """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (title MATCH_ANY 'apple' AND content MATCH_ANY 'search')
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_failed_and """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (title MATCH_ANY 'apple' AND content MATCH_ANY 'missing_token')
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_failed_and_reversed """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (content MATCH_ANY 'missing_token' AND title MATCH_ANY 'apple')
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_not_branch """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE NOT (title MATCH_ANY 'apple') OR tags MATCH_ALL 'database
analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ test {
+ sql """
+ SELECT id, score() AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (title MATCH_ANY 'apple' AND id > 0)
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY bm25_score DESC LIMIT 100
+ """
+ exception "Cannot score a compound predicate with incomplete index
evaluation"
+ }
+
+ // A failed scored child must remain an error even when its sibling adds
no scores.
+ for (boolean wand : [false, true]) {
+ sql "set enable_inverted_index_wand_query = ${wand}"
+ for (String sibling : ["tags MATCH_ANY 'missing_token'", "id < 0"]) {
+ for (boolean reverseChildren : [false, true]) {
+ String incomplete = "(title MATCH_ANY 'apple' AND id > 0)"
+ String predicate = reverseChildren ? "${sibling} OR
${incomplete}"
+ : "${incomplete} OR
${sibling}"
+ // Repeat the query to cover bitmap-cache reuse after the
error.
+ for (int attempt = 0; attempt < 2; ++attempt) {
+ test {
+ sql """
+ SELECT id, score() AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE ${predicate}
+ ORDER BY bm25_score DESC LIMIT 100
+ """
+ exception "Cannot score a compound predicate with
incomplete index evaluation"
+ }
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
