This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f121725b127 [feature](inverted index) add boolean query (#55721)
f121725b127 is described below
commit f121725b127f246622364db2c85698498cb171c0
Author: zzzxl <[email protected]>
AuthorDate: Thu Sep 11 20:03:59 2025 +0800
[feature](inverted index) add boolean query (#55721)
Problem Summary:
1. The current index query implementation has limitations when handling
multiple search conditions simultaneously. Users need the ability to
combine multiple term queries using boolean operators (AND, OR, NOT) for
more complex and flexible search scenarios.
2. Currently this is only the initial version, supporting term
combination queries.
---
be/src/olap/collection_statistics.h | 1 +
.../inverted_index/query_v2/boolean_query.cpp | 91 ------
.../inverted_index/query_v2/boolean_query.h | 56 ----
.../query_v2/boolean_query/boolean_query.h | 71 +++++
.../query_v2/boolean_query/boolean_weight.h | 63 ++++
.../query_v2/buffered_union_scorer.cpp | 258 +++++++++++++++++
.../query_v2/buffered_union_scorer.h | 65 +++++
.../inverted_index/query_v2/conjunction_op.cpp | 94 ------
.../inverted_index/query_v2/disjunction_op.cpp | 86 ------
.../inverted_index/query_v2/disjunction_op.h | 59 ----
.../segment_v2/inverted_index/query_v2/doc_set.h | 55 ++++
.../segment_v2/inverted_index/query_v2/factory.cpp | 36 ---
.../inverted_index/query_v2/factory.inline.h | 47 ---
.../query_v2/intersection_scorer.cpp | 158 +++++++++++
.../{conjunction_op.h => intersection_scorer.h} | 37 +--
.../segment_v2/inverted_index/query_v2/node.h | 128 ---------
.../segment_v2/inverted_index/query_v2/operator.h | 27 +-
.../inverted_index/query_v2/phrase_query.h | 38 ---
.../segment_v2/inverted_index/query_v2/query.h | 31 +-
.../inverted_index/query_v2/roaring_query.cpp | 25 --
.../inverted_index/query_v2/roaring_query.h | 62 ----
.../inverted_index/query_v2/score_combiner.h | 70 +++++
.../query_v2/{term_query.h => scorer.h} | 34 ++-
.../inverted_index/query_v2/segment_postings.h | 107 +++++++
.../inverted_index/query_v2/term_query.cpp | 36 ---
.../query_v2/term_query/term_query.h | 52 ++++
.../query_v2/term_query/term_scorer.h | 52 ++++
.../query_v2/term_query/term_weight.h | 73 +++++
.../query_v2/{factory.h => weight.h} | 25 +-
.../inverted_index/similarity/bm25_similarity.cpp | 29 +-
.../inverted_index/similarity/bm25_similarity.h | 3 +
.../inverted_index/similarity/similarity.h | 4 +-
.../segment_v2/inverted_index/util/term_iterator.h | 21 +-
.../inverted_index/util/term_position_iterator.h | 9 +-
.../segment_v2/inverted_index/util/tiny_set.h | 59 ++++
.../olap/rowset/segment_v2/inverted_index_common.h | 29 +-
be/test/olap/collection_statistics_test.cpp | 37 +++
.../inverted_index/query_v2/boolean_query_test.cpp | 316 +++++++++++++++++++++
.../inverted_index/query_v2/doc_set_test.cpp | 86 ++++++
.../inverted_index/query_v2/query_test.cpp | 183 ------------
.../query_v2/score_combiner_test.cpp | 103 +++++++
.../query_v2/segment_postings_test.cpp | 165 +++++++++++
42 files changed, 1897 insertions(+), 1084 deletions(-)
diff --git a/be/src/olap/collection_statistics.h
b/be/src/olap/collection_statistics.h
index 9b621d08d9b..87a91c755a3 100644
--- a/be/src/olap/collection_statistics.h
+++ b/be/src/olap/collection_statistics.h
@@ -88,6 +88,7 @@ private:
MOCK_DEFINE(friend class BM25SimilarityTest;)
MOCK_DEFINE(friend class CollectionStatisticsTest;)
+ MOCK_DEFINE(friend class BooleanQueryTest;)
};
using CollectionStatisticsPtr = std::shared_ptr<CollectionStatistics>;
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.cpp
deleted file mode 100644
index d503a118834..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.cpp
+++ /dev/null
@@ -1,91 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.h"
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/factory.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/node.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/operator.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/term_query.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-Status BooleanQuery::Builder::set_op(OperatorType type) {
- _op = DORIS_TRY(OperatorFactory::create(type));
- return Status::OK();
-}
-
-Status BooleanQuery::Builder::add(const Node& clause) {
- return visit_node(_op, OpAddChild {}, clause);
-}
-
-Result<Node> BooleanQuery::Builder::build() {
- auto query_ptr = std::make_shared<BooleanQuery>(std::move(_op));
- auto status = visit_node(query_ptr->_op, OpInit {});
- if (!status.ok()) {
- return ResultError(std::move(status));
- }
- return query_ptr;
-}
-
-BooleanQuery::BooleanQuery(Node op) : _op(std::move(op)) {}
-
-void BooleanQuery::execute(const std::shared_ptr<roaring::Roaring>& result) {
- bool use_skip = should_use_skip();
-
- if (use_skip) {
- search_by_skiplist(result);
- } else {
- search_by_bitmap(result);
- }
-}
-
-int32_t BooleanQuery::doc_id() const {
- return visit_node(_op, DocId {});
-}
-
-int32_t BooleanQuery::next_doc() const {
- return visit_node(_op, NextDoc {});
-}
-
-int32_t BooleanQuery::advance(int32_t target) const {
- return visit_node(_op, Advance {}, target);
-}
-
-int64_t BooleanQuery::cost() const {
- return visit_node(_op, Cost {});
-}
-
-bool BooleanQuery::should_use_skip() {
- return visit_node(_op, Cost {});
-}
-
-void BooleanQuery::search_by_skiplist(const std::shared_ptr<roaring::Roaring>&
result) {
- auto _next_doc = [](const auto& node) { return node->next_doc(); };
-
- int32_t doc = 0;
- while ((doc = visit_node(_op, _next_doc)) != INT32_MAX) {
- result->add(doc);
- }
-}
-
-void BooleanQuery::search_by_bitmap(const std::shared_ptr<roaring::Roaring>&
result) {}
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.h
deleted file mode 100644
index c66bf1d3536..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.h
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/node.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/query.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-enum class OperatorType;
-class BooleanQuery : public Query {
-public:
- BooleanQuery(Node op);
- ~BooleanQuery() override = default;
-
- void execute(const std::shared_ptr<roaring::Roaring>& result);
-
- int32_t doc_id() const;
- int32_t next_doc() const;
- int32_t advance(int32_t target) const;
- int64_t cost() const;
-
- class Builder {
- public:
- Status set_op(OperatorType type);
- Status add(const Node& clause);
- Result<Node> build();
-
- private:
- Node _op;
- };
-
-private:
- bool should_use_skip();
- void search_by_skiplist(const std::shared_ptr<roaring::Roaring>& result);
- void search_by_bitmap(const std::shared_ptr<roaring::Roaring>& result);
-
- Node _op;
-};
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query/boolean_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query/boolean_query.h
new file mode 100644
index 00000000000..d663b750cd4
--- /dev/null
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query/boolean_query.h
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include
"olap/rowset/segment_v2/inverted_index/query_v2/boolean_query/boolean_weight.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/operator.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/query.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/score_combiner.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+class BooleanQuery;
+using BooleanQueryPtr = std::shared_ptr<BooleanQuery>;
+
+class BooleanQuery : public Query {
+public:
+ BooleanQuery(OperatorType type, std::vector<QueryPtr> clauses)
+ : _type(type), _sub_queries(std::move(clauses)) {}
+ ~BooleanQuery() override = default;
+
+ WeightPtr weight(bool enable_scoring) override {
+ std::vector<WeightPtr> sub_weights;
+ for (const auto& query : _sub_queries) {
+ sub_weights.emplace_back(query->weight(enable_scoring));
+ }
+ if (enable_scoring) {
+ return std::make_shared<BooleanWeight<SumCombinerPtr>>(_type,
std::move(sub_weights),
+
std::make_shared<SumCombiner>());
+ } else {
+ return std::make_shared<BooleanWeight<DoNothingCombinerPtr>>(
+ _type, std::move(sub_weights),
std::make_shared<DoNothingCombiner>());
+ }
+ }
+
+ class Builder {
+ public:
+ Builder(OperatorType type) : _type(type) {}
+ ~Builder() = default;
+
+ void add(const QueryPtr& query) { _sub_queries.emplace_back(query); }
+
+ BooleanQueryPtr build() {
+ return std::make_shared<BooleanQuery>(_type,
std::move(_sub_queries));
+ }
+
+ private:
+ OperatorType _type;
+ std::vector<QueryPtr> _sub_queries;
+ };
+
+private:
+ OperatorType _type;
+ std::vector<QueryPtr> _sub_queries;
+};
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query/boolean_weight.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query/boolean_weight.h
new file mode 100644
index 00000000000..dd5b65837b8
--- /dev/null
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query/boolean_weight.h
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <vector>
+
+#include
"olap/rowset/segment_v2/inverted_index/query_v2/buffered_union_scorer.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/intersection_scorer.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/operator.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/weight.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+template <typename ScoreCombinerPtrT>
+class BooleanWeight : public Weight {
+public:
+ BooleanWeight(OperatorType type, std::vector<WeightPtr> sub_weights,
+ ScoreCombinerPtrT score_combiner)
+ : _type(type),
+ _sub_weights(std::move(sub_weights)),
+ _score_combiner(std::move(score_combiner)) {}
+ ~BooleanWeight() override = default;
+
+ ScorerPtr scorer(lucene::index::IndexReader* reader) override {
+ std::vector<ScorerPtr> sub_scorers = per_scorers(reader);
+ if (_type == OperatorType::OP_AND) {
+ return intersection_scorer_build(sub_scorers);
+ } else if (_type == OperatorType::OP_OR) {
+ return buffered_union_scorer_build<ScoreCombinerPtrT>(sub_scorers,
_score_combiner);
+ }
+ return nullptr;
+ }
+
+private:
+ std::vector<ScorerPtr> per_scorers(lucene::index::IndexReader* reader) {
+ std::vector<ScorerPtr> sub_scorers;
+ for (const auto& sub_weight : _sub_weights) {
+ sub_scorers.emplace_back(sub_weight->scorer(reader));
+ }
+ return sub_scorers;
+ }
+
+ OperatorType _type;
+ std::vector<WeightPtr> _sub_weights;
+ ScoreCombinerPtrT _score_combiner;
+};
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/buffered_union_scorer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/buffered_union_scorer.cpp
new file mode 100644
index 00000000000..16773f4b31f
--- /dev/null
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/buffered_union_scorer.cpp
@@ -0,0 +1,258 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include
"olap/rowset/segment_v2/inverted_index/query_v2/buffered_union_scorer.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/score_combiner.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/scorer.h"
+#include
"olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_scorer.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+static constexpr size_t HORIZON_NUM_TINYBITSETS = 64;
+static constexpr uint32_t HORIZON = static_cast<uint32_t>(64) *
HORIZON_NUM_TINYBITSETS;
+
+template <typename ScoreCombinerPtrU>
+ScorerPtr buffered_union_scorer_build(const std::vector<ScorerPtr>& scorers,
+ ScoreCombinerPtrU score_combiner) {
+ std::vector<ScorerPtr> non_empty_scorers;
+ for (const auto& docset : scorers) {
+ if (docset && docset->doc() != TERMINATED) {
+ non_empty_scorers.push_back(docset);
+ }
+ }
+ auto bitsets = std::vector<TinySetPtr>(HORIZON_NUM_TINYBITSETS);
+ std::ranges::generate(bitsets, []() { return std::make_shared<TinySet>(0);
});
+ auto scores = std::vector<ScoreCombinerPtrU>(HORIZON);
+ std::ranges::generate(scores, [&score_combiner]() { return
score_combiner->clone(); });
+
+ auto try_create_term_union = [&]<typename ScorerPtrT>() -> ScorerPtr {
+ std::vector<ScorerPtrT> term_scorers;
+ term_scorers.reserve(non_empty_scorers.size());
+
+ for (const auto& scorer : non_empty_scorers) {
+ if (auto term_scorer =
+ std::dynamic_pointer_cast<typename
ScorerPtrT::element_type>(scorer)) {
+ term_scorers.push_back(term_scorer);
+ } else {
+ return nullptr;
+ }
+ }
+
+ auto union_scorer = std::make_shared<BufferedUnionScorer<ScorerPtrT,
ScoreCombinerPtrU>>(
+ std::move(term_scorers), bitsets, scores,
HORIZON_NUM_TINYBITSETS, 0, 0);
+
+ if (union_scorer->refill()) {
+ union_scorer->advance();
+ } else {
+ union_scorer->_doc = TERMINATED;
+ }
+
+ return union_scorer;
+ };
+
+ if (auto result = try_create_term_union.template operator()<TS_Base>()) {
+ return result;
+ }
+ if (auto result = try_create_term_union.template operator()<TS_NoScore>())
{
+ return result;
+ }
+ if (auto result = try_create_term_union.template operator()<TS_Empty>()) {
+ return result;
+ }
+
+ auto union_scorer = std::make_shared<BufferedUnionScorer<ScorerPtr,
ScoreCombinerPtrU>>(
+ std::move(non_empty_scorers), std::move(bitsets),
std::move(scores),
+ HORIZON_NUM_TINYBITSETS, 0, 0);
+
+ if (union_scorer->refill()) {
+ union_scorer->advance();
+ } else {
+ union_scorer->_doc = TERMINATED;
+ }
+
+ return union_scorer;
+}
+
+template <typename T, typename Predicate>
+void unordered_drain_filter(std::vector<T>& v, Predicate predicate) {
+ size_t i = 0;
+ while (i < v.size()) {
+ if (predicate(v[i])) {
+ if (i < v.size() - 1) {
+ std::swap(v[i], v.back());
+ }
+ v.pop_back();
+ } else {
+ i++;
+ }
+ }
+}
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+BufferedUnionScorer<ScorerPtrT, ScoreCombinerPtrT>::BufferedUnionScorer(
+ std::vector<ScorerPtrT> scorers, std::vector<TinySetPtr> bitsets,
+ std::vector<ScoreCombinerPtrT> scores, size_t cursor, uint32_t offset,
uint32_t _doc)
+ : _scorers(std::move(scorers)), _bitsets(std::move(bitsets)),
_scores(std::move(scores)) {}
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+bool BufferedUnionScorer<ScorerPtrT, ScoreCombinerPtrT>::refill() {
+ if (_scorers.empty()) {
+ return false;
+ }
+ uint32_t min_doc = TERMINATED;
+ for (const auto& ds : _scorers) {
+ min_doc = std::min(min_doc, ds->doc());
+ }
+ if (min_doc == TERMINATED) {
+ return false;
+ }
+ _offset = min_doc;
+ _cursor = 0;
+ _doc = min_doc;
+ refill(_scorers, _bitsets, _scores, min_doc);
+ return true;
+}
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+void BufferedUnionScorer<ScorerPtrT, ScoreCombinerPtrT>::refill(
+ std::vector<ScorerPtrT>& scorers, const std::vector<TinySetPtr>&
bitsets,
+ std::vector<ScoreCombinerPtrT>& scores, uint32_t min_doc) {
+ unordered_drain_filter(scorers, [&](const ScorerPtrT& scorer) -> bool {
+ uint32_t horizon = min_doc + HORIZON;
+ while (true) {
+ uint32_t _doc = scorer->doc();
+ if (_doc >= horizon) {
+ return false;
+ }
+ uint32_t delta = _doc - min_doc;
+ bitsets[static_cast<size_t>(delta / 64)]->insert_mut(delta % 64);
+ if constexpr (!std::is_same_v<ScoreCombinerPtrT,
DoNothingCombinerPtr>) {
+ scores[static_cast<size_t>(delta)]->update(scorer);
+ }
+ if (scorer->advance() == TERMINATED) {
+ return true;
+ }
+ }
+ });
+}
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+bool BufferedUnionScorer<ScorerPtrT, ScoreCombinerPtrT>::advance_buffered() {
+ while (_cursor < HORIZON_NUM_TINYBITSETS) {
+ auto val_opt = _bitsets[_cursor]->pop_lowest();
+ if (val_opt.has_value()) {
+ uint32_t val = val_opt.value();
+ uint32_t delta = val + (static_cast<uint32_t>(_cursor) * 64);
+ _doc = _offset + delta;
+ if constexpr (!std::is_same_v<ScoreCombinerPtrT,
DoNothingCombinerPtr>) {
+ auto score_combiner = _scores[static_cast<size_t>(delta)];
+ _score = score_combiner->score();
+ score_combiner->clear();
+ }
+ return true;
+ } else {
+ _cursor++;
+ }
+ }
+ return false;
+}
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+uint32_t BufferedUnionScorer<ScorerPtrT, ScoreCombinerPtrT>::advance() {
+ if (advance_buffered()) {
+ return _doc;
+ }
+ if (!refill()) {
+ _doc = TERMINATED;
+ return TERMINATED;
+ }
+ if (!advance_buffered()) {
+ return TERMINATED;
+ }
+ return _doc;
+}
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+uint32_t BufferedUnionScorer<ScorerPtrT, ScoreCombinerPtrT>::seek(uint32_t
target) {
+ if (_doc >= target) {
+ return _doc;
+ }
+ uint32_t gap = target - _offset;
+ if (gap < HORIZON) {
+ size_t new_cursor = static_cast<size_t>(gap) / 64;
+ for (size_t i = _cursor; i < new_cursor; ++i) {
+ _bitsets[i]->clear();
+ }
+ for (size_t i = _cursor * 64; i < new_cursor * 64; ++i) {
+ _scores[i]->clear();
+ }
+ _cursor = new_cursor;
+ uint32_t current_doc = _doc;
+ while (current_doc < target) {
+ current_doc = advance();
+ }
+ return current_doc;
+ } else {
+ for (auto& tinyset : _bitsets) {
+ tinyset = std::make_shared<TinySet>(0);
+ }
+ for (auto& score_combiner : _scores) {
+ score_combiner->clear();
+ }
+ unordered_drain_filter(_scorers, [target](auto& docset) {
+ if (docset->doc() < target) {
+ docset->seek(target);
+ }
+ return docset->doc() == TERMINATED;
+ });
+ if (!refill()) {
+ _doc = TERMINATED;
+ return TERMINATED;
+ }
+ return advance();
+ }
+}
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+uint32_t BufferedUnionScorer<ScorerPtrT, ScoreCombinerPtrT>::doc() const {
+ return _doc;
+}
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+uint32_t BufferedUnionScorer<ScorerPtrT, ScoreCombinerPtrT>::size_hint() const
{
+ uint32_t max_hint = 0;
+ for (const auto& docset : _scorers) {
+ max_hint = std::max(max_hint, docset->size_hint());
+ }
+ return max_hint;
+}
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+float BufferedUnionScorer<ScorerPtrT, ScoreCombinerPtrT>::score() {
+ return _score;
+}
+
+template ScorerPtr buffered_union_scorer_build(const std::vector<ScorerPtr>&
scorers,
+ SumCombinerPtr score_combiner);
+template ScorerPtr buffered_union_scorer_build(const std::vector<ScorerPtr>&
scorers,
+ DoNothingCombinerPtr
score_combiner);
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/buffered_union_scorer.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/buffered_union_scorer.h
new file mode 100644
index 00000000000..27b89f00c7a
--- /dev/null
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/buffered_union_scorer.h
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <vector>
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/scorer.h"
+#include "olap/rowset/segment_v2/inverted_index/util/tiny_set.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+template <typename ScoreCombinerPtrU>
+ScorerPtr buffered_union_scorer_build(const std::vector<ScorerPtr>& scorers,
+ ScoreCombinerPtrU score_combiner);
+
+template <typename ScorerPtrT, typename ScoreCombinerPtrT>
+class BufferedUnionScorer : public Scorer {
+public:
+ BufferedUnionScorer(std::vector<ScorerPtrT> scorers,
std::vector<TinySetPtr> bitsets,
+ std::vector<ScoreCombinerPtrT> scores, size_t cursor,
uint32_t offset,
+ uint32_t doc);
+ ~BufferedUnionScorer() override = default;
+
+ uint32_t advance() override;
+ uint32_t seek(uint32_t target) override;
+ uint32_t doc() const override;
+ uint32_t size_hint() const override;
+
+ float score() override;
+
+ template <typename ScoreCombinerPtrU>
+ friend ScorerPtr buffered_union_scorer_build(const std::vector<ScorerPtr>&
scorers,
+ ScoreCombinerPtrU
score_combiner);
+
+private:
+ bool refill();
+ void refill(std::vector<ScorerPtrT>& scorers, const
std::vector<TinySetPtr>& bitsets,
+ std::vector<ScoreCombinerPtrT>& scores, uint32_t min_doc);
+ bool advance_buffered();
+
+ std::vector<ScorerPtrT> _scorers;
+ std::vector<TinySetPtr> _bitsets;
+ std::vector<ScoreCombinerPtrT> _scores;
+ size_t _cursor = 0;
+ uint32_t _offset = 0;
+ uint32_t _doc = 0;
+ float _score = 0.0F;
+};
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.cpp
deleted file mode 100644
index 4f32e1d8445..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.cpp
+++ /dev/null
@@ -1,94 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.h"
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/term_query.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-Status ConjunctionOp::init() {
- if (_childrens.size() < 2) {
- return Status::InternalError("_childrens must contain more than 2
elements.");
- }
-
- std::sort(_childrens.begin(), _childrens.end(), [](const Node& a, const
Node& b) {
- return visit_node(a, Cost {}) < visit_node(b, Cost {});
- });
-
- _lead1 = _childrens.data();
- _lead2 = _childrens.data() + 1;
-
- for (int32_t i = 2; i < _childrens.size(); i++) {
- _others.push_back(_childrens.data() + i);
- }
-
- return Status::OK();
-}
-
-int32_t ConjunctionOp::doc_id() const {
- return visit_node(*_lead1, DocId {});
-}
-
-int32_t ConjunctionOp::next_doc() const {
- return do_next(visit_node(*_lead1, NextDoc {}));
-}
-
-int32_t ConjunctionOp::advance(int32_t target) const {
- return do_next(visit_node(*_lead1, Advance {}, target));
-}
-
-int32_t ConjunctionOp::do_next(int32_t doc) const {
- while (true) {
- assert(doc == visit_node(*_lead1, DocId {}));
-
- int32_t next2 = visit_node(*_lead2, Advance {}, doc);
- if (next2 != doc) {
- doc = visit_node(*_lead1, Advance {}, next2);
- if (next2 != doc) {
- continue;
- }
- }
-
- bool advance_head = false;
- for (const auto& other : _others) {
- int32_t other_doc_id = visit_node(*other, DocId {});
- if (other_doc_id < doc) {
- int32_t next = visit_node(*other, Advance {}, doc);
- if (next > doc) {
- doc = visit_node(*_lead1, Advance {}, next);
- advance_head = true;
- break;
- }
- }
- }
- if (advance_head) {
- continue;
- }
-
- return doc;
- }
-}
-
-int64_t ConjunctionOp::cost() const {
- return visit_node(*_lead1, Cost {});
-}
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.cpp
deleted file mode 100644
index e4f196146f9..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.h"
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/node.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/term_query.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-DisjunctionOp::~DisjunctionOp() {
- while (!_pq.empty()) {
- auto* w = _pq.top();
- if (w != nullptr) {
- delete w;
- w = nullptr;
- }
- _pq.pop();
- }
-}
-
-Status DisjunctionOp::init() {
- if (_childrens.size() < 2) {
- return Status::InternalError("_childrens must contain more than 2
elements.");
- }
-
- for (const auto& child : _childrens) {
- auto* w = new DisiWrapper();
- w->_iter = &child;
- w->_cost = visit_node(*w->_iter, Cost {});
- this->_cost += w->_cost;
- _pq.push(w);
- }
-
- return Status::OK();
-}
-
-int32_t DisjunctionOp::doc_id() const {
- return visit_node(*_pq.top()->_iter, DocId {});
-}
-
-int32_t DisjunctionOp::next_doc() const {
- auto* top = _pq.top();
- int32_t doc = top->_doc;
- do {
- _pq.pop();
- top->_doc = visit_node(*top->_iter, NextDoc {});
- _pq.push(top);
- top = _pq.top();
- } while (top->_doc == doc);
- return top->_doc;
-}
-
-int32_t DisjunctionOp::advance(int32_t target) const {
- auto* top = _pq.top();
- do {
- _pq.pop();
- top->_doc = visit_node(*top->_iter, Advance {}, target);
- _pq.push(top);
- top = _pq.top();
- } while (top->_doc < target);
- return top->_doc;
-}
-
-int64_t DisjunctionOp::cost() const {
- return _cost;
-}
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.h
deleted file mode 100644
index 7abbef45efa..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.h
+++ /dev/null
@@ -1,59 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <queue>
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/operator.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-class DisjunctionOp : public Operator {
-public:
- DisjunctionOp() = default;
- ~DisjunctionOp() override;
-
- Status init();
-
- int32_t doc_id() const;
- int32_t next_doc() const;
- int32_t advance(int32_t target) const;
- int64_t cost() const;
-
-private:
- class DisiWrapper {
- public:
- DisiWrapper() = default;
- ~DisiWrapper() = default;
-
- int32_t _doc = -1;
- int64_t _cost = 0;
- const Node* _iter = nullptr;
- };
-
- struct CompareDisiWrapper {
- bool operator()(DisiWrapper* lhs, DisiWrapper* rhs) const { return
lhs->_doc > rhs->_doc; }
- };
-
- int64_t _cost = 0;
- mutable std::priority_queue<DisiWrapper*, std::vector<DisiWrapper*>,
CompareDisiWrapper> _pq;
-};
-
-using DisjunctionOpPtr = std::shared_ptr<DisjunctionOp>;
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/doc_set.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/doc_set.h
new file mode 100644
index 00000000000..120ba74e448
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/doc_set.h
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <climits>
+#include <cstdint>
+
+#include "common/exception.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+static constexpr uint32_t TERMINATED = static_cast<uint32_t>(INT_MAX);
+
+class DocSet {
+public:
+ DocSet() = default;
+ virtual ~DocSet() = default;
+
+ virtual uint32_t advance() {
+ throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR,
+ "advance() method not implemented in base
DocSet class");
+ }
+
+ virtual uint32_t seek(uint32_t target) {
+ throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR,
+ "seek() method not implemented in base DocSet
class");
+ }
+
+ virtual uint32_t doc() const {
+ throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR,
+ "doc() method not implemented in base DocSet
class");
+ }
+
+ virtual uint32_t size_hint() const {
+ throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR,
+ "size_hint() method not implemented in base
DocSet class");
+ }
+};
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/factory.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/factory.cpp
deleted file mode 100644
index 710b4b7ed54..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/factory.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/factory.h"
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/disjunction_op.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-Result<Node> OperatorFactory::create(OperatorType query_type) {
- switch (query_type) {
- case OperatorType::OP_AND:
- return std::make_shared<ConjunctionOp>();
- case OperatorType::OP_OR:
- return std::make_shared<DisjunctionOp>();
- default:
- return ResultError(Status::InternalError("failed create operator: {}",
query_type));
- }
-}
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/factory.inline.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/factory.inline.h
deleted file mode 100644
index fff60cc0626..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/factory.inline.h
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "factory.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.h"
-#include "olap/rowset/segment_v2/inverted_index/query_v2/term_query.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-template <typename... Args>
-Result<Node> QueryFactory::create(QueryType query_type, Args&&... args) {
- if constexpr (sizeof...(args) == 1) {
- switch (query_type) {
- case QueryType::ROARING_QUERY:
- return std::make_shared<RoaringQuery>(std::forward<Args>(args)...);
- default:
- return ResultError(Status::InternalError("failed create query:
{}", query_type));
- }
- } else if (sizeof...(args) == 3) {
- switch (query_type) {
- case QueryType::TERM_QUERY:
- return std::make_shared<TermQuery>(std::forward<Args>(args)...);
- case QueryType::PHRASE_QUERY:
- // return
std::make_shared<PhraseQuery>(std::forward<Args>(args)...);
- default:
- return ResultError(Status::InternalError("failed create query:
{}", query_type));
- }
- } else {
- return ResultError(Status::InternalError("Invalid arguments"));
- }
-}
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/intersection_scorer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/intersection_scorer.cpp
new file mode 100644
index 00000000000..f5e75a1c77b
--- /dev/null
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/intersection_scorer.cpp
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/intersection_scorer.h"
+
+#include <algorithm>
+#include <cassert>
+
+#include
"olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_scorer.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+uint32_t go_to_first_doc(const std::vector<ScorerPtr>& docsets) {
+ assert(!docsets.empty());
+
+ uint32_t candidate = docsets[0]->doc();
+ for (size_t i = 1; i < docsets.size(); ++i) {
+ candidate = std::max(candidate, docsets[i]->doc());
+ }
+
+ while (true) {
+ bool need_continue = false;
+
+ for (const auto& docset : docsets) {
+ uint32_t seek_doc = docset->seek(candidate);
+ if (seek_doc > candidate) {
+ candidate = docset->doc();
+ need_continue = true;
+ break;
+ }
+ }
+
+ if (!need_continue) {
+ return candidate;
+ }
+ }
+}
+
+ScorerPtr intersection_scorer_build(std::vector<ScorerPtr>& scorers) {
+ if (scorers.empty()) {
+ return std::make_shared<EmptyScorer>();
+ } else if (scorers.size() == 1) {
+ return scorers[0];
+ }
+ std::ranges::sort(scorers, [](const ScorerPtr& a, const ScorerPtr& b) {
+ return a->size_hint() < b->size_hint();
+ });
+ uint32_t doc = go_to_first_doc(scorers);
+ if (doc == TERMINATED) {
+ return std::make_shared<EmptyScorer>();
+ }
+ auto left = scorers[0];
+ auto right = scorers[1];
+ std::vector<ScorerPtr> others(scorers.begin() + 2, scorers.end());
+
+ auto try_create_term_intersection = [&]<typename TPtr>() -> ScorerPtr {
+ if (auto l = std::dynamic_pointer_cast<typename
TPtr::element_type>(left)) {
+ if (auto r = std::dynamic_pointer_cast<typename
TPtr::element_type>(right)) {
+ return std::make_shared<IntersectionScorer<TPtr>>(l, r,
others);
+ }
+ }
+ return nullptr;
+ };
+
+ if (auto result = try_create_term_intersection.template
operator()<TS_Base>()) {
+ return result;
+ }
+ if (auto result = try_create_term_intersection.template
operator()<TS_NoScore>()) {
+ return result;
+ }
+ if (auto result = try_create_term_intersection.template
operator()<TS_Empty>()) {
+ return result;
+ }
+
+ return std::make_shared<IntersectionScorer<ScorerPtr>>(std::move(left),
std::move(right),
+ std::move(others));
+}
+
+template <typename PivotScorerPtr>
+IntersectionScorer<PivotScorerPtr>::IntersectionScorer(PivotScorerPtr left,
PivotScorerPtr right,
+ std::vector<ScorerPtr>
others)
+ : _left(std::move(left)), _right(std::move(right)),
_others(std::move(others)) {}
+
+template <typename PivotScorerPtr>
+uint32_t IntersectionScorer<PivotScorerPtr>::advance() {
+ uint32_t candidate = _left->advance();
+
+ while (true) {
+ while (true) {
+ uint32_t right_doc = _right->seek(candidate);
+ candidate = _left->seek(right_doc);
+ if (candidate == right_doc) {
+ break;
+ }
+ }
+
+ bool need_continue = false;
+ for (const auto& docset : _others) {
+ uint32_t seek_doc = docset->seek(candidate);
+ if (seek_doc > candidate) {
+ candidate = _left->seek(seek_doc);
+ need_continue = true;
+ break;
+ }
+ }
+
+ if (!need_continue) {
+ return candidate;
+ }
+ }
+}
+
+template <typename PivotScorerPtr>
+uint32_t IntersectionScorer<PivotScorerPtr>::seek(uint32_t target) {
+ _left->seek(target);
+ std::vector<ScorerPtr> docsets;
+ docsets.push_back(_left);
+ docsets.push_back(_right);
+ for (auto& docset : _others) {
+ docsets.push_back(docset);
+ }
+ return go_to_first_doc(docsets);
+}
+
+template <typename PivotScorerPtr>
+uint32_t IntersectionScorer<PivotScorerPtr>::doc() const {
+ return _left->doc();
+}
+
+template <typename PivotScorerPtr>
+uint32_t IntersectionScorer<PivotScorerPtr>::size_hint() const {
+ return _left->size_hint();
+}
+
+template <typename PivotScorerPtr>
+float IntersectionScorer<PivotScorerPtr>::score() {
+ float total = _left->score() + _right->score();
+ for (auto& scorer : _others) {
+ total += scorer->score();
+ }
+ return total;
+}
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/intersection_scorer.h
similarity index 52%
rename from
be/src/olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.h
rename to
be/src/olap/rowset/segment_v2/inverted_index/query_v2/intersection_scorer.h
index 37f0eea0a63..45b40ad9c33 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/conjunction_op.h
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/intersection_scorer.h
@@ -17,30 +17,31 @@
#pragma once
-#include "olap/rowset/segment_v2/inverted_index/query_v2/operator.h"
+#include <vector>
-namespace doris::segment_v2::idx_query_v2 {
+#include "olap/rowset/segment_v2/inverted_index/query_v2/scorer.h"
-class ConjunctionOp : public Operator {
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+ScorerPtr intersection_scorer_build(std::vector<ScorerPtr>& scorers);
+
+template <typename PivotScorerPtr>
+class IntersectionScorer final : public Scorer {
public:
- ConjunctionOp() = default;
- ~ConjunctionOp() override = default;
+ IntersectionScorer(PivotScorerPtr left, PivotScorerPtr right,
std::vector<ScorerPtr> others);
+ ~IntersectionScorer() override = default;
- Status init();
+ uint32_t advance() override;
+ uint32_t seek(uint32_t target) override;
+ uint32_t doc() const override;
+ uint32_t size_hint() const override;
- int32_t doc_id() const;
- int32_t next_doc() const;
- int32_t advance(int32_t target) const;
- int64_t cost() const;
+ float score() override;
private:
- int32_t do_next(int32_t doc) const;
-
- const Node* _lead1 = nullptr;
- const Node* _lead2 = nullptr;
- std::vector<const Node*> _others;
+ PivotScorerPtr _left;
+ PivotScorerPtr _right;
+ std::vector<ScorerPtr> _others;
};
-using ConjunctionOpPtr = std::shared_ptr<ConjunctionOp>;
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/node.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/node.h
deleted file mode 100644
index 2e70f50548a..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/node.h
+++ /dev/null
@@ -1,128 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <cstdint>
-#include <memory>
-#include <roaring/roaring.hh>
-#include <variant>
-
-#include "common/status.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-class ConjunctionOp;
-class DisjunctionOp;
-class BooleanQuery;
-class TermQuery;
-class RoaringQuery;
-
-using ConjunctionOpPtr = std::shared_ptr<ConjunctionOp>;
-using DisjunctionOpPtr = std::shared_ptr<DisjunctionOp>;
-using TermQueryPtr = std::shared_ptr<TermQuery>;
-using RoaringQueryPtr = std::shared_ptr<RoaringQuery>;
-using BooleanQueryPtr = std::shared_ptr<BooleanQuery>;
-
-using Node = std::variant<ConjunctionOpPtr, DisjunctionOpPtr, BooleanQueryPtr,
TermQueryPtr,
- RoaringQueryPtr>;
-
-template <typename T>
-constexpr bool always_false = false;
-
-template <typename NodeType, typename Func, typename... Args>
-auto visit_node(NodeType&& node, Func&& func, Args&&... args) {
- using DecayedNodeType = std::remove_const_t<std::decay_t<NodeType>>;
- using NodeBaseType = std::remove_pointer_t<DecayedNodeType>;
-
- static_assert(!std::is_pointer_v<DecayedNodeType>, "NodeType must not be a
pointer");
- static_assert(std::is_same_v<std::remove_const_t<NodeBaseType>, Node>,
"NodeType must be Node");
-
- return std::visit(
- [&](const auto& opOrClause) {
- return std::forward<Func>(func)(opOrClause,
std::forward<Args>(args)...);
- },
- node);
-}
-
-template <typename T>
-concept IsOpPtr = std::is_same_v<T, ConjunctionOpPtr> || std::is_same_v<T,
DisjunctionOpPtr>;
-
-template <typename T>
-concept IsQueryPtr = std::is_same_v<T, BooleanQueryPtr> || std::is_same_v<T,
TermQueryPtr> ||
- std::is_same_v<T, RoaringQueryPtr>;
-
-struct OpAddChild {
- template <typename T>
- Status operator()(const T& op, const Node& clause) const {
- if constexpr (IsOpPtr<T>) {
- return op->add_child(clause);
- } else {
- return Status::OK();
- }
- }
-};
-
-struct OpInit {
- template <typename T>
- Status operator()(const T& op) const {
- if constexpr (IsOpPtr<T>) {
- return op->init();
- } else {
- return Status::OK();
- }
- }
-};
-
-struct QueryExecute {
- template <typename T>
- void operator()(const T& query, const std::shared_ptr<roaring::Roaring>&
result) const {
- if constexpr (IsQueryPtr<T>) {
- return query->execute(result);
- }
- }
-};
-
-struct DocId {
- template <typename T>
- int32_t operator()(const T& node) const {
- return node->doc_id();
- }
-};
-
-struct NextDoc {
- template <typename T>
- int32_t operator()(const T& node) const {
- return node->next_doc();
- }
-};
-
-struct Advance {
- template <typename T>
- int32_t operator()(const T& node, int32_t target) const {
- return node->advance(target);
- }
-};
-
-struct Cost {
- template <typename T>
- int64_t operator()(const T& node) const {
- return node->cost();
- }
-};
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/operator.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/operator.h
index 47eeaf3e1d7..a7fa382e612 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/operator.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/operator.h
@@ -17,29 +17,8 @@
#pragma once
-#include <vector>
+namespace doris::segment_v2::inverted_index::query_v2 {
-#include "olap/rowset/segment_v2/inverted_index/query_v2/node.h"
+enum class OperatorType { OP_AND = 0, OP_OR = 1, OP_NOT = 2 };
-namespace doris::segment_v2::idx_query_v2 {
-
-enum class OperatorType { OP_AND = 0, OP_OR };
-
-class Operator;
-using OperatorPtr = std::shared_ptr<Operator>;
-
-class Operator {
-public:
- Operator() = default;
- virtual ~Operator() = default;
-
- Status add_child(const Node& clause) {
- _childrens.emplace_back(clause);
- return Status::OK();
- }
-
-protected:
- std::vector<Node> _childrens;
-};
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/phrase_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/phrase_query.h
deleted file mode 100644
index fc9e54e498b..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/phrase_query.h
+++ /dev/null
@@ -1,38 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/query.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-class PhraseQuery : public Query {
-public:
- PhraseQuery(const std::shared_ptr<lucene::search::IndexSearcher>& searcher,
- const TQueryOptions& query_options, QueryInfo query_info) {}
- ~PhraseQuery() override = default;
-
- void execute(const std::shared_ptr<roaring::Roaring>& result) {}
-
- int32_t doc_id() const { return -1; }
- int32_t next_doc() const { return -1; }
- int32_t advance(int32_t target) const { return -1; }
- int64_t cost() const { return -1; }
-};
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/query.h
index bea8e7801c8..f902b5e7fd4 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/query.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/query.h
@@ -17,36 +17,17 @@
#pragma once
-#include <CLucene.h> // IWYU pragma: keep
-#include <CLucene/index/IndexReader.h>
-#include <CLucene/index/Term.h>
-#include <gen_cpp/PaloInternalService_types.h>
+#include "olap/rowset/segment_v2/inverted_index/query_v2/weight.h"
-#include <memory>
-#include <roaring/roaring.hh>
-#include <variant>
-
-#include "common/status.h"
-#include "olap/rowset/segment_v2/inverted_index/util/term_iterator.h"
-#include "olap/rowset/segment_v2/inverted_index/util/term_position_iterator.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-enum class QueryType { TERM_QUERY, PHRASE_QUERY, ROARING_QUERY };
-
-struct QueryInfo {
- std::wstring field_name;
- std::vector<std::string> terms;
- int32_t slop = 0;
-};
-
-class Query;
-using QueryPtr = std::shared_ptr<Query>;
+namespace doris::segment_v2::inverted_index::query_v2 {
class Query {
public:
Query() = default;
virtual ~Query() = default;
+
+ virtual WeightPtr weight(bool enable_scoring) = 0;
};
+using QueryPtr = std::shared_ptr<Query>;
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.cpp
deleted file mode 100644
index 48f53901559..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-RoaringQuery::RoaringQuery(const std::shared_ptr<roaring::Roaring>& roaring)
- : _roaring(roaring), _iter(_roaring->end()), _end(_roaring->end()) {}
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.h
deleted file mode 100644
index ad838512c97..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/roaring_query.h
+++ /dev/null
@@ -1,62 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <cstdint>
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/query.h"
-
-namespace doris::segment_v2::idx_query_v2 {
-
-class RoaringQuery : public Query {
-public:
- RoaringQuery(const std::shared_ptr<roaring::Roaring>& roaring);
- ~RoaringQuery() override = default;
-
- void execute(const std::shared_ptr<roaring::Roaring>& result) {}
-
- int32_t doc_id() const { return _doc; }
-
- int32_t next_doc() const {
- if (_iter == _end) {
- _iter = _roaring->begin();
- } else {
- ++_iter;
- }
- _doc = (_iter != _end) ? *_iter : INT_MAX;
- return _doc;
- }
-
- int32_t advance(int32_t target) const {
- while (_iter != _end && *_iter < target) {
- ++_iter;
- }
- _doc = (_iter != _end) ? *_iter : INT_MAX;
- return _doc;
- }
-
- int64_t cost() const { return _roaring->cardinality(); }
-
-private:
- mutable int32_t _doc = -1;
- std::shared_ptr<roaring::Roaring> _roaring;
- mutable roaring::Roaring::const_iterator _iter;
- roaring::Roaring::const_iterator _end;
-};
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/score_combiner.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/score_combiner.h
new file mode 100644
index 00000000000..e4ec4db8d13
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/score_combiner.h
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/scorer.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+class ScoreCombiner;
+using ScoreCombinerPtr = std::shared_ptr<ScoreCombiner>;
+
+class ScoreCombiner {
+public:
+ ScoreCombiner() = default;
+ virtual ~ScoreCombiner() = default;
+
+ virtual void update(const ScorerPtr& scorer) = 0;
+ virtual void clear() = 0;
+ virtual float score() const = 0;
+};
+
+class SumCombiner;
+using SumCombinerPtr = std::shared_ptr<SumCombiner>;
+
+class SumCombiner final : public ScoreCombiner {
+public:
+ SumCombiner() = default;
+ ~SumCombiner() override = default;
+
+ void update(const ScorerPtr& scorer) override { _score += scorer->score();
}
+ void clear() override { _score = 0; }
+ float score() const override { return _score; }
+
+ SumCombinerPtr clone() { return std::make_shared<SumCombiner>(); }
+
+private:
+ float _score = 0.0F;
+};
+
+class DoNothingCombiner;
+using DoNothingCombinerPtr = std::shared_ptr<DoNothingCombiner>;
+
+class DoNothingCombiner final : public ScoreCombiner {
+public:
+ DoNothingCombiner() = default;
+ ~DoNothingCombiner() override = default;
+
+ void update(const ScorerPtr& scorer) override {}
+ void clear() override {}
+ float score() const override { return 0.0F; }
+
+ DoNothingCombinerPtr clone() { return
std::make_shared<DoNothingCombiner>(); }
+};
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/scorer.h
similarity index 53%
rename from be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query.h
rename to be/src/olap/rowset/segment_v2/inverted_index/query_v2/scorer.h
index 115f321dd9d..397c967bbf9 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/scorer.h
@@ -19,26 +19,30 @@
#include <memory>
-#include "olap/rowset/segment_v2/inverted_index/query_v2/query.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/doc_set.h"
-namespace doris::segment_v2::idx_query_v2 {
+namespace doris::segment_v2::inverted_index::query_v2 {
-class TermQuery : public Query {
+class Scorer : public DocSet {
public:
- TermQuery(const std::shared_ptr<lucene::search::IndexSearcher>& searcher,
- const TQueryOptions& query_options, QueryInfo query_info);
- ~TermQuery() override;
+ Scorer() = default;
+ ~Scorer() override = default;
- void execute(const std::shared_ptr<roaring::Roaring>& result) {}
+ virtual float score() = 0;
+};
+using ScorerPtr = std::shared_ptr<Scorer>;
+
+class EmptyScorer : public Scorer {
+public:
+ EmptyScorer() = default;
+ ~EmptyScorer() override = default;
- int32_t doc_id() const { return _iter->doc_id(); }
- int32_t next_doc() const { return _iter->next_doc(); }
- int32_t advance(int32_t target) const { return _iter->advance(target); }
- int64_t cost() const { return _iter->doc_freq(); }
+ uint32_t advance() override { return TERMINATED; }
+ uint32_t seek(uint32_t) override { return TERMINATED; }
+ uint32_t doc() const override { return TERMINATED; }
+ uint32_t size_hint() const override { return 0; }
-private:
- TermDocs* _term_docs = nullptr;
- TermIterPtr _iter;
+ float score() override { return 0.0F; }
};
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/segment_postings.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/segment_postings.h
new file mode 100644
index 00000000000..3f1bc133c16
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/segment_postings.h
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/doc_set.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+template <typename TermIterator>
+class SegmentPostingsBase : public DocSet {
+public:
+ SegmentPostingsBase() = default;
+ SegmentPostingsBase(TermIterator iter) : _iter(std::move(iter)) {
+ if (_iter->next()) {
+ int32_t d = _iter->doc();
+ _doc = d >= INT_MAX ? TERMINATED : d;
+ } else {
+ _doc = TERMINATED;
+ }
+ }
+
+ uint32_t advance() override {
+ if (_iter->next()) {
+ return _doc = _iter->doc();
+ }
+ return _doc = TERMINATED;
+ }
+
+ uint32_t seek(uint32_t target) override {
+ if (target <= _doc) {
+ return _doc;
+ }
+ if (_iter->skipTo(target)) {
+ return _doc = _iter->doc();
+ }
+ return _doc = TERMINATED;
+ }
+
+ uint32_t doc() const override { return _doc; }
+
+ uint32_t size_hint() const override { return _iter->docFreq(); }
+
+ virtual int32_t freq() const {
+ throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR,
+ "freq() method not implemented in base
SegmentPostingsBase class");
+ }
+
+ virtual int32_t norm() const {
+ throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR,
+ "norm() method not implemented in base
SegmentPostingsBase class");
+ }
+
+protected:
+ TermIterator _iter;
+
+private:
+ uint32_t _doc = TERMINATED;
+};
+
+template <typename TermIterator>
+class SegmentPostings final : public SegmentPostingsBase<TermIterator> {
+public:
+ SegmentPostings(TermIterator iter) :
SegmentPostingsBase<TermIterator>(std::move(iter)) {}
+
+ int32_t freq() const override { return this->_iter->freq(); }
+ int32_t norm() const override { return this->_iter->norm(); }
+};
+
+template <typename TermIterator>
+class NoScoreSegmentPosting final : public SegmentPostingsBase<TermIterator> {
+public:
+ NoScoreSegmentPosting(TermIterator iter) :
SegmentPostingsBase<TermIterator>(std::move(iter)) {}
+
+ int32_t freq() const override { return 1; }
+ int32_t norm() const override { return 1; }
+};
+
+template <typename TermIterator>
+class EmptySegmentPosting final : public SegmentPostingsBase<TermIterator> {
+public:
+ EmptySegmentPosting() = default;
+
+ uint32_t advance() override { return TERMINATED; }
+ uint32_t seek(uint32_t) override { return TERMINATED; }
+ uint32_t doc() const override { return TERMINATED; }
+ uint32_t size_hint() const override { return 0; }
+
+ int32_t freq() const override { return 1; }
+ int32_t norm() const override { return 1; }
+};
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query.cpp
deleted file mode 100644
index ed12a05e9aa..00000000000
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/rowset/segment_v2/inverted_index/query_v2/term_query.h"
-
-#include <memory>
-
-namespace doris::segment_v2::idx_query_v2 {
-
-TermQuery::~TermQuery() {
- if (_term_docs) {
- _CLDELETE(_term_docs);
- }
-}
-
-TermQuery::TermQuery(const std::shared_ptr<lucene::search::IndexSearcher>&
searcher,
- const TQueryOptions& query_options, QueryInfo query_info)
{
- _iter = TermIterator::create(nullptr, false, searcher->getReader(),
query_info.field_name,
- query_info.terms[0]);
-}
-
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_query.h
new file mode 100644
index 00000000000..b1c57c125f6
--- /dev/null
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_query.h
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/query.h"
+#include
"olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_weight.h"
+#include "olap/rowset/segment_v2/inverted_index/similarity/bm25_similarity.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+class TermQuery : public Query {
+public:
+ TermQuery(IndexQueryContextPtr context, std::wstring field, std::wstring
term)
+ : _context(std::move(context)), _field(std::move(field)),
_term(std::move(term)) {}
+ ~TermQuery() override = default;
+
+ WeightPtr weight(bool enable_scoring) override {
+ SimilarityPtr bm25_similarity;
+ if (enable_scoring) {
+ bm25_similarity = std::make_shared<BM25Similarity>();
+ bm25_similarity->for_one_term(_context, _field, _term);
+ } else {
+ bm25_similarity = std::make_shared<BM25Similarity>(1.0F, 1.0F);
+ }
+ return std::make_shared<TermWeight>(std::move(_context),
std::move(_field),
+ std::move(_term),
std::move(bm25_similarity),
+ enable_scoring);
+ }
+
+private:
+ IndexQueryContextPtr _context;
+
+ std::wstring _field;
+ std::wstring _term;
+};
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_scorer.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_scorer.h
new file mode 100644
index 00000000000..c2bba205fd4
--- /dev/null
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_scorer.h
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/scorer.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/segment_postings.h"
+#include "olap/rowset/segment_v2/inverted_index/similarity/similarity.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+template <typename SegmentPostingsPtr>
+class TermScorer final : public Scorer {
+public:
+ TermScorer(SegmentPostingsPtr segment_postings, SimilarityPtr similarity)
+ : _segment_postings(std::move(segment_postings)),
_similarity(std::move(similarity)) {}
+
+ uint32_t advance() override { return _segment_postings->advance(); }
+ uint32_t seek(uint32_t target) override { return
_segment_postings->seek(target); }
+ uint32_t doc() const override { return _segment_postings->doc(); }
+ uint32_t size_hint() const override { return
_segment_postings->size_hint(); }
+
+ float score() override {
+ auto freq = _segment_postings->freq();
+ auto norm = _segment_postings->norm();
+ return _similarity->score(static_cast<float>(freq), norm);
+ }
+
+private:
+ SegmentPostingsPtr _segment_postings;
+ SimilarityPtr _similarity;
+};
+
+using TS_Base =
std::shared_ptr<TermScorer<std::shared_ptr<SegmentPostings<TermDocsPtr>>>>;
+using TS_NoScore =
std::shared_ptr<TermScorer<std::shared_ptr<NoScoreSegmentPosting<TermDocsPtr>>>>;
+using TS_Empty =
std::shared_ptr<TermScorer<std::shared_ptr<EmptySegmentPosting<TermDocsPtr>>>>;
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_weight.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_weight.h
new file mode 100644
index 00000000000..fb1985b7fd8
--- /dev/null
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_weight.h
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/segment_postings.h"
+#include
"olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_scorer.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/weight.h"
+#include "olap/rowset/segment_v2/inverted_index/similarity/similarity.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+class TermWeight : public Weight {
+public:
+ TermWeight(IndexQueryContextPtr context, std::wstring field, std::wstring
term,
+ SimilarityPtr similarity, bool enable_scoring)
+ : _context(std::move(context)),
+ _field(std::move(field)),
+ _term(std::move(term)),
+ _similarity(std::move(similarity)),
+ _enable_scoring(enable_scoring) {}
+ ~TermWeight() override = default;
+
+ ScorerPtr scorer(lucene::index::IndexReader* reader) override {
+ auto t = make_term_ptr(_field.c_str(), _term.c_str());
+ auto iter = make_term_doc_ptr(reader, t.get(), _enable_scoring,
_context->io_ctx);
+
+ auto make_scorer = [this](auto segment_postings) -> ScorerPtr {
+ using PostingsT = decltype(segment_postings);
+ return
std::make_shared<TermScorer<PostingsT>>(std::move(segment_postings),
+ _similarity);
+ };
+
+ if (iter) {
+ if (_enable_scoring) {
+ auto segment_postings =
+
std::make_shared<SegmentPostings<TermDocsPtr>>(std::move(iter));
+ return make_scorer(std::move(segment_postings));
+ } else {
+ auto segment_postings =
+
std::make_shared<NoScoreSegmentPosting<TermDocsPtr>>(std::move(iter));
+ return make_scorer(std::move(segment_postings));
+ }
+ } else {
+ auto segment_postings =
std::make_shared<EmptySegmentPosting<TermDocsPtr>>();
+ return make_scorer(std::move(segment_postings));
+ }
+ }
+
+private:
+ IndexQueryContextPtr _context;
+
+ std::wstring _field;
+ std::wstring _term;
+ SimilarityPtr _similarity;
+ bool _enable_scoring = false;
+};
+
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/factory.h
b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/weight.h
similarity index 63%
rename from be/src/olap/rowset/segment_v2/inverted_index/query_v2/factory.h
rename to be/src/olap/rowset/segment_v2/inverted_index/query_v2/weight.h
index 5432a10bbb6..a82bbeb57df 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query_v2/factory.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query_v2/weight.h
@@ -17,25 +17,22 @@
#pragma once
-#include "olap/rowset/segment_v2/inverted_index/query_v2/node.h"
+#include <CLucene.h>
+#include <CLucene/index/IndexReader.h>
-namespace doris::segment_v2::idx_query_v2 {
+#include "olap/rowset/segment_v2/inverted_index/query_v2/scorer.h"
-enum class QueryType;
-enum class OperatorType;
+CL_NS_USE(index)
-class Operator;
-using OperatorPtr = std::shared_ptr<Operator>;
+namespace doris::segment_v2::inverted_index::query_v2 {
-class QueryFactory {
+class Weight {
public:
- template <typename... Args>
- static Result<Node> create(QueryType query_type, Args&&... args);
-};
+ Weight() = default;
+ virtual ~Weight() = default;
-class OperatorFactory {
-public:
- static Result<Node> create(OperatorType query_type);
+ virtual ScorerPtr scorer(lucene::index::IndexReader* reader) = 0;
};
+using WeightPtr = std::shared_ptr<Weight>;
-} // namespace doris::segment_v2::idx_query_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/similarity/bm25_similarity.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/similarity/bm25_similarity.cpp
index 89f9d438866..37c4ca81903 100644
---
a/be/src/olap/rowset/segment_v2/inverted_index/similarity/bm25_similarity.cpp
+++
b/be/src/olap/rowset/segment_v2/inverted_index/similarity/bm25_similarity.cpp
@@ -17,8 +17,6 @@
#include "bm25_similarity.h"
-#include "olap/rowset/segment_v2/inverted_index/util/string_helper.h"
-
namespace doris::segment_v2 {
#include "common/compile_check_begin.h"
@@ -38,17 +36,25 @@ std::vector<float> BM25Similarity::LENGTH_TABLE = []() {
BM25Similarity::BM25Similarity() : _cache(256) {}
-void BM25Similarity::for_one_term(const IndexQueryContextPtr& context,
- const std::wstring& field_name, const
std::wstring& term) {
- _avgdl =
context->collection_statistics->get_or_calculate_avg_dl(field_name);
- _idf = context->collection_statistics->get_or_calculate_idf(field_name,
term);
+BM25Similarity::BM25Similarity(float idf, float avgdl) : _idf(idf),
_avgdl(avgdl), _cache(256) {
_weight = _boost * _idf * (_k1 + 1.0F);
+ compute_tf_cache();
+}
+void BM25Similarity::compute_tf_cache() {
for (int i = 0; i < _cache.size(); i++) {
_cache[i] = 1.0F / (_k1 * ((1 - _b) + _b * LENGTH_TABLE[i] / _avgdl));
}
}
+void BM25Similarity::for_one_term(const IndexQueryContextPtr& context,
+ const std::wstring& field_name, const
std::wstring& term) {
+ _avgdl =
context->collection_statistics->get_or_calculate_avg_dl(field_name);
+ _idf = context->collection_statistics->get_or_calculate_idf(field_name,
term);
+ _weight = _boost * _idf * (_k1 + 1.0F);
+ compute_tf_cache();
+}
+
float BM25Similarity::score(float freq, int64_t encoded_norm) {
float norm_inverse = _cache[((uint8_t)encoded_norm) & 0xFF];
return _weight - _weight / (1.0F + freq * norm_inverse);
@@ -58,16 +64,7 @@ int32_t BM25Similarity::number_of_leading_zeros(uint64_t
value) {
if (value == 0) {
return 64;
}
-#if defined(__GNUC__) || defined(__clang__)
- return __builtin_clzll(value);
-#else
- int32_t count = 0;
- for (uint64_t mask = 1ULL << 63; mask != 0; mask >>= 1) {
- if (value & mask) break;
- ++count;
- }
- return count;
-#endif
+ return std::countl_zero(value);
}
uint32_t BM25Similarity::long_to_int4(uint64_t i) {
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/similarity/bm25_similarity.h
b/be/src/olap/rowset/segment_v2/inverted_index/similarity/bm25_similarity.h
index cf87f6eb04c..df067e5353f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/similarity/bm25_similarity.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/similarity/bm25_similarity.h
@@ -33,6 +33,7 @@ namespace doris::segment_v2 {
class BM25Similarity : public Similarity {
public:
BM25Similarity();
+ BM25Similarity(float idf, float avgdl);
~BM25Similarity() override = default;
void for_one_term(const IndexQueryContextPtr& context, const std::wstring&
field_name,
@@ -48,6 +49,8 @@ private:
static uint32_t long_to_int4(uint64_t i);
static uint64_t int4_to_long(uint32_t i);
+ void compute_tf_cache();
+
static const int32_t MAX_INT32;
static const uint32_t MAX_INT4;
static const int32_t NUM_FREE_VALUES;
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/similarity/similarity.h
b/be/src/olap/rowset/segment_v2/inverted_index/similarity/similarity.h
index b59cc175ab3..221b29ecd98 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/similarity/similarity.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/similarity/similarity.h
@@ -20,9 +20,7 @@
#include <cstdint>
#include <memory>
-#include "olap/collection_similarity.h"
#include "olap/rowset/segment_v2/index_query_context.h"
-#include "olap/rowset/segment_v2/inverted_index/util/term_iterator.h"
namespace doris::segment_v2 {
#include "common/compile_check_begin.h"
@@ -37,7 +35,7 @@ public:
virtual float score(float freq, int64_t encoded_norm) = 0;
};
-using SimilarityPtr = std::unique_ptr<Similarity>;
+using SimilarityPtr = std::shared_ptr<Similarity>;
#include "common/compile_check_end.h"
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/util/term_iterator.h
b/be/src/olap/rowset/segment_v2/inverted_index/util/term_iterator.h
index 013e2829772..cd67d11cdf4 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/util/term_iterator.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/util/term_iterator.h
@@ -34,21 +34,11 @@ struct IOContext;
namespace doris::segment_v2 {
-struct CLuceneDeleter {
- void operator()(TermDocs* p) const {
- if (p) {
- _CLDELETE(p);
- }
- }
-};
-
class TermIterator;
using TermIterPtr = std::shared_ptr<TermIterator>;
class TermIterator {
public:
- using TermDocsPtr = std::unique_ptr<TermDocs, CLuceneDeleter>;
-
TermIterator() = default;
TermIterator(std::wstring term, TermDocsPtr term_docs)
: term_(std::move(term)), term_docs_(std::move(term_docs)) {}
@@ -95,17 +85,12 @@ public:
static TermIterPtr create(const io::IOContext* io_ctx, bool is_similarity,
lucene::index::IndexReader* reader, const
std::wstring& field_name,
const std::wstring& ws_term) {
- auto t = make_term(field_name, ws_term);
- auto* term_pos = reader->termDocs(t.get(), is_similarity, io_ctx);
- return std::make_shared<TermIterator>(ws_term, TermDocsPtr(term_pos,
CLuceneDeleter {}));
+ auto t = make_term_ptr(field_name.c_str(), ws_term.c_str());
+ auto term_docs = make_term_doc_ptr(reader, t.get(), is_similarity,
io_ctx);
+ return std::make_shared<TermIterator>(ws_term, std::move(term_docs));
}
protected:
- static TermPtr make_term(const std::wstring& field_name, const
std::wstring& ws_term) {
- return TermPtr(new lucene::index::Term(field_name.c_str(),
ws_term.c_str()),
- TermDeleter {});
- }
-
std::wstring term_;
TermDocsPtr term_docs_;
};
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/util/term_position_iterator.h
b/be/src/olap/rowset/segment_v2/inverted_index/util/term_position_iterator.h
index 8a2afd58314..01440fc24a1 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/util/term_position_iterator.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/util/term_position_iterator.h
@@ -32,8 +32,6 @@ using TermPositionsIterPtr =
std::shared_ptr<TermPositionsIterator>;
class TermPositionsIterator : public TermIterator {
public:
- using TermPositionsPtr = std::unique_ptr<TermPositions, CLuceneDeleter>;
-
TermPositionsIterator() = default;
TermPositionsIterator(std::wstring term, TermPositionsPtr term_positions)
: TermIterator(std::move(term), std::move(term_positions)) {
@@ -54,10 +52,9 @@ public:
lucene::index::IndexReader* reader,
const std::wstring& field_name,
const std::wstring& ws_term) {
- auto t = make_term(field_name, ws_term);
- auto* term_pos = reader->termPositions(t.get(), is_similarity, io_ctx);
- return std::make_shared<TermPositionsIterator>(
- ws_term, TermPositionsPtr(term_pos, CLuceneDeleter {}));
+ auto t = make_term_ptr(field_name.c_str(), ws_term.c_str());
+ auto term_pos = make_term_positions_ptr(reader, t.get(),
is_similarity, io_ctx);
+ return std::make_shared<TermPositionsIterator>(ws_term,
std::move(term_pos));
}
private:
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/util/tiny_set.h
b/be/src/olap/rowset/segment_v2/inverted_index/util/tiny_set.h
new file mode 100644
index 00000000000..f33b1cb7875
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/util/tiny_set.h
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstdint>
+#include <optional>
+
+namespace doris::segment_v2::inverted_index {
+
+class TinySet {
+public:
+ explicit TinySet(uint64_t value) : _bits(value) {}
+ ~TinySet() = default;
+
+ bool is_empty() const { return _bits == 0; }
+
+ void clear() { _bits = 0; }
+
+ bool insert_mut(uint32_t el) {
+ if (el >= 64) {
+ return false;
+ }
+ uint64_t old_bits = _bits;
+ _bits |= (1ULL << el);
+ return old_bits != _bits;
+ }
+
+ std::optional<uint32_t> pop_lowest() {
+ if (is_empty()) {
+ return std::nullopt;
+ }
+ uint32_t lowest = std::countr_zero(_bits);
+ _bits ^= (1ULL << lowest);
+ return lowest;
+ }
+
+ uint32_t len() const { return std::popcount(_bits); }
+
+private:
+ uint64_t _bits = 0;
+};
+using TinySetPtr = std::shared_ptr<TinySet>;
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_common.h
b/be/src/olap/rowset/segment_v2/inverted_index_common.h
index 2cabcda7395..9d5b232a100 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_common.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_common.h
@@ -23,10 +23,6 @@
#include "common/logging.h"
-namespace lucene::store {
-class Directory;
-} // namespace lucene::store
-
namespace doris::segment_v2 {
struct DirectoryDeleter {
@@ -38,6 +34,31 @@ struct TermDeleter {
};
using TermPtr = std::unique_ptr<lucene::index::Term, TermDeleter>;
+template <typename... Args>
+TermPtr make_term_ptr(Args&&... args) {
+ return TermPtr(new lucene::index::Term(std::forward<Args>(args)...));
+}
+
+struct CLuceneDeleter {
+ void operator()(lucene::index::TermDocs* p) const {
+ if (p) {
+ _CLDELETE(p);
+ }
+ }
+};
+using TermDocsPtr = std::unique_ptr<lucene::index::TermDocs, CLuceneDeleter>;
+using TermPositionsPtr = std::unique_ptr<lucene::index::TermPositions,
CLuceneDeleter>;
+
+template <typename... Args>
+TermDocsPtr make_term_doc_ptr(lucene::index::IndexReader* reader, Args&&...
args) {
+ return TermDocsPtr(reader->termDocs(std::forward<Args>(args)...));
+}
+
+template <typename... Args>
+TermPositionsPtr make_term_positions_ptr(lucene::index::IndexReader* reader,
Args&&... args) {
+ return
TermPositionsPtr(reader->termPositions(std::forward<Args>(args)...));
+}
+
struct ErrorContext {
std::string err_msg;
std::exception_ptr eptr;
diff --git a/be/test/olap/collection_statistics_test.cpp
b/be/test/olap/collection_statistics_test.cpp
index c83d69cf6d5..b0b8dd9153e 100644
--- a/be/test/olap/collection_statistics_test.cpp
+++ b/be/test/olap/collection_statistics_test.cpp
@@ -630,4 +630,41 @@ TEST_F(CollectionStatisticsTest,
FindSlotRefHandlesNullDirectCastAndNested) {
EXPECT_EQ(find_slot_ref(bin),
static_cast<vectorized::VSlotRef*>(slot_ref_nested.get()));
}
+TEST(TermInfoComparerTest, OrdersByTermAndDedups) {
+ using doris::TermInfoComparer;
+ using doris::segment_v2::TermInfo;
+
+ std::set<TermInfo, TermInfoComparer> terms;
+
+ TermInfo t1;
+ t1.term = std::string("banana");
+ t1.position = 2;
+
+ TermInfo t2;
+ t2.term = std::string("apple");
+ t2.position = 10;
+
+ TermInfo t3;
+ t3.term = std::string("cherry");
+ t3.position = 1;
+
+ TermInfo dup;
+ dup.term = std::string("banana");
+ dup.position = 100;
+
+ terms.insert(t1);
+ terms.insert(t2);
+ terms.insert(t3);
+ terms.insert(dup);
+
+ std::vector<std::string> ordered;
+ ordered.reserve(terms.size());
+ for (const auto& t : terms) {
+ ordered.push_back(t.get_single_term());
+ }
+
+ EXPECT_EQ(terms.size(), 3u);
+ EXPECT_THAT(ordered, ::testing::ElementsAre("apple", "banana", "cherry"));
+}
+
} // namespace doris
\ No newline at end of file
diff --git
a/be/test/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query_test.cpp
b/be/test/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query_test.cpp
new file mode 100644
index 00000000000..71c79789c1c
--- /dev/null
+++
b/be/test/olap/rowset/segment_v2/inverted_index/query_v2/boolean_query_test.cpp
@@ -0,0 +1,316 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include
"olap/rowset/segment_v2/inverted_index/query_v2/boolean_query/boolean_query.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <roaring/roaring.hh>
+#include <string>
+
+#include "common/status.h"
+#include "olap/rowset/segment_v2/index_query_context.h"
+#include "olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer.h"
+#include "olap/rowset/segment_v2/inverted_index/query_v2/operator.h"
+#include
"olap/rowset/segment_v2/inverted_index/query_v2/term_query/term_query.h"
+#include "olap/rowset/segment_v2/inverted_index/util/string_helper.h"
+
+CL_NS_USE(search)
+CL_NS_USE(store)
+
+namespace doris::segment_v2 {
+
+using namespace inverted_index;
+
+class BooleanQueryTest : public testing::Test {
+public:
+ const std::string kTestDir = "./ut_dir/query_test";
+ std::string field_name = "name";
+
+ void SetUp() override {
+ auto st = io::global_local_filesystem()->delete_directory(kTestDir);
+ ASSERT_TRUE(st.ok()) << st;
+ st = io::global_local_filesystem()->create_directory(kTestDir);
+ ASSERT_TRUE(st.ok()) << st;
+
+ create_test_index();
+ }
+ void TearDown() override {
+
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok());
+ }
+
+private:
+ void create_test_index() {
+ std::vector<std::string> test_data = {"apple banana orange", "apple
cherry grape",
+ "banana cherry kiwi", "orange
grape strawberry",
+ "apple orange kiwi", "cherry
banana grape",
+ "strawberry apple kiwi", "orange
cherry banana"};
+
+ CustomAnalyzerConfig::Builder builder;
+ builder.with_tokenizer_config("standard", {});
+ auto custom_analyzer_config = builder.build();
+ auto custom_analyzer =
CustomAnalyzer::build_custom_analyzer(custom_analyzer_config);
+
+ auto* indexwriter =
+ _CLNEW lucene::index::IndexWriter(kTestDir.c_str(),
custom_analyzer.get(), true);
+ indexwriter->setMaxBufferedDocs(100);
+ indexwriter->setRAMBufferSizeMB(-1);
+ indexwriter->setMaxFieldLength(0x7FFFFFFFL);
+ indexwriter->setMergeFactor(1000000000);
+ indexwriter->setUseCompoundFile(false);
+
+ auto* char_string_reader = _CLNEW lucene::util::SStringReader<char>;
+
+ auto* doc = _CLNEW lucene::document::Document();
+ int32_t field_config = lucene::document::Field::STORE_NO;
+ field_config |= lucene::document::Field::INDEX_NONORMS;
+ field_config |= lucene::document::Field::INDEX_TOKENIZED;
+ auto field_name_w = std::wstring(field_name.begin(), field_name.end());
+ auto* field = _CLNEW lucene::document::Field(field_name_w.c_str(),
field_config);
+ field->setOmitTermFreqAndPositions(false);
+ doc->add(*field);
+
+ for (int r = 0; r < 10; ++r) {
+ for (const auto& data : test_data) {
+ char_string_reader->init(data.data(), data.size(), false);
+ auto* stream =
+ custom_analyzer->reusableTokenStream(field->name(),
char_string_reader);
+ field->setValue(stream);
+ indexwriter->addDocument(doc);
+ }
+ }
+
+ indexwriter->close();
+
+ _CLLDELETE(indexwriter);
+ _CLLDELETE(doc);
+ _CLLDELETE(char_string_reader);
+ }
+};
+
+static Status boolean_query_search(
+ const std::string& name, lucene::index::IndexReader* reader,
+ const std::pair<std::vector<std::string>, std::vector<std::string>>&
terms,
+ query_v2::OperatorType op, roaring::Roaring& out_bitmap) {
+ std::wstring field = StringHelper::to_wstring(name);
+
+ auto context = std::make_shared<IndexQueryContext>();
+ context->collection_statistics = std::make_shared<CollectionStatistics>();
+ context->collection_similarity = std::make_shared<CollectionSimilarity>();
+
+ query_v2::BooleanQuery::Builder builder(op);
+ {
+ query_v2::BooleanQuery::Builder
builder_child(query_v2::OperatorType::OP_AND);
+ for (const auto& term : terms.first) {
+ std::wstring t = StringHelper::to_wstring(term);
+ auto clause = std::make_shared<query_v2::TermQuery>(context,
field, t);
+ builder_child.add(clause);
+ }
+ auto boolean_query = builder_child.build();
+ builder.add(boolean_query);
+ }
+ {
+ query_v2::BooleanQuery::Builder
builder_child(query_v2::OperatorType::OP_OR);
+ for (const auto& term : terms.second) {
+ std::wstring t = StringHelper::to_wstring(term);
+ auto clause = std::make_shared<query_v2::TermQuery>(context,
field, t);
+ builder_child.add(clause);
+ }
+ auto boolean_query = builder_child.build();
+ builder.add(boolean_query);
+ }
+ auto boolean_query = builder.build();
+ auto weight = boolean_query->weight(false);
+ auto scorer = weight->scorer(reader);
+
+ uint32_t doc = scorer->doc();
+ while (doc != query_v2::TERMINATED) {
+ out_bitmap.add(doc);
+ doc = scorer->advance();
+ }
+
+ return Status::OK();
+}
+
+std::vector<std::string> tokenize(const CustomAnalyzerPtr& custom_analyzer,
+ const std::string line) {
+ std::vector<std::string> results;
+ lucene::util::SStringReader<char> reader;
+ reader.init(line.data(), line.size(), false);
+ auto* token_stream = custom_analyzer->reusableTokenStream(L"", &reader);
+ token_stream->reset();
+ Token t;
+ while (token_stream->next(&t)) {
+ results.emplace_back(t.termBuffer<char>(), t.termLength<char>());
+ }
+ return results;
+}
+
+TEST_F(BooleanQueryTest, test_boolean_query) {
+ CustomAnalyzerConfig::Builder builder;
+ builder.with_tokenizer_config("standard", {});
+ auto custom_analyzer_config = builder.build();
+ auto custom_analyzer =
CustomAnalyzer::build_custom_analyzer(custom_analyzer_config);
+
+ std::vector<std::pair<std::vector<std::string>, std::vector<std::string>>>
test_cases = {
+ {{"apple"}, {"banana"}},
+ {{"orange"}, {"grape", "kiwi"}},
+ {{"cherry"}, {"strawberry"}},
+ {{"apple", "banana"}, {"kiwi"}}};
+
+ auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
+ auto* reader = IndexReader::open(dir, true);
+
+ ASSERT_TRUE(reader != nullptr) << "Failed to open index reader";
+ EXPECT_EQ(reader->numDocs(), 80) << "Index should contain 80 documents";
+
+ const std::vector<uint32_t> expected_cards = {10, 20, 0, 0};
+
+ for (size_t i = 0; i < test_cases.size(); ++i) {
+ const auto& terms = test_cases[i];
+ roaring::Roaring result;
+
+ try {
+ Status res = boolean_query_search(field_name, reader, terms,
+ query_v2::OperatorType::OP_AND,
result);
+ EXPECT_TRUE(res.ok()) << "Boolean query case " << i << " should
execute successfully";
+ EXPECT_EQ(result.cardinality(), expected_cards[i])
+ << "Unexpected result cardinality for AND case " << i;
+ } catch (const Exception& e) {
+ FAIL() << "Boolean query case " << i << " failed with exception: "
<< e.what();
+ }
+ }
+
+ reader->close();
+ _CLLDELETE(reader);
+ _CLDECDELETE(dir);
+}
+
+TEST_F(BooleanQueryTest, test_boolean_query_or_operation) {
+ CustomAnalyzerConfig::Builder builder;
+ builder.with_tokenizer_config("standard", {});
+ auto custom_analyzer_config = builder.build();
+ auto custom_analyzer =
CustomAnalyzer::build_custom_analyzer(custom_analyzer_config);
+
+ std::vector<std::pair<std::vector<std::string>, std::vector<std::string>>>
test_cases = {
+ {{"apple"}, {"banana"}}, {{"nonexistent"}, {"apple"}}};
+
+ auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
+ auto* reader = IndexReader::open(dir, true);
+
+ const std::vector<uint32_t> expected_cards = {70, 40};
+
+ for (size_t i = 0; i < test_cases.size(); ++i) {
+ const auto& terms = test_cases[i];
+ roaring::Roaring result;
+
+ try {
+ Status res = boolean_query_search(field_name, reader, terms,
+ query_v2::OperatorType::OP_OR,
result);
+ EXPECT_TRUE(res.ok()) << "Boolean OR query case " << i
+ << " should execute successfully";
+ EXPECT_EQ(result.cardinality(), expected_cards[i])
+ << "Unexpected result cardinality for OR case " << i;
+ } catch (const Exception& e) {
+ FAIL() << "Boolean OR query case " << i << " failed with
exception: " << e.what();
+ }
+ }
+
+ reader->close();
+ _CLLDELETE(reader);
+ _CLDECDELETE(dir);
+}
+
+TEST_F(BooleanQueryTest, test_boolean_query_scoring_or) {
+ std::wstring field = StringHelper::to_wstring(field_name);
+
+ auto context = std::make_shared<IndexQueryContext>();
+ context->collection_statistics = std::make_shared<CollectionStatistics>();
+ context->collection_similarity = std::make_shared<CollectionSimilarity>();
+
+ std::wstring ws_field = StringHelper::to_wstring(field_name);
+ // 直接访问成员填充统计信息
+ context->collection_statistics->_total_num_docs = 80;
+ context->collection_statistics->_total_num_tokens[ws_field] = 240; // 80*3
+ auto set_df = [&](const std::string& term, uint64_t df) {
+
context->collection_statistics->_term_doc_freqs[ws_field][StringHelper::to_wstring(term)]
=
+ df;
+ };
+ set_df("apple", 40);
+ set_df("banana", 40);
+ set_df("orange", 40);
+ set_df("cherry", 40);
+ set_df("grape", 30);
+ set_df("kiwi", 30);
+ set_df("strawberry", 20);
+
+ query_v2::BooleanQuery::Builder builder(query_v2::OperatorType::OP_OR);
+ {
+ query_v2::BooleanQuery::Builder
builder_child(query_v2::OperatorType::OP_AND);
+ auto clause = std::make_shared<query_v2::TermQuery>(context, field,
+
StringHelper::to_wstring("apple"));
+ builder_child.add(clause);
+ builder.add(builder_child.build());
+ }
+ {
+ query_v2::BooleanQuery::Builder
builder_child(query_v2::OperatorType::OP_OR);
+ auto clause = std::make_shared<query_v2::TermQuery>(context, field,
+
StringHelper::to_wstring("kiwi"));
+ builder_child.add(clause);
+ builder.add(builder_child.build());
+ }
+ auto boolean_query = builder.build();
+
+ auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
+ auto* reader = IndexReader::open(dir, true);
+ ASSERT_TRUE(reader != nullptr);
+
+ auto weight = boolean_query->weight(true);
+ auto scorer = weight->scorer(reader);
+
+ uint32_t doc = scorer->doc();
+ uint32_t count = 0;
+ float score_both = -1.0F;
+ float score_single = -1.0F;
+
+ while (doc != query_v2::TERMINATED) {
+ float s = scorer->score();
+ if ((doc % 8 == 4 || doc % 8 == 6) && score_both < 0.0F) {
+ score_both = s;
+ }
+ if ((doc % 8 == 0 || doc % 8 == 1 || doc % 8 == 2) && score_single <
0.0F) {
+ score_single = s;
+ }
+ ++count;
+ doc = scorer->advance();
+ }
+
+ std::cout << "count: " << count << std::endl;
+ std::cout << "score_single: " << score_single << std::endl;
+ std::cout << "score_both: " << score_both << std::endl;
+ EXPECT_EQ(count, 50);
+ EXPECT_GT(score_single, 0.0F);
+ EXPECT_GT(score_both, 0.0F);
+ EXPECT_GT(score_both, score_single);
+
+ reader->close();
+ _CLLDELETE(reader);
+ _CLDECDELETE(dir);
+}
+
+} // namespace doris::segment_v2
diff --git
a/be/test/olap/rowset/segment_v2/inverted_index/query_v2/doc_set_test.cpp
b/be/test/olap/rowset/segment_v2/inverted_index/query_v2/doc_set_test.cpp
new file mode 100644
index 00000000000..87e857985c1
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/inverted_index/query_v2/doc_set_test.cpp
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/doc_set.h"
+
+#include <gtest/gtest.h>
+
+#include "common/exception.h"
+
+namespace doris {
+
+using namespace segment_v2::inverted_index::query_v2;
+
+class DocSetTest : public testing::Test {
+public:
+ void SetUp() override {}
+ void TearDown() override {}
+};
+
+TEST_F(DocSetTest, TerminatorConstant) {
+ EXPECT_EQ(TERMINATED, static_cast<uint32_t>(INT_MAX));
+}
+
+TEST_F(DocSetTest, AdvanceNotImplemented) {
+ DocSet ds;
+ try {
+ (void)ds.advance();
+ FAIL() << "Expected doris::Exception for NOT_IMPLEMENTED_ERROR";
+ } catch (const Exception& e) {
+ EXPECT_EQ(e.code(), ErrorCode::NOT_IMPLEMENTED_ERROR);
+ } catch (...) {
+ FAIL() << "Expected doris::Exception";
+ }
+}
+
+TEST_F(DocSetTest, SeekNotImplemented) {
+ DocSet ds;
+ try {
+ (void)ds.seek(10);
+ FAIL() << "Expected doris::Exception for NOT_IMPLEMENTED_ERROR";
+ } catch (const Exception& e) {
+ EXPECT_EQ(e.code(), ErrorCode::NOT_IMPLEMENTED_ERROR);
+ } catch (...) {
+ FAIL() << "Expected doris::Exception";
+ }
+}
+
+TEST_F(DocSetTest, DocNotImplemented) {
+ DocSet ds;
+ try {
+ (void)ds.doc();
+ FAIL() << "Expected doris::Exception for NOT_IMPLEMENTED_ERROR";
+ } catch (const Exception& e) {
+ EXPECT_EQ(e.code(), ErrorCode::NOT_IMPLEMENTED_ERROR);
+ } catch (...) {
+ FAIL() << "Expected doris::Exception";
+ }
+}
+
+TEST_F(DocSetTest, SizeHintNotImplemented) {
+ DocSet ds;
+ try {
+ (void)ds.size_hint();
+ FAIL() << "Expected doris::Exception for NOT_IMPLEMENTED_ERROR";
+ } catch (const Exception& e) {
+ EXPECT_EQ(e.code(), ErrorCode::NOT_IMPLEMENTED_ERROR);
+ } catch (...) {
+ FAIL() << "Expected doris::Exception";
+ }
+}
+
+} // namespace doris
diff --git
a/be/test/olap/rowset/segment_v2/inverted_index/query_v2/query_test.cpp
b/be/test/olap/rowset/segment_v2/inverted_index/query_v2/query_test.cpp
deleted file mode 100644
index 13e462b96d3..00000000000
--- a/be/test/olap/rowset/segment_v2/inverted_index/query_v2/query_test.cpp
+++ /dev/null
@@ -1,183 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// #include "olap/rowset/segment_v2/inverted_index/query_v2/query.h"
-
-// #include <gtest/gtest.h>
-
-// #include <memory>
-// #include <string>
-
-// #include "common/status.h"
-// #include "olap/rowset/segment_v2/inverted_index/query_v2/boolean_query.h"
-// #include "olap/rowset/segment_v2/inverted_index/query_v2/factory.inline.h"
-// #include "olap/rowset/segment_v2/inverted_index/query_v2/node.h"
-// #include "olap/rowset/segment_v2/inverted_index/query_v2/operator.h"
-
-// #pragma GCC diagnostic push
-// #pragma GCC diagnostic ignored "-Wshadow-field"
-// #include <CLucene/util/stringUtil.h>
-
-// #include "CLucene/analysis/standard95/StandardAnalyzer.h"
-// #include "CLucene/store/FSDirectory.h"
-// #pragma GCC diagnostic pop
-
-// #include "common/logging.h"
-// #include "io/fs/local_file_system.h"
-
-// CL_NS_USE(search)
-// CL_NS_USE(store)
-
-// namespace doris::segment_v2 {
-
-// class QueryTest : public testing::Test {
-// public:
-// const std::string kTestDir = "./ut_dir/query_test";
-
-// void SetUp() override {
-// auto st = io::global_local_filesystem()->delete_directory(kTestDir);
-// ASSERT_TRUE(st.ok()) << st;
-// st = io::global_local_filesystem()->create_directory(kTestDir);
-// ASSERT_TRUE(st.ok()) << st;
-// }
-// void TearDown() override {
-//
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok());
-// }
-
-// QueryTest() = default;
-// ~QueryTest() override = default;
-// };
-
-// using namespace idx_query_v2;
-
-// static Status boolean_query_search(const std::string& name,
-// const std::shared_ptr<IndexSearcher>&
search) {
-// BooleanQuery::Builder builder;
-// RETURN_IF_ERROR(builder.set_op(OperatorType::OP_AND));
-// {
-// TQueryOptions options;
-// auto roaring = std::make_shared<roaring::Roaring>();
-// roaring->add(1);
-// roaring->add(3);
-// roaring->add(5);
-// roaring->add(7);
-// roaring->add(9);
-// auto clause =
DORIS_TRY(QueryFactory::create(QueryType::ROARING_QUERY, roaring));
-// RETURN_IF_ERROR(builder.add(clause));
-// }
-// {
-// BooleanQuery::Builder builder1;
-// RETURN_IF_ERROR(builder1.set_op(OperatorType::OP_OR));
-// {
-// TQueryOptions options;
-// QueryInfo query_info;
-// query_info.field_name = StringUtil::string_to_wstring(name);
-// query_info.terms.emplace_back("hm");
-// auto clause = DORIS_TRY(
-// QueryFactory::create(QueryType::TERM_QUERY, search,
options, query_info));
-// RETURN_IF_ERROR(builder1.add(clause));
-// }
-// {
-// TQueryOptions options;
-// QueryInfo query_info;
-// query_info.field_name = StringUtil::string_to_wstring(name);
-// query_info.terms.emplace_back("ac");
-// auto clause = DORIS_TRY(
-// QueryFactory::create(QueryType::TERM_QUERY, search,
options, query_info));
-// RETURN_IF_ERROR(builder1.add(clause));
-// }
-// auto boolean_query = DORIS_TRY(builder1.build());
-// RETURN_IF_ERROR(builder.add(boolean_query));
-// }
-// auto boolean_query = DORIS_TRY(builder.build());
-
-// auto result = std::make_shared<roaring::Roaring>();
-// visit_node(boolean_query, QueryExecute {}, result);
-// EXPECT_EQ(result->cardinality(), 3);
-// EXPECT_EQ(result->toString(), "{1,7,9}");
-
-// return Status::OK();
-// }
-
-// TEST_F(QueryTest, test_boolean_query) {
-// std::string name = "name";
-
-// // write
-// {
-// std::vector<std::string> datas;
-// datas.emplace_back("0 hm");
-// datas.emplace_back("1 hm");
-// datas.emplace_back("2 hm");
-// datas.emplace_back("3 bg");
-// datas.emplace_back("4 bg");
-// datas.emplace_back("5 bg");
-// datas.emplace_back("6 ac");
-// datas.emplace_back("7 ac");
-// datas.emplace_back("8 ac");
-// datas.emplace_back("9 ac");
-
-// auto* analyzer = _CLNEW
lucene::analysis::standard95::StandardAnalyzer();
-// analyzer->set_stopwords(nullptr);
-// auto* indexwriter = _CLNEW
lucene::index::IndexWriter(kTestDir.c_str(), analyzer, true);
-// indexwriter->setMaxBufferedDocs(100);
-// indexwriter->setRAMBufferSizeMB(-1);
-// indexwriter->setMaxFieldLength(0x7FFFFFFFL);
-// indexwriter->setMergeFactor(1000000000);
-// indexwriter->setUseCompoundFile(false);
-
-// auto* char_string_reader = _CLNEW lucene::util::SStringReader<char>;
-
-// auto* doc = _CLNEW lucene::document::Document();
-// int32_t field_config = lucene::document::Field::STORE_NO;
-// field_config |= lucene::document::Field::INDEX_NONORMS;
-// field_config |= lucene::document::Field::INDEX_TOKENIZED;
-// auto field_name = std::wstring(name.begin(), name.end());
-// auto* field = _CLNEW lucene::document::Field(field_name.c_str(),
field_config);
-// field->setOmitTermFreqAndPositions(false);
-// doc->add(*field);
-
-// for (const auto& data : datas) {
-// char_string_reader->init(data.data(), data.size(), false);
-// auto* stream = analyzer->reusableTokenStream(field->name(),
char_string_reader);
-// field->setValue(stream);
-// indexwriter->addDocument(doc);
-// }
-
-// indexwriter->close();
-
-// _CLLDELETE(indexwriter);
-// _CLLDELETE(doc);
-// _CLLDELETE(analyzer);
-// _CLLDELETE(char_string_reader);
-// }
-
-// // query
-// {
-// auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
-// auto* reader = IndexReader::open(dir, 1024 * 1024, true);
-// auto search = std::make_shared<IndexSearcher>(reader);
-
-// Status res = boolean_query_search(name, search);
-// EXPECT_TRUE(res.ok());
-
-// reader->close();
-// _CLLDELETE(reader);
-// _CLDECDELETE(dir);
-// }
-// }
-
-// } // namespace doris::segment_v2
diff --git
a/be/test/olap/rowset/segment_v2/inverted_index/query_v2/score_combiner_test.cpp
b/be/test/olap/rowset/segment_v2/inverted_index/query_v2/score_combiner_test.cpp
new file mode 100644
index 00000000000..5c7522984c1
--- /dev/null
+++
b/be/test/olap/rowset/segment_v2/inverted_index/query_v2/score_combiner_test.cpp
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/score_combiner.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/scorer.h"
+
+namespace doris {
+
+using segment_v2::inverted_index::query_v2::DoNothingCombiner;
+using segment_v2::inverted_index::query_v2::DoNothingCombinerPtr;
+using segment_v2::inverted_index::query_v2::EmptyScorer;
+using segment_v2::inverted_index::query_v2::Scorer;
+using segment_v2::inverted_index::query_v2::ScorerPtr;
+using segment_v2::inverted_index::query_v2::SumCombiner;
+using segment_v2::inverted_index::query_v2::SumCombinerPtr;
+
+class ConstScorer final : public Scorer {
+public:
+ explicit ConstScorer(float v) : _v(v) {}
+ ~ConstScorer() override = default;
+
+ float score() override { return _v; }
+
+private:
+ float _v;
+};
+
+class ScoreCombinerTest : public ::testing::Test {
+public:
+ void SetUp() override {}
+ void TearDown() override {}
+};
+
+TEST_F(ScoreCombinerTest, SumCombinerBasic) {
+ SumCombiner comb;
+ EXPECT_FLOAT_EQ(comb.score(), 0.0F);
+
+ auto s1 = std::make_shared<ConstScorer>(1.5F);
+ auto s2 = std::make_shared<ConstScorer>(2.25F);
+ comb.update(s1);
+ comb.update(s2);
+ EXPECT_FLOAT_EQ(comb.score(), 3.75F);
+
+ comb.clear();
+ EXPECT_FLOAT_EQ(comb.score(), 0.0F);
+
+ auto empty = std::make_shared<EmptyScorer>();
+ comb.update(empty);
+ EXPECT_FLOAT_EQ(comb.score(), 0.0F);
+}
+
+TEST_F(ScoreCombinerTest, SumCombinerCloneIndependence) {
+ SumCombiner c1;
+ c1.update(std::make_shared<ConstScorer>(5.0F));
+ EXPECT_FLOAT_EQ(c1.score(), 5.0F);
+
+ SumCombinerPtr c2 = c1.clone();
+ ASSERT_TRUE(c2 != nullptr);
+ EXPECT_FLOAT_EQ(c2->score(), 0.0F);
+
+ c2->update(std::make_shared<ConstScorer>(1.0F));
+ EXPECT_FLOAT_EQ(c2->score(), 1.0F);
+ EXPECT_FLOAT_EQ(c1.score(), 5.0F);
+}
+
+TEST_F(ScoreCombinerTest, DoNothingCombinerBehavior) {
+ DoNothingCombiner comb;
+ EXPECT_FLOAT_EQ(comb.score(), 0.0F);
+
+ comb.update(std::make_shared<ConstScorer>(10.0F));
+ EXPECT_FLOAT_EQ(comb.score(), 0.0F);
+
+ comb.clear();
+ EXPECT_FLOAT_EQ(comb.score(), 0.0F);
+
+ DoNothingCombinerPtr cloned = comb.clone();
+ ASSERT_TRUE(cloned != nullptr);
+ EXPECT_FLOAT_EQ(cloned->score(), 0.0F);
+
+ cloned->update(std::make_shared<ConstScorer>(3.14F));
+ EXPECT_FLOAT_EQ(cloned->score(), 0.0F);
+}
+
+} // namespace doris
diff --git
a/be/test/olap/rowset/segment_v2/inverted_index/query_v2/segment_postings_test.cpp
b/be/test/olap/rowset/segment_v2/inverted_index/query_v2/segment_postings_test.cpp
new file mode 100644
index 00000000000..f6fef2dae73
--- /dev/null
+++
b/be/test/olap/rowset/segment_v2/inverted_index/query_v2/segment_postings_test.cpp
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/query_v2/segment_postings.h"
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+namespace doris {
+
+using namespace segment_v2::inverted_index::query_v2;
+
+class SegmentPostingsTest : public ::testing::Test {
+public:
+ void SetUp() override {}
+ void TearDown() override {}
+};
+
+class FakeIter {
+public:
+ struct Entry {
+ int32_t d;
+ int32_t f;
+ int32_t n;
+ };
+
+ explicit FakeIter(std::vector<Entry> postings) :
_postings(std::move(postings)) {}
+
+ bool next() {
+ if (_idx + 1 < static_cast<int32_t>(_postings.size())) {
+ ++_idx;
+ return true;
+ }
+ return false;
+ }
+
+ bool skipTo(uint32_t target) {
+ int32_t start = std::max(_idx, 0);
+ for (int32_t j = start; j < static_cast<int32_t>(_postings.size());
++j) {
+ if (static_cast<uint32_t>(_postings[j].d) >= target) {
+ _idx = j;
+ return true;
+ }
+ }
+ return false;
+ }
+
+ int32_t doc() const { return _postings[_idx].d; }
+ int32_t freq() const { return _postings[_idx].f; }
+ int32_t norm() const { return _postings[_idx].n; }
+ uint32_t docFreq() const { return static_cast<uint32_t>(_postings.size());
}
+
+private:
+ std::vector<Entry> _postings;
+ int32_t _idx = -1;
+};
+
+TEST_F(SegmentPostingsTest, BasicIterationAndScore) {
+ using IterPtr = std::shared_ptr<FakeIter>;
+ std::vector<FakeIter::Entry> data = {{1, 2, 10}, {3, 1, 7}, {5, 4, 5}};
+ auto iter = std::make_shared<FakeIter>(data);
+
+ SegmentPostings<IterPtr> sp(iter);
+
+ EXPECT_EQ(sp.size_hint(), 3u);
+ EXPECT_EQ(sp.doc(), 1u);
+ EXPECT_EQ(sp.freq(), 2);
+ EXPECT_EQ(sp.norm(), 10);
+
+ EXPECT_EQ(sp.advance(), 3u);
+ EXPECT_EQ(sp.doc(), 3u);
+ EXPECT_EQ(sp.freq(), 1);
+ EXPECT_EQ(sp.norm(), 7);
+
+ EXPECT_EQ(sp.advance(), 5u);
+ EXPECT_EQ(sp.doc(), 5u);
+ EXPECT_EQ(sp.freq(), 4);
+ EXPECT_EQ(sp.norm(), 5);
+
+ EXPECT_EQ(sp.advance(), TERMINATED);
+ EXPECT_EQ(sp.doc(), TERMINATED);
+}
+
+TEST_F(SegmentPostingsTest, SeekBehavior) {
+ using IterPtr = std::shared_ptr<FakeIter>;
+ std::vector<FakeIter::Entry> data = {{2, 1, 1}, {10, 2, 3}, {15, 3, 9}};
+ auto iter = std::make_shared<FakeIter>(data);
+
+ SegmentPostings<IterPtr> sp(iter);
+
+ EXPECT_EQ(sp.doc(), 2u);
+ EXPECT_EQ(sp.seek(0), 2u);
+ EXPECT_EQ(sp.seek(2), 2u);
+ EXPECT_EQ(sp.seek(3), 10u);
+ EXPECT_EQ(sp.seek(10), 10u);
+ EXPECT_EQ(sp.seek(11), 15u);
+ EXPECT_EQ(sp.seek(100), TERMINATED);
+ EXPECT_EQ(sp.doc(), TERMINATED);
+}
+
+TEST_F(SegmentPostingsTest, NoScoreSegmentPostingAlwaysOne) {
+ using IterPtr = std::shared_ptr<FakeIter>;
+ std::vector<FakeIter::Entry> data = {{1, 100, 200}, {2, 300, 400}};
+ auto iter = std::make_shared<FakeIter>(data);
+
+ NoScoreSegmentPosting<IterPtr> sp(iter);
+ EXPECT_EQ(sp.doc(), 1u);
+ EXPECT_EQ(sp.freq(), 1);
+ EXPECT_EQ(sp.norm(), 1);
+
+ EXPECT_EQ(sp.advance(), 2u);
+ EXPECT_EQ(sp.freq(), 1);
+ EXPECT_EQ(sp.norm(), 1);
+
+ EXPECT_EQ(sp.advance(), TERMINATED);
+ EXPECT_EQ(sp.doc(), TERMINATED);
+}
+
+TEST_F(SegmentPostingsTest, EmptySegmentPostingAlwaysTerminated) {
+ EmptySegmentPosting<std::shared_ptr<FakeIter>> sp;
+ EXPECT_EQ(sp.size_hint(), 0u);
+ EXPECT_EQ(sp.doc(), TERMINATED);
+ EXPECT_EQ(sp.advance(), TERMINATED);
+ EXPECT_EQ(sp.seek(123), TERMINATED);
+ EXPECT_EQ(sp.freq(), 1);
+ EXPECT_EQ(sp.norm(), 1);
+}
+
+TEST_F(SegmentPostingsTest, ConstructorWithEmptyIterator) {
+ using IterPtr = std::shared_ptr<FakeIter>;
+ std::vector<FakeIter::Entry> data;
+ auto iter = std::make_shared<FakeIter>(data);
+
+ SegmentPostings<IterPtr> sp(iter);
+ EXPECT_EQ(sp.size_hint(), 0u);
+ EXPECT_EQ(sp.doc(), TERMINATED);
+}
+
+TEST_F(SegmentPostingsTest, IntMaxDocBecomesTerminatedOnInit) {
+ using IterPtr = std::shared_ptr<FakeIter>;
+ std::vector<FakeIter::Entry> data = {{INT_MAX, 1, 1}};
+ auto iter = std::make_shared<FakeIter>(data);
+
+ SegmentPostings<IterPtr> sp(iter);
+ EXPECT_EQ(sp.doc(), TERMINATED);
+}
+
+} // namespace doris
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
