This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c4b11b53e1c [fix](be) Keep BM25 statistics per index for MATCH on one
column (#69901)
c4b11b53e1c is described below
commit c4b11b53e1ceeb014d6a9be731261419d84edcdf
Author: Jack <[email protected]>
AuthorDate: Mon Nov 2 09:24:56 2026 +0800
[fix](be) Keep BM25 statistics per index for MATCH on one column (#69901)
### What problem does this PR solve?
Issue Number: None
Related PR: #69790
Problem Summary: score() over MATCH clauses that read different inverted
indexes of one column fails:
INDEX idx_english(name) USING INVERTED PROPERTIES("parser" = "english",
...),
INDEX idx_standard(name) USING INVERTED PROPERTIES("parser" =
"standard", ...)
SELECT id, score() FROM t
WHERE name MATCH 'apple' USING ANALYZER english
AND name MATCH 'apple' USING ANALYZER standard
ORDER BY score() DESC LIMIT 10;
-- Scoring predicates for one field select different inverted indexes
BM25 statistics are keyed by field (column unique id plus suffix path),
but each index of a column holds its own terms and lengths: over Apple
and apple, an index that folds case has df(apple) = 2 and one that does
not has df(apple) = 1. One entry cannot serve both, so the collector
refuses the query.
Fix: a MATCH keeps its statistics under the field and the index id.
CollectInfo records the field as indexed for opening the index, since
the
map key no longer is one. The readers bind their source to the real
field
and pass the statistics name to run_leaf, so the scorers and
similarities
need no change: they already look statistics up by the name they are
given. Each clause now scores with the statistics of its own index, and
the combined score is the sum of what each clause scores alone.
SEARCH keeps its own statistics names and its rejection of two indexes
on
one field; only MATCH changes. A MATCH and a SEARCH on one index keep
separate entries with the same values.
---
.../function/search/search_predicate_collector.cpp | 1 +
.../inverted/clucene/clucene_index_reader.cpp | 23 +++--
.../index/inverted/clucene/clucene_index_reader.h | 6 ++
be/src/storage/index/inverted/common/query_info.h | 7 ++
.../inverted/reader/collection_statistics.cpp | 2 +-
.../index/inverted/reader/predicate_collector.cpp | 5 +-
.../index/inverted/reader/predicate_collector.h | 2 +
.../index/inverted/reader/text_index_reader.h | 5 ++
.../index/inverted/snii/snii_index_reader.cpp | 4 +-
.../inverted/reader/collection_statistics_test.cpp | 99 +++++++++++++---------
.../snii/snii_index_reader_count_fallback_test.cpp | 4 +-
.../test_match_score_multi_index.out | 76 +++++++++++++++++
.../test_match_score_multi_index.groovy | 73 ++++++++++++++++
13 files changed, 256 insertions(+), 51 deletions(-)
diff --git a/be/src/exprs/function/search/search_predicate_collector.cpp
b/be/src/exprs/function/search/search_predicate_collector.cpp
index e25e1ee279d..a1d19c25b5f 100644
--- a/be/src/exprs/function/search/search_predicate_collector.cpp
+++ b/be/src/exprs/function/search/search_predicate_collector.cpp
@@ -241,6 +241,7 @@ Status SearchPredicateCollector::collect_from_leaf(const
TSearchClause& clause,
auto iter = collect_infos->find(ws_field_name);
if (iter == collect_infos->end()) {
CollectInfo collect_info;
+ collect_info.lucene_field = ws_field_name;
RETURN_IF_ERROR(append_scoring_leaf(&collect_info, term_infos));
preserve_selected_index_metadata(candidates, index_meta,
&collect_info);
(*collect_infos)[ws_field_name] = std::move(collect_info);
diff --git a/be/src/storage/index/inverted/clucene/clucene_index_reader.cpp
b/be/src/storage/index/inverted/clucene/clucene_index_reader.cpp
index 0e0661382b5..17e75604484 100644
--- a/be/src/storage/index/inverted/clucene/clucene_index_reader.cpp
+++ b/be/src/storage/index/inverted/clucene/clucene_index_reader.cpp
@@ -338,7 +338,8 @@ Status create_index_searcher(IndexSearcherBuilder*
index_searcher_builder,
}
Status run_clucene_leaf(const IndexQueryContextPtr& context, std::wstring
field,
- const logical::Node& leaf, const roaring::Roaring*
candidates, bool scoring,
+ const std::wstring& statistics_field, const
logical::Node& leaf,
+ const roaring::Roaring* candidates, bool scoring,
const FulltextIndexSearcherPtr& searcher,
const std::shared_ptr<roaring::Roaring>& result) {
auto* reader = searcher->getReader();
@@ -350,8 +351,7 @@ Status run_clucene_leaf(const IndexQueryContextPtr&
context, std::wstring field,
auto source =
clucene_index_source(std::shared_ptr<lucene::index::IndexReader>(searcher,
reader),
std::move(field), context->io_ctx);
- const auto& source_field = source->field();
- RETURN_IF_ERROR(run_leaf(context, source_field, leaf, candidates,
scoring,
+ RETURN_IF_ERROR(run_leaf(context, statistics_field, leaf, candidates,
scoring,
std::move(source), reader->maxDoc(), result));
} catch (const CLuceneError& e) {
return clucene_error_status(fmt::format("CLuceneError occurred: {}",
e.what()));
@@ -413,15 +413,26 @@ Status
CluceneTextIndexReader::_term_document_frequency(const std::string& colum
return Status::OK();
}
+Status run_clucene_leaf(const IndexQueryContextPtr& context, std::wstring
field,
+ const logical::Node& leaf, const roaring::Roaring*
candidates, bool scoring,
+ const FulltextIndexSearcherPtr& searcher,
+ const std::shared_ptr<roaring::Roaring>& result) {
+ const std::wstring statistics_field = field;
+ return run_clucene_leaf(context, std::move(field), statistics_field, leaf,
candidates, scoring,
+ searcher, result);
+}
+
Status CluceneTextIndexReader::_run_leaf(const IndexQueryContextPtr& context,
const std::string& column_name,
OpenedIndex& index,
const logical::Node& leaf,
const roaring::Roaring* candidates,
bool scoring,
std::shared_ptr<roaring::Roaring>*
out) {
auto result = std::make_shared<roaring::Roaring>();
- RETURN_IF_ERROR(run_clucene_leaf(context,
StringUtil::string_to_wstring(column_name), leaf,
- candidates, scoring,
-
static_cast<CluceneOpenedIndex&>(index).searcher, result));
+ auto field = StringUtil::string_to_wstring(column_name);
+ auto statistics_field = _statistics_field(field);
+ RETURN_IF_ERROR(run_clucene_leaf(context, std::move(field),
statistics_field, leaf, candidates,
+ scoring,
static_cast<CluceneOpenedIndex&>(index).searcher,
+ result));
result->runOptimize();
*out = std::move(result);
return Status::OK();
diff --git a/be/src/storage/index/inverted/clucene/clucene_index_reader.h
b/be/src/storage/index/inverted/clucene/clucene_index_reader.h
index 6319f2fc439..90bbc7c802d 100644
--- a/be/src/storage/index/inverted/clucene/clucene_index_reader.h
+++ b/be/src/storage/index/inverted/clucene/clucene_index_reader.h
@@ -59,6 +59,12 @@ Status read_clucene_null_bitmap(IndexFileReader&
file_reader, const TabletIndex&
lucene::store::Directory* dir = nullptr);
// run_leaf on the CLucene field `field` of `searcher`, its errors reported as
CLucene ones.
+// `statistics_field` names the leaf inside the query, as _statistics_field
does for an index.
+Status run_clucene_leaf(const IndexQueryContextPtr& context, std::wstring
field,
+ const std::wstring& statistics_field,
+ const index_query::logical::Node& leaf, const
roaring::Roaring* candidates,
+ bool scoring, const FulltextIndexSearcherPtr& searcher,
+ const std::shared_ptr<roaring::Roaring>& result);
Status run_clucene_leaf(const IndexQueryContextPtr& context, std::wstring
field,
const index_query::logical::Node& leaf, const
roaring::Roaring* candidates,
bool scoring, const FulltextIndexSearcherPtr& searcher,
diff --git a/be/src/storage/index/inverted/common/query_info.h
b/be/src/storage/index/inverted/common/query_info.h
index a5773ed8add..514280ecc1f 100644
--- a/be/src/storage/index/inverted/common/query_info.h
+++ b/be/src/storage/index/inverted/common/query_info.h
@@ -40,4 +40,11 @@ public:
}
};
+// The name BM25 statistics of a MATCH are kept under. Two indexes of one
column hold different
+// terms and lengths under the same field, so the field alone cannot key them:
over Apple and
+// apple, an index that folds case has df(apple) = 2 and one that does not has
df(apple) = 1.
+inline std::wstring match_statistics_field(const std::wstring& field, int64_t
index_id) {
+ return field + L"@" + std::to_wstring(index_id);
+}
+
} // namespace doris::segment_v2
diff --git a/be/src/storage/index/inverted/reader/collection_statistics.cpp
b/be/src/storage/index/inverted/reader/collection_statistics.cpp
index c163133fbde..e8e5c420f87 100644
--- a/be/src/storage/index/inverted/reader/collection_statistics.cpp
+++ b/be/src/storage/index/inverted/reader/collection_statistics.cpp
@@ -664,7 +664,7 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset,
index_reader = index_searcher->getReader();
#endif
auto source =
segment_v2::clucene_index_source(segment_v2::non_owning_reader(index_reader),
- ws_field_name, io_ctx);
+
collect_info.lucene_field, io_ctx);
index_query::IndexScoringStats scoring_stats;
RETURN_IF_ERROR(source->scoring_stats(&scoring_stats));
if (hidden_rows != nullptr) {
diff --git a/be/src/storage/index/inverted/reader/predicate_collector.cpp
b/be/src/storage/index/inverted/reader/predicate_collector.cpp
index 53c0fed022e..747e48d892f 100644
--- a/be/src/storage/index/inverted/reader/predicate_collector.cpp
+++ b/be/src/storage/index/inverted/reader/predicate_collector.cpp
@@ -391,11 +391,14 @@ Status MatchPredicateCollector::collect(RuntimeState*
state, const TabletSchemaS
std::string field_name =
build_field_name(index_meta->col_unique_ids()[0],
candidates.index_suffix_path);
- std::wstring ws_field_name =
inverted_index::StringHelper::to_wstring(field_name);
+ std::wstring lucene_field =
inverted_index::StringHelper::to_wstring(field_name);
+ std::wstring ws_field_name =
+ segment_v2::match_statistics_field(lucene_field,
index_meta->index_id());
auto iter = collect_infos->find(ws_field_name);
if (iter == collect_infos->end()) {
CollectInfo collect_info;
+ collect_info.lucene_field = std::move(lucene_field);
RETURN_IF_ERROR(append_scoring_leaf(&collect_info, term_infos));
preserve_selected_index_metadata(candidates, index_meta,
&collect_info);
iter = collect_infos->emplace(ws_field_name,
std::move(collect_info)).first;
diff --git a/be/src/storage/index/inverted/reader/predicate_collector.h
b/be/src/storage/index/inverted/reader/predicate_collector.h
index 211c7ca0f09..17da17d7aad 100644
--- a/be/src/storage/index/inverted/reader/predicate_collector.h
+++ b/be/src/storage/index/inverted/reader/predicate_collector.h
@@ -67,6 +67,8 @@ struct LogicalScoringPrefix {
};
struct CollectInfo {
+ // The field as indexed; the key of a CollectInfoMap may also name the
index.
+ std::wstring lucene_field;
std::vector<std::string> unique_terms;
std::unordered_map<std::string, uint32_t> unique_term_slots;
std::vector<LogicalScoringPrefix> scoring_prefixes;
diff --git a/be/src/storage/index/inverted/reader/text_index_reader.h
b/be/src/storage/index/inverted/reader/text_index_reader.h
index edd27cf961a..b2e95781e3b 100644
--- a/be/src/storage/index/inverted/reader/text_index_reader.h
+++ b/be/src/storage/index/inverted/reader/text_index_reader.h
@@ -20,6 +20,7 @@
#include <functional>
#include <string_view>
+#include "storage/index/inverted/common/query_info.h"
#include "storage/index/inverted/inverted_index_reader.h"
namespace doris::segment_v2 {
@@ -123,6 +124,10 @@ protected:
virtual Status _term_document_frequency(const std::string& column_name,
OpenedIndex& index,
const std::string& term, uint64_t*
df,
uint64_t* document_count) = 0;
+ // The name of this index's statistics in the query's CollectionStatistics.
+ std::wstring _statistics_field(const std::wstring& field) const {
+ return match_statistics_field(field, _index_meta.index_id());
+ }
// Runs `leaf` over the open index into `out`. With `candidates`, a phrase
matches only those
// rows; with `scoring`, the BM25 values reach the context's collection
similarity.
virtual Status _run_leaf(const IndexQueryContextPtr& context, const
std::string& column_name,
diff --git a/be/src/storage/index/inverted/snii/snii_index_reader.cpp
b/be/src/storage/index/inverted/snii/snii_index_reader.cpp
index d26e4fe030c..01bf82f46a1 100644
--- a/be/src/storage/index/inverted/snii/snii_index_reader.cpp
+++ b/be/src/storage/index/inverted/snii/snii_index_reader.cpp
@@ -311,8 +311,8 @@ Status SniiIndexReader::_run_leaf(const
IndexQueryContextPtr& context,
auto source = _bind_source(context, field, index);
const uint32_t doc_count = source->doc_count();
auto result = std::make_shared<roaring::Roaring>();
- RETURN_IF_ERROR_OR_CATCH_EXCEPTION(run_leaf(context, field, leaf,
candidates, scoring,
- std::move(source), doc_count,
result));
+ RETURN_IF_ERROR_OR_CATCH_EXCEPTION(run_leaf(context,
_statistics_field(field), leaf, candidates,
+ scoring, std::move(source),
doc_count, result));
result->runOptimize();
*out = std::move(result);
return Status::OK();
diff --git
a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
index 917950e7b62..c528d41bbde 100644
--- a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
+++ b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
@@ -80,6 +80,10 @@ using
collection_statistics_detail::sum_non_null_norm_lengths;
using collection_statistics_detail::resolve_snii_scoring_segment;
using collection_statistics_detail::SniiScoringSegmentStats;
+// Where a MATCH on the field of column 1 keeps the statistics of index 1 and
of index 2.
+const std::wstring kMatchField = segment_v2::match_statistics_field(L"1", 1);
+const std::wstring kMatchField2 = segment_v2::match_statistics_field(L"1", 2);
+
namespace collection_statistics {
class MockVExpr : public VExpr {
@@ -857,7 +861,7 @@ TEST_F(CollectionStatisticsTest,
CollectWithMockRowsetSplits) {
stats_->collect(runtime_state_.get(), splits, tablet_schema,
expr_contexts, nullptr);
EXPECT_TRUE(status.ok()) << status;
- expect_no_collected_tokens(L"1");
+ expect_no_collected_tokens(kMatchField);
}
TEST_F(CollectionStatisticsTest, CollectWithEmptySegments) {
@@ -913,8 +917,8 @@ TEST_F(CollectionStatisticsTest,
LegacyV3SegmentsUsePhysicalScoringStatistics) {
create_match_expr_contexts("alpha"),
nullptr);
ASSERT_TRUE(status.ok()) << status;
- expect_collected_stats(L"1", 2, 4);
- expect_collected_term(L"1", L"alpha", 2);
+ expect_collected_stats(kMatchField, 2, 4);
+ expect_collected_term(kMatchField, L"alpha", 2);
}
TEST_F(CollectionStatisticsTest,
MissingFieldIndexPreservesOtherFieldStatistics) {
@@ -1013,11 +1017,11 @@ TEST_F(CollectionStatisticsTest,
HiddenRowsAreCollectedLikeCompactedAway) {
create_match_expr_contexts("alpha gamma"),
nullptr, &hidden);
EXPECT_TRUE(status.ok()) << status;
return {.doc_count = stats.get_doc_num(),
- .token_count = stats.get_total_term_cnt_by_col(L"1"),
- .alpha_doc_frequency = stats.get_term_doc_freq_by_col(L"1",
L"alpha"),
- .gamma_doc_frequency = stats.get_term_doc_freq_by_col(L"1",
L"gamma"),
- .avg_dl = stats.get_or_calculate_avg_dl(L"1"),
- .alpha_idf = stats.get_or_calculate_idf(L"1", L"alpha")};
+ .token_count = stats.get_total_term_cnt_by_col(kMatchField),
+ .alpha_doc_frequency =
stats.get_term_doc_freq_by_col(kMatchField, L"alpha"),
+ .gamma_doc_frequency =
stats.get_term_doc_freq_by_col(kMatchField, L"gamma"),
+ .avg_dl = stats.get_or_calculate_avg_dl(kMatchField),
+ .alpha_idf = stats.get_or_calculate_idf(kMatchField,
L"alpha")};
};
const Collected uncompacted = collect(uncompacted_path, {1});
@@ -1043,8 +1047,8 @@ TEST_F(CollectionStatisticsTest,
HiddenRowsAreCollectedLikeCompactedAway) {
ASSERT_TRUE(stats_->collect(runtime_state_.get(), splits, tablet_schema,
create_match_expr_contexts("alpha gamma"),
nullptr)
.ok());
- expect_collected_stats(L"1", 3, 50);
- expect_collected_term(L"1", L"alpha", 2);
+ expect_collected_stats(kMatchField, 3, 50);
+ expect_collected_term(kMatchField, L"alpha", 2);
}
// A tablet whose rows were all deleted or overwritten still has segments to
collect from, but no
@@ -1073,9 +1077,9 @@ TEST_F(CollectionStatisticsTest,
AllRowsHiddenCollectsEmptyStatistics) {
create_match_expr_contexts("alpha"),
nullptr, &hidden);
ASSERT_TRUE(status.ok()) << status;
EXPECT_EQ(stats_->get_doc_num(), 0);
- expect_collected_term(L"1", L"alpha", 0);
- EXPECT_TRUE(std::isfinite(stats_->get_or_calculate_avg_dl(L"1")));
- EXPECT_TRUE(std::isfinite(stats_->get_or_calculate_idf(L"1", L"alpha")));
+ expect_collected_term(kMatchField, L"alpha", 0);
+ EXPECT_TRUE(std::isfinite(stats_->get_or_calculate_avg_dl(kMatchField)));
+ EXPECT_TRUE(std::isfinite(stats_->get_or_calculate_idf(kMatchField,
L"alpha")));
// Without any collected segment the statistics are still reported missing.
CollectionStatistics nothing_collected;
@@ -1102,7 +1106,7 @@ TEST_F(CollectionStatisticsTest,
NullRowsCountAsDocumentsWithoutTokens) {
const Status status = stats.collect(runtime_state_.get(), splits,
tablet_schema,
create_match_expr_contexts("alpha"), nullptr, hidden);
EXPECT_TRUE(status.ok()) << status;
- return std::make_pair(stats.get_doc_num(),
stats.get_total_term_cnt_by_col(L"1"));
+ return std::make_pair(stats.get_doc_num(),
stats.get_total_term_cnt_by_col(kMatchField));
};
const auto without_bitmap = collect(nullptr);
@@ -1354,8 +1358,8 @@ TEST_F(CollectionStatisticsTest,
LegacyV3SkipsMissingSegmentAfterCollectingAvail
create_match_expr_contexts("alpha"),
nullptr);
ASSERT_TRUE(status.ok()) << status;
- expect_collected_stats(L"1", 1, 2);
- expect_collected_term(L"1", L"alpha", 1);
+ expect_collected_stats(kMatchField, 1, 2);
+ expect_collected_term(kMatchField, L"alpha", 1);
}
TEST_F(CollectionStatisticsTest,
LegacyV3SkipsEmptySegmentAfterCollectingAvailableStatistics) {
@@ -1387,8 +1391,8 @@ TEST_F(CollectionStatisticsTest,
LegacyV3SkipsEmptySegmentAfterCollectingAvailab
create_match_expr_contexts("alpha"),
nullptr);
ASSERT_TRUE(status.ok()) << status;
- expect_collected_stats(L"1", 1, 2);
- expect_collected_term(L"1", L"alpha", 1);
+ expect_collected_stats(kMatchField, 1, 2);
+ expect_collected_term(kMatchField, L"alpha", 1);
}
// BM25 needs norms from every segment of an analyzed index: a segment written
without them makes
@@ -1419,11 +1423,11 @@ TEST_F(CollectionStatisticsTest,
LegacyV3RejectsSegmentWrittenWithoutNorms) {
Status status = collect({without_norms_path});
EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED) <<
status;
EXPECT_NE(status.to_string().find("written without norms"),
std::string::npos) << status;
- expect_no_collected_tokens(L"1");
+ expect_no_collected_tokens(kMatchField);
status = collect({with_norms_path, without_norms_path});
EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED) <<
status;
- expect_no_collected_tokens(L"1");
+ expect_no_collected_tokens(kMatchField);
}
TEST_F(CollectionStatisticsTest,
LegacyV3KeywordIndexWithoutNormsCollectsScoring) {
@@ -1486,8 +1490,8 @@ TEST_F(CollectionStatisticsTest,
SniiScoringUsesPhysicalStatistics) {
stats_->collect(runtime_state_.get(), splits, tablet_schema,
expr_contexts, nullptr);
ASSERT_TRUE(status.ok()) << status;
- expect_collected_stats(L"1", 2, 3);
- expect_collected_term(L"1", L"alpha", 2);
+ expect_collected_stats(kMatchField, 2, 3);
+ expect_collected_term(kMatchField, L"alpha", 2);
}
TEST_F(CollectionStatisticsTest,
SniiDocsOnlyNormalizerStillCollectsScoringStatistics) {
@@ -1507,8 +1511,8 @@ TEST_F(CollectionStatisticsTest,
SniiDocsOnlyNormalizerStillCollectsScoringStati
create_match_expr_contexts("alpha"),
nullptr);
ASSERT_TRUE(status.ok()) << status;
- expect_collected_stats(L"1", 2, 2);
- expect_collected_term(L"1", L"alpha", 2);
+ expect_collected_stats(kMatchField, 2, 2);
+ expect_collected_term(kMatchField, L"alpha", 2);
}
TEST_F(CollectionStatisticsTest, SniiScoringLookupUsesCallerIoContext) {
@@ -1619,7 +1623,7 @@ TEST_F(CollectionStatisticsTest,
SniiScoringRejectsMissingLogicalIndexForWholeCo
stats_->collect(runtime_state_.get(), splits, tablet_schema,
expr_contexts, nullptr);
EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
- expect_no_collected_tokens(L"1");
+ expect_no_collected_tokens(kMatchField2);
}
TEST_F(CollectionStatisticsTest, CollectWithMultipleRowsetSplits) {
@@ -2022,8 +2026,10 @@ TEST_F(CollectionStatisticsTest,
ExtractCollectInfoForVariantSubcolumnIndex) {
&collect_infos);
ASSERT_TRUE(status.ok()) << status.msg();
ASSERT_EQ(collect_infos.size(), 1u);
- auto it =
collect_infos.find(segment_v2::inverted_index::StringHelper::to_wstring(
- std::to_string(kVariantUid) + ".v.host"));
+ auto it = collect_infos.find(segment_v2::match_statistics_field(
+
segment_v2::inverted_index::StringHelper::to_wstring(std::to_string(kVariantUid)
+
+ ".v.host"),
+ 2001));
ASSERT_NE(it, collect_infos.end());
ASSERT_NE(it->second.index_meta, nullptr);
EXPECT_EQ(it->second.index_meta->index_name(), "variant_subcolumn_idx");
@@ -2092,8 +2098,10 @@ TEST_F(CollectionStatisticsTest,
MatchScoringUsesTextSemanticsForVariantParentIn
&collect_infos);
ASSERT_TRUE(status.ok()) << status;
ASSERT_EQ(collect_infos.size(), 1U);
- auto it =
collect_infos.find(segment_v2::inverted_index::StringHelper::to_wstring(
- std::to_string(kVariantUid) + ".v.key"));
+ auto it = collect_infos.find(segment_v2::match_statistics_field(
+
segment_v2::inverted_index::StringHelper::to_wstring(std::to_string(kVariantUid)
+
+ ".v.key"),
+ 2004));
ASSERT_NE(it, collect_infos.end());
ASSERT_NE(it->second.index_meta, nullptr);
ASSERT_NE(it->second.owned_index_meta, nullptr);
@@ -2178,8 +2186,10 @@ TEST_F(CollectionStatisticsTest,
ExtractCollectInfoForVariantFieldPatternIndex)
&collect_infos);
ASSERT_TRUE(status.ok()) << status.msg();
ASSERT_EQ(collect_infos.size(), 1u);
- auto it =
collect_infos.find(segment_v2::inverted_index::StringHelper::to_wstring(
- std::to_string(kVariantUid) + ".meta.host"));
+ auto it = collect_infos.find(segment_v2::match_statistics_field(
+
segment_v2::inverted_index::StringHelper::to_wstring(std::to_string(kVariantUid)
+
+ ".meta.host"),
+ 2002));
ASSERT_NE(it, collect_infos.end());
ASSERT_NE(it->second.index_meta, nullptr);
ASSERT_NE(it->second.owned_index_meta, nullptr);
@@ -2255,8 +2265,10 @@ TEST_F(CollectionStatisticsTest,
ExtractCollectInfoForVariantFieldPatternGlobInd
&collect_infos);
ASSERT_TRUE(status.ok()) << status.msg();
ASSERT_EQ(collect_infos.size(), 1u);
- auto it =
collect_infos.find(segment_v2::inverted_index::StringHelper::to_wstring(
- std::to_string(kVariantUid) + ".meta.user.name"));
+ auto it = collect_infos.find(segment_v2::match_statistics_field(
+
segment_v2::inverted_index::StringHelper::to_wstring(std::to_string(kVariantUid)
+
+
".meta.user.name"),
+ 2003));
ASSERT_NE(it, collect_infos.end());
ASSERT_NE(it->second.index_meta, nullptr);
ASSERT_NE(it->second.owned_index_meta, nullptr);
@@ -2386,8 +2398,8 @@ TEST_F(CollectionStatisticsTest,
CollectDirectIndexHitFromSchema) {
collector.collect(runtime_state_.get(), tablet_schema, match_expr,
&collect_infos);
ASSERT_TRUE(status.ok()) << status.msg();
ASSERT_EQ(collect_infos.size(), 1u);
- auto it = collect_infos.find(
-
segment_v2::inverted_index::StringHelper::to_wstring(std::to_string(kColUid)));
+ auto it = collect_infos.find(segment_v2::match_statistics_field(
+
segment_v2::inverted_index::StringHelper::to_wstring(std::to_string(kColUid)),
2100));
ASSERT_NE(it, collect_infos.end());
EXPECT_NE(it->second.index_meta, nullptr);
EXPECT_EQ(it->second.owned_index_meta, nullptr); // O1: schema-direct meta
is not owned
@@ -2648,8 +2660,8 @@ TEST_F(CollectionStatisticsTest,
CollectPreservesLogicalClauseShapesForSameField
build_match("alpha alpha beta"),
&collect_infos);
ASSERT_TRUE(second.ok()) << second.msg();
ASSERT_EQ(collect_infos.size(), 1u);
- auto it = collect_infos.find(
-
segment_v2::inverted_index::StringHelper::to_wstring(std::to_string(kColUid)));
+ auto it = collect_infos.find(segment_v2::match_statistics_field(
+
segment_v2::inverted_index::StringHelper::to_wstring(std::to_string(kColUid)),
2400));
ASSERT_NE(it, collect_infos.end());
ASSERT_EQ(it->second.unique_terms, std::vector<std::string>({"alpha",
"beta"}));
ASSERT_EQ(it->second.unique_term_slots.size(), 2u);
@@ -3275,7 +3287,8 @@ TEST_F(CollectionStatisticsTest,
NestedSearchScoringIsRejected) {
EXPECT_TRUE(collect_infos.empty());
}
-TEST_F(CollectionStatisticsTest,
OneScoringFieldCannotSelectDifferentPhysicalIndexes) {
+// Two indexes of one column hold different terms and lengths, so each keeps
its own statistics.
+TEST_F(CollectionStatisticsTest,
OneColumnKeepsTheStatisticsOfEachSelectedIndex) {
auto tablet_schema = create_tablet_schema_with_two_fulltext_indexes();
auto build_match = [](const std::string& analyzer_name,
InvertedIndexParserType parser_type) {
auto match_expr =
@@ -3310,7 +3323,15 @@ TEST_F(CollectionStatisticsTest,
OneScoringFieldCannotSelectDifferentPhysicalInd
build_match("english",
InvertedIndexParserType::PARSER_ENGLISH),
&collect_infos);
- EXPECT_EQ(second.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
+ ASSERT_TRUE(second.ok()) << second.msg();
+
+ ASSERT_EQ(collect_infos.size(), 2U);
+ for (const auto& [field, collect_info] : collect_infos) {
+ EXPECT_EQ(collect_info.lucene_field, L"1");
+ ASSERT_NE(collect_info.index_meta, nullptr);
+ EXPECT_EQ(field,
+ segment_v2::match_statistics_field(L"1",
collect_info.index_meta->index_id()));
+ }
}
// Test-only subclass that exposes the protected helpers of PredicateCollector.
diff --git
a/be/test/storage/index/inverted/snii/snii_index_reader_count_fallback_test.cpp
b/be/test/storage/index/inverted/snii/snii_index_reader_count_fallback_test.cpp
index c057bc18914..f0e43fd80d1 100644
---
a/be/test/storage/index/inverted/snii/snii_index_reader_count_fallback_test.cpp
+++
b/be/test/storage/index/inverted/snii/snii_index_reader_count_fallback_test.cpp
@@ -917,7 +917,7 @@ TEST_F(SniiIndexReaderCountFallback,
PublicBooleanQueriesScoreOnlyFinalBitmapWit
EXPECT_EQ(all_stats->idf_terms, (std::vector<std::wstring> {L"alpha",
L"beta"}));
// Each term's query reads the average length and its idf.
EXPECT_EQ(all_stats->fields,
- (std::vector<std::wstring> {L"content", L"content", L"content",
L"content"}));
+ (std::vector<std::wstring>(4, match_statistics_field(L"content",
31))));
EXPECT_EQ(match_all.stats.inverted_index_query_cache_lookup, 0);
auto any_stats = std::make_shared<FixedCollectionStatistics>();
@@ -934,7 +934,7 @@ TEST_F(SniiIndexReaderCountFallback,
PublicBooleanQueriesScoreOnlyFinalBitmapWit
(std::vector<uint32_t> {0, 1, 2}));
EXPECT_EQ(any_stats->idf_terms, (std::vector<std::wstring> {L"alpha",
L"beta"}));
EXPECT_EQ(any_stats->fields,
- (std::vector<std::wstring> {L"content", L"content", L"content",
L"content"}));
+ (std::vector<std::wstring>(4, match_statistics_field(L"content",
31))));
EXPECT_EQ(match_any.stats.inverted_index_query_cache_lookup, 0);
}
diff --git
a/regression-test/data/inverted_index_p0/test_match_score_multi_index.out
b/regression-test/data/inverted_index_p0/test_match_score_multi_index.out
new file mode 100644
index 00000000000..ff1a95bee87
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_match_score_multi_index.out
@@ -0,0 +1,76 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !V2_english --
+1 0.336981
+2 0.432503
+3 0.27602
+
+-- !V2_standard --
+1 1.137496
+
+-- !V2_both_and --
+1 1.474477
+
+-- !V2_both_or --
+1 1.474477
+2 0.432503
+3 0.27602
+
+-- !V2_same_index --
+1 0.336981
+2 0.432503
+3 0.27602
+4 1.459936
+
+-- !V2_phrase_and_term --
+1 2.611973
+
+-- !V3_english --
+1 0.336981
+2 0.432503
+3 0.27602
+
+-- !V3_standard --
+1 1.137496
+
+-- !V3_both_and --
+1 1.474477
+
+-- !V3_both_or --
+1 1.474477
+2 0.432503
+3 0.27602
+
+-- !V3_same_index --
+1 0.336981
+2 0.432503
+3 0.27602
+4 1.459936
+
+-- !V3_phrase_and_term --
+1 2.611973
+
+-- !SNII_english --
+1 0.336981
+2 0.432503
+3 0.27602
+
+-- !SNII_standard --
+1 1.137496
+
+-- !SNII_both_and --
+1 1.474477
+
+-- !SNII_both_or --
+1 1.474477
+2 0.432503
+3 0.27602
+
+-- !SNII_same_index --
+1 0.336981
+2 0.432503
+3 0.27602
+4 1.459936
+
+-- !SNII_phrase_and_term --
+1 2.611973
+
diff --git
a/regression-test/suites/inverted_index_p0/test_match_score_multi_index.groovy
b/regression-test/suites/inverted_index_p0/test_match_score_multi_index.groovy
new file mode 100644
index 00000000000..cea3ef03634
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/test_match_score_multi_index.groovy
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// score() over MATCH clauses that read different inverted indexes of one
column. Each index has
+// its own terms and lengths: the english index below folds case, so apple has
df 3 there and df 1
+// in the standard index. A clause scores with the statistics of its own
index, so the combined
+// score is the sum of what each clause scores alone; such a query used to
fail with "Scoring
+// predicates for one field select different inverted indexes".
+suite("test_match_score_multi_index", "p0") {
+ // MATCH needs index execution even when CI fuzzes these session defaults.
+ sql "set enable_common_expr_pushdown = true"
+ sql "set enable_inverted_index_query = true"
+
+ for (def storageFormat : ["V2", "V3", "SNII"]) {
+ sql "DROP TABLE IF EXISTS test_match_score_multi_index"
+ sql """
+ CREATE TABLE test_match_score_multi_index (
+ id INT NOT NULL,
+ name STRING NULL,
+ INDEX idx_english(name) USING INVERTED
+ PROPERTIES("parser" = "english", "support_phrase" =
"true", "lower_case" = "true"),
+ INDEX idx_standard(name) USING INVERTED
+ PROPERTIES("parser" = "standard", "support_phrase" =
"true", "lower_case" = "false")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "inverted_index_storage_format" = "${storageFormat}"
+ )
+ """
+ sql """INSERT INTO test_match_score_multi_index VALUES
+ (1, 'apple pie'), (2, 'Apple'), (3, 'APPLE tart today'), (4,
'pear')
+ """
+ sql "sync"
+
+ for (def query : [
+ ["english", "name MATCH 'apple' USING ANALYZER english"],
+ ["standard", "name MATCH 'apple' USING ANALYZER standard"],
+ // Row 1 matches both clauses and scores their sum.
+ ["both_and", """name MATCH 'apple' USING ANALYZER english
+ AND name MATCH 'apple' USING ANALYZER
standard"""],
+ // Rows 2 and 3 match only the english clause and score as it
does alone.
+ ["both_or", """name MATCH 'apple' USING ANALYZER english
+ OR name MATCH 'apple' USING ANALYZER
standard"""],
+ // Two clauses on one index still share its statistics.
+ ["same_index", """name MATCH 'apple' USING ANALYZER english
+ OR name MATCH 'pear' USING ANALYZER
english"""],
+ ["phrase_and_term", """name MATCH_PHRASE 'apple pie' USING
ANALYZER english
+ AND name MATCH 'pie' USING ANALYZER
standard"""]]) {
+ quickTest("${storageFormat}_${query[0]}", """
+ SELECT id, ROUND(s, 6) FROM (
+ SELECT id, score() AS s FROM test_match_score_multi_index
+ WHERE ${query[1]} ORDER BY s DESC LIMIT 10
+ ) hits ORDER BY id
+ """, true)
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
