This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new b2eb3c90469 branch-4.2: [fix](be) Score regexp and wildcard patterns
with a constant like Elasticsearch #69815 (#69922)
b2eb3c90469 is described below
commit b2eb3c90469d4352670c7573b96f81a96b092e1c
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Nov 2 08:51:30 2026 +0800
branch-4.2: [fix](be) Score regexp and wildcard patterns with a constant
like Elasticsearch #69815 (#69922)
Cherry-picked from #69815
Co-authored-by: Jack <[email protected]>
---
be/src/storage/index/index_reader_helper.cpp | 10 ++
.../query/engine/expand_query/expand_query.h | 5 +-
.../query/engine/expand_query/expand_weight.cpp | 69 +-----------
.../query/engine/expand_query/expand_weight.h | 9 +-
.../index/inverted/reader/predicate_collector.cpp | 5 +
.../exprs/function/search/function_search_test.cpp | 23 ++--
.../inverted/query/engine/prefix_query_test.cpp | 23 ----
.../inverted/query/engine/regexp_query_test.cpp | 48 ++------
.../inverted/query/engine/wildcard_query_test.cpp | 54 ++-------
.../inverted/reader/collection_statistics_test.cpp | 46 ++++++++
.../storage/segment/index_reader_helper_test.cpp | 26 +++++
.../search/test_search_score_pattern_constant.out | 124 +++++++++++++++++++++
.../test_search_score_pattern_constant.groovy | 80 +++++++++++++
13 files changed, 325 insertions(+), 197 deletions(-)
diff --git a/be/src/storage/index/index_reader_helper.cpp
b/be/src/storage/index/index_reader_helper.cpp
index 51ed6baa91d..efd92d9b2d3 100644
--- a/be/src/storage/index/index_reader_helper.cpp
+++ b/be/src/storage/index/index_reader_helper.cpp
@@ -35,6 +35,12 @@ bool can_score_match(const TabletIndex* index_meta) {
bool IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType
query_type,
const TabletIndex*
index_meta) {
+ // A regexp scores a constant 1 per matching row, as an Elasticsearch
regexp query does in
+ // query context. The constant reads no statistics, norms or positions, so
every index can
+ // score it: body MATCH 'apple' AND body MATCH_REGEXP 'ap.*' adds 1 to the
apple score.
+ if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) {
+ return true;
+ }
return (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY ||
@@ -44,6 +50,10 @@ bool
IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType query_ty
bool IndexReaderHelper::is_need_similarity_score(TExprOpcode::type query_type,
const TabletIndex*
index_meta) {
+ // See the InvertedIndexQueryType overload: a regexp scores a constant on
every index.
+ if (query_type == TExprOpcode::MATCH_REGEXP) {
+ return true;
+ }
return (query_type == TExprOpcode::MATCH_ANY || query_type ==
TExprOpcode::MATCH_ALL ||
query_type == TExprOpcode::MATCH_PHRASE ||
query_type == TExprOpcode::MATCH_PHRASE_PREFIX) &&
diff --git
a/be/src/storage/index/inverted/query/engine/expand_query/expand_query.h
b/be/src/storage/index/inverted/query/engine/expand_query/expand_query.h
index 155acf16c73..45816618fc9 100644
--- a/be/src/storage/index/inverted/query/engine/expand_query/expand_query.h
+++ b/be/src/storage/index/inverted/query/engine/expand_query/expand_query.h
@@ -38,8 +38,9 @@ public:
_pattern(std::move(pattern)) {}
~ExpandQuery() override = default;
- WeightPtr weight(bool enable_scoring) override {
- return std::make_shared<ExpandWeight>(_context, _field, _kind,
_pattern, enable_scoring);
+ // Scores are constant, so scoring changes nothing.
+ WeightPtr weight(bool /*enable_scoring*/) override {
+ return std::make_shared<ExpandWeight>(_context, _field, _kind,
_pattern);
}
private:
diff --git
a/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.cpp
b/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.cpp
index 85fd10a51d0..3c55f171ec4 100644
--- a/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.cpp
+++ b/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.cpp
@@ -17,7 +17,6 @@
#include "storage/index/inverted/query/engine/expand_query/expand_weight.h"
-#include <cmath>
#include <memory>
#include <roaring/roaring.hh>
#include <vector>
@@ -26,14 +25,10 @@
#include "storage/index/inverted/query/engine/bit_set_query/bit_set_scorer.h"
#include
"storage/index/inverted/query/engine/const_score_query/const_score_scorer.h"
#include "storage/index/inverted/query/engine/nullable_scorer.h"
-#include "storage/index/inverted/query/engine/score_combiner.h"
-#include "storage/index/inverted/query/engine/term_query/term_scorer.h"
-#include "storage/index/inverted/query/engine/union/buffered_union.h"
#include "storage/index/inverted/query/exec/block_doc_set.h"
#include "storage/index/inverted/query/exec/collect_postings.h"
#include "storage/index/inverted/query/exec/roaring_docid_sink.h"
#include "storage/index/inverted/query/exec/term_waves.h"
-#include "storage/index/inverted/query/scoring/bm25_similarity.h"
#include "storage/index/inverted/query/spi/postings_cursor.h"
namespace doris::segment_v2::inverted_index::query_v2 {
@@ -63,15 +58,11 @@ Status collect_expanded_rows(index_query::IndexSource&
source, index_query::Term
}
ExpandWeight::ExpandWeight(IndexQueryContextPtr context, std::wstring field,
- index_query::TermPatternKind kind, std::string
pattern,
- bool enable_scoring)
+ index_query::TermPatternKind kind, std::string
pattern)
: _context(std::move(context)),
_field(std::move(field)),
_kind(kind),
- _pattern(std::move(pattern)),
- // Prefix and phrase-edge expansions retain their constant-score
behavior.
- _enable_scoring(enable_scoring && (kind ==
index_query::TermPatternKind::kRegexp ||
- kind ==
index_query::TermPatternKind::kWildcard)) {}
+ _pattern(std::move(pattern)) {}
std::shared_ptr<roaring::Roaring> ExpandWeight::_rows(const
QueryExecutionContext& context,
const std::string&
binding_key,
@@ -94,68 +85,12 @@ std::shared_ptr<roaring::Roaring> ExpandWeight::_rows(const
QueryExecutionContex
ScorerPtr ExpandWeight::scorer(const QueryExecutionContext& context,
const std::string& binding_key) {
- // The listing fast path discards term frequencies, so scored patterns
need term scorers.
- if (_enable_scoring) {
- return make_nullable_scorer(_scored_terms(context, binding_key),
- logical_field_or_fallback(context,
binding_key, _field),
- context.null_resolver);
- }
auto scorer = std::make_shared<ConstScoreScorer<BitSetScorerPtr>>(
std::make_shared<BitSetScorer>(_rows(context, binding_key,
nullptr)));
return make_nullable_scorer(scorer, logical_field_or_fallback(context,
binding_key, _field),
context.null_resolver);
}
-ScorerPtr ExpandWeight::_scored_terms(const QueryExecutionContext& context,
- const std::string& binding_key) {
- index_query::TermPattern pattern;
- THROW_IF_ERROR(index_query::TermPattern::create(_kind, _pattern,
&pattern));
- auto source = lookup_source(_field, context, binding_key);
- // An unbound field has no postings and contributes no matches.
- if (source == nullptr) {
- return std::make_shared<EmptyScorer>();
- }
- std::vector<std::string> terms;
- THROW_IF_ERROR(source->expand_terms(
- pattern, index_query::expansion_limit(_kind,
index_query::max_expansions(*_context)),
- &terms));
- // A valid pattern may match no dictionary terms.
- if (terms.empty()) {
- return std::make_shared<EmptyScorer>();
- }
- // Expansion terms are absent from collection statistics; use the bound
source's statistics.
- index_query::IndexScoringStats stats;
- THROW_IF_ERROR(source->scoring_stats(&stats));
- DORIS_CHECK_GT(stats.doc_count, 0);
- // Docs-only SNII still reports its physical token count. Only legacy
sources without
- // that metadata report zero, so use unit average length only for those
sources.
- const uint64_t token_count = stats.token_count != 0 ? stats.token_count :
stats.doc_count;
- const float avgdl = static_cast<float>(token_count) / stats.doc_count;
- DORIS_CHECK_GT(avgdl, 0.0F);
- std::vector<ScorerPtr> scorers;
- scorers.reserve(terms.size());
- auto logical_field = logical_field_or_fallback(context, binding_key,
_field);
- for (const auto& term : terms) {
- uint64_t doc_freq = 0;
- THROW_IF_ERROR(source->doc_freq(term, &doc_freq));
- // The immutable source just enumerated this term and must have its
postings.
- DCHECK_GT(doc_freq, 0);
- const auto idf = static_cast<float>(std::log1p(
- (static_cast<double>(stats.doc_count) - doc_freq + 0.5) /
(doc_freq + 0.5)));
- auto similarity = std::make_shared<BM25Similarity>(idf, avgdl);
- auto postings = open_postings(*source, term, /*positions=*/false,
/*enable_scoring=*/true,
- similarity);
- DCHECK(postings != nullptr);
- scorers.emplace_back(std::make_shared<TermScorer>(std::move(postings),
-
std::move(similarity), logical_field));
- }
- // One expanded term needs no union; multiple terms add their per-document
BM25 scores.
- if (scorers.size() == 1) {
- return std::move(scorers.front());
- }
- return make_buffered_union(scorers, std::make_shared<SumCombiner>());
-}
-
index_query::TruthSet ExpandWeight::listed_rows(const QueryExecutionContext&
context,
const std::string& binding_key,
const roaring::Roaring*
candidates) {
diff --git
a/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.h
b/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.h
index 3bbe43531c3..b5e979aca5d 100644
--- a/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.h
+++ b/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.h
@@ -29,25 +29,23 @@ Status collect_expanded_rows(index_query::IndexSource&
source, index_query::Term
int32_t max_expansions, const roaring::Roaring*
candidates,
roaring::Roaring* rows);
-// Every document that holds an expanded term; scored regexp and wildcard
clauses sum BM25.
+// Every document that holds a term the pattern expands to, with a constant
score.
class ExpandWeight : public Weight {
public:
ExpandWeight(IndexQueryContextPtr context, std::wstring field,
- index_query::TermPatternKind kind, std::string pattern,
- bool enable_scoring = false);
+ index_query::TermPatternKind kind, std::string pattern);
~ExpandWeight() override = default;
ScorerPtr scorer(const QueryExecutionContext& context, const std::string&
binding_key) override;
bool lists_rows(const QueryExecutionContext&, const std::string&) const
override {
- return !_enable_scoring;
+ return true;
}
index_query::TruthSet listed_rows(const QueryExecutionContext& context,
const std::string& binding_key,
const roaring::Roaring* candidates)
override;
private:
- ScorerPtr _scored_terms(const QueryExecutionContext& context, const
std::string& binding_key);
std::shared_ptr<roaring::Roaring> _rows(const QueryExecutionContext&
context,
const std::string& binding_key,
const roaring::Roaring*
candidates);
@@ -55,7 +53,6 @@ private:
std::wstring _field;
index_query::TermPatternKind _kind;
std::string _pattern;
- bool _enable_scoring;
};
} // namespace doris::segment_v2::inverted_index::query_v2
diff --git a/be/src/storage/index/inverted/reader/predicate_collector.cpp
b/be/src/storage/index/inverted/reader/predicate_collector.cpp
index 376e7af5604..3d0b4fbbed6 100644
--- a/be/src/storage/index/inverted/reader/predicate_collector.cpp
+++ b/be/src/storage/index/inverted/reader/predicate_collector.cpp
@@ -369,6 +369,11 @@ Status MatchPredicateCollector::collect(RuntimeState*
state, const TabletSchemaS
!IndexReaderHelper::is_need_similarity_score(expr->op(), index_meta)) {
return Status::OK();
}
+ // A regexp scores a constant and reads no statistics. Its value is a
pattern, not text, so
+ // analyzing it would collect statistics for made-up terms: 'ap.*' would
become "ap".
+ if (expr->op() == TExprOpcode::MATCH_REGEXP) {
+ return Status::OK();
+ }
DORIS_CHECK(analyzer_ctx->analyzer_provider != nullptr);
auto options = DataTypeSerDe::get_default_format_options();
diff --git a/be/test/exprs/function/search/function_search_test.cpp
b/be/test/exprs/function/search/function_search_test.cpp
index cdee045e3c2..589104c3cc9 100644
--- a/be/test/exprs/function/search/function_search_test.cpp
+++ b/be/test/exprs/function/search/function_search_test.cpp
@@ -40,7 +40,6 @@
#include <gtest/gtest.h>
#include <chrono>
-#include <cmath>
#include <initializer_list>
#include <map>
#include <memory>
@@ -1888,8 +1887,6 @@ TEST_F(FunctionSearchTest,
TestBuildLeafQueryExecutesSelectedSniiWildcardReader)
std::make_shared<RecordingNativeInvertedIndexReader>(&decoy_meta,
decoy_file_reader);
auto selected_reader =
std::make_shared<RecordingNativeInvertedIndexReader>(
&selected_meta, selected_file_reader);
- selected_reader->fake_source->set_doc_count(4);
- selected_reader->fake_source->set_scoring_stats({.doc_count = 4,
.token_count = 3});
selected_reader->fake_source->add("alpha", {0, 2});
selected_reader->fake_source->add("gamma", {1});
decoy_reader->fake_source->add("alpha", {1});
@@ -1938,8 +1935,7 @@ TEST_F(FunctionSearchTest,
TestBuildLeafQueryExecutesSelectedSniiWildcardReader)
binding_key);
ASSERT_NE(nullptr, scorer);
EXPECT_EQ(0U, scorer->doc());
- // The source has three tokens over four rows, even without stored norms.
- EXPECT_NEAR(0.88F * std::log(2.0F), scorer->score(), 1e-6F);
+ EXPECT_FLOAT_EQ(1.0F, scorer->score());
expect_bitmap_eq(collect_docs(scorer), {0, 2});
// The pattern was lowercased by the selected analyzer and enumerated on
the selected
// index only; a leading wildcard has no prefix to enumerate from.
@@ -2533,8 +2529,7 @@ static void search_snii_pattern(const
std::map<std::string, std::string>& proper
ASSERT_TRUE(status.ok()) << status.to_string();
EXPECT_EQ(0, reader->query_calls) << value;
// The scorer enumerates the stem on the source; the reader itself is
never asked.
- // This helper checks normalization and routing, without requesting
scoring statistics.
- auto weight = query->weight(false);
+ auto weight = query->weight(true);
ASSERT_NE(nullptr, weight);
auto scorer =
weight->scorer(build_variant_search_query_execution_context(4, resolver,
nullptr),
binding_key);
@@ -2767,12 +2762,13 @@ TEST_F(FunctionSearchTest,
TestSniiScoredLeafRanksByTheEngineBm25) {
}
}
-// Scored wildcard leaves read frequencies and statistics through the unified
source,
-// even when the old reader query API publishes no per-document scores.
-TEST_F(FunctionSearchTest, TestSniiNativeWildcardUsesSourceScoringStatistics) {
+// A leaf for which the reader publishes no per-document score must keep the
constant that the
+// CLucene path also gives its own constant-score leaves, so that routing
scored clauses through a
+// new scorer does not quietly re-rank the unscored ones. The fake reader is
what withholds the
+// scores here, so this pins the "nothing published -> BitSetQuery -> 1.0"
behaviour; it does not
+// pin which query types the production is_need_similarity_score gate rejects.
+TEST_F(FunctionSearchTest,
TestSniiNativeLeafWithoutPublishedScoresKeepsConstantScore) {
SniiScoringFixture fixture(43, 4);
- fixture.reader->fake_source->set_doc_count(4);
- fixture.reader->fake_source->set_scoring_stats({.doc_count = 4,
.token_count = 2});
fixture.reader->fake_source->add("alpha", {0, 2});
inverted_index::query_v2::QueryPtr query;
@@ -2789,8 +2785,7 @@ TEST_F(FunctionSearchTest,
TestSniiNativeWildcardUsesSourceScoringStatistics) {
auto scorer = weight->scorer(exec_ctx, binding_key);
ASSERT_NE(nullptr, scorer);
EXPECT_EQ(0U, scorer->doc());
- // avgdl = 2/4 and implicit length = 1: BM25 is idf * 2.2 / 3.1.
- EXPECT_NEAR((2.2F / 3.1F) * std::log(2.0F), scorer->score(), 1e-6F);
+ EXPECT_FLOAT_EQ(1.0F, scorer->score());
expect_bitmap_eq(collect_docs(scorer), {0, 2});
}
diff --git a/be/test/storage/index/inverted/query/engine/prefix_query_test.cpp
b/be/test/storage/index/inverted/query/engine/prefix_query_test.cpp
index e711deb3ed9..4d68efdf47a 100644
--- a/be/test/storage/index/inverted/query/engine/prefix_query_test.cpp
+++ b/be/test/storage/index/inverted/query/engine/prefix_query_test.cpp
@@ -18,7 +18,6 @@
#include <CLucene.h>
#include <gtest/gtest.h>
-#include <cmath>
#include <memory>
#include <roaring/roaring.hh>
#include <string>
@@ -33,8 +32,6 @@
#include "storage/index/inverted/inverted_index_reader.h"
#include "storage/index/inverted/query/engine/expand_query/expand_query.h"
#include "storage/index/inverted/query/logical/node.h"
-#include "storage/index/inverted/query/scoring/bm25_similarity.h"
-#include "storage/index/inverted/query/scoring/collection_similarity.h"
#include "storage/index/inverted/query/spi/fake_index_source.h"
#include "storage/olap_common.h"
@@ -495,8 +492,6 @@ TEST(ExpansionLeafTest,
KeepsPatternLimitsAndLeavesRootCandidatesToTheCaller) {
auto source =
std::make_shared<index_query::testing::FakeIndexSource>();
source->batches = batches;
source->set_doc_count(10);
- // Scored regexp and wildcard leaves need statistics from their bound
source.
- source->set_scoring_stats({.doc_count = 10, .token_count = 10});
source->add("apple", {0, 4});
source->add("application", {1});
source->add("apply", {2, 4});
@@ -514,30 +509,12 @@ TEST(ExpansionLeafTest,
KeepsPatternLimitsAndLeavesRootCandidatesToTheCaller) {
expected.add(2);
}
for (const bool scoring : {false, true}) {
- context->collection_similarity =
- scoring ? std::make_shared<CollectionSimilarity>() :
nullptr;
auto result = std::make_shared<roaring::Roaring>();
result->add(9);
const auto status = run_leaf(context, L"content", leaf,
&candidates, scoring,
source, source->doc_count(),
result);
ASSERT_TRUE(status.ok()) << status;
EXPECT_EQ(*result, expected) << pattern << " scoring=" <<
scoring;
- if (scoring) {
- const auto& scores =
context->collection_similarity->_bm25_scores;
- EXPECT_EQ(scores.size(), expected.cardinality() - 1);
- const bool bm25 = kind == logical::ExpandKind::kWildcard ||
- kind == logical::ExpandKind::kRegexp;
- for (const auto& [doc, score] : scores) {
- // Row 9 predates this leaf and must not acquire a
score from it.
- EXPECT_NE(doc, 9U);
- EXPECT_TRUE(expected.contains(doc));
- const float df = doc == 1 ? 1.0F : 2.0F;
- const float idf = std::log1p((10.0F - df + 0.5F) / (df
+ 0.5F));
- const float expected_score =
- bm25 ? BM25Similarity(idf, 1.0F).score(1.0F,
1) : 1.0F;
- EXPECT_FLOAT_EQ(score, expected_score) << pattern << "
doc=" << doc;
- }
- }
}
}
}
diff --git a/be/test/storage/index/inverted/query/engine/regexp_query_test.cpp
b/be/test/storage/index/inverted/query/engine/regexp_query_test.cpp
index 9695f3da776..6de9c997661 100644
--- a/be/test/storage/index/inverted/query/engine/regexp_query_test.cpp
+++ b/be/test/storage/index/inverted/query/engine/regexp_query_test.cpp
@@ -49,15 +49,13 @@ public:
st = io::global_local_filesystem()->create_directory(kTestDir);
ASSERT_TRUE(st.ok()) << st;
std::string field_name = "content";
- create_test_index(field_name, kTestDir,
- {"apple123", "apple456", "banana789",
- "test123abc", "pattern456", "regex999",
- "match123", "search456", "prefix123",
- "suffix789", "apple_test", "banana_data",
- "test_pattern", "data_regex", "apple_banana",
- "test_match", "pattern_test", "prefix_suffix",
- "abc123xyz", "def456ghi", "apple789 apple789
apple456"},
- "standard");
+ create_test_index(
+ field_name, kTestDir,
+ {"apple123", "apple456", "banana789", "test123abc",
"pattern456",
+ "regex999", "match123", "search456", "prefix123",
"suffix789",
+ "apple_test", "banana_data", "test_pattern", "data_regex",
"apple_banana",
+ "test_match", "pattern_test", "prefix_suffix", "abc123xyz",
"def456ghi"},
+ "standard");
}
void TearDown() override {
@@ -181,36 +179,6 @@ TEST_F(RegexpQueryV2Test, test_regexp_query_with_scoring) {
ASSERT_NE(regexp_weight, nullptr);
}
-TEST_F(RegexpQueryV2Test, test_regexp_query_scores_matching_terms) {
- auto context = std::make_shared<IndexQueryContext>();
- context->collection_statistics = std::make_shared<CollectionStatistics>();
- context->collection_similarity = std::make_shared<CollectionSimilarity>();
-
- auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
- auto reader_holder =
make_shared_reader(lucene::index::IndexReader::open(dir, true));
- ASSERT_NE(reader_holder, nullptr);
-
- std::wstring field = StringHelper::to_wstring("content");
- auto query = std::make_shared<query_v2::ExpandQuery>(
- context, field, index_query::TermPatternKind::kRegexp,
anchored("apple.*"));
- auto weight = query->weight(true);
-
- query_v2::QueryExecutionContext exec_ctx;
- exec_ctx.segment_num_rows = reader_holder->maxDoc();
- exec_ctx.sources = {clucene_index_source(reader_holder, field, nullptr)};
- exec_ctx.field_sources.emplace(field, exec_ctx.sources.front());
-
- auto scorer = weight->scorer(exec_ctx);
- ASSERT_EQ(scorer->seek(0), 0);
- float single_term_score = scorer->score();
- EXPECT_NE(single_term_score, 1.0F);
- uint32_t last_doc = reader_holder->maxDoc() - 1;
- ASSERT_EQ(scorer->seek(last_doc), last_doc);
- EXPECT_GT(scorer->score(), single_term_score);
-
- _CLDECDELETE(dir);
-}
-
// Test regexp query execution
TEST_F(RegexpQueryV2Test, test_regexp_query_execution) {
auto context = std::make_shared<IndexQueryContext>();
@@ -659,7 +627,7 @@ TEST_F(RegexpQueryV2Test, test_whole_term_match_any_term) {
doc = scorer->advance();
}
- EXPECT_EQ(result.cardinality(), 21);
+ EXPECT_EQ(result.cardinality(), 20);
_CLDECDELETE(dir);
}
diff --git
a/be/test/storage/index/inverted/query/engine/wildcard_query_test.cpp
b/be/test/storage/index/inverted/query/engine/wildcard_query_test.cpp
index 1822c79e9e5..15d1b143ad4 100644
--- a/be/test/storage/index/inverted/query/engine/wildcard_query_test.cpp
+++ b/be/test/storage/index/inverted/query/engine/wildcard_query_test.cpp
@@ -58,20 +58,14 @@ public:
private:
void create_test_index(const std::string& field_name, const std::string&
dir) {
- std::vector<std::string> test_data = {"apple", "application",
"apply",
- "apricot", "banana",
"band",
- "bandana", "can",
"candy",
- "cat", "catalog",
"dog",
- "document", "door",
"hello",
- "help", "helpful",
"test",
- "testing", "testcase",
"wildcard",
- "wild", "wilderness",
"card",
- "cardboard", "abc",
"abcd",
- "abcde", "prefix123",
"123suffix",
- "pre123", "123suf",
"both123both",
- "star", "start",
"started",
- "starter", "question",
"quest",
- "query", "queries",
"test test testing"};
+ std::vector<std::string> test_data = {
+ "apple", "application", "apply", "apricot",
"banana", "band",
+ "bandana", "can", "candy", "cat",
"catalog", "dog",
+ "document", "door", "hello", "help",
"helpful", "test",
+ "testing", "testcase", "wildcard", "wild",
"wilderness", "card",
+ "cardboard", "abc", "abcd", "abcde",
"prefix123", "123suffix",
+ "pre123", "123suf", "both123both", "star", "start",
"started",
+ "starter", "question", "quest", "query",
"queries"};
CustomAnalyzerConfig::Builder builder;
builder.with_tokenizer_config("standard", {});
@@ -164,36 +158,6 @@ TEST_F(WildcardQueryV2Test,
test_wildcard_query_with_scoring) {
ASSERT_NE(wildcard_weight, nullptr);
}
-TEST_F(WildcardQueryV2Test, test_wildcard_query_scores_matching_terms) {
- auto context = std::make_shared<IndexQueryContext>();
- context->collection_statistics = std::make_shared<CollectionStatistics>();
- context->collection_similarity = std::make_shared<CollectionSimilarity>();
-
- auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
- auto reader_holder =
make_shared_reader(lucene::index::IndexReader::open(dir, true));
- ASSERT_NE(reader_holder, nullptr);
-
- std::wstring field = StringHelper::to_wstring("field");
- auto query = std::make_shared<query_v2::ExpandQuery>(
- context, field, index_query::TermPatternKind::kWildcard, "test*");
- auto weight = query->weight(true);
-
- query_v2::QueryExecutionContext exec_ctx;
- exec_ctx.segment_num_rows = reader_holder->maxDoc();
- exec_ctx.sources = {clucene_index_source(reader_holder, field, nullptr)};
- exec_ctx.field_sources.emplace(field, exec_ctx.sources.front());
-
- auto scorer = weight->scorer(exec_ctx);
- ASSERT_EQ(scorer->seek(17), 17);
- float single_term_score = scorer->score();
- EXPECT_NE(single_term_score, 1.0F);
- uint32_t last_doc = reader_holder->maxDoc() - 1;
- ASSERT_EQ(scorer->seek(last_doc), last_doc);
- EXPECT_GT(scorer->score(), single_term_score);
-
- _CLDECDELETE(dir);
-}
-
// Test wildcard query with asterisk prefix pattern
TEST_F(WildcardQueryV2Test, test_wildcard_query_prefix_pattern) {
auto context = std::make_shared<IndexQueryContext>();
@@ -575,4 +539,4 @@ TEST_F(WildcardQueryV2Test,
test_wildcard_query_move_semantics) {
ASSERT_NE(weight2, nullptr);
}
-} // namespace doris::segment_v2
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git
a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
index afabc73b9d7..8eef09fc51b 100644
--- a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
+++ b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
@@ -2515,6 +2515,52 @@ TEST_F(CollectionStatisticsTest,
CollectSkipsIndexWithoutSimilarityScore) {
EXPECT_TRUE(collect_infos.empty());
}
+// A regexp scores a constant, so it collects no statistics even on an index
whose MATCH_ANY does:
+// analyzing its pattern 'ap.*' would register the made-up term "ap".
+TEST_F(CollectionStatisticsTest, RegexpCollectsNoStatistics) {
+ auto tablet_schema = std::make_shared<TabletSchema>();
+
+ constexpr int32_t kColUid = 1360;
+ TabletColumn col;
+ col.set_unique_id(kColUid);
+ col.set_name("body");
+ col.set_type(FieldType::OLAP_FIELD_TYPE_STRING);
+ tablet_schema->append_column(col);
+
+ TabletIndexPB index_pb;
+ index_pb.set_index_id(2360);
+ index_pb.set_index_name("body_idx");
+ index_pb.set_index_type(IndexType::INVERTED);
+ index_pb.add_col_unique_id(kColUid);
+ (*index_pb.mutable_properties())["parser"] = "english";
+ (*index_pb.mutable_properties())["support_phrase"] = "true";
+ TabletIndex index;
+ index.init_from_pb(index_pb);
+ tablet_schema->append_index(std::move(index));
+
+ constexpr int kSlotId = 86;
+ runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId),
kColUid, "body", {});
+
+ auto match_expr =
std::make_shared<collection_statistics::MockVExpr>(TExprNodeType::MATCH_PRED);
+ match_expr->_children.push_back(
+ std::make_shared<collection_statistics::MockVSlotRef>("body",
SlotId(kSlotId)));
+
match_expr->_children.push_back(std::make_shared<collection_statistics::MockVLiteral>("ap.*"));
+
+ MatchPredicateCollector collector;
+ std::unordered_map<std::wstring, CollectInfo> collect_infos;
+ match_expr->set_opcode(TExprOpcode::MATCH_ANY);
+ auto status =
+ collector.collect(runtime_state_.get(), tablet_schema, match_expr,
&collect_infos);
+ ASSERT_TRUE(status.ok()) << status.msg();
+ EXPECT_EQ(collect_infos.size(), 1U);
+
+ collect_infos.clear();
+ match_expr->set_opcode(TExprOpcode::MATCH_REGEXP);
+ status = collector.collect(runtime_state_.get(), tablet_schema,
match_expr, &collect_infos);
+ ASSERT_TRUE(status.ok()) << status.msg();
+ EXPECT_TRUE(collect_infos.empty());
+}
+
TEST_F(CollectionStatisticsTest,
CollectPreservesLogicalClauseShapesForSameFieldName) {
auto tablet_schema = std::make_shared<TabletSchema>();
diff --git a/be/test/storage/segment/index_reader_helper_test.cpp
b/be/test/storage/segment/index_reader_helper_test.cpp
index 509a5eb4067..2fb699529ec 100644
--- a/be/test/storage/segment/index_reader_helper_test.cpp
+++ b/be/test/storage/segment/index_reader_helper_test.cpp
@@ -421,6 +421,32 @@ TEST_F(IndexReaderHelperTest,
IsNeedSimilarityScoreWithTExprOpcodeTest) {
EXPECT_FALSE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::LT,
&index_meta));
}
+TEST_F(IndexReaderHelperTest, RegexpScoresConstantOnEveryIndex) {
+ // A regexp scores a constant 1 and reads no statistics, so neither the
analyzer nor phrase
+ // support decides whether it scores.
+ for (const auto& properties : std::vector<std::map<std::string,
std::string>> {
+ {},
+ {{"parser", "none"}},
+ {{"parser", "english"}},
+ {{"parser", "english"}, {"support_phrase", "true"}},
+ {{"normalizer", "split_norm"}}}) {
+ SCOPED_TRACE(::testing::PrintToString(properties));
+ TabletIndexPB pb;
+ pb.set_index_id(1);
+ pb.set_index_name("regexp_index");
+ pb.set_index_type(IndexType::INVERTED);
+ pb.add_col_unique_id(1);
+ for (const auto& [key, value] : properties) {
+ (*pb.mutable_properties())[key] = value;
+ }
+ TabletIndex index;
+ index.init_from_pb(pb);
+
EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::MATCH_REGEXP,
&index));
+ EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(
+ InvertedIndexQueryType::MATCH_REGEXP_QUERY, &index));
+ }
+}
+
TEST_F(IndexReaderHelperTest, MatchScoringRequiresAnalyzedPhraseIndex) {
struct Case {
std::map<std::string, std::string> properties;
diff --git a/regression-test/data/search/test_search_score_pattern_constant.out
b/regression-test/data/search/test_search_score_pattern_constant.out
new file mode 100644
index 00000000000..e8603f53e4f
--- /dev/null
+++ b/regression-test/data/search/test_search_score_pattern_constant.out
@@ -0,0 +1,124 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !V2_search_regexp --
+2 1
+3 1
+6 1
+
+-- !V2_search_wildcard --
+2 1
+3 1
+6 1
+
+-- !V2_search_prefix --
+2 1
+3 1
+6 1
+
+-- !V2_search_keyword_regexp --
+6 1
+7 1
+
+-- !V2_match_regexp --
+2 1
+3 1
+6 1
+
+-- !V2_match_regexp_keyword --
+6 1
+7 1
+
+-- !V2_match_and_match_regexp --
+2 2.023573
+
+-- !V2_search_and_regexp --
+2 2.023573
+
+-- !V2_match_only --
+2 1.023573
+5 1.163151
+
+-- !V2_match_and_equality --
+2 1.023573
+
+-- !V3_search_regexp --
+2 1
+3 1
+6 1
+
+-- !V3_search_wildcard --
+2 1
+3 1
+6 1
+
+-- !V3_search_prefix --
+2 1
+3 1
+6 1
+
+-- !V3_search_keyword_regexp --
+6 1
+7 1
+
+-- !V3_match_regexp --
+2 1
+3 1
+6 1
+
+-- !V3_match_regexp_keyword --
+6 1
+7 1
+
+-- !V3_match_and_match_regexp --
+2 2.023573
+
+-- !V3_search_and_regexp --
+2 2.023573
+
+-- !V3_match_only --
+2 1.023573
+5 1.163151
+
+-- !V3_match_and_equality --
+2 1.023573
+
+-- !SNII_search_regexp --
+2 1
+3 1
+6 1
+
+-- !SNII_search_wildcard --
+2 1
+3 1
+6 1
+
+-- !SNII_search_prefix --
+2 1
+3 1
+6 1
+
+-- !SNII_search_keyword_regexp --
+6 1
+7 1
+
+-- !SNII_match_regexp --
+2 1
+3 1
+6 1
+
+-- !SNII_match_regexp_keyword --
+6 1
+7 1
+
+-- !SNII_match_and_match_regexp --
+2 2.023573
+
+-- !SNII_search_and_regexp --
+2 2.023573
+
+-- !SNII_match_only --
+2 1.023573
+5 1.163151
+
+-- !SNII_match_and_equality --
+2 1.023573
+
diff --git
a/regression-test/suites/search/test_search_score_pattern_constant.groovy
b/regression-test/suites/search/test_search_score_pattern_constant.groovy
new file mode 100644
index 00000000000..f3cf9615850
--- /dev/null
+++ b/regression-test/suites/search/test_search_score_pattern_constant.groovy
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// score() of pattern queries follows Elasticsearch's default rewrite: a
regexp, wildcard or
+// prefix in SEARCH and a MATCH_REGEXP score a constant 1 per matching row,
whatever segment the
+// row is in, and add exactly 1 next to a scored clause. An equality predicate
adds nothing.
+suite("test_search_score_pattern_constant", "p0") {
+ // SEARCH and MATCH need index execution even when CI fuzzes these session
defaults.
+ sql "set enable_common_expr_pushdown = true"
+ sql "set enable_inverted_index_query = true"
+
+ for (def storageFormat : ["V2", "V3", "SNII"]) {
+ sql "DROP TABLE IF EXISTS test_search_score_pattern_constant"
+ sql """
+ CREATE TABLE test_search_score_pattern_constant (
+ id BIGINT NOT NULL,
+ content STRING NULL,
+ tag STRING NULL,
+ INDEX idx_content(content) USING INVERTED
+ PROPERTIES("parser" = "english", "support_phrase" =
"true"),
+ INDEX idx_tag(tag) USING INVERTED
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "inverted_index_storage_format" = "${storageFormat}",
+ "disable_auto_compaction" = "true"
+ )
+ """
+ // Two inserts make two segments with different term statistics; a
constant score must not
+ // depend on them.
+ sql """INSERT INTO test_search_score_pattern_constant VALUES
+ (1, 'Apache Doris is a high performance analytical database',
'doris'),
+ (2, 'Inverted indexes are suitable for full text search', 'index'),
+ (3, 'The parser splits text into multiple terms', 'parser'),
+ (4, 'Doris supports fast real time data analysis', 'doris'),
+ (5, 'Users can quickly search product names', 'search')
+ """
+ sql """INSERT INTO test_search_score_pattern_constant VALUES
+ (6, 'text text text', 'text'), (7, 'plain words only', 'textual')
+ """
+ sql "sync"
+
+ for (def query : [
+ ["search_regexp", "SEARCH('content:/.*ext/')"],
+ ["search_wildcard", "SEARCH('content:*ext')"],
+ ["search_prefix", "SEARCH('content:te*')"],
+ ["search_keyword_regexp", "SEARCH('tag:/text.*/')"],
+ ["match_regexp", "content MATCH_REGEXP 'ext'"],
+ ["match_regexp_keyword", "tag MATCH_REGEXP 'text'"],
+ // A pattern next to a scored clause adds exactly 1, as an
Elasticsearch bool must
+ // with a regexp clause does.
+ ["match_and_match_regexp", "content MATCH 'search' AND content
MATCH_REGEXP 'ext'"],
+ ["search_and_regexp", "SEARCH('content:search AND
content:/.*ext/')"],
+ ["match_only", "content MATCH 'search'"],
+ ["match_and_equality", "content MATCH 'search' AND tag =
'index'"]]) {
+ quickTest("${storageFormat}_${query[0]}", """
+ SELECT id, ROUND(s, 6) FROM (
+ SELECT id, score() AS s FROM
test_search_score_pattern_constant
+ WHERE ${query[1]} ORDER BY s DESC LIMIT 10
+ ) hits ORDER BY id
+ """, true)
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
