This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 126771d7bc0 [fix](inverted index) Align phrase position defaults on
4.1 (#69190)
126771d7bc0 is described below
commit 126771d7bc018c3147c01cf6466cc3a8c137d372
Author: Jack <[email protected]>
AuthorDate: Sun Oct 18 11:35:19 2026 +0800
[fix](inverted index) Align phrase position defaults on 4.1 (#69190)
### What problem does this PR solve?
Issue Number: CIR-22263
Related PR: #69199
Problem Summary: FE persists `support_phrase=true` for new tokenized
indexes and leaves exact indexes docs-only. FE and BE retain the
docs-only interpretation of an omitted property on older persisted
analyzer indexes, preserving their postings and `TOKENIZE` output.
Built-in `lowercase` normalization emits one term, so phrase predicates
can use docs-only postings and BM25 can use unit frequency and document
length. Custom normalizers can emit multiple terms, so new indexes
retain positions by default, including when a legacy custom policy
shadows the built-in name.
This is the `branch-4.1` adaptation of #69199. It uses the branch's
CLucene scoring path and the keyword-index scoring support already
merged into `branch-4.1`. This branch has no SNII streamed compaction
path from #69192, so this PR does not eliminate docs-only rebuilds or
the tokenized T2 merge hotspot in CIR-22263.
### Release note
New exact and built-in lowercase-normalized indexes default to docs-only
postings. Custom normalizers retain phrase positions by default.
Lowercase-normalized indexes retain phrase matching and relevance
scoring without stored positions.
### Check List (For Author)
- Test
- [x] Unit Test: two targeted FE default and policy-binding tests and
eight BE collection-statistics, query-scoring, postings, and reader
tests on the current target branch.
- [x] Complete PR diff passed the English description check,
clang-format 16, and Checkstyle on the merged target tree.
- Behavior changed:
- [x] Yes. Exact indexes omit positions by default; custom normalizers
retain positions; built-in lowercase normalization keeps phrase and
scoring behavior with docs-only postings.
- Does this need documentation?
- [x] No.
Remote checks on the updated head are pending.
---
.../storage/compaction/collection_statistics.cpp | 10 ++-
be/src/storage/index/index_reader_helper.h | 25 +++++-
.../storage/index/inverted/query/query_helper.cpp | 18 +++--
.../compaction/collection_statistics_test.cpp | 92 ++++++++++++++++++++++
.../index/inverted/query/query_helper_test.cpp | 65 ++++++++++++++-
.../storage/index/inverted_index_parser_test.cpp | 24 +++++-
.../storage/segment/index_reader_helper_test.cpp | 43 ++++++++++
.../apache/doris/analysis/InvertedIndexUtil.java | 15 +++-
.../main/java/org/apache/doris/catalog/Index.java | 11 ++-
.../analysis/InvertedIndexPropertiesTest.java | 5 +-
.../java/org/apache/doris/catalog/IndexTest.java | 51 ++++++++++++
.../suites/index_p0/test_index_meta.groovy | 4 +-
.../search/test_search_score_raw_index.groovy | 3 +
13 files changed, 338 insertions(+), 28 deletions(-)
diff --git a/be/src/storage/compaction/collection_statistics.cpp
b/be/src/storage/compaction/collection_statistics.cpp
index 3aeb129d93f..6b59c1cc1ec 100644
--- a/be/src/storage/compaction/collection_statistics.cpp
+++ b/be/src/storage/compaction/collection_statistics.cpp
@@ -208,11 +208,12 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset, int3
// BM25 on an analyzed index needs the record length of every row, and
CLucene keeps
// them, together with the field's token count, in the norms. A
segment written without
// norms would feed a zero avgdl, or rank its rows as zero-length
documents next to the
- // segments that have norms, so refuse to score the collection. An
index that is not
- // analyzed never writes norms; use an average length of one when its
legacy
- // metadata does not include the total term frequency.
+ // segments that have norms, so refuse to score the collection.
Keyword indexes and
+ // built-in lowercase normalizers can use unit frequency and length
instead.
const auto token_count =
index_reader->sumTotalTermFreq(ws_field_name.c_str());
- if (!token_count.has_value() &&
+ const bool single_term_normalizer =
segment_v2::IndexReaderHelper::is_normalizer_only(
+ collect_info.index_meta->properties());
+ if (!token_count.has_value() && !single_term_normalizer &&
segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer(
collect_info.index_meta->properties())) {
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
@@ -221,6 +222,7 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset, int3
"for a variant path, when
inverted_index_skip_norms_for_variant is on",
seg_path, StringHelper::to_string(ws_field_name));
}
+ // Missing term totals imply one token per row for keyword and
lowercase indexes.
_total_num_tokens[ws_field_name] +=
token_count.value_or(index_reader->maxDoc());
for (const auto& term_info : collect_info.term_infos) {
diff --git a/be/src/storage/index/index_reader_helper.h
b/be/src/storage/index/index_reader_helper.h
index ad4c856fd26..988db5c73ec 100644
--- a/be/src/storage/index/index_reader_helper.h
+++ b/be/src/storage/index/index_reader_helper.h
@@ -17,6 +17,10 @@
#pragma once
+#include <boost/algorithm/string/predicate.hpp>
+#include <map>
+#include <string>
+
#include "storage/index/index_iterator.h"
#include "storage/index/inverted/inverted_index_reader.h"
@@ -25,6 +29,16 @@ namespace doris::segment_v2 {
class IndexReaderHelper {
public:
+ static bool is_normalizer_only(const std::map<std::string, std::string>&
properties) {
+ const auto normalizer =
properties.find(INVERTED_INDEX_NORMALIZER_NAME_KEY);
+ const auto analyzer =
properties.find(INVERTED_INDEX_ANALYZER_NAME_KEY);
+ // A custom normalizer can split a value into multiple terms that need
positions.
+ return !properties.contains(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY)
&&
+ normalizer != properties.end() &&
boost::iequals(normalizer->second, "lowercase") &&
+ (analyzer == properties.end() || analyzer->second.empty()) &&
+ get_parser_string_from_properties(properties) ==
INVERTED_INDEX_PARSER_NONE;
+ }
+
static bool is_fulltext_index(const IndexReaderPtr& reader) {
if (reader == nullptr || reader->index_type() != IndexType::INVERTED) {
return false;
@@ -60,7 +74,8 @@ public:
auto inverted_index_reader =
std::static_pointer_cast<InvertedIndexReader>(reader);
const auto& properties = inverted_index_reader->get_index_properties();
return get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES;
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
+ is_normalizer_only(properties);
}
// only string type or bkd index reader can be used for equal
@@ -97,7 +112,8 @@ public:
query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) {
const auto& properties = index_meta->properties();
if (get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
+ is_normalizer_only(properties)) {
return true;
}
}
@@ -111,7 +127,8 @@ public:
query_type == TExprOpcode::MATCH_PHRASE_PREFIX) {
const auto& properties = index_meta->properties();
if (get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
+ is_normalizer_only(properties)) {
return true;
}
}
@@ -120,4 +137,4 @@ public:
};
#include "common/compile_check_end.h"
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
diff --git a/be/src/storage/index/inverted/query/query_helper.cpp
b/be/src/storage/index/inverted/query/query_helper.cpp
index 1b44de0b1ce..1dd039783ce 100644
--- a/be/src/storage/index/inverted/query/query_helper.cpp
+++ b/be/src/storage/index/inverted/query/query_helper.cpp
@@ -17,6 +17,8 @@
#include "storage/index/inverted/query/query_helper.h"
+#include <algorithm>
+
namespace doris::segment_v2 {
#include "common/compile_check_begin.h"
@@ -24,8 +26,8 @@ void QueryHelper::collect(const IndexQueryContextPtr& context,
const std::vector<SimilarityPtr>& similarities,
const std::vector<TermIterPtr>& iterators, int32_t
doc) {
for (size_t i = 0; i < iterators.size(); i++) {
- auto freq = iterators[i]->freq();
- auto doc_length = iterators[i]->norm();
+ auto freq = std::max(iterators[i]->freq(), 1);
+ auto doc_length = std::max(iterators[i]->norm(), 1);
auto score = similarities[i]->score(static_cast<float>(freq),
doc_length);
context->collection_similarity->collect(doc, score);
}
@@ -38,8 +40,8 @@ void QueryHelper::collect(const IndexQueryContextPtr& context,
const auto& iter = iterators[i];
if (std::holds_alternative<TermPositionsIterPtr>(iter)) {
const auto& term_iter = std::get<TermPositionsIterPtr>(iter);
- auto freq = term_iter->freq();
- auto norm = term_iter->norm();
+ auto freq = std::max(term_iter->freq(), 1);
+ auto norm = std::max(term_iter->norm(), 1);
auto score = similarities[i]->score(static_cast<float>(freq),
norm);
context->collection_similarity->collect(doc, score);
}
@@ -50,8 +52,8 @@ void QueryHelper::collect_many(const IndexQueryContextPtr&
context, const Simila
const DocRange& doc_range) {
for (size_t j = 0; j < doc_range.doc_many_size_; j++) {
rowid_t row_id = (*doc_range.doc_many)[j];
- auto freq = (*doc_range.freq_many)[j];
- auto norm = (*doc_range.norm_many)[j];
+ auto freq = doc_range.freq_many ? (*doc_range.freq_many)[j] : 1;
+ auto norm = doc_range.norm_many ? std::max((*doc_range.norm_many)[j],
1U) : 1U;
auto score = similarity->score(static_cast<float>(freq), norm);
context->collection_similarity->collect(row_id, score);
}
@@ -62,8 +64,8 @@ void QueryHelper::collect_range(const IndexQueryContextPtr&
context,
const uint32_t docs_size = doc_range.doc_range.second -
doc_range.doc_range.first;
for (uint32_t j = 0; j < docs_size; j++) {
segment_v2::rowid_t row_id = doc_range.doc_range.first + j;
- auto freq = (*doc_range.freq_many)[j];
- auto norm = (*doc_range.norm_many)[j];
+ auto freq = doc_range.freq_many ? (*doc_range.freq_many)[j] : 1;
+ auto norm = doc_range.norm_many ? std::max((*doc_range.norm_many)[j],
1U) : 1U;
auto score = similarity->score(static_cast<float>(freq), norm);
context->collection_similarity->collect(row_id, score);
}
diff --git a/be/test/storage/compaction/collection_statistics_test.cpp
b/be/test/storage/compaction/collection_statistics_test.cpp
index 617f8dfe439..ae605c15f2e 100644
--- a/be/test/storage/compaction/collection_statistics_test.cpp
+++ b/be/test/storage/compaction/collection_statistics_test.cpp
@@ -38,12 +38,18 @@
#include "runtime/exec_env.h"
#include "runtime/index_policy/index_policy_mgr.h"
#include "storage/compaction/collection_statistics.cpp"
+#include "storage/index/index_file_reader.h"
+#include "storage/index/index_file_writer.h"
+#include "storage/index/index_writer.h"
+#include "storage/index/inverted/inverted_index_desc.h"
+#include "storage/index/inverted/util/term_iterator.h"
#include "storage/rowset/rowset.h"
#include "storage/rowset/rowset_meta.h"
#include "storage/rowset/rowset_reader.h"
#include "storage/tablet/tablet_schema.h"
#include "testutil/mock/mock_runtime_state.h"
#include "util/defer_op.h"
+#include "util/slice.h"
namespace doris {
@@ -350,6 +356,92 @@ TEST_F(CollectionStatisticsTest,
CollectWithEmptyRowsetSplits) {
EXPECT_TRUE(status.ok()) << status.msg();
}
+TEST_F(CollectionStatisticsTest, DocsOnlyNormalizerCollectsScoringStatistics) {
+ IndexPolicyMgr policy_mgr;
+ auto* exec_env = ExecEnv::GetInstance();
+ auto* original_policy_mgr = exec_env->index_policy_mgr();
+ exec_env->_index_policy_mgr = &policy_mgr;
+ Defer restore_policy_mgr([&] { exec_env->_index_policy_mgr =
original_policy_mgr; });
+
+ TabletSchemaPB schema_pb;
+ schema_pb.set_keys_type(DUP_KEYS);
+
schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3);
+ auto tablet_schema = std::make_shared<TabletSchema>();
+ tablet_schema->init_from_pb(schema_pb);
+
+ TabletColumn column;
+ column.set_unique_id(1);
+ column.set_name("content");
+ column.set_type(FieldType::OLAP_FIELD_TYPE_STRING);
+ tablet_schema->append_column(column);
+
+ TabletIndex index;
+ index._index_id = 1;
+ index._index_type = IndexType::INVERTED;
+ index._col_unique_ids.push_back(1);
+ index._properties["normalizer"] = "lowercase";
+ tablet_schema->append_index(std::move(index));
+
+ std::vector<StorePath> paths;
+ paths.emplace_back(test_dir_, 1024);
+ auto tmp_file_dirs = std::make_unique<segment_v2::TmpFileDirs>(paths);
+ ASSERT_TRUE(tmp_file_dirs->init().ok());
+ ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs));
+
+ const std::string segment_path = test_dir_ + "/normalized_0.dat";
+ const std::string prefix {
+
segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)};
+ io::FileWriterPtr compound_file;
+ io::FileWriterOptions options;
+ auto fs = io::global_local_filesystem();
+
ASSERT_TRUE(fs->create_file(segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(prefix),
+ &compound_file, &options)
+ .ok());
+ segment_v2::IndexFileWriter file_writer(fs, prefix, "normalized", 0,
+ InvertedIndexStorageFormatPB::V3,
+ std::move(compound_file));
+ const auto indexes = tablet_schema->inverted_indexs(1);
+ ASSERT_EQ(indexes.size(), 1);
+ std::unique_ptr<segment_v2::IndexColumnWriter> column_writer;
+ auto create_status = segment_v2::IndexColumnWriter::create(
+ &tablet_schema->column(0), &column_writer, &file_writer,
indexes[0]);
+ ASSERT_TRUE(create_status.ok()) << create_status;
+ std::vector<Slice> values {Slice("ALPHA")};
+ ASSERT_TRUE(column_writer->add_values("content", values.data(),
values.size()).ok());
+ ASSERT_TRUE(column_writer->finish().ok());
+ ASSERT_TRUE(file_writer.begin_close().ok());
+ ASSERT_TRUE(file_writer.finish_close().ok());
+
+ auto rowset_meta =
std::make_shared<collection_statistics::MockRowsetMeta>();
+ auto rowset =
std::make_shared<collection_statistics::MockRowset>(tablet_schema, rowset_meta);
+ rowset->set_num_segments(1);
+ rowset->set_segment_path(0, segment_path);
+ auto reader =
std::make_shared<collection_statistics::MockRowsetReader>(rowset);
+ std::vector<RowSetSplits> splits {RowSetSplits(reader)};
+
+ auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema,
+ create_match_expr_contexts("alpha"),
nullptr);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(stats_->get_doc_num(), 1U);
+ EXPECT_EQ(stats_->get_total_term_cnt_by_col(L"1"), 1U);
+ EXPECT_EQ(stats_->get_term_doc_freq_by_col(L"1", L"alpha"), 1U);
+
+ segment_v2::IndexFileReader file_reader(fs, prefix,
InvertedIndexStorageFormatPB::V3);
+ ASSERT_TRUE(file_reader.init().ok());
+ auto compound = file_reader.open(indexes[0]);
+ ASSERT_TRUE(compound.has_value()) << compound.error();
+ auto* raw_reader = lucene::index::IndexReader::open(compound->get());
+ auto searcher =
std::make_unique<lucene::search::IndexSearcher>(raw_reader, true);
+ auto term_iter =
+ segment_v2::TermIterator::create(nullptr, true,
searcher->getReader(), L"1", "alpha");
+ ::DocRange range;
+ ASSERT_TRUE(term_iter->read_range(&range));
+ ASSERT_EQ(range.doc_many_size_, 1);
+ ASSERT_EQ(range.freq_many, nullptr);
+ ASSERT_NE(range.norm_many, nullptr);
+ EXPECT_EQ((*range.norm_many)[0], 1U);
+}
+
TEST_F(CollectionStatisticsTest, CollectWithEmptyExpressions) {
auto tablet_schema = create_tablet_schema_with_inverted_index();
VExprContextSPtrs empty_contexts;
diff --git a/be/test/storage/index/inverted/query/query_helper_test.cpp
b/be/test/storage/index/inverted/query/query_helper_test.cpp
index 95c1a3a6fad..d6278ce58b5 100644
--- a/be/test/storage/index/inverted/query/query_helper_test.cpp
+++ b/be/test/storage/index/inverted/query/query_helper_test.cpp
@@ -52,6 +52,23 @@ private:
float _score_value;
};
+class RecordingSimilarity : public doris::segment_v2::Similarity {
+public:
+ MOCK_FUNCTION void for_one_term(const IndexQueryContextPtr&, const
std::wstring&,
+ const std::wstring&) override {}
+ MOCK_FUNCTION void for_terms(const IndexQueryContextPtr&, const
std::wstring&,
+ const std::vector<std::wstring>&) override {}
+ MOCK_FUNCTION float score(float freq, int64_t encoded_norm) override {
+ last_freq = freq;
+ last_norm = encoded_norm;
+ return freq;
+ }
+ MOCK_FUNCTION float max_score() override { return 1.0F; }
+
+ float last_freq = 0;
+ int64_t last_norm = 0;
+};
+
class MockTermIterator : public TermIterator {
public:
MockTermIterator(int32_t freq_val, int32_t norm_val) : _freq(freq_val),
_norm(norm_val) {}
@@ -180,6 +197,52 @@ TEST_F(QueryHelperTest, CollectMany) {
ASSERT_TRUE(_context->collection_similarity->_bm25_scores.count(30) > 0);
}
+TEST_F(QueryHelperTest, CollectManyWithoutPostingsStatistics) {
+ SimilarityPtr similarity = std::make_unique<RecordingSimilarity>();
+ auto* recording = static_cast<RecordingSimilarity*>(similarity.get());
+ DocRange doc_range;
+ doc_range.type_ = DocRangeType::kMany;
+ doc_range.doc_many_size_ = 1;
+ std::vector<uint32_t> docs {10};
+ std::vector<uint32_t> norms {0};
+ doc_range.doc_many = &docs;
+ doc_range.norm_many = &norms;
+
+ QueryHelper::collect_many(_context, similarity, doc_range);
+
+ EXPECT_FLOAT_EQ(recording->last_freq, 1.0F);
+ EXPECT_EQ(recording->last_norm, 1);
+ EXPECT_FLOAT_EQ(_context->collection_similarity->_bm25_scores.at(10),
1.0F);
+}
+
+TEST_F(QueryHelperTest, CollectRangeWithoutPostingsStatistics) {
+ SimilarityPtr similarity = std::make_unique<RecordingSimilarity>();
+ auto* recording = static_cast<RecordingSimilarity*>(similarity.get());
+ DocRange doc_range;
+ doc_range.type_ = DocRangeType::kRange;
+ doc_range.doc_range = std::make_pair(10, 11);
+
+ QueryHelper::collect_range(_context, similarity, doc_range);
+
+ EXPECT_FLOAT_EQ(recording->last_freq, 1.0F);
+ EXPECT_EQ(recording->last_norm, 1);
+ EXPECT_FLOAT_EQ(_context->collection_similarity->_bm25_scores.at(10),
1.0F);
+}
+
+TEST_F(QueryHelperTest, CollectSingleTermWithoutPostingsStatistics) {
+ std::vector<SimilarityPtr> similarities;
+ similarities.push_back(std::make_unique<RecordingSimilarity>());
+ auto* recording =
static_cast<RecordingSimilarity*>(similarities.front().get());
+ std::vector<TermIterPtr> iterators;
+ iterators.push_back(std::make_shared<MockTermIterator>(0, 0));
+
+ QueryHelper::collect(_context, similarities, iterators, 10);
+
+ EXPECT_FLOAT_EQ(recording->last_freq, 1.0F);
+ EXPECT_EQ(recording->last_norm, 1);
+ EXPECT_FLOAT_EQ(_context->collection_similarity->_bm25_scores.at(10),
1.0F);
+}
+
TEST_F(QueryHelperTest, CollectManyWithZeroSize) {
SimilarityPtr similarity = std::make_unique<MockSimilarity>(4.0f);
@@ -276,4 +339,4 @@ TEST_F(QueryHelperTest, CollectMixedScenario) {
ASSERT_TRUE(_context->collection_similarity->_bm25_scores.count(1002) > 0);
ASSERT_TRUE(_context->collection_similarity->_bm25_scores.count(2000) > 0);
ASSERT_TRUE(_context->collection_similarity->_bm25_scores.count(2001) > 0);
-}
\ No newline at end of file
+}
diff --git a/be/test/storage/index/inverted_index_parser_test.cpp
b/be/test/storage/index/inverted_index_parser_test.cpp
index 174ef1226a1..d35aed9fd0d 100644
--- a/be/test/storage/index/inverted_index_parser_test.cpp
+++ b/be/test/storage/index/inverted_index_parser_test.cpp
@@ -126,7 +126,29 @@ TEST_F(InvertedIndexParserTest,
TestGetParserModeStringFromProperties) {
TEST_F(InvertedIndexParserTest,
TestGetParserPhraseSupportStringFromProperties) {
std::map<std::string, std::string> properties;
- // Test with empty properties
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
+ properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_NONE;
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
+ properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_ENGLISH;
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
+ properties.clear();
+ properties[INVERTED_INDEX_ANALYZER_NAME_KEY] = "english";
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
+ properties.clear();
+ properties[INVERTED_INDEX_PARSER_KEY_ALIAS] = INVERTED_INDEX_PARSER_NONE;
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
+ properties.clear();
+ properties[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "lowercase";
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
diff --git a/be/test/storage/segment/index_reader_helper_test.cpp
b/be/test/storage/segment/index_reader_helper_test.cpp
index d52e036f229..ca7a1ea5c80 100644
--- a/be/test/storage/segment/index_reader_helper_test.cpp
+++ b/be/test/storage/segment/index_reader_helper_test.cpp
@@ -396,6 +396,49 @@ TEST_F(IndexReaderHelperTest,
IsNeedSimilarityScoreWithInvertedIndexQueryTypeTes
InvertedIndexQueryType::LESS_THAN_QUERY, &index_meta));
}
+TEST_F(IndexReaderHelperTest, NormalizerOnlyPhraseAndScoringNeedNoPositions) {
+ TabletIndexPB index_pb;
+ index_pb.set_index_type(IndexType::INVERTED);
+ index_pb.set_index_id(2);
+ index_pb.set_index_name("normalized");
+ index_pb.add_col_unique_id(1);
+ (*index_pb.mutable_properties())[INVERTED_INDEX_NORMALIZER_NAME_KEY] =
"lowercase";
+ TabletIndex index_meta;
+ index_meta.init_from_pb(index_pb);
+
+ auto reader =
std::make_shared<MockInvertedIndexReader>(InvertedIndexReaderType::FULLTEXT,
+
index_meta.properties());
+
EXPECT_EQ(get_parser_phrase_support_string_from_properties(index_meta.properties()),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+ EXPECT_TRUE(IndexReaderHelper::is_support_phrase(reader));
+
EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType::MATCH_ANY_QUERY,
+ &index_meta));
+
EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::MATCH_ANY,
&index_meta));
+
+ auto explicit_disabled = index_meta.properties();
+ explicit_disabled[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] =
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
+ auto disabled_reader = std::make_shared<MockInvertedIndexReader>(
+ InvertedIndexReaderType::FULLTEXT, explicit_disabled);
+ EXPECT_FALSE(IndexReaderHelper::is_support_phrase(disabled_reader));
+
+ auto builtin_alias = index_meta.properties();
+ builtin_alias[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "LowerCase";
+ auto alias_reader =
std::make_shared<MockInvertedIndexReader>(InvertedIndexReaderType::FULLTEXT,
+
builtin_alias);
+ EXPECT_TRUE(IndexReaderHelper::is_support_phrase(alias_reader));
+
+ TabletIndexPB custom_pb = index_pb;
+ (*custom_pb.mutable_properties())[INVERTED_INDEX_NORMALIZER_NAME_KEY] =
"split_norm";
+ TabletIndex custom_meta;
+ custom_meta.init_from_pb(custom_pb);
+ auto custom_reader = std::make_shared<MockInvertedIndexReader>(
+ InvertedIndexReaderType::FULLTEXT, custom_meta.properties());
+ EXPECT_FALSE(IndexReaderHelper::is_support_phrase(custom_reader));
+ EXPECT_FALSE(IndexReaderHelper::is_need_similarity_score(
+ InvertedIndexQueryType::MATCH_ANY_QUERY, &custom_meta));
+}
+
TEST_F(IndexReaderHelperTest, IsNeedSimilarityScoreWithTExprOpcodeTest) {
TabletIndex index_meta;
auto index_meta_pb = std::make_unique<TabletIndexPB>();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
index fb9a5ddb41c..867c4b86832 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
@@ -131,7 +131,7 @@ public class InvertedIndexUtil {
public static boolean getInvertedIndexSupportPhrase(Map<String, String>
properties) {
String supportPhrase = properties == null ? null :
properties.get(INVERTED_INDEX_SUPPORT_PHRASE_KEY);
- return supportPhrase != null ? Boolean.parseBoolean(supportPhrase) :
true;
+ return supportPhrase != null && Boolean.parseBoolean(supportPhrase);
}
public static String getPreferredAnalyzer(Map<String, String> properties) {
@@ -419,11 +419,20 @@ public class InvertedIndexUtil {
}
/**
- * Canonicalize analyzer and normalizer names in index properties. Legacy
metadata may contain
- * case-distinct policy names, so a resolved custom policy must keep its
exact stored name.
+ * Resolve policy names and retain positions for custom normalizers that
may split terms.
+ * Preserve exact custom policy spellings when legacy names differ only by
case.
*/
public static void normalizeInvertedIndexProperties(Map<String, String>
properties) {
resolvePolicyNames(properties);
+ String normalizer = properties.get(INVERTED_INDEX_NORMALIZER_NAME_KEY);
+ if (normalizer != null && !normalizer.isEmpty()
+ && !properties.containsKey(INVERTED_INDEX_SUPPORT_PHRASE_KEY))
{
+ boolean builtin = Env.getCurrentEnv().getIndexPolicyMgr()
+ .getTopLevelBuiltin(normalizer,
IndexPolicy.BUILTIN_NORMALIZERS) != null;
+ if (!builtin) {
+ properties.put(INVERTED_INDEX_SUPPORT_PHRASE_KEY, "true");
+ }
+ }
AnalyzerKeyNormalizer.normalizeInvertedIndexProperties(
properties,
INVERTED_INDEX_PARSER_KEY,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
index 352f89c7959..556218fbac2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
@@ -77,10 +77,13 @@ public class Index implements Writable {
this.comment = comment;
if (indexType == IndexDef.IndexType.INVERTED) {
if (this.properties != null && !this.properties.isEmpty()) {
- if
(this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)
- ||
this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY_ALIAS)
- ||
this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_ANALYZER_NAME_KEY)
- ||
this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_NORMALIZER_NAME_KEY))
{
+ String normalizerName = this.properties.get(
+ InvertedIndexUtil.INVERTED_INDEX_NORMALIZER_NAME_KEY);
+ if
(!InvertedIndexUtil.INVERTED_INDEX_PARSER_NONE.equalsIgnoreCase(getInvertedIndexParser())
+ || StringUtils.isNotEmpty(this.properties.get(
+
InvertedIndexUtil.INVERTED_INDEX_ANALYZER_NAME_KEY))
+ || (StringUtils.isNotEmpty(normalizerName)
+ &&
!"lowercase".equalsIgnoreCase(normalizerName))) {
String supportPhraseKey = InvertedIndexUtil
.INVERTED_INDEX_SUPPORT_PHRASE_KEY;
if (!this.properties.containsKey(supportPhraseKey)) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
index 7ad2b49f704..81b91a0850d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
@@ -848,10 +848,11 @@ public class InvertedIndexPropertiesTest {
policyMgr.replayCreateIndexPolicy(new IndexPolicy(
92, "norm_lower", IndexPolicyTypeEnum.NORMALIZER,
ImmutableMap.of("token_filter", "lowercase")));
Map<String, String> mixedNormalizer = new
HashMap<>(ImmutableMap.of("normalizer", "LowerCase"));
+ Map<String, String> customNormalizer = new
HashMap<>(ImmutableMap.of("normalizer", "lowercase"));
Map<String, String> mixedAnalyzer = new
HashMap<>(ImmutableMap.of("analyzer", "Standard"));
withIndexPolicyManager(policyMgr, () -> {
- for (Map<String, String> properties :
ImmutableList.of(mixedNormalizer, mixedAnalyzer)) {
+ for (Map<String, String> properties :
ImmutableList.of(mixedNormalizer, customNormalizer, mixedAnalyzer)) {
Assertions.assertDoesNotThrow(() ->
InvertedIndexUtil.checkInvertedIndexParser("c",
PrimitiveType.VARCHAR, properties,
TInvertedIndexFileStorageFormat.V3));
}
@@ -859,6 +860,8 @@ public class InvertedIndexPropertiesTest {
// The exact policy shadows the canonical name, so only
the mixed spelling still
// reaches the built-in normalizer on BE.
() -> Assertions.assertEquals("LowerCase",
mixedNormalizer.get("normalizer")),
+ () ->
Assertions.assertFalse(mixedNormalizer.containsKey("support_phrase")),
+ () -> Assertions.assertEquals("true",
customNormalizer.get("support_phrase")),
() -> Assertions.assertEquals("LowerCase",
InvertedIndexUtil.resolveAnalyzerName("LowerCase")),
() -> Assertions.assertEquals("lowercase",
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java
index 776c97ca76b..f753150858b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java
@@ -18,15 +18,66 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.IndexDef;
+import org.apache.doris.analysis.InvertedIndexUtil;
import org.junit.Assert;
import org.junit.Test;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
public class IndexTest {
+ @Test
+ public void testInvertedIndexPhraseDefaultsFollowTokenization() {
+ Index exact = new Index(1, "exact", Collections.singletonList("col"),
+ IndexDef.IndexType.INVERTED, Collections.emptyMap(), "");
+
Assert.assertFalse(exact.getProperties().containsKey("support_phrase"));
+
Assert.assertFalse(InvertedIndexUtil.getInvertedIndexSupportPhrase(Collections.emptyMap()));
+
+ Index keyword = new Index(2, "keyword",
Collections.singletonList("col"),
+ IndexDef.IndexType.INVERTED,
Collections.singletonMap("parser", "none"), "");
+
Assert.assertFalse(keyword.getProperties().containsKey("support_phrase"));
+
+ Index normalized = new Index(3, "normalized",
Collections.singletonList("col"),
+ IndexDef.IndexType.INVERTED,
Collections.singletonMap("normalizer", "lowercase"), "");
+
Assert.assertFalse(normalized.getProperties().containsKey("support_phrase"));
+ Assert.assertFalse(InvertedIndexUtil.getInvertedIndexSupportPhrase(
+ Collections.singletonMap("normalizer", "lowercase")));
+
+ Index customNormalizer = new Index(7, "custom_normalizer",
Collections.singletonList("col"),
+ IndexDef.IndexType.INVERTED,
Collections.singletonMap("normalizer", "split_norm"), "");
+ Assert.assertEquals("true",
customNormalizer.getProperties().get("support_phrase"));
+
+ Map<String, String> disabledCustomProperties = new HashMap<>();
+ disabledCustomProperties.put("normalizer", "split_norm");
+ disabledCustomProperties.put("support_phrase", "false");
+ Index disabledCustomNormalizer = new Index(8,
"disabled_custom_normalizer",
+ Collections.singletonList("col"), IndexDef.IndexType.INVERTED,
+ disabledCustomProperties, "");
+ Assert.assertEquals("false",
disabledCustomNormalizer.getProperties().get("support_phrase"));
+
+ Index parsed = new Index(4, "parsed", Collections.singletonList("col"),
+ IndexDef.IndexType.INVERTED,
Collections.singletonMap("parser", "english"), "");
+ Assert.assertEquals("true",
parsed.getProperties().get("support_phrase"));
+ Assert.assertFalse(InvertedIndexUtil.getInvertedIndexSupportPhrase(
+ Collections.singletonMap("parser", "english")));
+
+ Index analyzed = new Index(5, "analyzed",
Collections.singletonList("col"),
+ IndexDef.IndexType.INVERTED,
Collections.singletonMap("analyzer", "english"), "");
+ Assert.assertEquals("true",
analyzed.getProperties().get("support_phrase"));
+
+ Map<String, String> disabledProperties = new HashMap<>();
+ disabledProperties.put("parser", "english");
+ disabledProperties.put("support_phrase", "false");
+ Index disabled = new Index(6, "disabled",
Collections.singletonList("col"),
+ IndexDef.IndexType.INVERTED, disabledProperties, "");
+ Assert.assertEquals("false",
disabled.getProperties().get("support_phrase"));
+ }
+
@Test
public void testGetColumnUniqueIds() {
// Create test columns with unique IDs
diff --git a/regression-test/suites/index_p0/test_index_meta.groovy
b/regression-test/suites/index_p0/test_index_meta.groovy
index d7647f2a9a3..bcaaf2922b1 100644
--- a/regression-test/suites/index_p0/test_index_meta.groovy
+++ b/regression-test/suites/index_p0/test_index_meta.groovy
@@ -71,7 +71,7 @@ suite("index_meta", "p0") {
assertEquals(show_result[1][4], "name")
assertEquals(show_result[1][10], "INVERTED")
assertEquals(show_result[1][11], "index for name")
- assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" =
\"none\", \"support_phrase\" = \"true\")")
+ assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" =
\"none\")")
// add index on column description
sql "create index idx_desc on ${tableName}(description) USING INVERTED
PROPERTIES(\"parser\"=\"standard\") COMMENT 'index for description';"
@@ -90,7 +90,7 @@ suite("index_meta", "p0") {
assertEquals(show_result[1][4], "name")
assertEquals(show_result[1][10], "INVERTED")
assertEquals(show_result[1][11], "index for name")
- assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" =
\"none\", \"support_phrase\" = \"true\")")
+ assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" =
\"none\")")
assertEquals(show_result[2][2], "idx_desc")
assertEquals(show_result[2][4], "description")
assertEquals(show_result[2][10], "INVERTED")
diff --git a/regression-test/suites/search/test_search_score_raw_index.groovy
b/regression-test/suites/search/test_search_score_raw_index.groovy
index fb80fcf2637..ec10b1fabbc 100644
--- a/regression-test/suites/search/test_search_score_raw_index.groovy
+++ b/regression-test/suites/search/test_search_score_raw_index.groovy
@@ -16,6 +16,9 @@
// under the License.
suite("test_search_score_raw_index", "p0") {
+ // SEARCH requires inverted-index execution even under fuzzy session
settings.
+ sql "set enable_common_expr_pushdown = true"
+ sql "set enable_inverted_index_query = true"
// Score TopN pushdown accepts one ordering key. Sort ties outside the
bounded query.
def scoredQuery = { hint, table, predicate, direction, limit ->
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
