This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 74b4418d738 [fix](be) Validate SNII analyzer context before query
cache (#66869)
74b4418d738 is described below
commit 74b4418d738ba2aa034f5ac5bd0294cf48278de2
Author: Jack <[email protected]>
AuthorDate: Mon Aug 24 11:24:47 2026 +0800
[fix](be) Validate SNII analyzer context before query cache (#66869)
### What problem does this PR solve?
Issue Number: None
Related PR: #66052
Problem Summary: SEARCH selected the correct SNII reader but passed no
analyzer context to native queries. A cold CommonGrams query therefore
bypassed while a warm raw-query cache entry could be returned before the
immutable segment analyzer contract was validated. This PR builds the
analyzer context from the selected reader properties, passes it to
analyzed native queries, and defers analyzed raw-cache lookup until
segment admission. Analyzer-independent wildcard and regexp queries keep
their early cache path.
### Release note
Fix SNII SEARCH queries on CommonGrams indexes so cold and cached
queries validate the same analyzer contract.
---
be/src/exprs/function/function_search.cpp | 16 +++-
.../function/variant_inverted_index_search.cpp | 37 +++++++++
.../exprs/function/variant_inverted_index_search.h | 5 ++
be/src/storage/index/snii/snii_index_reader.cpp | 16 ++--
be/test/exprs/function/function_search_test.cpp | 71 +++++++++++++++-
...inverted_index_reader_analysis_purpose_test.cpp | 19 +++--
.../snii/snii_index_reader_count_fallback_test.cpp | 95 ++++++++++++++++++++++
7 files changed, 240 insertions(+), 19 deletions(-)
diff --git a/be/src/exprs/function/function_search.cpp
b/be/src/exprs/function/function_search.cpp
index af6d01fb342..a96983945cd 100644
--- a/be/src/exprs/function/function_search.cpp
+++ b/be/src/exprs/function/function_search.cpp
@@ -773,7 +773,12 @@ Status FunctionSearch::build_leaf_query(const
TSearchClause& clause,
};
FieldReaderBinding binding;
- RETURN_IF_ERROR(resolver.resolve(field_name, query_type, &binding));
+ const bool require_analyzer_context = clause_type != "WILDCARD" &&
clause_type != "REGEXP";
+ if (require_analyzer_context) {
+ RETURN_IF_ERROR(resolver.resolve_with_analyzer_context(field_name,
query_type, &binding));
+ } else {
+ RETURN_IF_ERROR(resolver.resolve(field_name, query_type, &binding));
+ }
if (!binding.is_bound()) {
LOG(INFO) << "search: No inverted index for field '" << field_name
@@ -905,9 +910,12 @@ Status FunctionSearch::build_leaf_query(const
TSearchClause& clause,
? normalize_wildcard_pattern(value,
binding.index_properties)
: value;
Field query_value = Field::create_field<TYPE_STRING>(pattern);
- RETURN_IF_ERROR(binding.inverted_reader->query(reader_context,
-
binding.stored_field_name, query_value,
- snii_query_type,
data_bitmap, nullptr));
+ const bool raw_pattern_query =
+ snii_query_type == InvertedIndexQueryType::WILDCARD_QUERY
||
+ snii_query_type ==
InvertedIndexQueryType::MATCH_REGEXP_QUERY;
+ RETURN_IF_ERROR(binding.inverted_reader->query(
+ reader_context, binding.stored_field_name, query_value,
snii_query_type,
+ data_bitmap, raw_pattern_query ? nullptr :
binding.analyzer_context.get()));
// Reply-direction fields land on the copy the reader was given,
so they have to be
// folded back. Today this is unreachable rather than
load-bearing: the count-only
// fast path requires the scan to have no score runtime, while the
similarity that
diff --git a/be/src/exprs/function/variant_inverted_index_search.cpp
b/be/src/exprs/function/variant_inverted_index_search.cpp
index 1e5d8f24489..5f44ac124df 100644
--- a/be/src/exprs/function/variant_inverted_index_search.cpp
+++ b/be/src/exprs/function/variant_inverted_index_search.cpp
@@ -65,6 +65,29 @@ void add_search_binding_diagnostic(const
std::shared_ptr<IndexQueryContext>& con
}
}
+InvertedIndexAnalyzerCtxSPtr build_analyzer_context(
+ const std::map<std::string, std::string>& properties, const
std::string& analyzer_key) {
+ InvertedIndexAnalyzerConfig config;
+ config.analyzer_name = get_analyzer_name_from_properties(properties);
+ config.parser_type = get_inverted_index_parser_type_from_string(
+ get_parser_string_from_properties(properties));
+ config.parser_mode = get_parser_mode_string_from_properties(properties);
+ config.lower_case = get_parser_lowercase_from_properties(properties);
+ config.stop_words = get_parser_stopwords_from_properties(properties);
+ config.char_filter_map =
get_parser_char_filter_map_from_properties(properties);
+
+ auto analyzer_context = std::make_shared<InvertedIndexAnalyzerCtx>();
+ analyzer_context->analyzer_key = analyzer_key;
+ analyzer_context->analyzer_name = config.analyzer_name;
+ analyzer_context->parser_type = config.parser_type;
+ analyzer_context->char_filter_map = config.char_filter_map;
+ if (analyzer_context->requires_analysis()) {
+ analyzer_context->analyzer_provider =
+
inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config);
+ }
+ return analyzer_context;
+}
+
} // namespace
FieldReaderResolver::FieldReaderResolver(
@@ -384,6 +407,20 @@ Status FieldReaderResolver::resolve(const std::string&
field_name,
return Status::OK();
}
+Status FieldReaderResolver::resolve_with_analyzer_context(const std::string&
field_name,
+
InvertedIndexQueryType query_type,
+ FieldReaderBinding*
binding) {
+ RETURN_IF_ERROR(resolve(field_name, query_type, binding));
+ if (!binding->use_snii_native_reader() || binding->analyzer_context !=
nullptr) {
+ return Status::OK();
+ }
+
+ binding->analyzer_context =
+ build_analyzer_context(binding->index_properties,
binding->analyzer_key);
+ _cache.at(binding->binding_key).analyzer_context =
binding->analyzer_context;
+ return Status::OK();
+}
+
segment_v2::IndexIterator* VariantSearchNullBitmapAdapter::iterator_for(
const query_v2::Scorer& /*scorer*/, const std::string& logical_field)
const {
if (logical_field.empty()) {
diff --git a/be/src/exprs/function/variant_inverted_index_search.h
b/be/src/exprs/function/variant_inverted_index_search.h
index d03dbc486d8..a468a9c0034 100644
--- a/be/src/exprs/function/variant_inverted_index_search.h
+++ b/be/src/exprs/function/variant_inverted_index_search.h
@@ -86,6 +86,7 @@ struct FieldReaderBinding {
std::map<std::string, std::string> index_properties;
std::string binding_key;
std::string analyzer_key;
+ InvertedIndexAnalyzerCtxSPtr analyzer_context;
SearchFieldBindingState state =
SearchFieldBindingState::MISSING_IN_SEGMENT;
SearchFieldExecutionMode execution_mode =
SearchFieldExecutionMode::UNBOUND;
@@ -112,6 +113,10 @@ public:
Status resolve(const std::string& field_name, InvertedIndexQueryType
query_type,
FieldReaderBinding* binding);
+ Status resolve_with_analyzer_context(const std::string& field_name,
+ InvertedIndexQueryType query_type,
+ FieldReaderBinding* binding);
+
bool is_variant_subcolumn(const std::string& field_name) const {
return _variant_subcolumn_fields.count(field_name) > 0;
}
diff --git a/be/src/storage/index/snii/snii_index_reader.cpp
b/be/src/storage/index/snii/snii_index_reader.cpp
index d701c314efb..3aeebe54588 100644
--- a/be/src/storage/index/snii/snii_index_reader.cpp
+++ b/be/src/storage/index/snii/snii_index_reader.cpp
@@ -642,6 +642,12 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr&
context, const std::s
const bool common_grams_query_eligible = common_grams_phrase_shape &&
!actual_similarity;
const bool raw_pattern_query = query_type ==
InvertedIndexQueryType::MATCH_REGEXP_QUERY ||
query_type ==
InvertedIndexQueryType::WILDCARD_QUERY;
+ // A physical keyword-lane index has no analyzer contract for the open
below to validate:
+ // SniiIndexColumnWriter::init() refuses a CommonGrams metadata seed
whenever should_analyzer()
+ // is false, so such a segment can never carry gram terms. Key this on the
writer-side
+ // predicate itself rather than on the reader type it happens to select.
+ const bool keyword_lane_query =
+
!inverted_index::InvertedIndexAnalyzer::should_analyzer(_index_meta.properties());
// Lucene-style CommonGrams: the plan decision is local to the segment and
query. Snapshot the
// switch once so this query's plan and cache identity use the same mode.
const bool common_grams_query_plan_enabled =
config::enable_common_grams_query_plan;
@@ -656,11 +662,11 @@ Status SniiIndexReader::_query(const
IndexQueryContextPtr& context, const std::s
ctx->has_complete_common_grams_identity();
};
const bool safety_requires_plain = !common_grams_query_plan_enabled;
- // The raw cache key cannot prove whether the immutable segment analyzer
has CommonGrams until
- // its metadata is open. Delay every eligible forced-plain lookup, then
restore ordinary cache
- // access below only for a segment that cannot contain gram terms.
- const bool initial_force_plain = common_grams_query_eligible &&
safety_requires_plain;
- const bool initial_allow_result_cache = !actual_similarity &&
!initial_force_plain;
+ // An analyzed raw query can only share a cached result after the
immutable segment analyzer
+ // contract has been validated below. Patterns and the keyword lane are
analyzer-independent
+ // and can still use the cache before opening the logical reader.
+ const bool initial_allow_result_cache =
+ !actual_similarity && (raw_pattern_query || keyword_lane_query);
const bool defer_result_cache_lookup = !actual_similarity &&
!initial_allow_result_cache;
const InvertedIndexRawQuerySemantic raw_semantic {
.raw_query_bytes = search_str,
diff --git a/be/test/exprs/function/function_search_test.cpp
b/be/test/exprs/function/function_search_test.cpp
index 4ac10cd5101..0edc417eb8b 100644
--- a/be/test/exprs/function/function_search_test.cpp
+++ b/be/test/exprs/function/function_search_test.cpp
@@ -2346,7 +2346,8 @@ TEST_F(FunctionSearchTest,
TestBuildLeafQueryExecutesSelectedSniiWildcardReader)
{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_ENGLISH},
{INVERTED_INDEX_PARSER_LOWERCASE_KEY, INVERTED_INDEX_PARSER_TRUE}};
std::map<std::string, std::string> selected_properties {
- {INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD},
+ {INVERTED_INDEX_ANALYZER_NAME_KEY,
"unregistered_search_wildcard_analyzer"},
+ {INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_NONE},
{INVERTED_INDEX_PARSER_LOWERCASE_KEY, INVERTED_INDEX_PARSER_TRUE}};
auto decoy_meta = make_test_inverted_index(15, decoy_properties);
auto selected_meta = make_test_inverted_index(16, selected_properties);
@@ -2530,6 +2531,74 @@ TEST_F(FunctionSearchTest,
TestSniiNativeForwardsTermClauseAsEqualQuery) {
expect_bitmap_eq(collect_docs(scorer), {0, 2});
}
+TEST_F(FunctionSearchTest, TestSniiNativePassesSelectedAnalyzerContext) {
+ auto* exec_env = ExecEnv::GetInstance();
+ auto* previous_policy_mgr = exec_env->index_policy_mgr();
+ IndexPolicyMgr scoped_policy_mgr;
+ exec_env->_index_policy_mgr = &scoped_policy_mgr;
+ DEFER(exec_env->_index_policy_mgr = previous_policy_mgr);
+
+ TIndexPolicy tokenizer;
+ tokenizer.id = 910030;
+ tokenizer.name = "function_search_context_tokenizer";
+ tokenizer.type = TIndexPolicyType::TOKENIZER;
+ tokenizer.properties["type"] = "char_group";
+ tokenizer.properties["tokenize_on_chars"] = "[whitespace]";
+
+ TIndexPolicy common_grams;
+ common_grams.id = 910031;
+ common_grams.name = "function_search_context_common_grams";
+ common_grams.type = TIndexPolicyType::TOKEN_FILTER;
+ common_grams.properties["type"] = "common_grams";
+
+ TIndexPolicy analyzer;
+ analyzer.id = 910032;
+ analyzer.name = "function_search_context_analyzer";
+ analyzer.type = TIndexPolicyType::ANALYZER;
+ analyzer.properties["tokenizer"] = tokenizer.name;
+ analyzer.properties["token_filter"] = "lowercase," + common_grams.name;
+ scoped_policy_mgr.apply_policy_changes({tokenizer, common_grams,
analyzer}, {});
+
+ auto context = std::make_shared<IndexQueryContext>();
+ std::map<std::string, std::string> properties {
+ {INVERTED_INDEX_ANALYZER_NAME_KEY, analyzer.name},
+ {INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_NONE}};
+ auto index_meta = make_test_inverted_index(45, properties);
+ auto index_file_reader =
std::make_shared<RejectingCluceneIndexFileReader>();
+ auto reader =
+ std::make_shared<RecordingNativeInvertedIndexReader>(&index_meta,
index_file_reader);
+ reader->set_query_result("running quickly", make_bitmap({1}));
+ segment_v2::InvertedIndexIterator iterator;
+ iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader);
+
+ std::unordered_map<std::string, IndexFieldNameAndTypePair>
data_type_with_names;
+ data_type_with_names.emplace(
+ "body", IndexFieldNameAndTypePair {"body",
std::make_shared<DataTypeString>()});
+ std::unordered_map<std::string, IndexIterator*> iterators;
+ iterators["body"] = &iterator;
+ TSearchFieldBinding field_binding;
+ field_binding.field_name = "body";
+ field_binding.index_properties = properties;
+ field_binding.__isset.index_properties = true;
+ FieldReaderResolver resolver(data_type_with_names, iterators, context,
{field_binding});
+
+ inverted_index::query_v2::QueryPtr query;
+ std::string binding_key;
+ auto status =
+ function_search->build_leaf_query(make_leaf_clause("MATCH",
"running quickly"), context,
+ resolver, &query, &binding_key,
"OR", 0, 3);
+
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_NE(reader->last_analyzer_ctx, nullptr);
+ EXPECT_EQ(reader->last_analyzer_ctx->analyzer_key, analyzer.name);
+ EXPECT_EQ(reader->last_analyzer_ctx->analyzer_name, analyzer.name);
+ EXPECT_EQ(reader->last_analyzer_ctx->parser_type,
InvertedIndexParserType::PARSER_NONE);
+ EXPECT_TRUE(reader->last_analyzer_ctx->requires_analysis());
+ ASSERT_NE(reader->last_analyzer_ctx->analyzer_provider, nullptr);
+
EXPECT_TRUE(reader->last_analyzer_ctx->analyzer_provider->uses_common_grams());
+
EXPECT_FALSE(reader->last_analyzer_ctx->analyzer_provider->base_analyzer_fingerprint().empty());
+}
+
// default_operator "and" maps a multi-token TERM clause onto MATCH_ALL_QUERY
instead of the
// default EQUAL_QUERY (which is an OR of terms) -- SNII has no boolean query
tree to build, so
// this is expressed entirely as which query type gets forwarded to the reader.
diff --git
a/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp
b/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp
index f7075c30601..eb903b1b3d3 100644
---
a/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp
+++
b/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp
@@ -407,10 +407,10 @@ protected:
}
template <typename Reader, typename Provider>
- void expect_raw_cache_hit_before_analysis(const std::shared_ptr<Reader>&
reader,
- const
std::shared_ptr<IndexFileReader>& file_reader,
- const std::shared_ptr<Provider>&
provider,
- bool
common_grams_query_plan_enabled) {
+ void expect_raw_cache_hit_after_segment_admission(
+ const std::shared_ptr<Reader>& reader,
+ const std::shared_ptr<IndexFileReader>& file_reader,
+ const std::shared_ptr<Provider>& provider, bool
common_grams_query_plan_enabled) {
QueryExecutionContext execution(/*scoring=*/false);
InvertedIndexAnalyzerCtx analyzer_ctx;
analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH;
@@ -447,7 +447,7 @@ protected:
EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 0);
EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 1);
EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0);
- EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, 0);
+ EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, 1);
EXPECT_EQ(execution.stats.inverted_index_searcher_cache_miss, 0);
}
@@ -562,6 +562,7 @@ TEST(InvertedIndexAnalyzerCtxTest,
UsesProviderCommonGramsIdentityWithoutCopying
TEST_F(InvertedIndexReaderAnalysisPurposeTest,
SniiRawCacheLookupIsIndependentOfRequestAnalyzerIdentity) {
+ preload_legacy_searcher_cache_entries();
const bool original = config::enable_common_grams_query_plan;
Defer restore([original] {
EXPECT_TRUE(config::set_config("enable_common_grams_query_plan",
@@ -577,14 +578,14 @@ TEST_F(InvertedIndexReaderAnalysisPurposeTest,
.common_grams_fingerprint = "grams:complete"};
const inverted_index::CommonGramsQueryIdentity empty_identity;
for (const auto& identity : {complete_identity, empty_identity}) {
- expect_raw_cache_hit_before_analysis(
+ expect_raw_cache_hit_after_segment_admission(
_snii_reader, _snii_file_reader,
std::make_shared<IdentityFailingAnalyzerProvider>(identity),
config::enable_common_grams_query_plan);
}
- expect_raw_cache_hit_before_analysis(_snii_reader, _snii_file_reader,
-
std::make_shared<RecordingFailingAnalyzerProvider>(),
-
config::enable_common_grams_query_plan);
+ expect_raw_cache_hit_after_segment_admission(
+ _snii_reader, _snii_file_reader,
std::make_shared<RecordingFailingAnalyzerProvider>(),
+ config::enable_common_grams_query_plan);
}
TEST_F(InvertedIndexReaderAnalysisPurposeTest,
DisabledResultCacheDoesNotLookupCountOrInsert) {
diff --git
a/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp
b/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp
index 455073febd6..657ff963f85 100644
--- a/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp
+++ b/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp
@@ -48,6 +48,7 @@
#include "runtime/thread_context.h"
#include "storage/index/index_file_reader.h"
#include "storage/index/index_iterator.h" // for IndexIterator
+#include "storage/index/inverted/analyzer/analyzer.h"
#include "storage/index/inverted/analyzer/custom_analyzer.h"
#include "storage/index/inverted/common_grams/common_grams_key_codec.h"
#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h"
@@ -1005,6 +1006,50 @@ TEST_F(SniiIndexReaderCountFallback,
CustomAnalyzerWithNoneParserRetainsAnalyzed
EXPECT_EQ(second.stats.inverted_index_query_cache_insert, 0);
}
+TEST_F(SniiIndexReaderCountFallback,
+ CommonGramsAnalyzerContractIsValidatedBeforeColdAndWarmCacheLookup) {
+ const auto provider = make_common_grams_provider();
+ ASSERT_NE(provider->common_grams_identity(), nullptr);
+ constexpr std::string_view kPathPrefix =
+
"./ut_dir/snii_index_reader_count_fallback_test/analyzer_contract_cache";
+ assert_ok(write_common_grams_segment(kPathPrefix,
*provider->common_grams_identity(),
+
inverted_index::CommonGramsCoverage::kComplete,
+ /*include_gram=*/true));
+ OpenedSniiIndex opened;
+ assert_ok(open_snii_index(&_meta, std::string(kPathPrefix), &opened));
+
+ const Field query_value =
Field::create_field<TYPE_STRING>(std::string("alpha"));
+ QueryExecutionContext cold_missing_context(/*enable_query_cache=*/true);
+ std::shared_ptr<roaring::Roaring> cold_bitmap;
+ const Status cold_status = opened.index_reader->query(
+ cold_missing_context.context, "analyzer_contract_content",
query_value,
+ InvertedIndexQueryType::MATCH_ANY_QUERY, cold_bitmap, nullptr);
+ EXPECT_EQ(cold_status.code(), ErrorCode::INVERTED_INDEX_BYPASS) <<
cold_status;
+ EXPECT_EQ(cold_missing_context.stats.inverted_index_query_cache_lookup, 0);
+
+ InvertedIndexAnalyzerCtx analyzer_ctx;
+ analyzer_ctx.analyzer_name = "test_common_grams";
+ analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_NONE;
+ analyzer_ctx.analyzer_provider = provider;
+ QueryExecutionContext admitted(/*enable_query_cache=*/true);
+ std::shared_ptr<roaring::Roaring> admitted_bitmap;
+ assert_ok(opened.index_reader->query(admitted.context,
"analyzer_contract_content", query_value,
+
InvertedIndexQueryType::MATCH_ANY_QUERY, admitted_bitmap,
+ &analyzer_ctx));
+ ASSERT_NE(admitted_bitmap, nullptr);
+ EXPECT_EQ(bitmap_docids(*admitted_bitmap), (std::vector<uint32_t> {1}));
+ EXPECT_EQ(admitted.stats.inverted_index_query_cache_insert, 1);
+
+ QueryExecutionContext warm_missing_context(/*enable_query_cache=*/true);
+ std::shared_ptr<roaring::Roaring> warm_bitmap;
+ const Status warm_status = opened.index_reader->query(
+ warm_missing_context.context, "analyzer_contract_content",
query_value,
+ InvertedIndexQueryType::MATCH_ANY_QUERY, warm_bitmap, nullptr);
+ EXPECT_EQ(warm_status.code(), ErrorCode::INVERTED_INDEX_BYPASS) <<
warm_status;
+ EXPECT_EQ(warm_missing_context.stats.inverted_index_query_cache_lookup, 0);
+ EXPECT_EQ(warm_missing_context.stats.inverted_index_query_cache_hit, 0);
+}
+
TEST_F(SniiIndexReaderCountFallback,
CustomKeywordAnalyzerWithNoneParserNormalizesSingleTerm) {
inverted_index::CustomAnalyzerConfig::Builder builder;
builder.with_tokenizer_config("keyword", {});
@@ -1785,5 +1830,55 @@ TEST_F(SniiIndexReaderCountFallback,
MultiTermPhraseUsesNormalPositionalQuery) {
EXPECT_EQ(phrase_docids, (std::vector<uint32_t> {0, 3, 5}));
}
+TEST_F(SniiIndexReaderCountFallback,
KeywordLaneWarmQueryCacheHitSkipsSegmentOpen) {
+ // A physical keyword-lane index carries no analyzer contract for the
segment open to
+ // validate: SniiIndexColumnWriter::init() rejects a CommonGrams metadata
seed whenever
+ // should_analyzer() is false, the same split that picks STRING_TYPE over
FULLTEXT in
+ // ColumnReader. Its warm raw-query cache entry must therefore still be
served before the
+ // logical reader is opened, including when the independent searcher cache
is disabled.
+ TabletIndex keyword_meta;
+ {
+ TabletIndexPB pb;
+ pb.set_index_type(IndexType::INVERTED);
+ pb.set_index_id(kIndexId);
+ pb.set_index_name("keyword_idx");
+ pb.add_col_unique_id(0);
+ keyword_meta.init_from_pb(pb);
+ }
+
ASSERT_FALSE(inverted_index::InvertedIndexAnalyzer::should_analyzer(keyword_meta.properties()));
+
+ OpenedSniiIndex opened;
+ opened.file_reader = std::make_shared<IndexFileReader>(
+ io::global_local_filesystem(), kIndexPathPrefix,
InvertedIndexStorageFormatPB::SNII);
+ assert_ok(opened.file_reader->init());
+ opened.index_reader = SniiIndexReader::create_shared(&keyword_meta,
opened.file_reader,
+
InvertedIndexReaderType::STRING_TYPE);
+
+ std::atomic<uint32_t> searcher_opens {0};
+
opened.index_reader->set_searcher_open_observer_for_test(record_searcher_open,
&searcher_opens);
+
+ const Field query_value =
Field::create_field<TYPE_STRING>(std::string("failed"));
+
+ QueryExecutionContext cold(/*enable_query_cache=*/true);
+ std::shared_ptr<roaring::Roaring> cold_bitmap;
+ assert_ok(opened.index_reader->query(cold.context, "keyword_content",
query_value,
+ InvertedIndexQueryType::EQUAL_QUERY,
cold_bitmap));
+ ASSERT_NE(cold_bitmap, nullptr);
+ EXPECT_EQ(bitmap_docids(*cold_bitmap), (std::vector<uint32_t> {0, 1, 2, 3,
4, 5}));
+ EXPECT_EQ(cold.stats.inverted_index_query_cache_miss, 1);
+ EXPECT_EQ(cold.stats.inverted_index_query_cache_insert, 1);
+ EXPECT_EQ(searcher_opens.load(std::memory_order_relaxed), 1U);
+
+ QueryExecutionContext warm(/*enable_query_cache=*/true);
+ std::shared_ptr<roaring::Roaring> warm_bitmap;
+ assert_ok(opened.index_reader->query(warm.context, "keyword_content",
query_value,
+ InvertedIndexQueryType::EQUAL_QUERY,
warm_bitmap));
+ ASSERT_NE(warm_bitmap, nullptr);
+ EXPECT_EQ(bitmap_docids(*warm_bitmap), bitmap_docids(*cold_bitmap));
+ EXPECT_EQ(warm.stats.inverted_index_query_cache_hit, 1);
+ EXPECT_EQ(searcher_opens.load(std::memory_order_relaxed), 1U)
+ << "warm keyword-lane query re-opened the segment before the
query-cache hit";
+}
+
} // namespace
} // namespace doris::segment_v2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
