github-actions[bot] commented on code in PR #68758:
URL: https://github.com/apache/doris/pull/68758#discussion_r4255167674
##########
be/src/exprs/function/search/variant_inverted_index_search.cpp:
##########
@@ -294,117 +297,19 @@ Status FieldReaderResolver::resolve(const std::string&
field_name,
return Status::OK();
}
- if (index_file_reader->get_storage_format() ==
InvertedIndexStorageFormatPB::SNII) {
- resolved.execution_mode = SearchFieldExecutionMode::SNII_NATIVE;
- _cache.emplace(binding_key, resolved);
- if (is_variant_sub) {
- add_search_binding_diagnostic(
- _context,
- fmt::format("[VariantSearchBinding] phase=field_resolve "
- "result=selected_snii_native logical_field={}
stored_field={} "
- "query_type={} effective_query_type={}
index_id={} suffix={} "
- "reader_type={} analyzer_key={} index_file={}",
- field_name, stored_field_name,
query_type_to_string(query_type),
- query_type_to_string(effective_query_type),
- inverted_reader->get_index_id(),
-
inverted_reader->get_index_meta().get_index_suffix(),
- reader_type_to_string(inverted_reader->type()),
- resolved.analyzer_key,
- index_file_reader->get_index_file_path(
- &inverted_reader->get_index_meta())));
- }
- *binding = resolved;
- return Status::OK();
- }
-
- auto index_file_key =
-
index_file_reader->get_index_file_cache_key(&inverted_reader->get_index_meta());
- InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key);
- InvertedIndexCacheHandle searcher_cache_handle;
-
- bool searcher_cache_enabled =
- _context->runtime_state != nullptr &&
-
_context->runtime_state->query_options().enable_inverted_index_searcher_cache;
-
- bool cache_hit = false;
- if (searcher_cache_enabled) {
- int64_t lookup_dummy = 0;
- SCOPED_RAW_TIMER(_context->stats ?
&_context->stats->inverted_index_lookup_timer
- : &lookup_dummy);
- cache_hit =
InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key,
-
&searcher_cache_handle);
- }
-
- std::shared_ptr<lucene::index::IndexReader> reader_holder;
- if (cache_hit) {
- if (_context->stats) {
- _context->stats->inverted_index_searcher_cache_hit++;
- }
- auto searcher_variant = searcher_cache_handle.get_index_searcher();
- auto* searcher_ptr =
std::get_if<FulltextIndexSearcherPtr>(&searcher_variant);
- if (searcher_ptr != nullptr && *searcher_ptr != nullptr) {
- reader_holder = std::shared_ptr<lucene::index::IndexReader>(
- (*searcher_ptr)->getReader(),
[](lucene::index::IndexReader*) {});
- }
- }
-
- if (!reader_holder) {
- if (_context->stats) {
- _context->stats->inverted_index_searcher_cache_miss++;
- }
- int64_t dummy_timer = 0;
- SCOPED_RAW_TIMER(_context->stats ?
&_context->stats->inverted_index_searcher_open_timer
- : &dummy_timer);
- RETURN_IF_ERROR(
-
index_file_reader->init(config::inverted_index_read_buffer_size,
_context->io_ctx));
- auto directory = DORIS_TRY(
- index_file_reader->open(&inverted_reader->get_index_meta(),
_context->io_ctx));
- auto index_searcher_builder = DORIS_TRY(
-
IndexSearcherBuilder::create_index_searcher_builder(inverted_reader->type()));
- auto searcher_result =
-
DORIS_TRY(index_searcher_builder->get_index_searcher(directory.get()));
- auto reader_size = index_searcher_builder->get_reader_size();
-
- auto* stream =
static_cast<DorisCompoundReader*>(directory.get())->getDorisIndexInput();
- DBUG_EXECUTE_IF(
- "FieldReaderResolver.resolve.io_ctx", ({
- const auto* cur_io_ctx = (const
io::IOContext*)stream->getIoContext();
- if (cur_io_ctx->file_cache_stats) {
- if (cur_io_ctx->file_cache_stats !=
&_context->stats->file_cache_stats) {
- LOG(FATAL) << "search: io_ctx file_cache_stats
mismatch: "
- << cur_io_ctx->file_cache_stats << " vs
"
- << &_context->stats->file_cache_stats;
- }
- }
- }));
- stream->setIoContext(nullptr);
- stream->setIndexFile(false);
-
- auto* cache_value = new
InvertedIndexSearcherCache::CacheValue(std::move(searcher_result),
-
reader_size, UnixMillis());
- InvertedIndexSearcherCache::instance()->insert(searcher_cache_key,
cache_value,
- &searcher_cache_handle);
-
- auto new_variant = searcher_cache_handle.get_index_searcher();
- auto* new_ptr = std::get_if<FulltextIndexSearcherPtr>(&new_variant);
- if (new_ptr != nullptr && *new_ptr != nullptr) {
- reader_holder = std::shared_ptr<lucene::index::IndexReader>(
- (*new_ptr)->getReader(), [](lucene::index::IndexReader*)
{});
- }
-
- if (!reader_holder) {
- return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "failed to build IndexSearcher for field '{}'",
field_name);
- }
- }
-
- _searcher_cache_handles.push_back(std::move(searcher_cache_handle));
-
- resolved.lucene_reader = reader_holder;
- resolved.execution_mode = SearchFieldExecutionMode::CLUCENE;
- _binding_readers[binding_key] = reader_holder;
- _field_readers[resolved.stored_field_wstr] = reader_holder;
- _readers.emplace_back(reader_holder);
+ // The index opens through the reader's own cache path, the one MATCH
opens with, and binds
+ // the field as the engine's source.
+ std::unique_ptr<segment_v2::OpenedIndex> opened;
+ index_query::IndexSourcePtr source;
+ RETURN_IF_ERROR(
+ inverted_reader->open_source(_context, resolved.stored_field_wstr,
&opened, &source));
Review Comment:
[P1] Preserve physical gram admission when opening SEARCH sources
After backing up a dense3 SNII table, dropping it, recreating its
analyzer/tokenizer names as legacy ngram2, and restoring the table,
`is_gram_family()` sees only the current policy and returns false. This
`open_source()` path bypasses `_admit/check_open`, so `search('body:abcdef')`
probes the persisted trigram dictionary with bigrams and silently returns no
rows. The former native leaf called `reader->query()`, which rejected that
mismatch;
`SniiGramCacheTest.AnalyzedQueriesDeclineWhenTheAnalyzerNoLongerCutsGrams`
already covers this recovery state for reader queries. Check the opened
segment's recorded `GramScheme` before admitting SEARCH, which has no row
fallback, and add the same case through the real SEARCH resolver.
##########
be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp:
##########
@@ -255,6 +297,61 @@ std::vector<TermInfo>
InvertedIndexAnalyzer::get_analyse_result(
return get_analyse_result(reader, analyzer.get());
}
+Status InvertedIndexAnalyzer::analyze(std::string_view value, const
InvertedIndexAnalyzerCtx* ctx,
+ const std::map<std::string,
std::string>& properties,
+ std::vector<TermInfo>* out) {
+ try {
+ if (ctx == nullptr) {
+ *out = get_analyse_result(value, properties);
+ } else if (!ctx->requires_analysis()) {
+ // Every string, the empty one included, is a term of an
untokenized index.
+ *out = {TermInfo {.term = std::string(value)}};
+ } else if (const auto analyzer = ctx->get_analyzer(); analyzer !=
nullptr) {
+ auto reader = create_reader(ctx->char_filter_map);
+ reader->init(value.data(), static_cast<int32_t>(value.size()),
true);
+ *out = get_analyse_result(reader, analyzer.get());
+ } else {
+ *out = get_analyse_result(value, properties);
+ }
+ } catch (const CLuceneError& e) {
+ return
Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("analyzing '{}' failed:
{}",
+ value,
e.what());
+ } catch (const Exception& e) {
+ return
Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("analyzing '{}' failed:
{}",
+ value,
e.what());
+ }
+ return Status::OK();
+}
+
+std::string InvertedIndexAnalyzer::normalize(const std::string& value,
+ const std::map<std::string,
std::string>& properties) {
+ const std::string analyzer_name =
get_analyzer_name_from_properties(properties);
+ if (analyzer_name.empty() || is_builtin_analyzer(analyzer_name)) {
+ // The index writers read a builtin analyzer's text through the
index's character filter.
+ const InvertedIndexParserType parser_type =
get_inverted_index_parser_type_from_string(
+ analyzer_name.empty() ?
get_parser_string_from_properties(properties)
+ : analyzer_name);
+ return normalize_builtin_term(
+ parser_type,
+ get_parser_lowercase_from_properties<true>(properties) ==
+ INVERTED_INDEX_PARSER_TRUE,
+
filter_characters(get_parser_char_filter_map_from_properties(properties),
value));
+ }
+ auto* index_policy_mgr = doris::ExecEnv::GetInstance()->index_policy_mgr();
+ if (index_policy_mgr == nullptr) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "Index policy manager is not initialized");
+ }
+ const auto normalizer =
index_policy_mgr->get_normalizer_by_name(analyzer_name);
+ if (normalizer == nullptr) {
+ return value;
+ }
+ auto reader = create_reader({});
Review Comment:
[P1] Apply the index character filter in the custom-policy branch
A named analyzer with `tokenizer=standard` and `token_filter=lowercase` can
also use an index-level `char_replace` filter replacing `-` with `x`. The
writer stores `A-BCD` as `axbcd`, but `search('body:A-B*')` reaches this empty
reader and normalizes the prefix to `a-b`, missing the row. Unlike the builtin
branch fixed for discussion 4244238493, this separate named-policy branch only
applies filters declared in the policy; the `normalizer == nullptr` return also
skips the outer filter. Apply the index-level filter before the policy
normalizer and cover their combination through SEARCH.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
