airborne12 commented on code in PR #68758:
URL: https://github.com/apache/doris/pull/68758#discussion_r4256553584
##########
be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp:
##########
@@ -255,6 +297,61 @@ std::vector<TermInfo>
InvertedIndexAnalyzer::get_analyse_result(
return get_analyse_result(reader, analyzer.get());
}
+Status InvertedIndexAnalyzer::analyze(std::string_view value, const
InvertedIndexAnalyzerCtx* ctx,
+ const std::map<std::string,
std::string>& properties,
+ std::vector<TermInfo>* out) {
+ try {
+ if (ctx == nullptr) {
+ *out = get_analyse_result(value, properties);
+ } else if (!ctx->requires_analysis()) {
+ // Every string, the empty one included, is a term of an
untokenized index.
+ *out = {TermInfo {.term = std::string(value)}};
+ } else if (const auto analyzer = ctx->get_analyzer(); analyzer !=
nullptr) {
+ auto reader = create_reader(ctx->char_filter_map);
+ reader->init(value.data(), static_cast<int32_t>(value.size()),
true);
+ *out = get_analyse_result(reader, analyzer.get());
+ } else {
+ *out = get_analyse_result(value, properties);
+ }
+ } catch (const CLuceneError& e) {
+ return
Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("analyzing '{}' failed:
{}",
+ value,
e.what());
+ } catch (const Exception& e) {
+ return
Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("analyzing '{}' failed:
{}",
+ value,
e.what());
+ }
+ return Status::OK();
+}
+
+std::string InvertedIndexAnalyzer::normalize(const std::string& value,
+ const std::map<std::string,
std::string>& properties) {
+ const std::string analyzer_name =
get_analyzer_name_from_properties(properties);
+ if (analyzer_name.empty() || is_builtin_analyzer(analyzer_name)) {
+ // The index writers read a builtin analyzer's text through the
index's character filter.
+ const InvertedIndexParserType parser_type =
get_inverted_index_parser_type_from_string(
+ analyzer_name.empty() ?
get_parser_string_from_properties(properties)
+ : analyzer_name);
+ return normalize_builtin_term(
+ parser_type,
+ get_parser_lowercase_from_properties<true>(properties) ==
+ INVERTED_INDEX_PARSER_TRUE,
+
filter_characters(get_parser_char_filter_map_from_properties(properties),
value));
+ }
+ auto* index_policy_mgr = doris::ExecEnv::GetInstance()->index_policy_mgr();
+ if (index_policy_mgr == nullptr) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "Index policy manager is not initialized");
+ }
+ const auto normalizer =
index_policy_mgr->get_normalizer_by_name(analyzer_name);
+ if (normalizer == nullptr) {
+ return value;
+ }
+ auto reader = create_reader({});
Review Comment:
Confirmed and fixed in 0750c734ed8. The custom-analyzer branch of
`InvertedIndexAnalyzer::normalize` now reads the pattern through the index's
character filter, with the same `create_reader(char_filter_map)` the writers
use, before the analyzer's normalizer, and applies the filter alone when the
analyzer has no per-character filters.
`FunctionSearchTest.TestSniiNativePrefixAppliesTheIndexCharacterFilterToACustomAnalyzer`
runs `A-B*` through SEARCH on a `standard` + `lowercase` analyzer with an
index-level `-` to `x` filter and expects the stem `axb`; it was `a-b` before
the fix.
`AnalyzerTest.TestNormalizeAppliesTheIndexCharacterFilterToCustomAnalyzers`
compares normalize with the terms the real custom analyzers write, with and
without per-character filters (`axb` and `AxB`; they were `a-b` and `A-B`).
##########
be/src/exprs/function/search/variant_inverted_index_search.cpp:
##########
@@ -294,117 +297,19 @@ Status FieldReaderResolver::resolve(const std::string&
field_name,
return Status::OK();
}
- if (index_file_reader->get_storage_format() ==
InvertedIndexStorageFormatPB::SNII) {
- resolved.execution_mode = SearchFieldExecutionMode::SNII_NATIVE;
- _cache.emplace(binding_key, resolved);
- if (is_variant_sub) {
- add_search_binding_diagnostic(
- _context,
- fmt::format("[VariantSearchBinding] phase=field_resolve "
- "result=selected_snii_native logical_field={}
stored_field={} "
- "query_type={} effective_query_type={}
index_id={} suffix={} "
- "reader_type={} analyzer_key={} index_file={}",
- field_name, stored_field_name,
query_type_to_string(query_type),
- query_type_to_string(effective_query_type),
- inverted_reader->get_index_id(),
-
inverted_reader->get_index_meta().get_index_suffix(),
- reader_type_to_string(inverted_reader->type()),
- resolved.analyzer_key,
- index_file_reader->get_index_file_path(
- &inverted_reader->get_index_meta())));
- }
- *binding = resolved;
- return Status::OK();
- }
-
- auto index_file_key =
-
index_file_reader->get_index_file_cache_key(&inverted_reader->get_index_meta());
- InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key);
- InvertedIndexCacheHandle searcher_cache_handle;
-
- bool searcher_cache_enabled =
- _context->runtime_state != nullptr &&
-
_context->runtime_state->query_options().enable_inverted_index_searcher_cache;
-
- bool cache_hit = false;
- if (searcher_cache_enabled) {
- int64_t lookup_dummy = 0;
- SCOPED_RAW_TIMER(_context->stats ?
&_context->stats->inverted_index_lookup_timer
- : &lookup_dummy);
- cache_hit =
InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key,
-
&searcher_cache_handle);
- }
-
- std::shared_ptr<lucene::index::IndexReader> reader_holder;
- if (cache_hit) {
- if (_context->stats) {
- _context->stats->inverted_index_searcher_cache_hit++;
- }
- auto searcher_variant = searcher_cache_handle.get_index_searcher();
- auto* searcher_ptr =
std::get_if<FulltextIndexSearcherPtr>(&searcher_variant);
- if (searcher_ptr != nullptr && *searcher_ptr != nullptr) {
- reader_holder = std::shared_ptr<lucene::index::IndexReader>(
- (*searcher_ptr)->getReader(),
[](lucene::index::IndexReader*) {});
- }
- }
-
- if (!reader_holder) {
- if (_context->stats) {
- _context->stats->inverted_index_searcher_cache_miss++;
- }
- int64_t dummy_timer = 0;
- SCOPED_RAW_TIMER(_context->stats ?
&_context->stats->inverted_index_searcher_open_timer
- : &dummy_timer);
- RETURN_IF_ERROR(
-
index_file_reader->init(config::inverted_index_read_buffer_size,
_context->io_ctx));
- auto directory = DORIS_TRY(
- index_file_reader->open(&inverted_reader->get_index_meta(),
_context->io_ctx));
- auto index_searcher_builder = DORIS_TRY(
-
IndexSearcherBuilder::create_index_searcher_builder(inverted_reader->type()));
- auto searcher_result =
-
DORIS_TRY(index_searcher_builder->get_index_searcher(directory.get()));
- auto reader_size = index_searcher_builder->get_reader_size();
-
- auto* stream =
static_cast<DorisCompoundReader*>(directory.get())->getDorisIndexInput();
- DBUG_EXECUTE_IF(
- "FieldReaderResolver.resolve.io_ctx", ({
- const auto* cur_io_ctx = (const
io::IOContext*)stream->getIoContext();
- if (cur_io_ctx->file_cache_stats) {
- if (cur_io_ctx->file_cache_stats !=
&_context->stats->file_cache_stats) {
- LOG(FATAL) << "search: io_ctx file_cache_stats
mismatch: "
- << cur_io_ctx->file_cache_stats << " vs
"
- << &_context->stats->file_cache_stats;
- }
- }
- }));
- stream->setIoContext(nullptr);
- stream->setIndexFile(false);
-
- auto* cache_value = new
InvertedIndexSearcherCache::CacheValue(std::move(searcher_result),
-
reader_size, UnixMillis());
- InvertedIndexSearcherCache::instance()->insert(searcher_cache_key,
cache_value,
- &searcher_cache_handle);
-
- auto new_variant = searcher_cache_handle.get_index_searcher();
- auto* new_ptr = std::get_if<FulltextIndexSearcherPtr>(&new_variant);
- if (new_ptr != nullptr && *new_ptr != nullptr) {
- reader_holder = std::shared_ptr<lucene::index::IndexReader>(
- (*new_ptr)->getReader(), [](lucene::index::IndexReader*)
{});
- }
-
- if (!reader_holder) {
- return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "failed to build IndexSearcher for field '{}'",
field_name);
- }
- }
-
- _searcher_cache_handles.push_back(std::move(searcher_cache_handle));
-
- resolved.lucene_reader = reader_holder;
- resolved.execution_mode = SearchFieldExecutionMode::CLUCENE;
- _binding_readers[binding_key] = reader_holder;
- _field_readers[resolved.stored_field_wstr] = reader_holder;
- _readers.emplace_back(reader_holder);
+ // The index opens through the reader's own cache path, the one MATCH
opens with, and binds
+ // the field as the engine's source.
+ std::unique_ptr<segment_v2::OpenedIndex> opened;
+ index_query::IndexSourcePtr source;
+ RETURN_IF_ERROR(
+ inverted_reader->open_source(_context, resolved.stored_field_wstr,
&opened, &source));
Review Comment:
Confirmed and fixed in 7b6a05bcab9. `OpenedIndex` now reports whether its
segment was written as a gram index, and the SNII one answers from the scheme
the segment recorded. After opening the source, the SEARCH resolver refuses
such a segment with the same `INVERTED_INDEX_NOT_SUPPORTED` it returns for a
gram policy, so a segment restored under a recreated non-gram policy is no
longer probed with bigrams.
`SniiGramCacheTest.SearchRefusesASegmentWrittenAsAGramIndex` recreates the
dense3 names as a legacy bigram tokenizer, the state
`AnalyzedQueriesDeclineWhenTheAnalyzerNoLongerCutsGrams` covers for reader
queries, and resolves the field through the real `FieldReaderResolver`. It
returned OK before the fix.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
