airborne12 commented on code in PR #68758: URL: https://github.com/apache/doris/pull/68758#discussion_r4265801972
########## be/src/storage/index/inverted/reader/text_index_reader.cpp: ########## @@ -0,0 +1,676 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/reader/text_index_reader.h" + +#include <algorithm> +#include <map> +#include <memory> +#include <roaring/roaring.hh> +#include <set> +#include <string> +#include <string_view> +#include <type_traits> + +#include "common/config.h" +#include "common/exception.h" +#include "common/logging.h" +#include "common/status.h" +#include "core/data_type/primitive_type.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/analysis/analyzer/analyzer.h" +#include "storage/index/inverted/analysis/inverted_index_parser.h" +#include "storage/index/inverted/common/inverted_index_cache.h" +#include "storage/index/inverted/common/inverted_index_query_type.h" +#include "storage/index/inverted/common/single_flight.h" +#include "storage/index/inverted/common/string_helper.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/inverted/query/engine/all_query/all_query.h" +#include "storage/index/inverted/query/engine/bit_set_query/bit_set_query.h" +#include "storage/index/inverted/query/engine/boolean_query/boolean_query_builder.h" +#include "storage/index/inverted/query/engine/boolean_query/listed_terms.h" +#include "storage/index/inverted/query/engine/boolean_query/operator.h" +#include "storage/index/inverted/query/engine/collect/doc_set_collector.h" +#include "storage/index/inverted/query/engine/complete_null_bitmap.h" +#include "storage/index/inverted/query/engine/expand_query/expand_query.h" +#include "storage/index/inverted/query/engine/phrase_prefix_query/phrase_prefix_query.h" +#include "storage/index/inverted/query/engine/phrase_query/multi_phrase_query.h" +#include "storage/index/inverted/query/engine/phrase_query/phrase_query.h" +#include "storage/index/inverted/query/engine/term_query/term_query.h" +#include "storage/index/inverted/query/exec/docid_set_ops.h" +#include "storage/index/inverted/query/exec/roaring_docid_sink.h" +#include "storage/index/inverted/query/exec/term_pattern.h" +#include "storage/index/inverted/query/logical/match_lowering.h" +#include "storage/index/inverted/query/scoring/bm25_similarity.h" +#include "storage/olap_common.h" +#include "util/defer_op.h" + +namespace doris::segment_v2 { + +namespace logical = index_query::logical; +namespace query_v2 = inverted_index::query_v2; + +namespace { + +namespace logical = index_query::logical; +namespace query_v2 = inverted_index::query_v2; + +query_v2::QueryPtr term_query(const IndexQueryContextPtr& context, const std::wstring& field, + const std::string& term) { + return std::make_shared<query_v2::TermQuery>(context, field, term); +} + +// One term is queried as itself; several form the boolean the set asks for. SEARCH counts a +// threshold before a set reaches here. +query_v2::QueryPtr term_set_query(const IndexQueryContextPtr& context, const std::wstring& field, + const std::string& binding_key, const logical::TermSet& set) { + DORIS_CHECK(set.min_should_match == 0); + if (set.terms.size() == 1) { + return term_query(context, field, set.terms.front()); + } + auto builder = query_v2::create_operator_boolean_query_builder( + set.require_all ? query_v2::OperatorType::OP_AND : query_v2::OperatorType::OP_OR); + for (const auto& term : set.terms) { + builder->add(term_query(context, field, term), binding_key); + } + return builder->build(); +} + +Status phrase_query(const IndexQueryContextPtr& context, const std::wstring& field, + const logical::Phrase& phrase, const roaring::Roaring* candidates, + query_v2::QueryPtr* out) { + const bool single_terms = std::ranges::all_of( + phrase.slots, [](const TermInfo& slot) { return slot.is_single_term(); }); + if (phrase.prefix) { + if (!single_terms) { + return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>( + "a phrase prefix with several terms at one position is not supported"); + } + *out = std::make_shared<query_v2::PhrasePrefixQuery>(context, field, phrase.slots, + candidates, phrase.suffix); + return Status::OK(); + } + const index_query::PhraseQueryOptions options { + .slop = static_cast<uint32_t>(phrase.slop), + .ordered = phrase.ordered, + .candidates = candidates, + .candidate_rows_consumed = + candidates == nullptr ? nullptr : &context->candidate_rows_consumed}; + if (single_terms) { + *out = std::make_shared<query_v2::PhraseQuery>(context, field, phrase.slots, options); + } else { + *out = std::make_shared<query_v2::MultiPhraseQuery>(context, field, phrase.slots, options); + } + return Status::OK(); +} + +index_query::TermPatternKind pattern_kind(logical::ExpandKind kind) { + switch (kind) { + case logical::ExpandKind::kPrefix: + return index_query::TermPatternKind::kPrefix; + case logical::ExpandKind::kRegexp: + return index_query::TermPatternKind::kRegexp; + case logical::ExpandKind::kContains: + return index_query::TermPatternKind::kContains; + case logical::ExpandKind::kWildcard: + default: + return index_query::TermPatternKind::kWildcard; + } +} + +} // namespace + +Status TextIndexReader::open_source(const IndexQueryContextPtr& context, const std::wstring& field, + std::unique_ptr<OpenedIndex>* opened, + index_query::IndexSourcePtr* source) { + RETURN_IF_ERROR(_open_index(context, opened)); + *source = _bind_source(context, field, **opened); + return Status::OK(); +} + +Status run_leaf(const IndexQueryContextPtr& context, const std::wstring& field, + const logical::Node& leaf, const roaring::Roaring* candidates, bool scoring, + index_query::IndexSourcePtr source, uint32_t doc_count, + const std::shared_ptr<roaring::Roaring>& result) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_timer); + if (const auto* expand = leaf.as<logical::Expand>(); expand != nullptr && !scoring) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_exec_timer); + if (doc_count == 0) { + return Status::OK(); + } + const auto kind = pattern_kind(expand->kind); + index_query::TermPattern pattern; + THROW_IF_ERROR(index_query::TermPattern::create(kind, expand->pattern, &pattern)); + THROW_IF_ERROR(query_v2::collect_expanded_rows( + *source, pattern, + index_query::expansion_limit(kind, index_query::max_expansions(*context)), nullptr, + result.get())); + return Status::OK(); + } + std::span<const std::string> terms; + if (const auto* term = leaf.as<logical::Term>(); term != nullptr) { + terms = std::span(&term->term, 1); + } else if (const auto* set = leaf.as<logical::TermSet>(); + set != nullptr && (!set->require_all || set->terms.size() == 1) && + set->min_should_match == 0) { + terms = set->terms; + } + if (!terms.empty() && (!scoring || terms.size() > 1)) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_exec_timer); + if (!scoring) { + index_query::RoaringDocIdSink sink(*result); + return source->collect_terms(terms, sink); + } + query_v2::ListedTerms listed(std::move(source), nullptr); + for (size_t i = 0; i < terms.size(); ++i) { + auto similarity = std::make_shared<BM25Similarity>(); + similarity->for_one_term(context, field, + inverted_index::StringHelper::to_wstring(terms[i])); + listed.add(i, terms[i], std::move(similarity)); + } + const auto scorer = listed.scored_disjunction(); + if (context->collection_similarity != nullptr) { + *result |= query_v2::collect_scored_rows(scorer, 0, *context->collection_similarity); + } else { + query_v2::collect_true_rows(scorer, result.get()); + } + return Status::OK(); + } + query_v2::WeightPtr weight; + { + SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_init_timer); + query_v2::QueryPtr query; + RETURN_IF_ERROR(plan_query(leaf, context, field, "", candidates, &query)); + weight = query->weight(scoring); + } + SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_exec_timer); + query_v2::QueryExecutionContext exec_ctx; + exec_ctx.segment_num_rows = doc_count; + exec_ctx.field_sources.emplace(field, std::move(source)); + query_v2::collect_multi_segment_doc_set(weight, exec_ctx, "", result, + context->collection_similarity, scoring); + return Status::OK(); +} + +Status plan_query(const logical::Node& leaf, const IndexQueryContextPtr& context, + const std::wstring& field, const std::string& binding_key, + const roaring::Roaring* candidates, query_v2::QueryPtr* out) { + if (const auto* term = leaf.as<logical::Term>()) { + *out = term_query(context, field, term->term); + } else if (const auto* set = leaf.as<logical::TermSet>()) { + *out = term_set_query(context, field, binding_key, *set); + } else if (const auto* phrase = leaf.as<logical::Phrase>()) { + return phrase_query(context, field, *phrase, candidates, out); + } else if (const auto* expand = leaf.as<logical::Expand>()) { + *out = std::make_shared<query_v2::ExpandQuery>(context, field, pattern_kind(expand->kind), + expand->pattern); + } else if (leaf.as<logical::Exists>() != nullptr) { + *out = std::make_shared<query_v2::AllQuery>(field, /*nullable=*/true); + } else if (leaf.as<logical::Empty>() != nullptr) { + *out = std::make_shared<query_v2::BitSetQuery>(roaring::Roaring()); Review Comment: By design, SEARCH is going to stop handling NULL and ignore it: it will not track NULL or UNKNOWN rows for nullable fields. We are therefore leaving its current NULL handling as is in this PR instead of fixing it. Without NULL tracking, an analyzer-empty leaf is an empty result with no NULL bitmap, and `NOT` treats the NULL row like any other row the clause does not match. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
