This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 511a8a0f893 [fix](be) Normalize SNII prefix syntax before analysis
(#66874)
511a8a0f893 is described below
commit 511a8a0f8931955ee7a69c328bfe45654472f0d3
Author: Jack <[email protected]>
AuthorDate: Mon Aug 24 11:42:58 2026 +0800
[fix](be) Normalize SNII prefix syntax before analysis (#66874)
### What problem does this PR solve?
Issue Number: None
Related PR: #66052
Problem Summary:
`SEARCH ... PREFIX` values retain their trailing DSL `*` marker in FE.
SNII classifies a custom keyword analyzer as analyzed, but that analyzer
preserves the marker as a literal byte. Consequently, a predicate such
as `fail*` sent `fail*` to the phrase-prefix reader instead of the
indexed prefix `fail` and returned no rows, while V3 matched the
expected documents.
This PR strips exactly the trailing DSL marker before SNII phrase-prefix
analysis. Non-analyzed keyword fields continue to use `WILDCARD_QUERY`
with the marker intact, ordinary WILDCARD clauses are unchanged, and the
V3 path is unchanged. The change is query-only: it does not alter index
writes or the on-disk format, so existing SNII indexes are fixed after
upgrade without rebuilding.
TDD evidence:
- RED: a real custom keyword tokenizer/analyzer preserved `fail*`;
`FunctionSearch::build_leaf_query` sent `fail*` and produced an empty
bitmap instead of `{0, 2}`.
- GREEN: the same path sends `fail`, retains
`MATCH_PHRASE_PREFIX_QUERY`, and produces `{0, 2}`.
- All 98 `FunctionSearchTest` cases pass.
- `./build.sh --be -j 192` passes with ASAN.
- `build-support/check-format.sh` passes.
### Release note
Fix SEARCH PREFIX queries on SNII indexes that use a custom keyword
analyzer.
---
be/src/exprs/function/function_search.cpp | 34 ++++++------
be/test/exprs/function/function_search_test.cpp | 74 +++++++++++++++++++++++++
2 files changed, 91 insertions(+), 17 deletions(-)
diff --git a/be/src/exprs/function/function_search.cpp
b/be/src/exprs/function/function_search.cpp
index a96983945cd..bb1cceda64d 100644
--- a/be/src/exprs/function/function_search.cpp
+++ b/be/src/exprs/function/function_search.cpp
@@ -860,14 +860,10 @@ Status FunctionSearch::build_leaf_query(const
TSearchClause& clause,
!inverted_index::InvertedIndexAnalyzer::should_analyzer(
binding.index_properties)) {
// FE keeps the trailing '*' in the PREFIX value unstripped
(SearchDslParser.java).
- // On an analysed field the tokenizer drops it, leaving a clean
single prefix term,
- // so the default clause_type_to_query_type mapping
(MATCH_PHRASE_PREFIX_QUERY) is
- // correct as-is. On a keyword (non-analysed) field the whole
string -- '*' included
- // -- becomes one literal term
(InvertedIndexAnalyzer::get_analyse_result), so
- // MATCH_PHRASE_PREFIX_QUERY would search for a term that can
never exist. Route
- // those to WILDCARD_QUERY instead, exactly like the CLucene path's
- // WildcardQuery(value) for PREFIX
(function_search.cpp:1075-1076): the reader
- // forwards a WILDCARD_QUERY value unanalysed, so the trailing '*'
works the same way.
+ // A non-analysed keyword field needs that marker for
WILDCARD_QUERY, matching the
+ // CLucene path's WildcardQuery(value) for PREFIX
(function_search.cpp:1075-1076).
+ // Analysed fields stay on MATCH_PHRASE_PREFIX_QUERY; its reader
input is normalized
+ // below because a custom keyword tokenizer preserves '*' as a
literal byte.
snii_query_type = InvertedIndexQueryType::WILDCARD_QUERY;
}
@@ -903,12 +899,17 @@ Status FunctionSearch::build_leaf_query(const
TSearchClause& clause,
if (clause_type == "WILDCARD" && value == "*") {
data_bitmap->addRange(0, num_rows);
} else {
- // Wildcard patterns carry the analyzer's lower_case semantics;
every other clause
- // passes its value through untouched, since the reader analyses
it.
- std::string pattern =
- clause_type == "WILDCARD"
- ? normalize_wildcard_pattern(value,
binding.index_properties)
- : value;
+ // Wildcard patterns carry the analyzer's lower_case semantics.
Phrase-prefix analysis
+ // consumes the prefix text, not the DSL's trailing '*' syntax
marker. This must happen
+ // before analysis because custom keyword tokenizers preserve the
marker as a literal.
+ std::string pattern = value;
+ if (clause_type == "WILDCARD") {
+ pattern = normalize_wildcard_pattern(value,
binding.index_properties);
+ } else if (snii_query_type ==
InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) {
+ DORIS_CHECK(clause_type == "PREFIX");
+ DORIS_CHECK(pattern.ends_with('*'));
+ pattern.pop_back();
+ }
Field query_value = Field::create_field<TYPE_STRING>(pattern);
const bool raw_pattern_query =
snii_query_type == InvertedIndexQueryType::WILDCARD_QUERY
||
@@ -925,10 +926,9 @@ Status FunctionSearch::build_leaf_query(const
TSearchClause& clause,
if (reader_context != context) {
context->merge_reader_outputs(*reader_context);
}
- // Restore the pre-normalization value for WILDCARD so the trace
still shows what the
- // caller actually asked for, not just what was sent to the reader.
+ // Preserve the caller's pre-normalization value in the trace.
std::string log_suffix =
- clause_type == "WILDCARD" ? (" (original='" + value +
"')") : std::string();
+ pattern != value ? (" (original='" + value + "')") :
std::string();
VLOG_DEBUG << "search: SNII clause processed, type=" << clause_type
<< ", field=" << field_name << ", value='" << pattern
<< "'" << log_suffix;
}
diff --git a/be/test/exprs/function/function_search_test.cpp
b/be/test/exprs/function/function_search_test.cpp
index 0edc417eb8b..791c28968e0 100644
--- a/be/test/exprs/function/function_search_test.cpp
+++ b/be/test/exprs/function/function_search_test.cpp
@@ -42,6 +42,7 @@
#include "runtime/index_policy/index_policy_mgr.h"
#include "storage/index/index_file_reader.h"
#include "storage/index/index_iterator.h"
+#include "storage/index/inverted/analyzer/analyzer.h"
#include "storage/index/inverted/inverted_index_iterator.h"
#include "storage/index/inverted/inverted_index_parser.h"
#include "storage/index/inverted/query_v2/collect/doc_set_collector.h"
@@ -2824,6 +2825,79 @@ TEST_F(FunctionSearchTest,
TestSniiNativeKeywordPrefixRoutesToWildcardQuery) {
expect_bitmap_eq(collect_docs(scorer), {0, 2});
}
+TEST_F(FunctionSearchTest,
TestSniiNativeCustomKeywordPrefixStripsDslSuffixBeforeAnalysis) {
+ auto* exec_env = ExecEnv::GetInstance();
+ auto* previous_policy_mgr = exec_env->index_policy_mgr();
+ IndexPolicyMgr scoped_policy_mgr;
+ exec_env->_index_policy_mgr = &scoped_policy_mgr;
+ DEFER(exec_env->_index_policy_mgr = previous_policy_mgr);
+
+ TIndexPolicy tokenizer;
+ tokenizer.id = 910030;
+ tokenizer.name = "function_search_keyword_tokenizer";
+ tokenizer.type = TIndexPolicyType::TOKENIZER;
+ tokenizer.properties["type"] = "keyword";
+
+ TIndexPolicy analyzer;
+ analyzer.id = 910031;
+ analyzer.name = "function_search_keyword_analyzer";
+ analyzer.type = TIndexPolicyType::ANALYZER;
+ analyzer.properties["tokenizer"] = tokenizer.name;
+ scoped_policy_mgr.apply_policy_changes({tokenizer, analyzer}, {});
+
+ std::map<std::string, std::string> properties {
+ {INVERTED_INDEX_ANALYZER_NAME_KEY, analyzer.name}};
+
ASSERT_TRUE(inverted_index::InvertedIndexAnalyzer::should_analyzer(properties));
+ auto raw_terms = inverted_index::InvertedIndexAnalyzer::get_analyse_result(
+ "fail*", properties,
inverted_index::AnalysisPurpose::kPhrasePrefixQuery);
+ ASSERT_EQ(1, raw_terms.size());
+ EXPECT_EQ("fail*", raw_terms[0].get_single_term());
+
+ OlapReaderStatistics stats;
+ auto context = std::make_shared<IndexQueryContext>();
+ context->stats = &stats;
+ auto index_meta = make_test_inverted_index(24, properties);
+ auto index_file_reader =
std::make_shared<RejectingCluceneIndexFileReader>();
+ auto reader =
+ std::make_shared<RecordingNativeInvertedIndexReader>(&index_meta,
index_file_reader);
+ reader->set_query_result("fail", make_bitmap({0, 2}));
+ segment_v2::InvertedIndexIterator iterator;
+ iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader);
+
+ std::unordered_map<std::string, IndexFieldNameAndTypePair>
data_type_with_names;
+ data_type_with_names.emplace(
+ "body", IndexFieldNameAndTypePair {"body",
std::make_shared<DataTypeString>()});
+ std::unordered_map<std::string, IndexIterator*> iterators;
+ iterators["body"] = &iterator;
+ TSearchFieldBinding field_binding;
+ field_binding.field_name = "body";
+ field_binding.index_properties = properties;
+ field_binding.__isset.index_properties = true;
+ FieldReaderResolver resolver(data_type_with_names, iterators, context,
{field_binding});
+
+ auto clause = make_leaf_clause("PREFIX", "fail*");
+ inverted_index::query_v2::QueryPtr query;
+ std::string binding_key;
+ auto status = function_search->build_leaf_query(clause, context, resolver,
&query, &binding_key,
+ "OR", 0, 4);
+
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ ASSERT_NE(nullptr, query);
+ EXPECT_EQ(1, reader->query_calls);
+ EXPECT_EQ(InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY,
reader->last_query_type);
+ EXPECT_EQ("fail", reader->last_query_value);
+
+ auto weight = query->weight(false);
+ ASSERT_NE(nullptr, weight);
+ inverted_index::query_v2::QueryExecutionContext exec_ctx;
+ exec_ctx.segment_num_rows = 4;
+ auto scorer = weight->scorer(exec_ctx, binding_key);
+ ASSERT_NE(nullptr, scorer);
+ expect_bitmap_eq(collect_docs(scorer), {0, 2});
+
+ scoped_policy_mgr.apply_policy_changes({}, {tokenizer.id, analyzer.id});
+}
+
// Shared wiring for the SNII native SEARCH scoring tests: one fake SNII
reader bound to field
// "body" behind a standard analyzer, plus the resolver build_leaf_query
needs. The resolver keeps
// references to the maps, so they must be owned by something that outlives it.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
