This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 3d43e02e95c branch-3.1: [fix](inverted index) enable custom analyzer
support for match without index #55858 (#56267)
3d43e02e95c is described below
commit 3d43e02e95c89db305b3b9e1b8aec3f766328310
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Sep 25 17:21:47 2025 +0800
branch-3.1: [fix](inverted index) enable custom analyzer support for match
without index #55858 (#56267)
Cherry-picked from #55858
Co-authored-by: zzzxl <[email protected]>
---
be/src/vec/functions/match.cpp | 11 ++++-------
.../analyzer/test_custom_analyzer.out | 3 +++
.../analyzer/test_custom_analyzer.groovy | 23 ++++++++++++++++++++++
3 files changed, 30 insertions(+), 7 deletions(-)
diff --git a/be/src/vec/functions/match.cpp b/be/src/vec/functions/match.cpp
index f992c0bb324..166109775eb 100644
--- a/be/src/vec/functions/match.cpp
+++ b/be/src/vec/functions/match.cpp
@@ -103,11 +103,6 @@ Status FunctionMatchBase::execute_impl(FunctionContext*
context, Block& block,
context->get_function_state(FunctionContext::FRAGMENT_LOCAL));
}
- if (!inverted_index_ctx->custom_analyzer.empty()) {
- return Status::NotSupported(
- "Custom analyzer is not supported for unindexed MATCH
operations.");
- }
-
const ColumnPtr source_col =
block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
const auto* values = check_and_get_column<ColumnString>(source_col.get());
@@ -181,7 +176,8 @@ std::vector<TermInfo>
FunctionMatchBase::analyse_query_str_token(
if (inverted_index_ctx == nullptr) {
return query_tokens;
}
- if (inverted_index_ctx->parser_type ==
InvertedIndexParserType::PARSER_NONE) {
+ if (inverted_index_ctx->parser_type ==
InvertedIndexParserType::PARSER_NONE &&
+ inverted_index_ctx->custom_analyzer.empty()) {
query_tokens.emplace_back(match_query_str);
return query_tokens;
}
@@ -216,7 +212,8 @@ inline std::vector<TermInfo>
FunctionMatchBase::analyse_data_token(
}
} else {
const auto& str_ref = string_col->get_data_at(current_block_row_idx);
- if (inverted_index_ctx->parser_type ==
InvertedIndexParserType::PARSER_NONE) {
+ if (inverted_index_ctx->parser_type ==
InvertedIndexParserType::PARSER_NONE &&
+ inverted_index_ctx->custom_analyzer.empty()) {
data_tokens.emplace_back(str_ref.to_string());
} else {
auto reader =
doris::segment_v2::inverted_index::InvertedIndexAnalyzer::create_reader(
diff --git
a/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer.out
b/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer.out
index a7c849ec5d5..4c22ab33fd2 100644
--- a/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer.out
+++ b/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer.out
@@ -38,3 +38,6 @@
-- !sql --
3 Wikipedia;Miscellaneous-Jj102786 / 3tle Born Oct 27th 1986 @
Blytheville, Arkansas @ 9:14pm 23 yrs of age male,white Cucassion American
raised Religion:Pentocostal,Church of God
+-- !sql --
+1 GET /french/images/nav_venue_off.gif HTTP/1.0
+
diff --git
a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer.groovy
b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer.groovy
index a2c0ba453bf..9c2315b27eb 100644
---
a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer.groovy
+++
b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer.groovy
@@ -138,4 +138,27 @@ suite("test_custom_analyzer", "p0") {
logger.info("used by index")
}
}
+
+
+
+ try {
+ sql "DROP TABLE IF EXISTS test_custom_analyzer_3"
+ sql """
+ CREATE TABLE test_custom_analyzer_3 (
+ `a` bigint NOT NULL AUTO_INCREMENT(1),
+ `ch` text NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`a`)
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """ insert into test_custom_analyzer_3 values(1, "GET
/french/images/nav_venue_off.gif HTTP/1.0"); """
+ sql """ alter table test_custom_analyzer_3 add index idx_ch(`ch`)
using inverted properties("support_phrase" = "true", "analyzer" =
"lowercase_delimited"); """
+
+ qt_sql """ select * from test_custom_analyzer_3 where ch match
'nav_venue_off.gif'; """
+ } catch (SQLException e) {
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
