This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 93a21ae81a2 branch-4.2: [improvement](scan) Align scanner split to
segments for MATCH #67915 (#70775)
93a21ae81a2 is described below
commit 93a21ae81a2b6cce0702f1cd576dc833b014c277
Author: Chenyang Sun <[email protected]>
AuthorDate: Fri Nov 20 17:46:03 2026 +0800
branch-4.2: [improvement](scan) Align scanner split to segments for MATCH
#67915 (#70775)
pick from master #67915
Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
---
be/src/exec/operator/olap_scan_operator.cpp | 39 +++++++++++++++++-----
be/src/exec/operator/olap_scan_operator.h | 3 ++
.../analyzer/test_custom_analyzer1.groovy | 8 ++---
.../analyzer/test_ik_analyzer.groovy | 16 ++++-----
.../suites/inverted_index_p0/test_stopwords.groovy | 16 ++++-----
.../suites/search/test_search_dsl_syntax.groovy | 6 ++--
6 files changed, 56 insertions(+), 32 deletions(-)
diff --git a/be/src/exec/operator/olap_scan_operator.cpp
b/be/src/exec/operator/olap_scan_operator.cpp
index 0043a58fd9a..cfcdf630056 100644
--- a/be/src/exec/operator/olap_scan_operator.cpp
+++ b/be/src/exec/operator/olap_scan_operator.cpp
@@ -449,6 +449,17 @@ static bool contains_expr_node_type(const VExprSPtr& expr,
TExprNodeType::type n
});
}
+// Find MATCH recursively; ones nested in AND / OR / NOT count too.
+static bool is_match_expr(const VExprSPtr& expr) {
+ DORIS_CHECK(expr != nullptr);
+ if (expr->node_type() == TExprNodeType::MATCH_PRED ||
+ expr->node_type() == TExprNodeType::SEARCH_EXPR ||
expr->can_push_down_to_index()) {
+ return true;
+ }
+ return std::ranges::any_of(expr->children(),
+ [](const auto& child) { return
is_match_expr(child); });
+}
+
static Status validate_residual_scan_conjuncts(RuntimeState* state,
TPushAggOp::type
push_down_agg_type,
const VExprContextSPtrs&
conjuncts) {
@@ -579,6 +590,24 @@ bool OlapScanLocalState::_read_mor_as_dup() {
return p._olap_scan_node.__isset.read_mor_as_dup &&
p._olap_scan_node.read_mor_as_dup;
}
+// Give each segment a scanner of its own for queries like:
+// SELECT k1 FROM t ORDER BY l2_distance_approximate(embedding, [1.0, 2.0])
LIMIT 2
+// SELECT k1 FROM t WHERE msg MATCH_PHRASE 'error timeout'
+bool OlapScanLocalState::_use_scan_parallelism_by_per_segment() {
+ if (!state()->query_options().__isset.optimize_index_scan_parallelism ||
+ !state()->query_options().optimize_index_scan_parallelism) {
+ return false;
+ }
+ // TODO: Use optimize_index_scan_parallelism for ann range search in the
future.
+ // Currently, ann topn is enough
+ if (_ann_topn_runtime != nullptr) {
+ return true;
+ }
+ return config::is_cloud_mode() &&
+ std::ranges::any_of(_common_expr_ctxs_push_down,
+ [](const auto& ctx) { return
is_match_expr(ctx->root()); });
+}
+
Status OlapScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
if (_scan_ranges.empty()) {
_eos = true;
@@ -643,15 +672,7 @@ Status
OlapScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
scanner_builder.set_min_rows_per_scanner(min_rows_per_scanner);
// BM25 statistics must not depend on how a tablet is split into
scanners.
scanner_builder.set_share_tablet_collection_statistics(_score_runtime
!= nullptr);
- // If the session variable is set, force one scanner per segment.
- if (state()->query_options().__isset.optimize_index_scan_parallelism &&
- state()->query_options().optimize_index_scan_parallelism) {
- // TODO: Use optimize_index_scan_parallelism for ann range search
in the future.
- // Currently, ann topn is enough
- if (_ann_topn_runtime != nullptr) {
- scanner_builder.set_scan_parallelism_by_segment(true);
- }
- }
+
scanner_builder.set_scan_parallelism_by_segment(_use_scan_parallelism_by_per_segment());
RETURN_IF_ERROR(scanner_builder.build_scanners(*scanners));
for (auto& scanner : *scanners) {
diff --git a/be/src/exec/operator/olap_scan_operator.h
b/be/src/exec/operator/olap_scan_operator.h
index 461e13a4cfa..d7354522ff1 100644
--- a/be/src/exec/operator/olap_scan_operator.h
+++ b/be/src/exec/operator/olap_scan_operator.h
@@ -116,6 +116,9 @@ private:
Status _init_scanners(std::list<ScannerSPtr>* scanners) override;
+ // Whether each segment should be scanned by a scanner of its own.
+ bool _use_scan_parallelism_by_per_segment();
+
Status _build_key_ranges_and_filters();
std::vector<std::unique_ptr<TPaloScanRange>> _scan_ranges;
diff --git
a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
index 83d51ad7831..e6c40c479f4 100644
---
a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
+++
b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
@@ -107,11 +107,11 @@ suite("test_custom_analyzer1", "p0") {
sql "sync"
sql """ set enable_segment_limit_pushdown = true; """
- qt_sql """ select * from test_custom_analyzer1 where ch match 'with';
"""
- qt_sql """ select * from test_custom_analyzer1 where ch match 'the';
"""
+ order_qt_sql """ select * from test_custom_analyzer1 where ch match
'with'; """
+ order_qt_sql """ select * from test_custom_analyzer1 where ch match
'the'; """
- qt_sql """ select * from test_custom_analyzer2 where ch match 'hm'; """
- qt_sql """ select * from test_custom_analyzer2 where ch match 'bg'; """
+ order_qt_sql """ select * from test_custom_analyzer2 where ch match
'hm'; """
+ order_qt_sql """ select * from test_custom_analyzer2 where ch match
'bg'; """
} finally {
}
}
\ No newline at end of file
diff --git
a/regression-test/suites/inverted_index_p0/analyzer/test_ik_analyzer.groovy
b/regression-test/suites/inverted_index_p0/analyzer/test_ik_analyzer.groovy
index e806a339fb0..250d8da7b87 100644
--- a/regression-test/suites/inverted_index_p0/analyzer/test_ik_analyzer.groovy
+++ b/regression-test/suites/inverted_index_p0/analyzer/test_ik_analyzer.groovy
@@ -70,17 +70,17 @@ suite("test_ik_analyzer", "p0") {
// Testing ik smart mode
println "Testing ik smart mode:"
- qt_sql """ select * from ${tableNameSmart} where content match_phrase
'北京'; """
- qt_sql """ select * from ${tableNameSmart} where content match_phrase
'计算机科学'; """
- qt_sql """ select * from ${tableNameSmart} where content match_phrase
'数据库管理系统'; """
- qt_sql """ select * from ${tableNameSmart} where content match_phrase
'中华人民共和国'; """
+ order_qt_sql """ select * from ${tableNameSmart} where content
match_phrase '北京'; """
+ order_qt_sql """ select * from ${tableNameSmart} where content
match_phrase '计算机科学'; """
+ order_qt_sql """ select * from ${tableNameSmart} where content
match_phrase '数据库管理系统'; """
+ order_qt_sql """ select * from ${tableNameSmart} where content
match_phrase '中华人民共和国'; """
// Testing ik max_word mode
println "Testing ik max_word mode:"
- qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '北京'; """
- qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '计算机科学'; """
- qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '数据库管理系统'; """
- qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '中华人民共和国'; """
+ order_qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '北京'; """
+ order_qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '计算机科学'; """
+ order_qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '数据库管理系统'; """
+ order_qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '中华人民共和国'; """
} finally {
sql "DROP TABLE IF EXISTS ${tableNameSmart}"
diff --git a/regression-test/suites/inverted_index_p0/test_stopwords.groovy
b/regression-test/suites/inverted_index_p0/test_stopwords.groovy
index 31e30cf6e48..2ff7dd3f556 100644
--- a/regression-test/suites/inverted_index_p0/test_stopwords.groovy
+++ b/regression-test/suites/inverted_index_p0/test_stopwords.groovy
@@ -44,16 +44,16 @@ suite("test_stopwords", "p0"){
sql "sync"
sql """ set enable_segment_limit_pushdown = true """
- qt_sql """ select * from ${indexTbName} where b match 'a'; """
- qt_sql """ select * from ${indexTbName} where b match 'are'; """
- qt_sql """ select * from ${indexTbName} where b match 'to'; """
+ order_qt_sql """ select * from ${indexTbName} where b match 'a'; """
+ order_qt_sql """ select * from ${indexTbName} where b match 'are'; """
+ order_qt_sql """ select * from ${indexTbName} where b match 'to'; """
- qt_sql """ select * from ${indexTbName} where c match 'a'; """
- qt_sql """ select * from ${indexTbName} where c match 'are'; """
- qt_sql """ select * from ${indexTbName} where c match 'to'; """
+ order_qt_sql """ select * from ${indexTbName} where c match 'a'; """
+ order_qt_sql """ select * from ${indexTbName} where c match 'are'; """
+ order_qt_sql """ select * from ${indexTbName} where c match 'to'; """
- qt_sql """ select * from ${indexTbName} where b match_phrase 'like a
box'; """
- qt_sql """ select * from ${indexTbName} where c match_phrase 'like a
box'; """
+ order_qt_sql """ select * from ${indexTbName} where b match_phrase
'like a box'; """
+ order_qt_sql """ select * from ${indexTbName} where c match_phrase
'like a box'; """
} finally {
//try_sql("DROP TABLE IF EXISTS ${testTable}")
diff --git a/regression-test/suites/search/test_search_dsl_syntax.groovy
b/regression-test/suites/search/test_search_dsl_syntax.groovy
index 34484e80993..2e8b84b3fbb 100644
--- a/regression-test/suites/search/test_search_dsl_syntax.groovy
+++ b/regression-test/suites/search/test_search_dsl_syntax.groovy
@@ -92,16 +92,16 @@ suite("test_search_dsl_syntax", "p0") {
qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:Machine') ORDER BY id"
// DSL Syntax Test 2: Phrase queries with quotes
- qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:\"Machine Learning\"')"
+ qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:\"Machine Learning\"') ORDER BY id"
// DSL Syntax Test 3: Wildcard queries
qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:*Learning') ORDER BY id"
// DSL Syntax Test 4: Prefix queries
- qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:Data*')"
+ qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:Data*') ORDER BY id"
// DSL Syntax Test 5: Regular expression queries
- qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:/.*[Dd]ata.*/')"
+ qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:/.*[Dd]ata.*/') ORDER BY id"
// DSL Syntax Test 6: Boolean AND operator
qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('category:Technology AND tags:AI') ORDER BY id"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]