This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 211f6165aba branch-4.1: [fix](score) disable search topn with extra
predicates #65821 (#67327)
211f6165aba is described below
commit 211f6165abaf3cd0bd86417713dd81523f2a8a52
Author: Jack <[email protected]>
AuthorDate: Tue Sep 1 09:25:41 2026 +0800
branch-4.1: [fix](score) disable search topn with extra predicates #65821
(#67327)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #65821 (master), picked from commit
2b6a45e7e2cb12d091ce0253366e50fca4efd5d3
Problem Summary:
Backport of #65821 to branch-4.1. Search score TopN pushdown may return
incorrect results when the search predicate is combined with additional
predicates, because the pushed TopN limit can be applied before the
remaining predicates are evaluated. This change disables the pushed
search TopN limit in those cases while preserving the virtual score
column pushdown, and adds regression coverage (search + equality / range
/ match / score range / multiple search predicates, plus limit+offset
overflow).
**Hunk audit (source diff → this PR):**
| Source hunk | Status |
|---|---|
| `PushDownScoreTopNIntoOlapScan.java` `@@ -194,17 +194,22 @@` (overflow
guard rework + pushedScoreLimit) | **Adapted ×2**: ① branch-4.1 never
had the #64633 overflow-guard block, so the hunk's removed lines have no
counterpart here; ② `Utils.addOverflows` does not exist on 4.1 (#64633
not backported) — inlined the equivalent check `topN.getLimit() >
Long.MAX_VALUE - topN.getOffset()` (identical to the master helper's
implementation). |
| `PushDownScoreTopNIntoOlapScan.java` `@@ -243,6 +248,19 @@`
(`shouldDisableSearchTopN` helper) | Ported |
| `test_search_score_topn_predicates.out` (new) | Ported (verbatim) |
| `test_search_score_topn_predicates.groovy` (new) | **Adapted**:
dropped `set enable_segment_limit_pushdown = true` — the variable comes
from #62222 which is not on 4.1; it defaults to true on master and only
controls a BE-side segment limit optimization, unrelated to this
FE-plan-level fix. |
**Local verification on this branch:** full ASAN BE+FE build green;
`run-regression-test.sh -d inverted_index_p0 -s
test_search_score_topn_predicates` → 1 suite, 0 failed against a local
1FE+1BE cluster built from this PR. No FE UT exists for this rule on 4.1
and the source PR added none (its coverage is the regression suite
above).
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [x] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
Co-authored-by: liangj777
<[email protected]>
---
.../rewrite/PushDownScoreTopNIntoOlapScan.java | 26 ++-
.../test_search_score_topn_predicates.out | 38 ++++
.../test_search_score_topn_predicates.groovy | 203 +++++++++++++++++++++
3 files changed, 266 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java
index d24a6018438..824716f2a45 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java
@@ -188,12 +188,23 @@ public class PushDownScoreTopNIntoOlapScan implements
RewriteRuleFactory {
+ " for push down optimization");
}
+ // When limit + offset overflows the long range, the pushed scan limit
would wrap to a
+ // negative value. Fail with the same error as ordinary TopN instead
of leaving score()
+ // unmaterialized and reporting an unrelated score() usage error.
+ if (topN.getLimit() > Long.MAX_VALUE - topN.getOffset()) {
+ throw new AnalysisException("limit + offset overflows the long
range");
+ }
+
+ long scoreLimit = topN.getLimit() + topN.getOffset();
+ long pushedScoreLimit = shouldDisableSearchTopN(filter.getConjuncts(),
extractedScorePredicate)
+ ? 0L : scoreLimit;
+
// All conditions met, perform the push down.
// This is the core action: push score() as a virtual column and also
push the
// topN info.
Plan newScan =
scan.appendVirtualColumnsAndTopN(ImmutableList.of(scoreAlias),
ImmutableList.of(), Optional.empty(),
- topN.getOrderKeys(), Optional.of(topN.getLimit() +
topN.getOffset()),
+ topN.getOrderKeys(), Optional.of(pushedScoreLimit),
scoreRangeInfo);
// Rebuild the plan tree above the new scan.
@@ -232,6 +243,19 @@ public class PushDownScoreTopNIntoOlapScan implements
RewriteRuleFactory {
return topN.withChildren(newProject);
}
+ private boolean shouldDisableSearchTopN(Set<Expression> conjuncts,
Expression extractedScorePredicate) {
+ List<Expression> nonScoreConjuncts = conjuncts.stream()
+ .filter(conjunct -> extractedScorePredicate == null ||
!conjunct.equals(extractedScorePredicate))
+ .collect(ImmutableList.toImmutableList());
+
+ boolean hasSearchPredicate = nonScoreConjuncts.stream()
+ .anyMatch(conjunct -> !conjunct.collect(e -> e instanceof
SearchExpression).isEmpty());
+ if (!hasSearchPredicate) {
+ return false;
+ }
+ return nonScoreConjuncts.size() > 1 || !(nonScoreConjuncts.get(0)
instanceof SearchExpression);
+ }
+
/**
* Extract score range info from a single score predicate.
* Only supports min_score semantics (similar to Elasticsearch):
diff --git
a/regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out
b/regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out
new file mode 100644
index 00000000000..645855dd631
--- /dev/null
+++
b/regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out
@@ -0,0 +1,38 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !single_search --
+1
+
+-- !search_with_equal_predicate --
+3
+
+-- !search_with_plain_equal_predicate --
+3
+
+-- !search_with_equal_limit_two --
+3
+4
+
+-- !search_with_equal_offset --
+4
+
+-- !search_with_match_predicate --
+3
+
+-- !search_with_range_predicate --
+3
+
+-- !multiple_search_predicates --
+4
+
+-- !search_with_score_range_only --
+1
+3
+
+-- !search_with_score_range_and_other_predicate --
+3
+
+-- !nested_search_with_other_predicate --
+7
+
+-- !not_search_with_other_search --
+6
diff --git
a/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy
b/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy
new file mode 100644
index 00000000000..1357a9b6c3a
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy
@@ -0,0 +1,203 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_search_score_topn_predicates", "p0") {
+ sql "DROP TABLE IF EXISTS test_search_score_topn_predicates"
+
+ sql """
+ CREATE TABLE test_search_score_topn_predicates (
+ id INT,
+ status VARCHAR(20),
+ plain_status VARCHAR(20),
+ category VARCHAR(20),
+ title TEXT,
+ body TEXT,
+ INDEX idx_status (status) USING INVERTED,
+ INDEX idx_category (category) USING INVERTED,
+ INDEX idx_title (title) USING INVERTED PROPERTIES("parser" =
"english", "support_phrase" = "true"),
+ INDEX idx_body (body) USING INVERTED PROPERTIES("parser" =
"english", "support_phrase" = "true")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "disable_auto_compaction" = "true"
+ )
+ """
+
+ sql """
+ INSERT INTO test_search_score_topn_predicates VALUES
+ (1, 'drop', 'drop', 'plain', 'apple apple apple apple apple apple
apple apple apple apple apple apple', 'alpha'),
+ (2, 'keep', 'keep', 'plain', 'apple', 'alpha'),
+ (3, 'keep', 'keep', 'plain', 'apple apple apple apple apple',
'alpha'),
+ (4, 'keep', 'keep', 'plain', 'apple apple apple', 'beta beta
beta'),
+ (5, 'drop', 'drop', 'plain', 'banana', 'beta beta beta beta beta
beta beta beta beta beta beta beta'),
+ (6, 'keep', 'keep', 'plain', 'pear', 'beta beta beta beta beta'),
+ (7, 'keep', 'keep', 'special', 'cherry cherry cherry cherry',
'gamma'),
+ (8, 'drop', 'drop', 'special', 'cherry cherry cherry cherry cherry
cherry cherry cherry cherry cherry', 'gamma')
+ """
+
+ sql "sync"
+ sql "set enable_nereids_planner = true"
+ sql "set enable_fallback_to_original_planner = false"
+ sql "set enable_inverted_index_query_cache = false"
+
+ qt_single_search """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple')
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_equal_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_plain_equal_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND plain_status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_equal_limit_two """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND status = 'keep'
+ ORDER BY s DESC
+ LIMIT 2
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_equal_offset """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1 OFFSET 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_match_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND status MATCH 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_range_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND id > 1
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_multiple_search_predicates """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND search('body:beta')
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_score_range_only """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND score() > 0
+ ORDER BY s DESC
+ LIMIT 2
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_score_range_and_other_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND score() > 0 AND status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_nested_search_with_other_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE (search('title:cherry') OR category MATCH 'special') AND
status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_not_search_with_other_search """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE NOT search('title:apple') AND search('body:beta') AND status
= 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ // limit + offset overflows the long range. score() must report the
standard TopN
+ // overflow error instead of skipping score pushdown and reporting a
score() usage error.
+ test {
+ sql """
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple')
+ ORDER BY s DESC
+ LIMIT 9223372036854775807 OFFSET 9223372036854775807
+ """
+ exception "limit + offset overflows the long range"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
