This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new d0f922d047c branch-4.1: [fix](fe) Merge TopN with child prefix order
keys #64685 (#64989)
d0f922d047c is described below
commit d0f922d047cad5af1a3b239c49438d0df1daf158
Author: morrySnow <[email protected]>
AuthorDate: Wed Jul 1 17:44:55 2026 +0800
branch-4.1: [fix](fe) Merge TopN with child prefix order keys #64685
(#64989)
Backport #64685
---
.../doris/nereids/rules/rewrite/MergeTopNs.java | 18 ++++--
.../nereids/rules/rewrite/MergeTopNsTest.java | 21 +++++++
.../limit_push_down/merge_topn_prefix_key.out | 11 ++++
.../limit_push_down/merge_topn_prefix_key.groovy | 64 ++++++++++++++++++++++
4 files changed, 110 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeTopNs.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeTopNs.java
index 3614434459d..167115b9e6d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeTopNs.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeTopNs.java
@@ -29,7 +29,8 @@ import java.util.List;
* this rule aims to merge consecutive topN
* <p>
* topN - child topN
- * If child topN orderby list is prefix subList of topN =>
+ * If one TopN orderby list is a prefix of the other =>
+ * merged TopN uses the longer orderby list, with
* topN with limit = min(topN.limit, max(childTopN.limit - topN.offset, 0)),
* offset = topN.offset + childTopN.offset
*/
@@ -41,8 +42,17 @@ public class MergeTopNs extends OneRewriteRuleFactory {
LogicalTopN<Plan> childTopN = topN.child();
List<OrderKey> orderKeys = topN.getOrderKeys();
List<OrderKey> childOrderKeys = childTopN.getOrderKeys();
- if (!orderKeys.subList(0,
childOrderKeys.size()).equals(childOrderKeys)) {
- return null;
+ List<OrderKey> mergedOrderKeys = orderKeys;
+ int shortKeyLength = Math.min(orderKeys.size(),
childOrderKeys.size());
+ if (orderKeys.size() < childOrderKeys.size()) {
+ if (!childOrderKeys.subList(0,
shortKeyLength).equals(orderKeys)) {
+ return null;
+ }
+ mergedOrderKeys = childOrderKeys;
+ } else {
+ if (!orderKeys.subList(0,
childOrderKeys.size()).equals(childOrderKeys)) {
+ return null;
+ }
}
long offset = topN.getOffset();
long limit = topN.getLimit();
@@ -55,7 +65,7 @@ public class MergeTopNs extends OneRewriteRuleFactory {
// when the parent has a non-zero offset (DORIS-26301).
Same semantics as
// MergeLimits.mergeLimit for consecutive limits.
long newLimit = Math.min(limit, Math.max(childLimit -
offset, 0));
- return topN.withLimitChild(newLimit, newOffset,
childTopN.child());
+ return topN.withLimitOrderKeyAndChild(newLimit, newOffset,
mergedOrderKeys, childTopN.child());
}).toRule(RuleType.MERGE_TOP_N);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeTopNsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeTopNsTest.java
index 63b2c97ac31..35ced912045 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeTopNsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeTopNsTest.java
@@ -27,6 +27,7 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.nereids.util.PlanConstructor;
import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/**
@@ -61,6 +62,26 @@ class MergeTopNsTest implements MemoPatternMatchSupported {
);
}
+ @Test
+ void testMergeChildMoreOrderKeys() {
+ LogicalPlan plan = new LogicalPlanBuilder(score)
+ .topN(20, 0, ImmutableList.of(0, 1))
+ .topN(10, 0, ImmutableList.of(0))
+ .build();
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new MergeTopNs())
+ .matches(
+ logicalTopN(logicalOlapScan()).when(topN -> {
+ Assertions.assertEquals(10, topN.getLimit());
+ Assertions.assertEquals(0, topN.getOffset());
+ Assertions.assertEquals(2,
topN.getOrderKeys().size());
+ Assertions.assertEquals(score.getOutput().get(0),
topN.getOrderKeys().get(0).getExpr());
+ Assertions.assertEquals(score.getOutput().get(1),
topN.getOrderKeys().get(1).getExpr());
+ return true;
+ })
+ );
+ }
+
@Test
void testOffset() {
LogicalPlan plan = new LogicalPlanBuilder(score)
diff --git
a/regression-test/data/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.out
b/regression-test/data/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.out
new file mode 100644
index 00000000000..7055064bf13
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !merge_child_more_order_keys_shape --
+PhysicalResultSink
+--PhysicalTopN[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalTopN[LOCAL_SORT]
+--------PhysicalOlapScan[merge_topn_prefix_key]
+
+-- !merge_child_more_order_keys --
+12 1 2
+13 1 3
diff --git
a/regression-test/suites/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.groovy
b/regression-test/suites/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.groovy
new file mode 100644
index 00000000000..eae38564dca
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.groovy
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("merge_topn_prefix_key") {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+ sql "SET runtime_filter_mode=OFF"
+ sql "SET topn_lazy_materialization_threshold=-1"
+ sql "SET enable_two_phase_read_opt=false"
+
+ sql "DROP TABLE IF EXISTS merge_topn_prefix_key"
+
+ sql """
+ CREATE TABLE merge_topn_prefix_key (
+ id INT,
+ a INT,
+ b INT
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ )
+ """
+
+ sql """
+ INSERT INTO merge_topn_prefix_key VALUES
+ (11, 1, 1),
+ (12, 1, 2),
+ (13, 1, 3),
+ (21, 2, 1),
+ (22, 2, 2),
+ (31, 3, 1)
+ """
+
+ qt_merge_child_more_order_keys_shape """
+ EXPLAIN SHAPE PLAN
+ SELECT id, a, b FROM (
+ SELECT id, a, b FROM merge_topn_prefix_key ORDER BY a, b LIMIT 4
+ ) t ORDER BY a LIMIT 2 OFFSET 1
+ """
+
+ qt_merge_child_more_order_keys """
+ SELECT * FROM (
+ SELECT id, a, b FROM (
+ SELECT id, a, b FROM merge_topn_prefix_key ORDER BY a, b LIMIT
4
+ ) t ORDER BY a LIMIT 2 OFFSET 1
+ ) r ORDER BY a, b, id
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
