This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 960967b6eab [fix](rbo) Preserve column mappings when merging nested
set operations (#69131)
960967b6eab is described below
commit 960967b6eabc4065b2f1c12bdb145555e8f65e6d
Author: feiniaofeiafei <[email protected]>
AuthorDate: Thu Oct 22 10:22:00 2026 +0800
[fix](rbo) Preserve column mappings when merging nested set operations
(#69131)
### What problem does this PR solve?
Merging nested set operations can lose the parent’s input column mapping
and return values from the wrong columns. For example, a query grouping
by country can return campaign names after pruning an unused date
column.
Each set operation’s regularChildrenOutputs[i] specifies which child
slots supply its output columns, in output order. This mapping can
differ from the child’s own output order.
In the following plan, the parent reads the nested UNION as [country#63,
campaign#62], while the nested UNION’s own output is [campaign#62,
country#63].
Before MERGE_SET_OPERATION:
```sql
Union
outputs: [country#70, tag#71]
child mappings:
[country#46, tag#49]
[country#63, campaign#62]
├─ Project [country#46, campaign#45 AS tag#49]
│ └─ Scan
└─ Union
outputs: [campaign#62, country#63]
child mappings:
[campaign#51, country#52]
[campaign#57, country#58]
├─ Project [campaign#51, country#52]
│ └─ Scan
└─ Project [campaign#57, country#58]
└─ Scan
```
The original rewrite directly copied the nested UNION’s child mappings
into the parent, discarding the parent’s required column permutation.
After the original rewrite — incorrect:
```sql
Union
outputs: [country#70, tag#71]
child mappings:
[country#46, tag#49]
[campaign#51, country#52] ← country#70 receives campaign#51
[campaign#57, country#58] ← country#70 receives campaign#57
├─ Project [country#46, campaign#45 AS tag#49]
│ └─ Scan
├─ Project [campaign#51, country#52]
│ └─ Scan
└─ Project [campaign#57, country#58]
└─ Scan
```
The fix composes the parent’s input mapping with the nested operation’s
output ordinals before flattening its children.
After the fixed rewrite — correct:
```sql
Union
outputs: [country#70, tag#71]
child mappings:
[country#46, tag#49]
[country#52, campaign#51]
[country#58, campaign#57]
├─ Project [country#46, campaign#45 AS tag#49]
│ └─ Scan
├─ Project [campaign#51, country#52]
│ └─ Scan
└─ Project [campaign#57, country#58]
└─ Scan
The child Project nodes keep their original output order. The corrected
mappings ensure that country#70 receives country values from every branch.
```
The same mapping composition is applied to nested INTERSECT and
left-deep EXCEPT operations. UNION constant rows are reordered
consistently, and the parent EXCEPT’s remaining inputs retain their
original mappings.
---
.../nereids/rules/rewrite/MergeSetOperations.java | 41 +++++++++-
.../rules/rewrite/MergeSetOperationsExcept.java | 5 +-
.../rewrite/SetOperationOutputMappingTest.java | 53 +++++++++++++
.../merge_union_output_mapping.out | 12 +++
.../merge_union_output_mapping.groovy | 88 ++++++++++++++++++++++
5 files changed, 195 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperations.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperations.java
index 57fedeb350e..53817b658e4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperations.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperations.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -27,9 +28,12 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Merge nodes of the same type and same qualifier.
@@ -83,8 +87,13 @@ public class MergeSetOperations implements
RewriteRuleFactory {
if (canMerge(parentUnion, child) && child.getOutput().size() ==
parentUnion.getOutput().size()) {
LogicalUnion childUnion = (LogicalUnion) child;
newChildren.addAll(childUnion.children());
-
newChildrenOutputs.addAll(childUnion.getRegularChildrenOutputs());
- constantList.addAll(childUnion.getConstantExprsList());
+ List<Integer> outputOrder = getChildOutputOrder(parentUnion,
i, childUnion);
+ for (List<SlotReference> outputs :
childUnion.getRegularChildrenOutputs()) {
+ newChildrenOutputs.add(reorderOutputs(outputs,
outputOrder));
+ }
+ for (List<NamedExpression> constants :
childUnion.getConstantExprsList()) {
+ constantList.add(reorderOutputs(constants, outputOrder));
+ }
needMerge = true;
} else {
newChildren.add(child);
@@ -108,7 +117,10 @@ public class MergeSetOperations implements
RewriteRuleFactory {
if (canMerge(parentSetIntersect, child)) {
LogicalIntersect childIntersect = (LogicalIntersect) child;
newChildren.addAll(childIntersect.children());
-
newChildrenOutputs.addAll(childIntersect.getRegularChildrenOutputs());
+ List<Integer> outputOrder =
getChildOutputOrder(parentSetIntersect, i, childIntersect);
+ for (List<SlotReference> outputs :
childIntersect.getRegularChildrenOutputs()) {
+ newChildrenOutputs.add(reorderOutputs(outputs,
outputOrder));
+ }
} else {
newChildren.add(child);
newChildrenOutputs.add(parentSetIntersect.getRegularChildOutput(i));
@@ -118,6 +130,29 @@ public class MergeSetOperations implements
RewriteRuleFactory {
newChildren.build(), newChildrenOutputs.build());
}
+ static List<Integer> getChildOutputOrder(
+ LogicalSetOperation parent, int childIndex, LogicalSetOperation
child) {
+ Map<ExprId, Integer> outputIndexes = new HashMap<>();
+ for (int i = 0; i < child.getOutput().size(); i++) {
+ outputIndexes.put(child.getOutput().get(i).getExprId(), i);
+ }
+ ImmutableList.Builder<Integer> order = ImmutableList.builder();
+ for (SlotReference slot : parent.getRegularChildOutput(childIndex)) {
+ Integer index = outputIndexes.get(slot.getExprId());
+ Preconditions.checkState(index != null, "Set operation input must
belong to child output");
+ order.add(index);
+ }
+ return order.build();
+ }
+
+ static <T> List<T> reorderOutputs(List<T> outputs, List<Integer> order) {
+ ImmutableList.Builder<T> reordered =
ImmutableList.builderWithExpectedSize(order.size());
+ for (int index : order) {
+ reordered.add(outputs.get(index));
+ }
+ return reordered.build();
+ }
+
/** canMerge */
public static boolean canMerge(LogicalSetOperation parent) {
Plan left = parent.child(0);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperationsExcept.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperationsExcept.java
index 2eb284e69c5..79d62285bca 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperationsExcept.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperationsExcept.java
@@ -54,7 +54,10 @@ public class MergeSetOperationsExcept extends
OneRewriteRuleFactory {
newChildren.addAll(childExcept.children());
newChildren.addAll(parentExcept.children().subList(1,
parentExcept.children().size()));
-
newChildrenOutputs.addAll(childExcept.getRegularChildrenOutputs());
+ List<Integer> outputOrder =
MergeSetOperations.getChildOutputOrder(parentExcept, 0, childExcept);
+ for (List<SlotReference> outputs :
childExcept.getRegularChildrenOutputs()) {
+
newChildrenOutputs.add(MergeSetOperations.reorderOutputs(outputs, outputOrder));
+ }
newChildrenOutputs.addAll(
parentExcept.getRegularChildrenOutputs().subList(1,
parentExcept.children().size()));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SetOperationOutputMappingTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SetOperationOutputMappingTest.java
index a42c974dabf..e1d4489e18c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SetOperationOutputMappingTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SetOperationOutputMappingTest.java
@@ -26,6 +26,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
+import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.types.IntegerType;
@@ -63,6 +64,58 @@ class SetOperationOutputMappingTest extends
TestWithFeService implements MemoPat
StatementScopeIdGenerator.clear();
}
+ @Test
+ void testMergeSetOperationsComposesChildOutputMapping() {
+ Alias first = new Alias(new ExprId(101), new IntegerLiteral(10),
"campaign");
+ Alias second = new Alias(new ExprId(102), new IntegerLiteral(20),
"country");
+ LogicalOneRowRelation row = new LogicalOneRowRelation(
+ new RelationId(101), ImmutableList.of(first, second));
+ SlotReference campaign = new SlotReference(new ExprId(103), "campaign",
+ IntegerType.INSTANCE, false, ImmutableList.of());
+ SlotReference country = new SlotReference(new ExprId(104), "country",
+ IntegerType.INSTANCE, false, ImmutableList.of());
+ LogicalUnion child = new LogicalUnion(Qualifier.ALL,
ImmutableList.of(campaign, country),
+ ImmutableList.of(ImmutableList.of((SlotReference)
first.toSlot(), (SlotReference) second.toSlot())),
+ ImmutableList.of(ImmutableList.of(first, second)), false,
ImmutableList.of(row));
+ LogicalUnion parent = new LogicalUnion(Qualifier.ALL,
ImmutableList.of(country, campaign),
+ ImmutableList.of(ImmutableList.of(country, campaign)),
ImmutableList.of(),
+ false, ImmutableList.of(child));
+
+ LogicalUnion rewritten = (LogicalUnion)
PlanChecker.from(MemoTestUtils.createConnectContext(), parent)
+ .applyTopDown(new MergeSetOperations()).getPlan();
+ Assertions.assertEquals(ImmutableList.of(row), rewritten.children());
+ Assertions.assertEquals(ImmutableList.of(second.toSlot(),
first.toSlot()),
+ rewritten.getRegularChildOutput(0));
+ Assertions.assertEquals(ImmutableList.of(second, first),
rewritten.getConstantExprsList().get(0));
+
+ Alias rightFirst = new Alias(new ExprId(105), new IntegerLiteral(30),
"right_first");
+ Alias rightSecond = new Alias(new ExprId(106), new IntegerLiteral(40),
"right_second");
+ LogicalOneRowRelation right = new LogicalOneRowRelation(
+ new RelationId(102), ImmutableList.of(rightFirst,
rightSecond));
+ List<SlotReference> rightOutputs = ImmutableList.of(
+ (SlotReference) rightFirst.toSlot(), (SlotReference)
rightSecond.toSlot());
+ LogicalExcept childExcept = new LogicalExcept(Qualifier.DISTINCT,
ImmutableList.of(campaign, country),
+ ImmutableList.of(child.getRegularChildOutput(0),
rightOutputs), ImmutableList.of(row, right));
+ Alias outerFirst = new Alias(new ExprId(107), new IntegerLiteral(50),
"outer_first");
+ Alias outerSecond = new Alias(new ExprId(108), new IntegerLiteral(60),
"outer_second");
+ LogicalOneRowRelation outerRight = new LogicalOneRowRelation(
+ new RelationId(103), ImmutableList.of(outerFirst,
outerSecond));
+ List<SlotReference> outerRightOutputs = ImmutableList.of(
+ (SlotReference) outerFirst.toSlot(), (SlotReference)
outerSecond.toSlot());
+ LogicalExcept parentExcept = new LogicalExcept(Qualifier.DISTINCT,
ImmutableList.of(country, campaign),
+ ImmutableList.of(ImmutableList.of(country, campaign),
outerRightOutputs),
+ ImmutableList.of(childExcept, outerRight));
+ LogicalExcept rewrittenExcept = (LogicalExcept) PlanChecker.from(
+ MemoTestUtils.createConnectContext(), parentExcept)
+ .applyTopDown(new MergeSetOperationsExcept()).getPlan();
+ Assertions.assertEquals(ImmutableList.of(row, right, outerRight),
rewrittenExcept.children());
+ Assertions.assertEquals(ImmutableList.of(second.toSlot(),
first.toSlot()),
+ rewrittenExcept.getRegularChildOutput(0));
+ Assertions.assertEquals(ImmutableList.of(rightSecond.toSlot(),
rightFirst.toSlot()),
+ rewrittenExcept.getRegularChildOutput(1));
+ Assertions.assertEquals(outerRightOutputs,
rewrittenExcept.getRegularChildOutput(2));
+ }
+
@Test
void testMergeOneRowRelationUsesRegularChildOutput() {
Alias firstProject = new Alias(new ExprId(1), new IntegerLiteral(10),
"first_col");
diff --git
a/regression-test/data/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.out
b/regression-test/data/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.out
new file mode 100644
index 00000000000..41dab4ef646
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !country --
+FR
+
+-- !country_date --
+FR 2026-01-01
+
+-- !except_reordered --
+FR AD_A
+
+-- !except_removed --
+
diff --git
a/regression-test/suites/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.groovy
b/regression-test/suites/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.groovy
new file mode 100644
index 00000000000..6a95bceba9a
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.groovy
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("merge_union_output_mapping", "p0") {
+ sql "DROP TABLE IF EXISTS merge_union_output_mapping"
+ sql """CREATE TABLE merge_union_output_mapping (
+ id INT, campaign VARCHAR(40), country VARCHAR(40), dt VARCHAR(40)
+ ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ('replication_num' = '1')"""
+ sql "INSERT INTO merge_union_output_mapping VALUES (1, 'AD_A', 'FR',
'2026-01-01')"
+ sql "SET enable_sql_cache=false"
+ order_qt_country """WITH sa AS (
+ SELECT CAST(dt AS DATE) AS dt, campaign, country, campaign AS tag FROM
merge_union_output_mapping
+ UNION ALL
+ SELECT CAST(dt AS DATE), campaign, country, campaign
+ FROM (
+ SELECT campaign, country, dt, id AS x0, id AS x1 FROM
merge_union_output_mapping
+ UNION ALL
+ SELECT campaign, country, dt, id AS x0, id AS x1 FROM
merge_union_output_mapping
+ ) u
+), c AS (
+ SELECT dt, country, tag, 'SA' AS ad_type FROM sa
+), e AS (
+ SELECT 'FR' AS country
+)
+SELECT a.country
+FROM c a
+LEFT JOIN e ON CASE
+ WHEN a.country = 'PL' AND a.ad_type = 'DSP' THEN 'PL_DSP'
+ ELSE a.country
+END = e.country
+LEFT JOIN c b ON a.tag = b.tag
+GROUP BY a.country"""
+ order_qt_country_date """WITH sa AS (
+ SELECT CAST(dt AS DATE) AS dt, campaign, country, campaign AS tag FROM
merge_union_output_mapping
+ UNION ALL
+ SELECT CAST(dt AS DATE), campaign, country, campaign
+ FROM (
+ SELECT campaign, country, dt, id AS x0, id AS x1 FROM
merge_union_output_mapping
+ UNION ALL
+ SELECT campaign, country, dt, id AS x0, id AS x1 FROM
merge_union_output_mapping
+ ) u
+), c AS (
+ SELECT dt, country, tag, 'SA' AS ad_type FROM sa
+), e AS (
+ SELECT 'FR' AS country
+)
+SELECT a.country, a.dt
+FROM c a
+LEFT JOIN e ON CASE
+ WHEN a.country = 'PL' AND a.ad_type = 'DSP' THEN 'PL_DSP'
+ ELSE a.country
+END = e.country
+LEFT JOIN c b ON a.tag = b.tag
+GROUP BY a.country, a.dt"""
+ order_qt_except_reordered """
+ SELECT country, campaign FROM (
+ SELECT campaign, country FROM merge_union_output_mapping
+ EXCEPT
+ SELECT campaign, country FROM merge_union_output_mapping WHERE id
< 0
+ ) u
+ EXCEPT
+ SELECT country, campaign FROM merge_union_output_mapping WHERE id < 0
+ """
+ order_qt_except_removed """
+ SELECT country, campaign FROM (
+ SELECT campaign, country FROM merge_union_output_mapping
+ EXCEPT
+ SELECT campaign, country FROM merge_union_output_mapping WHERE id
< 0
+ ) u
+ EXCEPT
+ SELECT country, campaign FROM merge_union_output_mapping
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
