This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 53104945146 [fix](local shuffle) fix bucket shuffle + set operation +
local shuffle compute wrong result (#60823)
53104945146 is described below
commit 53104945146522c15936ef9556253896c7bba3f9
Author: 924060929 <[email protected]>
AuthorDate: Thu Feb 26 18:19:03 2026 +0800
[fix](local shuffle) fix bucket shuffle + set operation + local shuffle
compute wrong result (#60823)
fix bucket shuffle + set operation + local shuffle compute wrong result,
because backend can not plan the correct local shuffle type, introduced
by #59006.
so we should disable bucket shuffle for set operation now, after support
plan local shuffle in frontend, we can support this feature.
---
.../glue/translator/PhysicalPlanTranslator.java | 15 +-
.../properties/ChildOutputPropertyDeriver.java | 96 +++++++------
.../properties/ChildrenPropertiesRegulator.java | 136 +++++++++---------
.../nereids/properties/RequestPropertyDeriver.java | 3 +-
.../infer_set_operator_distinct.out | 154 +++++++++++----------
.../runtime_filter/test_pushdown_setop.out | 32 ++---
.../bucket_shuffle_set_operation.groovy | 3 +
7 files changed, 228 insertions(+), 211 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 941b2708f0a..892441752c5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -2418,13 +2418,14 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
setOperationNode.setColocate(true);
}
- for (Plan child : setOperation.children()) {
- PhysicalPlan childPhysicalPlan = (PhysicalPlan) child;
- if
(JoinUtils.isStorageBucketed(childPhysicalPlan.getPhysicalProperties())) {
-
setOperationNode.setDistributionMode(DistributionMode.BUCKET_SHUFFLE);
- break;
- }
- }
+ // TODO: open comment when support `enable_local_shuffle_planner`
+ // for (Plan child : setOperation.children()) {
+ // PhysicalPlan childPhysicalPlan = (PhysicalPlan) child;
+ // if
(JoinUtils.isStorageBucketed(childPhysicalPlan.getPhysicalProperties())) {
+ //
setOperationNode.setDistributionMode(DistributionMode.BUCKET_SHUFFLE);
+ // break;
+ // }
+ // }
return setOperationFragment;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index 43ada3b40ec..8ed28a9e9b1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -30,7 +30,6 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import
org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.algebra.Union;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor;
@@ -74,11 +73,9 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -459,52 +456,53 @@ public class ChildOutputPropertyDeriver extends
PlanVisitor<PhysicalProperties,
return PhysicalProperties.GATHER;
}
- int distributeToChildIndex
- =
setOperation.<Integer>getMutableState(PhysicalSetOperation.DISTRIBUTE_TO_CHILD_INDEX).orElse(-1);
- if (distributeToChildIndex >= 0
- && childrenDistribution.get(distributeToChildIndex) instanceof
DistributionSpecHash) {
- DistributionSpecHash childDistribution
- = (DistributionSpecHash)
childrenDistribution.get(distributeToChildIndex);
- List<SlotReference> childToIndex =
setOperation.getRegularChildrenOutputs().get(distributeToChildIndex);
- Map<ExprId, Integer> idToOutputIndex = new LinkedHashMap<>();
- for (int j = 0; j < childToIndex.size(); j++) {
- idToOutputIndex.put(childToIndex.get(j).getExprId(), j);
- }
-
- List<ExprId> orderedShuffledColumns =
childDistribution.getOrderedShuffledColumns();
- List<ExprId> setOperationDistributeColumnIds = new ArrayList<>();
- for (ExprId tableDistributeColumnId : orderedShuffledColumns) {
- Integer index = idToOutputIndex.get(tableDistributeColumnId);
- if (index == null) {
- break;
- }
-
setOperationDistributeColumnIds.add(setOperation.getOutput().get(index).getExprId());
- }
- // check whether the set operation output all distribution columns
of the child
- if (setOperationDistributeColumnIds.size() ==
orderedShuffledColumns.size()) {
- boolean isUnion = setOperation instanceof Union;
- boolean shuffleToRight = distributeToChildIndex > 0;
- if (!isUnion && shuffleToRight) {
- return new PhysicalProperties(
- new DistributionSpecHash(
- setOperationDistributeColumnIds,
- ShuffleType.EXECUTION_BUCKETED
- )
- );
- } else {
- // keep the distribution as the child
- return new PhysicalProperties(
- new DistributionSpecHash(
- setOperationDistributeColumnIds,
- childDistribution.getShuffleType(),
- childDistribution.getTableId(),
- childDistribution.getSelectedIndexId(),
- childDistribution.getPartitionIds()
- )
- );
- }
- }
- }
+ // TODO: open comment when support `enable_local_shuffle_planner`
+ // int distributeToChildIndex
+ // =
setOperation.<Integer>getMutableState(PhysicalSetOperation.DISTRIBUTE_TO_CHILD_INDEX).orElse(-1);
+ // if (distributeToChildIndex >= 0
+ // && childrenDistribution.get(distributeToChildIndex)
instanceof DistributionSpecHash) {
+ // DistributionSpecHash childDistribution
+ // = (DistributionSpecHash)
childrenDistribution.get(distributeToChildIndex);
+ // List<SlotReference> childToIndex =
setOperation.getRegularChildrenOutputs().get(distributeToChildIndex);
+ // Map<ExprId, Integer> idToOutputIndex = new LinkedHashMap<>();
+ // for (int j = 0; j < childToIndex.size(); j++) {
+ // idToOutputIndex.put(childToIndex.get(j).getExprId(), j);
+ // }
+ //
+ // List<ExprId> orderedShuffledColumns =
childDistribution.getOrderedShuffledColumns();
+ // List<ExprId> setOperationDistributeColumnIds = new
ArrayList<>();
+ // for (ExprId tableDistributeColumnId : orderedShuffledColumns) {
+ // Integer index =
idToOutputIndex.get(tableDistributeColumnId);
+ // if (index == null) {
+ // break;
+ // }
+ //
setOperationDistributeColumnIds.add(setOperation.getOutput().get(index).getExprId());
+ // }
+ // // check whether the set operation output all distribution
columns of the child
+ // if (setOperationDistributeColumnIds.size() ==
orderedShuffledColumns.size()) {
+ // boolean isUnion = setOperation instanceof Union;
+ // boolean shuffleToRight = distributeToChildIndex > 0;
+ // if (!isUnion && shuffleToRight) {
+ // return new PhysicalProperties(
+ // new DistributionSpecHash(
+ // setOperationDistributeColumnIds,
+ // ShuffleType.EXECUTION_BUCKETED
+ // )
+ // );
+ // } else {
+ // // keep the distribution as the child
+ // return new PhysicalProperties(
+ // new DistributionSpecHash(
+ // setOperationDistributeColumnIds,
+ // childDistribution.getShuffleType(),
+ // childDistribution.getTableId(),
+ // childDistribution.getSelectedIndexId(),
+ // childDistribution.getPartitionIds()
+ // )
+ // );
+ // }
+ // }
+ // }
for (int i = 0; i < childrenDistribution.size(); i++) {
DistributionSpec childDistribution = childrenDistribution.get(i);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
index 8db30640797..aa30308e38e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
@@ -55,7 +55,6 @@ import org.apache.doris.statistics.Statistics;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -626,80 +625,83 @@ public class ChildrenPropertiesRegulator extends
PlanVisitor<List<List<PhysicalP
} else if (requiredDistributionSpec instanceof DistributionSpecHash) {
// TODO: should use the most common hash spec as basic
DistributionSpecHash basic = (DistributionSpecHash)
requiredDistributionSpec;
- int bucketShuffleBasicIndex = -1;
- double basicRowCount = -1;
+ // TODO: open comment when support `enable_local_shuffle_planner`
+ // int bucketShuffleBasicIndex = -1;
+ // double basicRowCount = -1;
// find the bucket shuffle basic index
- try {
- ImmutableSet<ShuffleType> supportBucketShuffleTypes =
ImmutableSet.of(
- ShuffleType.NATURAL,
- ShuffleType.STORAGE_BUCKETED
- );
- for (int i = 0; i < originChildrenProperties.size(); i++) {
- PhysicalProperties originChildrenProperty =
originChildrenProperties.get(i);
- DistributionSpec childDistribution =
originChildrenProperty.getDistributionSpec();
- if (childDistribution instanceof DistributionSpecHash
- && supportBucketShuffleTypes.contains(
- ((DistributionSpecHash)
childDistribution).getShuffleType())
- &&
!(isBucketShuffleDownGrade(setOperation.child(i)))) {
- Statistics stats = setOperation.child(i).getStats();
- double rowCount = stats.getRowCount();
- if (rowCount > basicRowCount) {
- basicRowCount = rowCount;
- bucketShuffleBasicIndex = i;
- }
- }
- }
- } catch (Throwable t) {
- // catch stats exception
- LOG.warn("Can not find the most (bucket num, rowCount): " + t,
t);
- bucketShuffleBasicIndex = -1;
- }
+ // try {
+ // ImmutableSet<ShuffleType> supportBucketShuffleTypes =
ImmutableSet.of(
+ // ShuffleType.NATURAL,
+ // ShuffleType.STORAGE_BUCKETED
+ // );
+ // for (int i = 0; i < originChildrenProperties.size(); i++) {
+ // PhysicalProperties originChildrenProperty =
originChildrenProperties.get(i);
+ // DistributionSpec childDistribution =
originChildrenProperty.getDistributionSpec();
+ // if (childDistribution instanceof DistributionSpecHash
+ // && supportBucketShuffleTypes.contains(
+ // ((DistributionSpecHash)
childDistribution).getShuffleType())
+ // &&
!(isBucketShuffleDownGrade(setOperation.child(i)))) {
+ // Statistics stats = setOperation.child(i).getStats();
+ // double rowCount = stats.getRowCount();
+ // if (rowCount > basicRowCount) {
+ // basicRowCount = rowCount;
+ // bucketShuffleBasicIndex = i;
+ // }
+ // }
+ // }
+ // } catch (Throwable t) {
+ // // catch stats exception
+ // LOG.warn("Can not find the most (bucket num, rowCount): " +
t, t);
+ // bucketShuffleBasicIndex = -1;
+ // }
// use bucket shuffle
- if (bucketShuffleBasicIndex >= 0) {
- DistributionSpecHash notShuffleSideRequire
- = (DistributionSpecHash)
requiredProperties.get(bucketShuffleBasicIndex).getDistributionSpec();
-
- DistributionSpecHash notNeedShuffleOutput
- = (DistributionSpecHash)
originChildrenProperties.get(bucketShuffleBasicIndex)
- .getDistributionSpec();
-
- for (int i = 0; i < originChildrenProperties.size(); i++) {
- DistributionSpecHash current
- = (DistributionSpecHash)
originChildrenProperties.get(i).getDistributionSpec();
- if (i == bucketShuffleBasicIndex) {
- continue;
- }
-
- DistributionSpecHash currentRequire
- = (DistributionSpecHash)
requiredProperties.get(i).getDistributionSpec();
-
- PhysicalProperties target = calAnotherSideRequired(
- ShuffleType.STORAGE_BUCKETED,
- notNeedShuffleOutput, current,
- notShuffleSideRequire,
- currentRequire);
- updateChildEnforceAndCost(i, target);
- }
-
setOperation.setMutableState(PhysicalSetOperation.DISTRIBUTE_TO_CHILD_INDEX,
bucketShuffleBasicIndex);
+ // if (bucketShuffleBasicIndex >= 0) {
+ // DistributionSpecHash notShuffleSideRequire
+ // = (DistributionSpecHash)
requiredProperties.get(bucketShuffleBasicIndex)
+ // .getDistributionSpec();
+ //
+ // DistributionSpecHash notNeedShuffleOutput
+ // = (DistributionSpecHash)
originChildrenProperties.get(bucketShuffleBasicIndex)
+ // .getDistributionSpec();
+ //
+ // for (int i = 0; i < originChildrenProperties.size(); i++) {
+ // DistributionSpecHash current
+ // = (DistributionSpecHash)
originChildrenProperties.get(i).getDistributionSpec();
+ // if (i == bucketShuffleBasicIndex) {
+ // continue;
+ // }
+ //
+ // DistributionSpecHash currentRequire
+ // = (DistributionSpecHash)
requiredProperties.get(i).getDistributionSpec();
+ //
+ // PhysicalProperties target = calAnotherSideRequired(
+ // ShuffleType.STORAGE_BUCKETED,
+ // notNeedShuffleOutput, current,
+ // notShuffleSideRequire,
+ // currentRequire);
+ // updateChildEnforceAndCost(i, target);
+ // }
+ // setOperation.setMutableState(
+ // PhysicalSetOperation.DISTRIBUTE_TO_CHILD_INDEX,
bucketShuffleBasicIndex);
// use partitioned shuffle
- } else {
- for (int i = 0; i < originChildrenProperties.size(); i++) {
- DistributionSpecHash current
- = (DistributionSpecHash)
originChildrenProperties.get(i).getDistributionSpec();
- if (current.getShuffleType() !=
ShuffleType.EXECUTION_BUCKETED
- || !bothSideShuffleKeysAreSameOrder(basic, current,
+ // } else {
+ for (int i = 0; i < originChildrenProperties.size(); i++) {
+ DistributionSpecHash current
+ = (DistributionSpecHash)
originChildrenProperties.get(i).getDistributionSpec();
+ if (current.getShuffleType() != ShuffleType.EXECUTION_BUCKETED
+ || !bothSideShuffleKeysAreSameOrder(basic, current,
+ (DistributionSpecHash)
requiredProperties.get(0).getDistributionSpec(),
+ (DistributionSpecHash)
requiredProperties.get(i).getDistributionSpec())) {
+ PhysicalProperties target = calAnotherSideRequired(
+ ShuffleType.EXECUTION_BUCKETED, basic, current,
(DistributionSpecHash)
requiredProperties.get(0).getDistributionSpec(),
- (DistributionSpecHash)
requiredProperties.get(i).getDistributionSpec())) {
- PhysicalProperties target = calAnotherSideRequired(
- ShuffleType.EXECUTION_BUCKETED, basic, current,
- (DistributionSpecHash)
requiredProperties.get(0).getDistributionSpec(),
- (DistributionSpecHash)
requiredProperties.get(i).getDistributionSpec());
- updateChildEnforceAndCost(i, target);
- }
+ (DistributionSpecHash)
requiredProperties.get(i).getDistributionSpec());
+ updateChildEnforceAndCost(i, target);
}
}
+ // }
}
return ImmutableList.of(originChildrenProperties);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
index d8cd91ac1e7..d79d4590fa5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
@@ -307,13 +307,14 @@ public class RequestPropertyDeriver extends
PlanVisitor<Void, PlanContext> {
// shuffle all column
// TODO: for wide table, may be we should add a upper limit of
shuffle columns
+ // TODO: open comment when support `enable_local_shuffle_planner`
and change to REQUIRE
// intersect/except always need hash distribution, we use REQUIRE
to auto select
// bucket shuffle or execution shuffle
addRequestPropertyToChildren(setOperation.getRegularChildrenOutputs().stream()
.map(childOutputs -> childOutputs.stream()
.map(SlotReference::getExprId)
.collect(ImmutableList.toImmutableList()))
- .map(l -> PhysicalProperties.createHash(l,
ShuffleType.REQUIRE))
+ .map(l -> PhysicalProperties.createHash(l,
ShuffleType.EXECUTION_BUCKETED))
.collect(Collectors.toList()));
}
return null;
diff --git
a/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out
b/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out
index e5022abafe2..73e7df6bfa6 100644
---
a/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out
+++
b/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out
@@ -42,8 +42,9 @@ PhysicalResultSink
-- !except_distinct --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalExcept[bucketShuffle]
-------PhysicalOlapScan[t1]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------PhysicalOlapScan[t2]
@@ -58,8 +59,9 @@ PhysicalResultSink
-- !intersect_distinct --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalIntersect[bucketShuffle]
-------PhysicalOlapScan[t1]
+----PhysicalIntersect
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------PhysicalOlapScan[t2]
@@ -84,20 +86,20 @@ PhysicalResultSink
-- !mixed_set_operators --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalIntersect[bucketShuffle]
-------PhysicalDistribute[DistributionSpecHash]
---------PhysicalExcept[bucketShuffle]
+----PhysicalIntersect
+------PhysicalExcept
+--------hashAgg[GLOBAL]
----------PhysicalDistribute[DistributionSpecHash]
-------------hashAgg[GLOBAL]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashAgg[LOCAL]
-------------------PhysicalUnion
---------------------PhysicalDistribute[DistributionSpecExecutionAny]
-----------------------PhysicalOlapScan[t1]
---------------------PhysicalDistribute[DistributionSpecExecutionAny]
-----------------------PhysicalOlapScan[t2]
+------------hashAgg[LOCAL]
+--------------PhysicalUnion
+----------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------PhysicalOlapScan[t1]
+----------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------PhysicalOlapScan[t2]
+--------PhysicalDistribute[DistributionSpecHash]
----------PhysicalOlapScan[t3]
-------PhysicalOlapScan[t4]
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalOlapScan[t4]
-- !join_with_union --
PhysicalResultSink
@@ -241,8 +243,9 @@ PhysicalResultSink
-- !except_with_subquery --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalExcept[bucketShuffle]
-------PhysicalOlapScan[t1]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------filter((t2.score > 10))
----------PhysicalOlapScan[t2]
@@ -292,9 +295,10 @@ PhysicalResultSink
-- !except_complex_subquery --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalExcept[bucketShuffle]
-------PhysicalProject
---------PhysicalOlapScan[t1]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------PhysicalProject
----------filter((t2.score > 20))
@@ -385,9 +389,10 @@ SyntaxError:
-- !with_hint_except_distinct --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalExcept[bucketShuffle]
-------hashAgg[GLOBAL]
---------PhysicalOlapScan[t1]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------hashAgg[GLOBAL]
+----------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------hashAgg[GLOBAL]
----------PhysicalOlapScan[t2]
@@ -413,9 +418,10 @@ SyntaxError:
-- !with_hint_intersect_distinct --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalIntersect[bucketShuffle]
-------hashAgg[GLOBAL]
---------PhysicalOlapScan[t1]
+----PhysicalIntersect
+------PhysicalDistribute[DistributionSpecHash]
+--------hashAgg[GLOBAL]
+----------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------hashAgg[GLOBAL]
----------PhysicalOlapScan[t2]
@@ -451,25 +457,25 @@ SyntaxError:
-- !with_hint_mixed_set_operators --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalIntersect[bucketShuffle]
-------PhysicalDistribute[DistributionSpecHash]
---------hashAgg[GLOBAL]
-----------PhysicalExcept[bucketShuffle]
+----PhysicalIntersect
+------hashAgg[GLOBAL]
+--------PhysicalExcept
+----------hashAgg[GLOBAL]
------------PhysicalDistribute[DistributionSpecHash]
---------------hashAgg[GLOBAL]
-----------------PhysicalDistribute[DistributionSpecHash]
-------------------hashAgg[LOCAL]
---------------------PhysicalUnion
-----------------------PhysicalDistribute[DistributionSpecExecutionAny]
-------------------------hashAgg[GLOBAL]
---------------------------PhysicalOlapScan[t1]
-----------------------PhysicalDistribute[DistributionSpecExecutionAny]
-------------------------hashAgg[GLOBAL]
---------------------------PhysicalOlapScan[t2]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalDistribute[DistributionSpecExecutionAny]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalOlapScan[t1]
+------------------PhysicalDistribute[DistributionSpecExecutionAny]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalOlapScan[t2]
+----------PhysicalDistribute[DistributionSpecHash]
------------hashAgg[GLOBAL]
--------------PhysicalOlapScan[t3]
-------hashAgg[GLOBAL]
---------PhysicalOlapScan[t4]
+------PhysicalDistribute[DistributionSpecHash]
+--------hashAgg[GLOBAL]
+----------PhysicalOlapScan[t4]
Hint log:
Used: use_INFER_SET_OPERATOR_DISTINCT
@@ -687,9 +693,10 @@ SyntaxError:
-- !with_hint_except_with_subquery --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalExcept[bucketShuffle]
-------hashAgg[GLOBAL]
---------PhysicalOlapScan[t1]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------hashAgg[GLOBAL]
+----------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------hashAgg[GLOBAL]
----------filter((t2.score > 10))
@@ -764,10 +771,11 @@ SyntaxError:
-- !with_hint_except_complex_subquery --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalExcept[bucketShuffle]
-------hashAgg[GLOBAL]
---------PhysicalProject
-----------PhysicalOlapScan[t1]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------hashAgg[GLOBAL]
----------PhysicalProject
@@ -860,8 +868,9 @@ SyntaxError:
-- !with_hint_no_except_distinct --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalExcept[bucketShuffle]
-------PhysicalOlapScan[t1]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------PhysicalOlapScan[t2]
@@ -886,8 +895,9 @@ SyntaxError:
-- !with_hint_no_intersect_distinct --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalIntersect[bucketShuffle]
-------PhysicalOlapScan[t1]
+----PhysicalIntersect
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------PhysicalOlapScan[t2]
@@ -922,20 +932,20 @@ SyntaxError:
-- !with_hint_no_mixed_set_operators --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalIntersect[bucketShuffle]
-------PhysicalDistribute[DistributionSpecHash]
---------PhysicalExcept[bucketShuffle]
+----PhysicalIntersect
+------PhysicalExcept
+--------hashAgg[GLOBAL]
----------PhysicalDistribute[DistributionSpecHash]
-------------hashAgg[GLOBAL]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashAgg[LOCAL]
-------------------PhysicalUnion
---------------------PhysicalDistribute[DistributionSpecExecutionAny]
-----------------------PhysicalOlapScan[t1]
---------------------PhysicalDistribute[DistributionSpecExecutionAny]
-----------------------PhysicalOlapScan[t2]
+------------hashAgg[LOCAL]
+--------------PhysicalUnion
+----------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------PhysicalOlapScan[t1]
+----------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------PhysicalOlapScan[t2]
+--------PhysicalDistribute[DistributionSpecHash]
----------PhysicalOlapScan[t3]
-------PhysicalOlapScan[t4]
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalOlapScan[t4]
Hint log:
Used:
@@ -1129,8 +1139,9 @@ SyntaxError:
-- !with_hint_no_except_with_subquery --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalExcept[bucketShuffle]
-------PhysicalOlapScan[t1]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------filter((t2.score > 10))
----------PhysicalOlapScan[t2]
@@ -1200,9 +1211,10 @@ SyntaxError:
-- !with_hint_no_except_complex_subquery --
PhysicalResultSink
--PhysicalDistribute[DistributionSpecGather]
-----PhysicalExcept[bucketShuffle]
-------PhysicalProject
---------PhysicalOlapScan[t1]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
------PhysicalDistribute[DistributionSpecHash]
--------PhysicalProject
----------filter((t2.score > 20))
diff --git
a/regression-test/data/shape_check/tpch_sf1000/runtime_filter/test_pushdown_setop.out
b/regression-test/data/shape_check/tpch_sf1000/runtime_filter/test_pushdown_setop.out
index 5a6b91c6e66..1cb60a27a39 100644
---
a/regression-test/data/shape_check/tpch_sf1000/runtime_filter/test_pushdown_setop.out
+++
b/regression-test/data/shape_check/tpch_sf1000/runtime_filter/test_pushdown_setop.out
@@ -6,16 +6,16 @@ PhysicalResultSink
------hashAgg[LOCAL]
--------PhysicalProject
----------hashJoin[INNER_JOIN broadcast] hashCondition=((T.l_linenumber =
expr_cast(r_regionkey as BIGINT))) otherCondition=() build RFs:RF0
expr_cast(r_regionkey as BIGINT)->[cast(l_linenumber as BIGINT),o_orderkey]
-------------PhysicalExcept[bucketShuffle] RFV2: RF1[l_linenumber->o_orderkey]
+------------PhysicalExcept RFV2: RF1[l_linenumber->o_orderkey]
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[lineitem] apply RFs: RF0
--------------PhysicalDistribute[DistributionSpecHash]
----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------PhysicalOlapScan[lineitem] apply RFs: RF0
---------------hashAgg[GLOBAL]
-----------------PhysicalProject
-------------------PhysicalOlapScan[orders] apply RFs: RF0 RFV2: RF1
+------------------PhysicalProject
+--------------------PhysicalOlapScan[orders] apply RFs: RF0 RFV2: RF1
------------PhysicalProject
--------------PhysicalOlapScan[region]
@@ -27,16 +27,16 @@ PhysicalResultSink
--------PhysicalProject
----------hashJoin[INNER_JOIN broadcast]
hashCondition=((expr_abs(l_linenumber) = expr_cast(r_regionkey as LARGEINT)))
otherCondition=() build RFs:RF0 expr_cast(r_regionkey as
LARGEINT)->[abs(cast(l_linenumber as BIGINT)),abs(o_orderkey)]
------------PhysicalProject
---------------PhysicalExcept[bucketShuffle] RFV2: RF1[l_linenumber->o_orderkey]
+--------------PhysicalExcept RFV2: RF1[l_linenumber->o_orderkey]
+----------------hashAgg[GLOBAL]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------hashAgg[LOCAL]
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[lineitem] apply RFs: RF0
----------------PhysicalDistribute[DistributionSpecHash]
------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[lineitem] apply RFs: RF0
-----------------hashAgg[GLOBAL]
-------------------PhysicalProject
---------------------PhysicalOlapScan[orders] apply RFs: RF0 RFV2: RF1
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[orders] apply RFs: RF0 RFV2: RF1
------------PhysicalProject
--------------PhysicalOlapScan[region]
diff --git
a/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy
b/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy
index 81ac60baca7..5533853eaa9 100644
---
a/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy
+++
b/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy
@@ -16,6 +16,9 @@
// under the License.
suite("bucket_shuffle_set_operation") {
+ // TODO: open comment when support `enable_local_shuffle_planner` and
change to REQUIRE
+ return
+
multi_sql """
drop table if exists bucket_shuffle_set_operation1;
create table bucket_shuffle_set_operation1(id int, value int)
distributed by hash(id) buckets 10 properties('replication_num'='1');
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
