This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ce13a1d951b [fix](nereids) make runtime filter order stable #29203
ce13a1d951b is described below
commit ce13a1d951b46e232086168ff779a0952b571bd2
Author: minghong <[email protected]>
AuthorDate: Fri Dec 29 01:00:27 2023 +0800
[fix](nereids) make runtime filter order stable #29203
---
.../apache/doris/nereids/processor/post/RuntimeFilterContext.java | 6 +++---
.../apache/doris/nereids/processor/post/RuntimeFilterGenerator.java | 6 ++++--
.../data/nereids_tpcds_shape_sf1000_p0/shape/query95.out | 4 ++--
.../data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out | 4 ++--
.../data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out | 4 ++--
.../data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out | 4 ++--
regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out | 4 ++--
7 files changed, 17 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java
index 05467588f80..4360e5659c5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java
@@ -121,17 +121,17 @@ public class RuntimeFilterContext {
// you can see disjoint set data structure to learn the processing
detailed.
private final Map<NamedExpression, Pair<PhysicalRelation, Slot>>
aliasTransferMap = Maps.newHashMap();
- private final Map<Slot, ScanNode> scanNodeOfLegacyRuntimeFilterTarget =
Maps.newHashMap();
+ private final Map<Slot, ScanNode> scanNodeOfLegacyRuntimeFilterTarget =
Maps.newLinkedHashMap();
private final Set<Plan> effectiveSrcNodes = Sets.newHashSet();
// cte to related joins map which can extract common runtime filter to cte
inside
- private final Map<CTEId, Set<PhysicalHashJoin>> cteToJoinsMap =
Maps.newHashMap();
+ private final Map<CTEId, Set<PhysicalHashJoin>> cteToJoinsMap =
Maps.newLinkedHashMap();
// cte candidates which can be pushed into common runtime filter into from
outside
private final Map<PhysicalCTEProducer, Map<EqualTo, PhysicalHashJoin>>
cteRFPushDownMap = Maps.newLinkedHashMap();
- private final Map<CTEId, PhysicalCTEProducer> cteProducerMap =
Maps.newHashMap();
+ private final Map<CTEId, PhysicalCTEProducer> cteProducerMap =
Maps.newLinkedHashMap();
// cte whose runtime filter has been extracted
private final Set<CTEId> processedCTE = Sets.newHashSet();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
index bae483c2939..45a14d3b633 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
@@ -71,6 +71,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -392,7 +393,7 @@ public class RuntimeFilterGenerator extends
PlanPostProcessor {
private void collectPushDownCTEInfos(PhysicalHashJoin<? extends Plan, ?
extends Plan> join,
CascadesContext context) {
RuntimeFilterContext ctx = context.getRuntimeFilterContext();
- Set<CTEId> cteIds = new HashSet<>();
+ Set<CTEId> cteIds = new LinkedHashSet<>(); // use LinkedHashSet to
make runtime filter order stable
PhysicalPlan leftChild = (PhysicalPlan) join.left();
PhysicalPlan rightChild = (PhysicalPlan) join.right();
@@ -404,7 +405,8 @@ public class RuntimeFilterGenerator extends
PlanPostProcessor {
if ((leftHasCTE && !rightHasCTE) || (!leftHasCTE && rightHasCTE)) {
for (CTEId id : cteIds) {
if (ctx.getCteToJoinsMap().get(id) == null) {
- Set<PhysicalHashJoin> newJoin = new HashSet<>();
+ // use LinkedHashSet to make runtime filter order stable
+ Set<PhysicalHashJoin> newJoin = new LinkedHashSet<>();
newJoin.add(join);
ctx.getCteToJoinsMap().put(id, newJoin);
} else {
diff --git
a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
index 6720aab10b8..c50e58dd00e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------------------PhysicalProject
--------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = web_returns.wr_order_number))
otherCondition=()
----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6
wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7
wr_order_number->[ws_order_number,ws_order_number]
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF7
ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF6
ws_order_number->[ws_order_number,ws_order_number]
------------------------PhysicalDistribute
--------------------------PhysicalProject
----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
diff --git
a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
index d2ecc4813b5..9fb38391e5e 100644
---
a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
+++
b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------------------PhysicalProject
--------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = web_returns.wr_order_number))
otherCondition=()
----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6
wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7
wr_order_number->[ws_order_number,ws_order_number]
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF7
ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF6
ws_order_number->[ws_order_number,ws_order_number]
------------------------PhysicalDistribute
--------------------------PhysicalProject
----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
diff --git
a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
index d2ecc4813b5..9fb38391e5e 100644
---
a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
+++
b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------------------PhysicalProject
--------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = web_returns.wr_order_number))
otherCondition=()
----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6
wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7
wr_order_number->[ws_order_number,ws_order_number]
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF7
ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF6
ws_order_number->[ws_order_number,ws_order_number]
------------------------PhysicalDistribute
--------------------------PhysicalProject
----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
diff --git
a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
index bde9dbef392..acee79a67ec 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------------------PhysicalProject
--------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = web_returns.wr_order_number))
otherCondition=()
----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6
wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7
wr_order_number->[ws_order_number,ws_order_number]
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF7
ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF6
ws_order_number->[ws_order_number,ws_order_number]
------------------------PhysicalDistribute
--------------------------PhysicalProject
----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
diff --git
a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
index bde9dbef392..acee79a67ec 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------------------PhysicalProject
--------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = web_returns.wr_order_number))
otherCondition=()
----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6
wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN]
hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number))
otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7
wr_order_number->[ws_order_number,ws_order_number]
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF7
ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN]
hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=()
build RFs:RF4 ws_order_number->[ws_order_number];RF6
ws_order_number->[ws_order_number,ws_order_number]
------------------------PhysicalDistribute
--------------------------PhysicalProject
----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
