This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0-beta in repository https://gitbox.apache.org/repos/asf/doris.git
commit 31bd9ba03cc26818bd2d440702815c2b7956e8e4 Author: jakevin <[email protected]> AuthorDate: Mon Jun 5 16:56:25 2023 +0800 [fix](Nereids): fix filter can't be pushdown unionAll (#20310) --- .../logical/PushdownFilterThroughSetOperation.java | 21 ++--- .../nereids/trees/plans/logical/LogicalExcept.java | 5 - .../trees/plans/logical/LogicalIntersect.java | 5 - .../trees/plans/logical/LogicalSetOperation.java | 3 - .../nereids/trees/plans/logical/LogicalUnion.java | 5 - .../nereids_tpcds_shape_sf100_p0/shape/query75.out | 104 ++++++++++----------- 6 files changed, 58 insertions(+), 85 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java index 218bf9e2ee..f678951d78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java @@ -45,11 +45,11 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalFilter(logicalSetOperation()).then(filter -> { - LogicalSetOperation setOperation = filter.child(); + return logicalFilter(logicalSetOperation()).when(f -> f.child().getQualifier() == Qualifier.ALL).then(f -> { + LogicalSetOperation setOperation = f.child(); if (setOperation instanceof LogicalUnion && ((LogicalUnion) setOperation).hasPushedFilter()) { - return filter; + return f; } List<Plan> newChildren = new ArrayList<>(); @@ -70,26 +70,19 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory { replaceMap.put(output, child.getOutput().get(i)); } - Set<Expression> newFilterPredicates = filter.getConjuncts().stream().map(conjunct -> + Set<Expression> newFilterPredicates = f.getConjuncts().stream().map(conjunct -> ExpressionUtils.replace(conjunct, replaceMap)).collect(ImmutableSet.toImmutableSet()); newChildren.add(new LogicalFilter<>(newFilterPredicates, child)); } - if (allOneRowRelation) { - return filter; - } - - if (setOperation instanceof LogicalUnion && setOperation.getQualifier() == Qualifier.DISTINCT) { - return new LogicalFilter<>(filter.getConjuncts(), - ((LogicalUnion) setOperation).withHasPushedFilter().withChildren(newChildren)); + return f; } if (hasOneRowRelation) { // If there are some `OneRowRelation` exists, we need to keep the `filter`. - return filter.withChildren( - ((LogicalUnion) setOperation).withHasPushedFilter().withNewChildren(newChildren)); + return f.withChildren(((LogicalUnion) setOperation).withHasPushedFilter().withChildren(newChildren)); } - return setOperation.withNewChildren(newChildren); + return setOperation.withChildren(newChildren); }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_SET_OPERATION); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java index cd55848405..f11e9782a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java @@ -80,9 +80,4 @@ public class LogicalExcept extends LogicalSetOperation { public LogicalExcept withNewOutputs(List<NamedExpression> newOutputs) { return new LogicalExcept(qualifier, newOutputs, Optional.empty(), Optional.empty(), children); } - - @Override - public LogicalExcept withNewChildren(List<Plan> children) { - return withChildren(children); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java index 2f3b37453e..6c831da1d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java @@ -82,9 +82,4 @@ public class LogicalIntersect extends LogicalSetOperation { return new LogicalIntersect(qualifier, newOutputs, Optional.empty(), Optional.empty(), children); } - - @Override - public LogicalIntersect withNewChildren(List<Plan> children) { - return withChildren(children); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index 31d8ab6dcf..308cd2a9e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -224,7 +224,4 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements public int getArity() { return children.size(); } - - public abstract LogicalSetOperation withNewChildren(List<Plan> children); - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index 1f53f89d6d..934df533f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -116,11 +116,6 @@ public class LogicalUnion extends LogicalSetOperation implements OutputPrunable return new LogicalUnion(qualifier, outputs, true, Optional.empty(), Optional.empty(), children); } - @Override - public LogicalUnion withNewChildren(List<Plan> children) { - return withChildren(children); - } - @Override public LogicalUnion pruneOutputs(List<NamedExpression> prunedOutputs) { return withNewOutputs(prunedOutputs); diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out index 319e8a5922..4997e1c25c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out @@ -8,64 +8,62 @@ CteAnchor[cteId= ( CTEId#3=] ) ----------hashAgg[GLOBAL] ------------PhysicalDistribute --------------hashAgg[LOCAL] -----------------filter(((all_sales.d_year = 1998) OR (all_sales.d_year = 1999))) -------------------PhysicalUnion ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute -------------------------hashAgg[LOCAL] ---------------------------filter(((d_year = 1998) OR (d_year = 1999))) -----------------------------PhysicalUnion +----------------PhysicalUnion +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute +----------------------hashAgg[LOCAL] +------------------------PhysicalUnion +--------------------------PhysicalProject +----------------------------hashJoin[RIGHT_OUTER_JOIN](catalog_sales.cs_item_sk = catalog_returns.cr_item_sk)(catalog_sales.cs_order_number = catalog_returns.cr_order_number) ------------------------------PhysicalProject ---------------------------------hashJoin[RIGHT_OUTER_JOIN](catalog_sales.cs_item_sk = catalog_returns.cr_item_sk)(catalog_sales.cs_order_number = catalog_returns.cr_order_number) -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[catalog_returns] -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk) ---------------------------------------hashJoin[INNER_JOIN](item.i_item_sk = catalog_sales.cs_item_sk) -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[catalog_sales] -----------------------------------------PhysicalDistribute -------------------------------------------PhysicalProject ---------------------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) -----------------------------------------------PhysicalOlapScan[item] ---------------------------------------PhysicalDistribute -----------------------------------------PhysicalProject -------------------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) ---------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------PhysicalOlapScan[catalog_returns] ------------------------------PhysicalProject ---------------------------------hashJoin[RIGHT_OUTER_JOIN](store_sales.ss_item_sk = store_returns.sr_item_sk)(store_sales.ss_ticket_number = store_returns.sr_ticket_number) -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[store_returns] -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) ---------------------------------------hashJoin[INNER_JOIN](item.i_item_sk = store_sales.ss_item_sk) -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] -----------------------------------------PhysicalDistribute -------------------------------------------PhysicalProject ---------------------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) -----------------------------------------------PhysicalOlapScan[item] ---------------------------------------PhysicalDistribute -----------------------------------------PhysicalProject -------------------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) ---------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------hashJoin[RIGHT_OUTER_JOIN](web_sales.ws_item_sk = web_returns.wr_item_sk)(web_sales.ws_order_number = web_returns.wr_order_number) -------------------------PhysicalProject ---------------------------PhysicalOlapScan[web_returns] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk) -----------------------------hashJoin[INNER_JOIN](item.i_item_sk = web_sales.ws_item_sk) +--------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk) +----------------------------------hashJoin[INNER_JOIN](item.i_item_sk = catalog_sales.cs_item_sk) +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[catalog_sales] +------------------------------------PhysicalDistribute +--------------------------------------PhysicalProject +----------------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) +------------------------------------------PhysicalOlapScan[item] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) +----------------------------------------PhysicalOlapScan[date_dim] +--------------------------PhysicalProject +----------------------------hashJoin[RIGHT_OUTER_JOIN](store_sales.ss_item_sk = store_returns.sr_item_sk)(store_sales.ss_ticket_number = store_returns.sr_ticket_number) ------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[web_sales] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) -------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalOlapScan[store_returns] +------------------------------PhysicalProject +--------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) +----------------------------------hashJoin[INNER_JOIN](item.i_item_sk = store_sales.ss_item_sk) +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[store_sales] +------------------------------------PhysicalDistribute +--------------------------------------PhysicalProject +----------------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) +------------------------------------------PhysicalOlapScan[item] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) +----------------------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------hashJoin[RIGHT_OUTER_JOIN](web_sales.ws_item_sk = web_returns.wr_item_sk)(web_sales.ws_order_number = web_returns.wr_order_number) +----------------------PhysicalProject +------------------------PhysicalOlapScan[web_returns] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk) +--------------------------hashJoin[INNER_JOIN](item.i_item_sk = web_sales.ws_item_sk) +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[web_sales] ----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) -----------------------------------PhysicalOlapScan[date_dim] +--------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) +----------------------------------PhysicalOlapScan[item] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) +--------------------------------PhysicalOlapScan[date_dim] --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
