This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 369dd5d597e42f637ac06068e03e1f72b85f4ac5 Author: starocean999 <[email protected]> AuthorDate: Wed Oct 11 16:12:24 2023 +0800 [enhancement](nereids)add eliminate filter on one row relation rule (#24980) 1.simplify PushdownFilterThroughSetOperation rule 2.add eliminate filter on one row relation rule --- .../org/apache/doris/nereids/rules/RuleType.java | 1 + .../nereids/rules/rewrite/EliminateFilter.java | 61 ++++++++++++++++++---- .../rewrite/PushdownFilterThroughSetOperation.java | 27 +--------- .../data/nereids_p0/test_filter_pushdown_set.out | 24 +++++++++ .../nereids_p0/test_filter_pushdown_set.groovy | 24 +++++++-- 5 files changed, 98 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 89de2182767..b1489e47165 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -194,6 +194,7 @@ public enum RuleType { ELIMINATE_LIMIT_ON_ONE_ROW_RELATION(RuleTypeClass.REWRITE), ELIMINATE_LIMIT_ON_EMPTY_RELATION(RuleTypeClass.REWRITE), ELIMINATE_FILTER(RuleTypeClass.REWRITE), + ELIMINATE_FILTER_ON_ONE_RELATION(RuleTypeClass.REWRITE), ELIMINATE_NOT_NULL(RuleTypeClass.REWRITE), ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE), ELIMINATE_OUTER_JOIN(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java index 5f5d8b39a83..2f46c43ecff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java @@ -19,27 +19,36 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.util.ExpressionUtils; -import com.google.common.collect.Sets; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; -import java.util.Set; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** * Eliminate filter which is FALSE or TRUE. */ -public class EliminateFilter extends OneRewriteRuleFactory { +public class EliminateFilter implements RewriteRuleFactory { @Override - public Rule build() { - return logicalFilter() - .when(filter -> filter.getConjuncts().stream().anyMatch(BooleanLiteral.class::isInstance)) + public List<Rule> buildRules() { + return ImmutableList.of(logicalFilter().when( + filter -> filter.getConjuncts().stream().anyMatch(BooleanLiteral.class::isInstance)) .thenApply(ctx -> { LogicalFilter<Plan> filter = ctx.root; - Set<Expression> newConjuncts = Sets.newHashSetWithExpectedSize(filter.getConjuncts().size()); + ImmutableSet.Builder newConjuncts = ImmutableSet.builder(); for (Expression expression : filter.getConjuncts()) { if (expression == BooleanLiteral.FALSE) { return new LogicalEmptyRelation(ctx.statementContext.getNextRelationId(), @@ -48,12 +57,44 @@ public class EliminateFilter extends OneRewriteRuleFactory { newConjuncts.add(expression); } } - if (newConjuncts.isEmpty()) { + + ImmutableSet<Expression> conjuncts = newConjuncts.build(); + if (conjuncts.isEmpty()) { + return filter.child(); + } else { + return new LogicalFilter<>(conjuncts, filter.child()); + } + }) + .toRule(RuleType.ELIMINATE_FILTER), + logicalFilter(logicalOneRowRelation()).thenApply(ctx -> { + LogicalFilter<LogicalOneRowRelation> filter = ctx.root; + Map<Expression, Expression> replaceMap = + filter.child().getOutputs().stream().filter(e -> e instanceof Alias) + .collect(Collectors.toMap(NamedExpression::toSlot, e -> ((Alias) e).child())); + + ImmutableSet.Builder newConjuncts = ImmutableSet.builder(); + ExpressionRewriteContext context = + new ExpressionRewriteContext(ctx.cascadesContext); + for (Expression expression : filter.getConjuncts()) { + Expression newExpr = ExpressionUtils.replace(expression, replaceMap); + Expression foldExpression = + FoldConstantRule.INSTANCE.rewrite(newExpr, context); + + if (foldExpression == BooleanLiteral.FALSE) { + return new LogicalEmptyRelation( + ctx.statementContext.getNextRelationId(), filter.getOutput()); + } else if (foldExpression != BooleanLiteral.TRUE) { + newConjuncts.add(expression); + } + } + + ImmutableSet<Expression> conjuncts = newConjuncts.build(); + if (conjuncts.isEmpty()) { return filter.child(); } else { - return new LogicalFilter<>(newConjuncts, filter.child()); + return new LogicalFilter<>(conjuncts, filter.child()); } }) - .toRule(RuleType.ELIMINATE_FILTER); + .toRule(RuleType.ELIMINATE_FILTER_ON_ONE_RELATION)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java index 19f5243fc95..6b78ab1f367 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java @@ -22,11 +22,8 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; -import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; -import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableSet; @@ -44,25 +41,10 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalFilter(logicalSetOperation()).when(f -> f.child().getQualifier() == Qualifier.ALL).then(f -> { + return logicalFilter(logicalSetOperation()).then(f -> { LogicalSetOperation setOperation = f.child(); - - if (setOperation instanceof LogicalUnion && ((LogicalUnion) setOperation).hasPushedFilter()) { - return f; - } - List<Plan> newChildren = new ArrayList<>(); - boolean allOneRowRelation = true; - boolean hasOneRowRelation = false; for (Plan child : setOperation.children()) { - if (child instanceof OneRowRelation) { - // We shouldn't push down the 'filter' to 'oneRowRelation'. - hasOneRowRelation = true; - newChildren.add(child); - continue; - } else { - allOneRowRelation = false; - } Map<Expression, Expression> replaceMap = new HashMap<>(); for (int i = 0; i < setOperation.getOutputs().size(); ++i) { NamedExpression output = setOperation.getOutputs().get(i); @@ -73,14 +55,7 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory { ExpressionUtils.replace(conjunct, replaceMap)).collect(ImmutableSet.toImmutableSet()); newChildren.add(new LogicalFilter<>(newFilterPredicates, child)); } - if (allOneRowRelation) { - return f; - } - if (hasOneRowRelation) { - // If there are some `OneRowRelation` exists, we need to keep the `filter`. - return f.withChildren(((LogicalUnion) setOperation).withHasPushedFilter().withChildren(newChildren)); - } return setOperation.withChildren(newChildren); }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_SET_OPERATION); } diff --git a/regression-test/data/nereids_p0/test_filter_pushdown_set.out b/regression-test/data/nereids_p0/test_filter_pushdown_set.out index 42ecbc5ad88..6f6a6062fef 100644 --- a/regression-test/data/nereids_p0/test_filter_pushdown_set.out +++ b/regression-test/data/nereids_p0/test_filter_pushdown_set.out @@ -1,10 +1,34 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select1 -- 1 2 +1 3 +1 3 -- !select2 -- 1 2 +1 3 -- !select3 -- 1 2 +-- !select4 -- +1 3 + +-- !select5 -- +1 2 +1 2 + +-- !select6 -- +1 2 + +-- !select7 -- +1 2 + +-- !select8 -- + +-- !select22 -- +1 2 + +-- !select23 -- +1 2 + diff --git a/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy b/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy index 03c1b257981..7d25aade293 100644 --- a/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy +++ b/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy @@ -18,8 +18,26 @@ suite("test_filter_pushdown_set") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" - qt_select1 'select * from (select 1 as a, 2 as b union all select 3, 3) t where a = 1;' + qt_select1 'select * from (select 1 as a, 2 as b union all select 1, 3 union all select 1, 3) t where a = 1 order by a, b;' + qt_select2 'select * from (select 1 as a, 2 as b union select 1, 3 union select 1, 3) t where a = 1 order by a, b;' + qt_select3 'select * from ((select 1 as a, 2 as b union all select 1, 3) intersect select 1, 2) t where a = 1 order by a, b;' + qt_select4 'select * from ((select 1 as a, 2 as b union all select 1, 3) except select 1, 2 ) t where a = 1 order by a, b;' + qt_select5 'select * from (select 1 as a, 2 as b union all select 1, 3 union all select 1, 2) t where b = 2 order by a, b;' + qt_select6 'select * from (select 1 as a, 2 as b union select 1, 3 union select 1, 2) t where b = 2 order by a, b;' + qt_select7 'select * from ((select 1 as a, 2 as b union all select 1, 3) intersect select 1, 2) t where b = 2 order by a, b;' + qt_select8 'select * from ((select 1 as a, 2 as b union all select 1, 3) except select 1, 2 ) t where b = 2 order by a, b;' + + explain { + sql("select * from ((select 1 as a, 2 as b union all select 1, 3) intersect select 1, 2) t where a = 1;") + notContains "VSELECT" + } + + explain { + sql("select * from ((select 1 as a, 2 as b union all select 1, 3) except select 1, 2 ) t where b = 2;") + notContains "1 | 3" + } + sql "SET enable_nereids_planner=false" - qt_select2 'select * from (select 1 as a, 2 as b union all select 3, 3) t where a = 1;' - qt_select3 'select * from (select 1 as a, 2 as b union select 3, 3) t where a = 1;' + qt_select22 'select * from (select 1 as a, 2 as b union all select 3, 3) t where a = 1 order by a, b;' + qt_select23 'select * from (select 1 as a, 2 as b union select 3, 3) t where a = 1 order by a, b;' } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
