This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 4cc6076814b branch-3.1: [opt](nereids) logical filter replace null to
false #49457 (#51985)
4cc6076814b is described below
commit 4cc6076814b237f712b13feee4b821fa38702bd1
Author: yujun <[email protected]>
AuthorDate: Mon Jun 23 19:38:54 2025 +0800
branch-3.1: [opt](nereids) logical filter replace null to false #49457
(#51985)
Cherry-pick from #49457
---
.../rules/expression/ExpressionOptimization.java | 14 ++-
.../rules/expression/rules/FoldConstantRule.java | 9 +-
.../nereids/rules/rewrite/EliminateFilter.java | 34 ++++++-
.../expression/rules/ExpressionRewriteSqlTest.java | 65 +++++++++++++
.../nereids/rules/rewrite/EliminateFilterTest.java | 105 +++++++++++++++++++++
5 files changed, 220 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
index a5b94d216e9..b91f9a3d0ec 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
@@ -42,13 +42,19 @@ import java.util.List;
public class ExpressionOptimization extends ExpressionRewrite {
public static final List<ExpressionRewriteRule> OPTIMIZE_REWRITE_RULES =
ImmutableList.of(
bottomUp(
- ExtractCommonFactorRule.INSTANCE,
- DistinctPredicatesRule.INSTANCE,
- SimplifyComparisonPredicate.INSTANCE,
SimplifyInPredicate.INSTANCE,
- SimplifyDecimalV3Comparison.INSTANCE,
+
+ // comparison predicates
+ SimplifyComparisonPredicate.INSTANCE,
+
+ // compound predicates
SimplifyRange.INSTANCE,
SimplifyConflictCompound.INSTANCE,
+ DistinctPredicatesRule.INSTANCE,
+ ExtractCommonFactorRule.INSTANCE,
+
+ SimplifyDecimalV3Comparison.INSTANCE,
+
DateFunctionRewrite.INSTANCE,
ArrayContainToArrayOverlap.INSTANCE,
CaseWhenToIf.INSTANCE,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRule.java
index cd10b6a7e37..a4874d7ad31 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRule.java
@@ -23,6 +23,7 @@ import
org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
import org.apache.doris.nereids.rules.expression.ExpressionRewrite;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.qe.SessionVariable;
import com.google.common.collect.ImmutableList;
@@ -51,9 +52,13 @@ public class FoldConstantRule implements
ExpressionPatternRuleFactory {
/** evaluate by visitor */
public static Expression evaluate(Expression expr,
ExpressionRewriteContext ctx) {
- if (ctx.cascadesContext != null
+ SessionVariable sessionVariable = ctx.cascadesContext != null
&& ctx.cascadesContext.getConnectContext() != null
- &&
ctx.cascadesContext.getConnectContext().getSessionVariable().isEnableFoldConstantByBe())
{
+ ?
ctx.cascadesContext.getConnectContext().getSessionVariable() : null;
+ if (sessionVariable != null &&
sessionVariable.isDebugSkipFoldConstant()) {
+ return expr;
+ }
+ if (sessionVariable != null &&
sessionVariable.isEnableFoldConstantByBe()) {
return FULL_FOLD_REWRITER.rewrite(expr, ctx);
} else {
return FoldConstantRuleOnFE.VISITOR_INSTANCE.rewrite(expr, ctx);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
index 1413faf3bb0..6b4668fffad 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
@@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
+import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
@@ -31,6 +32,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.util.ExpressionUtils;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -49,7 +51,10 @@ public class EliminateFilter implements RewriteRuleFactory {
.thenApply(ctx -> {
LogicalFilter<Plan> filter = ctx.root;
ImmutableSet.Builder<Expression> newConjuncts =
ImmutableSet.builder();
+ ExpressionRewriteContext context =
+ new ExpressionRewriteContext(ctx.cascadesContext);
for (Expression expression : filter.getConjuncts()) {
+ expression =
FoldConstantRule.evaluate(eliminateNullLiteral(expression), context);
if (expression == BooleanLiteral.FALSE ||
expression.isNullLiteral()) {
return new
LogicalEmptyRelation(ctx.statementContext.getNextRelationId(),
filter.getOutput());
@@ -75,7 +80,7 @@ public class EliminateFilter implements RewriteRuleFactory {
new ExpressionRewriteContext(ctx.cascadesContext);
for (Expression expression : filter.getConjuncts()) {
Expression newExpr =
ExpressionUtils.replace(expression, replaceMap);
- Expression foldExpression =
FoldConstantRule.evaluate(newExpr, context);
+ Expression foldExpression =
FoldConstantRule.evaluate(eliminateNullLiteral(newExpr), context);
if (foldExpression == BooleanLiteral.FALSE ||
expression.isNullLiteral()) {
return new LogicalEmptyRelation(
@@ -94,4 +99,31 @@ public class EliminateFilter implements RewriteRuleFactory {
})
.toRule(RuleType.ELIMINATE_FILTER_ON_ONE_RELATION));
}
+
+ @VisibleForTesting
+ public Expression eliminateNullLiteral(Expression expression) {
+ if (!expression.anyMatch(e -> ((Expression) e).isNullLiteral())) {
+ return expression;
+ }
+
+ return replaceNullToFalse(expression);
+ }
+
+ // only replace null which its ancestors are all and/or
+ // NOTICE: NOT's type is boolean too, if replace null to false in NOT,
will got NOT(NULL) = NOT(FALSE) = TRUE,
+ // but it is wrong, NOT(NULL) = NULL. For a filter, only the AND / OR,
can keep NULL as FALSE.
+ private Expression replaceNullToFalse(Expression expression) {
+ if (expression.isNullLiteral()) {
+ return BooleanLiteral.FALSE;
+ }
+
+ if (expression instanceof CompoundPredicate) {
+ ImmutableList.Builder<Expression> builder =
ImmutableList.builderWithExpectedSize(
+ expression.children().size());
+ expression.children().forEach(e ->
builder.add(replaceNullToFalse(e)));
+ return expression.withChildren(builder.build());
+ }
+
+ return expression;
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/ExpressionRewriteSqlTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/ExpressionRewriteSqlTest.java
new file mode 100644
index 00000000000..fcd0505fbe6
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/ExpressionRewriteSqlTest.java
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.util.PlanChecker;
+
+import org.junit.jupiter.api.Test;
+
+class ExpressionRewriteSqlTest extends SqlTestBase {
+ @Test
+ public void testSimplifyRangeAndExtractCommonFactor() {
+ String sql = "select * from T1 where id > 1 and score > 1 or id > 1
and score > 10";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matches(
+ logicalFilter().when(f ->
f.getPredicate().toSql().equals(
+ "AND[(id > 1),(score > 1)]"
+ )));
+
+ sql = "select * from T1 where id > 1 and score > 1 or id > 1 and id <
0";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matches(
+ logicalFilter().when(f ->
f.getPredicate().toSql().equals(
+ "AND[(id > 1),(score > 1)]"
+ )));
+
+ sql = "select * from T1 where id > 1 and id < 0 or score > 1 and score
< 0";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matches(logicalEmptyRelation());
+
+ sql = "select * from T1 where id > 1 and id < 0 and score > 1 and
score < 0";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matches(logicalEmptyRelation());
+
+ sql = "select * from T1 where not(id > 1 and id < 0 or score > 1 and
score < 0)";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matches(logicalFilter().when(
+ f -> f.getPredicate().toSql().equals("AND[( not id IS
NULL),( not score IS NULL)]")));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
index d3d1316eaac..692f6532541 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
@@ -19,21 +19,31 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.expression.ExpressionNormalization;
import org.apache.doris.nereids.trees.expressions.And;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.types.IntegerType;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.LogicalPlanBuilder;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.nereids.util.PlanConstructor;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.Arrays;
+import java.util.List;
+
/**
* Tests for {@link EliminateFilter}.
*/
@@ -62,6 +72,80 @@ class EliminateFilterTest implements
MemoPatternMatchSupported {
.matches(logicalEmptyRelation());
}
+ @Test
+ void testEliminateFilterReduceNull() {
+ List<Expression> exprList = Arrays.asList(
+ new EqualTo(scan1.getOutput().get(0), Literal.of(1)),
+ new GreaterThan(scan1.getOutput().get(1), Literal.of(1)),
+ NullLiteral.INSTANCE,
+ NullLiteral.INSTANCE);
+ Expression expr = new
Or(ExpressionUtils.falseOrNull(scan1.getOutput().get(1)),
+ new GreaterThan(scan1.getOutput().get(0), Literal.of(1)));
+ LogicalPlan filter = new LogicalPlanBuilder(scan1)
+ .filter(expr)
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filter)
+ .applyTopDown(new EliminateFilter())
+ .matches(
+ logicalFilter().when(f ->
f.getPredicate().toSql().equals("(id > 1)"))
+ );
+
+ expr = new Or(ExpressionUtils.falseOrNull(scan1.getOutput().get(0)),
+ ExpressionUtils.falseOrNull(scan1.getOutput().get(1)));
+ filter = new LogicalPlanBuilder(scan1)
+ .filter(expr)
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filter)
+ .applyTopDown(new EliminateFilter())
+ .matches(logicalEmptyRelation());
+
+ filter = new LogicalPlanBuilder(scan1)
+ .filter(new And(exprList))
+ .build();
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filter)
+ .applyTopDown(new EliminateFilter())
+ .matches(logicalEmptyRelation());
+
+ filter = new LogicalPlanBuilder(scan1)
+ .filter(new And(NullLiteral.INSTANCE, new Or(exprList)))
+ .build();
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filter)
+ .applyTopDown(new EliminateFilter())
+ .matches(logicalEmptyRelation());
+
+ filter = new LogicalPlanBuilder(scan1)
+ .filter(new Or(NullLiteral.INSTANCE, NullLiteral.INSTANCE))
+ .build();
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filter)
+ .applyTopDown(new EliminateFilter())
+ .matches(logicalEmptyRelation());
+
+ filter = new LogicalPlanBuilder(scan1)
+ .filter(NullLiteral.INSTANCE)
+ .build();
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filter)
+ .applyTopDown(new EliminateFilter())
+ .matches(logicalEmptyRelation());
+
+ filter = new LogicalPlanBuilder(scan1)
+ .filter(new Not(NullLiteral.INSTANCE))
+ .build();
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filter)
+ .applyTopDown(new EliminateFilter())
+ .matches(logicalEmptyRelation());
+
+ filter = new LogicalPlanBuilder(scan1)
+ .filter(new Not(new And(exprList)))
+ .build();
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filter)
+ .applyTopDown(new EliminateFilter())
+ .matches(logicalFilter().when(
+ f -> f.getPredicate().toSql().equals("( not AND[(id =
1),(name > 1),NULL,NULL])"))
+ );
+ }
+
@Test
void testEliminateFilterTrue() {
LogicalPlan filterTrue = new LogicalPlanBuilder(scan1)
@@ -102,4 +186,25 @@ class EliminateFilterTest implements
MemoPatternMatchSupported {
logicalFilter(logicalOlapScan()).when(f ->
f.getPredicate() instanceof GreaterThan)
);
}
+
+ @Test
+ void testEliminateNullLiteral() {
+ Expression a = new SlotReference("a", IntegerType.INSTANCE);
+ Expression b = new SlotReference("b", IntegerType.INSTANCE);
+ Expression one = Literal.of(1);
+ Expression two = Literal.of(2);
+ Expression expression = new And(Arrays.asList(
+ new And(new GreaterThan(a, one), new
NullLiteral(IntegerType.INSTANCE)),
+ new Or(Arrays.asList(new GreaterThan(b, two), new
NullLiteral(IntegerType.INSTANCE),
+ new EqualTo(a, new NullLiteral(IntegerType.INSTANCE)))),
+ new Not(new And(new GreaterThan(a, one), new
NullLiteral(IntegerType.INSTANCE)))
+ ));
+ Expression expectExpression = new And(Arrays.asList(
+ new And(new GreaterThan(a, one), BooleanLiteral.FALSE),
+ new Or(Arrays.asList(new GreaterThan(b, two),
BooleanLiteral.FALSE,
+ new EqualTo(a, new
NullLiteral(IntegerType.INSTANCE)))),
+ new Not(new And(new GreaterThan(a, one), new
NullLiteral(IntegerType.INSTANCE)))
+ ));
+ Assertions.assertEquals(expectExpression, new
EliminateFilter().eliminateNullLiteral(expression));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
