This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 6525aeff33a [opt](Nereids) add where Null rule to create empty
relation as where false (#38135) (#38362)
6525aeff33a is described below
commit 6525aeff33a124c093bb1ef0ff8e42e183e77d1c
Author: LiBinfeng <[email protected]>
AuthorDate: Mon Aug 5 13:49:20 2024 +0800
[opt](Nereids) add where Null rule to create empty relation as where false
(#38135) (#38362)
cherry-pick: #38135
explain shape plan select * from table2 where Null; explain shape plan
select * from table2 where false; in this case, null literal can be
regard as same as false literal
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
---
.../nereids/rules/rewrite/EliminateFilter.java | 8 +-
.../nereids/rules/rewrite/EliminateFilterTest.java | 12 ++
.../data/empty_relation/eliminate_empty.out | 89 +++++++++++++
.../suites/empty_relation/eliminate_empty.groovy | 139 ++++++++++++++++++++-
.../test_multi_range_partition.groovy | 8 +-
.../test_simplify_comparison.groovy | 18 ---
6 files changed, 248 insertions(+), 26 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
index 2f46c43ecff..157555d5591 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
@@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -45,12 +46,13 @@ public class EliminateFilter implements RewriteRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(logicalFilter().when(
- filter ->
filter.getConjuncts().stream().anyMatch(BooleanLiteral.class::isInstance))
+ filter ->
filter.getConjuncts().stream().anyMatch(BooleanLiteral.class::isInstance)
+ ||
filter.getConjuncts().stream().anyMatch(NullLiteral.class::isInstance))
.thenApply(ctx -> {
LogicalFilter<Plan> filter = ctx.root;
ImmutableSet.Builder newConjuncts = ImmutableSet.builder();
for (Expression expression : filter.getConjuncts()) {
- if (expression == BooleanLiteral.FALSE) {
+ if (expression == BooleanLiteral.FALSE ||
expression.isNullLiteral()) {
return new
LogicalEmptyRelation(ctx.statementContext.getNextRelationId(),
filter.getOutput());
} else if (expression != BooleanLiteral.TRUE) {
@@ -80,7 +82,7 @@ public class EliminateFilter implements RewriteRuleFactory {
Expression foldExpression =
FoldConstantRule.INSTANCE.rewrite(newExpr,
context);
- if (foldExpression == BooleanLiteral.FALSE) {
+ if (foldExpression == BooleanLiteral.FALSE ||
expression.isNullLiteral()) {
return new LogicalEmptyRelation(
ctx.statementContext.getNextRelationId(),
filter.getOutput());
} else if (foldExpression != BooleanLiteral.TRUE) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
index 9f35a0c0253..6b4b85d3f80 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.util.LogicalPlanBuilder;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
@@ -42,6 +43,17 @@ class EliminateFilterTest implements
MemoPatternMatchSupported {
.matches(logicalEmptyRelation());
}
+ @Test
+ void testEliminateFilterNull() {
+ LogicalPlan filterNull = new
LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0))
+ .filter(NullLiteral.INSTANCE)
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filterNull)
+ .applyTopDown(new EliminateFilter())
+ .matches(logicalEmptyRelation());
+ }
+
@Test
void testEliminateFilterTrue() {
LogicalPlan filterTrue = new
LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0))
diff --git a/regression-test/data/empty_relation/eliminate_empty.out
b/regression-test/data/empty_relation/eliminate_empty.out
index f01d0a947ee..acb17696ebf 100644
--- a/regression-test/data/empty_relation/eliminate_empty.out
+++ b/regression-test/data/empty_relation/eliminate_empty.out
@@ -70,6 +70,95 @@ PhysicalResultSink
-- !except_empty_data --
+-- !null_join --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_explain_union_empty_data --
+PhysicalResultSink
+--PhysicalDistribute
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[nation]
+
+-- !null_union_empty_data --
+1
+
+-- !null_explain_union_empty_empty --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_union_empty_empty --
+
+-- !null_union_emtpy_onerow --
+10
+
+-- !null_explain_intersect_data_empty --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_explain_intersect_empty_data --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_explain_except_data_empty --
+PhysicalResultSink
+--PhysicalDistribute
+----PhysicalProject
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[nation]
+
+-- !null_explain_except_data_empty_data --
+PhysicalResultSink
+--PhysicalDistribute
+----PhysicalExcept
+------PhysicalDistribute
+--------PhysicalProject
+----------PhysicalOlapScan[nation]
+------PhysicalDistribute
+--------PhysicalProject
+----------filter(( not (n_nationkey = 1)))
+------------PhysicalOlapScan[nation]
+
+-- !null_except_data_empty_data --
+1
+
+-- !null_explain_except_empty_data --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_intersect_data_empty --
+
+-- !null_intersect_empty_data --
+
+-- !null_except_data_empty --
+1
+
+-- !null_except_empty_data --
+
+-- !prune_partition1 --
+PhysicalResultSink
+--PhysicalDistribute
+----PhysicalProject
+------hashAgg[GLOBAL]
+--------PhysicalDistribute
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((eliminate_partition_prune.k1 = 100))
+----------------PhysicalOlapScan[eliminate_partition_prune]
+
+-- !prune_partition2 --
+PhysicalResultSink
+--PhysicalDistribute
+----PhysicalProject
+------hashAgg[GLOBAL]
+--------PhysicalDistribute
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((eliminate_partition_prune.k1 = 100))
+----------------PhysicalOlapScan[eliminate_partition_prune]
+
-- !join_with_empty_child --
2 8 e v 3
4 7 at 2
diff --git a/regression-test/suites/empty_relation/eliminate_empty.groovy
b/regression-test/suites/empty_relation/eliminate_empty.groovy
index dc602a466c7..8d5f1baea7b 100644
--- a/regression-test/suites/empty_relation/eliminate_empty.groovy
+++ b/regression-test/suites/empty_relation/eliminate_empty.groovy
@@ -128,6 +128,143 @@ suite("eliminate_empty") {
select r_regionkey from region where false except select n_nationkey
from nation
"""
+ qt_null_join """
+ explain shape plan
+ select *
+ from
+ nation
+ join
+ (select * from region where Null) R
+ """
+
+ qt_null_explain_union_empty_data """
+ explain shape plan
+ select *
+ from (select n_nationkey from nation union select r_regionkey from
region where Null) T
+ """
+ qt_null_union_empty_data """
+ select *
+ from (select n_nationkey from nation union select r_regionkey from
region where Null) T
+ """
+
+ qt_null_explain_union_empty_empty """
+ explain shape plan
+ select *
+ from (
+ select n_nationkey from nation where Null
+ union
+ select r_regionkey from region where Null
+ ) T
+ """
+ qt_null_union_empty_empty """
+ select *
+ from (
+ select n_nationkey from nation where Null
+ union
+ select r_regionkey from region where Null
+ ) T
+ """
+ qt_null_union_emtpy_onerow """
+ select *
+ from (
+ select n_nationkey from nation where Null
+ union
+ select 10
+ union
+ select 10
+ )T
+ """
+
+ qt_null_explain_intersect_data_empty """
+ explain shape plan
+ select n_nationkey from nation intersect select r_regionkey from
region where Null
+ """
+
+ qt_null_explain_intersect_empty_data """
+ explain shape plan
+ select r_regionkey from region where Null intersect select n_nationkey
from nation
+ """
+
+ qt_null_explain_except_data_empty """
+ explain shape plan
+ select n_nationkey from nation except select r_regionkey from region
where Null
+ """
+
+ qt_null_explain_except_data_empty_data """
+ explain shape plan
+ select n_nationkey from nation
+ except
+ select r_regionkey from region where Null
+ except
+ select n_nationkey from nation where n_nationkey != 1;
+ """
+
+ qt_null_except_data_empty_data """
+ select n_nationkey from nation
+ except
+ select r_regionkey from region where Null
+ except
+ select n_nationkey from nation where n_nationkey != 1;
+ """
+
+ qt_null_explain_except_empty_data """
+ explain shape plan
+ select r_regionkey from region where Null except select n_nationkey
from nation
+ """
+
+
+ qt_null_intersect_data_empty """
+ select n_nationkey from nation intersect select r_regionkey from
region where Null
+ """
+
+ qt_null_intersect_empty_data """
+ select r_regionkey from region where Null intersect select n_nationkey
from nation
+ """
+
+ qt_null_except_data_empty """
+ select n_nationkey from nation except select r_regionkey from region
where Null
+ """
+
+ qt_null_except_empty_data """
+ select r_regionkey from region where Null except select n_nationkey
from nation
+ """
+
+ sql """
+ drop table if exists eliminate_partition_prune;
+ """
+ sql """
+ CREATE TABLE `eliminate_partition_prune` (
+ `k1` int(11) NULL COMMENT "",
+ `k2` int(11) NULL COMMENT "",
+ `k3` int(11) NULL COMMENT ""
+ )
+ PARTITION BY RANGE(`k1`, `k2`)
+ (PARTITION p1 VALUES LESS THAN ("3", "1"),
+ PARTITION p2 VALUES [("3", "1"), ("7", "10")),
+ PARTITION p3 VALUES [("7", "10"), ("10", "15")))
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 10
+ PROPERTIES ('replication_num' = '1');
+ """
+
+
+ qt_prune_partition1 """
+ explain shape plan
+ select sum(k2)
+ from
+ (select * from eliminate_partition_prune where k1=100) T
+ group by k3;
+ """
+ sql """
+ insert into eliminate_partition_prune values (7, 0, 0)
+ """
+ qt_prune_partition2 """
+ explain shape plan
+ select sum(k2)
+ from
+ (select * from eliminate_partition_prune where k1=100) T
+ group by k3;
+ """
+
sql """drop table if exists table_5_undef_partitions2_keys3"""
sql """drop table if exists table_10_undef_partitions2_keys3"""
@@ -179,4 +316,4 @@ suite("eliminate_empty") {
sql """drop table if exists table_5_undef_partitions2_keys3"""
sql """drop table if exists table_10_undef_partitions2_keys3"""
}
-}
\ No newline at end of file
+}
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
index d82eaa19eb6..1c6a68eb85c 100644
---
a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
@@ -169,12 +169,12 @@ suite("test_multi_range_partition") {
explain {
sql "select * from pt where k1=7 and k2 in (null);"
- contains "partitions=0/3"
+ contains "VEMPTYSET"
}
explain {
sql "select * from pt where k1=7 and k2 not in (null);"
- contains "partitions=0/3"
+ contains "VEMPTYSET"
}
explain {
@@ -189,13 +189,13 @@ suite("test_multi_range_partition") {
explain {
sql "select * from pt where k2 in (null);"
- contains "partitions=0/3"
+ contains "VEMPTYSET"
}
// p1/p2/p3 NOT pruned
explain {
sql "select * from pt where k2 not in (null)"
- contains "partitions=0/3"
+ contains "VEMPTYSET"
}
explain {
diff --git
a/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy
b/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy
index 70dffc3a1dc..b0e98d37bb6 100644
--- a/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy
+++ b/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy
@@ -88,15 +88,6 @@ suite("test_simplify_comparison") {
contains "CAST"
}
- explain {
- sql "verbose select * from simple_test_table_t where a = cast(1.1 as
double) and b = cast(1.1 as double) and c = cast(1.1 as double) and d =
cast(1.1 as double);"
- contains "a[#0] IS NULL"
- contains "b[#1] IS NULL"
- contains "c[#2] IS NULL"
- contains "d[#3] IS NULL"
- contains "AND NULL"
- }
-
explain {
sql "verbose select * from simple_test_table_t where e = cast(1.1 as
double);"
contains "CAST(e[#4] AS double) = 1.1"
@@ -204,15 +195,6 @@ suite("test_simplify_comparison") {
contains "CAST"
}
- explain {
- sql "verbose select * from simple_test_table_t where a = 1.1 and b =
1.1 and c = 1.1 and d = 1.1;"
- contains "a[#0] IS NULL"
- contains "b[#1] IS NULL"
- contains "c[#2] IS NULL"
- contains "d[#3] IS NULL"
- contains "AND NULL"
- }
-
explain {
sql "verbose select * from simple_test_table_t where e = 1.1;"
contains "CAST(e[#4] AS double) = 1.1"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
