This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b6ccdd36e9c [opt](Nereids) add where Null rule to create empty
relation as where false (#38135)
b6ccdd36e9c is described below
commit b6ccdd36e9c190fc3513492154c6cdb5b01119f0
Author: LiBinfeng <[email protected]>
AuthorDate: Thu Jul 25 11:36:11 2024 +0800
[opt](Nereids) add where Null rule to create empty relation as where false
(#38135)
explain shape plan select * from table2 where Null;
explain shape plan select * from table2 where false;
in this case, null literal can be regard as same as false literal
---
.../nereids/rules/rewrite/EliminateFilter.java | 8 +-
.../nereids/rules/rewrite/EliminateFilterTest.java | 12 +++
.../data/empty_relation/eliminate_empty.out | 67 ++++++++++++++
.../infer_predicate/pull_up_predicate_literal.out | 20 +---
.../suites/empty_relation/eliminate_empty.groovy | 103 ++++++++++++++++++++-
.../test_simplify_comparison.groovy | 20 +---
6 files changed, 189 insertions(+), 41 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
index ef9e418f58d..1413faf3bb0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
@@ -24,6 +24,7 @@ import
org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -43,12 +44,13 @@ public class EliminateFilter implements RewriteRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(logicalFilter().when(
- filter -> ExpressionUtils.containsType(filter.getConjuncts(),
BooleanLiteral.class))
+ filter -> ExpressionUtils.containsType(filter.getConjuncts(),
BooleanLiteral.class)
+ || ExpressionUtils.containsType(filter.getConjuncts(),
NullLiteral.class))
.thenApply(ctx -> {
LogicalFilter<Plan> filter = ctx.root;
ImmutableSet.Builder<Expression> newConjuncts =
ImmutableSet.builder();
for (Expression expression : filter.getConjuncts()) {
- if (expression == BooleanLiteral.FALSE) {
+ if (expression == BooleanLiteral.FALSE ||
expression.isNullLiteral()) {
return new
LogicalEmptyRelation(ctx.statementContext.getNextRelationId(),
filter.getOutput());
} else if (expression != BooleanLiteral.TRUE) {
@@ -75,7 +77,7 @@ public class EliminateFilter implements RewriteRuleFactory {
Expression newExpr =
ExpressionUtils.replace(expression, replaceMap);
Expression foldExpression =
FoldConstantRule.evaluate(newExpr, context);
- if (foldExpression == BooleanLiteral.FALSE) {
+ if (foldExpression == BooleanLiteral.FALSE ||
expression.isNullLiteral()) {
return new LogicalEmptyRelation(
ctx.statementContext.getNextRelationId(),
filter.getOutput());
} else if (foldExpression != BooleanLiteral.TRUE) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
index cbdc95db559..d3d1316eaac 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java
@@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.util.LogicalPlanBuilder;
@@ -50,6 +51,17 @@ class EliminateFilterTest implements
MemoPatternMatchSupported {
.matches(logicalEmptyRelation());
}
+ @Test
+ void testEliminateFilterNull() {
+ LogicalPlan filterNull = new LogicalPlanBuilder(scan1)
+ .filter(NullLiteral.INSTANCE)
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), filterNull)
+ .applyTopDown(new EliminateFilter())
+ .matches(logicalEmptyRelation());
+ }
+
@Test
void testEliminateFilterTrue() {
LogicalPlan filterTrue = new LogicalPlanBuilder(scan1)
diff --git a/regression-test/data/empty_relation/eliminate_empty.out
b/regression-test/data/empty_relation/eliminate_empty.out
index ca0d2a4641a..56917b8042d 100644
--- a/regression-test/data/empty_relation/eliminate_empty.out
+++ b/regression-test/data/empty_relation/eliminate_empty.out
@@ -70,6 +70,73 @@ PhysicalResultSink
-- !except_empty_data --
+-- !null_join --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_explain_union_empty_data --
+PhysicalResultSink
+--PhysicalDistribute[DistributionSpecGather]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[nation]
+
+-- !null_union_empty_data --
+1
+
+-- !null_explain_union_empty_empty --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_union_empty_empty --
+
+-- !null_union_emtpy_onerow --
+10
+
+-- !null_explain_intersect_data_empty --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_explain_intersect_empty_data --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_explain_except_data_empty --
+PhysicalResultSink
+--PhysicalDistribute[DistributionSpecGather]
+----PhysicalProject
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[nation]
+
+-- !null_explain_except_data_empty_data --
+PhysicalResultSink
+--PhysicalDistribute[DistributionSpecGather]
+----PhysicalExcept
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalProject
+----------PhysicalOlapScan[nation]
+------PhysicalDistribute[DistributionSpecHash]
+--------PhysicalProject
+----------filter(( not (n_nationkey = 1)))
+------------PhysicalOlapScan[nation]
+
+-- !null_except_data_empty_data --
+1
+
+-- !null_explain_except_empty_data --
+PhysicalResultSink
+--PhysicalEmptyRelation
+
+-- !null_intersect_data_empty --
+
+-- !null_intersect_empty_data --
+
+-- !null_except_data_empty --
+1
+
+-- !null_except_empty_data --
+
-- !prune_partition1 --
PhysicalResultSink
--PhysicalEmptyRelation
diff --git
a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
index d7c54f36d46..bb0367626de 100644
---
a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
+++
b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
@@ -1345,27 +1345,11 @@ PhysicalResultSink
-- !const_value_and_join_column_type128 --
PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_tinyint as DECIMALV3(21,
16)) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter(NULL and d_tinyint IS NULL)
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
-- !const_value_and_join_column_type129 --
PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_smallint as DECIMALV3(21,
16)) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter(NULL and d_smallint IS NULL)
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
-- !const_value_and_join_column_type130 --
PhysicalResultSink
diff --git a/regression-test/suites/empty_relation/eliminate_empty.groovy
b/regression-test/suites/empty_relation/eliminate_empty.groovy
index 47246b633ca..afe0a1ce487 100644
--- a/regression-test/suites/empty_relation/eliminate_empty.groovy
+++ b/regression-test/suites/empty_relation/eliminate_empty.groovy
@@ -132,6 +132,107 @@ suite("eliminate_empty") {
select r_regionkey from region where false except select n_nationkey
from nation
"""
+ qt_null_join """
+ explain shape plan
+ select *
+ from
+ nation
+ join
+ (select * from region where Null) R
+ """
+
+ qt_null_explain_union_empty_data """
+ explain shape plan
+ select *
+ from (select n_nationkey from nation union select r_regionkey from
region where Null) T
+ """
+ qt_null_union_empty_data """
+ select *
+ from (select n_nationkey from nation union select r_regionkey from
region where Null) T
+ """
+
+ qt_null_explain_union_empty_empty """
+ explain shape plan
+ select *
+ from (
+ select n_nationkey from nation where Null
+ union
+ select r_regionkey from region where Null
+ ) T
+ """
+ qt_null_union_empty_empty """
+ select *
+ from (
+ select n_nationkey from nation where Null
+ union
+ select r_regionkey from region where Null
+ ) T
+ """
+ qt_null_union_emtpy_onerow """
+ select *
+ from (
+ select n_nationkey from nation where Null
+ union
+ select 10
+ union
+ select 10
+ )T
+ """
+
+ qt_null_explain_intersect_data_empty """
+ explain shape plan
+ select n_nationkey from nation intersect select r_regionkey from
region where Null
+ """
+
+ qt_null_explain_intersect_empty_data """
+ explain shape plan
+ select r_regionkey from region where Null intersect select n_nationkey
from nation
+ """
+
+ qt_null_explain_except_data_empty """
+ explain shape plan
+ select n_nationkey from nation except select r_regionkey from region
where Null
+ """
+
+ qt_null_explain_except_data_empty_data """
+ explain shape plan
+ select n_nationkey from nation
+ except
+ select r_regionkey from region where Null
+ except
+ select n_nationkey from nation where n_nationkey != 1;
+ """
+
+ qt_null_except_data_empty_data """
+ select n_nationkey from nation
+ except
+ select r_regionkey from region where Null
+ except
+ select n_nationkey from nation where n_nationkey != 1;
+ """
+
+ qt_null_explain_except_empty_data """
+ explain shape plan
+ select r_regionkey from region where Null except select n_nationkey
from nation
+ """
+
+
+ qt_null_intersect_data_empty """
+ select n_nationkey from nation intersect select r_regionkey from
region where Null
+ """
+
+ qt_null_intersect_empty_data """
+ select r_regionkey from region where Null intersect select n_nationkey
from nation
+ """
+
+ qt_null_except_data_empty """
+ select n_nationkey from nation except select r_regionkey from region
where Null
+ """
+
+ qt_null_except_empty_data """
+ select r_regionkey from region where Null except select n_nationkey
from nation
+ """
+
sql """
drop table if exists eliminate_partition_prune;
"""
@@ -219,4 +320,4 @@ suite("eliminate_empty") {
sql """drop table if exists table_5_undef_partitions2_keys3"""
sql """drop table if exists table_10_undef_partitions2_keys3"""
}
-}
\ No newline at end of file
+}
diff --git
a/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy
b/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy
index cb756c9173d..4e416f41bb2 100644
--- a/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy
+++ b/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy
@@ -89,15 +89,6 @@ suite("test_simplify_comparison") {
contains "CAST"
}
- explain {
- sql "verbose select * from simple_test_table_t where a = cast(1.1 as
double) and b = cast(1.1 as double) and c = cast(1.1 as double) and d =
cast(1.1 as double);"
- contains "a[#0] IS NULL"
- contains "b[#1] IS NULL"
- contains "c[#2] IS NULL"
- contains "d[#3] IS NULL"
- contains "AND NULL"
- }
-
explain {
sql "verbose select * from simple_test_table_t where e = cast(1.1 as
double);"
contains "CAST(e[#4] AS double) = 1.1"
@@ -205,15 +196,6 @@ suite("test_simplify_comparison") {
contains "CAST"
}
- explain {
- sql "verbose select * from simple_test_table_t where a = 1.1 and b =
1.1 and c = 1.1 and d = 1.1;"
- contains "a[#0] IS NULL"
- contains "b[#1] IS NULL"
- contains "c[#2] IS NULL"
- contains "d[#3] IS NULL"
- contains "AND NULL"
- }
-
explain {
sql "verbose select * from simple_test_table_t where e = 1.1;"
contains "CAST(e[#4] AS double) = 1.1"
@@ -272,4 +254,4 @@ suite("test_simplify_comparison") {
}
qt_select1 """select * from simple_test_table_t where cast(a as
decimal(5,1)) = 10.0;"""
qt_select2 """select a.col1, cast(a.col1 as decimal(7,2)) col3, case when
a.col1 is null then 15 when cast(a.col1 as decimal(7,2)) < -99997.99 then 18
when cast(a.col1 as decimal(7,2)) < 1.001 then 3 else -55 end col2 from (select
1 as col1) a;"""
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
