This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 48d41a8c8b7 [feature](Nereids): support comparing mv with inferred
predicate (#29132)
48d41a8c8b7 is described below
commit 48d41a8c8b76ce22a8a9235e92862b06a5171aa6
Author: 谢健 <[email protected]>
AuthorDate: Fri Dec 29 10:38:53 2023 +0800
[feature](Nereids): support comparing mv with inferred predicate (#29132)
---
.../rules/exploration/mv/ComparisonResult.java | 11 ++++
.../rules/exploration/mv/HyperGraphComparator.java | 16 +++--
.../joinorder/hypergraph/CompareOuterJoinTest.java | 2 -
.../joinorder/hypergraph/InferPredicateTest.java | 72 ++++++++++++++++++++++
.../joinorder/hypergraph/PullupExpressionTest.java | 3 +-
.../agg_with_roll_up/aggregate_with_roll_up.groovy | 2 +-
.../aggregate_without_roll_up.groovy | 2 +-
.../mv/join/inner/inner_join.groovy | 2 +-
.../mv/join/left_outer/outer_join.groovy | 2 +-
.../mv/partition_mv_rewrite.groovy | 2 +-
10 files changed, 101 insertions(+), 13 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
index 8836745465e..1eb49cbfc0e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
@@ -116,4 +116,15 @@ public class ComparisonResult {
viewNoNullableSlotBuilder.build(), valid);
}
}
+
+ @Override
+ public String toString() {
+ if (isInvalid()) {
+ return "INVALID";
+ }
+ return String.format("viewExpressions: %s \n "
+ + "queryExpressions :%s \n "
+ + "viewNoNullableSlot :%s \n",
+ viewExpressions, queryExpressions, viewNoNullableSlot);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
index a869fe729a9..04efecc9c25 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
@@ -27,6 +27,7 @@ import
org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -41,6 +42,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* HyperGraphComparator
@@ -108,16 +110,22 @@ public class HyperGraphComparator {
private ComparisonResult buildComparisonRes() {
ComparisonResult.Builder builder = new ComparisonResult.Builder();
for (Entry<Edge, List<? extends Expression>> e :
pullUpQueryExprWithEdge.entrySet()) {
- if (!e.getValue().isEmpty() && !canPullUp(e.getKey())) {
+ List<? extends Expression> rawFilter = e.getValue().stream()
+ .filter(expr -> !ExpressionUtils.isInferred(expr))
+ .collect(Collectors.toList());
+ if (!rawFilter.isEmpty() && !canPullUp(e.getKey())) {
return ComparisonResult.INVALID;
}
- builder.addQueryExpressions(e.getValue());
+ builder.addQueryExpressions(rawFilter);
}
for (Entry<Edge, List<? extends Expression>> e :
pullUpViewExprWithEdge.entrySet()) {
- if (!e.getValue().isEmpty() && !canPullUp(e.getKey())) {
+ List<? extends Expression> rawFilter = e.getValue().stream()
+ .filter(expr -> !ExpressionUtils.isInferred(expr))
+ .collect(Collectors.toList());
+ if (!rawFilter.isEmpty() && !canPullUp(e.getKey())) {
return ComparisonResult.INVALID;
}
- builder.addViewExpressions(e.getValue());
+ builder.addViewExpressions(rawFilter);
}
for (Pair<JoinType, Set<Slot>> inferredCond :
inferredViewEdgeMap.values()) {
builder.addViewNoNullableSlot(inferredCond.second);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
index 105a5d450c3..8914f4ac80b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
@@ -33,7 +33,6 @@ import org.apache.doris.nereids.util.HyperGraphBuilder;
import org.apache.doris.nereids.util.PlanChecker;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.util.List;
@@ -113,7 +112,6 @@ class CompareOuterJoinTest extends SqlTestBase {
Assertions.assertEquals("(id = 0)",
res.getQueryExpressions().get(0).toSql());
}
- @Disabled
@Test
void testInnerJoinWithFilter2() {
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
new file mode 100644
index 00000000000..bd18f31a7c0
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.jobs.joinorder.hypergraph;
+
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.rules.RuleSet;
+import
org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule;
+import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
+import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator;
+import
org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
+import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.util.PlanChecker;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class InferPredicateTest extends SqlTestBase {
+ @Test
+ void testPullUpQueryFilter() {
+ CascadesContext c1 = createCascadesContext(
+ "select * from T1 left join T2 on T1.id = T2.id where T1.id =
1",
+ connectContext
+ );
+ Plan p1 = PlanChecker.from(c1)
+ .analyze()
+ .rewrite()
+ .getPlan().child(0);
+ CascadesContext c2 = createCascadesContext(
+ "select * from T1 left join T2 on T1.id = T2.id",
+ connectContext
+ );
+ Plan p2 = PlanChecker.from(c2)
+ .analyze()
+ .rewrite()
+ .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+ .getAllPlan().get(0).child(0);
+ HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+ HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ Assertions.assertFalse(res.isInvalid());
+ Assertions.assertEquals("(id = 1)",
res.getQueryExpressions().get(0).toSql());
+ }
+
+ LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
+ StructInfo st1 = AbstractMaterializedViewRule.extractStructInfo(p1,
+ null).get(0);
+ StructInfo st2 = AbstractMaterializedViewRule.extractStructInfo(p2,
+ null).get(0);
+ RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
+ SlotMapping sm = SlotMapping.generate(rm);
+ return LogicalCompatibilityContext.from(rm, sm, st1, st2);
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
index 4da4905f8aa..5489f06379c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
@@ -56,9 +56,8 @@ class PullupExpressionTest extends SqlTestBase {
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
- Assertions.assertEquals(2, res.getQueryExpressions().size());
+ Assertions.assertEquals(1, res.getQueryExpressions().size());
Assertions.assertEquals("(id = 1)",
res.getQueryExpressions().get(0).toSql());
- Assertions.assertEquals("(id = 1)",
res.getQueryExpressions().get(1).toSql());
}
@Test
diff --git
a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
index d6b5c37e59c..fd3c02408d9 100644
---
a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
@@ -23,7 +23,7 @@ suite("aggregate_with_roll_up") {
sql "SET enable_materialized_view_rewrite=true"
sql "SET enable_nereids_timeout = false"
// tmp disable to rewrite, will be removed in the future
- sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+ sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
sql """
drop table if exists orders
diff --git
a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
index 5d40aa03138..65da58cd5bd 100644
---
a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
@@ -23,7 +23,7 @@ suite("aggregate_without_roll_up") {
sql "SET enable_materialized_view_rewrite=true"
sql "SET enable_nereids_timeout = false"
// tmp disable to rewrite, will be removed in the future
- sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+ sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
sql "SET global enable_auto_analyze = false"
sql """
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
index ccf6a83f2c6..80ca73df03b 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
@@ -23,7 +23,7 @@ suite("inner_join") {
sql "SET enable_materialized_view_rewrite=true"
sql "SET enable_nereids_timeout = false"
// tmp disable to rewrite, will be removed in the future
- sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+ sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
sql """
drop table if exists orders
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
index e8b1da2572a..37cf716c942 100644
---
a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
@@ -23,7 +23,7 @@ suite("outer_join") {
sql "SET enable_materialized_view_rewrite=true"
sql "SET enable_nereids_timeout = false"
// tmp disable to rewrite, will be removed in the future
- sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+ sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
sql """
drop table if exists orders
diff --git
a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
index e0144fe6369..6da7ad89b77 100644
--- a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
@@ -23,7 +23,7 @@ suite("partition_mv_rewrite") {
sql "SET enable_materialized_view_rewrite=true"
sql "SET enable_nereids_timeout = false"
// tmp disable to rewrite, will be removed in the future
- sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+ sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
sql """
drop table if exists orders
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
