This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 245efbe4786 extract agg in struct info node (#29853)
245efbe4786 is described below
commit 245efbe4786f0f4409d0c4398ffc533b675e898c
Author: 谢健 <[email protected]>
AuthorDate: Mon Jan 15 13:32:21 2024 +0800
extract agg in struct info node (#29853)
---
.../joinorder/hypergraph/node/StructInfoNode.java | 62 ++++++++++++++
.../rules/exploration/mv/HyperGraphComparator.java | 35 +++++++-
.../rules/exploration/mv/HyperGraphAggTest.java | 98 ++++++++++++++++++++++
3 files changed, 191 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java
index 424ca6a5f10..9e3886bfdca 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java
@@ -19,14 +19,26 @@ package
org.apache.doris.nereids.jobs.joinorder.hypergraph.node;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.GroupPlan;
+import org.apache.doris.nereids.trees.plans.LeafPlan;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
/**
* HyperGraph Node.
@@ -34,9 +46,13 @@ import java.util.List;
public class StructInfoNode extends AbstractNode {
private List<HyperGraph> graphs = new ArrayList<>();
+ private final List<Set<Expression>> expressions;
+ private final Set<CatalogRelation> relationSet;
public StructInfoNode(int index, Plan plan, List<Edge> edges) {
super(extractPlan(plan), index, edges);
+ relationSet = plan.collect(CatalogRelation.class::isInstance);
+ expressions = collectExpressions(plan);
}
public StructInfoNode(int index, Plan plan) {
@@ -48,6 +64,52 @@ public class StructInfoNode extends AbstractNode {
this.graphs = graphs;
}
+ private @Nullable List<Set<Expression>> collectExpressions(Plan plan) {
+ if (plan instanceof LeafPlan) {
+ return ImmutableList.of();
+ }
+ List<Set<Expression>> childExpressions =
collectExpressions(plan.child(0));
+ if (!isValidNodePlan(plan) || childExpressions == null) {
+ return null;
+ }
+ if (plan instanceof LogicalAggregate) {
+ return ImmutableList.<Set<Expression>>builder()
+ .add(ImmutableSet.copyOf(plan.getExpressions()))
+ .add(ImmutableSet.copyOf(((LogicalAggregate<?>)
plan).getGroupByExpressions()))
+ .addAll(childExpressions)
+ .build();
+ }
+ return ImmutableList.<Set<Expression>>builder()
+ .add(ImmutableSet.copyOf(plan.getExpressions()))
+ .addAll(childExpressions)
+ .build();
+ }
+
+ private boolean isValidNodePlan(Plan plan) {
+ return plan instanceof LogicalProject || plan instanceof
LogicalAggregate
+ || plan instanceof LogicalFilter || plan instanceof
LogicalCatalogRelation;
+ }
+
+ /**
+ * get all expressions of nodes
+ */
+ public @Nullable List<Expression> getExpressions() {
+ return expressions.stream()
+ .flatMap(Collection::stream)
+ .collect(Collectors.toList());
+ }
+
+ public @Nullable List<Set<Expression>> getExprSetList() {
+ return expressions;
+ }
+
+ /**
+ * return catalog relation
+ */
+ public Set<CatalogRelation> getCatalogRelation() {
+ return relationSet;
+ }
+
private static Plan extractPlan(Plan plan) {
if (plan instanceof GroupPlan) {
//TODO: Note mv can be in logicalExpression, how can we choose it
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
index 7817475c7b8..bf48926348f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
@@ -24,6 +24,7 @@ import
org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -90,14 +91,21 @@ public class HyperGraphComparator {
}
private ComparisonResult isLogicCompatible() {
- // 1 try to construct a map which can be mapped from edge to edge
+ // 1 compare nodes
+ boolean nodeMatches =
logicalCompatibilityContext.getQueryToViewNodeMapping().entrySet()
+ .stream().allMatch(e -> compareNodeWithExpr(e.getKey(),
e.getValue()));
+ if (!nodeMatches) {
+ return
ComparisonResult.newInvalidResWithErrorMessage("StructInfoNode are not
compatible\n");
+ }
+
+ // 2 try to construct a map which can be mapped from edge to edge
Map<Edge, Edge> queryToView = constructQueryToViewMapWithExpr();
if (!makeViewJoinCompatible(queryToView)) {
return ComparisonResult.newInvalidResWithErrorMessage("Join types
are not compatible\n");
}
refreshViewEdges();
- // 2. compare them by expression and nodes. Note compare edges after
inferring for nodes
+ // 3. compare them by expression and nodes. Note compare edges after
inferring for nodes
boolean matchNodes = queryToView.entrySet().stream()
.allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue()));
if (!matchNodes) {
@@ -105,7 +113,7 @@ public class HyperGraphComparator {
}
queryToView.forEach(this::compareEdgeWithExpr);
- // 3. process residual edges
+ // 1. process residual edges
Sets.difference(getQueryJoinEdgeSet(), queryToView.keySet())
.forEach(e -> pullUpQueryExprWithEdge.put(e,
e.getExpressions()));
Sets.difference(getQueryFilterEdgeSet(), queryToView.keySet())
@@ -118,6 +126,25 @@ public class HyperGraphComparator {
return buildComparisonRes();
}
+ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode
view) {
+ List<Set<Expression>> queryExprSetList = query.getExprSetList();
+ List<Set<Expression>> viewExprSetList = view.getExprSetList();
+ if (queryExprSetList == null || viewExprSetList == null
+ || queryExprSetList.size() != viewExprSetList.size()) {
+ return false;
+ }
+ int size = queryExprSetList.size();
+ for (int i = 0; i < size; i++) {
+ Set<Expression> mappingQueryExprSet =
queryExprSetList.get(i).stream()
+ .map(e ->
logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping().get(e))
+ .collect(Collectors.toSet());
+ if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
private ComparisonResult buildComparisonRes() {
ComparisonResult.Builder builder = new ComparisonResult.Builder();
for (Entry<Edge, List<? extends Expression>> e :
pullUpQueryExprWithEdge.entrySet()) {
@@ -134,7 +161,7 @@ public class HyperGraphComparator {
.filter(expr -> !ExpressionUtils.isInferred(expr))
.collect(Collectors.toList());
if (!rawFilter.isEmpty() &&
!canPullUp(getViewEdgeAfterInferring(e.getKey()))) {
- return
ComparisonResult.newInvalidResWithErrorMessage(getErrorMessage() + "with error
edge\n" + e);
+ return
ComparisonResult.newInvalidResWithErrorMessage(getErrorMessage() + "\nwith
error edge\n" + e);
}
builder.addViewExpressions(rawFilter);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
new file mode 100644
index 00000000000..29d3e0edc36
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
+import org.apache.doris.nereids.rules.RuleSet;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.util.PlanChecker;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.Objects;
+
+class HyperGraphAggTest extends SqlTestBase {
+ @Test
+ void testJoinWithAgg() {
+ CascadesContext c2 = createCascadesContext(
+ "select * from T1 inner join"
+ + "("
+ + "select id from T2 group by id"
+ + ") T2 "
+ + "on T1.id = T2.id ",
+ connectContext
+ );
+ Plan p2 = PlanChecker.from(c2)
+ .analyze()
+ .rewrite()
+ .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+ .getAllPlan().get(0).child(0);
+ HyperGraph h1 = HyperGraph.toStructInfo(p2).get(0);
+ Assertions.assertEquals("id", Objects.requireNonNull(((StructInfoNode)
h1.getNode(1)).getExpressions()).get(0).toSql());
+ }
+
+ @Disabled
+ @Test
+ void testIJWithAgg() {
+
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+ CascadesContext c1 = createCascadesContext(
+ "select * from T1 inner join T2 "
+ + "on T1.id = T2.id",
+ connectContext
+ );
+ Plan p1 = PlanChecker.from(c1)
+ .analyze()
+ .rewrite()
+ .getPlan().child(0);
+ CascadesContext c2 = createCascadesContext(
+ "select * from T1 inner join"
+ + "("
+ + "select id from T2 group by id"
+ + ") T2 "
+ + "on T1.id = T2.id ",
+ connectContext
+ );
+ Plan p2 = PlanChecker.from(c2)
+ .analyze()
+ .rewrite()
+ .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+ .getAllPlan().get(0).child(0);
+ HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+ HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ Assertions.assertTrue(!res.isInvalid());
+ Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
+ }
+
+ LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
+ StructInfo st1 = AbstractMaterializedViewRule.extractStructInfo(p1,
+ null).get(0);
+ StructInfo st2 = AbstractMaterializedViewRule.extractStructInfo(p2,
+ null).get(0);
+ RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
+ SlotMapping sm = SlotMapping.generate(rm);
+ return LogicalCompatibilityContext.from(rm, sm, st1, st2);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
