This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 9488297efce branch-4.1: [fix](fe) Backport null-reject MV rewrite
fixes (#62492, #63268) (#63593)
9488297efce is described below
commit 9488297efce7a58bf5b29df54c4c37bdf6de5bf8
Author: seawinde <[email protected]>
AuthorDate: Thu Jun 4 15:38:46 2026 +0800
branch-4.1: [fix](fe) Backport null-reject MV rewrite fixes (#62492,
#63268) (#63593)
pr: #62492, #63268
commitId: 673b028a, 76bbe581
Backport #62492 and its correctness follow-up #63268 together to
branch-4.1.
#62492 infers null-reject evidence from INNER JoinEdge for multi-hop
outer join MV rewrite.
#63268 adds the required real IS NOT NULL compensation when INNER
JoinEdge proof is needed for OUTER JOIN MV rewrite, preventing extra
null-padded rows.
Tests:
- git diff --check upstream/branch-4.1..HEAD
- Tried ./run-fe-ut.sh --run
org.apache.doris.nereids.rules.exploration.mv.NullRejectInferenceTest
twice. The first run failed during fe-core compilation because generated
thrift classes such as TIcebergPartitionField/TRecCTENode/TMCCommitData
were not available from fe-common classes. After generated thrift
sources were refreshed, the second run failed before tests during
fe-core generate-patterns with NoClassDefFoundError:
org/apache/doris/nereids/pattern/generator/ExpressionTypeMappingGenerator.
No test case was executed locally.
---------
Co-authored-by: Copilot <[email protected]>
Co-authored-by: seawinde <[email protected]>
---
.../mv/AbstractMaterializedViewRule.java | 184 +++++++++++++----
.../exploration/mv/NullRejectInferenceTest.java | 209 ++++++++++++++++++++
.../mv/dimension/dimension_1.groovy | 2 +-
.../mv/dimension/dimension_2_left_join.groovy | 8 +-
.../mv/dimension/dimension_2_right_join.groovy | 8 +-
.../mv/dimension/dimension_self_conn.groovy | 5 +-
.../mv/dimension_predicate/left_join_filter.groovy | 4 +-
.../dimension_predicate/right_join_filter.groovy | 4 +-
.../outer_join_two_hop_null_reject.groovy | 138 +++++++++++++
.../join_elim_line_pattern.groovy | 4 +-
.../join_elim_star_pattern.groovy | 16 +-
.../inner_join_infer_and_derive.groovy | 24 +--
.../inner_join_null_reject_compensation.groovy | 217 +++++++++++++++++++++
.../left_join_infer_and_derive.groovy | 6 +-
.../right_join_infer_and_derive.groovy | 6 +-
15 files changed, 750 insertions(+), 85 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 9f76c242dcf..9429ac30645 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.MTMV;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Id;
import org.apache.doris.common.Pair;
@@ -29,6 +30,7 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.jobs.executor.Rewriter;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory;
@@ -43,6 +45,7 @@ import
org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
import org.apache.doris.nereids.rules.rewrite.MergeProjectable;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -821,21 +824,28 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
Set<Set<Slot>> requireNoNullableViewSlot =
comparisonResult.getViewNoNullableSlot();
// check query is use the null reject slot which view comparison need
if (!requireNoNullableViewSlot.isEmpty()) {
+ // Required null-reject slots are recorded on the view side. Map
query slots to view slots
+ // before checking whether query predicates or INNER JoinEdges can
reject those null rows.
SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse();
- // try to use
- boolean valid = containsNullRejectSlot(requireNoNullableViewSlot,
- queryStructInfo.getPredicates().getPulledUpPredicates(),
queryToViewMapping, queryStructInfo,
- viewStructInfo, cascadesContext);
- if (!valid) {
+ Optional<Set<Expression>>
queryBasedNullRejectCompensationPredicates =
+ getQueryBasedNullRejectCompensationPredicates(
+ requireNoNullableViewSlot,
+
queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping,
+ queryStructInfo, viewStructInfo,
viewToQuerySlotMapping, cascadesContext);
+ if (!queryBasedNullRejectCompensationPredicates.isPresent()) {
queryStructInfo =
queryStructInfo.withPredicates(queryStructInfo.getPredicates()
.mergePulledUpPredicates(comparisonResult.getQueryAllPulledUpExpressions()));
- valid = containsNullRejectSlot(requireNoNullableViewSlot,
-
queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping,
- queryStructInfo, viewStructInfo, cascadesContext);
+ queryBasedNullRejectCompensationPredicates =
getQueryBasedNullRejectCompensationPredicates(
+ requireNoNullableViewSlot,
queryStructInfo.getPredicates().getPulledUpPredicates(),
+ queryToViewMapping, queryStructInfo, viewStructInfo,
viewToQuerySlotMapping, cascadesContext);
}
- if (!valid) {
+ if (!queryBasedNullRejectCompensationPredicates.isPresent()) {
return SplitPredicate.INVALID_INSTANCE;
}
+ if (!queryBasedNullRejectCompensationPredicates.get().isEmpty()) {
+ queryStructInfo =
queryStructInfo.withPredicates(queryStructInfo.getPredicates()
+
.mergePulledUpPredicates(queryBasedNullRejectCompensationPredicates.get()));
+ }
}
// compensate couldNot PulledUp Conjunctions
Map<Expression, ExpressionInfo> couldNotPulledUpCompensateConjunctions
=
@@ -863,46 +873,106 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
}
/**
- * Check the queryPredicates contains the required nullable slot
+ * Check whether query-side null-reject evidence covers each required
view-side slot set.
+ *
+ * <p>The check is view-based because the required null-reject slots come
from the MV join graph.
+ * The returned compensation predicates are query-based because they will
be merged into queryStructInfo.
+ *
+ * <p>Return meanings:
+ * Optional.empty(): no valid proof, or no safe output slot can carry the
compensation predicate.
+ * Optional.of(emptySet()): existing query predicates already provide the
required null-reject.
+ * Optional.of(nonEmptySet): INNER JoinEdge proof must be materialized as
these IS NOT NULL predicates.
*/
- private boolean containsNullRejectSlot(Set<Set<Slot>>
requireNoNullableViewSlot,
+ private Optional<Set<Expression>>
getQueryBasedNullRejectCompensationPredicates(
+ Set<Set<Slot>> requireNoNullableViewSlot,
Set<Expression> queryPredicates,
SlotMapping queryToViewMapping,
StructInfo queryStructInfo,
StructInfo viewStructInfo,
+ SlotMapping viewToQueryMapping,
CascadesContext cascadesContext) {
- Set<Expression> queryPulledUpPredicates = queryPredicates.stream()
- .flatMap(expr ->
ExpressionUtils.extractConjunction(expr).stream())
- .map(expr -> {
- // NOTICE inferNotNull generate Not with
isGeneratedIsNotNull = false,
- // so, we need set this flag to false before comparison.
- if (expr instanceof Not) {
- return ((Not) expr).withGeneratedIsNotNull(false);
- }
- return expr;
- })
+ Set<Slot> predicateNullRejectViewSlots = getViewBasedNullRejectSlots(
+ getPredicateNullRejectSlots(queryPredicates, cascadesContext),
queryToViewMapping, queryStructInfo);
+ Set<Slot> innerJoinNullRejectViewSlots = getViewBasedNullRejectSlots(
+ getInnerJoinNullRejectSlots(queryStructInfo, cascadesContext),
queryToViewMapping, queryStructInfo);
+ Set<Slot> allNullRejectViewSlots = new
HashSet<>(predicateNullRejectViewSlots);
+ allNullRejectViewSlots.addAll(innerJoinNullRejectViewSlots);
+ if (allNullRejectViewSlots.isEmpty()) {
+ return Optional.empty();
+ }
+ Set<Slot> viewOutputSlots =
viewStructInfo.getPlanOutputShuttledExpressions().stream()
+ .filter(Slot.class::isInstance)
+ .map(Slot.class::cast)
.collect(Collectors.toSet());
- Set<Expression> queryNullRejectPredicates =
- ExpressionUtils.inferNotNull(queryPulledUpPredicates,
cascadesContext);
- if (queryPulledUpPredicates.containsAll(queryNullRejectPredicates)) {
- // Query has no null reject predicates, return
- return false;
+ Map<SlotReference, SlotReference> viewToQuerySlotReferenceMap =
viewToQueryMapping.toSlotReferenceMap();
+ Set<Expression> compensationPredicates = new HashSet<>();
+ for (Set<Slot> requiredViewSlots :
getShuttledRequireNoNullableViewSlots(
+ requireNoNullableViewSlot, viewStructInfo)) {
+ if (Sets.intersection(requiredViewSlots,
allNullRejectViewSlots).isEmpty()) {
+ return Optional.empty();
+ }
+ if (!Sets.intersection(requiredViewSlots,
predicateNullRejectViewSlots).isEmpty()) {
+ continue;
+ }
+ Optional<Slot> compensationViewSlot = findCompensationViewSlot(
+ requiredViewSlots, viewOutputSlots,
innerJoinNullRejectViewSlots);
+ if (!compensationViewSlot.isPresent()) {
+ return Optional.empty();
+ }
+ Slot querySlot =
viewToQuerySlotReferenceMap.get(compensationViewSlot.get());
+ if (querySlot == null) {
+ return Optional.empty();
+ }
+ compensationPredicates.add(new Not(new IsNull(querySlot), false));
+ }
+ return Optional.of(compensationPredicates);
+ }
+
+ private Set<Slot> getPredicateNullRejectSlots(Set<Expression>
queryPredicates, CascadesContext cascadesContext) {
+ Set<Slot> nullRejectSlots = new HashSet<>();
+ for (Expression queryPredicate : queryPredicates) {
+
TypeUtils.isNotNull(queryPredicate).ifPresent(nullRejectSlots::add);
+ }
+ for (Expression inferredNotNull :
ExpressionUtils.inferNotNull(queryPredicates, cascadesContext)) {
+
TypeUtils.isNotNull(inferredNotNull).ifPresent(nullRejectSlots::add);
}
- // Get query null reject predicate slots
- Set<Expression> queryNullRejectSlotSet = new HashSet<>();
- for (Expression queryNullRejectPredicate : queryNullRejectPredicates) {
- Optional<Slot> notNullSlot =
TypeUtils.isNotNull(queryNullRejectPredicate);
- if (!notNullSlot.isPresent()) {
+ return nullRejectSlots;
+ }
+
+ private Set<Slot> getInnerJoinNullRejectSlots(StructInfo queryStructInfo,
CascadesContext cascadesContext) {
+ Set<Slot> nullRejectSlots = new HashSet<>();
+ // INNER JOIN conditions guarantee NOT NULL on join-key slots.
+ // After EliminateOuterJoin converts LEFT to INNER, the JoinEdge
objects in the HyperGraph
+ // retain the INNER type even though EliminateNotNull removes
filter-level NOT NULL predicates.
+ for (JoinEdge joinEdge :
queryStructInfo.getHyperGraph().getJoinEdges()) {
+ if (joinEdge.getJoinType().isInnerJoin()) {
+ nullRejectSlots.addAll(ExpressionUtils.inferNotNullSlots(
+ ImmutableSet.copyOf(joinEdge.getExpressions()),
cascadesContext));
+ }
+ }
+ return nullRejectSlots;
+ }
+
+ private Set<Slot> getViewBasedNullRejectSlots(Set<Slot>
queryNullRejectSlots,
+ SlotMapping queryToViewMapping, StructInfo queryStructInfo) {
+ Set<Slot> viewBasedSlots = new HashSet<>();
+ for (Slot queryNullRejectSlot : queryNullRejectSlots) {
+ Expression shuttledQuerySlot =
ExpressionUtils.shuttleExpressionWithLineage(
+ queryNullRejectSlot, queryStructInfo.getTopPlan());
+ if (!(shuttledQuerySlot instanceof Slot)) {
continue;
}
- queryNullRejectSlotSet.add(notNullSlot.get());
+ Expression viewSlot = ExpressionUtils.replace(shuttledQuerySlot,
+ queryToViewMapping.toSlotReferenceMap());
+ if (viewSlot instanceof Slot) {
+ viewBasedSlots.add((Slot) viewSlot);
+ }
}
- // query slot need shuttle to use table slot, avoid alias influence
- Set<Expression> queryUsedNeedRejectNullSlotsViewBased =
ExpressionUtils.shuttleExpressionWithLineage(
- new ArrayList<>(queryNullRejectSlotSet),
queryStructInfo.getTopPlan()).stream()
- .map(expr -> ExpressionUtils.replace(expr,
queryToViewMapping.toSlotReferenceMap()))
- .collect(Collectors.toSet());
- // view slot need shuttle to use table slot, avoid alias influence
+ return viewBasedSlots;
+ }
+
+ private Set<Set<Slot>>
getShuttledRequireNoNullableViewSlots(Set<Set<Slot>> requireNoNullableViewSlot,
+ StructInfo viewStructInfo) {
Set<Set<Slot>> shuttledRequireNoNullableViewSlot = new HashSet<>();
for (Set<Slot> requireNullableSlots : requireNoNullableViewSlot) {
shuttledRequireNoNullableViewSlot.add(
@@ -910,9 +980,41 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
viewStructInfo.getTopPlan()).stream().map(Slot.class::cast)
.collect(Collectors.toSet()));
}
- // query pulledUp predicates should have null reject predicates and
contains any require noNullable slot
- return
shuttledRequireNoNullableViewSlot.stream().noneMatch(viewRequiredNullSlotSet ->
- Sets.intersection(viewRequiredNullSlotSet,
queryUsedNeedRejectNullSlotsViewBased).isEmpty());
+ return shuttledRequireNoNullableViewSlot;
+ }
+
+ private Optional<Slot> findCompensationViewSlot(Set<Slot>
requiredViewSlots, Set<Slot> viewOutputSlots,
+ Set<Slot> innerJoinNullRejectViewSlots) {
+ Set<Slot> outputRequiredSlots = Sets.intersection(requiredViewSlots,
viewOutputSlots);
+ Optional<Slot> compensationViewSlot = outputRequiredSlots.stream()
+ .filter(innerJoinNullRejectViewSlots::contains)
+ .findFirst();
+ if (compensationViewSlot.isPresent()) {
+ return compensationViewSlot;
+ }
+ return outputRequiredSlots.stream()
+ .filter(slot ->
isOriginalNonNullableSlotOnInnerJoinProofTable(slot,
innerJoinNullRejectViewSlots))
+ .findFirst();
+ }
+
+ private boolean isOriginalNonNullableSlotOnInnerJoinProofTable(Slot slot,
Set<Slot> innerJoinNullRejectViewSlots) {
+ if (!(slot instanceof SlotReference)) {
+ return false;
+ }
+ SlotReference slotReference = (SlotReference) slot;
+ if (!slotReference.getOriginalColumn().map(column ->
!column.isAllowNull()).orElse(!slot.nullable())) {
+ return false;
+ }
+ Optional<TableIf> originalTable = slotReference.getOriginalTable();
+ if (!originalTable.isPresent()) {
+ return false;
+ }
+ return innerJoinNullRejectViewSlots.stream()
+ .filter(SlotReference.class::isInstance)
+ .map(SlotReference.class::cast)
+ .map(SlotReference::getOriginalTable)
+ .anyMatch(referenceTable -> referenceTable.isPresent()
+ && referenceTable.get().equals(originalTable.get()));
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java
new file mode 100644
index 00000000000..5fd7628096b
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/NullRejectInferenceTest.java
@@ -0,0 +1,209 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleSet;
+import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.util.PlanChecker;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+class NullRejectInferenceTest extends SqlTestBase {
+
+ private static final TestMaterializedViewRule TEST_RULE = new
TestMaterializedViewRule();
+
+ @Test
+ void testTwoHopNullRejectFromInnerJoinConditions() {
+
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION");
+ CascadesContext queryContext = createCascadesContext(
+ "select lineitem.l_orderkey, supplier.s_name, nation.n_name
from lineitem "
+ + "inner join supplier on lineitem.l_suppkey =
supplier.s_suppkey "
+ + "inner join nation on supplier.s_nationkey =
nation.n_nationkey "
+ + "where nation.n_name = 'CHINA'",
+ connectContext
+ );
+ Plan queryPlan = PlanChecker.from(queryContext)
+ .analyze()
+ .rewrite()
+ .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+ .getAllPlan().get(0).child(0);
+
+ CascadesContext viewContext = createCascadesContext(
+ "select lineitem.l_orderkey, supplier.s_name, nation.n_name
from lineitem "
+ + "left outer join supplier on lineitem.l_suppkey =
supplier.s_suppkey "
+ + "left outer join nation on supplier.s_nationkey =
nation.n_nationkey",
+ connectContext
+ );
+ Plan viewPlan = PlanChecker.from(viewContext)
+ .analyze()
+ .rewrite()
+ .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+ .getAllPlan().get(0).child(0);
+
+ StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan,
queryContext);
+ StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan,
viewContext);
+ RelationMapping relationMapping = RelationMapping.generate(
+ queryStructInfo.getRelations(), viewStructInfo.getRelations(),
8).get(0);
+ SlotMapping queryToView = SlotMapping.generate(relationMapping);
+ SlotMapping viewToQuery = queryToView.inverse();
+ LogicalCompatibilityContext compatibilityContext =
LogicalCompatibilityContext.from(
+ relationMapping, viewToQuery, queryStructInfo, viewStructInfo);
+ ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(
+ queryStructInfo, viewStructInfo, compatibilityContext);
+
+ Assertions.assertFalse(comparisonResult.isInvalid());
+
Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty());
+
+ SplitPredicate compensatePredicates =
TEST_RULE.predicatesCompensateForTest(
+ queryStructInfo, viewStructInfo, viewToQuery,
comparisonResult, queryContext);
+ Assertions.assertFalse(compensatePredicates.isInvalid());
+ Assertions.assertTrue(compensatePredicates.toList().stream()
+ .anyMatch(expression -> isNotNullOnSlot(expression,
"s_name")));
+ }
+
+ @Test
+ void testNullRejectCompensationForInnerJoinFullJoinRewrite() {
+
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION");
+ CascadesContext queryContext = createCascadesContext(
+ "select lineitem.l_shipdate, orders.o_orderdate from lineitem "
+ + "inner join orders on lineitem.l_orderkey =
orders.o_orderkey "
+ + "where orders.o_orderdate = '2023-10-17'",
+ connectContext
+ );
+ Plan queryPlan = PlanChecker.from(queryContext)
+ .analyze()
+ .rewrite()
+ .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+ .getAllPlan().get(0).child(0);
+
+ CascadesContext viewContext = createCascadesContext(
+ "select lineitem.l_shipdate, orders.o_orderdate from lineitem "
+ + "full outer join orders on lineitem.l_orderkey =
orders.o_orderkey",
+ connectContext
+ );
+ Plan viewPlan = PlanChecker.from(viewContext)
+ .analyze()
+ .rewrite()
+ .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+ .getAllPlan().get(0).child(0);
+
+ StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan,
queryContext);
+ StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan,
viewContext);
+ RelationMapping relationMapping = RelationMapping.generate(
+ queryStructInfo.getRelations(), viewStructInfo.getRelations(),
8).get(0);
+ SlotMapping queryToView = SlotMapping.generate(relationMapping);
+ SlotMapping viewToQuery = queryToView.inverse();
+ LogicalCompatibilityContext compatibilityContext =
LogicalCompatibilityContext.from(
+ relationMapping, viewToQuery, queryStructInfo, viewStructInfo);
+ ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(
+ queryStructInfo, viewStructInfo, compatibilityContext);
+
+ Assertions.assertFalse(comparisonResult.isInvalid());
+
Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty());
+
+ SplitPredicate compensatePredicates =
TEST_RULE.predicatesCompensateForTest(
+ queryStructInfo, viewStructInfo, viewToQuery,
comparisonResult, queryContext);
+ Assertions.assertFalse(compensatePredicates.isInvalid());
+ Assertions.assertTrue(compensatePredicates.toList().stream()
+ .anyMatch(expression -> isNotNullOnSlot(expression,
"l_shipdate")));
+ }
+
+ @Test
+ void testNullRejectCompensationForInnerJoinFullJoinRewriteOnRightSide() {
+
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,PRUNE_EMPTY_PARTITION");
+ CascadesContext queryContext = createCascadesContext(
+ "select lineitem.l_shipdate, orders.o_orderdate from lineitem "
+ + "inner join orders on lineitem.l_orderkey =
orders.o_orderkey "
+ + "where lineitem.l_shipdate = '2023-10-17'",
+ connectContext
+ );
+ Plan queryPlan = PlanChecker.from(queryContext)
+ .analyze()
+ .rewrite()
+ .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+ .getAllPlan().get(0).child(0);
+
+ CascadesContext viewContext = createCascadesContext(
+ "select lineitem.l_shipdate, orders.o_orderdate from lineitem "
+ + "full outer join orders on lineitem.l_orderkey =
orders.o_orderkey",
+ connectContext
+ );
+ Plan viewPlan = PlanChecker.from(viewContext)
+ .analyze()
+ .rewrite()
+ .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+ .getAllPlan().get(0).child(0);
+
+ StructInfo queryStructInfo = StructInfo.of(queryPlan, queryPlan,
queryContext);
+ StructInfo viewStructInfo = StructInfo.of(viewPlan, viewPlan,
viewContext);
+ RelationMapping relationMapping = RelationMapping.generate(
+ queryStructInfo.getRelations(), viewStructInfo.getRelations(),
8).get(0);
+ SlotMapping queryToView = SlotMapping.generate(relationMapping);
+ SlotMapping viewToQuery = queryToView.inverse();
+ LogicalCompatibilityContext compatibilityContext =
LogicalCompatibilityContext.from(
+ relationMapping, viewToQuery, queryStructInfo, viewStructInfo);
+ ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(
+ queryStructInfo, viewStructInfo, compatibilityContext);
+
+ Assertions.assertFalse(comparisonResult.isInvalid());
+
Assertions.assertFalse(comparisonResult.getViewNoNullableSlot().isEmpty());
+
+ SplitPredicate compensatePredicates =
TEST_RULE.predicatesCompensateForTest(
+ queryStructInfo, viewStructInfo, viewToQuery,
comparisonResult, queryContext);
+ Assertions.assertFalse(compensatePredicates.isInvalid());
+ Assertions.assertTrue(compensatePredicates.toList().stream()
+ .anyMatch(expression -> isNotNullOnSlot(expression,
"o_orderdate")));
+ }
+
+ private static boolean isNotNullOnSlot(Expression expression, String
slotName) {
+ if (!(expression instanceof Not) || ((Not)
expression).isGeneratedIsNotNull()
+ || !(((Not) expression).child() instanceof IsNull)) {
+ return false;
+ }
+ Expression slot = ((IsNull) ((Not) expression).child()).child();
+ return slot instanceof SlotReference &&
slotName.equals(((SlotReference) slot).getName());
+ }
+
+ private static class TestMaterializedViewRule extends
AbstractMaterializedViewRule {
+ @Override
+ public List<Rule> buildRules() {
+ return ImmutableList.of();
+ }
+
+ private SplitPredicate predicatesCompensateForTest(StructInfo
queryStructInfo,
+ StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping,
+ ComparisonResult comparisonResult, CascadesContext
cascadesContext) {
+ return predicatesCompensate(queryStructInfo, viewStructInfo,
viewToQuerySlotMapping,
+ comparisonResult, cascadesContext);
+ }
+ }
+}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy
b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy
index c5f205638e6..73cdc618d86 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy
@@ -378,7 +378,7 @@ suite("partition_mv_rewrite_dimension_1") {
waitingMTMVTaskFinished(job_name)
for (int j = 0; j < join_type_stmt_list.size(); j++) {
logger.info("j:" + j)
- if (i == j) {
+ if (i == j || (j == 1 && i in [0, 2, 3])) {
mv_rewrite_success(join_type_stmt_list[j], join_type_mv)
compare_res(join_type_stmt_list[j] + " order by 1,2,3,4")
} else {
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy
b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy
index 70a45b3fd97..8a6c0a70292 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy
@@ -210,7 +210,7 @@ suite("partition_mv_rewrite_dimension_2_1") {
if (i == 0) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [ 0, 2, 4, 5, 10, 11]) {
+ if (j in [ 0, 2, 4, 5, 8, 10, 11]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
@@ -230,7 +230,7 @@ suite("partition_mv_rewrite_dimension_2_1") {
} else if (i == 2) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [0, 2, 4, 5, 10, 11]) {
+ if (j in [0, 2, 4, 5, 8, 10, 11]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
@@ -281,7 +281,7 @@ suite("partition_mv_rewrite_dimension_2_1") {
} else if (i == 7) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [4, 5, 7, 9, 10, 11]) {
+ if (j in [3, 4, 5, 7, 9, 10, 11]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
@@ -301,7 +301,7 @@ suite("partition_mv_rewrite_dimension_2_1") {
} else if (i == 9) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [4, 5, 7, 9, 10, 11]) {
+ if (j in [3, 4, 5, 7, 9, 10, 11]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy
b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy
index f3fd0795b4b..b9c27e366fa 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy
@@ -211,7 +211,7 @@ suite("partition_mv_rewrite_dimension_2_right_join") {
if (i == 0) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [ 0, 2, 4, 5, 10, 11]) {
+ if (j in [ 0, 2, 4, 5, 8, 10, 11]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
@@ -231,7 +231,7 @@ suite("partition_mv_rewrite_dimension_2_right_join") {
} else if (i == 2) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [0, 2, 4, 5, 10, 11]) {
+ if (j in [0, 2, 4, 5, 8, 10, 11]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
@@ -282,7 +282,7 @@ suite("partition_mv_rewrite_dimension_2_right_join") {
} else if (i == 7) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [4, 5, 7, 9, 10, 11]) {
+ if (j in [3, 4, 5, 7, 9, 10, 11]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
@@ -302,7 +302,7 @@ suite("partition_mv_rewrite_dimension_2_right_join") {
} else if (i == 9) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [4, 5, 7, 9, 10, 11]) {
+ if (j in [3, 4, 5, 7, 9, 10, 11]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
index e722b6eeb60..7a4995ad80c 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
@@ -293,7 +293,10 @@ suite("partition_mv_rewrite_dimension_self_conn") {
} else {
for (int j = 0; j < join_type_stmt_list.size(); j++) {
logger.info("j:" + j)
- if (i == j) {
+ // INNER query can use an OUTER JOIN MV only when the MV
output has a safe slot
+ // to filter null-generated rows. Here only RIGHT JOIN MV
exposes a non-nullable
+ // left-side slot for that compensation.
+ if (i == j || (j == 1 && i == 2)) {
mv_rewrite_success(join_type_stmt_list[j],
join_type_self_conn_mv)
compare_res(join_type_stmt_list[j] + " order by 1,2,3")
} else {
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_join_filter.groovy
b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_join_filter.groovy
index 37c9d3bfde4..74972d1fedc 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_join_filter.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_join_filter.groovy
@@ -216,7 +216,7 @@ suite("left_join_filter") {
} else if (i == 2) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [0, 2, 4, 9]) {
+ if (j in [0, 2, 4, 7, 9]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
@@ -276,7 +276,7 @@ suite("left_join_filter") {
} else if (i == 8) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [4, 6, 8, 9]) {
+ if (j in [3, 4, 6, 8, 9]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_join_filter.groovy
b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_join_filter.groovy
index c29effd36cb..20d4c125a93 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_join_filter.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_join_filter.groovy
@@ -226,7 +226,7 @@ suite("right_join_filter") {
} else if (i == 3) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [1, 3, 4, 9]) {
+ if (j in [1, 3, 4, 8, 9]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
@@ -266,7 +266,7 @@ suite("right_join_filter") {
} else if (i == 7) {
for (int j = 0; j < mv_list_1.size(); j++) {
logger.info("j:" + j)
- if (j in [4, 5, 7, 9]) {
+ if (j in [2, 4, 5, 7, 9]) {
mv_rewrite_success(mv_list_1[j], mv_name)
compare_res(mv_list_1[j] + " order by 1,2,3,4,5")
} else {
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join_two_hop_null_reject.groovy
b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join_two_hop_null_reject.groovy
new file mode 100644
index 00000000000..d481394777c
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join_two_hop_null_reject.groovy
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("outer_join_two_hop_null_reject") {
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+ sql "set runtime_filter_mode=OFF"
+ sql "set enable_nereids_planner=true"
+ sql "set enable_fallback_to_original_planner=false"
+ sql "set enable_materialized_view_rewrite=true"
+ sql "set enable_nereids_timeout=false"
+
+ sql """drop table if exists fact_orders_2hop"""
+ sql """drop table if exists dim_stores_2hop"""
+ sql """drop table if exists dim_regions_2hop"""
+
+ sql """
+ CREATE TABLE IF NOT EXISTS fact_orders_2hop (
+ order_date DATE NOT NULL,
+ store_id INT NOT NULL,
+ amount DECIMALV3(10, 2) NOT NULL
+ )
+ DUPLICATE KEY(order_date, store_id)
+ DISTRIBUTED BY HASH(store_id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS dim_stores_2hop (
+ id INT NOT NULL,
+ store_name VARCHAR(32) NOT NULL
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS dim_regions_2hop (
+ store_id INT NOT NULL,
+ region_name VARCHAR(32) NOT NULL
+ )
+ DUPLICATE KEY(store_id)
+ DISTRIBUTED BY HASH(store_id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """
+ insert into fact_orders_2hop values
+ ('2024-01-01', 1, 100.00),
+ ('2024-01-01', 2, 200.00),
+ ('2024-01-02', 3, 300.00),
+ ('2024-01-02', 4, 400.00)
+ """
+
+ sql """
+ insert into dim_stores_2hop values
+ (1, 'Store A'),
+ (2, 'Store B'),
+ (3, 'Store C')
+ """
+
+ sql """
+ insert into dim_regions_2hop values
+ (1, 'West'),
+ (2, 'East'),
+ (3, 'West')
+ """
+
+ sql """analyze table fact_orders_2hop with sync"""
+ sql """analyze table dim_stores_2hop with sync"""
+ sql """analyze table dim_regions_2hop with sync"""
+
+ def compare_res = { def stmt ->
+ sql "set enable_materialized_view_rewrite=false"
+ def origin_res = sql stmt
+ logger.info("origin_res: " + origin_res)
+ sql "set enable_materialized_view_rewrite=true"
+ def mv_origin_res = sql stmt
+ logger.info("mv_origin_res: " + mv_origin_res)
+ assertTrue((mv_origin_res == [] && origin_res == []) ||
(mv_origin_res.size() == origin_res.size()))
+ for (int row = 0; row < mv_origin_res.size(); row++) {
+ assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+ for (int col = 0; col < mv_origin_res[row].size(); col++) {
+ assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+ }
+ }
+ }
+
+ def mvName = "mv_orders_2hop_null_reject"
+ def mvSql = """
+ select o.order_date, o.store_id, d.store_name, r.region_name,
+ sum(o.amount) as total_amount
+ from fact_orders_2hop o
+ left join dim_stores_2hop d
+ on o.store_id = d.id
+ left join dim_regions_2hop r
+ on d.id = r.store_id
+ group by o.order_date, o.store_id, d.store_name, r.region_name
+ """
+ def querySql = """
+ select o.order_date, sum(o.amount)
+ from fact_orders_2hop o
+ left join dim_stores_2hop d
+ on o.store_id = d.id
+ left join dim_regions_2hop r
+ on d.id = r.store_id
+ where r.region_name = 'West'
+ group by o.order_date
+ order by 1
+ """
+
+ create_async_mv(db, mvName, mvSql)
+ mv_rewrite_success_without_check_chosen(querySql, mvName)
+ compare_res(querySql)
+
+ sql """drop materialized view if exists ${mvName}"""
+}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join_elim_p_f_key/join_elim_line_pattern.groovy
b/regression-test/suites/nereids_rules_p0/mv/join_elim_p_f_key/join_elim_line_pattern.groovy
index 9e1618d18dd..b5dda41c914 100644
---
a/regression-test/suites/nereids_rules_p0/mv/join_elim_p_f_key/join_elim_line_pattern.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/join_elim_p_f_key/join_elim_line_pattern.groovy
@@ -316,7 +316,7 @@ suite("join_elim_line_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [3, 5, 7]) {
+ if (j in [2, 3, 4, 5, 6, 7]) {
mv_rewrite_success_without_check_chosen(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 5)
} else {
@@ -484,7 +484,7 @@ suite("join_elim_line_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [3, 5, 7]) {
+ if (j in [2, 3, 4, 5, 6, 7]) {
mv_rewrite_success_without_check_chosen(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 5)
} else {
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join_elim_p_f_key/join_elim_star_pattern.groovy
b/regression-test/suites/nereids_rules_p0/mv/join_elim_p_f_key/join_elim_star_pattern.groovy
index d8850796a3e..c3fe74ce768 100644
---
a/regression-test/suites/nereids_rules_p0/mv/join_elim_p_f_key/join_elim_star_pattern.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/join_elim_p_f_key/join_elim_star_pattern.groovy
@@ -295,7 +295,7 @@ suite("join_elim_star_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13]) {
+ if (j in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]) {
mv_rewrite_success(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 4)
} else {
@@ -326,7 +326,7 @@ suite("join_elim_star_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [9, 11, 13]) {
+ if (j in [8, 9, 10, 11, 12, 13]) {
mv_rewrite_success(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 4)
} else {
@@ -358,7 +358,7 @@ suite("join_elim_star_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [3, 5, 7]) {
+ if (j in [2, 3, 4, 5, 6, 7]) {
mv_rewrite_success(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 4)
} else {
@@ -393,7 +393,7 @@ suite("join_elim_star_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13]) {
+ if (j in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]) {
mv_rewrite_success(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 4)
} else {
@@ -428,7 +428,7 @@ suite("join_elim_star_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [1, 2, 3, 4, 5, 6, 7, 9, 11, 13]) {
+ if (j in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]) {
mv_rewrite_success(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 4)
} else {
@@ -462,7 +462,7 @@ suite("join_elim_star_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [1, 2, 3, 4, 5, 6, 7, 9, 11, 13]) {
+ if (j in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]) {
mv_rewrite_success(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 4)
} else {
@@ -492,7 +492,7 @@ suite("join_elim_star_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [9, 11, 13]) {
+ if (j in [8, 9, 10, 11, 12, 13]) {
mv_rewrite_success(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 4)
} else {
@@ -523,7 +523,7 @@ suite("join_elim_star_pattern") {
create_async_mv(db, mv_name, mv_stmt_2)
for (int j = 1; j < query_list.size() + 1; j++) {
logger.info("left mv current query index: " + j)
- if (j in [3, 5, 7]) {
+ if (j in [2, 3, 4, 5, 6, 7]) {
mv_rewrite_success(query_list[j - 1], mv_name)
compare_res(query_list[j - 1], 4)
} else {
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy
b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy
index 15709ed1f6f..c5768f1b95d 100644
---
a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy
@@ -221,32 +221,20 @@ suite("inner_join_infer_and_derive") {
if (mtmv_it == 0) {
for (int i = 0; i < query_list.size(); i++) {
logger.info("i: " + i)
- if (i in [0, 2, 5, 6, 8, 11]) {
- mv_rewrite_fail(query_list[i], mv_name_1)
- } else {
- mv_rewrite_success(query_list[i], mv_name_1)
- compare_res(query_list[i] + order_stmt)
- }
+ mv_rewrite_success(query_list[i], mv_name_1)
+ compare_res(query_list[i] + order_stmt)
}
} else if (mtmv_it == 1) {
for (int i = 0; i < query_list.size(); i++) {
logger.info("i: " + i)
- if (i in [1, 3, 4, 7, 9, 10]) {
- mv_rewrite_fail(query_list[i], mv_name_1)
- } else {
- mv_rewrite_success(query_list[i], mv_name_1)
- compare_res(query_list[i] + order_stmt)
- }
+ mv_rewrite_success(query_list[i], mv_name_1)
+ compare_res(query_list[i] + order_stmt)
}
} else if (mtmv_it == 2) {
for (int i = 0; i < query_list.size(); i++) {
logger.info("i: " + i)
- if (i in [12, 13]) {
- mv_rewrite_success(query_list[i], mv_name_1)
- compare_res(query_list[i] + order_stmt)
- } else {
- mv_rewrite_fail(query_list[i], mv_name_1)
- }
+ mv_rewrite_success(query_list[i], mv_name_1)
+ compare_res(query_list[i] + order_stmt)
}
}
sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name_1};"""
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_null_reject_compensation.groovy
b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_null_reject_compensation.groovy
new file mode 100644
index 00000000000..a678dd461c8
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_null_reject_compensation.groovy
@@ -0,0 +1,217 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("inner_join_null_reject_compensation") {
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+ sql "set runtime_filter_mode=OFF"
+ sql "set enable_nereids_planner=true"
+ sql "set enable_fallback_to_original_planner=false"
+ sql "set enable_materialized_view_rewrite=true"
+ sql "set pre_materialized_view_rewrite_strategy=FORCE_IN_RBO"
+ sql "set enable_nereids_timeout=false"
+
+ sql """drop materialized view if exists
mv_inner_join_null_reject_compensation"""
+ sql """drop materialized view if exists mv_repro_left_join"""
+ sql """drop materialized view if exists
mv_repro_left_join_missing_right_output"""
+ sql """drop materialized view if exists
mv_repro_left_join_nullable_right_output"""
+ sql """drop table if exists mv_repro_a"""
+ sql """drop table if exists mv_repro_b"""
+ sql """drop table if exists orders_inner_join_null_reject"""
+ sql """drop table if exists lineitem_inner_join_null_reject"""
+
+ sql """
+ create table lineitem_inner_join_null_reject (
+ l_orderkey int not null,
+ l_shipdate date not null,
+ l_suppkey int not null
+ )
+ duplicate key(l_orderkey)
+ distributed by hash(l_orderkey) buckets 1
+ properties (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """
+ create table orders_inner_join_null_reject (
+ o_orderkey int not null,
+ o_orderdate date not null
+ )
+ duplicate key(o_orderkey)
+ distributed by hash(o_orderkey) buckets 1
+ properties (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """
+ create table mv_repro_a (
+ id int null,
+ k int null
+ )
+ duplicate key(id)
+ distributed by hash(id) buckets 1
+ properties (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """
+ create table mv_repro_b (
+ k int null,
+ v int null
+ )
+ duplicate key(k)
+ distributed by hash(k) buckets 1
+ properties (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """
+ insert into lineitem_inner_join_null_reject values
+ (1, '2023-10-17', 10),
+ (999, '2023-10-17', 20)
+ """
+
+ sql """
+ insert into orders_inner_join_null_reject values
+ (1, '2023-10-17'),
+ (888, '2023-10-17')
+ """
+
+ sql """
+ insert into mv_repro_a values
+ (1, 10),
+ (2, 20)
+ """
+
+ sql """
+ insert into mv_repro_b values
+ (10, 100)
+ """
+
+ sql """analyze table lineitem_inner_join_null_reject with sync"""
+ sql """analyze table orders_inner_join_null_reject with sync"""
+ sql """analyze table mv_repro_a with sync"""
+ sql """analyze table mv_repro_b with sync"""
+
+ def withUseMvHint = { def stmt, def mvName ->
+ stmt.replaceFirst("(?i)\\bselect\\b", "select /*+ use_mv(${mvName})
*/")
+ }
+
+ def compare_res_with_forced_mv = { def stmt, def mvName ->
+ def stmtWithUseMvHint = withUseMvHint(stmt, mvName)
+ sql "set enable_materialized_view_rewrite=false"
+ def origin_res = sql stmt
+ logger.info("origin_res: " + origin_res)
+ sql "set enable_materialized_view_rewrite=true"
+ mv_rewrite_success(stmtWithUseMvHint, mvName)
+ def mv_origin_res = sql stmtWithUseMvHint
+ logger.info("mv_origin_res: " + mv_origin_res)
+ assertTrue((mv_origin_res == [] && origin_res == []) ||
(mv_origin_res.size() == origin_res.size()))
+ for (int row = 0; row < mv_origin_res.size(); row++) {
+ assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+ for (int col = 0; col < mv_origin_res[row].size(); col++) {
+ assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+ }
+ }
+ }
+
+ def mvName = "mv_inner_join_null_reject_compensation"
+ def mvSql = """
+ select l.l_shipdate, l.l_suppkey, o.o_orderdate
+ from lineitem_inner_join_null_reject l
+ full outer join orders_inner_join_null_reject o
+ on l.l_orderkey = o.o_orderkey
+ """
+
+ create_async_mv(db, mvName, mvSql)
+
+ def queryNeedLeftSideCompensation = """
+ select l.l_shipdate, l.l_suppkey, o.o_orderdate
+ from lineitem_inner_join_null_reject l
+ inner join orders_inner_join_null_reject o
+ on l.l_orderkey = o.o_orderkey
+ where o.o_orderdate = '2023-10-17'
+ order by 1, 2, 3
+ """
+
+ def queryNeedRightSideCompensation = """
+ select l.l_shipdate, l.l_suppkey, o.o_orderdate
+ from lineitem_inner_join_null_reject l
+ inner join orders_inner_join_null_reject o
+ on l.l_orderkey = o.o_orderkey
+ where l.l_shipdate = '2023-10-17'
+ order by 1, 2, 3
+ """
+
+ compare_res_with_forced_mv(queryNeedLeftSideCompensation, mvName)
+ compare_res_with_forced_mv(queryNeedRightSideCompensation, mvName)
+
+ def leftJoinMvName = "mv_repro_left_join"
+ def leftJoinMvSql = """
+ select
+ a.id as a_id,
+ a.k as a_k,
+ b.k as b_k,
+ b.v as b_v
+ from mv_repro_a a
+ left join mv_repro_b b
+ on a.k = b.k
+ """
+
+ create_async_mv(db, leftJoinMvName, leftJoinMvSql)
+
+ def innerJoinQueryOnLeftJoinMv = """
+ select a.id
+ from mv_repro_a a
+ inner join mv_repro_b b
+ on a.k = b.k
+ order by 1
+ """
+
+ compare_res_with_forced_mv(innerJoinQueryOnLeftJoinMv, leftJoinMvName)
+
+ def leftJoinMvWithoutRightOutputName =
"mv_repro_left_join_missing_right_output"
+ def leftJoinMvWithoutRightOutputSql = """
+ select
+ a.id as a_id,
+ a.k as a_k
+ from mv_repro_a a
+ left join mv_repro_b b
+ on a.k = b.k
+ """
+
+ create_async_mv(db, leftJoinMvWithoutRightOutputName,
leftJoinMvWithoutRightOutputSql)
+ mv_rewrite_fail(innerJoinQueryOnLeftJoinMv,
leftJoinMvWithoutRightOutputName)
+
+ def leftJoinMvWithNullableRightOutputName =
"mv_repro_left_join_nullable_right_output"
+ def leftJoinMvWithNullableRightOutputSql = """
+ select
+ a.id as a_id,
+ a.k as a_k,
+ b.v as b_v
+ from mv_repro_a a
+ left join mv_repro_b b
+ on a.k = b.k
+ """
+
+ create_async_mv(db, leftJoinMvWithNullableRightOutputName,
leftJoinMvWithNullableRightOutputSql)
+ mv_rewrite_fail(innerJoinQueryOnLeftJoinMv,
leftJoinMvWithNullableRightOutputName)
+}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy
b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy
index 7e51847c3b1..000e1caef1d 100644
---
a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy
@@ -211,7 +211,11 @@ suite("left_join_infer_and_derive") {
if (mtmv_it == 0) {
for (int i = 0; i < query_list.size(); i++) {
logger.info("i: " + i)
- if (i in [1, 3, 4, 6, 8, 11]) {
+ // For query_stmt_4, the LEFT JOIN is converted to INNER JOIN
before MV rewrite,
+ // but the join condition can be simplified away, leaving the
INNER JoinEdge with
+ // no expressions. Without JoinEdge expressions, null-reject
slots cannot be
+ // inferred, so this case is expected to fail.
+ if (i in [1, 4, 6]) {
mv_rewrite_fail(query_list[i], mv_name_1)
} else {
mv_rewrite_success(query_list[i], mv_name_1)
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy
b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy
index 1761efa5ef4..e43cf21e0cc 100644
---
a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy
@@ -211,7 +211,11 @@ suite("right_join_infer_and_derive") {
if (mtmv_it == 0) {
for (int i = 0; i < query_list.size(); i++) {
logger.info("i: " + i)
- if (i in [0, 2, 5, 7, 9, 10]) {
+ // For query_stmt_10, the RIGHT JOIN is converted to INNER
JOIN before MV rewrite,
+ // but the join condition can be simplified away, leaving the
INNER JoinEdge with
+ // no expressions. Without JoinEdge expressions, null-reject
slots cannot be
+ // inferred, so this case is expected to fail.
+ if (i in [0, 7, 10]) {
mv_rewrite_fail(query_list[i], mv_name_1)
} else {
mv_rewrite_success(query_list[i], mv_name_1)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
