This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7f0d3d9dcbdf56832ec1d936587a5e3edeeea457 Author: seawinde <[email protected]> AuthorDate: Thu Feb 1 18:33:55 2024 +0800 [Fix](nereids)Disable getting partition related table and column when self join (#30650) * add left anti join ut * forbidden the self join partition column get * [Fix](nereids) Disable getting partition related table and column when self join * fix code style --- .../exploration/mv/MaterializedViewUtils.java | 56 ++++++++----- .../exploration/mv/MaterializedViewUtilsTest.java | 92 +++++++++++++++++++++- 2 files changed, 126 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index e68f9d833c9..e7b11b5fd17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -43,6 +43,9 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -72,9 +75,6 @@ public class MaterializedViewUtils { break; } } - if (columnExpr == null) { - return Optional.empty(); - } if (!(columnExpr instanceof SlotReference)) { return Optional.empty(); } @@ -85,13 +85,14 @@ public class MaterializedViewUtils { // check sql pattern IncrementCheckerContext context = new IncrementCheckerContext(columnSlot); materializedViewPlan.accept(MaterializedViewIncrementChecker.INSTANCE, context); - if (context.getTableColumnList().isEmpty() || !context.isPctPossible()) { + if (context.getPartitionRelatedTableAndColumnList().isEmpty() || !context.isPctPossible()) { return Optional.empty(); } // TODO support to return only one related table info, support multi later - return Optional.of(new RelatedTableInfo(new BaseTableInfo(context.getTableColumnList().get(0).key()), + Pair<TableIf, Column> tableIfColumnPair = context.getPartitionRelatedTableAndColumnList().get(0); + return Optional.of(new RelatedTableInfo(new BaseTableInfo(tableIfColumnPair.key()), context.isPctPossible(), - context.getTableColumnList().get(0).value().getName())); + tableIfColumnPair.value().getName())); } /** @@ -226,19 +227,29 @@ public class MaterializedViewUtils { @Override public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerContext context) { - if (!(relation instanceof LogicalCatalogRelation) || !context.getTableColumnList().isEmpty()) { - return visit(relation, context); + if (!(relation instanceof LogicalCatalogRelation)) { + return null; } LogicalCatalogRelation logicalCatalogRelation = (LogicalCatalogRelation) relation; TableIf table = logicalCatalogRelation.getTable(); + // if self join, can't infer partition column + if (!context.getTableIdAndRelationMapping().get(table.getId()).isEmpty()) { + context.setPctPossible(false); + return null; + } + // record tableId and relation, to check the self join + context.addTableIdAndRelation(((LogicalCatalogRelation) relation).getTable().getId(), relation); + // TODO: 2024/1/31 support only one partition referenced column, support multi later + if (!context.getPartitionRelatedTableAndColumnList().isEmpty()) { + return null; + } if (!(table instanceof MTMVRelatedTableIf)) { - return visit(relation, context); + return null; } MTMVRelatedTableIf relatedTable = (MTMVRelatedTableIf) table; PartitionType type = relatedTable.getPartitionType(); - if (PartitionType.UNPARTITIONED.equals(type)) { - return visit(relation, context); + return null; } Set<Column> partitionColumnSet = new HashSet<>(relatedTable.getPartitionColumns()); Column mvReferenceColumn = context.getMvPartitionColumn().getColumn().get(); @@ -254,7 +265,8 @@ public class MaterializedViewUtils { IncrementCheckerContext context) { Set<Expression> groupByExprSet = new HashSet<>(aggregate.getGroupByExpressions()); if (groupByExprSet.isEmpty()) { - return visit(aggregate, context); + context.setPctPossible(false); + return null; } Set<Column> originalGroupbyExprSet = new HashSet<>(); groupByExprSet.forEach(groupExpr -> { @@ -264,6 +276,7 @@ public class MaterializedViewUtils { }); if (!originalGroupbyExprSet.contains(context.getMvPartitionColumn().getColumn().get())) { context.setPctPossible(false); + return null; } return visit(aggregate, context); } @@ -317,8 +330,9 @@ public class MaterializedViewUtils { private static final class IncrementCheckerContext { private final SlotReference mvPartitionColumn; private boolean pctPossible = true; - private final List<Pair<TableIf, Column>> tableColumnList = new ArrayList<>(); - private boolean joinNullGenerateSide; + private final List<Pair<TableIf, Column>> partitionRelatedTableAndColumnList = new ArrayList<>(); + // This record the table id and relation mapping, because a table maybe used repeatedly. + private final Multimap<Long, LogicalRelation> tableIdAndRelationMapping = HashMultimap.create(); public IncrementCheckerContext(SlotReference mvPartitionColumn) { this.mvPartitionColumn = mvPartitionColumn; @@ -337,19 +351,19 @@ public class MaterializedViewUtils { } public void addTableColumn(TableIf relatedTable, Column partitionColumn) { - tableColumnList.add(Pair.of(relatedTable, partitionColumn)); + partitionRelatedTableAndColumnList.add(Pair.of(relatedTable, partitionColumn)); } - public List<Pair<TableIf, Column>> getTableColumnList() { - return tableColumnList; + public List<Pair<TableIf, Column>> getPartitionRelatedTableAndColumnList() { + return partitionRelatedTableAndColumnList; } - public boolean isJoinNullGenerateSide() { - return joinNullGenerateSide; + public Multimap<Long, LogicalRelation> getTableIdAndRelationMapping() { + return tableIdAndRelationMapping; } - public void setJoinNullGenerateSide(boolean joinNullGenerateSide) { - this.joinNullGenerateSide = joinNullGenerateSide; + public void addTableIdAndRelation(Long tableId, LogicalRelation relation) { + tableIdAndRelationMapping.put(tableId, relation); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 8bf8ea14ea5..4204bbe0221 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -65,6 +65,37 @@ public class MaterializedViewUtilsTest extends TestWithFeService { + "PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" + ")"); + + createTable("CREATE TABLE `lineitem_list_partition` (\n" + + " `l_orderkey` BIGINT not NULL,\n" + + " `l_linenumber` INT NULL,\n" + + " `l_partkey` INT NULL,\n" + + " `l_suppkey` INT NULL,\n" + + " `l_quantity` DECIMAL(15, 2) NULL,\n" + + " `l_extendedprice` DECIMAL(15, 2) NULL,\n" + + " `l_discount` DECIMAL(15, 2) NULL,\n" + + " `l_tax` DECIMAL(15, 2) NULL,\n" + + " `l_returnflag` VARCHAR(1) NULL,\n" + + " `l_linestatus` VARCHAR(1) NULL,\n" + + " `l_commitdate` DATE NULL,\n" + + " `l_receiptdate` DATE NULL,\n" + + " `l_shipinstruct` VARCHAR(25) NULL,\n" + + " `l_shipmode` VARCHAR(10) NULL,\n" + + " `l_comment` VARCHAR(44) NULL,\n" + + " `l_shipdate` DATE NULL\n" + + " ) ENGINE=OLAP\n" + + " DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )\n" + + " COMMENT 'OLAP'\n" + + " PARTITION BY list(l_orderkey) (\n" + + " PARTITION p1 VALUES in ('1'),\n" + + " PARTITION p2 VALUES in ('2'),\n" + + " PARTITION p3 VALUES in ('3')\n" + + " )\n" + + " DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 3\n" + + " PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + " )"); + createTable("CREATE TABLE IF NOT EXISTS orders (\n" + " O_ORDERKEY INTEGER NOT NULL,\n" + " O_CUSTKEY INTEGER NOT NULL,\n" @@ -82,6 +113,30 @@ public class MaterializedViewUtilsTest extends TestWithFeService { + "PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" + ")"); + + createTable("CREATE TABLE `orders_list_partition` (\n" + + " `o_orderkey` BIGINT not NULL,\n" + + " `o_custkey` INT NULL,\n" + + " `o_orderstatus` VARCHAR(1) NULL,\n" + + " `o_totalprice` DECIMAL(15, 2) NULL,\n" + + " `o_orderpriority` VARCHAR(15) NULL,\n" + + " `o_clerk` VARCHAR(15) NULL,\n" + + " `o_shippriority` INT NULL,\n" + + " `o_comment` VARCHAR(79) NULL,\n" + + " `o_orderdate` DATE NULL\n" + + " ) ENGINE=OLAP\n" + + " DUPLICATE KEY(`o_orderkey`, `o_custkey`)\n" + + " COMMENT 'OLAP'\n" + + " PARTITION BY list(o_orderkey) (\n" + + " PARTITION p1 VALUES in ('1'),\n" + + " PARTITION p2 VALUES in ('2'),\n" + + " PARTITION p3 VALUES in ('3'),\n" + + " PARTITION p4 VALUES in ('4')\n" + + " )\n" + + " DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 3\n" + + " PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + " )"); createTable("CREATE TABLE IF NOT EXISTS partsupp (\n" + " PS_PARTKEY INTEGER NOT NULL,\n" + " PS_SUPPKEY INTEGER NOT NULL,\n" @@ -141,7 +196,7 @@ public class MaterializedViewUtilsTest extends TestWithFeService { + " DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )\n" + " COMMENT 'OLAP'\n" + " AUTO PARTITION BY range date_trunc(`l_shipdate`, 'day') ()\n" - + " DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96\n" + + " DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 3\n" + " PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" + " );\n" @@ -276,6 +331,41 @@ public class MaterializedViewUtilsTest extends TestWithFeService { }); } + @Test + public void getRelatedTableInfoLeftAntiJoinTest() { + PlanChecker.from(connectContext) + .checkExplain(" select l_shipdate, l_orderkey, count(l_shipdate), count(l_orderkey) \n" + + " from lineitem_list_partition\n" + + " left anti join orders_list_partition\n" + + " on l_shipdate = o_orderdate\n" + + " group by l_shipdate, l_orderkey", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + Optional<RelatedTableInfo> relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan); + checkRelatedTableInfo(relatedTableInfo, + "lineitem_list_partition", + "l_orderkey", + true); + }); + } + + @Test + public void getRelatedTableInfoSelfJoinTest() { + PlanChecker.from(connectContext) + .checkExplain(" select t1.l_shipdate, t1.l_orderkey, t1.l_partkey, t1.l_suppkey, 1\n" + + " from lineitem_list_partition t1\n" + + " join lineitem_list_partition t2\n" + + " on t1.l_shipdate = t2.l_shipdate\n" + + " group by t1.l_shipdate, t1.l_orderkey, t1.l_partkey, t1.l_suppkey", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + Optional<RelatedTableInfo> relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan); + Assertions.assertFalse(relatedTableInfo.isPresent()); + }); + } + @Test public void getRelatedTableInfoUseRightTest() { PlanChecker.from(connectContext) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
