This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new bf2d5280480 [Improvement](materialized-view) adjust priority of materialized view match rule (#33305) (#33348) bf2d5280480 is described below commit bf2d5280480c5ce91933479b7d7dcab4cd8a2226 Author: Pxl <pxl...@qq.com> AuthorDate: Mon Apr 8 13:15:20 2024 +0800 [Improvement](materialized-view) adjust priority of materialized view match rule (#33305) (#33348) adjust priority of materialized view match rule --- .../java/org/apache/doris/catalog/OlapTable.java | 8 +++ .../mv/AbstractSelectMaterializedIndexRule.java | 25 +++++++++- regression-test/data/mv_p0/test_base/test_base.out | 9 ++++ .../suites/mv_p0/test_base/test_base.groovy | 57 ++++++++++++++++++++++ .../suites/nereids_p0/test_prune_tablet_mv.groovy | 2 +- 5 files changed, 98 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 8bb31d32efc..536b8032a99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -462,6 +462,14 @@ public class OlapTable extends Table { : Collections.emptyList(); } + public MaterializedIndex getBaseIndex() { + Optional<Partition> partition = idToPartition.values().stream().findFirst(); + if (!partition.isPresent()) { + partition = tempPartitions.getAllPartitions().stream().findFirst(); + } + return partition.isPresent() ? partition.get().getBaseIndex() : null; + } + public Column getVisibleColumn(String columnName) { for (MaterializedIndexMeta meta : getVisibleIndexIdToMeta().values()) { Column target = meta.getColumnByDefineName(columnName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index da73094e5a5..17eadbd319b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.rewrite.mv; import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; @@ -59,7 +60,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Lists; -import org.apache.commons.collections.CollectionUtils; import java.util.Comparator; import java.util.HashMap; @@ -185,6 +185,9 @@ public abstract class AbstractSelectMaterializedIndexRule { return scan.getTable().getBaseIndexId(); } + MaterializedIndex baseIndex = scan.getTable().getBaseIndex(); + candidates.add(baseIndex); + OlapTable table = scan.getTable(); // Scan slot exprId -> slot name Map<ExprId, String> exprIdToName = scan.getOutput() @@ -205,11 +208,13 @@ public abstract class AbstractSelectMaterializedIndexRule { .sum()) // compare by column count .thenComparing(rid -> table.getSchemaByIndexId((Long) rid).size()) + // prioritize using non-base index + .thenComparing(rid -> (Long) rid == baseIndex.getId()) // compare by index id .thenComparing(rid -> (Long) rid)) .collect(Collectors.toList()); - return CollectionUtils.isEmpty(sortedIndexIds) ? scan.getTable().getBaseIndexId() : sortedIndexIds.get(0); + return sortedIndexIds.get(0); } protected List<MaterializedIndex> matchPrefixMost( @@ -223,6 +228,22 @@ public abstract class AbstractSelectMaterializedIndexRule { Set<String> nonEqualColNames = split.getOrDefault(false, ImmutableSet.of()).stream() .map(String::toLowerCase).collect(Collectors.toSet()); + // prioritize using index with where clause + if (candidate.stream() + .anyMatch(index -> scan.getTable().getIndexMetaByIndexId(index.getId()).getWhereClause() != null)) { + candidate = candidate.stream() + .filter(index -> scan.getTable().getIndexMetaByIndexId(index.getId()).getWhereClause() != null) + .collect(Collectors.toList()); + } + + // prioritize using index with pre agg + if (candidate.stream().anyMatch( + index -> scan.getTable().getIndexMetaByIndexId(index.getId()).getKeysType() != KeysType.DUP_KEYS)) { + candidate = candidate.stream().filter( + index -> scan.getTable().getIndexMetaByIndexId(index.getId()).getKeysType() != KeysType.DUP_KEYS) + .collect(Collectors.toList()); + } + if (!(equalColNames.isEmpty() && nonEqualColNames.isEmpty())) { List<MaterializedIndex> matchingResult = matchKeyPrefixMost(scan.getTable(), candidate, equalColNames, nonEqualColNames); diff --git a/regression-test/data/mv_p0/test_base/test_base.out b/regression-test/data/mv_p0/test_base/test_base.out new file mode 100644 index 00000000000..82e8aacf086 --- /dev/null +++ b/regression-test/data/mv_p0/test_base/test_base.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_mv -- +\N 1 +\N 2 + +-- !select_mv -- +1 \N +2 \N + diff --git a/regression-test/suites/mv_p0/test_base/test_base.groovy b/regression-test/suites/mv_p0/test_base/test_base.groovy new file mode 100644 index 00000000000..f3229813e2f --- /dev/null +++ b/regression-test/suites/mv_p0/test_base/test_base.groovy @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_base") { + sql """set enable_nereids_planner=true""" + sql """SET enable_fallback_to_original_planner=false""" + sql """ drop table if exists dwd;""" + + sql """ + CREATE TABLE `dwd` ( + `id` bigint(20) NULL COMMENT 'id', + `created_at` datetime NULL, + `dt` date NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """insert into dwd(id) values(1);""" + + createMV (""" + create materialized view dwd_mv as SELECT created_at, id FROM dwd; + """) + + sql """insert into dwd(id) values(2);""" + + explain { + sql("SELECT created_at, id FROM dwd order by 1, 2;") + contains "(dwd_mv)" + } + qt_select_mv "SELECT created_at, id FROM dwd order by 1, 2;" + + explain { + sql("SELECT id,created_at FROM dwd order by 1, 2;") + contains "(dwd)" + } + qt_select_mv "SELECT id,created_at FROM dwd order by 1, 2;" +} diff --git a/regression-test/suites/nereids_p0/test_prune_tablet_mv.groovy b/regression-test/suites/nereids_p0/test_prune_tablet_mv.groovy index 2a5c1349ac9..680b568de44 100644 --- a/regression-test/suites/nereids_p0/test_prune_tablet_mv.groovy +++ b/regression-test/suites/nereids_p0/test_prune_tablet_mv.groovy @@ -36,7 +36,7 @@ suite("test_prune_tablet_mv") { sql "insert into test_prune_tablet_t2 values(1,0),(2,0),(3,0),(4,0),(5,0),(6,0),(7,0);" explain { - sql("select * from test_prune_tablet_t2 where id = 3;") + sql("select * from test_prune_tablet_t2 where c1 = 0 and id = 3;") contains "mv_t2" contains "tablets=1/16" } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org