This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3716f8a171f0aea70720313d7a1676ec487659af Author: Pxl <[email protected]> AuthorDate: Wed Mar 6 20:47:06 2024 +0800 [Bug](partition) fix npe when prune partition with not exist partition column in mv #31860 --- .../rules/rewrite/PruneOlapScanPartition.java | 15 +++-- .../mv_p0/test_mv_partition/test_mv_partition.out | 4 ++ .../test_mv_partition/test_mv_partition.groovy | 64 ++++++++++++++++++++++ 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java index e9e59f1c113..eef8dc33fa0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; @@ -62,10 +63,16 @@ public class PruneOlapScanPartition extends OneRewriteRuleFactory { .collect(Collectors.toMap(slot -> slot.getName().toLowerCase(), Function.identity())); PartitionInfo partitionInfo = table.getPartitionInfo(); - List<Slot> partitionSlots = partitionInfo.getPartitionColumns() - .stream() - .map(column -> scanOutput.get(column.getName().toLowerCase())) - .collect(Collectors.toList()); + List<Slot> partitionSlots = new ArrayList<>(); + for (Column column : partitionInfo.getPartitionColumns()) { + Slot slot = scanOutput.get(column.getName().toLowerCase()); + if (slot == null) { + return filter; + } else { + partitionSlots.add(slot); + } + } + List<Long> manuallySpecifiedPartitions = scan.getManuallySpecifiedPartitions(); Map<Long, PartitionItem> idToPartitions; diff --git a/regression-test/data/mv_p0/test_mv_partition/test_mv_partition.out b/regression-test/data/mv_p0/test_mv_partition/test_mv_partition.out new file mode 100644 index 00000000000..352a8c8050f --- /dev/null +++ b/regression-test/data/mv_p0/test_mv_partition/test_mv_partition.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_mv -- +2024-03-04 2 + diff --git a/regression-test/suites/mv_p0/test_mv_partition/test_mv_partition.groovy b/regression-test/suites/mv_p0/test_mv_partition/test_mv_partition.groovy new file mode 100644 index 00000000000..510156e01c6 --- /dev/null +++ b/regression-test/suites/mv_p0/test_mv_partition/test_mv_partition.groovy @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_mv_partition") { + sql """set enable_nereids_planner=true;""" + sql """set enable_fallback_to_original_planner=false;""" + sql """ DROP TABLE IF EXISTS chh; """ + + sql """ + CREATE TABLE `chh` ( + `event_id` varchar(50) NULL COMMENT '', + `time_stamp` datetime NULL COMMENT '', + `device_id` varchar(150) NULL DEFAULT "" COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`event_id`) + PARTITION BY RANGE(`time_stamp`)() + DISTRIBUTED BY HASH(`device_id`) BUCKETS AUTO + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "is_being_synced" = "false", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.time_zone" = "Asia/Shanghai", + "dynamic_partition.start" = "-30", + "dynamic_partition.end" = "1", + "dynamic_partition.prefix" = "p", + "dynamic_partition.replication_allocation" = "tag.location.default: 1", + "dynamic_partition.buckets" = "20", + "dynamic_partition.create_history_partition" = "true", + "dynamic_partition.history_partition_num" = "-1", + "dynamic_partition.hot_partition_num" = "0", + "dynamic_partition.reserved_history_periods" = "NULL", + "dynamic_partition.storage_policy" = "", + "dynamic_partition.storage_medium" = "HDD", + "storage_format" = "V2", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false" + ); + """ + + sql """insert into chh(event_id,time_stamp,device_id) values('ad_sdk_request','2024-03-04 00:00:00','a');""" + + createMV("create materialized view m_view as select to_date(time_stamp),count(device_id) from chh group by to_date(time_stamp);") + + sql """insert into chh(event_id,time_stamp,device_id) values('ad_sdk_request','2024-03-04 00:00:00','a');""" + + qt_select_mv "select * from chh index m_view where `mv_to_date(time_stamp)` = '2024-03-04';" +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
