This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 42e91149e4df989a5cbdbcdeb72b5377e58ed653 Author: zclllyybb <[email protected]> AuthorDate: Fri Apr 19 20:15:01 2024 +0800 [enhancement](auto-partition) Forbid use Auto and Dynamic partition at the same time (#33736) --- .../org/apache/doris/analysis/PartitionDesc.java | 32 ++++++++----- .../doris/common/util/DynamicPartitionUtil.java | 6 +++ .../apache/doris/datasource/InternalCatalog.java | 17 +------ .../doris/nereids/parser/PartitionTableInfo.java | 6 ++- .../test_auto_partition_behavior.groovy | 53 +++++++++++++++++++++- 5 files changed, 84 insertions(+), 30 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java index 288d5a8457a..697394b3203 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java @@ -102,7 +102,6 @@ public class PartitionDesc { // 1. partition by list (column) : now support one slotRef // 2. partition by range(column/function(column)) : support slotRef and some // special function eg: date_trunc, date_floor/ceil - // not only for auto partition. maybe we should check for project partitiion also public static List<String> getColNamesFromExpr(ArrayList<Expr> exprs, boolean isListPartition, boolean isAutoPartition) throws AnalysisException { @@ -206,16 +205,27 @@ public class PartitionDesc { throw new AnalysisException( "The partition column must be NOT NULL with allow_partition_column_nullable OFF"); } - if (this instanceof RangePartitionDesc && isAutoCreatePartitions && columnDef.isAllowNull()) { - throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column"); - } - if (this instanceof RangePartitionDesc && partitionExprs != null) { - if (partitionExprs.get(0) instanceof FunctionCallExpr) { - if (!columnDef.getType().isDateType()) { - throw new AnalysisException( - "Auto range partition needs Date/DateV2/" - + "Datetime/DatetimeV2 column as partition column" - + partitionExprs.get(0).toSql()); + if (this instanceof RangePartitionDesc && isAutoCreatePartitions) { + if (columnDef.isAllowNull()) { + throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column"); + } + if (partitionExprs != null) { + for (Expr expr : partitionExprs) { + if (!(expr instanceof FunctionCallExpr) || !RANGE_PARTITION_FUNCTIONS + .contains(((FunctionCallExpr) expr).getFnName().getFunction())) { + throw new AnalysisException( + "auto create partition only support slotRef and " + + "date_trunc/date_floor/date_ceil function in range partitions. " + + expr.toSql()); + } + } + if (partitionExprs.get(0) instanceof FunctionCallExpr) { + if (!columnDef.getType().isDateType()) { + throw new AnalysisException( + "Auto range partition needs Date/DateV2/" + + "Datetime/DatetimeV2 column as partition column but got" + + partitionExprs.get(0).toSql()); + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 1dc2d81dd7c..4464473788f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -535,6 +535,12 @@ public class DynamicPartitionUtil { analyzedProperties.put(DynamicPartitionProperty.ENABLE, enableValue); } + if (Boolean.parseBoolean(analyzedProperties.getOrDefault(DynamicPartitionProperty.ENABLE, "true")) + && olapTable.getPartitionInfo().enableAutomaticPartition()) { + throw new AnalysisException( + "Can't use Dynamic Partition and Auto Partition at the same time"); + } + // If dynamic property "start" is not specified, use Integer.MIN_VALUE as default int start = DynamicPartitionProperty.MIN_START_OFFSET; if (properties.containsKey(DynamicPartitionProperty.START)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index c22cbf38c6f..ea33e30f861 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2645,21 +2645,8 @@ public class InternalCatalog implements CatalogIf<Database> { .getDynamicPartitionProperty(); if (dynamicProperty.isExist() && dynamicProperty.getEnable() && partitionDesc.isAutoCreatePartitions()) { - String dynamicUnit = dynamicProperty.getTimeUnit(); - ArrayList<Expr> autoExprs = partitionDesc.getPartitionExprs(); - for (Expr autoExpr : autoExprs) { - Expr func = (FunctionCallExpr) autoExpr; - for (Expr child : func.getChildren()) { - if (child instanceof LiteralExpr) { - String autoUnit = ((LiteralExpr) child).getStringValue(); - if (!dynamicUnit.equalsIgnoreCase(autoUnit)) { - throw new AnalysisException( - "If support auto partition and dynamic partition at same time, " - + "they must have the same interval unit."); - } - } - } - } + throw new AnalysisException( + "Can't use Dynamic Partition and Auto Partition at the same time"); } } catch (AnalysisException e) { throw new DdlException(e.getMessage()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java index fcf35b739c4..dee77d5ad85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java @@ -140,8 +140,10 @@ public class PartitionTableInfo { throw new AnalysisException( "The partition column must be NOT NULL with allow_partition_column_nullable OFF"); } - if (isAutoPartition && partitionType.equalsIgnoreCase(PartitionType.RANGE.name()) && column.isNullable()) { - throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column"); + if (partitionType.equalsIgnoreCase(PartitionType.RANGE.name()) && isAutoPartition) { + if (column.isNullable()) { + throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column"); + } } } diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy index e3300266ada..13c96cd48bc 100644 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy @@ -240,6 +240,16 @@ suite("test_auto_partition_behavior") { exception "AUTO RANGE PARTITION doesn't support NULL column" } + sql "drop table if exists test_dynamic" + sql """ + create table test_dynamic( + k0 DATE not null + ) + auto partition by range (date_trunc(k0, 'year')) () + DISTRIBUTED BY HASH(`k0`) BUCKETS auto + properties("replication_num" = "1"); + """ + // PROHIBIT different timeunit of interval when use both auto & dynamic partition sql "set experimental_enable_nereids_planner=true;" test{ @@ -263,7 +273,19 @@ suite("test_auto_partition_behavior") { "dynamic_partition.buckets" = "8" ); """ - exception "If support auto partition and dynamic partition at same time, they must have the same interval unit." + exception "Can't use Dynamic Partition and Auto Partition at the same time" + } + test { + sql """ + ALTER TABLE test_dynamic set ( + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "32" + ); + """ + exception "Can't use Dynamic Partition and Auto Partition at the same time" } sql "set experimental_enable_nereids_planner=false;" @@ -288,7 +310,19 @@ suite("test_auto_partition_behavior") { "dynamic_partition.buckets" = "8" ); """ - exception "If support auto partition and dynamic partition at same time, they must have the same interval unit." + exception "Can't use Dynamic Partition and Auto Partition at the same time" + } + test { + sql """ + ALTER TABLE test_dynamic set ( + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "32" + ); + """ + exception "Can't use Dynamic Partition and Auto Partition at the same time" } // prohibit too long value for partition column @@ -359,4 +393,19 @@ suite("test_auto_partition_behavior") { """ exception "partition expr date_trunc is illegal!" } + sql "set experimental_enable_nereids_planner=false;" + test{ + sql """ + create table illegal( + k0 datetime(6) NOT null, + k1 int NOT null + ) + auto partition by range (date_trunc(k1, 'hour')) + ( + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS 2 + properties("replication_num" = "1"); + """ + exception "Auto range partition needs Date/DateV2/Datetime/DatetimeV2 column as partition column" + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
