This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit a2ea72eb6ea2750d5a7cc36f799f6910ef752f7d Author: Jack Drogon <[email protected]> AuthorDate: Sun Apr 2 15:51:21 2023 +0800 [improvement](dynamic partition) Fix dynamic partition no bucket (#18300) Signed-off-by: Jack Drogon <[email protected]> --- .../apache/doris/alter/SchemaChangeHandler.java | 2 +- .../doris/common/util/DynamicPartitionUtil.java | 12 +++- .../doris/catalog/DynamicPartitionTableTest.java | 6 +- .../suites/autobucket/test_autobucket.groovy | 3 +- ...vy => test_autobucket_dynamic_partition.groovy} | 31 +++++---- .../test_dynamic_partition.groovy | 73 ++++++++++++++-------- 6 files changed, 81 insertions(+), 46 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index dafd366d25..0ba0a70ae5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1695,7 +1695,7 @@ public class SchemaChangeHandler extends AlterHandler { if (!olapTable.dynamicPartitionExists()) { try { DynamicPartitionUtil.checkInputDynamicPartitionProperties(properties, - olapTable.getPartitionInfo()); + olapTable); } catch (DdlException e) { // This table is not a dynamic partition table // and didn't supply all dynamic partition properties diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index b51aeaf9bc..b36932f691 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -20,6 +20,7 @@ package org.apache.doris.common.util; import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.DynamicPartitionProperty; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; @@ -398,10 +399,12 @@ public class DynamicPartitionUtil { // Check if all requried properties has been set. // And also check all optional properties, if not set, set them to default value. public static boolean checkInputDynamicPartitionProperties(Map<String, String> properties, - PartitionInfo partitionInfo) throws DdlException { + OlapTable olapTable) throws DdlException { if (properties == null || properties.isEmpty()) { return false; } + + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); if (partitionInfo.getType() != PartitionType.RANGE || partitionInfo.isMultiColumnPartition()) { throw new DdlException("Dynamic partition only support single-column range partition"); } @@ -442,7 +445,9 @@ public class DynamicPartitionUtil { throw new DdlException("Must assign dynamic_partition.end properties"); } if (Strings.isNullOrEmpty(buckets)) { - throw new DdlException("Must assign dynamic_partition.buckets properties"); + DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); + buckets = String.valueOf(distributionInfo.getBucketNum()); + properties.put(DynamicPartitionProperty.BUCKETS, buckets); } if (Strings.isNullOrEmpty(timeZone)) { properties.put(DynamicPartitionProperty.TIME_ZONE, TimeUtils.getSystemTimeZone().getID()); @@ -505,6 +510,7 @@ public class DynamicPartitionUtil { properties.remove(DynamicPartitionProperty.BUCKETS); analyzedProperties.put(DynamicPartitionProperty.BUCKETS, bucketsValue); } + if (properties.containsKey(DynamicPartitionProperty.ENABLE)) { String enableValue = properties.get(DynamicPartitionProperty.ENABLE); checkEnable(enableValue); @@ -674,7 +680,7 @@ public class DynamicPartitionUtil { */ public static void checkAndSetDynamicPartitionProperty(OlapTable olapTable, Map<String, String> properties, Database db) throws UserException { - if (DynamicPartitionUtil.checkInputDynamicPartitionProperties(properties, olapTable.getPartitionInfo())) { + if (DynamicPartitionUtil.checkInputDynamicPartitionProperties(properties, olapTable)) { Map<String, String> dynamicPartitionProperties = DynamicPartitionUtil.analyzeDynamicPartition(properties, olapTable, db); TableProperty tableProperty = olapTable.getTableProperty(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 3a522ad3a6..ff6d1f687d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -258,7 +258,7 @@ public class DynamicPartitionTableTest { @Test public void testMissBuckets() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n" + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_miss_buckets` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" @@ -282,14 +282,12 @@ public class DynamicPartitionTableTest { + "\"dynamic_partition.time_unit\" = \"day\",\n" + "\"dynamic_partition.prefix\" = \"p\"\n" + ");"; - expectedException.expect(DdlException.class); - expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.buckets properties"); createTable(createOlapTblStmt); } @Test public void testNotAllowed() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n" + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_not_allowed` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" diff --git a/regression-test/suites/autobucket/test_autobucket.groovy b/regression-test/suites/autobucket/test_autobucket.groovy index 29945e0f9a..ab0ae99658 100644 --- a/regression-test/suites/autobucket/test_autobucket.groovy +++ b/regression-test/suites/autobucket/test_autobucket.groovy @@ -25,7 +25,7 @@ suite("test_autobucket") { COMMENT 'OLAP' DISTRIBUTED BY HASH(`user_id`) BUCKETS AUTO PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_allocation" = "tag.location.default: 1" ) """ @@ -35,6 +35,7 @@ suite("test_autobucket") { result = sql "show partitions from autobucket_test" logger.info("${result}") // XXX: buckets at pos(8), next maybe impl by sql meta + // 10 is the default buckets without partition size assertEquals(Integer.valueOf(result.get(0).get(8)), 10) sql "drop table if exists autobucket_test" diff --git a/regression-test/suites/autobucket/test_autobucket.groovy b/regression-test/suites/autobucket/test_autobucket_dynamic_partition.groovy similarity index 52% copy from regression-test/suites/autobucket/test_autobucket.groovy copy to regression-test/suites/autobucket/test_autobucket_dynamic_partition.groovy index 29945e0f9a..80679cb07d 100644 --- a/regression-test/suites/autobucket/test_autobucket.groovy +++ b/regression-test/suites/autobucket/test_autobucket_dynamic_partition.groovy @@ -15,27 +15,34 @@ // specific language governing permissions and limitations // under the License. -suite("test_autobucket") { - sql "drop table if exists autobucket_test" +suite("test_autobucket_dynamic_partition") { + sql "drop table if exists test_autobucket_dynamic_partition" result = sql """ - CREATE TABLE `autobucket_test` ( - `user_id` largeint(40) NOT NULL - ) ENGINE=OLAP - DUPLICATE KEY(`user_id`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`user_id`) BUCKETS AUTO + CREATE TABLE + test_autobucket_dynamic_partition (k1 DATETIME) + PARTITION BY + RANGE (k1) () DISTRIBUTED BY HASH (k1) BUCKETS AUTO PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "WEEK", + "dynamic_partition.start" = "-2", + "dynamic_partition.end" = "2", + "dynamic_partition.prefix" = "p", + "replication_allocation" = "tag.location.default: 1" ) """ - result = sql "show create table autobucket_test" + result = sql "show create table test_autobucket_dynamic_partition" assertTrue(result.toString().containsIgnoreCase("BUCKETS AUTO")) - result = sql "show partitions from autobucket_test" + result = sql "show partitions from test_autobucket_dynamic_partition" logger.info("${result}") // XXX: buckets at pos(8), next maybe impl by sql meta + // 10 is the default buckets without partition size + assertEquals(result.size(), 3) assertEquals(Integer.valueOf(result.get(0).get(8)), 10) + assertEquals(Integer.valueOf(result.get(1).get(8)), 10) + assertEquals(Integer.valueOf(result.get(2).get(8)), 10) - sql "drop table if exists autobucket_test" + sql "drop table if exists test_autobucket_dynamic_partition" } diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy index f80de41841..16246bf629 100644 --- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy +++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy @@ -14,44 +14,68 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - suite("test_dynamic_partition") { // todo: test dynamic partition sql "drop table if exists dy_par" sql """ - CREATE TABLE IF NOT EXISTS dy_par ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL ) - AGGREGATE KEY(k1,k2) - PARTITION BY RANGE(k1) ( ) - DISTRIBUTED BY HASH(k1) BUCKETS 3 - PROPERTIES ( - "dynamic_partition.enable"="true", - "dynamic_partition.end"="3", - "dynamic_partition.buckets"="10", - "dynamic_partition.start"="-3", - "dynamic_partition.prefix"="p", - "dynamic_partition.time_unit"="DAY", + CREATE TABLE IF NOT EXISTS dy_par ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL ) + AGGREGATE KEY(k1,k2) + PARTITION BY RANGE(k1) ( ) + DISTRIBUTED BY HASH(k1) BUCKETS 3 + PROPERTIES ( + "dynamic_partition.enable"="true", + "dynamic_partition.end"="3", + "dynamic_partition.buckets"="10", + "dynamic_partition.start"="-3", + "dynamic_partition.prefix"="p", + "dynamic_partition.time_unit"="DAY", "dynamic_partition.create_history_partition"="true", "dynamic_partition.replication_allocation" = "tag.location.default: 1") """ List<List<Object>> result = sql "show tables like 'dy_par'" logger.info("${result}") assertEquals(result.size(), 1) + result = sql "show partitions from dy_par" + // XXX: buckets at pos(8), next maybe impl by sql meta + assertEquals(Integer.valueOf(result.get(0).get(8)), 10) sql "drop table dy_par" - // + sql "drop table if exists dy_par_bucket_set_by_distribution" + sql """ + CREATE TABLE IF NOT EXISTS dy_par_bucket_set_by_distribution + ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL ) + AGGREGATE KEY(k1,k2) + PARTITION BY RANGE(k1) ( ) + DISTRIBUTED BY HASH(k1) BUCKETS 3 + PROPERTIES ( + "dynamic_partition.enable"="true", + "dynamic_partition.end"="3", + "dynamic_partition.start"="-3", + "dynamic_partition.prefix"="p", + "dynamic_partition.time_unit"="DAY", + "dynamic_partition.create_history_partition"="true", + "dynamic_partition.replication_allocation" = "tag.location.default: 1") + """ + result = sql "show tables like 'dy_par_bucket_set_by_distribution'" + logger.info("${result}") + assertEquals(result.size(), 1) + result = sql "show partitions from dy_par_bucket_set_by_distribution" + // XXX: buckets at pos(8), next maybe impl by sql meta + assertEquals(Integer.valueOf(result.get(0).get(8)), 3) + sql "drop table dy_par_bucket_set_by_distribution" sql "drop table if exists dy_par_bad" test { sql """ - CREATE TABLE IF NOT EXISTS dy_par_bad ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL ) - AGGREGATE KEY(k1,k2) - PARTITION BY RANGE(k1) ( ) - DISTRIBUTED BY HASH(k1) BUCKETS 3 - PROPERTIES ( - "dynamic_partition.enable"="true", - "dynamic_partition.end"="3", - "dynamic_partition.buckets"="10", - "dynamic_partition.start"="-3", - "dynamic_partition.prefix"="p", - "dynamic_partition.time_unit"="DAY", + CREATE TABLE IF NOT EXISTS dy_par_bad ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL ) + AGGREGATE KEY(k1,k2) + PARTITION BY RANGE(k1) ( ) + DISTRIBUTED BY HASH(k1) BUCKETS 3 + PROPERTIES ( + "dynamic_partition.enable"="true", + "dynamic_partition.end"="3", + "dynamic_partition.buckets"="10", + "dynamic_partition.start"="-3", + "dynamic_partition.prefix"="p", + "dynamic_partition.time_unit"="DAY", "dynamic_partition.create_history_partition"="true", "dynamic_partition.replication_allocation" = "tag.location.not_exist_tag: 1") """ @@ -59,7 +83,6 @@ suite("test_dynamic_partition") { exception "errCode = 2," } sql "drop table if exists dy_par_bad" - sql """ CREATE TABLE IF NOT EXISTS dy_par ( k1 datev2 NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL ) AGGREGATE KEY(k1,k2) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
