This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9983feddcbb7b91effc25b3a7845593c6443b469 Author: Mingyu Chen <[email protected]> AuthorDate: Tue Feb 14 15:14:52 2023 +0800 [improvement](dynamic-partition) add storage_medium property for dynamic partition (#16648) --- .../docs/advanced/partition/dynamic-partition.md | 7 +++++ .../docs/advanced/partition/dynamic-partition.md | 8 ++++++ .../doris/catalog/DynamicPartitionProperty.java | 11 +++++++- .../doris/clone/DynamicPartitionScheduler.java | 32 ++++++++++++++++------ .../doris/common/util/DynamicPartitionUtil.java | 17 ++++++++++++ 5 files changed, 66 insertions(+), 9 deletions(-) diff --git a/docs/en/docs/advanced/partition/dynamic-partition.md b/docs/en/docs/advanced/partition/dynamic-partition.md index ce6bc02ba8..92546f4d95 100644 --- a/docs/en/docs/advanced/partition/dynamic-partition.md +++ b/docs/en/docs/advanced/partition/dynamic-partition.md @@ -181,6 +181,13 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`: Otherwise, every `[...,...]` in `reserved_history_periods` is a couple of properties, and they should be set at the same time. And the first date can't be larger than the second one. +- `dynamic_partition.storage_medium` + + <version since="dev"></version> + + Specifies the default storage medium for the created dynamic partition. HDD is the default, SSD can be selected. + + Note that when set to SSD, the `hot_partition_num` property will no longer take effect, all partitions will default to SSD storage media and the cooldown time will be 9999-12-31 23:59:59. #### Create History Partition Rules diff --git a/docs/zh-CN/docs/advanced/partition/dynamic-partition.md b/docs/zh-CN/docs/advanced/partition/dynamic-partition.md index 5db9a504b7..1832544447 100644 --- a/docs/zh-CN/docs/advanced/partition/dynamic-partition.md +++ b/docs/zh-CN/docs/advanced/partition/dynamic-partition.md @@ -173,6 +173,14 @@ under the License. 这两个时间段的分区。其中，`reserved_history_periods` 的每一个 `[...,...]` 是一对设置项，两者需要同时被设置，且第一个时间不能大于第二个时间。 +- `dynamic_partition.storage_medium` + + <version since="dev"></version> + + 指定创建的动态分区的默认存储介质。默认是 HDD，可选择 SSD。 + + 注意，当设置为SSD时，`hot_partition_num` 属性将不再生效，所有分区将默认为 SSD 存储介质并且冷却时间为 9999-12-31 23:59:59。 + #### 创建历史分区规则 当 `create_history_partition` 为 `true`，即开启创建历史分区功能时，Doris 会根据 `dynamic_partition.start` 和 `dynamic_partition.history_partition_num` 来决定创建历史分区的个数。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index 88446394e0..8f5e9d359f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -19,6 +19,7 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.DynamicPartitionUtil; @@ -47,6 +48,7 @@ public class DynamicPartitionProperty { public static final String HOT_PARTITION_NUM = "dynamic_partition.hot_partition_num"; public static final String RESERVED_HISTORY_PERIODS = "dynamic_partition.reserved_history_periods"; public static final String STORAGE_POLICY = "dynamic_partition.storage_policy"; + public static final String STORAGE_MEDIUM = "dynamic_partition.storage_medium"; public static final int MIN_START_OFFSET = Integer.MIN_VALUE; public static final int MAX_END_OFFSET = Integer.MAX_VALUE; @@ -74,6 +76,7 @@ public class DynamicPartitionProperty { private int hotPartitionNum; private String reservedHistoryPeriods; private String storagePolicy; + private String storageMedium; // ssd or hdd public DynamicPartitionProperty(Map<String, String> properties) { if (properties != null && !properties.isEmpty()) { @@ -94,6 +97,7 @@ public class DynamicPartitionProperty { this.reservedHistoryPeriods = properties.getOrDefault( RESERVED_HISTORY_PERIODS, NOT_SET_RESERVED_HISTORY_PERIODS); this.storagePolicy = properties.getOrDefault(STORAGE_POLICY, ""); + this.storageMedium = properties.getOrDefault(STORAGE_MEDIUM, Config.default_storage_medium); createStartOfs(properties); } else { this.exist = false; @@ -177,6 +181,10 @@ public class DynamicPartitionProperty { return storagePolicy; } + public String getStorageMedium() { + return storageMedium; + } + public String getStartOfInfo() { if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) { return startOfWeek.toDisplayInfo(); @@ -220,7 +228,8 @@ public class DynamicPartitionProperty { + ",\n\"" + HISTORY_PARTITION_NUM + "\" = \"" + historyPartitionNum + "\"" + ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum + "\"" + ",\n\"" + RESERVED_HISTORY_PERIODS + "\" = \"" + reservedHistoryPeriods + "\"" - + ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\""; + + ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\"" + + ",\n\"" + STORAGE_MEDIUM + "\" = \"" + storageMedium + "\""; if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) { res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\""; } else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 3cbbe27c17..7c20455742 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -26,6 +26,7 @@ import org.apache.doris.analysis.PartitionValue; import org.apache.doris.analysis.RandomDistributionDesc; import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DataProperty; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.DynamicPartitionProperty; @@ -298,10 +299,8 @@ public class DynamicPartitionScheduler extends MasterDaemon { dynamicPartitionProperty.getReplicaAllocation().toCreateStmt()); } - if (hotPartitionNum > 0) { - // set storage_medium and storage_cooldown_time based on dynamic_partition.hot_partition_num - setStorageMediumProperty(partitionProperties, dynamicPartitionProperty, now, hotPartitionNum, idx); - } + // set storage_medium and storage_cooldown_time based on dynamic_partition.hot_partition_num + setStorageMediumProperty(partitionProperties, dynamicPartitionProperty, now, hotPartitionNum, idx); if (StringUtils.isNotEmpty(storagePolicyName)) { setStoragePolicyProperty(partitionProperties, dynamicPartitionProperty, now, idx, storagePolicyName); @@ -309,7 +308,7 @@ public class DynamicPartitionScheduler extends MasterDaemon { String partitionName = dynamicPartitionProperty.getPrefix() + DynamicPartitionUtil.getFormattedPartitionName(dynamicPartitionProperty.getTimeZone(), - prevBorder, dynamicPartitionProperty.getTimeUnit()); + prevBorder, dynamicPartitionProperty.getTimeUnit()); SinglePartitionDesc rangePartitionDesc = new SinglePartitionDesc(true, partitionName, partitionKeyDesc, partitionProperties); @@ -332,14 +331,31 @@ public class DynamicPartitionScheduler extends MasterDaemon { return addPartitionClauses; } + /** + * If dynamic_partition.storage_medium is set to SSD, + * ignore hot_partition_num property and set to (SSD, 9999-12-31 23:59:59) + * Else, if hot partition num is set, set storage medium to SSD due to time. + * + * @param partitionProperties + * @param property + * @param now + * @param hotPartitionNum + * @param offset + */ private void setStorageMediumProperty(HashMap<String, String> partitionProperties, DynamicPartitionProperty property, ZonedDateTime now, int hotPartitionNum, int offset) { - if (offset + hotPartitionNum <= 0) { + if ((hotPartitionNum <= 0 || offset + hotPartitionNum <= 0) && !property.getStorageMedium() + .equalsIgnoreCase("ssd")) { return; } + String cooldownTime; + if (property.getStorageMedium().equalsIgnoreCase("ssd")) { + cooldownTime = TimeUtils.longToTimeString(DataProperty.MAX_COOLDOWN_TIME_MS); + } else { + cooldownTime = DynamicPartitionUtil.getPartitionRangeString( + property, now, offset + hotPartitionNum, DynamicPartitionUtil.DATETIME_FORMAT); + } partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, TStorageMedium.SSD.name()); - String cooldownTime = DynamicPartitionUtil.getPartitionRangeString( - property, now, offset + hotPartitionNum, DynamicPartitionUtil.DATETIME_FORMAT); partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME, cooldownTime); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 60247b4ec7..b51aeaf9bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -39,6 +39,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; import org.apache.doris.policy.StoragePolicy; +import org.apache.doris.thrift.TStorageMedium; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -365,6 +366,14 @@ public class DynamicPartitionUtil { } } + private static void checkStorageMedium(String storageMedium) throws DdlException { + try { + TStorageMedium.valueOf(storageMedium.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new DdlException("invalid storage medium: " + storageMedium + ". Should be SSD or HDD"); + } + } + private static SimpleDateFormat getSimpleDateFormat(String timeUnit) { if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { return new SimpleDateFormat(DATETIME_FORMAT); @@ -624,6 +633,14 @@ public class DynamicPartitionUtil { analyzedProperties.put(DynamicPartitionProperty.STORAGE_POLICY, remoteStoragePolicy); } } + if (properties.containsKey(DynamicPartitionProperty.STORAGE_MEDIUM)) { + String storageMedium = properties.get(DynamicPartitionProperty.STORAGE_MEDIUM); + checkStorageMedium(storageMedium); + properties.remove(DynamicPartitionProperty.STORAGE_MEDIUM); + if (!Strings.isNullOrEmpty(storageMedium)) { + analyzedProperties.put(DynamicPartitionProperty.STORAGE_MEDIUM, storageMedium); + } + } return analyzedProperties; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
