This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 2b8b4ad1b8a [branch-2.0](dynamic partition) fix dynamic partition
storage medium not working #29490 (#29806)
2b8b4ad1b8a is described below
commit 2b8b4ad1b8a9c2bdbb8eaeffd29380097dd332c8
Author: yujun <[email protected]>
AuthorDate: Wed Jan 10 22:25:00 2024 +0800
[branch-2.0](dynamic partition) fix dynamic partition storage medium not
working #29490 (#29806)
---
.../doris/catalog/DynamicPartitionProperty.java | 11 +++++---
.../doris/clone/DynamicPartitionScheduler.java | 30 ++++++++++++++--------
2 files changed, 27 insertions(+), 14 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
index 8f5e9d359f9..54c49c1ee8d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
@@ -19,7 +19,6 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.DynamicPartitionUtil;
@@ -27,6 +26,8 @@ import
org.apache.doris.common.util.DynamicPartitionUtil.StartOfDate;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.TimeUtils;
+import com.google.common.base.Strings;
+
import java.util.Map;
import java.util.TimeZone;
@@ -97,7 +98,7 @@ public class DynamicPartitionProperty {
this.reservedHistoryPeriods = properties.getOrDefault(
RESERVED_HISTORY_PERIODS,
NOT_SET_RESERVED_HISTORY_PERIODS);
this.storagePolicy = properties.getOrDefault(STORAGE_POLICY, "");
- this.storageMedium = properties.getOrDefault(STORAGE_MEDIUM,
Config.default_storage_medium);
+ this.storageMedium = properties.getOrDefault(STORAGE_MEDIUM, "");
createStartOfs(properties);
} else {
this.exist = false;
@@ -228,8 +229,10 @@ public class DynamicPartitionProperty {
+ ",\n\"" + HISTORY_PARTITION_NUM + "\" = \"" +
historyPartitionNum + "\""
+ ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum +
"\""
+ ",\n\"" + RESERVED_HISTORY_PERIODS + "\" = \"" +
reservedHistoryPeriods + "\""
- + ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\""
- + ",\n\"" + STORAGE_MEDIUM + "\" = \"" + storageMedium + "\"";
+ + ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\"";
+ if (!Strings.isNullOrEmpty(storageMedium)) {
+ res += ",\n\"" + STORAGE_MEDIUM + "\" = \"" + storageMedium + "\"";
+ }
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" +
startOfWeek.dayOfWeek + "\"";
} else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index dc03ecf8233..565f4c066f8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -52,6 +52,7 @@ import org.apache.doris.common.util.RangeUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.thrift.TStorageMedium;
+import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
@@ -351,19 +352,28 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
*/
private void setStorageMediumProperty(HashMap<String, String>
partitionProperties,
DynamicPartitionProperty property, ZonedDateTime now, int
hotPartitionNum, int offset) {
- if ((hotPartitionNum <= 0 || offset + hotPartitionNum <= 0) &&
!property.getStorageMedium()
- .equalsIgnoreCase("ssd")) {
- return;
- }
- String cooldownTime;
- if (property.getStorageMedium().equalsIgnoreCase("ssd")) {
- cooldownTime =
TimeUtils.longToTimeString(DataProperty.MAX_COOLDOWN_TIME_MS);
+ // 1. no hot partition, then use dynamic_partition.storage_medium
+ // 2. has hot partition
+ // 1) dynamic_partition.storage_medium = 'ssd', then use ssd;
+ // 2) otherwise
+ // a. cooldown partition, then use hdd
+ // b. hot partition. then use ssd
+ if (hotPartitionNum <= 0 ||
property.getStorageMedium().equalsIgnoreCase("ssd")) {
+ if (!Strings.isNullOrEmpty(property.getStorageMedium())) {
+
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM,
property.getStorageMedium());
+
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
+
TimeUtils.longToTimeString(DataProperty.MAX_COOLDOWN_TIME_MS));
+ }
+ } else if (offset + hotPartitionNum <= 0) {
+
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM,
TStorageMedium.HDD.name());
+
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
+
TimeUtils.longToTimeString(DataProperty.MAX_COOLDOWN_TIME_MS));
} else {
- cooldownTime = DynamicPartitionUtil.getPartitionRangeString(
+ String cooldownTime = DynamicPartitionUtil.getPartitionRangeString(
property, now, offset + hotPartitionNum,
DynamicPartitionUtil.DATETIME_FORMAT);
+
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM,
TStorageMedium.SSD.name());
+
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
cooldownTime);
}
- partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM,
TStorageMedium.SSD.name());
-
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
cooldownTime);
}
private void setStoragePolicyProperty(HashMap<String, String>
partitionProperties,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
