This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 03d9f6d [Feature] support hour time unit with dynamic parition (#4514)
03d9f6d is described below
commit 03d9f6d8b4f8bdcd5c8b920e27c052d4e91a290c
Author: caoyang10 <[email protected]>
AuthorDate: Sun Sep 6 20:25:27 2020 +0800
[Feature] support hour time unit with dynamic parition (#4514)
Many tables are so large that need seperate partitions with "HOUR" time
unit.
But now dynamic partition doesn't support "HOUR" time unit and it was
marked as "TODO".
So I support the feature and it works.
---
docs/en/administrator-guide/dynamic-partition.md | 4 ++-
.../zh-CN/administrator-guide/dynamic-partition.md | 4 ++-
.../java/org/apache/doris/common/ErrorCode.java | 2 +-
.../doris/common/util/DynamicPartitionUtil.java | 30 ++++++++++++++++++++--
4 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/docs/en/administrator-guide/dynamic-partition.md
b/docs/en/administrator-guide/dynamic-partition.md
index 078eece..0e989ca 100644
--- a/docs/en/administrator-guide/dynamic-partition.md
+++ b/docs/en/administrator-guide/dynamic-partition.md
@@ -81,7 +81,9 @@ The rules of dynamic partition are prefixed with
`dynamic_partition.`:
* `dynamic_partition.time_unit`
- The unit for dynamic partition scheduling. Can be specified as `DAY`,`
WEEK`, and `MONTH`, means to create or delete partitions by day, week, and
month, respectively.
+ The unit for dynamic partition scheduling. Can be specified as
`HOUR`,`DAY`,` WEEK`, and `MONTH`, means to create or delete partitions by
hour, day, week, and month, respectively.
+
+ When specified as `HOUR`, the suffix format of the dynamically created
partition name is `yyyyMMddHH`, for example, `2020032501`.
When specified as `DAY`, the suffix format of the dynamically created
partition name is `yyyyMMdd`, for example, `20200325`.
diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md
b/docs/zh-CN/administrator-guide/dynamic-partition.md
index 167e137..7f77cdf 100644
--- a/docs/zh-CN/administrator-guide/dynamic-partition.md
+++ b/docs/zh-CN/administrator-guide/dynamic-partition.md
@@ -79,8 +79,10 @@ under the License.
* `dynamic_partition.time_unit`
- 动态分区调度的单位。可指定为 `DAY`、`WEEK`、`MONTH`。分别表示按天、按星期、按月进行分区创建或删除。
+ 动态分区调度的单位。可指定为 `HOUR`、`DAY`、`WEEK`、`MONTH`。分别表示按天、按星期、按月进行分区创建或删除。
+ 当指定为 `HOUR` 时，动态创建的分区名后缀格式为 `yyyyMMddHH`，例如`2020032501`。
+
当指定为 `DAY` 时，动态创建的分区名后缀格式为 `yyyyMMdd`，例如`20200325`。
当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`。即当前日期属于这一年的第几周，例如 `2020-03-25`
创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
index 186899c..75ae600 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
@@ -210,7 +210,7 @@ public enum ErrorCode {
"Table %s is not a colocated table"),
ERR_INVALID_OPERATION(5065, new byte[] { '4', '2', '0', '0', '0' },
"Operation %s is invalid"),
ERROR_DYNAMIC_PARTITION_TIME_UNIT(5065, new byte[] {'4', '2', '0', '0',
'0'},
- "Unsupported time unit %s. Expect DAY/WEEK/MONTH."),
+ "Unsupported time unit %s. Expect HOUR/DAY/WEEK/MONTH."),
ERROR_DYNAMIC_PARTITION_START_ZERO(5066, new byte[] {'4', '2', '0', '0',
'0'},
"Dynamic partition start must less than 0"),
ERROR_DYNAMIC_PARTITION_START_FORMAT(5066, new byte[] {'4', '2', '0', '0',
'0'},
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index c83eac0..5e68161 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -63,6 +63,7 @@ public class DynamicPartitionUtil {
public static void checkTimeUnit(String timeUnit) throws DdlException {
if (Strings.isNullOrEmpty(timeUnit)
|| !(timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())
+ || timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())
|| timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())
|| timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString()))) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_TIME_UNIT,
timeUnit);
@@ -365,6 +366,8 @@ public class DynamicPartitionUtil {
return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
return formattedDateStr.substring(0, 6);
+ } else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
+ return formattedDateStr.substring(0, 10);
} else {
formattedDateStr = formattedDateStr.substring(0, 8);
Calendar calendar = Calendar.getInstance(tz);
@@ -385,7 +388,8 @@ public class DynamicPartitionUtil {
}
// return the partition range date string formatted as yyyy-MM-dd[
HH:mm::ss]
- // TODO: support HOUR and YEAR
+ // add support: HOUR by caoyang10
+ // TODO: support YEAR
public static String getPartitionRangeString(DynamicPartitionProperty
property, ZonedDateTime current,
int offset, String format) {
String timeUnit = property.getTimeUnit();
@@ -393,10 +397,27 @@ public class DynamicPartitionUtil {
return getPartitionRangeOfDay(current, offset, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
return getPartitionRangeOfWeek(current, offset,
property.getStartOfWeek(), format);
- } else { // MONTH
+ } else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { //
MONTH
+ return getPartitionRangeOfHour(current, offset, format);
+ } else {
return getPartitionRangeOfMonth(current, offset,
property.getStartOfMonth(), format);
}
}
+
+ /**
+ * return formatted string of partition range in HOUR granularity.
+ * offset: The offset from the current hour. 0 means current hour, 1 means
next hour, -1 last hour.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24 00:12:34, offset = -1
+ * It will return 2020-05-23 23:00:00
+ * Today is 2020-05-24 00, offset = 1
+ * It will return 2020-05-24 01:00:00
+ */
+ private static String getPartitionRangeOfHour(ZonedDateTime current, int
offset, String format) {
+ return getFormattedTimeWithoutMinuteSecond(current.plusHours(offset),
format);
+ }
/**
* return formatted string of partition range in DAY granularity.
@@ -460,6 +481,11 @@ public class DynamicPartitionUtil {
return
DateTimeFormatter.ofPattern(format).format(timeWithoutHourMinuteSecond);
}
+ private static String getFormattedTimeWithoutMinuteSecond(ZonedDateTime
zonedDateTime, String format) {
+ ZonedDateTime timeWithoutMinuteSecond =
zonedDateTime.withMinute(0).withSecond(0);
+ return
DateTimeFormatter.ofPattern(format).format(timeWithoutMinuteSecond);
+ }
+
/**
* Used to indicate the start date.
* Taking the year as the granularity, it can indicate the month and day
as the start date.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
