This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 11cce06 [Feature] Support create history dynamic partition (#5703)
11cce06 is described below
commit 11cce0696259ef5b290fe21cedae3144c406f600
Author: Mingyu Chen <[email protected]>
AuthorDate: Sat May 8 12:05:19 2021 +0800
[Feature] Support create history dynamic partition (#5703)
1. Add a new dynamic partition property `create_history_partition`.
If set to true, Doris will create all partitions from `start` to `end`.
2. Add a new FE config `max_dynamic_partition_num`
To limit the number of partitions created when creating one table.
---
be/test/exec/tablet_sink_test.cpp | 6 +-
docs/en/administrator-guide/config/fe_config.md | 5 +
docs/en/administrator-guide/dynamic-partition.md | 6 +
docs/zh-CN/administrator-guide/config/fe_config.md | 5 +
.../zh-CN/administrator-guide/dynamic-partition.md | 12 +-
fe/fe-core/pom.xml | 1 +
.../doris/catalog/DynamicPartitionProperty.java | 11 +-
.../doris/clone/DynamicPartitionScheduler.java | 16 +-
.../main/java/org/apache/doris/common/Config.java | 8 +
.../java/org/apache/doris/common/ErrorCode.java | 4 +-
.../doris/common/util/DynamicPartitionUtil.java | 84 ++++--
.../doris/catalog/DynamicPartitionTableTest.java | 291 ++++++++++++++++++++-
12 files changed, 412 insertions(+), 37 deletions(-)
diff --git a/be/test/exec/tablet_sink_test.cpp
b/be/test/exec/tablet_sink_test.cpp
index 7b4e112..f973b04 100644
--- a/be/test/exec/tablet_sink_test.cpp
+++ b/be/test/exec/tablet_sink_test.cpp
@@ -453,7 +453,7 @@ TEST_F(OlapTableSinkTest, normal) {
ASSERT_TRUE(st.ok());
// close
st = sink.close(&state, Status::OK());
- ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: already stopped,
skip waiting for close. cancelled/!eos: : 1/1") << st.to_string();
+ ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: wait close
failed. ") << st.to_string();
// each node has a eof
ASSERT_EQ(2, service->_eof_counters);
@@ -586,7 +586,7 @@ TEST_F(OlapTableSinkTest, convert) {
ASSERT_TRUE(st.ok());
// close
st = sink.close(&state, Status::OK());
- ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: already stopped,
skip waiting for close. cancelled/!eos: : 1/1") << st.to_string();
+ ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: wait close
failed. ") << st.to_string();
// each node has a eof
ASSERT_EQ(2, service->_eof_counters);
@@ -966,7 +966,7 @@ TEST_F(OlapTableSinkTest, decimal) {
ASSERT_TRUE(st.ok());
// close
st = sink.close(&state, Status::OK());
- ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: already stopped,
skip waiting for close. cancelled/!eos: : 1/1") << st.to_string();
+ ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: wait close
failed. ") << st.to_string();
ASSERT_EQ(2, output_set.size());
ASSERT_TRUE(output_set.count("[(12 12.3)]") > 0);
diff --git a/docs/en/administrator-guide/config/fe_config.md
b/docs/en/administrator-guide/config/fe_config.md
index f976f5c..2139fe4 100644
--- a/docs/en/administrator-guide/config/fe_config.md
+++ b/docs/en/administrator-guide/config/fe_config.md
@@ -789,3 +789,8 @@ And the new UI will also try to get this base path first to
assemble the URL.
Only valid when `enable_http_server_v2` is true.
The default is empty, that is, not set.
+
+### `max_dynamic_partition_num`
+
+When creating a dynamic partition table, the maximum number of partitions
allowed to be automatically created. To prevent creating too many partitions at
once.
+The default is 500.
diff --git a/docs/en/administrator-guide/dynamic-partition.md
b/docs/en/administrator-guide/dynamic-partition.md
index 09f3a27..a0afe4e 100644
--- a/docs/en/administrator-guide/dynamic-partition.md
+++ b/docs/en/administrator-guide/dynamic-partition.md
@@ -125,6 +125,12 @@ The rules of dynamic partition are prefixed with
`dynamic_partition.`:
When `time_unit` is` MONTH`, this parameter is used to specify the start
date of each month. The value ranges from 1 to 28. 1 means the 1st of every
month, and 28 means the 28th of every month. The default is 1, which means that
every month starts at 1st. The 29, 30 and 31 are not supported at the moment to
avoid ambiguity caused by lunar years or months.
+* `dynamic_partition.create_history_partition`
+
+ The default is false. When set to true, Doris will automatically create
all partitions from start to end. At the same time, the parameter
`max_dynamic_partition_num` of FE will limit the total number of partitions to
avoid creating too many partitions at once. If (end - start) is larger than
`max_dynamic_partition_num`, the operation will fail.
+
+ When the `start` attribute is not specified, this parameter has no effect.
+
### Notice
If some partitions between `dynamic_partition.start` and
`dynamic_partition.end` are lost due to some unexpected circumstances when
using dynamic partition, the lost partitions between the current time and
`dynamic_partition.end` will be recreated, but the lost partitions between
`dynamic_partition.start` and the current time will not be recreated.
diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md
b/docs/zh-CN/administrator-guide/config/fe_config.md
index 64836ce..d7665b5 100644
--- a/docs/zh-CN/administrator-guide/config/fe_config.md
+++ b/docs/zh-CN/administrator-guide/config/fe_config.md
@@ -794,3 +794,8 @@ thrift_client_timeout_ms 的值被设置为大于0来避免线程卡在java.net.
仅在 `enable_http_server_v2` 为 true 的情况下才有效。
默认为空，即不设置。
+
+### `max_dynamic_partition_num`
+
+在创建动态分区表时，允许自动创建的最大分区数量。以防止一次性创建过多的分区。
+默认为 500。
diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md
b/docs/zh-CN/administrator-guide/dynamic-partition.md
index dfbc085..ddbd55f 100644
--- a/docs/zh-CN/administrator-guide/dynamic-partition.md
+++ b/docs/zh-CN/administrator-guide/dynamic-partition.md
@@ -110,7 +110,7 @@ under the License.
* `dynamic_partition.buckets`
动态创建的分区所对应的分桶数量。
-
+
* `dynamic_partition.replication_num`
动态创建的分区所对应的副本数量，如果不填写，则默认为该表创建时指定的副本数量。
@@ -122,6 +122,12 @@ under the License.
* `dynamic_partition.start_day_of_month`
当 `time_unit` 为 `MONTH` 时，该参数用于指定每月的起始日期。取值为 1 到 28。其中 1 表示每月1号，28
表示每月28号。默认为 1，即表示每月以1号位起始点。暂不支持以29、30、31号为起始日，以避免因闰年或闰月带来的歧义。
+
+* `dynamic_partition.create_history_partition`
+
+ 默认为 false。当置为 true 时，Doris 会自动创建由 start 到 end 的所有分区。同时，FE 的参数
`max_dynamic_partition_num` 会限制总分区数量，以避免一次性创建过多分区。当 end - start 的值大于
`max_dynamic_partition_num` 值时，操作将被禁止。
+
+ 当不指定 `start` 属性时，该参数不生效。
### 注意事项
@@ -346,6 +352,6 @@ mysql> SHOW DYNAMIC PARTITION TABLES;
1. 创建动态分区表后提示 ```Could not create table with dynamic partition when fe config
dynamic_partition_enable is false```
- 由于动态分区的总开关，也就是 FE 的配置 ```dynamic_partition_enable``` 为
false，导致无法创建动态分区表。
+ 由于动态分区的总开关，也就是 FE 的配置 ```dynamic_partition_enable``` 为 false，导致无法创建动态分区表。
- 这时候请修改 FE 的配置文件，增加一行 ```dynamic_partition_enable=true```，并重启 FE。或者执行命令
ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true") 将动态分区开关打开即可。
+ 这时候请修改 FE 的配置文件，增加一行 ```dynamic_partition_enable=true```，并重启 FE。或者执行命令
ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true") 将动态分区开关打开即可。
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 0711f55..7c21c9c 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -737,6 +737,7 @@ under the License.
<sources>
<!-- add arbitrary num of src dirs here -->
<source>${basedir}/target/generated-sources/build/</source>
+
<source>${basedir}/target/generated-sources/</source>
</sources>
</configuration>
</execution>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
index 3092da6..4fa2319 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
@@ -36,8 +36,10 @@ public class DynamicPartitionProperty {
public static final String START_DAY_OF_MONTH =
"dynamic_partition.start_day_of_month";
public static final String TIME_ZONE = "dynamic_partition.time_zone";
public static final String REPLICATION_NUM =
"dynamic_partition.replication_num";
+ public static final String CREATE_HISTORY_PARTITION =
"dynamic_partition.create_history_partition";
public static final int MIN_START_OFFSET = Integer.MIN_VALUE;
+ public static final int MAX_END_OFFSET = Integer.MAX_VALUE;
public static final int NOT_SET_REPLICATION_NUM = -1;
private boolean exist;
@@ -52,6 +54,7 @@ public class DynamicPartitionProperty {
private StartOfDate startOfMonth;
private TimeZone tz = TimeUtils.getSystemTimeZone();
private int replicationNum;
+ private boolean createHistoryPartition = false;
public DynamicPartitionProperty(Map<String, String> properties) {
if (properties != null && !properties.isEmpty()) {
@@ -65,6 +68,7 @@ public class DynamicPartitionProperty {
this.prefix = properties.get(PREFIX);
this.buckets = Integer.parseInt(properties.get(BUCKETS));
this.replicationNum =
Integer.parseInt(properties.getOrDefault(REPLICATION_NUM,
String.valueOf(NOT_SET_REPLICATION_NUM)));
+ this.createHistoryPartition =
Boolean.parseBoolean(properties.get(CREATE_HISTORY_PARTITION));
createStartOfs(properties);
} else {
this.exist = false;
@@ -123,6 +127,10 @@ public class DynamicPartitionProperty {
return startOfMonth;
}
+ public boolean isCreateHistoryPartition() {
+ return createHistoryPartition;
+ }
+
public String getStartOfInfo() {
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
return startOfWeek.toDisplayInfo();
@@ -156,7 +164,8 @@ public class DynamicPartitionProperty {
",\n\"" + END + "\" = \"" + end + "\"" +
",\n\"" + PREFIX + "\" = \"" + prefix + "\"" +
",\n\"" + REPLICATION_NUM + "\" = \"" + useReplicationNum +
"\"" +
- ",\n\"" + BUCKETS + "\" = \"" + buckets + "\"";
+ ",\n\"" + BUCKETS + "\" = \"" + buckets + "\"" +
+ ",\n\"" + CREATE_HISTORY_PARTITION + "\" = \"" +
createHistoryPartition + "\"";
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" +
startOfWeek.dayOfWeek + "\"";
} else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 4855554..78e02cf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -45,14 +45,14 @@ import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.common.util.RangeUtils;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
@@ -142,9 +142,13 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
DynamicPartitionProperty dynamicPartitionProperty =
olapTable.getTableProperty().getDynamicPartitionProperty();
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo)
olapTable.getPartitionInfo();
ZonedDateTime now =
ZonedDateTime.now(dynamicPartitionProperty.getTimeZone().toZoneId());
- for (int i = 0; i <= dynamicPartitionProperty.getEnd(); i++) {
- String prevBorder =
DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, i,
partitionFormat);
- String nextBorder =
DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, i +
1, partitionFormat);
+
+ boolean createHistoryPartition =
dynamicPartitionProperty.isCreateHistoryPartition();
+ int idx = createHistoryPartition ? dynamicPartitionProperty.getStart()
: 0;
+
+ for (; idx <= dynamicPartitionProperty.getEnd(); idx++) {
+ String prevBorder =
DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now,
idx, partitionFormat);
+ String nextBorder =
DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, idx
+ 1, partitionFormat);
PartitionValue lowerValue = new PartitionValue(prevBorder);
PartitionValue upperValue = new PartitionValue(nextBorder);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index 9d177e7..f535c9c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -1379,4 +1379,12 @@ public class Config extends ConfigBase {
*/
@ConfField
public static int grpc_max_message_size_bytes = 1 * 1024 * 1024 * 1024; //
1GB
+
+ /**
+ * Used to limit the maximum number of partitions that can be created when
creating a dynamic partition table,
+ * to avoid creating too many partitions at one time.
+ * The number is determined by "start" and "end" in the dynamic partition
parameters.
+ */
+ @ConfField(mutable = true, masterOnly = true)
+ public static int max_dynamic_partition_num = 500;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
index 18d8ec1..71447c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
@@ -239,7 +239,9 @@ public enum ErrorCode {
ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_FORMAT(5072, new byte[] {'4', '2',
'0', '0', '0'},
"Invalid dynamic partition replication num: %s."),
ERROR_CREATE_TABLE_LIKE_EMPTY(5073, new byte[] {'4', '2', '0', '0', '0'},
- "Origin create table stmt is empty");
+ "Origin create table stmt is empty"),
+ ERROR_DYNAMIC_PARTITION_CREATE_HISTORY_PARTITION(5074, new byte[]{'4',
'2', '0', '0', '0'},
+ "Invalid dynamic partition create_history_partition: %s. Expected
true or false");
ErrorCode(int code, byte[] sqlState, String errorMsg) {
this.code = code;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index 81906d1..4ae5225 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -30,18 +30,19 @@ import org.apache.doris.catalog.RangePartitionInfo;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableProperty;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeNameFormat;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.DayOfWeek;
@@ -77,6 +78,12 @@ public class DynamicPartitionUtil {
ErrorReport.reportDdlException(DynamicPartitionProperty.TIME_UNIT
+ " could not be "
+ TimeUnit.HOUR.toString() + " when type of partition
column "
+ partitionColumn.getDisplayName() + " is " +
PrimitiveType.DATE.toString());
+ } else if
(PrimitiveType.getIntegerTypes().contains(partitionColumn.getDataType())
+ && timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
+ // The partition column's type is INT, not support HOUR
+ ErrorReport.reportDdlException(DynamicPartitionProperty.TIME_UNIT
+ " could not be "
+ + TimeUnit.HOUR.toString() + " when type of partition
column "
+ + partitionColumn.getDisplayName() + " is Integer");
}
}
@@ -88,27 +95,33 @@ public class DynamicPartitionUtil {
}
}
- private static void checkStart(String start) throws DdlException {
+ private static int checkStart(String start) throws DdlException {
try {
- if (Integer.parseInt(start) >= 0) {
+ int startInt = Integer.parseInt(start);
+ if (startInt >= 0) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_START_ZERO,
start);
}
+ return startInt;
} catch (NumberFormatException e) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_START_FORMAT,
start);
}
+ return DynamicPartitionProperty.MIN_START_OFFSET;
}
- private static void checkEnd(String end) throws DdlException {
+ private static int checkEnd(String end) throws DdlException {
if (Strings.isNullOrEmpty(end)) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_EMPTY);
}
try {
- if (Integer.parseInt(end) <= 0) {
+ int endInt = Integer.parseInt(end);
+ if (endInt <= 0) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_ZERO, end);
}
+ return endInt;
} catch (NumberFormatException e) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_FORMAT,
end);
}
+ return DynamicPartitionProperty.MAX_END_OFFSET;
}
private static void checkBuckets(String buckets) throws DdlException {
@@ -131,6 +144,14 @@ public class DynamicPartitionUtil {
}
}
+ private static boolean checkCreateHistoryPartition(String create) throws
DdlException {
+ if (Strings.isNullOrEmpty(create)
+ || (!Boolean.TRUE.toString().equalsIgnoreCase(create) &&
!Boolean.FALSE.toString().equalsIgnoreCase(create))) {
+
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_CREATE_HISTORY_PARTITION,
create);
+ }
+ return Boolean.valueOf(create);
+ }
+
private static void checkStartDayOfMonth(String val) throws DdlException {
if (Strings.isNullOrEmpty(val)) {
throw new DdlException("Invalid properties: " +
DynamicPartitionProperty.START_DAY_OF_MONTH);
@@ -204,13 +225,15 @@ public class DynamicPartitionUtil {
String end = properties.get(DynamicPartitionProperty.END);
String buckets = properties.get(DynamicPartitionProperty.BUCKETS);
String enable = properties.get(DynamicPartitionProperty.ENABLE);
+ String createHistoryPartition =
properties.get(DynamicPartitionProperty.CREATE_HISTORY_PARTITION);
if (!(Strings.isNullOrEmpty(enable) &&
Strings.isNullOrEmpty(timeUnit) &&
Strings.isNullOrEmpty(timeZone) &&
Strings.isNullOrEmpty(prefix) &&
Strings.isNullOrEmpty(start) &&
Strings.isNullOrEmpty(end) &&
- Strings.isNullOrEmpty(buckets))) {
+ Strings.isNullOrEmpty(buckets) &&
+ Strings.isNullOrEmpty(createHistoryPartition))) {
if (Strings.isNullOrEmpty(enable)) {
properties.put(DynamicPartitionProperty.ENABLE, "true");
}
@@ -232,6 +255,9 @@ public class DynamicPartitionUtil {
if (Strings.isNullOrEmpty(timeZone)) {
properties.put(DynamicPartitionProperty.TIME_ZONE,
TimeUtils.getSystemTimeZone().getID());
}
+ if (Strings.isNullOrEmpty(createHistoryPartition)) {
+
properties.put(DynamicPartitionProperty.CREATE_HISTORY_PARTITION, "false");
+ }
}
return true;
}
@@ -267,12 +293,7 @@ public class DynamicPartitionUtil {
properties.remove(DynamicPartitionProperty.PREFIX);
analyzedProperties.put(DynamicPartitionProperty.PREFIX,
prefixValue);
}
- if (properties.containsKey(DynamicPartitionProperty.END)) {
- String endValue = properties.get(DynamicPartitionProperty.END);
- checkEnd(endValue);
- properties.remove(DynamicPartitionProperty.END);
- analyzedProperties.put(DynamicPartitionProperty.END, endValue);
- }
+
if (properties.containsKey(DynamicPartitionProperty.BUCKETS)) {
String bucketsValue =
properties.get(DynamicPartitionProperty.BUCKETS);
checkBuckets(bucketsValue);
@@ -285,14 +306,41 @@ public class DynamicPartitionUtil {
properties.remove(DynamicPartitionProperty.ENABLE);
analyzedProperties.put(DynamicPartitionProperty.ENABLE,
enableValue);
}
- // If dynamic property is not specified.Use Integer.MIN_VALUE as
default
+
+ // If dynamic property "start" is not specified, use Integer.MIN_VALUE
as default
+ int start = DynamicPartitionProperty.MIN_START_OFFSET;
if (properties.containsKey(DynamicPartitionProperty.START)) {
String startValue = properties.get(DynamicPartitionProperty.START);
- checkStart(startValue);
+ start = checkStart(startValue);
properties.remove(DynamicPartitionProperty.START);
analyzedProperties.put(DynamicPartitionProperty.START, startValue);
}
+ int end = DynamicPartitionProperty.MAX_END_OFFSET;
+ if (properties.containsKey(DynamicPartitionProperty.END)) {
+ String endValue = properties.get(DynamicPartitionProperty.END);
+ end = checkEnd(endValue);
+ properties.remove(DynamicPartitionProperty.END);
+ analyzedProperties.put(DynamicPartitionProperty.END, endValue);
+ }
+
+ boolean createHistoryPartition = false;
+ if
(properties.containsKey(DynamicPartitionProperty.CREATE_HISTORY_PARTITION)) {
+ String val =
properties.get(DynamicPartitionProperty.CREATE_HISTORY_PARTITION);
+ createHistoryPartition = checkCreateHistoryPartition(val);
+
properties.remove(DynamicPartitionProperty.CREATE_HISTORY_PARTITION);
+
analyzedProperties.put(DynamicPartitionProperty.CREATE_HISTORY_PARTITION, val);
+ }
+
+ // Check the number of dynamic partitions that need to be created to
avoid creating too many partitions at once.
+ // If create_history_partition is false, history partition is not
considered.
+ if (!createHistoryPartition) {
+ start = 0;
+ }
+ if (end - start > Config.max_dynamic_partition_num) {
+ throw new DdlException("Too many dynamic partitions: " + (end -
start) + ". Limit: " + Config.max_dynamic_partition_num);
+ }
+
if
(properties.containsKey(DynamicPartitionProperty.START_DAY_OF_MONTH)) {
String val =
properties.get(DynamicPartitionProperty.START_DAY_OF_MONTH);
checkStartDayOfMonth(val);
@@ -414,9 +462,9 @@ public class DynamicPartitionUtil {
return getPartitionRangeOfDay(current, offset, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
return getPartitionRangeOfWeek(current, offset,
property.getStartOfWeek(), format);
- } else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { //
MONTH
+ } else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
return getPartitionRangeOfHour(current, offset, format);
- } else {
+ } else { // MONTH
return getPartitionRangeOfMonth(current, offset,
property.getStartOfMonth(), format);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index d7dc04c..28e890b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -19,11 +19,14 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
-
import org.apache.doris.utframe.UtFrameUtils;
+
+import com.clearspring.analytics.util.Lists;
+
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -33,9 +36,11 @@ import org.junit.rules.ExpectedException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Iterator;
+import java.util.List;
import java.util.UUID;
public class DynamicPartitionTableTest {
@@ -468,16 +473,75 @@ public class DynamicPartitionTableTest {
"\"dynamic_partition.buckets\" = \"1\"\n" +
");";
createTable(createOlapTblStmt);
- OlapTable emptyDynamicTable =
(OlapTable)Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("empty_dynamic_partition");
+ OlapTable emptyDynamicTable = (OlapTable)
Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("empty_dynamic_partition");
Assert.assertTrue(emptyDynamicTable.getAllPartitions().size() == 4);
+ Iterator<Partition> partitionIterator =
emptyDynamicTable.getAllPartitions().iterator();
+ List<String> partNames = Lists.newArrayList();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ partNames.add(partitionName.substring(1));
+ }
+ Collections.sort(partNames);
+
int partitionCount = 0;
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+ for (String partName : partNames) {
+ Date partitionDate = sdf.parse(partName);
+ Date date = new Date();
+ Calendar calendar = new GregorianCalendar();
+ calendar.setTime(date);
+ calendar.add(calendar.DATE, partitionCount);
+ date = calendar.getTime();
+
+ Assert.assertEquals(partitionDate.getYear(), date.getYear());
+ Assert.assertEquals(partitionDate.getMonth(), date.getMonth());
+ Assert.assertEquals(partitionDate.getDay(), date.getDay());
+
+ partitionCount++;
+ }
+ }
+
+ @Test
+ public void testFillHistoryDynamicPartition() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`histo_dynamic_partition` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ OlapTable emptyDynamicTable = (OlapTable)
Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("histo_dynamic_partition");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
Iterator<Partition> partitionIterator =
emptyDynamicTable.getAllPartitions().iterator();
- while (partitionCount < 4) {
+ List<String> partNames = Lists.newArrayList();
+ while (partitionIterator.hasNext()) {
String partitionName = partitionIterator.next().getName();
- SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
- Date partitionDate = sdf.parse(partitionName.substring(1));
+ partNames.add(partitionName.substring(1));
+ }
+ Collections.sort(partNames);
+ int partitionCount = -3;
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+ for (String partName : partNames) {
+ Date partitionDate = sdf.parse(partName);
Date date = new Date();
Calendar calendar = new GregorianCalendar();
calendar.setTime(date);
@@ -491,4 +555,221 @@ public class DynamicPartitionTableTest {
partitionCount++;
}
}
+
+ @Test(expected = DdlException.class)
+ public void testFillHistoryDynamicPartition2() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`histo_dynamic_partition2` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3000\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ // exceed the max dynamic partition limit
+ Config.max_dynamic_partition_num = 1000;
+ createTable(createOlapTblStmt);
+ }
+
+ @Test
+ public void testAllTypeDynamicPartition() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`hour_dynamic_partition`
(\n" +
+ " `k1` datetime NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+
+ "\"dynamic_partition.time_unit\" = \"hour\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ OlapTable emptyDynamicTable = (OlapTable)
Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("hour_dynamic_partition");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ Iterator<Partition> partitionIterator =
emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(11, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`week_dynamic_partition` (\n" +
+ " `k1` datetime NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+
+ "\"dynamic_partition.time_unit\" = \"week\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable)
Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("week_dynamic_partition");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(8, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`month_dynamic_partition` (\n" +
+ " `k1` datetime NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+
+ "\"dynamic_partition.time_unit\" = \"month\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable)
Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("month_dynamic_partition");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(7, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`int_dynamic_partition_day`
(\n" +
+ " `k1` int NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable)
Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("int_dynamic_partition_day");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(9, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`int_dynamic_partition_week`
(\n" +
+ " `k1` int NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+
+ "\"dynamic_partition.time_unit\" = \"week\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable)
Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("int_dynamic_partition_week");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(8, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`int_dynamic_partition_month`
(\n" +
+ " `k1` int NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+
+ "\"dynamic_partition.time_unit\" = \"month\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable)
Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("int_dynamic_partition_month");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(7, partitionName.length());
+ }
+ }
+
+ @Test(expected = DdlException.class)
+ public void testHourDynamicPartitionWithIntType() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`int_dynamic_partition_hour` (\n" +
+ " `k1` int NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+
+ "\"dynamic_partition.time_unit\" = \"hour\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
