This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8a8a3adb2fb [Enhancement](partition) Support use Auto and Dynamic
partition at the same time (#39580)
8a8a3adb2fb is described below
commit 8a8a3adb2fbad944a49bf19df05e252736ef0f3d
Author: zclllhhjj <[email protected]>
AuthorDate: Tue Aug 27 16:43:01 2024 +0800
[Enhancement](partition) Support use Auto and Dynamic partition at the same
time (#39580)
## Proposed changes
before we forbid it in https://github.com/apache/doris/pull/33736, now
it's time to let it go.
now we support:
```sql
mysql> create table auto_dynamic(
-> k0 datetime(6) NOT NULL
-> )
-> auto partition by range (date_trunc(k0, 'year'))
-> (
-> )
-> DISTRIBUTED BY HASH(`k0`) BUCKETS 2
-> properties(
-> "dynamic_partition.enable" = "true",
-> "dynamic_partition.prefix" = "p",
-> "dynamic_partition.start" = "-50",
-> "dynamic_partition.end" = "0",
-> "dynamic_partition.time_unit" = "year",
-> "replication_num" = "1"
-> );
Query OK, 0 rows affected (0.12 sec)
```
with this, the users could manage their data more flexible
we will refactor the docs of it, and the only recommended way it set end
to zero.
---
.../doris/common/util/DynamicPartitionUtil.java | 41 ++++-
.../apache/doris/datasource/InternalCatalog.java | 6 +-
.../test_auto_partition_behavior.out | 10 +-
.../test_auto_partition_behavior.groovy | 195 +++++++++------------
.../test_dynamic_partition.groovy | 31 ++--
5 files changed, 143 insertions(+), 140 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index d3c50be141f..7fc13392cef 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -17,6 +17,9 @@
package org.apache.doris.common.util;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
@@ -128,13 +131,14 @@ public class DynamicPartitionUtil {
return DynamicPartitionProperty.MIN_START_OFFSET;
}
- private static int checkEnd(String end) throws DdlException {
+ private static int checkEnd(String end, boolean enableAutoPartition)
throws DdlException {
if (Strings.isNullOrEmpty(end)) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_EMPTY);
}
try {
int endInt = Integer.parseInt(end);
- if (endInt <= 0) {
+ // with auto partition sometime we dont like to create future
partition by dynamic partition.
+ if (endInt < 0 || endInt == 0 && !enableAutoPartition) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_ZERO, end);
}
return endInt;
@@ -521,6 +525,25 @@ public class DynamicPartitionUtil {
}
}
+ public static void partitionIntervalCompatible(String dynamicUnit,
ArrayList<Expr> autoExprs)
+ throws AnalysisException {
+ if (autoExprs == null) {
+ return;
+ }
+ for (Expr autoExpr : autoExprs) {
+ Expr func = (FunctionCallExpr) autoExpr;
+ for (Expr child : func.getChildren()) {
+ if (child instanceof LiteralExpr) {
+ String autoUnit = ((LiteralExpr) child).getStringValue();
+ if (!dynamicUnit.equalsIgnoreCase(autoUnit)) {
+ throw new AnalysisException("If support auto partition
and dynamic partition at same time, "
+ + "they must have the same interval unit.");
+ }
+ }
+ }
+ }
+ }
+
// Analyze all properties to check their validation
public static Map<String, String> analyzeDynamicPartition(Map<String,
String> properties,
OlapTable olapTable, Database db) throws UserException {
@@ -529,6 +552,12 @@ public class DynamicPartitionUtil {
if (properties.containsKey(DynamicPartitionProperty.TIME_UNIT)) {
String timeUnitValue =
properties.get(DynamicPartitionProperty.TIME_UNIT);
checkTimeUnit(timeUnitValue, olapTable.getPartitionInfo());
+
+ // if both enabled, must use same interval.
+ if (olapTable.getPartitionInfo().enableAutomaticPartition()) {
+ partitionIntervalCompatible(timeUnitValue,
olapTable.getPartitionInfo().getPartitionExprs());
+ }
+
properties.remove(DynamicPartitionProperty.TIME_UNIT);
analyzedProperties.put(DynamicPartitionProperty.TIME_UNIT,
timeUnitValue);
}
@@ -553,11 +582,7 @@ public class DynamicPartitionUtil {
analyzedProperties.put(DynamicPartitionProperty.ENABLE,
enableValue);
}
- if
(Boolean.parseBoolean(analyzedProperties.getOrDefault(DynamicPartitionProperty.ENABLE,
"true"))
- && olapTable.getPartitionInfo().enableAutomaticPartition()) {
- throw new AnalysisException(
- "Can't use Dynamic Partition and Auto Partition at the
same time");
- }
+ boolean enableAutoPartition =
olapTable.getPartitionInfo().enableAutomaticPartition();
// If dynamic property "start" is not specified, use Integer.MIN_VALUE
as default
int start = DynamicPartitionProperty.MIN_START_OFFSET;
@@ -572,7 +597,7 @@ public class DynamicPartitionUtil {
boolean hasEnd = false;
if (properties.containsKey(DynamicPartitionProperty.END)) {
String endValue = properties.get(DynamicPartitionProperty.END);
- end = checkEnd(endValue);
+ end = checkEnd(endValue, enableAutoPartition);
properties.remove(DynamicPartitionProperty.END);
analyzedProperties.put(DynamicPartitionProperty.END, endValue);
hasEnd = true;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 3b28f01f2b4..2e1729d8bb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -3073,8 +3073,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
.getDynamicPartitionProperty();
if (dynamicProperty.isExist() &&
dynamicProperty.getEnable()
&& partitionDesc.isAutoCreatePartitions()) {
- throw new AnalysisException(
- "Can't use Dynamic Partition and Auto
Partition at the same time");
+ String dynamicUnit = dynamicProperty.getTimeUnit();
+ ArrayList<Expr> autoExprs =
partitionDesc.getPartitionExprs();
+ // check same interval. fail will leading to
AnalysisException
+
DynamicPartitionUtil.partitionIntervalCompatible(dynamicUnit, autoExprs);
}
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
diff --git
a/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out
b/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out
index 40ae8a97a89..27d77c500f2 100644
---
a/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out
+++
b/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out
@@ -102,10 +102,7 @@ xxX 3
2013-12-12T00:00 2013-12-12T00:00
2020-12-12T00:00 2020-12-12T12:12:12.123456
--- !sql_overwrite1 --
-Yyy
-
--- !sql_overwrite2 --
+-- !sql_overwrite --
Xxx
-- !sql_non_order1 --
@@ -117,3 +114,8 @@ Xxx
-- !sql_non_order3 --
3 2013-12-12T00:00
+-- !sql_dynamic_auto --
+2024-01-01T00:00
+2900-01-01T00:00
+3000-01-01T00:00
+
diff --git
a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
index 4c3f184e332..fb8be0b5510 100644
---
a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
+++
b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
@@ -16,6 +16,9 @@
// under the License.
suite("test_auto_partition_behavior") {
+ sql "set experimental_enable_nereids_planner=true;"
+ sql "set enable_fallback_to_original_planner=false;"
+
/// unique key table
sql "drop table if exists unique_table"
sql """
@@ -165,18 +168,6 @@ suite("test_auto_partition_behavior") {
);
"""
sql """ insert into rewrite values ("Xxx"); """
- // legacy planner
- sql " set experimental_enable_nereids_planner=false "
- try {
- sql """ insert overwrite table rewrite partition(p1) values ("XXX") """
- fail()
- } catch (Exception e) {
- assertTrue(e.getMessage().contains("Insert has filtered data in strict
mode"))
- }
- sql """ insert overwrite table rewrite partition(p1) values ("Yyy") """
- qt_sql_overwrite1 """ select * from rewrite """ // Yyy
- // nereids planner
- sql " set experimental_enable_nereids_planner=true "
try {
sql """ insert overwrite table rewrite partition(p1) values ("") """
fail()
@@ -184,7 +175,7 @@ suite("test_auto_partition_behavior") {
assertTrue(e.getMessage().contains("Insert has filtered data in strict
mode"))
}
sql """ insert overwrite table rewrite partition(p1) values ("Xxx") """
- qt_sql_overwrite2 """ select * from rewrite """ // Xxx
+ qt_sql_overwrite """ select * from rewrite """ // Xxx
sql " drop table if exists non_order; "
sql """
@@ -209,22 +200,6 @@ suite("test_auto_partition_behavior") {
qt_sql_non_order3 """ select * from non_order where k1 = '2013-12-12'; """
// range partition can't auto create null partition
- sql " set experimental_enable_nereids_planner=true "
- sql "drop table if exists invalid_null_range"
- test {
- sql """
- create table invalid_null_range(
- k0 datetime(6) null
- )
- auto partition by range (date_trunc(k0, 'hour'))
- (
- )
- DISTRIBUTED BY HASH(`k0`) BUCKETS 2
- properties("replication_num" = "1");
- """
- exception "AUTO RANGE PARTITION doesn't support NULL column"
- }
- sql " set experimental_enable_nereids_planner=false "
sql "drop table if exists invalid_null_range"
test {
sql """
@@ -240,18 +215,11 @@ suite("test_auto_partition_behavior") {
exception "AUTO RANGE PARTITION doesn't support NULL column"
}
- sql "drop table if exists test_dynamic"
- sql """
- create table test_dynamic(
- k0 DATE not null
- )
- auto partition by range (date_trunc(k0, 'year')) ()
- DISTRIBUTED BY HASH(`k0`) BUCKETS auto
- properties("replication_num" = "1");
- """
+
+ // dynamic + auto partition
+ sql """ admin set frontend config
('dynamic_partition_check_interval_seconds' = '1') """
// PROHIBIT different timeunit of interval when use both auto & dynamic
partition
- sql "set experimental_enable_nereids_planner=true;"
test{
sql """
CREATE TABLE tbl3
@@ -273,45 +241,18 @@ suite("test_auto_partition_behavior") {
"dynamic_partition.buckets" = "8"
);
"""
- exception "Can't use Dynamic Partition and Auto Partition at the same
time"
- }
- test {
- sql """
- ALTER TABLE test_dynamic set (
- "dynamic_partition.enable" = "true",
- "dynamic_partition.time_unit" = "DAY",
- "dynamic_partition.end" = "3",
- "dynamic_partition.prefix" = "p",
- "dynamic_partition.buckets" = "32"
- );
- """
- exception "Can't use Dynamic Partition and Auto Partition at the same
time"
+ exception "If support auto partition and dynamic partition at same
time, they must have the same interval unit."
}
- sql "set experimental_enable_nereids_planner=false;"
- test{
- sql """
- CREATE TABLE tbl3
- (
- k1 DATETIME NOT NULL,
- col1 int
+ sql " drop table if exists test_dynamic "
+ sql """
+ create table test_dynamic(
+ k0 DATE not null
)
- auto partition by range (date_trunc(`k1`, 'year')) ()
- DISTRIBUTED BY HASH(k1)
- PROPERTIES
- (
- "replication_num" = "1",
- "dynamic_partition.create_history_partition"="true",
- "dynamic_partition.enable" = "true",
- "dynamic_partition.time_unit" = "HOUR",
- "dynamic_partition.start" = "-2",
- "dynamic_partition.end" = "2",
- "dynamic_partition.prefix" = "p",
- "dynamic_partition.buckets" = "8"
- );
+ auto partition by range (date_trunc(k0, 'year')) ()
+ DISTRIBUTED BY HASH(`k0`) BUCKETS auto
+ properties("replication_num" = "1");
"""
- exception "Can't use Dynamic Partition and Auto Partition at the same
time"
- }
test {
sql """
ALTER TABLE test_dynamic set (
@@ -322,8 +263,70 @@ suite("test_auto_partition_behavior") {
"dynamic_partition.buckets" = "32"
);
"""
- exception "Can't use Dynamic Partition and Auto Partition at the same
time"
+ exception "If support auto partition and dynamic partition at same
time, they must have the same interval unit."
}
+ sql """
+ ALTER TABLE test_dynamic set (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "YeAr",
+ "dynamic_partition.end" = "3",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "32"
+ );
+ """
+
+ sql " drop table if exists auto_dynamic "
+ sql """
+ create table auto_dynamic(
+ k0 datetime(6) NOT NULL
+ )
+ auto partition by range (date_trunc(k0, 'hour'))
+ (
+ )
+ DISTRIBUTED BY HASH(`k0`) BUCKETS 2
+ properties(
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.create_history_partition" = "true",
+ "dynamic_partition.start" = "-5",
+ "dynamic_partition.end" = "0",
+ "dynamic_partition.time_unit" = "hour",
+ "replication_num" = "1"
+ );
+ """
+ def part_result = sql " show partitions from auto_dynamic "
+ assertEquals(part_result.size, 6)
+
+ sql " drop table if exists auto_dynamic "
+ sql """
+ create table auto_dynamic(
+ k0 datetime(6) NOT NULL
+ )
+ auto partition by range (date_trunc(k0, 'year'))
+ (
+ )
+ DISTRIBUTED BY HASH(`k0`) BUCKETS 2
+ properties(
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.start" = "-50",
+ "dynamic_partition.end" = "0",
+ "dynamic_partition.time_unit" = "year",
+ "replication_num" = "1"
+ );
+ """
+ part_result = sql " show partitions from auto_dynamic "
+ assertEquals(part_result.size, 1)
+ sql " insert into auto_dynamic values ('2024-01-01'), ('2900-01-01'),
('1900-01-01'), ('3000-01-01'); "
+ sleep(3000)
+ part_result = sql " show partitions from auto_dynamic "
+ log.info("${part_result}".toString())
+ assertEquals(part_result.size, 3)
+ qt_sql_dynamic_auto "select * from auto_dynamic order by k0;"
+ sql """ admin set frontend config
('dynamic_partition_check_interval_seconds' = '600') """
+
+
+
// prohibit too long value for partition column
sql "drop table if exists `long_value`"
@@ -345,25 +348,10 @@ suite("test_auto_partition_behavior") {
exception exception_str
}
- // illegal partiton definetion
- sql "set experimental_enable_nereids_planner=false;"
- test{
- sql """
- create table illegal(
- k0 datetime(6) NOT null,
- k1 datetime(6) NOT null
- )
- auto partition by range (date_trunc(k0, k1, 'hour'))
- (
- )
- DISTRIBUTED BY HASH(`k0`) BUCKETS 2
- properties("replication_num" = "1");
- """
- exception "auto create partition only support one slotRef in function
expr"
- }
- sql "set experimental_enable_nereids_planner=true;"
- sql "set enable_fallback_to_original_planner=false;"
+
+
+ /// illegal partition exprs
test{
sql """
create table illegal(
@@ -393,24 +381,11 @@ suite("test_auto_partition_behavior") {
"""
exception "partition expr date_trunc is illegal!"
}
- sql "set experimental_enable_nereids_planner=false;"
- test{
- sql """
- create table illegal(
- k0 datetime(6) NOT null,
- k1 int NOT null
- )
- auto partition by range (date_trunc(k1, 'hour'))
- (
- )
- DISTRIBUTED BY HASH(`k0`) BUCKETS 2
- properties("replication_num" = "1");
- """
- exception "Auto range partition needs Date/DateV2/Datetime/DatetimeV2
column as partition column"
- }
- sql "set experimental_enable_nereids_planner=true;"
+
+
+ // altering table property effects new partitions.
sql " drop table if exists test_change "
sql """
create table test_change(
@@ -426,7 +401,7 @@ suite("test_auto_partition_behavior") {
logger.info("get table replica num: " + replicaNum)
sql """ insert into test_change values ("20201212"); """
- def part_result = sql " show tablets from test_change "
+ part_result = sql " show tablets from test_change "
assertEquals(part_result.size, 2 * replicaNum)
sql """ ALTER TABLE test_change MODIFY DISTRIBUTION DISTRIBUTED BY
HASH(k0) BUCKETS 50; """
sql """ insert into test_change values ("20001212"); """
diff --git
a/regression-test/suites/partition_p1/dynamic_partition/test_dynamic_partition.groovy
b/regression-test/suites/partition_p1/dynamic_partition/test_dynamic_partition.groovy
index a0434bb0c10..a9dfcce8eac 100644
---
a/regression-test/suites/partition_p1/dynamic_partition/test_dynamic_partition.groovy
+++
b/regression-test/suites/partition_p1/dynamic_partition/test_dynamic_partition.groovy
@@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_dynamic_partition_with_update","nonConcurrent") {
- def tbl = "test_dynamic_partition_with_update"
- sql "drop table if exists ${tbl}"
+suite("test_dynamic_partition_with_update", "nonConcurrent") {
+ sql "drop table if exists test_dynamic_partition_with_update"
sql """
- CREATE TABLE IF NOT EXISTS ${tbl}
+ CREATE TABLE IF NOT EXISTS test_dynamic_partition_with_update
( k1 date NOT NULL )
PARTITION BY RANGE(k1) ( )
DISTRIBUTED BY HASH(k1) BUCKETS 1
@@ -36,38 +35,38 @@ suite("test_dynamic_partition_with_update","nonConcurrent")
{
"""
// set check interval time
- sql """ admin set frontend config
('dynamic_partition_check_interval_seconds' = '2') """
+ sql """ admin set frontend config
('dynamic_partition_check_interval_seconds' = '1') """
// check table init
- def result = sql "show partitions from ${tbl}"
+ def result = sql "show partitions from test_dynamic_partition_with_update"
assertEquals(7, result.size())
result = sql "show dynamic partition tables"
assertEquals("true",result.get(0).get(1))
// disable dynamic partition to insert partition
- sql """ alter table ${tbl} set ('dynamic_partition.enable' = 'false') """
+ sql """ alter table test_dynamic_partition_with_update set
('dynamic_partition.enable' = 'false') """
result = sql "show dynamic partition tables"
assertEquals("false",result.get(0).get(1))
// manually insert partition
- sql """ alter table ${tbl} add partition p1 values [("2020-01-02"),
("2020-01-05")) """
- sql """ alter table ${tbl} add partition p2 values [("2020-05-02"),
("2020-06-06")) """
- sql """ alter table ${tbl} add partition p3 values [("2020-07-04"),
("2020-07-28")) """
- sql """ alter table ${tbl} add partition p4 values [("2999-04-25"),
("2999-04-28")) """
+ sql """ alter table test_dynamic_partition_with_update add partition p1
values [("2020-01-02"), ("2020-01-05")) """
+ sql """ alter table test_dynamic_partition_with_update add partition p2
values [("2020-05-02"), ("2020-06-06")) """
+ sql """ alter table test_dynamic_partition_with_update add partition p3
values [("2020-07-04"), ("2020-07-28")) """
+ sql """ alter table test_dynamic_partition_with_update add partition p4
values [("2999-04-25"), ("2999-04-28")) """
// check size
- result = sql "show partitions from ${tbl}"
+ result = sql "show partitions from test_dynamic_partition_with_update"
assertEquals(11, result.size())
- sql """ alter table ${tbl} set ('dynamic_partition.enable' = 'true') """
+ sql """ alter table test_dynamic_partition_with_update set
('dynamic_partition.enable' = 'true') """
result = sql "show dynamic partition tables"
assertEquals("true",result.get(0).get(1))
// check and update
- sleep(5000);
+ sleep(3000)
// check size
- result = sql "show partitions from ${tbl}"
+ result = sql "show partitions from test_dynamic_partition_with_update"
assertEquals(8, result.size())
- sql "drop table ${tbl}"
+ sql """ admin set frontend config
('dynamic_partition_check_interval_seconds' = '600') """
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
