This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ebd2a4b707 [fix](dynamic partition) fix create hot partition failed
without error response (#20996)
ebd2a4b707 is described below
commit ebd2a4b7076e32ad970dafc363cfcf165156d3d6
Author: yujun <[email protected]>
AuthorDate: Tue Jul 18 10:56:37 2023 +0800
[fix](dynamic partition) fix create hot partition failed without error
response (#20996)
---
.../doris/common/util/DynamicPartitionUtil.java | 40 ++++++++++++++------
.../doris/catalog/DynamicPartitionTableTest.java | 44 +++++++++++++++++++++-
.../test_outer_join_with_window_function.groovy | 1 -
...t_outer_join_with_window_function_datev2.groovy | 1 -
.../test_dynamic_partition.groovy | 22 +++++++++++
5 files changed, 93 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index 552ff2decb..7d0e5833fb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -231,11 +231,24 @@ public class DynamicPartitionUtil {
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc,
null);
}
- private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc,
Database db) throws DdlException {
+ private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc,
int hotPartitionNum,
+ Database db) throws DdlException {
if (replicaAlloc.getTotalReplicaNum() <= 0) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_ZERO);
}
+
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc,
null);
+ if (hotPartitionNum <= 0) {
+ return;
+ }
+
+ try {
+
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc,
TStorageMedium.SSD);
+ } catch (DdlException e) {
+ throw new DdlException("Failed to find enough backend for ssd
storage medium. When setting "
+ + DynamicPartitionProperty.HOT_PARTITION_NUM + " > 0, the
hot partitions will store "
+ + "in ssd. Please check the replication num,replication
tag and storage medium.");
+ }
}
private static void checkHotPartitionNum(String val) throws DdlException {
@@ -600,28 +613,33 @@ public class DynamicPartitionUtil {
analyzedProperties.put(DynamicPartitionProperty.TIME_ZONE, val);
}
+ int hotPartitionNum = 0;
+ if
(properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) {
+ String val =
properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM);
+ checkHotPartitionNum(val);
+ hotPartitionNum = Integer.parseInt(val);
+ properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM);
+ analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM,
val);
+ }
+
// check replication_allocation first, then replciation_num
+ ReplicaAllocation replicaAlloc = null;
if
(properties.containsKey(DynamicPartitionProperty.REPLICATION_ALLOCATION)) {
- ReplicaAllocation replicaAlloc =
PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition");
- checkReplicaAllocation(replicaAlloc, db);
+ replicaAlloc =
PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition");
properties.remove(DynamicPartitionProperty.REPLICATION_ALLOCATION);
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
replicaAlloc.toCreateStmt());
} else if
(properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM)) {
String val =
properties.get(DynamicPartitionProperty.REPLICATION_NUM);
checkReplicationNum(val, db);
properties.remove(DynamicPartitionProperty.REPLICATION_NUM);
+ replicaAlloc = new ReplicaAllocation(Short.valueOf(val));
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
- new ReplicaAllocation(Short.valueOf(val)).toCreateStmt());
+ replicaAlloc.toCreateStmt());
} else {
- checkReplicaAllocation(olapTable.getDefaultReplicaAllocation(),
db);
+ replicaAlloc = olapTable.getDefaultReplicaAllocation();
}
+ checkReplicaAllocation(replicaAlloc, hotPartitionNum, db);
- if
(properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) {
- String val =
properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM);
- checkHotPartitionNum(val);
- properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM);
- analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM,
val);
- }
if
(properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS)) {
String reservedHistoryPeriods =
properties.get(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS);
checkReservedHistoryPeriodValidate(reservedHistoryPeriods,
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index 3609e56592..22c19fb159 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -28,6 +28,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;
@@ -82,6 +83,15 @@ public class DynamicPartitionTableTest {
UtFrameUtils.cleanDorisFeDir(runningDir);
}
+ private static void changeBeDisk(TStorageMedium storageMedium) {
+ List<Backend> backends = Env.getCurrentSystemInfo().getAllBackends();
+ for (Backend be : backends) {
+ for (DiskInfo diskInfo : be.getDisks().values()) {
+ diskInfo.setStorageMedium(storageMedium);
+ }
+ }
+ }
+
private static void createTable(String sql) throws Exception {
CreateTableStmt createTableStmt = (CreateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Env.getCurrentEnv().createTable(createTableStmt);
@@ -970,6 +980,8 @@ public class DynamicPartitionTableTest {
@Test
public void testHotPartitionNum() throws Exception {
+ changeBeDisk(TStorageMedium.SSD);
+
Database testDb =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test");
// 1. hour
@@ -1188,9 +1200,11 @@ public class DynamicPartitionTableTest {
}
@Test(expected = DdlException.class)
- public void testHotPartitionNumAbnormal() throws Exception {
+ public void testHotPartitionNumAbnormalLT0() throws Exception {
+ changeBeDisk(TStorageMedium.SSD);
+
// dynamic_partition.hot_partition_num must larger than 0.
- String createOlapTblStmt = "CREATE TABLE
test.`hot_partition_hour_tbl1` (\n"
+ String createOlapTblStmt = "CREATE TABLE
test.`hot_partition_hour_tbl1_lt0` (\n"
+ " `k1` datetime NULL COMMENT \"\",\n"
+ " `k2` int NULL COMMENT \"\"\n"
+ ") ENGINE=OLAP\n"
@@ -1211,6 +1225,32 @@ public class DynamicPartitionTableTest {
createTable(createOlapTblStmt);
}
+ @Test(expected = DdlException.class)
+ public void testHotPartitionNumAbnormalMissSSD() throws Exception {
+ changeBeDisk(TStorageMedium.HDD);
+
+ // when dynamic_partition.hot_partition_num > 0, it require ssd
storage medium.
+ String createOlapTblStmt = "CREATE TABLE
test.`hot_partition_hour_tbl1_miss_ssd` (\n"
+ + " `k1` datetime NULL COMMENT \"\",\n"
+ + " `k2` int NULL COMMENT \"\"\n"
+ + ") ENGINE=OLAP\n"
+ + "PARTITION BY RANGE(`k1`)\n"
+ + "()\n"
+ + "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\",\n"
+ + "\"dynamic_partition.enable\" = \"true\",\n"
+ + "\"dynamic_partition.start\" = \"-3\",\n"
+ + "\"dynamic_partition.end\" = \"3\",\n"
+ + "\"dynamic_partition.create_history_partition\" =
\"true\",\n"
+ + "\"dynamic_partition.time_unit\" = \"hour\",\n"
+ + "\"dynamic_partition.prefix\" = \"p\",\n"
+ + "\"dynamic_partition.buckets\" = \"1\",\n"
+ + "\"dynamic_partition.hot_partition_num\" = \"1\"\n"
+ + ");";
+ createTable(createOlapTblStmt);
+ }
+
@Test
public void testRuntimeInfo() throws Exception {
DynamicPartitionScheduler scheduler = new
DynamicPartitionScheduler("test", 10);
diff --git
a/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy
b/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy
index dfbea21562..3c195713ae 100644
---
a/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy
+++
b/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy
@@ -218,7 +218,6 @@ suite("test_outer_join_with_with_window_function") {
"dynamic_partition.buckets" = "4",
"dynamic_partition.create_history_partition" = "true",
"dynamic_partition.history_partition_num" = "50",
- "dynamic_partition.hot_partition_num" = "2",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.start_day_of_month" = "1",
"in_memory" = "false",
diff --git
a/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy
b/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy
index 169e92c8ba..16abf0fb8a 100644
---
a/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy
+++
b/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy
@@ -222,7 +222,6 @@ suite("test_outer_join_with_window_function_datev2") {
"dynamic_partition.buckets" = "4",
"dynamic_partition.create_history_partition" = "true",
"dynamic_partition.history_partition_num" = "50",
- "dynamic_partition.hot_partition_num" = "2",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.start_day_of_month" = "1",
"in_memory" = "false",
diff --git
a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
index 1a69dda4c7..f4680ff681 100644
---
a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
+++
b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
@@ -149,4 +149,26 @@ suite("test_dynamic_partition") {
exception "errCode = 2,"
}
sql "drop table if exists dy_par_bad"
+ test {
+ sql """
+ CREATE TABLE IF NOT EXISTS dy_par_bad
+ ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
+ AGGREGATE KEY(k1,k2)
+ PARTITION BY RANGE(k1) ( )
+ DISTRIBUTED BY HASH(k1) BUCKETS 3
+ PROPERTIES (
+ "dynamic_partition.enable"="true",
+ "dynamic_partition.end"="3",
+ "dynamic_partition.start"="-3",
+ "dynamic_partition.prefix"="p",
+ "dynamic_partition.time_unit"="DAY",
+ "dynamic_partition.create_history_partition"="true",
+ "dynamic_partition.hot_partition_num" = "2",
+ "dynamic_partition.replication_allocation" =
"tag.location.default: 1")
+ """
+ // check exception message contains
+ // dynamic_partition.hot_partition_num > 0 will require ssd storage
medium
+ exception "errCode = 2,"
+ }
+ sql "drop table if exists dy_par_bad"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
