This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0dca9eff22a [Enhancement](alter) support add partitions in batch
(#37114)
0dca9eff22a is described below
commit 0dca9eff22ac911e523eceefce6ac94ad9535320
Author: lw112 <[email protected]>
AuthorDate: Wed Aug 14 23:20:41 2024 +0800
[Enhancement](alter) support add partitions in batch (#37114)
## Proposed changes
Issue Number: close #32524
---
fe/fe-core/src/main/cup/sql_parser.cup | 8 ++
.../main/java/org/apache/doris/alter/Alter.java | 10 +-
.../doris/analysis/AlterMultiPartitionClause.java | 73 ++++++++++++++
.../main/java/org/apache/doris/catalog/Env.java | 6 ++
.../apache/doris/datasource/InternalCatalog.java | 20 ++++
.../alter_p0/test_alter_add_multi_partition.groovy | 110 +++++++++++++++++++++
6 files changed, 226 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index 619f13e6278..891a72645cd 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -1746,6 +1746,14 @@ alter_table_clause ::=
{:
RESULT = new ModifyEngineClause(engine, properties);
:}
+ | KW_ADD opt_tmp:isTempPartition KW_PARTITIONS KW_FROM LPAREN
partition_key_list:lower RPAREN KW_TO LPAREN partition_key_list:upper RPAREN
KW_INTERVAL INTEGER_LITERAL:time_interval ident:time_unit
opt_properties:properties
+ {:
+ RESULT = new
AlterMultiPartitionClause(PartitionKeyDesc.createMultiFixed(lower, upper,
time_interval, time_unit), properties, isTempPartition);
+ :}
+ | KW_ADD opt_tmp:isTempPartition KW_PARTITIONS KW_FROM LPAREN
partition_key_list:lower RPAREN KW_TO LPAREN partition_key_list:upper RPAREN
KW_INTERVAL INTEGER_LITERAL:num_interval opt_properties:properties
+ {:
+ RESULT = new
AlterMultiPartitionClause(PartitionKeyDesc.createMultiFixed(lower, upper,
num_interval), properties, isTempPartition);
+ :}
;
opt_enable_feature_properties ::=
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 1fcb4fe65c3..ced14ad4b8d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -20,6 +20,7 @@ package org.apache.doris.alter;
import org.apache.doris.analysis.AddPartitionClause;
import org.apache.doris.analysis.AddPartitionLikeClause;
import org.apache.doris.analysis.AlterClause;
+import org.apache.doris.analysis.AlterMultiPartitionClause;
import org.apache.doris.analysis.AlterSystemStmt;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.AlterViewStmt;
@@ -252,7 +253,8 @@ public class Alter {
} else if (alterClause instanceof
DropPartitionFromIndexClause) {
// do nothing
} else if (alterClause instanceof AddPartitionClause
- || alterClause instanceof AddPartitionLikeClause) {
+ || alterClause instanceof AddPartitionLikeClause
+ || alterClause instanceof
AlterMultiPartitionClause) {
needProcessOutsideTableLock = true;
} else {
throw new DdlException("Invalid alter operation: " +
alterClause.getOpType());
@@ -508,6 +510,12 @@ public class Alter {
} else if (alterClause instanceof ModifyTablePropertiesClause) {
Map<String, String> properties = alterClause.getProperties();
((SchemaChangeHandler)
schemaChangeHandler).updateTableProperties(db, tableName, properties);
+ } else if (alterClause instanceof AlterMultiPartitionClause) {
+ if (!((AlterMultiPartitionClause)
alterClause).isTempPartition()) {
+ DynamicPartitionUtil.checkAlterAllowed(
+ (OlapTable) db.getTableOrMetaException(tableName,
TableType.OLAP));
+ }
+ Env.getCurrentEnv().addMultiPartitions(db, tableName,
(AlterMultiPartitionClause) alterClause);
} else {
throw new DdlException("Invalid alter operation: " +
alterClause.getOpType());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMultiPartitionClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMultiPartitionClause.java
new file mode 100644
index 00000000000..f7231e6da3e
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMultiPartitionClause.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.alter.AlterOpType;
+import org.apache.doris.common.AnalysisException;
+
+import java.util.Map;
+
+public class AlterMultiPartitionClause extends AlterTableClause {
+ private PartitionKeyDesc partitionKeyDesc;
+ private Map<String, String> properties;
+ private boolean isTempPartition;
+
+ public AlterMultiPartitionClause(PartitionKeyDesc partitionKeyDesc,
Map<String, String> properties,
+ boolean isTempPartition) {
+ super(AlterOpType.ADD_PARTITION);
+ this.partitionKeyDesc = partitionKeyDesc;
+ this.properties = properties;
+ this.isTempPartition = isTempPartition;
+ }
+
+ @Override
+ public void analyze(Analyzer analyzer) throws AnalysisException {
+ }
+
+ @Override
+ public String toSql() {
+ return String.format("ADD PARTITIONS %s", partitionKeyDesc.toSql());
+ }
+
+ @Override
+ public String toString() {
+ return toSql();
+ }
+
+ @Override
+ public boolean allowOpMTMV() {
+ return false;
+ }
+
+ @Override
+ public boolean needChangeMTMVState() {
+ return false;
+ }
+
+ public PartitionKeyDesc getPartitionKeyDesc() {
+ return partitionKeyDesc;
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public boolean isTempPartition() {
+ return isTempPartition;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 907f2573920..121228f569a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -38,6 +38,7 @@ import org.apache.doris.analysis.AlterDatabasePropertyStmt;
import org.apache.doris.analysis.AlterDatabaseQuotaStmt;
import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType;
import org.apache.doris.analysis.AlterDatabaseRename;
+import org.apache.doris.analysis.AlterMultiPartitionClause;
import org.apache.doris.analysis.AlterSystemStmt;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.AlterViewStmt;
@@ -3278,6 +3279,11 @@ public class Env {
isCreateTable, generatedPartitionId, writeEditLog);
}
+ public void addMultiPartitions(Database db, String tableName,
AlterMultiPartitionClause multiPartitionClause)
+ throws DdlException {
+ getInternalCatalog().addMultiPartitions(db, tableName,
multiPartitionClause);
+ }
+
public void addPartitionLike(Database db, String tableName,
AddPartitionLikeClause addPartitionLikeClause)
throws DdlException {
getInternalCatalog().addPartitionLike(db, tableName,
addPartitionLikeClause);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 12093bb6696..234119b992b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -25,6 +25,7 @@ import org.apache.doris.analysis.AlterDatabasePropertyStmt;
import org.apache.doris.analysis.AlterDatabaseQuotaStmt;
import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType;
import org.apache.doris.analysis.AlterDatabaseRename;
+import org.apache.doris.analysis.AlterMultiPartitionClause;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.ColumnDef;
import org.apache.doris.analysis.ColumnDef.DefaultValue;
@@ -42,6 +43,7 @@ import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.HashDistributionDesc;
import org.apache.doris.analysis.KeysDesc;
import org.apache.doris.analysis.LiteralExpr;
+import org.apache.doris.analysis.MultiPartitionDesc;
import org.apache.doris.analysis.PartitionDesc;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.PartitionNames;
@@ -1867,6 +1869,24 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
}
+ public void addMultiPartitions(Database db, String tableName,
AlterMultiPartitionClause multiPartitionClause)
+ throws DdlException {
+ List<SinglePartitionDesc> singlePartitionDescs;
+ try {
+ MultiPartitionDesc multiPartitionDesc = new
MultiPartitionDesc(multiPartitionClause.getPartitionKeyDesc(),
+ multiPartitionClause.getProperties());
+ singlePartitionDescs =
multiPartitionDesc.getSinglePartitionDescList();
+ } catch (AnalysisException e) {
+ throw new DdlException("Failed to analyze multi partition clause:
" + e.getMessage());
+ }
+
+ for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) {
+ AddPartitionClause addPartitionClause = new
AddPartitionClause(singlePartitionDesc, null,
+ multiPartitionClause.getProperties(), false);
+ addPartition(db, tableName, addPartitionClause, false, 0, true);
+ }
+ }
+
public void replayAddPartition(PartitionPersistInfo info) throws
MetaNotFoundException {
Database db = (Database) getDbOrMetaException(info.getDbId());
OlapTable olapTable = (OlapTable)
db.getTableOrMetaException(info.getTableId(),
diff --git
a/regression-test/suites/alter_p0/test_alter_add_multi_partition.groovy
b/regression-test/suites/alter_p0/test_alter_add_multi_partition.groovy
new file mode 100644
index 00000000000..a5a55d94bea
--- /dev/null
+++ b/regression-test/suites/alter_p0/test_alter_add_multi_partition.groovy
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_alter_add_multi_partition") {
+ sql "DROP TABLE IF EXISTS add_multi_partition FORCE"
+
+ // Check if you can create a partition for a leap month
+ sql """
+ CREATE TABLE IF NOT EXISTS add_multi_partition
+ (
+ `k1` LARGEINT NOT NULL,
+ `date` DATE NOT NULL,
+ `k2` VARCHAR(20)
+ )
+ ENGINE=OLAP
+ UNIQUE KEY(`k1`, `date`)
+ PARTITION BY RANGE(`date`)
+ (
+ PARTITION `p_20000201` VALUES [("2000-02-01"), ("2000-02-05"))
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ );
+ """
+ List<List<Object>> result1 = sql "show partitions from
add_multi_partition;"
+ assertEquals(result1.size(), 1)
+ sql "ALTER TABLE add_multi_partition ADD PARTITIONS FROM ('2000-02-05') TO
('2000-03-01') INTERVAL 4 DAY;"
+ List<List<Object>> result2 = sql "show partitions from
add_multi_partition;"
+ assertEquals(result2.size(), 8)
+ def partitionName = sql "show partitions from add_multi_partition where
PartitionName = 'p_20000229';"
+ for (pn in partitionName) {
+ assertTrue(pn[1] == "p_20000229")
+ }
+ sql "DROP TABLE IF EXISTS add_multi_partition FORCE"
+
+
+ sql """
+ CREATE TABLE `add_multi_partition` (
+ `k1` LARGEINT NOT NULL,
+ `date` DATE NOT NULL,
+ `k2` VARCHAR(20) NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`k1`, `date`)
+ PARTITION BY RANGE(`date`)
+ (PARTITION p_2024_01 VALUES [('2024-01-01'), ('2024-01-08')),
+ PARTITION p_2024_02 VALUES [('2024-01-08'), ('2024-01-15')),
+ PARTITION p_2024_03 VALUES [('2024-01-15'), ('2024-01-22')),
+ PARTITION p_2024_04 VALUES [('2024-01-22'), ('2024-01-29')),
+ PARTITION p_2024_05 VALUES [('2024-01-29'), ('2024-02-05')),
+ PARTITION p_2024_06 VALUES [('2024-02-05'), ('2024-02-12')),
+ PARTITION p_2024_07 VALUES [('2024-02-12'), ('2024-02-19')),
+ PARTITION p_2024_08 VALUES [('2024-02-19'), ('2024-02-26')),
+ PARTITION p_2024_09 VALUES [('2024-02-26'), ('2024-03-01')))
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ );
+ """
+ List<List<Object>> result3 = sql "show partitions from
add_multi_partition;"
+ assertEquals(result3.size(), 9)
+ sql "ALTER TABLE add_multi_partition ADD PARTITIONS FROM ('2024-04-01') TO
('2025-01-01') INTERVAL 1 WEEK;"
+ List<List<Object>> result4 = sql "show partitions from
add_multi_partition;"
+ assertEquals(result4.size(), 49)
+ sql "DROP TABLE IF EXISTS add_multi_partition FORCE"
+
+
+ sql """
+ CREATE TABLE IF NOT EXISTS add_multi_partition
+ (
+ `k1` LARGEINT NOT NULL,
+ `age` SMALLINT,
+ `k2` VARCHAR(20)
+ )
+ ENGINE=OLAP
+ UNIQUE KEY(`k1`, `age`)
+ PARTITION BY RANGE(`age`)
+ (
+ FROM (1) TO (100) INTERVAL 10
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ );
+ """
+ List<List<Object>> result7 = sql "show partitions from
add_multi_partition;"
+ assertEquals(result7.size(), 10)
+ sql "ALTER TABLE add_multi_partition ADD PARTITIONS FROM (100) TO (200)
INTERVAL 10;"
+ List<List<Object>> result8 = sql "show partitions from
add_multi_partition;"
+ assertEquals(result8.size(), 20)
+ sql "DROP TABLE IF EXISTS add_multi_partition FORCE"
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
