This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 078fec73f64 branch-4.0: [fix](iow) force drop partition in INSERT
OVERWRITE #62510 (#65029)
078fec73f64 is described below
commit 078fec73f64983e46b1de9251c735169d7d3724e
Author: morrySnow <[email protected]>
AuthorDate: Thu Jul 2 11:47:17 2026 +0800
branch-4.0: [fix](iow) force drop partition in INSERT OVERWRITE #62510
(#65029)
Backport #62510
---
.../insertoverwrite/InsertOverwriteManager.java | 4 +-
.../insert/InsertOverwriteTableCommand.java | 4 +-
.../test_insert_overwrite_multiple.out | 15 ----
.../test_insert_overwrite_recover.out | 14 ----
.../test_insert_overwrite_recover_multiple.out | 23 ------
.../test_insert_overwrite_recover_no_partition.out | 14 ----
.../test_insert_overwrite_multiple.groovy | 68 -----------------
.../test_insert_overwrite_recover.groovy | 64 ----------------
.../test_insert_overwrite_recover_multiple.groovy | 87 ----------------------
...st_insert_overwrite_recover_no_partition.groovy | 78 -------------------
10 files changed, 4 insertions(+), 367 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
index 3f48a2130dc..45b4878984a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
@@ -177,7 +177,7 @@ public class InsertOverwriteManager extends MasterDaemon
implements Writable {
}
// here we will make all raplacement of this group visiable. if someone
fails, nothing happen.
- public void taskGroupSuccess(long groupId, OlapTable targetTable) throws
DdlException {
+ public void taskGroupSuccess(long groupId, OlapTable targetTable, boolean
forceDropPartition) throws DdlException {
try {
Map<Long, Long> relations = partitionPairs.get(groupId);
ArrayList<String> oldNames = new ArrayList<>();
@@ -186,7 +186,7 @@ public class InsertOverwriteManager extends MasterDaemon
implements Writable {
oldNames.add(targetTable.getPartition(partitionPair.getKey()).getName());
newNames.add(targetTable.getPartition(partitionPair.getValue()).getName());
}
- InsertOverwriteUtil.replacePartition(targetTable, oldNames,
newNames);
+ InsertOverwriteUtil.replacePartition(targetTable, oldNames,
newNames, forceDropPartition);
} catch (Exception e) {
LOG.warn("insert overwrite task making replacement failed because
" + e.getMessage()
+ "all new partition will not be visible and will be
recycled by partition GC.");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java
index d002207dffe..2a774384414 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java
@@ -221,7 +221,7 @@ public class InsertOverwriteTableCommand extends Command
implements NeedAuditEnc
// partitions and return. for transactional, the replacement
will really occur when insert successed,
// i.e. `insertInto` finished. then we call taskGroupSuccess
to make replacement.
insertIntoAutoDetect(ctx, executor, taskId);
- insertOverwriteManager.taskGroupSuccess(taskId, (OlapTable)
targetTable);
+ insertOverwriteManager.taskGroupSuccess(taskId, (OlapTable)
targetTable, isForceDropPartition());
} else {
// it's overwrite table(as all partitions) or specific
partition(s)
List<String> tempPartitionNames =
InsertOverwriteUtil.generateTempPartitionNames(partitionNames);
@@ -433,7 +433,7 @@ public class InsertOverwriteTableCommand extends Command
implements NeedAuditEnc
}
public boolean isForceDropPartition() {
- return false;
+ return true;
}
@Override
diff --git
a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_multiple.out
b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_multiple.out
deleted file mode 100644
index 64fd2a2dabd..00000000000
---
a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_multiple.out
+++ /dev/null
@@ -1,15 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !select_check_1 --
-1 a 2022-01-02
-2 a 2023-01-02
-3 a 2024-01-02
-
--- !select_check__value_should_be_3 --
-3 a 2024-01-02
-
--- !select_check__value_should_be_2 --
-2 a 2024-01-02
-
--- !select_check__value_should_be_3 --
-3 a 2024-01-02
-
diff --git
a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover.out
b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover.out
deleted file mode 100644
index eae52360da6..00000000000
---
a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover.out
+++ /dev/null
@@ -1,14 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !select_check_1 --
-1 a 2022-01-02
-2 a 2023-01-02
-3 a 2024-01-02
-
--- !select_check_1 --
-3 a 2024-01-02
-
--- !select_check_1 --
-1 a 2022-01-02
-2 a 2023-01-02
-3 a 2024-01-02
-
diff --git
a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.out
b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.out
deleted file mode 100644
index 4930be08bd1..00000000000
---
a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.out
+++ /dev/null
@@ -1,23 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !select_check_1 --
-1 a 2022-01-02
-2 a 2023-01-02
-3 a 2024-01-02
-
--- !select_check_2 --
-3 a 2024-01-02
-
--- !select_check_3 --
-4 a 2024-01-02
-
--- !select_check_recover_1_value4 --
-4 a 2024-01-02
-
--- !select_check_recover_2_value3 --
-3 a 2024-01-02
-
--- !select_check_recover_3_value123 --
-1 a 2022-01-02
-2 a 2023-01-02
-3 a 2024-01-02
-
diff --git
a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.out
b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.out
deleted file mode 100644
index a3effb99a08..00000000000
---
a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.out
+++ /dev/null
@@ -1,14 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !select_check_1 --
-1 a 2022-01-02
-2 a 2023-01-02
-3 a 2024-01-02
-
--- !select_check_2 --
-3 a 2024-01-02
-
--- !select_check_3 --
-1 a 2022-01-02
-2 a 2023-01-02
-3 a 2024-01-02
-
diff --git
a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_multiple.groovy
b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_multiple.groovy
deleted file mode 100644
index 94bc3f96ec4..00000000000
---
a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_multiple.groovy
+++ /dev/null
@@ -1,68 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_insert_overwrite_multiple") {
- def table = "test_insert_overwrite_multiple"
-
- // create table and insert data
- sql """ drop table if exists ${table}"""
- sql """
- create table ${table} (
- `id` int(11),
- `name` varchar(128),
- `da` date
- )
- engine=olap
- duplicate key(id)
- partition by range(da)(
- PARTITION p3 VALUES LESS THAN ('2023-01-01'),
- PARTITION p4 VALUES LESS THAN ('2024-01-01'),
- PARTITION p5 VALUES LESS THAN ('2025-01-01')
- )
- distributed by hash(id) buckets 2
- properties(
- "replication_num"="1",
- "light_schema_change"="true"
- );
- """
-
- sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
- sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
- sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
- sql """ SYNC;"""
-
- qt_select_check_1 """ select * from ${table} order by id,name,da; """
-
- sql """ insert overwrite table ${table} values(3, 'a', '2024-01-02'); """
-
- qt_select_check__value_should_be_3 """ select * from ${table} order by
id,name,da; """
-
- sql """ insert overwrite table ${table} values(2, 'a', '2024-01-02'); """
-
- qt_select_check__value_should_be_2 """ select * from ${table} order by
id,name,da; """
-
- sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
- sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
- sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
-
- sql """ recover partition p3 from ${table}; """
- sql """ recover partition p4 from ${table}; """
- sql """ recover partition p5 from ${table}; """
-
- qt_select_check__value_should_be_3 """ select * from ${table} order by
id,name,da; """
-
-}
diff --git
a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover.groovy
b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover.groovy
deleted file mode 100644
index 2668f1b2a71..00000000000
---
a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover.groovy
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_insert_overwrite_recover") {
- def table = "test_insert_overwrite_recover"
-
- // create table and insert data
- sql """ drop table if exists ${table}"""
- sql """
- create table ${table} (
- `id` int(11),
- `name` varchar(128),
- `da` date
- )
- engine=olap
- duplicate key(id)
- partition by range(da)(
- PARTITION p3 VALUES LESS THAN ('2023-01-01'),
- PARTITION p4 VALUES LESS THAN ('2024-01-01'),
- PARTITION p5 VALUES LESS THAN ('2025-01-01')
- )
- distributed by hash(id) buckets 2
- properties(
- "replication_num"="1",
- "light_schema_change"="true"
- );
- """
-
- sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
- sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
- sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
- sql """ SYNC;"""
-
- qt_select_check_1 """ select * from ${table} order by id,name,da; """
-
- sql """ insert overwrite table ${table} values(3, 'a', '2024-01-02'); """
-
- qt_select_check_1 """ select * from ${table} order by id,name,da; """
-
- sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
- sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
- sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
-
- sql """ recover partition p3 from ${table}; """
- sql """ recover partition p4 from ${table}; """
- sql """ recover partition p5 from ${table}; """
-
- qt_select_check_1 """ select * from ${table} order by id,name,da; """
-
-}
diff --git
a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.groovy
b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.groovy
deleted file mode 100644
index e7f65e82610..00000000000
---
a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.groovy
+++ /dev/null
@@ -1,87 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_insert_overwrite_recover_multiple") {
- def table = "test_insert_overwrite_recover_multiple"
-
- // create table and insert data
- sql """ drop table if exists ${table}"""
- sql """
- create table ${table} (
- `id` int(11),
- `name` varchar(128),
- `da` date
- )
- engine=olap
- duplicate key(id)
- partition by range(da)(
- PARTITION p3 VALUES LESS THAN ('2023-01-01'),
- PARTITION p4 VALUES LESS THAN ('2024-01-01'),
- PARTITION p5 VALUES LESS THAN ('2025-01-01')
- )
- distributed by hash(id) buckets 2
- properties(
- "replication_num"="1",
- "light_schema_change"="true"
- );
- """
-
- sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
- sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
- sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
- sql """ SYNC;"""
-
- qt_select_check_1 """ select * from ${table} order by id,name,da; """
-
- sql """ insert overwrite table ${table} values(3, 'a', '2024-01-02'); """
-
- qt_select_check_2 """ select * from ${table} order by id,name,da; """
-
- sql """ insert overwrite table ${table} values(4, 'a', '2024-01-02'); """
-
- qt_select_check_3 """ select * from ${table} order by id,name,da; """
-
- sql """ ALTER TABLE ${table} DROP PARTITION p3; """
- sql """ ALTER TABLE ${table} DROP PARTITION p4; """
- sql """ ALTER TABLE ${table} DROP PARTITION p5; """
-
- sql """ recover partition p3 from ${table}; """
- sql """ recover partition p4 from ${table}; """
- sql """ recover partition p5 from ${table}; """
-
- qt_select_check_recover_1_value4 """ select * from ${table} order by
id,name,da; """
- sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
- sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
- sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
-
- sql """ recover partition p3 from ${table}; """
- sql """ recover partition p4 from ${table}; """
- sql """ recover partition p5 from ${table}; """
-
- qt_select_check_recover_2_value3 """ select * from ${table} order by
id,name,da; """
- sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
- sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
- sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
-
- sql """ recover partition p3 from ${table}; """
- sql """ recover partition p4 from ${table}; """
- sql """ recover partition p5 from ${table}; """
-
- qt_select_check_recover_3_value123 """ select * from ${table} order by
id,name,da; """
-
-
-}
diff --git
a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
deleted file mode 100644
index af5526a22b2..00000000000
---
a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
+++ /dev/null
@@ -1,78 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_insert_overwrite_recover_no_partition") {
- def table = "test_insert_overwrite_recover_no_partition"
- def table_bk = "test_insert_overwrite_recover_no_partition_backup"
- // create table and insert data for range.
- sql """ drop table if exists ${table} force"""
- sql """
- create table ${table} (
- `id` int(11),
- `name` varchar(128),
- `da` date
- )
- engine=olap
- duplicate key(id)
- distributed by hash(id) buckets 2
- properties(
- "replication_num"="1",
- "light_schema_change"="true"
- );
- """
-
- sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
- sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
- sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
- sql """ SYNC;"""
-
- qt_select_check_1 """ select * from ${table} order by id,name,da; """
-
- sql """ insert overwrite table ${table} values(3, 'a', '2024-01-02'); """
-
-
- qt_select_check_2 """ select * from ${table} order by id,name,da; """
-
- // now unpartition data is kept inside the recycle bin.
- // we need to recover it as another partition in the table.
- sql """ recover partition ${table} as p2 from ${table}; """
-
- // create a table to copy the data only for partition p2.
-
- sql """ drop table if exists ${table_bk} force"""
- sql """
- create table ${table_bk} (
- `id` int(11),
- `name` varchar(128),
- `da` date
- )
- engine=olap
- duplicate key(id)
- distributed by hash(id) buckets 2
- properties(
- "replication_num"="1",
- "light_schema_change"="true"
- );
- """
- sql """ insert into ${table_bk} select * from ${table} partition p2; """
-
- sql """ alter table ${table} replace with table ${table_bk}; """
-
- // data from the select should be same as data before overwrite.
- qt_select_check_3 """ select * from ${table} order by id,name,da; """
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
