This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 94f09818efb [fix](catalog recycle bin) Forbid recover partition if
table schame is changed #31146
94f09818efb is described below
commit 94f09818efb3605794fe20778d9d562265f5527d
Author: meiyi <[email protected]>
AuthorDate: Tue Feb 20 22:18:52 2024 +0800
[fix](catalog recycle bin) Forbid recover partition if table schame is
changed #31146
---
.../apache/doris/catalog/CatalogRecycleBin.java | 11 +-
.../recover_with_schema_change.out | 39 +++++++
.../doris/regression/action/TestAction.groovy | 1 +
.../recover_with_schema_change.groovy | 127 +++++++++++++++++++++
4 files changed, 177 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
index 177d4641fc0..3ae63074454 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
@@ -810,8 +810,17 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
throw new DdlException("Can not recover partition[" +
partitionName + "]. Partition item conflict.");
}
- // check if partition name exists
+ // check if schema change
Partition recoverPartition = recoverPartitionInfo.getPartition();
+ Set<Long> tableIndex = table.getIndexIdToMeta().keySet();
+ Set<Long> partitionIndex =
recoverPartition.getMaterializedIndices(IndexExtState.ALL).stream()
+ .map(i -> i.getId()).collect(Collectors.toSet());
+ if (!tableIndex.equals(partitionIndex)) {
+ throw new DdlException("table's index not equal with partition's
index. table's index=" + tableIndex
+ + ", partition's index=" + partitionIndex);
+ }
+
+ // check if partition name exists
Preconditions.checkState(recoverPartition.getName().equalsIgnoreCase(partitionName));
if (!Strings.isNullOrEmpty(newPartitionName)) {
if (table.checkPartitionNameExist(newPartitionName)) {
diff --git
a/regression-test/data/catalog_recycle_bin_p0/recover_with_schema_change.out
b/regression-test/data/catalog_recycle_bin_p0/recover_with_schema_change.out
new file mode 100644
index 00000000000..11667bab84a
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/recover_with_schema_change.out
@@ -0,0 +1,39 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1 a \N 2022-01-02
+2 a \N 2023-01-02
+3 a \N 2024-01-02
+4 a 10 2022-01-02
+
+-- !sql --
+a
+a
+
+-- !sql --
+2 2023-01-02 a
+3 2024-01-02 a
+
+-- !sql --
+a
+a
+
+-- !sql --
+2 a 2023-01-02
+3 a 2024-01-02
+
+-- !sql --
+a
+a
+
+-- !sql --
+2 a 2023-01-02
+3 a 2024-01-02
+
+-- !sql --
+a
+a
+
+-- !sql --
+2 a 2023-01-02
+3 a 2024-01-02
+
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy
index c6b19bccd23..684cb9d5beb 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy
@@ -68,6 +68,7 @@ class TestAction implements SuiteAction {
} else {
if (exception != null || result.exception != null) {
def msg = result.exception?.toString()
+ log.info("Exception: ${msg}")
Assert.assertTrue("Expect exception msg contains
'${exception}', but meet '${msg}'",
msg != null && exception != null &&
msg.contains(exception))
}
diff --git
a/regression-test/suites/catalog_recycle_bin_p0/recover_with_schema_change.groovy
b/regression-test/suites/catalog_recycle_bin_p0/recover_with_schema_change.groovy
new file mode 100644
index 00000000000..a23c449ab95
--- /dev/null
+++
b/regression-test/suites/catalog_recycle_bin_p0/recover_with_schema_change.groovy
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("recover_with_schema_change") {
+ def tableName = "recover_with_schema_change"
+
+ for (int i = 0; i <= 4; i++) {
+ def table = tableName + i
+
+ // create table and insert data
+ sql """ drop table if exists ${table} """
+ sql """
+ create table ${table} (
+ `id` int(11),
+ `name` varchar(128),
+ `da` date
+ )
+ engine=olap
+ duplicate key(id)
+ partition by range(da)(
+ PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+ PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+ PARTITION p5 VALUES LESS THAN ('2025-01-01')
+ )
+ distributed by hash(id) buckets 2
+ properties(
+ "replication_num"="1",
+ "light_schema_change"="true"
+ );
+ """
+
+ sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+ sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+ sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+
+ if (i == 3 || i == 4) {
+ // create mv
+ createMV """ create materialized view mv_${table} as select name
from ${table}; """
+ }
+
+ // drop partition
+ sql """ ALTER TABLE ${table} DROP PARTITION p3; """
+
+ if (i == 0) {
+ // do light weight schema change
+ sql """ ALTER TABLE ${table} add column `age` int after name; """
+ waitForSchemaChangeDone {
+ sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${table}'
ORDER BY createtime DESC LIMIT 1 """
+ time 60
+ }
+
+ // recover partition should success
+ sql """ recover partition p3 from
regression_test_catalog_recycle_bin_p0.${table}; """
+ } else if (i == 1) {
+ // do hard weight schema change
+ sql """ ALTER TABLE ${table} order by(`id`, `da`, `name`); """
+ waitForSchemaChangeDone {
+ sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${table}'
ORDER BY createtime DESC LIMIT 1 """
+ time 60
+ }
+
+ // recover partition should fail
+ test {
+ sql """ recover partition p3 from
regression_test_catalog_recycle_bin_p0.${table}; """
+ exception "table's index not equal with partition's index"
+ }
+ } else if (i == 2) {
+ // create mv
+ createMV """ create materialized view mv_${table} as select name
from ${table}; """
+
+ // recover partition should fail
+ test {
+ sql """ recover partition p3 from
regression_test_catalog_recycle_bin_p0.${table}; """
+ exception "table's index not equal with partition's index"
+ }
+ } else if (i == 3) {
+ // drop mv
+ sql """ drop materialized view mv_${table} on ${table}; """
+
+ // recover partition should fail
+ test {
+ sql """ recover partition p3 from
regression_test_catalog_recycle_bin_p0.${table}; """
+ exception "table's index not equal with partition's index"
+ }
+ } else if (i == 4) {
+ // drop mv
+ sql """ drop materialized view mv_${table} on ${table}; """
+
+ // create mv
+ createMV """ create materialized view mv_${table} as select name
from ${table}; """
+
+ // recover partition should fail
+ test {
+ sql """ recover partition p3 from
regression_test_catalog_recycle_bin_p0.${table}; """
+ exception "table's index not equal with partition's index"
+ }
+ }
+
+ // write data
+ if (i == 0) {
+ sql """ insert into ${table} values(4, 'a', 10, '2022-01-02'); """
+ } else {
+ test {
+ sql """ insert into ${table} values(4, 'b', '2022-01-02'); """
+ exception "Insert has filtered data in strict mode"
+ }
+ order_qt_sql """ select name from ${table}; """
+ }
+
+ // read data
+ order_qt_sql """ select * from ${table}; """
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
