This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new adeae3c4cc2 [fix](cluster key) some data type is not supported for
cluster key (#38966)
adeae3c4cc2 is described below
commit adeae3c4cc209fba619e56bd744988becd7957c8
Author: meiyi <[email protected]>
AuthorDate: Wed Aug 7 23:51:13 2024 +0800
[fix](cluster key) some data type is not supported for cluster key (#38966)
1. some data type is not supported for key column, it's the same for
cluster key column
2. modify some cluster key cases as the mow cases
---
.../plans/commands/info/ColumnDefinition.java | 61 +++++----
.../trees/plans/commands/info/CreateMTMVInfo.java | 2 +-
.../trees/plans/commands/info/CreateTableInfo.java | 4 +-
.../data/unique_with_mow_c_p0/test_delete_sign.out | 69 ++++++++++
.../test_mow_full_clone_exception.out | 37 ++++++
.../test_point_query_cluster_key.groovy | 2 +-
.../ssb_unique_sql_zstd/ddl/customer_create.sql | 1 -
.../ssb_unique_sql_zstd/ddl/date_create.sql | 1 -
.../ssb_unique_sql_zstd/ddl/lineorder_create.sql | 1 -
.../ssb_unique_sql_zstd/ddl/part_create.sql | 1 -
.../ssb_unique_sql_zstd/ddl/supplier_create.sql | 1 -
.../unique_with_mow_c_p0/test_create_table.groovy | 20 +++
.../unique_with_mow_c_p0/test_delete_sign.groovy | 141 +++++++++++++++++++++
.../test_mow_full_clone_exception.groovy | 139 ++++++++++++++++++++
.../test_mow_with_null_sequence.groovy | 2 -
.../test_primary_key_simple_case.groovy | 1 -
.../unique_with_mow_c_p0/test_schema_change.groovy | 1 -
.../test_unique_mow_sequence.groovy | 1 -
.../ssb_unique_sql_zstd/ddl/customer_create.sql | 1 -
.../ssb_unique_sql_zstd/ddl/date_create.sql | 1 -
.../ssb_unique_sql_zstd/ddl/lineorder_create.sql | 1 -
.../ssb_unique_sql_zstd/ddl/part_create.sql | 1 -
.../ssb_unique_sql_zstd/ddl/supplier_create.sql | 1 -
23 files changed, 444 insertions(+), 46 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
index 1d1f82f3fd6..0b2694cc311 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
@@ -199,10 +199,37 @@ public class ColumnDefinition {
}
}
+ private void checkKeyColumnType(boolean isOlap) {
+ if (isOlap) {
+ if (type.isFloatLikeType()) {
+ throw new AnalysisException("Float or double can not used as a
key, use decimal instead.");
+ } else if (type.isStringType()) {
+ throw new AnalysisException("String Type should not be used in
key column[" + name + "]");
+ } else if (type.isArrayType()) {
+ throw new AnalysisException("Array can only be used in the
non-key column of"
+ + " the duplicate table at present.");
+ }
+ }
+ if (type.isBitmapType() || type.isHllType() ||
type.isQuantileStateType()) {
+ throw new AnalysisException("Key column can not set complex type:"
+ name);
+ } else if (type.isJsonType()) {
+ throw new AnalysisException("JsonType type should not be used in
key column[" + getName() + "].");
+ } else if (type.isVariantType()) {
+ throw new AnalysisException("Variant type should not be used in
key column[" + getName() + "].");
+ } else if (type.isMapType()) {
+ throw new AnalysisException("Map can only be used in the non-key
column of"
+ + " the duplicate table at present.");
+ } else if (type.isStructType()) {
+ throw new AnalysisException("Struct can only be used in the
non-key column of"
+ + " the duplicate table at present.");
+ }
+ }
+
/**
* validate column definition and analyze
*/
- public void validate(boolean isOlap, Set<String> keysSet, boolean
isEnableMergeOnWrite, KeysType keysType) {
+ public void validate(boolean isOlap, Set<String> keysSet, Set<String>
clusterKeySet, boolean isEnableMergeOnWrite,
+ KeysType keysType) {
try {
FeNameFormat.checkColumnName(name);
} catch (Exception e) {
@@ -234,33 +261,7 @@ public class ColumnDefinition {
throw new AnalysisException(
String.format("Key column %s can not set aggregation
type", name));
}
- if (isOlap) {
- if (type.isFloatLikeType()) {
- throw new AnalysisException(
- "Float or double can not used as a key, use
decimal instead.");
- } else if (type.isStringType()) {
- throw new AnalysisException(
- "String Type should not be used in key column[" +
name + "]");
- } else if (type.isArrayType()) {
- throw new AnalysisException("Array can only be used in the
non-key column of"
- + " the duplicate table at present.");
- }
- }
- if (type.isBitmapType() || type.isHllType() ||
type.isQuantileStateType()) {
- throw new AnalysisException("Key column can not set complex
type:" + name);
- } else if (type.isJsonType()) {
- throw new AnalysisException(
- "JsonType type should not be used in key column[" +
getName() + "].");
- } else if (type.isVariantType()) {
- throw new AnalysisException(
- "Variant type should not be used in key column[" +
getName() + "].");
- } else if (type.isMapType()) {
- throw new AnalysisException("Map can only be used in the
non-key column of"
- + " the duplicate table at present.");
- } else if (type.isStructType()) {
- throw new AnalysisException("Struct can only be used in the
non-key column of"
- + " the duplicate table at present.");
- }
+ checkKeyColumnType(isOlap);
} else if (aggType == null && isOlap) {
Preconditions.checkState(keysType != null, "keysType is null");
if (keysType.equals(KeysType.DUP_KEYS)) {
@@ -274,6 +275,10 @@ public class ColumnDefinition {
}
}
+ if (clusterKeySet.contains(name)) {
+ checkKeyColumnType(isOlap);
+ }
+
if (aggType != null) {
// check if aggregate type is valid
if (aggType != AggregateType.GENERIC
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index 6ccc6a08fb9..2e8774f4f8a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -162,7 +162,7 @@ public class CreateMTMVInfo {
final boolean finalEnableMergeOnWrite = false;
Set<String> keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
keysSet.addAll(keys);
- columns.forEach(c -> c.validate(true, keysSet,
finalEnableMergeOnWrite, KeysType.DUP_KEYS));
+ columns.forEach(c -> c.validate(true, keysSet, Sets.newHashSet(),
finalEnableMergeOnWrite, KeysType.DUP_KEYS));
if (distribution == null) {
throw new AnalysisException("Create async materialized view should
contain distribution desc");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 4bbae8d4e78..3bc8bb91d62 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -558,7 +558,9 @@ public class CreateTableInfo {
final boolean finalEnableMergeOnWrite = isEnableMergeOnWrite;
Set<String> keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
keysSet.addAll(keys);
- columns.forEach(c -> c.validate(engineName.equals(ENGINE_OLAP),
keysSet, finalEnableMergeOnWrite,
+ Set<String> clusterKeySet =
Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
+ clusterKeySet.addAll(clusterKeysColumnNames);
+ columns.forEach(c -> c.validate(engineName.equals(ENGINE_OLAP),
keysSet, clusterKeySet, finalEnableMergeOnWrite,
keysType));
// validate index
diff --git a/regression-test/data/unique_with_mow_c_p0/test_delete_sign.out
b/regression-test/data/unique_with_mow_c_p0/test_delete_sign.out
new file mode 100644
index 00000000000..60bf812800b
--- /dev/null
+++ b/regression-test/data/unique_with_mow_c_p0/test_delete_sign.out
@@ -0,0 +1,69 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_0 --
+true 1 10 {"c":"c"}
+
+-- !select_1 --
+
+-- !select_2 --
+
+-- !select_3 --
+true 1 10 {"c":"c"}
+
+-- !select_4 --
+
+-- !select_5 --
+
+-- !select_6 --
+
+-- !select_7 --
+true 1 10 {"c":"c"}
+
+-- !select_8 --
+true 1 10 {"c":"c"}
+
+-- !select_9 --
+true 1 30 {"b":"b"}
+
+-- !select_10 --
+true 1 10 {"c":"c"}
+
+-- !select_11 --
+true 1 10 {"c":"c"}
+
+-- !select_12 --
+true 1 30 {"b":"b"}
+
+-- !select_0 --
+true 1 10 {"c":"c"}
+
+-- !select_1 --
+
+-- !select_2 --
+
+-- !select_3 --
+true 1 10 {"c":"c"}
+
+-- !select_4 --
+
+-- !select_5 --
+
+-- !select_6 --
+
+-- !select_7 --
+true 1 10 {"c":"c"}
+
+-- !select_8 --
+true 1 10 {"c":"c"}
+
+-- !select_9 --
+true 1 30 {"b":"b"}
+
+-- !select_10 --
+true 1 10 {"c":"c"}
+
+-- !select_11 --
+true 1 10 {"c":"c"}
+
+-- !select_12 --
+true 1 30 {"b":"b"}
+
diff --git
a/regression-test/data/unique_with_mow_c_p0/test_mow_full_clone_exception.out
b/regression-test/data/unique_with_mow_c_p0/test_mow_full_clone_exception.out
new file mode 100644
index 00000000000..f11c60b41e1
--- /dev/null
+++
b/regression-test/data/unique_with_mow_c_p0/test_mow_full_clone_exception.out
@@ -0,0 +1,37 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1 10
+2 200
+3 30
+4 400
+5 500
+6 600
+7 7
+8 8
+9 9
+10 10
+
+-- !sql --
+1 10
+2 200
+3 30
+4 400
+5 500
+6 600
+7 7
+8 8
+9 9
+10 10
+
+-- !sql --
+1 10
+2 200
+3 30
+4 400
+5 500
+6 600
+7 7
+8 8
+9 9
+10 10
+
diff --git
a/regression-test/suites/point_query_p0/test_point_query_cluster_key.groovy
b/regression-test/suites/point_query_p0/test_point_query_cluster_key.groovy
index 0ce1ddaddec..7497fc8b5e4 100644
--- a/regression-test/suites/point_query_p0/test_point_query_cluster_key.groovy
+++ b/regression-test/suites/point_query_p0/test_point_query_cluster_key.groovy
@@ -239,7 +239,7 @@ suite("test_point_query_cluster_key") {
sql """CREATE TABLE ${tableName} (
`customer_key` bigint(20) NULL,
`customer_btm_value_0` text NULL,
- `customer_btm_value_1` text NULL,
+ `customer_btm_value_1` VARCHAR(1000) NULL,
`customer_btm_value_2` text NULL
) ENGINE=OLAP
UNIQUE KEY(`customer_key`)
diff --git
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/customer_create.sql
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/customer_create.sql
index 3640400704c..1d593fb15ba 100644
---
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/customer_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/customer_create.sql
@@ -14,6 +14,5 @@ DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
diff --git
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/date_create.sql
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/date_create.sql
index 6a065537829..e46caac95dc 100644
---
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/date_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/date_create.sql
@@ -23,6 +23,5 @@ DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
diff --git
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/lineorder_create.sql
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/lineorder_create.sql
index d56c8aee33c..d3015f954c9 100644
---
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/lineorder_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/lineorder_create.sql
@@ -31,6 +31,5 @@ DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 48
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
diff --git
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/part_create.sql
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/part_create.sql
index 34a1555fa52..5ec4981cf10 100644
---
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/part_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/part_create.sql
@@ -15,6 +15,5 @@ DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 10
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
diff --git
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/supplier_create.sql
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/supplier_create.sql
index 662aca9847d..266a2119eb6 100644
---
a/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/supplier_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p0/ssb_unique_sql_zstd/ddl/supplier_create.sql
@@ -13,6 +13,5 @@ DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
diff --git
a/regression-test/suites/unique_with_mow_c_p0/test_create_table.groovy
b/regression-test/suites/unique_with_mow_c_p0/test_create_table.groovy
index c7a530b9143..8cd7cb6d198 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_create_table.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_create_table.groovy
@@ -227,4 +227,24 @@ suite("test_create_table") {
"""
exception "Cluster keys only support unique keys table"
}
+
+ // cluster key contains complex type
+ test {
+ sql """
+ CREATE TABLE `$tableName` (
+ `c_custkey` int(11) NOT NULL COMMENT "",
+ `c_name` varchar(26) NOT NULL COMMENT "",
+ `c_address` varchar(41) NOT NULL COMMENT "",
+ `c_city` variant NOT NULL COMMENT ""
+ )
+ UNIQUE KEY (`c_custkey`)
+ CLUSTER BY (`c_name`, `c_city`, `c_address`)
+ DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
+ exception "Variant type should not be used in key column"
+ }
}
diff --git
a/regression-test/suites/unique_with_mow_c_p0/test_delete_sign.groovy
b/regression-test/suites/unique_with_mow_c_p0/test_delete_sign.groovy
new file mode 100644
index 00000000000..8192aa2de1d
--- /dev/null
+++ b/regression-test/suites/unique_with_mow_c_p0/test_delete_sign.groovy
@@ -0,0 +1,141 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_delete_sign", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+ def tableName = "test_delete_sign"
+ // test delete sigin X sequence column
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """ CREATE TABLE ${tableName} (
+ col1 BOOLEAN,
+ col2 INT,
+ col3 INT,
+ col4 variant
+ ) unique key(col1, col2)
+ CLUSTER BY (`col3`, `col2`)
+ distributed by hash(col1) buckets 1
+ properties(
+ "replication_num" = "1",
+ "function_column.sequence_col" = 'col3',
+ "store_row_column" = "${use_row_store}"
+ ); """
+
+ sql """insert into ${tableName} values(true, 1, 1, '{"a":"a"}');"""
+ sql """insert into ${tableName} values(true, 1, 10,
'{"c":"c"}');"""
+ sql """insert into ${tableName} values(true, 1, 2, '{"b":"b"}');"""
+ qt_select_0 "select * from ${tableName};"
+ sql """insert into ${tableName}
(col1,col2,col3,col4,__DORIS_DELETE_SIGN__)values(true, 1, 10,
'{"c":"c"}',1);"""
+ qt_select_1 "select * from ${tableName};"
+ sql """insert into ${tableName} values(true, 1, 3, '{"c":"c"}');"""
+ qt_select_2 "select * from ${tableName};"
+
+ // test delete sigin X update
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """ CREATE TABLE ${tableName} (
+ col1 BOOLEAN,
+ col2 INT,
+ col3 INT,
+ col4 variant
+ ) unique key(col1, col2)
+ CLUSTER BY (`col2`, `col3`)
+ distributed by hash(col1) buckets 1
+ properties(
+ "replication_num" = "1",
+ "function_column.sequence_col" = 'col3',
+ "store_row_column" = "${use_row_store}"
+ ); """
+
+ sql """insert into ${tableName} values(true, 1, 1, '{"a":"a"}');"""
+ sql """insert into ${tableName} values(true, 1, 10,
'{"c":"c"}');"""
+ sql """insert into ${tableName} values(true, 1, 2, '{"b":"b"}');"""
+ qt_select_3 "select * from ${tableName};"
+ sql """insert into ${tableName}
(col1,col2,col3,col4,__DORIS_DELETE_SIGN__)values(true, 1, 10,
'{"c":"c"}',1);"""
+ qt_select_4 "select * from ${tableName};"
+ sql """insert into ${tableName} values(true, 1, 3, '{"c":"c"}');"""
+ qt_select_5 "select * from ${tableName};"
+ //sql """update ${tableName} set __DORIS_DELETE_SIGN__=0 where
col3=10;"""
+ qt_select_6 "select * from ${tableName};"
+ sql """insert into ${tableName} values(true, 1, 5, '{"c":"c"}');"""
+
+ // test delete sigin X default value
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """ CREATE TABLE ${tableName} (
+ col1 BOOLEAN,
+ col2 INT,
+ col3 INT,
+ col4 variant NULL
+ ) unique key(col1, col2)
+ CLUSTER BY (`col1`, `col3`)
+ distributed by hash(col1) buckets 1
+ properties(
+ "replication_num" = "1",
+ "function_column.sequence_col" = 'col3',
+ "store_row_column" = "${use_row_store}"
+ ); """
+
+ sql """insert into ${tableName} values(true, 1, 1, '{"a":"a"}');"""
+ sql """insert into ${tableName} values(true, 1, 10,
'{"c":"c"}');"""
+ sql """insert into ${tableName} values(true, 1, 2, '{"b":"b"}');"""
+ qt_select_7 "select * from ${tableName};"
+ sql """insert into ${tableName} (col1,col2,col3)values(true, 1,
1);"""
+ qt_select_8 "select * from ${tableName};"
+ sql """insert into ${tableName} values(true, 1, 30,
'{"b":"b"}');"""
+ qt_select_9 "select * from ${tableName};"
+
+ // test delete sigin X txn
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """ CREATE TABLE ${tableName} (
+ col1 BOOLEAN,
+ col2 INT,
+ col3 INT,
+ col4 variant default NULL,
+ INDEX idx_col3 (`col3`) USING INVERTED,
+ ) unique key(col1, col2)
+ CLUSTER BY (`col3`, `col1`, `col2`)
+ distributed by hash(col1) buckets 1
+ properties(
+ "replication_num" = "1",
+ "function_column.sequence_col" = 'col3',
+ "store_row_column" = "${use_row_store}"
+ ); """
+
+ sql """begin"""
+ sql """insert into ${tableName} values(true, 1, 1, '{"a":"a"}');"""
+ sql """insert into ${tableName} values(true, 1, 10,
'{"c":"c"}');"""
+ sql """insert into ${tableName} values(true, 1, 2, '{"b":"b"}');"""
+ sql """commit"""
+ qt_select_10 "select * from ${tableName};"
+ sql """begin"""
+ sql """insert into ${tableName} (col1,col2,col3)values(true, 1,
1);"""
+ sql """commit"""
+ qt_select_11 "select * from ${tableName};"
+ sql """begin"""
+ sql """insert into ${tableName} values(true, 1, 30,
'{"b":"b"}');"""
+ sql """commit"""
+ qt_select_12 "select * from ${tableName};"
+ }
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_c_p0/test_mow_full_clone_exception.groovy
b/regression-test/suites/unique_with_mow_c_p0/test_mow_full_clone_exception.groovy
new file mode 100644
index 00000000000..516cff9d4f5
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_c_p0/test_mow_full_clone_exception.groovy
@@ -0,0 +1,139 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+import org.apache.doris.regression.util.NodeType
+
+// This case can reproduce an clone issue which may produce duplicate keys in
mow table
+// the bug would be triggered in the following condition:
+// 1. replica 0 miss version
+// 2. replica 0 try to do full clone from other replicas
+// 3. the full clone failed and the delete bitmap is overrided incorrectly
+// 4. replica 0 try to do incremental clone again and this time the clone
succeed
+// 5. incremental clone can't fix the delete bitmap overrided by previous
failed full clone
+// 6. duplicate key occurred
+//
+// the bug is fixed in #37001
+
+suite('test_full_clone_exception') {
+ def options = new ClusterOptions()
+ options.feConfigs += [
+ 'disable_tablet_scheduler=true',
+ 'tablet_checker_interval_ms=500',
+ 'schedule_batch_size=1000',
+ 'schedule_slot_num_per_hdd_path=1000',
+ ]
+ options.beConfigs += [
+ 'disable_auto_compaction=true',
+ 'report_tablet_interval_seconds=1',
+ 'enable_java_support=false',
+ ]
+
+ options.enableDebugPoints()
+ options.cloudMode = false
+ docker(options) {
+ def txnFailureInject = 'TxnManager.prepare_txn.random_failed'
+ def fullCloneInject =
'SnapshotManager.create_snapshot_files.allow_inc_clone'
+ def reviseTabletMetaInject='Tablet.revise_tablet_meta_fail'
+ def be1 = sql_return_maparray('show backends').get(0)
+ def be2 = sql_return_maparray('show backends').get(1)
+ def be3 = sql_return_maparray('show backends').get(2)
+
+ def addInjectToBE = { beIdx, injectName, param ->
+ def be = sql_return_maparray('show backends').get(beIdx)
+ GetDebugPoint().enableDebugPoint(be.Host, be.HttpPort as int,
NodeType.BE, injectName, param)
+ }
+
+ def deleteInjectOnBE = { beIdx, injectName ->
+ def be = sql_return_maparray('show backends').get(beIdx)
+ GetDebugPoint().disableDebugPoint(be.Host, be.HttpPort as int,
NodeType.BE, injectName)
+ }
+
+ sql """
+ CREATE TABLE IF NOT EXISTS t (
+ k int,
+ v int
+ )
+ UNIQUE KEY(k)
+ CLUSTER BY(v)
+ DISTRIBUTED BY HASH(k) BUCKETS 1 properties(
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
+
+ sql 'INSERT INTO t VALUES(1,1),(2,2),(3,3),(4,4)'
+ sql 'INSERT INTO t VALUES(1,10),(2,20),(3,30),(4,40)'
+
+ // inject txn failure, make replica 0 miss version
+ addInjectToBE(0, txnFailureInject, [percent:1.0])
+
+ sql 'INSERT INTO t VALUES(2,200),(4,400),(5,500),(6,600)'
+ sql 'INSERT INTO t VALUES(7,7)'
+ sql 'INSERT INTO t VALUES(8,8)'
+ sql 'INSERT INTO t VALUES(9,9)'
+
+ deleteInjectOnBE(0, txnFailureInject)
+
+ sql 'INSERT INTO t VALUES(10,10)'
+
+ sleep 5000
+
+ // check replica 0 miss version
+ def replica1 = sql_return_maparray('show tablets from t').find {
it.BackendId.toLong().equals(be1.BackendId.toLong()) }
+ assertNotNull(replica1)
+ assertEquals(3, replica1.VersionCount.toInteger())
+ assertEquals(3, replica1.Version.toInteger())
+ assertEquals(8, replica1.LstFailedVersion.toInteger())
+
+ def tabletId = replica1.TabletId
+ // inject failure on replica 0, which can't clone succeed
+ addInjectToBE(0, reviseTabletMetaInject, [tablet_id:tabletId])
+ // inject on replica 1, force replica 0 full clone from them
+ addInjectToBE(1, fullCloneInject, [tablet_id:tabletId,
is_full_clone:true])
+ addInjectToBE(2, fullCloneInject, [tablet_id:tabletId,
is_full_clone:true])
+
+ // start clone
+ setFeConfig('disable_tablet_scheduler', false)
+
+ sleep 10000
+
+ // now, there's lots of full clone failures, remove all debug points,
make
+ // replica 0 can do normal incremental clone from other replicas
+ deleteInjectOnBE(0, reviseTabletMetaInject)
+ deleteInjectOnBE(1, fullCloneInject)
+ deleteInjectOnBE(2, fullCloneInject)
+
+ sleep 10000
+
+ // make sure the clone succeed
+ replica1 = sql_return_maparray('show tablets from t').find {
it.BackendId.toLong().equals(be1.BackendId.toLong()) }
+ assertNotNull(replica1)
+ assertEquals(8, replica1.VersionCount.toInteger())
+ assertEquals(8, replica1.Version.toInteger())
+ assertEquals(-1, replica1.LstFailedVersion.toInteger())
+
+ // three replica's content should be consistent
+ sql 'set use_fix_replica=0'
+ qt_sql 'select * from t order by k'
+
+ sql 'set use_fix_replica=1'
+ qt_sql 'select * from t order by k'
+
+ sql 'set use_fix_replica=2'
+ qt_sql 'select * from t order by k'
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_c_p0/test_mow_with_null_sequence.groovy
b/regression-test/suites/unique_with_mow_c_p0/test_mow_with_null_sequence.groovy
index bf3ce215a90..3e6f7cce599 100644
---
a/regression-test/suites/unique_with_mow_c_p0/test_mow_with_null_sequence.groovy
+++
b/regression-test/suites/unique_with_mow_c_p0/test_mow_with_null_sequence.groovy
@@ -31,7 +31,6 @@ suite("test_mow_with_null_sequence") {
PROPERTIES (
"function_column.sequence_col" = 'c_date',
"replication_num" = "1",
- "disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
"""
@@ -65,7 +64,6 @@ suite("test_mow_with_null_sequence") {
PROPERTIES (
"function_column.sequence_col" = 'c_int',
"replication_num" = "1",
- "disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
"""
diff --git
a/regression-test/suites/unique_with_mow_c_p0/test_primary_key_simple_case.groovy
b/regression-test/suites/unique_with_mow_c_p0/test_primary_key_simple_case.groovy
index 94b3051cba3..8e5d683cea5 100644
---
a/regression-test/suites/unique_with_mow_c_p0/test_primary_key_simple_case.groovy
+++
b/regression-test/suites/unique_with_mow_c_p0/test_primary_key_simple_case.groovy
@@ -41,7 +41,6 @@ suite("test_primary_key_simple_case") {
CLUSTER BY(`user_id`, `age`, `cost`, `sex`)
DISTRIBUTED BY HASH(`user_id`)
PROPERTIES ( "replication_num" = "1",
- "disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
"""
diff --git
a/regression-test/suites/unique_with_mow_c_p0/test_schema_change.groovy
b/regression-test/suites/unique_with_mow_c_p0/test_schema_change.groovy
index 26fa94ab4e5..9abee82f7c0 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_schema_change.groovy
@@ -50,7 +50,6 @@ suite("test_schema_change") {
CLUSTER BY(`cost`, `comment`)
DISTRIBUTED BY HASH(`user_id`)
PROPERTIES ( "replication_num" = "1",
- "disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
"""
diff --git
a/regression-test/suites/unique_with_mow_c_p0/test_unique_mow_sequence.groovy
b/regression-test/suites/unique_with_mow_c_p0/test_unique_mow_sequence.groovy
index c3ded30c048..d9e7c53312f 100644
---
a/regression-test/suites/unique_with_mow_c_p0/test_unique_mow_sequence.groovy
+++
b/regression-test/suites/unique_with_mow_c_p0/test_unique_mow_sequence.groovy
@@ -36,7 +36,6 @@ suite("test_unique_mow_sequence") {
"function_column.sequence_type" = 'int',
"compression"="zstd",
"replication_num" = "1",
- "disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
"""
diff --git
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/customer_create.sql
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/customer_create.sql
index 8240bd709ce..9e201b44646 100644
---
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/customer_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/customer_create.sql
@@ -14,6 +14,5 @@ DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
diff --git
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/date_create.sql
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/date_create.sql
index 1ff610fd690..3d12170cf99 100644
---
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/date_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/date_create.sql
@@ -23,6 +23,5 @@ DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
diff --git
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/lineorder_create.sql
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/lineorder_create.sql
index 829b8d65bd4..b9481142be1 100644
---
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/lineorder_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/lineorder_create.sql
@@ -31,6 +31,5 @@ DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 48
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
diff --git
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/part_create.sql
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/part_create.sql
index 9dda02c7b72..3975ff83f09 100644
---
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/part_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/part_create.sql
@@ -15,6 +15,5 @@ DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 10
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
diff --git
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/supplier_create.sql
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/supplier_create.sql
index b827e9b6db4..7e101c5667f 100644
---
a/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/supplier_create.sql
+++
b/regression-test/suites/unique_with_mow_c_p2/ssb_unique_sql_zstd/ddl/supplier_create.sql
@@ -13,6 +13,5 @@ DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10
PROPERTIES (
"compression"="zstd",
"replication_num" = "1",
-"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true"
);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
