This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 44a7efff4f7 [branch-2.1] Picks "[Opt](delete) Skip newly inserted rows
check in non-strict mode partial update if the row's delete sign is marked
#40322" (#40383)
44a7efff4f7 is described below
commit 44a7efff4f7e31658009e5c787a93f7031d55471
Author: bobhan1 <[email protected]>
AuthorDate: Mon Sep 9 16:32:24 2024 +0800
[branch-2.1] Picks "[Opt](delete) Skip newly inserted rows check in
non-strict mode partial update if the row's delete sign is marked #40322"
(#40383)
picks https://github.com/apache/doris/pull/40322
---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 3 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 3 +-
.../partial_update/partial_update_delete.csv | 3 +-
.../partial_update/test_partial_update_delete.out | 338 +++++++++++++++++++--
.../test_partial_update_delete.groovy | 300 ++++++++++++------
5 files changed, 532 insertions(+), 115 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 54527a69b76..b9a46ec97ff 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -495,7 +495,8 @@ Status
SegmentWriter::append_block_with_partial_content(const vectorized::Block*
segment_pos);
} else {
- if
(!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update)
{
+ if
(!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update
&&
+ !have_delete_sign) {
std::string error_column;
for (auto cid :
_opts.rowset_ctx->partial_update_info->missing_cids) {
const TabletColumn& col = _tablet_schema->column(cid);
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index 4f9691b2b38..c2418194ed6 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -435,7 +435,8 @@ Status
VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da
DeleteBitmap::TEMP_VERSION_COMMON},
segment_pos);
} else {
- if
(!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update)
{
+ if
(!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update
&&
+ !have_delete_sign) {
std::string error_column;
for (auto cid :
_opts.rowset_ctx->partial_update_info->missing_cids) {
const TabletColumn& col = _tablet_schema->column(cid);
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv
b/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv
index 5f5fbe759f1..e22b5b747a6 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv
+++
b/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv
@@ -1,3 +1,4 @@
1
2
-3
\ No newline at end of file
+3
+7
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out
index e1623d42460..7a639b8d6e2 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out
@@ -1,21 +1,21 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
--- !sql --
+-- !sql1 --
1 1 1 1 1
2 2 2 2 2
3 3 3 3 3
4 4 4 4 4
5 5 5 5 5
--- !sql --
+-- !sql1 --
2 2 2 2 2
4 4 4 4 4
5 5 5 5 5
--- !sql --
+-- !sql1 --
4 4 4 4 4
5 5 5 5 5
--- !with_delete_sign --
+-- !with_delete_sign1 --
1 \N \N 0 \N 1
1 1 1 1 1 0
2 \N \N 0 \N 1
@@ -25,44 +25,291 @@
4 4 4 4 4 0
5 5 5 5 5 0
--- !sql --
+-- !sql2 --
1 1 1 1 1
2 2 2 2 2
3 3 3 3 3
4 4 4 4 4
5 5 5 5 5
+6 6 6 6 6
--- !sql --
+-- !sql2 --
4 4 4 4 4
5 5 5 5 5
+6 6 6 6 6
--- !sql --
-1 \N \N \N \N 1
+-- !sql2 --
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql2 --
+1 \N \N 0 \N 1
+1 1 1 1 1 0
+2 \N \N 0 \N 1
+2 2 2 2 2 0
+3 \N \N 0 \N 1
+3 3 3 3 3 0
+4 \N \N 0 \N 1
+4 4 4 4 4 0
+5 5 5 5 5 0
+6 6 6 6 6 0
+7 \N \N 0 \N 1
+8 \N \N 0 \N 1
+9 \N \N 0 \N 1
+
+-- !sql3 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+1 1 1 1 1 0 1
+1 1 1 1 1 1 1
+2 2 2 2 2 0 2
+2 2 2 2 2 1 2
+3 3 3 3 3 0 3
+3 3 3 3 3 1 3
+4 4 4 4 4 0 4
+4 4 4 4 4 1 4
+5 5 5 5 5 0 5
+6 6 6 6 6 0 6
+7 \N \N 0 \N 1 \N
+8 \N \N 0 \N 1 \N
+9 \N \N 0 \N 1 \N
+
+-- !sql4 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql1 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+
+-- !sql1 --
+2 2 2 2 2
+4 4 4 4 4
+5 5 5 5 5
+
+-- !sql1 --
+4 4 4 4 4
+5 5 5 5 5
+
+-- !with_delete_sign1 --
+1 \N \N 0 \N 1
+1 1 1 1 1 0
+2 \N \N 0 \N 1
+2 2 2 2 2 0
+3 \N \N 0 \N 1
+3 3 3 3 3 0
+4 4 4 4 4 0
+5 5 5 5 5 0
+
+-- !sql2 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql2 --
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql2 --
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql2 --
+1 \N \N 0 \N 1
+1 1 1 1 1 0
+2 \N \N 0 \N 1
+2 2 2 2 2 0
+3 \N \N 0 \N 1
+3 3 3 3 3 0
+4 \N \N 0 \N 1
+4 4 4 4 4 0
+5 5 5 5 5 0
+6 6 6 6 6 0
+7 \N \N 0 \N 1
+8 \N \N 0 \N 1
+9 \N \N 0 \N 1
+
+-- !sql3 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+1 1 1 1 1 0 1
+1 1 1 1 1 1 1
+2 2 2 2 2 0 2
+2 2 2 2 2 1 2
+3 3 3 3 3 0 3
+3 3 3 3 3 1 3
+4 4 4 4 4 0 4
+4 4 4 4 4 1 4
+5 5 5 5 5 0 5
+6 6 6 6 6 0 6
+7 \N \N 0 \N 1 \N
+8 \N \N 0 \N 1 \N
+9 \N \N 0 \N 1 \N
+
+-- !sql4 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql1 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+
+-- !sql1 --
+2 2 2 2 2
+4 4 4 4 4
+5 5 5 5 5
+
+-- !sql1 --
+4 4 4 4 4
+5 5 5 5 5
+
+-- !with_delete_sign1 --
+1 \N \N 0 \N 1
1 1 1 1 1 0
-2 \N \N \N \N 1
+2 \N \N 0 \N 1
2 2 2 2 2 0
-3 \N \N \N \N 1
+3 \N \N 0 \N 1
3 3 3 3 3 0
4 4 4 4 4 0
5 5 5 5 5 0
--- !sql --
+-- !sql2 --
1 1 1 1 1
2 2 2 2 2
3 3 3 3 3
4 4 4 4 4
5 5 5 5 5
+6 6 6 6 6
+
+-- !sql2 --
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql2 --
+5 5 5 5 5
+6 6 6 6 6
--- !sql --
+-- !sql2 --
+1 \N \N 0 \N 1
+1 1 1 1 1 0
+2 \N \N 0 \N 1
+2 2 2 2 2 0
+3 \N \N 0 \N 1
+3 3 3 3 3 0
+4 \N \N 0 \N 1
+4 4 4 4 4 0
+5 5 5 5 5 0
+6 6 6 6 6 0
+7 \N \N 0 \N 1
+8 \N \N 0 \N 1
+9 \N \N 0 \N 1
+
+-- !sql3 --
+1 1 1 1 1
2 2 2 2 2
+3 3 3 3 3
4 4 4 4 4
5 5 5 5 5
+6 6 6 6 6
--- !sql --
+-- !sql3 --
4 4 4 4 4
5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+5 5 5 5 5
+6 6 6 6 6
--- !with_delete_sign --
+-- !sql3 --
+1 1 1 1 1 0 1
+1 1 1 1 1 1 1
+2 2 2 2 2 0 2
+2 2 2 2 2 1 2
+3 3 3 3 3 0 3
+3 3 3 3 3 1 3
+4 4 4 4 4 0 4
+4 4 4 4 4 1 4
+5 5 5 5 5 0 5
+6 6 6 6 6 0 6
+7 \N \N 0 \N 1 \N
+8 \N \N 0 \N 1 \N
+9 \N \N 0 \N 1 \N
+
+-- !sql4 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql1 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+
+-- !sql1 --
+2 2 2 2 2
+4 4 4 4 4
+5 5 5 5 5
+
+-- !sql1 --
+4 4 4 4 4
+5 5 5 5 5
+
+-- !with_delete_sign1 --
1 \N \N 0 \N 1
1 1 1 1 1 0
2 \N \N 0 \N 1
@@ -72,24 +319,75 @@
4 4 4 4 4 0
5 5 5 5 5 0
--- !sql --
+-- !sql2 --
1 1 1 1 1
2 2 2 2 2
3 3 3 3 3
4 4 4 4 4
5 5 5 5 5
+6 6 6 6 6
--- !sql --
+-- !sql2 --
4 4 4 4 4
5 5 5 5 5
+6 6 6 6 6
--- !sql --
-1 \N \N \N \N 1
+-- !sql2 --
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql2 --
+1 \N \N 0 \N 1
1 1 1 1 1 0
-2 \N \N \N \N 1
+2 \N \N 0 \N 1
2 2 2 2 2 0
-3 \N \N \N \N 1
+3 \N \N 0 \N 1
3 3 3 3 3 0
+4 \N \N 0 \N 1
4 4 4 4 4 0
5 5 5 5 5 0
+6 6 6 6 6 0
+7 \N \N 0 \N 1
+8 \N \N 0 \N 1
+9 \N \N 0 \N 1
+
+-- !sql3 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+5 5 5 5 5
+6 6 6 6 6
+
+-- !sql3 --
+1 1 1 1 1 0 1
+1 1 1 1 1 1 1
+2 2 2 2 2 0 2
+2 2 2 2 2 1 2
+3 3 3 3 3 0 3
+3 3 3 3 3 1 3
+4 4 4 4 4 0 4
+4 4 4 4 4 1 4
+5 5 5 5 5 0 5
+6 6 6 6 6 0 6
+7 \N \N 0 \N 1 \N
+8 \N \N 0 \N 1 \N
+9 \N \N 0 \N 1 \N
+
+-- !sql4 --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy
index 7db618798bb..66fc6b7e2fa 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy
@@ -20,99 +20,215 @@ suite('test_partial_update_delete') {
String db = context.config.getDbNameByFile(context.file)
sql "select 1;" // to create database
- for (def use_row_store : [false, true]) {
- logger.info("current params: use_row_store: ${use_row_store}")
-
- connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
- sql "use ${db};"
-
- def tableName1 = "test_partial_update_delete1"
- sql "DROP TABLE IF EXISTS ${tableName1};"
- sql """ CREATE TABLE IF NOT EXISTS ${tableName1} (
- `k1` int NOT NULL,
- `c1` int,
- `c2` int,
- `c3` int NOT NULL,
- `c4` int
- )UNIQUE KEY(k1)
- DISTRIBUTED BY HASH(k1) BUCKETS 1
- PROPERTIES (
- "enable_unique_key_merge_on_write" = "true",
- "disable_auto_compaction" = "true",
- "enable_mow_light_delete" = "false",
- "replication_num" = "1",
- "store_row_column" = "${use_row_store}"); """
-
- def tableName2 = "test_partial_update_delete2"
- sql "DROP TABLE IF EXISTS ${tableName2};"
- sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
- `k` BIGINT NULL
- ) UNIQUE KEY(k)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- PROPERTIES (
- "enable_unique_key_merge_on_write" = "true",
- "disable_auto_compaction" = "true",
- "enable_mow_light_delete" = "false",
- "replication_num" = "1",
- "store_row_column" = "${use_row_store}"); """
-
- sql "insert into ${tableName1}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
- qt_sql "select * from ${tableName1} order by k1;"
- sql "insert into ${tableName2} values(1),(3);"
- sql "delete from ${tableName1} A using ${tableName2} B where
A.k1=B.k;"
- qt_sql "select * from ${tableName1} order by k1;"
-
- sql "delete from ${tableName1} where c2=2;"
- qt_sql "select * from ${tableName1} order by k1;"
-
- sql "set skip_delete_sign=true;"
- sql "set skip_storage_engine_merge=true;"
- sql "set skip_delete_bitmap=true;"
- qt_with_delete_sign "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__
from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
- sql "drop table if exists ${tableName1};"
- sql "drop table if exists ${tableName2};"
-
- sql "set skip_delete_sign=false;"
- sql "set skip_storage_engine_merge=false;"
- sql "set skip_delete_bitmap=false;"
- def tableName3 = "test_partial_update_delete3"
- sql "DROP TABLE IF EXISTS ${tableName3};"
- sql """ CREATE TABLE IF NOT EXISTS ${tableName3} (
- `k1` int NOT NULL,
- `c1` int,
- `c2` int,
- `c3` int,
- `c4` int
- )UNIQUE KEY(k1)
- DISTRIBUTED BY HASH(k1) BUCKETS 1
- PROPERTIES (
- "enable_unique_key_merge_on_write" = "true",
- "disable_auto_compaction" = "true",
- "enable_mow_light_delete" = "false",
- "replication_num" = "1",
- "store_row_column" = "${use_row_store}"); """
-
- sql "insert into ${tableName3}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
- qt_sql "select k1,c1,c2,c3,c4 from ${tableName3} order by
k1,c1,c2,c3,c4;"
- streamLoad {
- table "${tableName3}"
-
- set 'column_separator', ','
- set 'format', 'csv'
- set 'columns', 'k1'
- set 'partial_columns', 'true'
- set 'merge_type', 'DELETE'
-
- file 'partial_update_delete.csv'
- time 10000
+ for (def use_nereids : [true, false]) {
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_nereids: ${use_nereids},
use_row_store: ${use_row_store}")
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+ if (use_nereids) {
+ sql "set enable_nereids_planner=true"
+ sql "set enable_fallback_to_original_planner=false"
+ } else {
+ sql "set enable_nereids_planner=false"
+ }
+ sql "sync;"
+
+ def tableName1 = "test_partial_update_delete1"
+ sql "DROP TABLE IF EXISTS ${tableName1};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int NOT NULL,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_mow_light_delete" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+
+
+ def tableName2 = "test_partial_update_delete2"
+ sql "DROP TABLE IF EXISTS ${tableName2};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
+ `k` BIGINT NULL
+ ) UNIQUE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_mow_light_delete" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+
+ sql "insert into ${tableName1}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
+ qt_sql1 "select * from ${tableName1} order by k1;"
+ sql "insert into ${tableName2} values(1),(3);"
+ sql "delete from ${tableName1} A using ${tableName2} B where
A.k1=B.k;"
+ qt_sql1 "select * from ${tableName1} order by k1;"
+
+ sql "delete from ${tableName1} where c2=2;"
+ qt_sql1 "select * from ${tableName1} order by k1;"
+
+ sql "set skip_delete_sign=true;"
+ sql "set skip_storage_engine_merge=true;"
+ sql "set skip_delete_bitmap=true;"
+ qt_with_delete_sign1 "select
k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by
k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ sql "drop table if exists ${tableName1};"
+ sql "drop table if exists ${tableName2};"
+
+
+ sql "set skip_delete_sign=false;"
+ sql "set skip_storage_engine_merge=false;"
+ sql "set skip_delete_bitmap=false;"
+ def tableName3 = "test_partial_update_delete3"
+ sql "DROP TABLE IF EXISTS ${tableName3};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName3} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int NOT NULL,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_mow_light_delete" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+
+ sql "insert into ${tableName3}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);"
+ qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by
k1,c1,c2,c3,c4;"
+ streamLoad {
+ table "${tableName3}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k1'
+ set 'partial_columns', 'true'
+ set 'merge_type', 'DELETE'
+ file 'partial_update_delete.csv'
+ time 10000
+ }
+ sql "sync"
+ qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by
k1,c1,c2,c3,c4;"
+
+ sql "set enable_insert_strict=false;"
+ sql "set enable_unique_key_partial_update=true;"
+ sql "sync;"
+ sql "insert into ${tableName3}(k1, __DORIS_DELETE_SIGN__)
values(8,1),(4,1),(9,1);"
+ qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by
k1,c1,c2,c3,c4;"
+ sql "set enable_insert_strict=true;"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "sync;"
+
+ sql "set skip_delete_sign=true;"
+ sql "set skip_storage_engine_merge=true;"
+ sql "set skip_delete_bitmap=true;"
+ qt_sql2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from
${tableName3} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ sql "drop table if exists ${tableName3};"
+
+
+ sql "set skip_delete_sign=false;"
+ sql "set skip_storage_engine_merge=false;"
+ sql "set skip_delete_bitmap=false;"
+ def tableName4 = "test_partial_update_delete4"
+ sql "DROP TABLE IF EXISTS ${tableName4};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName4} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int NOT NULL,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_mow_light_delete" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}",
+ "function_column.sequence_col" = "c3"); """
+
+ sql "insert into ${tableName4}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);"
+ qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by
k1,c1,c2,c3,c4;"
+ // if the table has sequence map col, can not set sequence map
col when merge_type=delete
+ streamLoad {
+ table "${tableName4}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k1'
+ set 'partial_columns', 'true'
+ set 'merge_type', 'DELETE'
+ file 'partial_update_delete.csv'
+ time 10000
+ }
+ sql "sync"
+ qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by
k1,c1,c2,c3,c4;"
+
+ sql "set enable_insert_strict=false;"
+ sql "set enable_unique_key_partial_update=true;"
+ sql "sync;"
+ sql "insert into ${tableName4}(k1, __DORIS_DELETE_SIGN__)
values(8,1),(4,1),(9,1);"
+ qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by
k1,c1,c2,c3,c4;"
+ sql "set enable_insert_strict=true;"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "sync;"
+
+ sql "set skip_delete_sign=true;"
+ sql "set skip_storage_engine_merge=true;"
+ sql "set skip_delete_bitmap=true;"
+ qt_sql3 "select
k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName4}
order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ sql "drop table if exists ${tableName4};"
+
+
+ sql "set skip_delete_sign=false;"
+ sql "set skip_storage_engine_merge=false;"
+ sql "set skip_delete_bitmap=false;"
+ def tableName5 = "test_partial_update_delete5"
+ sql "DROP TABLE IF EXISTS ${tableName5};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName5} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int NOT NULL,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_mow_light_delete" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}",
+ "function_column.sequence_type" = "int"); """
+ sql "insert into
${tableName5}(k1,c1,c2,c3,c4,__DORIS_SEQUENCE_COL__)
values(1,1,1,1,1,1),(2,2,2,2,2,2),(3,3,3,3,3,3),(4,4,4,4,4,4),(5,5,5,5,5,5),(6,6,6,6,6,6);"
+ qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by
k1,c1,c2,c3,c4;"
+ // if the table has sequence type col, users must set sequence
col even if merge_type=delete
+ streamLoad {
+ table "${tableName5}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k1'
+ set 'partial_columns', 'true'
+ set 'merge_type', 'DELETE'
+ file 'partial_update_delete.csv'
+ time 10000
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertTrue(json.Message.contains('need to specify the
sequence column'))
+ }
+ }
+ sql "drop table if exists ${tableName5};"
}
- sql "sync"
- qt_sql "select k1,c1,c2,c3,c4 from ${tableName3} order by
k1,c1,c2,c3,c4;"
- sql "set skip_delete_sign=true;"
- sql "set skip_storage_engine_merge=true;"
- sql "set skip_delete_bitmap=true;"
- qt_sql "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from
${tableName3} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
- sql "drop table if exists ${tableName3};"
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
