This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9e1a4b59b0329b2737b858857d5fb752625f48d6 Author: bobhan1 <[email protected]> AuthorDate: Sun Aug 13 10:32:21 2023 +0800 [Enhancement](partial update) Support including delete sign column in partial update stream load (#22874) --- .../apache/doris/planner/StreamLoadPlanner.java | 6 +-- .../partial_update/delete_sign.csv | 3 ++ .../test_partial_update_delete_sign.out | 22 ++++++++ .../test_partial_update_delete_sign.groovy | 60 ++++++++++++++++++++++ 4 files changed, 88 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 96d5cfa6c2..675d2b42d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -153,9 +153,9 @@ public class StreamLoadPlanner { for (ImportColumnDesc importColumnDesc : taskInfo.getColumnExprDescs().descs) { if (importColumnDesc.getColumnName() != null && importColumnDesc.getColumnName().equals(col.getName())) { - if (!col.isVisible()) { - throw new UserException("Partial update should not include invisible column: " - + col.getName()); + if (!col.isVisible() && !Column.DELETE_SIGN.equals(col.getName())) { + throw new UserException("Partial update should not include invisible column except" + + " delete sign column: " + col.getName()); } partialUpdateInputColumns.add(col.getName()); if (destTable.hasSequenceCol() && (taskInfo.hasSequenceCol() || ( diff --git a/regression-test/data/unique_with_mow_p0/partial_update/delete_sign.csv b/regression-test/data/unique_with_mow_p0/partial_update/delete_sign.csv new file mode 100644 index 0000000000..712d86b9ab --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/delete_sign.csv @@ -0,0 +1,3 @@ +1,1 +3,1 +5,1 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out new file mode 100644 index 0000000000..f66f4b6873 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out @@ -0,0 +1,22 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !after_delete -- +2 2 2 2 2 +4 4 4 4 4 + +-- !with_delete_sign -- +1 \N \N \N \N 1 +1 1 1 1 1 0 +2 2 2 2 2 0 +3 \N \N \N \N 1 +3 3 3 3 3 0 +4 4 4 4 4 0 +5 \N \N \N \N 1 +5 5 5 5 5 0 + diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy new file mode 100644 index 0000000000..1aa651c48c --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_partial_update_delete_sign') { + sql 'set enable_nereids_planner=false' + sql "set experimental_enable_nereids_planner=false;" + sql 'set enable_nereids_dml=false' + + def tableName1 = "test_partial_update_delete_sign1" + sql "DROP TABLE IF EXISTS ${tableName1};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1" + );""" + + sql "insert into ${tableName1} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" + qt_sql "select * from ${tableName1} order by k1,c1,c2,c3,c4;" + streamLoad { + table "${tableName1}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'partial_columns', 'true' + set 'columns', 'k1,__DORIS_DELETE_SIGN__' + + file 'delete_sign.csv' + time 10000 // limit inflight 10s + } + sql "sync" + qt_after_delete "select * from ${tableName1} order by k1,c1,c2,c3,c4;" + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_with_delete_sign "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName1};" +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
