This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new bddab94121 [Enhancement](partial update) Support including delete sign
column in partial update stream load (#22874)
bddab94121 is described below
commit bddab941217cf2e8ece3f468b5cf6804f2341c63
Author: bobhan1 <[email protected]>
AuthorDate: Sun Aug 13 10:32:21 2023 +0800
[Enhancement](partial update) Support including delete sign column in
partial update stream load (#22874)
---
.../apache/doris/planner/StreamLoadPlanner.java | 6 +--
.../partial_update/delete_sign.csv | 3 ++
.../test_partial_update_delete_sign.out | 22 ++++++++
.../test_partial_update_delete_sign.groovy | 60 ++++++++++++++++++++++
4 files changed, 88 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
index ccdd7b336e..07d54037d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
@@ -153,9 +153,9 @@ public class StreamLoadPlanner {
for (ImportColumnDesc importColumnDesc :
taskInfo.getColumnExprDescs().descs) {
if (importColumnDesc.getColumnName() != null
&&
importColumnDesc.getColumnName().equals(col.getName())) {
- if (!col.isVisible()) {
- throw new UserException("Partial update should not
include invisible column: "
- + col.getName());
+ if (!col.isVisible() &&
!Column.DELETE_SIGN.equals(col.getName())) {
+ throw new UserException("Partial update should not
include invisible column except"
+ + " delete sign column: " + col.getName());
}
partialUpdateInputColumns.add(col.getName());
if (destTable.hasSequenceCol() &&
(taskInfo.hasSequenceCol() || (
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/delete_sign.csv
b/regression-test/data/unique_with_mow_p0/partial_update/delete_sign.csv
new file mode 100644
index 0000000000..712d86b9ab
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/delete_sign.csv
@@ -0,0 +1,3 @@
+1,1
+3,1
+5,1
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
new file mode 100644
index 0000000000..f66f4b6873
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+
+-- !after_delete --
+2 2 2 2 2
+4 4 4 4 4
+
+-- !with_delete_sign --
+1 \N \N \N \N 1
+1 1 1 1 1 0
+2 2 2 2 2 0
+3 \N \N \N \N 1
+3 3 3 3 3 0
+4 4 4 4 4 0
+5 \N \N \N \N 1
+5 5 5 5 5 0
+
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
new file mode 100644
index 0000000000..1aa651c48c
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('test_partial_update_delete_sign') {
+ sql 'set enable_nereids_planner=false'
+ sql "set experimental_enable_nereids_planner=false;"
+ sql 'set enable_nereids_dml=false'
+
+ def tableName1 = "test_partial_update_delete_sign1"
+ sql "DROP TABLE IF EXISTS ${tableName1};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1"
+ );"""
+
+ sql "insert into ${tableName1}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
+ qt_sql "select * from ${tableName1} order by k1,c1,c2,c3,c4;"
+ streamLoad {
+ table "${tableName1}"
+
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'partial_columns', 'true'
+ set 'columns', 'k1,__DORIS_DELETE_SIGN__'
+
+ file 'delete_sign.csv'
+ time 10000 // limit inflight 10s
+ }
+ sql "sync"
+ qt_after_delete "select * from ${tableName1} order by k1,c1,c2,c3,c4;"
+ sql "set skip_delete_sign=true;"
+ sql "set skip_storage_engine_merge=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ qt_with_delete_sign "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from
${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ sql "drop table if exists ${tableName1};"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
