This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3cd6c4cb9c5 [opt](partial update) Allow to only specify key columns in
partial update (#40736)
3cd6c4cb9c5 is described below
commit 3cd6c4cb9c5369145858ca618ea2b1fc9367da0b
Author: bobhan1 <[email protected]>
AuthorDate: Sat Sep 14 15:30:25 2024 +0800
[opt](partial update) Allow to only specify key columns in partial update
(#40736)
---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 2 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 2 +-
.../schema_change/load_with_key_column.csv | 2 +-
.../test_partial_update_only_keys.out | 59 ++++++++++++++++++++
...artial_update_insert_light_schema_change.groovy | 4 +-
...test_partial_update_insert_schema_change.groovy | 4 +-
.../test_partial_update_only_keys.groovy | 62 ++++++++++++++++++++++
.../test_partial_update_schema_change.groovy | 8 +--
...t_partial_update_schema_change_row_store.groovy | 8 +--
9 files changed, 138 insertions(+), 13 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 84fa6c9e004..0e9b55d99b8 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -535,7 +535,7 @@ Status SegmentWriter::probe_key_for_mow(
// 3. set columns to data convertor and then write all columns
Status SegmentWriter::append_block_with_partial_content(const
vectorized::Block* block,
size_t row_pos, size_t
num_rows) {
- if (block->columns() <= _tablet_schema->num_key_columns() ||
+ if (block->columns() < _tablet_schema->num_key_columns() ||
block->columns() >= _tablet_schema->num_columns()) {
return Status::InvalidArgument(
fmt::format("illegal partial update block columns: {}, num key
columns: {}, total "
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index cae16d98b7c..5663c3779df 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -555,7 +555,7 @@ Status VerticalSegmentWriter::batch_block(const
vectorized::Block* block, size_t
_opts.rowset_ctx->partial_update_info->is_partial_update &&
_opts.write_type == DataWriteType::TYPE_DIRECT &&
!_opts.rowset_ctx->is_transient_rowset_writer) {
- if (block->columns() <= _tablet_schema->num_key_columns() ||
+ if (block->columns() < _tablet_schema->num_key_columns() ||
block->columns() >= _tablet_schema->num_columns()) {
return Status::InvalidArgument(fmt::format(
"illegal partial update block columns: {}, num key
columns: {}, total "
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
b/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
index 128ecf3838d..75456df9f6c 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
+++
b/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
@@ -1 +1 @@
-1, 1
\ No newline at end of file
+2, 2
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_only_keys.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_only_keys.out
new file mode 100644
index 00000000000..9ec27274af4
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_only_keys.out
@@ -0,0 +1,59 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
index 6c15c9562d9..3388b78eb72 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
@@ -264,11 +264,11 @@ suite("test_partial_update_insert_light_schema_change",
"p0") {
sql "sync"
// test insert data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
sql "set enable_unique_key_partial_update=true;"
test {
sql "insert into ${tableName}(c0,c1) values(1, 1);"
- exception "INTERNAL_ERROR"
+ exception "Insert has filtered data in strict mode"
}
sql "insert into ${tableName}(c0,c1,c2) values(1,0,10);"
sql "set enable_unique_key_partial_update=false;"
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
index b648243bb78..85b4328b76f 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
@@ -249,11 +249,11 @@ suite("test_partial_update_insert_schema_change", "p0") {
sql "sync"
// test insert data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
sql "set enable_unique_key_partial_update=true;"
test {
sql "insert into ${tableName}(c0,c1) values(1, 1);"
- exception "INTERNAL_ERROR"
+ exception "Insert has filtered data in strict mode"
}
sql "insert into ${tableName}(c0,c1,c2) values(1,0,10);"
sql "set enable_unique_key_partial_update=false;"
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
new file mode 100644
index 00000000000..1f56643b6e8
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
@@ -0,0 +1,62 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_partial_update_only_keys", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+ def tableName = "test_primary_key_partial_update"
+ sql """ DROP TABLE IF EXISTS ${tableName} force"""
+ sql """ CREATE TABLE ${tableName} (
+ `k` BIGINT NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int)
+ UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "store_row_column" = "${use_row_store}"); """
+ sql """insert into ${tableName} select number,number,number,number
from numbers("number"="3");"""
+ qt_sql """select * from ${tableName} order by k;"""
+ // new rows will be appended
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync"
+ sql "insert into ${tableName}(k) values(0),(1),(4),(5),(6);"
+ qt_sql """select * from ${tableName} order by k;"""
+
+ // fail if has new rows
+ sql "set enable_insert_strict=true;"
+ sql "sync"
+ sql "insert into ${tableName}(k) values(0),(1),(4),(5),(6);"
+ qt_sql """select * from ${tableName} order by k;"""
+ test {
+ sql "insert into ${tableName}(k) values(0),(1),(10),(11);"
+ exception "Insert has filtered data in strict mode"
+ }
+ qt_sql """select * from ${tableName} order by k;"""
+ }
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
index 75fcaff36ad..914244d1fa4 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
@@ -415,12 +415,13 @@ suite("test_partial_update_schema_change", "p0") {
});
// test load data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
streamLoad {
table "${tableName}"
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -434,7 +435,7 @@ suite("test_partial_update_schema_change", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(1, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
@@ -1001,6 +1002,7 @@ suite("test_partial_update_schema_change", "p0") {
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -1014,7 +1016,7 @@ suite("test_partial_update_schema_change", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(1, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
index 5fba5367a01..54e51591725 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
@@ -419,12 +419,13 @@ suite("test_partial_update_row_store_schema_change",
"p0") {
});
// test load data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
streamLoad {
table "${tableName}"
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -438,7 +439,7 @@ suite("test_partial_update_row_store_schema_change", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(1, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
@@ -1014,6 +1015,7 @@ suite("test_partial_update_row_store_schema_change",
"p0") {
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -1027,7 +1029,7 @@ suite("test_partial_update_row_store_schema_change",
"p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(1, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
