This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a0e1d4c08c2 [branch-2.0] Picks "[opt](partial update) Allow to only
specify key columns in partial update #40736" (#40864)
a0e1d4c08c2 is described below
commit a0e1d4c08c2f280c320efed78ec31b7b4c60ef0b
Author: bobhan1 <[email protected]>
AuthorDate: Wed Sep 18 23:30:38 2024 +0800
[branch-2.0] Picks "[opt](partial update) Allow to only specify key columns
in partial update #40736" (#40864)
picks https://github.com/apache/doris/pull/40736
---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 2 +-
.../schema_change/load_with_key_column.csv | 2 +-
.../test_partial_update_only_keys.out | 59 ++++++++++++++++++++
...artial_update_insert_light_schema_change.groovy | 4 +-
...test_partial_update_insert_schema_change.groovy | 4 +-
.../test_partial_update_only_keys.groovy | 62 ++++++++++++++++++++++
.../test_partial_update_schema_change.groovy | 12 +++--
...t_partial_update_schema_change_row_store.groovy | 12 +++--
8 files changed, 141 insertions(+), 16 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 894e0739b66..6aab6c464fd 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -336,7 +336,7 @@ void
SegmentWriter::_serialize_block_to_row_column(vectorized::Block& block) {
// 3. set columns to data convertor and then write all columns
Status SegmentWriter::append_block_with_partial_content(const
vectorized::Block* block,
size_t row_pos, size_t
num_rows) {
- if (block->columns() <= _tablet_schema->num_key_columns() ||
+ if (block->columns() < _tablet_schema->num_key_columns() ||
block->columns() >= _tablet_schema->num_columns()) {
return Status::InternalError(
fmt::format("illegal partial update block columns: {}, num key
columns: {}, total "
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
b/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
index 128ecf3838d..75456df9f6c 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
+++
b/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
@@ -1 +1 @@
-1, 1
\ No newline at end of file
+2, 2
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_only_keys.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_only_keys.out
new file mode 100644
index 00000000000..9ec27274af4
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_only_keys.out
@@ -0,0 +1,59 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
index 22a063c6dcc..19ba9a6fbf9 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
@@ -273,11 +273,11 @@ suite("test_partial_update_insert_light_schema_change",
"p0") {
sql "sync"
// test insert data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
sql "set enable_unique_key_partial_update=true;"
test {
sql "insert into ${tableName}(c0,c1) values(1, 1);"
- exception "INTERNAL_ERROR"
+ exception "Insert has filtered data in strict mode"
}
sql "insert into ${tableName}(c0,c1,c2) values(1,0,10);"
sql "set enable_unique_key_partial_update=false;"
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
index 61f07c8e10f..d7cbcf1fb21 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
@@ -258,11 +258,11 @@ suite("test_partial_update_insert_schema_change", "p0") {
sql "sync"
// test insert data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
sql "set enable_unique_key_partial_update=true;"
test {
sql "insert into ${tableName}(c0,c1) values(1, 1);"
- exception "INTERNAL_ERROR"
+ exception "Insert has filtered data in strict mode"
}
sql "insert into ${tableName}(c0,c1,c2) values(1,0,10);"
sql "set enable_unique_key_partial_update=false;"
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
new file mode 100644
index 00000000000..1f56643b6e8
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
@@ -0,0 +1,62 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_partial_update_only_keys", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+ def tableName = "test_primary_key_partial_update"
+ sql """ DROP TABLE IF EXISTS ${tableName} force"""
+ sql """ CREATE TABLE ${tableName} (
+ `k` BIGINT NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int)
+ UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "store_row_column" = "${use_row_store}"); """
+ sql """insert into ${tableName} select number,number,number,number
from numbers("number"="3");"""
+ qt_sql """select * from ${tableName} order by k;"""
+ // new rows will be appended
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync"
+ sql "insert into ${tableName}(k) values(0),(1),(4),(5),(6);"
+ qt_sql """select * from ${tableName} order by k;"""
+
+ // fail if has new rows
+ sql "set enable_insert_strict=true;"
+ sql "sync"
+ sql "insert into ${tableName}(k) values(0),(1),(4),(5),(6);"
+ qt_sql """select * from ${tableName} order by k;"""
+ test {
+ sql "insert into ${tableName}(k) values(0),(1),(10),(11);"
+ exception "Insert has filtered data in strict mode"
+ }
+ qt_sql """select * from ${tableName} order by k;"""
+ }
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
index 4bce896ed48..6fd866f378f 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
@@ -422,12 +422,13 @@ suite("test_partial_update_schema_change", "p0") {
}
// test load data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
streamLoad {
table "${tableName}"
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -440,8 +441,8 @@ suite("test_partial_update_schema_change", "p0") {
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
- assertEquals(0, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
@@ -1021,6 +1022,7 @@ suite("test_partial_update_schema_change", "p0") {
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -1033,8 +1035,8 @@ suite("test_partial_update_schema_change", "p0") {
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
- assertEquals(0, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
index 1fa4ec39d36..6803d75d840 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
@@ -426,12 +426,13 @@ suite("test_partial_update_row_store_schema_change",
"p0") {
}
// test load data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
streamLoad {
table "${tableName}"
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -444,8 +445,8 @@ suite("test_partial_update_row_store_schema_change", "p0") {
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
- assertEquals(0, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
@@ -1031,6 +1032,7 @@ suite("test_partial_update_row_store_schema_change",
"p0") {
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -1043,8 +1045,8 @@ suite("test_partial_update_row_store_schema_change",
"p0") {
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
- assertEquals(0, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
