This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new fed9185a2a3 [branch-2.0](cherry-pick) add complex type regression
cases for partial update (#28758) (#29240)
fed9185a2a3 is described below
commit fed9185a2a3e59b1549750e5b0419cb587024100
Author: zhannngchen <[email protected]>
AuthorDate: Thu Dec 28 19:35:14 2023 +0800
[branch-2.0](cherry-pick) add complex type regression cases for partial
update (#28758) (#29240)
cherry-pick #28758 from master
---
.../partial_update/complex_type/array.tsv | 3 +
.../partial_update/complex_type/jsonb.tsv | 3 +
.../partial_update/complex_type/map.tsv | 3 +
.../partial_update/complex_type/struct.tsv | 3 +
.../partial_update/complex_type/varchar.tsv | 2 +
.../test_partial_update_complex_type.out | 45 +++++
...t_partial_update_complex_type_schema_change.out | 67 ++++++++
.../test_partial_update_complex_type.groovy | 127 +++++++++++++++
...artial_update_complex_type_schema_change.groovy | 181 +++++++++++++++++++++
9 files changed, 434 insertions(+)
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/complex_type/array.tsv
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/array.tsv
new file mode 100644
index 00000000000..d5319e25d6f
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/array.tsv
@@ -0,0 +1,3 @@
+2 [1,2,3,5,8,13]
+1 [1,3,5,7,9,11]
+4 [1,2,3,4,5]
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/complex_type/jsonb.tsv
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/jsonb.tsv
new file mode 100644
index 00000000000..6d40308f4ca
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/jsonb.tsv
@@ -0,0 +1,3 @@
+2 {"name": "tom", "age": 35}
+1 {"name": "micheal", "age": 28}
+3 {"name": "kk", "age": 42}
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/complex_type/map.tsv
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/map.tsv
new file mode 100644
index 00000000000..ef06c23eaaa
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/map.tsv
@@ -0,0 +1,3 @@
+2 {"count": 100}
+1 {"num": 9}
+5 {"answer": 42}
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/complex_type/struct.tsv
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/struct.tsv
new file mode 100644
index 00000000000..e4819b5a33f
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/struct.tsv
@@ -0,0 +1,3 @@
+2 {100,1000}
+1 {1, 10}
+6 {1000000, 10000000}
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/complex_type/varchar.tsv
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/varchar.tsv
new file mode 100644
index 00000000000..6eee41dcebf
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/complex_type/varchar.tsv
@@ -0,0 +1,2 @@
+1 apache doris
+2 apache doris 2.0
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_complex_type.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_complex_type.out
new file mode 100644
index 00000000000..0cdd6833745
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_complex_type.out
@@ -0,0 +1,45 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !update_varchar --
+1 apache doris {"jsonk1":123,"jsonk2":456} [100, 200] {"a":
1, "b": 2}
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [300, 400]
{"a": 3, "b": 4}
+
+-- !update_jsonb --
+1 apache doris {"name":"micheal","age":28} [100, 200] {"a":
1, "b": 2}
+2 apache doris 2.0 {"name":"tom","age":35} [300, 400] {"a":
3, "b": 4}
+3 \N {"name":"kk","age":42} [] \N
+
+-- !update_array --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
{"a": 1, "b": 2}
+2 apache doris 2.0 {"name":"tom","age":35} [1, 2, 3, 5, 8, 13]
{"a": 3, "b": 4}
+3 \N {"name":"kk","age":42} [] \N
+4 \N \N [1, 2, 3, 4, 5] \N
+
+-- !update_struct --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
{"a": 1, "b": 10}
+2 apache doris 2.0 {"name":"tom","age":35} [1, 2, 3, 5, 8, 13]
{"a": 100, "b": 1000}
+3 \N {"name":"kk","age":42} [] \N
+4 \N \N [1, 2, 3, 4, 5] \N
+6 \N \N [] {"a": 1000000, "b": 10000000}
+
+-- !update_varchar --
+1 apache doris {"jsonk1":123,"jsonk2":456} [100, 200] {"a":
1, "b": 2}
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [300, 400]
{"a": 3, "b": 4}
+
+-- !update_jsonb --
+1 apache doris {"name":"micheal","age":28} [100, 200] {"a":
1, "b": 2}
+2 apache doris 2.0 {"name":"tom","age":35} [300, 400] {"a":
3, "b": 4}
+3 \N {"name":"kk","age":42} [] \N
+
+-- !update_array --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
{"a": 1, "b": 2}
+2 apache doris 2.0 {"name":"tom","age":35} [1, 2, 3, 5, 8, 13]
{"a": 3, "b": 4}
+3 \N {"name":"kk","age":42} [] \N
+4 \N \N [1, 2, 3, 4, 5] \N
+
+-- !update_struct --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
{"a": 1, "b": 10}
+2 apache doris 2.0 {"name":"tom","age":35} [1, 2, 3, 5, 8, 13]
{"a": 100, "b": 1000}
+3 \N {"name":"kk","age":42} [] \N
+4 \N \N [1, 2, 3, 4, 5] \N
+6 \N \N [] {"a": 1000000, "b": 10000000}
+
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.out
new file mode 100644
index 00000000000..71c30a0cd11
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.out
@@ -0,0 +1,67 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !update_varchar --
+1 apache doris \N
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444}
+
+-- !update_jsonb --
+1 apache doris {"name":"micheal","age":28}
+2 apache doris 2.0 {"name":"tom","age":35}
+3 \N {"name":"kk","age":42}
+
+-- !update_varchar --
+1 apache doris {"name":"micheal","age":28} \N
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [300, 400]
+3 \N {"name":"kk","age":42} \N
+
+-- !update_array --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [1, 2, 3, 5, 8,
13]
+3 \N {"name":"kk","age":42} \N
+4 \N \N [1, 2, 3, 4, 5]
+
+-- !update_varchar --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
\N
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [300, 400]
{"a": 3, "b": 4}
+3 \N {"name":"kk","age":42} \N \N
+4 \N \N [1, 2, 3, 4, 5] \N
+
+-- !update_struct --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
{"a": 1, "b": 10}
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [300, 400]
{"a": 100, "b": 1000}
+3 \N {"name":"kk","age":42} \N \N
+4 \N \N [1, 2, 3, 4, 5] \N
+6 \N \N \N {"a": 1000000, "b": 10000000}
+
+-- !update_varchar --
+1 apache doris \N
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444}
+
+-- !update_jsonb --
+1 apache doris {"name":"micheal","age":28}
+2 apache doris 2.0 {"name":"tom","age":35}
+3 \N {"name":"kk","age":42}
+
+-- !update_varchar --
+1 apache doris {"name":"micheal","age":28} \N
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [300, 400]
+3 \N {"name":"kk","age":42} \N
+
+-- !update_array --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [1, 2, 3, 5, 8,
13]
+3 \N {"name":"kk","age":42} \N
+4 \N \N [1, 2, 3, 4, 5]
+
+-- !update_varchar --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
\N
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [300, 400]
{"a": 3, "b": 4}
+3 \N {"name":"kk","age":42} \N \N
+4 \N \N [1, 2, 3, 4, 5] \N
+
+-- !update_struct --
+1 apache doris {"name":"micheal","age":28} [1, 3, 5, 7, 9, 11]
{"a": 1, "b": 10}
+2 apache doris 2.0 {"jsonk3":333,"jsonk4":444} [300, 400]
{"a": 100, "b": 1000}
+3 \N {"name":"kk","age":42} \N \N
+4 \N \N [1, 2, 3, 4, 5] \N
+6 \N \N \N {"a": 1000000, "b": 10000000}
+
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy
new file mode 100644
index 00000000000..ba13bfbeef9
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy
@@ -0,0 +1,127 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_primary_key_partial_update_complex_type", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+ def tableName = "test_primary_key_partial_update_complex_type"
+ // NOTE:
+ // 1. variant type don't support partial update
+ // 2. the combination of map type and row store may result in
bugs, so we skip map type in temporary
+ //
+ // create table
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """ CREATE TABLE ${tableName} (
+ `id` int(11) NOT NULL COMMENT "用户 ID",
+ `c_varchar` varchar(65533) NULL COMMENT "用户姓名",
+ `c_jsonb` JSONB NULL,
+ `c_array` ARRAY<INT> NULL DEFAULT "[]",
+ `c_struct` STRUCT<a:INT, b:INT> NULL)
+ UNIQUE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES("replication_num" = "1",
"enable_unique_key_merge_on_write" = "true",
+ "store_row_column" = "${use_row_store}"); """
+
+ // insert 2 lines
+ sql """
+ insert into ${tableName} values(2, "doris2", '{"jsonk3": 333,
"jsonk4": 444}', [300, 400], {3, 4})
+ """
+
+ sql """
+ insert into ${tableName} values(1, "doris1", '{"jsonk1": 123,
"jsonk2": 456}', [100, 200], {1, 2})
+ """
+
+ // update varchar column
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_varchar'
+
+ file 'complex_type/varchar.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_varchar"""
+ select * from ${tableName} order by id;
+ """
+
+ // update jsonb column, update 2 rows, add 1 new row
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_jsonb'
+
+ file 'complex_type/jsonb.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_jsonb"""
+ select * from ${tableName} order by id;
+ """
+
+ // update array column, update 2 rows, add 1 new row
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_array'
+
+ file 'complex_type/array.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_array"""
+ select * from ${tableName} order by id;
+ """
+
+ // update struct column, update 2 rows, add 1 new row
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_struct'
+
+ file 'complex_type/struct.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_struct"""
+ select * from ${tableName} order by id;
+ """
+
+ // drop table
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ }
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.groovy
new file mode 100644
index 00000000000..6604ced55bc
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.groovy
@@ -0,0 +1,181 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_primary_key_partial_update_complex_type_schema_change", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+ def tableName =
"test_primary_key_partial_update_complex_type_schema_change"
+ // create table
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """ CREATE TABLE ${tableName} (
+ `id` int(11) NOT NULL COMMENT "用户 ID",
+ `c_varchar` varchar(65533) NULL COMMENT "用户姓名")
+ UNIQUE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES("replication_num" = "1",
"enable_unique_key_merge_on_write" = "true",
+ "store_row_column" = "${use_row_store}"); """
+
+ // insert 2 lines
+ sql """
+ insert into ${tableName} values(2, "doris2")
+ """
+
+ sql """
+ insert into ${tableName} values(1, "doris1")
+ """
+
+ // add new jsonb column
+ sql """
+ ALTER TABLE ${tableName} ADD COLUMN c_jsonb JSONB NULL;
+ """
+
+ sql """
+ insert into ${tableName} values(2, "doris3", '{"jsonk3": 333,
"jsonk4": 444}')
+ """
+
+ // update varchar column
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_varchar'
+
+ file 'complex_type/varchar.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_varchar"""
+ select * from ${tableName} order by id;
+ """
+
+ // update jsonb column, update 2 rows, add 1 new row
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_jsonb'
+
+ file 'complex_type/jsonb.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_jsonb"""
+ select * from ${tableName} order by id;
+ """
+
+ // add new array column
+ sql """
+ ALTER TABLE ${tableName} ADD COLUMN c_array ARRAY<INT> NULL;
+ """
+
+ sql """
+ insert into ${tableName} values(2, "doris3", '{"jsonk3": 333,
"jsonk4": 444}', [300, 400])
+ """
+
+ // update varchar column
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_varchar'
+
+ file 'complex_type/varchar.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_varchar"""
+ select * from ${tableName} order by id;
+ """
+
+ // update array column, update 2 rows, add 1 new row
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_array'
+
+ file 'complex_type/array.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_array"""
+ select * from ${tableName} order by id;
+ """
+
+ // add new struct column
+ sql """
+ ALTER TABLE ${tableName} ADD COLUMN c_struct STRUCT<a:INT,
b:INT> NULL;
+ """
+
+ sql """
+ insert into ${tableName} values(2, "doris3", '{"jsonk3": 333,
"jsonk4": 444}', [300, 400], {3, 4})
+ """
+
+ // update varchar column
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_varchar'
+
+ file 'complex_type/varchar.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_varchar"""
+ select * from ${tableName} order by id;
+ """
+
+ // update struct column, update 2 rows, add 1 new row
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'columns', 'id,c_struct'
+
+ file 'complex_type/struct.tsv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_update_struct"""
+ select * from ${tableName} order by id;
+ """
+
+ // drop table
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
