This is an automated email from the ASF dual-hosted git repository.
hellostephen pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 519e5c88cc6 [test](inverted index)Add upgrade and downgrade test cases
for inverted_index (#37783)
519e5c88cc6 is described below
commit 519e5c88cc615a9f3c4cd25ba5a7b1efcec42b64
Author: ChenPeng2013 <[email protected]>
AuthorDate: Fri Aug 30 10:19:34 2024 +0800
[test](inverted index)Add upgrade and downgrade test cases for
inverted_index (#37783)
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
add case
---
.../inverted_index.groovy | 213 +++++++++++++++++++++
.../suites/inverted_index_up_down_p0/load.groovy | 113 +++++++++++
2 files changed, 326 insertions(+)
diff --git
a/regression-test/suites/inverted_index_up_down_p0/inverted_index.groovy
b/regression-test/suites/inverted_index_up_down_p0/inverted_index.groovy
new file mode 100644
index 00000000000..913602905e7
--- /dev/null
+++ b/regression-test/suites/inverted_index_up_down_p0/inverted_index.groovy
@@ -0,0 +1,213 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_upgrade_downgrade_compatibility_inverted_index","p0,inverted_index,restart_fe")
{
+ def timeout = 120000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+
+ sql "SET enable_match_without_inverted_index = false"
+
+ def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName =
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ sleep(3000) // wait change table state to normal
+ logger.info(table_name + " latest alter job finished, detail:
" + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
+ }
+
+ def wait_for_build_index_on_partition_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW BUILD INDEX WHERE TableName =
"${table_name}";"""
+ def expected_finished_num = alter_res.size();
+ def finished_num = 0;
+ for (int i = 0; i < expected_finished_num; i++) {
+ logger.info(table_name + " build index job state: " +
alter_res[i][7] + i)
+ if (alter_res[i][7] == "FINISHED") {
+ ++finished_num;
+ }
+ }
+ if (finished_num == expected_finished_num) {
+ logger.info(table_name + " all build index jobs finished,
detail: " + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout,
"wait_for_latest_build_index_on_partition_finish timeout")
+ }
+
+ for (version in ["V1", "V2"]) {
+ // duplicate table
+ def tableName = "t_up_down_inverted_index${version}_duplicate"
+ def result = sql(String.format("show create table %s", tableName))
+ if
(!result[0][1].toString().contains(String.format('''"inverted_index_storage_format"
= "%s"''', version))) {
+ throw new IllegalStateException("${tableName}
inverted_index_storage_format exception")
+ }
+ result = sql("select count(*) from ${tableName}")
+ if (result[0][0] != 10) {
+ throw new IllegalStateException("count error")
+ }
+ result = sql("select count(*) from ${tableName} where en in ('I see',
'Let go')")
+ if (result[0][0] != 2) {
+ throw new IllegalStateException("count error")
+ }
+ result = sql("select count(*) from ${tableName} where en match_any
('on')")
+ if (result[0][0] != 2) {
+ throw new IllegalStateException("count error")
+ }
+ result = sql("select count(*) from ${tableName} where ch = '等一等'")
+ if (result[0][0] != 1) {
+ throw new IllegalStateException("count error")
+ }
+
+ sql "alter table ${tableName} add index idx_b(b)"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+ sql "build index idx_b on ${tableName}"
+ wait_for_build_index_on_partition_finish(tableName, timeout)
+
+ sql "insert into ${tableName} values(10001, 10001, 10001, 'Not yet',
'还没')"
+ sql "insert into ${tableName} values(10002, 10002, 10002, 'So long',
'再见')"
+ sql "insert into ${tableName} values(10003, 10003, 10003, 'Why not',
'为什么不')"
+ result = sql("select a, b, c from ${tableName} where a = 10001")
+ if (result.size() != 1 || result[0][0] != 10001 || result[0][1] !=
10001 || result[0][2] != 10001) {
+ throw new IllegalStateException(String.format("%d %d %d",
result[0][0], result[0][1], result[0][2]))
+ }
+ result = sql("select a, b, c from ${tableName} where b = 10001")
+ if (result.size() != 1 || result[0][0] != 10001 || result[0][1] !=
10001 || result[0][2] != 10001) {
+ throw new IllegalStateException(String.format("%d %d %d",
result[0][0], result[0][1], result[0][2]))
+ }
+ sql "delete from ${tableName} where a = 10001"
+ sql "delete from ${tableName} where b = 10002"
+ sql "delete from ${tableName} where c = 10003"
+ for (String columnName in ["a", "b", "c"]) {
+ for (String columnValue in ["10001", "10002", "10003"]) {
+ result = sql("select ${columnName} from ${tableName} where
${columnName} = ${columnValue}")
+ if (result.size() != 0) {
+ throw new IllegalStateException("${result.size()}")
+ }
+ }
+ }
+ sql "alter table ${tableName} drop index idx_b"
+
+ // unique table
+ tableName = "t_up_down_inverted_index${version}_unique"
+ result = sql(String.format("show create table %s", tableName))
+ if
(!result[0][1].toString().contains(String.format('''"inverted_index_storage_format"
= "%s"''', version))) {
+ throw new IllegalStateException("${tableName}
inverted_index_storage_format exception")
+ }
+ result = sql("select a, b, c from ${tableName} where a = 10")
+ if (result.size() != 1 || result[0][0] != 10 || result[0][1] != 10 ||
result[0][2] != 10) {
+ throw new IllegalStateException(String.format("%d %d %d",
result[0][0], result[0][1], result[0][2]))
+ }
+ result = sql("select count(*) from ${tableName} where d =
'2022-10-22'")
+ if (result[0][0] != 1) {
+ throw new IllegalStateException("count err")
+ }
+
+ sql "alter table ${tableName} add index idx_b(b)"
+ sql "alter table ${tableName} add index idx_en(en) using inverted
properties(\"parser\" = \"english\", \"support_phrase\" = \"true\")"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+ sql "build index idx_b on ${tableName}"
+ sql "build index idx_en on ${tableName}"
+ wait_for_build_index_on_partition_finish(tableName, timeout)
+ sql "insert into ${tableName} values(10001, 10001, 10001, '2024-1-1',
'Not yet', '还没')"
+ sql "insert into ${tableName} values(10002, 10002, 10002, '2024-2-1',
'So long', '再见')"
+ sql "insert into ${tableName} values(10003, 10003, 10003, '2024-3-1',
'Why not', '为什么不')"
+ result = sql("select a, b, c from ${tableName} where a = 10001")
+ if (result.size() != 1 || result[0][0] != 10001 || result[0][1] !=
10001 || result[0][2] != 10001) {
+ throw new IllegalStateException(String.format("%d %d %d",
result[0][0], result[0][1], result[0][2]))
+ }
+ result = sql("select a, b, c from ${tableName} where b = 10001")
+ if (result.size() != 1 || result[0][0] != 10001 || result[0][1] !=
10001 || result[0][2] != 10001) {
+ throw new IllegalStateException(String.format("%d %d %d",
result[0][0], result[0][1], result[0][2]))
+ }
+ result = sql("select count(*) from ${tableName} where ch
match_any('什么')")
+ if (result[0][0] != 1) {
+ throw new IllegalStateException("count err")
+ }
+ result = sql("select count(*) from ${tableName} where en
MATCH_PHRASE_PREFIX('lon')")
+ if (result[0][0] != 1) {
+ throw new IllegalStateException("count err")
+ }
+
+ sql "delete from ${tableName} partition p3 where a = 10001"
+ sql "delete from ${tableName} partition p3 where b = 10002"
+ sql "delete from ${tableName} partition p3 where c = 10003"
+ for (String columnName in ["a", "b", "c"]) {
+ for (String columnValue in ["10001", "10002", "10003"]) {
+ result = sql("select ${columnName} from ${tableName} where
${columnName} = ${columnValue}")
+ if (result.size() != 0) {
+ throw new IllegalStateException("${result.size()}")
+ }
+ }
+ }
+ sql "alter table ${tableName} drop index idx_b"
+ sql "alter table ${tableName} drop index idx_en"
+
+ // agg table
+ tableName = "t_up_down_inverted_index${version}_agg"
+ result = sql(String.format("show create table %s", tableName))
+ if
(!result[0][1].toString().contains(String.format('''"inverted_index_storage_format"
= "%s"''', version))) {
+ throw new IllegalStateException("${tableName}
inverted_index_storage_format exception")
+ }
+ result = sql("select a,b,c from ${tableName} where a = \"10\"")
+ if (result.size() != 1 || result[0][0] != "10" || result[0][1] != 10
|| result[0][2] != 10) {
+ throw new IllegalStateException(String.format("%s %d %d",
result[0][0], result[0][1], result[0][2]))
+ }
+ sql "alter table ${tableName} add index idx_b(b)"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+ sql "build index idx_b on ${tableName}"
+ wait_for_build_index_on_partition_finish(tableName, timeout)
+ sql "show index from ${tableName}"
+ sql "insert into ${tableName} values(\"10001\", 10001, 10001, 10001,
10001, 10001)"
+ sql "insert into ${tableName} values(\"10002\", 10002, 10002, 10002,
10002, 10002)"
+ sql "insert into ${tableName} values(\"10003\", 10003, 10003, 10003,
10003, 10003)"
+ result = sql("select a, b, c from ${tableName} where a = \"10001\"")
+ if (result.size() != 1 || result[0][0] != "10001" || result[0][1] !=
10001 || result[0][2] != 10001) {
+ throw new IllegalStateException(String.format("%d %d %d",
result[0][0], result[0][1], result[0][2]))
+ }
+ result = sql("select a, b, c from ${tableName} where b = 10001")
+ if (result.size() != 1 || result[0][0] != "10001" || result[0][1] !=
10001 || result[0][2] != 10001) {
+ throw new IllegalStateException(String.format("%d %d %d",
result[0][0], result[0][1], result[0][2]))
+ }
+ sql "delete from ${tableName} where a = \"10001\""
+ sql "delete from ${tableName} where b = 10002"
+ sql "delete from ${tableName} where c = 10003"
+ result = sql("select a, b, c from ${tableName} where a = \"10001\"")
+ if (result.size() != 0) {
+ throw new IllegalStateException(String.format("%d", result.size()))
+ }
+ for (String columnName in ["b", "c"]) {
+ for (String columnValue in ["10002", "10003"]) {
+ result = sql("select ${columnName} from ${tableName} where
${columnName} = ${columnValue}")
+ if (result.size() != 0) {
+ throw new IllegalStateException("${result.size()}")
+ }
+ }
+ }
+ sql "alter table ${tableName} drop index idx_b"
+ }
+}
diff --git a/regression-test/suites/inverted_index_up_down_p0/load.groovy
b/regression-test/suites/inverted_index_up_down_p0/load.groovy
new file mode 100644
index 00000000000..1c96a20903d
--- /dev/null
+++ b/regression-test/suites/inverted_index_up_down_p0/load.groovy
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_upgrade_downgrade_prepare_inverted_index","p0,inverted_index,restart_fe")
{
+ for (version in ["V1", "V2"]) {
+ sql "drop table if exists t_up_down_inverted_index${version}_duplicate"
+ sql String.format('''
+ create table t_up_down_inverted_index%s_duplicate(
+ a int,
+ b int,
+ c int,
+ en varchar(1024),
+ ch varchar(1024),
+ index idx_a(a),
+ index idx_en(en) using inverted properties("parser" = "english",
"support_phrase" = "false"),
+ index idx_ch(ch) using inverted properties("parser" = "chinese")
+ )
+ DUPLICATE KEY(a, b)
+ PROPERTIES (
+ "replication_num" = "1",
+ "inverted_index_storage_format" = "%s"
+ );''', version, version)
+ sql String.format('''insert into t_up_down_inverted_index%s_duplicate
values
+ (1,1,1,"I see","我明白了"),
+ (2,2,2,"I quit","我不干了"),
+ (3,3,3,"Let go","放手"),
+ (4,4,4,"Me too","我也是"),
+ (5,5,5,"My god","天哪"),
+ (6,6,6,"No way","不行"),
+ (7,7,7,"Come on","来吧赶快"),
+ (8,8,8,"Hold on","等一等"),
+ (9,9,9,"Not bad","还不错"),
+ (1,1,1,"I agree","我同意")''', version)
+
+ sql "drop table if exists t_up_down_inverted_index${version}_unique"
+ sql String.format('''
+ create table t_up_down_inverted_index%s_unique(
+ a int,
+ b int,
+ c int,
+ d date,
+ en varchar(1024),
+ ch varchar(1024),
+ index idx_d(d),
+ index idx_ch(ch) using inverted properties("parser" = "unicode",
"support_phrase" = "true"),
+ index idx_a(a)
+ )
+ UNIQUE KEY(a, b)
+ PARTITION BY RANGE(`a`)
+ (
+ PARTITION `p0` VALUES LESS THAN (100),
+ PARTITION `p1` VALUES LESS THAN (200),
+ PARTITION `p2` VALUES LESS THAN (300),
+ PARTITION `p3` VALUES LESS THAN MAXVALUE
+ )
+ DISTRIBUTED BY HASH(`a`) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1",
+ "inverted_index_storage_format" = "%s"
+ );''', version, version)
+
+ sql String.format('''insert into t_up_down_inverted_index%s_unique
values
+ (1,1,1,"2022-1-22","I see","我明白了"),
+ (2,2,2,"2022-2-22","I quit","我不干了"),
+ (3,3,3,"2022-3-22","Let go","放手"),
+ (4,4,4,"2022-4-22","Me too","我也是"),
+ (5,5,5,"2022-5-22","My god","天哪"),
+ (6,6,6,"2022-6-22","No way","不行"),
+ (7,7,7,"2022-7-22","Come on","来吧赶快"),
+ (8,8,8,"2022-8-22","Hold on","等一等"),
+ (9,9,9,"2022-9-22","Not bad","还不错"),
+ (10,10,10,"2022-10-22","I agree","我同意")''', version)
+
+ sql "drop table if exists t_up_down_inverted_index${version}_agg"
+ sql String.format('''
+ create table t_up_down_inverted_index%s_agg(
+ a varchar(255),
+ b int,
+ c int,
+ d int sum,
+ e int max,
+ f int min,
+ index idx_a(a)
+ )
+ AGGREGATE KEY(a, b, c)
+ DISTRIBUTED BY HASH(`a`) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1",
+ "inverted_index_storage_format" = "%s"
+ );''', version, version)
+ def values = new ArrayList()
+ for (int i = 0; i < 1000; i++ ) {
+ values.add(String.format("(\"%d\",%d,%d, %d, %d, %d)", i, i, i, i,
i, i))
+ }
+ for (int i = 0; i < 10; i++ ) {
+ sql String.format("insert into t_up_down_inverted_index%s_agg
values%s", version, values.join(","))
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
