This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f236261256b [fix](regression) compaction cases adapt
force_olap_table_replica_num option (#28136)
f236261256b is described below
commit f236261256b2b11160e793f28f60160d0e5d1df6
Author: Xiaocc <[email protected]>
AuthorDate: Mon Dec 11 10:08:21 2023 +0800
[fix](regression) compaction cases adapt force_olap_table_replica_num
option (#28136)
---
.../plugins/plugin_get_table_properties.groovy | 30 ++++++++++++++++++++
.../compaction/test_compacation_with_delete.groovy | 4 ++-
.../compaction/test_compaction_agg_keys.groovy | 4 ++-
.../test_compaction_agg_keys_with_delete.groovy | 4 ++-
.../compaction/test_compaction_dup_keys.groovy | 4 ++-
.../test_compaction_dup_keys_with_delete.groovy | 4 ++-
...compaction_uniq_cluster_keys_with_delete.groovy | 4 ++-
.../compaction/test_compaction_uniq_keys.groovy | 5 +++-
.../test_compaction_uniq_keys_cluster_key.groovy | 4 ++-
.../test_compaction_uniq_keys_row_store.groovy | 4 ++-
.../test_compaction_uniq_keys_with_delete.groovy | 4 ++-
.../suites/compaction/test_full_compaction.groovy | 6 ++--
.../test_single_replica_compaction.groovy | 33 ++++++++++++++++++----
.../test_vertical_compaction_agg_keys.groovy | 4 ++-
.../test_vertical_compaction_dup_keys.groovy | 4 ++-
.../test_vertical_compaction_uniq_keys.groovy | 4 ++-
16 files changed, 102 insertions(+), 20 deletions(-)
diff --git a/regression-test/plugins/plugin_get_table_properties.groovy
b/regression-test/plugins/plugin_get_table_properties.groovy
new file mode 100644
index 00000000000..84d16d2d45d
--- /dev/null
+++ b/regression-test/plugins/plugin_get_table_properties.groovy
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.Suite
+
+Suite.metaClass.get_table_replica_num = {String tb_name /* param */ ->
+ Suite suite = delegate as Suite
+ def result = sql """ show create table ${tb_name} """
+ def createTbl = result[0][1].toString()
+ def regexPattern = /"replication_allocation" = "tag.location.default:
(\d+)"/
+ def matcher = (createTbl =~ regexPattern)
+ assertTrue(matcher.find())
+ return matcher.group(1).toInteger()
+}
+
+logger.info("Added 'get_table_replica_num' function to Suite")
\ No newline at end of file
diff --git
a/regression-test/suites/compaction/test_compacation_with_delete.groovy
b/regression-test/suites/compaction/test_compacation_with_delete.groovy
index 74c6c218e12..6530ed57a85 100644
--- a/regression-test/suites/compaction/test_compacation_with_delete.groovy
+++ b/regression-test/suites/compaction/test_compacation_with_delete.groovy
@@ -131,6 +131,8 @@ suite("test_compaction_with_delete") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -144,7 +146,7 @@ suite("test_compaction_with_delete") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount <= 8)
+ assert (rowCount <= 8 * replicaNum)
qt_select_default3 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git a/regression-test/suites/compaction/test_compaction_agg_keys.groovy
b/regression-test/suites/compaction/test_compaction_agg_keys.groovy
index c076327cd5b..20a54bb2d4f 100644
--- a/regression-test/suites/compaction/test_compaction_agg_keys.groovy
+++ b/regression-test/suites/compaction/test_compaction_agg_keys.groovy
@@ -137,6 +137,8 @@ suite("test_compaction_agg_keys") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -152,7 +154,7 @@ suite("test_compaction_agg_keys") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount < 8)
+ assert (rowCount < 8 * replicaNum)
qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy
b/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy
index 4499ce1f5e6..36a8383dda2 100644
---
a/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy
+++
b/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy
@@ -147,6 +147,8 @@ suite("test_compaction_agg_keys_with_delete") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -162,7 +164,7 @@ suite("test_compaction_agg_keys_with_delete") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount < 8)
+ assert (rowCount < 8 * replicaNum)
qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git a/regression-test/suites/compaction/test_compaction_dup_keys.groovy
b/regression-test/suites/compaction/test_compaction_dup_keys.groovy
index fc6a0511bff..d9d19d7bb86 100644
--- a/regression-test/suites/compaction/test_compaction_dup_keys.groovy
+++ b/regression-test/suites/compaction/test_compaction_dup_keys.groovy
@@ -136,6 +136,8 @@ suite("test_compaction_dup_keys") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -151,7 +153,7 @@ suite("test_compaction_dup_keys") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount <= 8)
+ assert (rowCount <= 8 * replicaNum)
qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy
b/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy
index 27eea610bbd..18e88851226 100644
---
a/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy
+++
b/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy
@@ -149,6 +149,8 @@ suite("test_compaction_dup_keys_with_delete") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -164,7 +166,7 @@ suite("test_compaction_dup_keys_with_delete") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount <= 8)
+ assert (rowCount <= 8 * replicaNum)
qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/compaction/test_compaction_uniq_cluster_keys_with_delete.groovy
b/regression-test/suites/compaction/test_compaction_uniq_cluster_keys_with_delete.groovy
index dc6cfe72082..76347524005 100644
---
a/regression-test/suites/compaction/test_compaction_uniq_cluster_keys_with_delete.groovy
+++
b/regression-test/suites/compaction/test_compaction_uniq_cluster_keys_with_delete.groovy
@@ -156,6 +156,8 @@ suite("test_compaction_uniq_cluster_keys_with_delete") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -169,7 +171,7 @@ suite("test_compaction_uniq_cluster_keys_with_delete") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount < 8)
+ assert (rowCount < 8 * replicaNum)
qt_select_default3 """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys.groovy
b/regression-test/suites/compaction/test_compaction_uniq_keys.groovy
index 316229f58fc..34b357627d8 100644
--- a/regression-test/suites/compaction/test_compaction_uniq_keys.groovy
+++ b/regression-test/suites/compaction/test_compaction_uniq_keys.groovy
@@ -135,6 +135,9 @@ suite("test_compaction_uniq_keys") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
+
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -148,7 +151,7 @@ suite("test_compaction_uniq_keys") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount < 8)
+ assert (rowCount < 8 * replicaNum)
qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/compaction/test_compaction_uniq_keys_cluster_key.groovy
b/regression-test/suites/compaction/test_compaction_uniq_keys_cluster_key.groovy
index 39d485c35f3..3a8fee9aba2 100644
---
a/regression-test/suites/compaction/test_compaction_uniq_keys_cluster_key.groovy
+++
b/regression-test/suites/compaction/test_compaction_uniq_keys_cluster_key.groovy
@@ -140,6 +140,8 @@ suite("test_compaction_uniq_keys_cluster_key") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -153,7 +155,7 @@ suite("test_compaction_uniq_keys_cluster_key") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount < 8)
+ assert (rowCount < 8 * replicaNum)
qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
} finally {
// try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy
b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy
index 7187e0d4333..484ae9a7f47 100644
---
a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy
+++
b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy
@@ -192,6 +192,8 @@ suite("test_compaction_uniq_keys_row_store") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -205,7 +207,7 @@ suite("test_compaction_uniq_keys_row_store") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount < 8)
+ assert (rowCount < 8 * replicaNum)
checkValue()
} finally {
// try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy
b/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy
index b45f90683aa..3902264de28 100644
---
a/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy
+++
b/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy
@@ -151,6 +151,8 @@ suite("test_compaction_uniq_keys_with_delete") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -164,7 +166,7 @@ suite("test_compaction_uniq_keys_with_delete") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount < 8)
+ assert (rowCount < 8 * replicaNum)
qt_select_default3 """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git a/regression-test/suites/compaction/test_full_compaction.groovy
b/regression-test/suites/compaction/test_full_compaction.groovy
index eaebd33377d..81f22bf37c5 100644
--- a/regression-test/suites/compaction/test_full_compaction.groovy
+++ b/regression-test/suites/compaction/test_full_compaction.groovy
@@ -100,6 +100,8 @@ suite("test_full_compaction") {
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
// before full compaction, there are 7 rowsets.
int rowsetCount = 0
for (String[] tablet in tablets) {
@@ -112,7 +114,7 @@ suite("test_full_compaction") {
assert tabletJson.rowsets instanceof List
rowsetCount +=((List<String>) tabletJson.rowsets).size()
}
- assert (rowsetCount == 7)
+ assert (rowsetCount == 7 * replicaNum)
// trigger full compactions for all tablets in ${tableName}
for (String[] tablet in tablets) {
@@ -166,7 +168,7 @@ suite("test_full_compaction") {
assert tabletJson.rowsets instanceof List
rowsetCount +=((List<String>) tabletJson.rowsets).size()
}
- assert (rowsetCount == 1)
+ assert (rowsetCount == 1 * replicaNum)
// make sure all hidden data has been deleted
// (1,100)(2,200)
diff --git
a/regression-test/suites/compaction/test_single_replica_compaction.groovy
b/regression-test/suites/compaction/test_single_replica_compaction.groovy
index 4e3fba73660..ea76246dc0f 100644
--- a/regression-test/suites/compaction/test_single_replica_compaction.groovy
+++ b/regression-test/suites/compaction/test_single_replica_compaction.groovy
@@ -77,6 +77,26 @@ suite("test_single_replica_compaction", "p2") {
return out
}
+ def triggerFullCompaction = { be_host, be_http_port, table_id ->
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl -X POST http://${be_host}:${be_http_port}";)
+ sb.append("/api/compaction/run?table_id=")
+ sb.append(table_id)
+ sb.append("&compact_type=full")
+
+ String command = sb.toString()
+ logger.info(command)
+ process = command.execute()
+ code = process.waitFor()
+ err = IOGroovyMethods.getText(new BufferedReader(new
InputStreamReader(process.getErrorStream())));
+ out = process.getText()
+ logger.info("Run compaction: code=" + code + ", out=" + out + ",
disableAutoCompaction " + disableAutoCompaction + ", err=" + err)
+ if (!disableAutoCompaction) {
+ return "Success, " + out
+ }
+ assertEquals(code, 0)
+ return out
+ }
def waitForCompaction = { be_host, be_http_port, tablet_id ->
boolean running = true
do {
@@ -129,7 +149,7 @@ suite("test_single_replica_compaction", "p2") {
UNIQUE KEY(`id`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`id`) BUCKETS 1
- PROPERTIES ( "replication_num" = "3",
"enable_single_replica_compaction" = "true" );
+ PROPERTIES ( "replication_num" = "3",
"enable_single_replica_compaction" = "true", "enable_unique_key_merge_on_write"
= "false" );
"""
String[][] tablets = sql """ show tablets from ${tableName}; """
@@ -146,6 +166,9 @@ suite("test_single_replica_compaction", "p2") {
// and `show tablets` will return 3 different replicas with the same
tablet.
// So we can use the same tablet_id to get tablet/trigger compaction
with different backends.
String tablet_id = tablets[0][0]
+ String[][] tablet_info = sql """ show tablet ${tablet_id}; """
+ logger.info("tablet: " + tablet_info)
+ def table_id = tablet_info[0][5]
for (String[] tablet in tablets) {
String trigger_backend_id = tablet[2]
def tablet_status =
getTabletStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
@@ -213,8 +236,8 @@ suite("test_single_replica_compaction", "p2") {
// trigger follower be to fetch compaction result
for (String id in follower_backend_id) {
- assertTrue(triggerCompaction(backendId_to_backendIP[id],
backendId_to_backendHttpPort[id],
- "cumulative", tablet_id).contains("Success"));
+ assertTrue(triggerFullCompaction(backendId_to_backendIP[id],
backendId_to_backendHttpPort[id],
+ table_id).contains("Success"));
waitForCompaction(backendId_to_backendIP[id],
backendId_to_backendHttpPort[id], tablet_id)
}
@@ -228,8 +251,8 @@ suite("test_single_replica_compaction", "p2") {
// // trigger follower be to fetch compaction result
for (String id in follower_backend_id) {
- assertTrue(triggerCompaction(backendId_to_backendIP[id],
backendId_to_backendHttpPort[id],
- "base", tablet_id).contains("Success"));
+ assertTrue(triggerFullCompaction(backendId_to_backendIP[id],
backendId_to_backendHttpPort[id],
+ table_id).contains("Success"));
waitForCompaction(backendId_to_backendIP[id],
backendId_to_backendHttpPort[id], tablet_id)
}
diff --git
a/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy
b/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy
index 01553550154..8aa8ba387b7 100644
--- a/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy
+++ b/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy
@@ -148,6 +148,8 @@ suite("test_vertical_compaction_agg_keys") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -161,7 +163,7 @@ suite("test_vertical_compaction_agg_keys") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount < 8)
+ assert (rowCount < 8 * replicaNum)
qt_select_default3 """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy
b/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy
index fdd43ff92c6..00951981326 100644
--- a/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy
+++ b/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy
@@ -148,6 +148,8 @@ suite("test_vertical_compaction_dup_keys") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -161,7 +163,7 @@ suite("test_vertical_compaction_dup_keys") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount <= 8)
+ assert (rowCount <= 8 * replicaNum)
qt_select_default3 """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy
b/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy
index c417b50b6a1..dec38b56607 100644
---
a/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy
+++
b/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy
@@ -145,6 +145,8 @@ suite("test_vertical_compaction_uniq_keys") {
} while (running)
}
+ def replicaNum = get_table_replica_num(tableName)
+ logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
@@ -158,7 +160,7 @@ suite("test_vertical_compaction_uniq_keys") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount < 8)
+ assert (rowCount < 8 * replicaNum)
qt_select_default3 """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
