This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1f806997f9e [Test](snapshot) Add cleanup snapshot test for recycler
(#58345)
1f806997f9e is described below
commit 1f806997f9eb9b93b19a847ef7fafbc0496c240e
Author: Yixuan Wang <[email protected]>
AuthorDate: Thu Nov 27 10:22:32 2025 +0800
[Test](snapshot) Add cleanup snapshot test for recycler (#58345)
if there is no snapshot, the recycler will recycle all resources
---
.../org/apache/doris/regression/Config.groovy | 24 ++-
.../cloud_p0/conf/regression-conf-custom.groovy | 1 +
.../plugins/cloud_recycler_plugin.groovy | 10 +-
.../recycler/test_recycler_cleanup_snapshot.groovy | 165 +++++++++++++++++++++
4 files changed, 197 insertions(+), 3 deletions(-)
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 1b98e808987..c9061975584 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -176,6 +176,10 @@ class Config {
public String tdeAlgorithm
public String tdeKeyId
+ // for multi version status and cluster snapshot
+ public boolean enableMultiVersionStatus
+ public boolean enableClusterSnapshot
+
Config() {}
Config(
@@ -240,7 +244,9 @@ class Config {
String tdeKeyRegion,
String tdeKeyProvider,
String tdeAlgorithm,
- String tdeKeyId) {
+ String tdeKeyId,
+ boolean enableMultiVersionStatus,
+ boolean enableClusterSnapshot) {
this.s3Source = s3Source
this.caseNamePrefix = caseNamePrefix
this.validateBackupPrefix = validateBackupPrefix
@@ -303,6 +309,8 @@ class Config {
this.tdeKeyProvider = tdeKeyProvider
this.tdeAlgorithm = tdeAlgorithm
this.tdeKeyId = tdeKeyId
+ this.enableMultiVersionStatus = enableMultiVersionStatus
+ this.enableClusterSnapshot = enableClusterSnapshot
}
static String removeDirectoryPrefix(String str) {
@@ -653,7 +661,9 @@ class Config {
configToString(obj.tdeKeyRegion),
configToString(obj.tdeKeyProvider),
configToString(obj.tdeAlgorithm),
- configToString(obj.tdeKeyId)
+ configToString(obj.tdeKeyId),
+ configToBoolean(obj.enableMultiVersionStatus),
+ configToBoolean(obj.enableClusterSnapshot)
)
config.ccrDownstreamUrl = configToString(obj.ccrDownstreamUrl)
@@ -1023,6 +1033,16 @@ class Config {
config.actionParallel = 10
log.info("Set actionParallel to 10 because not
specify.".toString())
}
+
+ if (config.enableMultiVersionStatus == null) {
+ config.enableMultiVersionStatus = false
+ log.info("Set enableMultiVersionStatus to
'${config.enableMultiVersionStatus}' because not specify.".toString())
+ }
+
+ if (config.enableClusterSnapshot == null) {
+ config.enableClusterSnapshot = false
+ log.info("Set enableClusterSnapshot to
'${config.enableClusterSnapshot}' because not specify.".toString())
+ }
}
static String configToString(Object obj) {
diff --git
a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
index d47e204324a..ca79ea9d20a 100644
--- a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
+++ b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
@@ -67,6 +67,7 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + //
keep this line as th
"test_recycler_with_truncate_table," +
"test_recycler_with_txn_label," +
"test_recycler," +
+ "test_recycler_cleanup_snapshot," +
"zzz_the_end_sentinel_do_not_touch" // keep this line as the last line
excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this
line as the first line
diff --git a/regression-test/plugins/cloud_recycler_plugin.groovy
b/regression-test/plugins/cloud_recycler_plugin.groovy
index 25dc6d331fc..3f37bc50fc1 100644
--- a/regression-test/plugins/cloud_recycler_plugin.groovy
+++ b/regression-test/plugins/cloud_recycler_plugin.groovy
@@ -73,10 +73,18 @@ logger.info("Added 'triggerRecycle' function to Suite")
//cloud mode recycler plugin
Suite.metaClass.checkRecycleTable = { String token, String instanceId, String
cloudUniqueId, String tableName,
- Collection<String> tabletIdList /* param */ ->
+ Collection<String> tabletIdList, boolean isSkip = true /* param */ ->
// which suite invoke current function?
Suite suite = delegate as Suite
+ def enableMultiVersionStatus = context.config.enableMultiVersionStatus
+ def enableClusterSnapshot = context.config.enableClusterSnapshot
+
+ if((enableMultiVersionStatus || enableClusterSnapshot) && isSkip) {
+ logger.info("enableMultiVersionStatus or enableClusterSnapshot is true
or isSkip is true, skip checkRecycleTable, it will test recycle in special
cases")
+ return true
+ }
+
// function body
suite.getLogger().info("""Test plugin: suiteName: ${suite.name},
tableName: ${tableName}, instanceId: ${instanceId}, token:${token},
cloudUniqueId:${cloudUniqueId}""".toString())
diff --git
a/regression-test/suites/cloud_p0/recycler/test_recycler_cleanup_snapshot.groovy
b/regression-test/suites/cloud_p0/recycler/test_recycler_cleanup_snapshot.groovy
new file mode 100644
index 00000000000..abe69c5710c
--- /dev/null
+++
b/regression-test/suites/cloud_p0/recycler/test_recycler_cleanup_snapshot.groovy
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonOutput
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_recycler_cleanup_snapshot") {
+ def enableMultiVersionStatus = context.config.enableMultiVersionStatus
+ def enableClusterSnapshot = context.config.enableClusterSnapshot
+
+ if(!enableMultiVersionStatus || !enableClusterSnapshot) {
+ logger.info("enableMultiVersionStatus or enableClusterSnapshot is not
true, skip cleanup snapshot")
+ return
+ }
+
+ def token = "greedisgood9999"
+ def instanceId = context.config.instanceId;
+ def cloudUniqueId = context.config.cloudUniqueId;
+ def dbName = context.config.getDbNameByFile(context.file);
+ def caseStartTime = System.currentTimeMillis()
+ def recyclerLastSuccessTime = -1
+ def recyclerLastFinishTime = -1
+
+ // Make sure to complete at least one round of recycling
+ def getRecycleJobInfo = {
+ def recycleJobInfoApi = { checkFunc ->
+ httpTest {
+ endpoint context.config.recycleServiceHttpAddress
+ uri
"/RecyclerService/http/recycle_job_info?token=$token&instance_id=$instanceId"
+ op "get"
+ check checkFunc
+ }
+ }
+ recycleJobInfoApi.call() {
+ respCode, body ->
+ logger.info("http cli result: ${body} ${respCode}")
+ def recycleJobInfoResult = body
+ logger.info("recycleJobInfoResult:${recycleJobInfoResult}")
+ assertEquals(respCode, 200)
+ def info = parseJson(recycleJobInfoResult.trim())
+ if (info.last_finish_time_ms != null) {
+ recyclerLastFinishTime =
Long.parseLong(info.last_finish_time_ms)
+ }
+ if (info.last_success_time_ms != null) {
+ recyclerLastSuccessTime =
Long.parseLong(info.last_success_time_ms)
+ }
+ }
+ }
+
+ def getCreatedSnapshot = {
+ def snapshotInfo = sql """ SELECT * FROM
information_schema.cluster_snapshots """
+ logger.info("snapshotInfo:${snapshotInfo}")
+ return snapshotInfo
+ }
+
+ def deleteCreatedSnapshot = {
+ def snapshotList = getCreatedSnapshot()
+ for (snapshot : snapshotList) {
+ sql """ ADMIN DROP CLUSTER SNAPSHOT WHERE snapshot_id =
'${snapshot.snapshot_id}'; """
+ }
+ }
+
+ do {
+ triggerRecycle(token, instanceId)
+ Thread.sleep(10000)
+ getRecycleJobInfo()
+ logger.info("caseStartTime=${caseStartTime},
recyclerLastSuccessTime=${recyclerLastSuccessTime}")
+ if (recyclerLastFinishTime > caseStartTime) {
+ break
+ }
+ } while (true)
+ assertEquals(recyclerLastFinishTime, recyclerLastSuccessTime)
+
+ // Make sure to complete at least one round of checking
+ def checkerLastSuccessTime = -1
+ def checkerLastFinishTime = -1
+
+ def triggerChecker = {
+ def triggerCheckerApi = { checkFunc ->
+ httpTest {
+ endpoint context.config.recycleServiceHttpAddress
+ uri
"/RecyclerService/http/check_instance?token=$token&instance_id=$instanceId"
+ op "get"
+ check checkFunc
+ }
+ }
+ triggerCheckerApi.call() {
+ respCode, body ->
+ log.info("http cli result: ${body} ${respCode}".toString())
+ def triggerCheckerResult = body
+
logger.info("triggerCheckerResult:${triggerCheckerResult}".toString())
+ assertTrue(triggerCheckerResult.trim().equalsIgnoreCase("OK"))
+ }
+ }
+ def getCheckJobInfo = {
+ def checkJobInfoApi = { checkFunc ->
+ httpTest {
+ endpoint context.config.recycleServiceHttpAddress
+ uri
"/RecyclerService/http/check_job_info?token=$token&instance_id=$instanceId"
+ op "get"
+ check checkFunc
+ }
+ }
+ checkJobInfoApi.call() {
+ respCode, body ->
+ logger.info("http cli result: ${body} ${respCode}")
+ def checkJobInfoResult = body
+ logger.info("checkJobInfoResult:${checkJobInfoResult}")
+ assertEquals(respCode, 200)
+ def info = parseJson(checkJobInfoResult.trim())
+ if (info.last_finish_time_ms != null) { // Check done
+ checkerLastFinishTime =
Long.parseLong(info.last_finish_time_ms)
+ }
+ if(info.last_success_time_ms != null) {
+ checkerLastSuccessTime =
Long.parseLong(info.last_success_time_ms)
+ }
+ }
+ }
+
+ // collect all tables of tablet ids and tablet names and drop table
+ def allTabletIds = []
+ def allTableNames = []
+ def allTables = sql " SHOW TABLES FROM ${dbName}"
+
+ for (tableInfo : allTables) {
+ def table_name = tableInfo[0]
+
+ def tabletIds = sql " SHOW TABLETS FROM ${dbName}.${table_name}"
+ for (tabletInfo : tabletIds) {
+ allTabletIds.add(tabletInfo[0])
+ }
+ allTableNames.add(table_name)
+
+ logger.info("table name: ${table_name}, tablet ids: ${allTabletIds}")
+ sql " DROP TABLE IF EXISTS ${dbName}.${table_name} FORCE"
+ }
+
+ deleteCreatedSnapshot()
+
+ def retry = 15
+ def success = false
+ // recycle data
+ do {
+ triggerRecycle(token, instanceId)
+ Thread.sleep(20000) // 20s
+ if (checkRecycleTable(token, instanceId, cloudUniqueId, "",
allTabletIds, false)) {
+ success = true
+ break
+ }
+ } while (retry--)
+ assertTrue(success)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
