This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch setupcbtcleanser
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/setupcbtcleanser by this push:
new 675107f80e0 Add task
675107f80e0 is described below
commit 675107f80e02d167b0bdd9b7f70e7829b426f836
Author: Yi Hu <ya...@google.com>
AuthorDate: Thu Nov 2 15:11:40 2023 -0400
Add task
---
.test-infra/tools/build.gradle | 5 ++
.test-infra/tools/stale_cbt_instances_cleaner.sh | 58 ++++++++++++++++++++++++
2 files changed, 63 insertions(+)
diff --git a/.test-infra/tools/build.gradle b/.test-infra/tools/build.gradle
index 5ed6ebfc18f..048fc0a8d02 100644
--- a/.test-infra/tools/build.gradle
+++ b/.test-infra/tools/build.gradle
@@ -28,6 +28,10 @@ task removeStaleBqDatasets(type: Exec) {
commandLine './stale_bq_datasets_cleaner.sh'
}
+task removeStaleCbtInstances(type: Exec) {
+ commandLine './stale_cbt_instances_cleaner.sh'
+}
+
task removeStaleK8sWorkload(type: Exec) {
commandLine './stale_k8s_workload_cleaner.sh'
}
@@ -35,5 +39,6 @@ task removeStaleK8sWorkload(type: Exec) {
task cleanupOtherStaleResources {
// declared as finalizedBy dependency so that other task continue even if
one dep task fails
finalizedBy tasks.removeStaleBqDatasets
+ finalizedBy tasks.removeStaleCbtInstances
finalizedBy tasks.removeStaleK8sWorkload
}
diff --git a/.test-infra/tools/stale_cbt_instances_cleaner.sh
b/.test-infra/tools/stale_cbt_instances_cleaner.sh
new file mode 100755
index 00000000000..6996c9b0545
--- /dev/null
+++ b/.test-infra/tools/stale_cbt_instances_cleaner.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Deletes stale and old BQ datasets that are left after tests.
+#
+
+set -euo pipefail
+
+PROJECT=apache-beam-testing
+
+# get first 50 instances
+CBT_INSTANCES=`cbt -project=$PROJECT listinstances | awk 'NR>2 {print $1}
NR==52{exit}'`
+
+CLEANUP_INSTANCE_TEMPLATES=(bt-read-tests bt-write-xlang test[a-z]+)
+
+# A grace period of 5 days
+GRACE_PERIOD=$((`date +%s` - 24 * 3600 * 5))
+# count number of failed api calls
+declare -i failed_calls=0
+
+for instance in ${CBT_INSTANCES[@]}; do
+ for template in ${CLEANUP_INSTANCE_TEMPLATES[@]}; do
+ pattern=$template-"([0-9]{8})"-
+ if [[ $instance =~ $pattern ]]; then
+ CREATE_DATE=${BASH_REMATCH[1]}
+ # skip if not a valid date
+ CREATED=`date -ju -f "%Y%m%d-%H%M%S" ${CREATE_DATE}-000000 +%s` ||
continue
+ if [[ $GRACE_PERIOD -gt $CREATED ]]; then
+ if cbt -project=$PROJECT deleteinstance $instance; then
+ echo "Deleted $instance (created $CREATE_DATE)"
+ else
+ failed_calls+=1
+ fi
+ fi
+ break
+ fi
+ done
+done
+
+# fail the script if failed_calls is nonzero
+if [[ failed_calls -ne 0 ]]; then
+ echo "Failed delete $failed_calls instances"
+ exit 1
+fi
