This is an automated email from the ASF dual-hosted git repository.
wyxxxcat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 73d9733c78a [fix](cloud) Fix recycler checks for shared storage in
clone and rollback scenarios (#69492)
73d9733c78a is described below
commit 73d9733c78a48c82f04aa5f70fe118bebd11b6f4
Author: Yixuan Wang <[email protected]>
AuthorDate: Wed Oct 28 10:08:55 2026 +0800
[fix](cloud) Fix recycler checks for shared storage in clone and rollback
scenarios (#69492)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
cloud/src/recycler/checker.cpp | 207 +++++++++++++++++++--
cloud/src/recycler/checker.h | 92 ++++++++-
cloud/src/recycler/recycler_service.cpp | 2 +-
.../plugins/snapshot_recycler_plugin.groovy | 15 +-
4 files changed, 282 insertions(+), 34 deletions(-)
diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp
index e122bbfb861..f1caa677b75 100644
--- a/cloud/src/recycler/checker.cpp
+++ b/cloud/src/recycler/checker.cpp
@@ -130,7 +130,8 @@ TxnErrorCode collect_pending_table_stream_drops(
return iter->error_code();
}
-Checker::Checker(std::shared_ptr<TxnKv> txn_kv) : txn_kv_(std::move(txn_kv)) {
+Checker::Checker(std::shared_ptr<TxnKv> txn_kv)
+ : txn_kv_(std::move(txn_kv)),
resource_mgr_(std::make_shared<ResourceManager>(txn_kv_)) {
ip_port_ = std::string(butil::my_ip_cstr()) + ":" +
std::to_string(config::brpc_listen_port);
}
@@ -150,13 +151,18 @@ int Checker::start() {
std::chrono::seconds(config::recycler_sleep_before_scheduling_seconds));
while (!stopped()) {
std::vector<InstanceInfoPB> instances;
- get_all_instances(txn_kv_.get(), instances);
+ if (get_all_instances(txn_kv_.get(), instances) != 0) {
+ // Do not publish a partial owner index after a failed scan.
+ instances.clear();
+ }
LOG(INFO) << "Checker get instances: " << [&instances] {
std::stringstream ss;
for (auto& i : instances) ss << ' ' << i.instance_id();
return ss.str();
}();
if (!instances.empty()) {
+ auto shared_storage_index =
+ std::make_shared<SharedStorageIndex>(txn_kv_,
instances);
// enqueue instances
std::lock_guard lock(mtx_);
for (auto& instance : instances) {
@@ -169,7 +175,8 @@ int Checker::start() {
pending_instance_map_.insert({instance.instance_id(), enqueue_time_s});
// skip instance already in pending queue
if (success) {
- pending_instance_queue_.push_back(std::move(instance));
+ pending_instance_queue_.push_back(
+ {std::move(instance), shared_storage_index});
}
}
pending_instance_cond_.notify_all();
@@ -193,6 +200,7 @@ int Checker::start() {
while (!stopped()) {
// fetch instance to check
InstanceInfoPB instance;
+ std::shared_ptr<SharedStorageIndex> shared_storage_index;
long enqueue_time_s = 0;
{
std::unique_lock lock(mtx_);
@@ -202,7 +210,9 @@ int Checker::start() {
if (stopped()) {
return;
}
- instance = std::move(pending_instance_queue_.front());
+ instance = std::move(pending_instance_queue_.front().instance);
+ shared_storage_index =
+
std::move(pending_instance_queue_.front().shared_storage_index);
pending_instance_queue_.pop_front();
enqueue_time_s = pending_instance_map_[instance.instance_id()];
pending_instance_map_.erase(instance.instance_id());
@@ -217,7 +227,9 @@ int Checker::start() {
continue;
}
}
- auto checker = std::make_shared<InstanceChecker>(txn_kv_,
instance.instance_id());
+ auto checker = std::make_shared<InstanceChecker>(txn_kv_,
instance.instance_id(),
+ resource_mgr_,
+
std::move(shared_storage_index));
if (checker->init(instance) != 0) {
LOG(WARNING) << "failed to init instance checker, instance_id="
<< instance.instance_id();
@@ -424,7 +436,7 @@ void Checker::do_inspect(const InstanceInfoPB& instance) {
<< " key=" << hex(check_job_key);
return;
}
- auto checker = InstanceChecker(txn_kv_, instance.instance_id());
+ auto checker = InstanceChecker(txn_kv_, instance.instance_id(),
resource_mgr_);
if (checker.init(instance) != 0) {
LOG_CHECK_INTERVAL_ALARM << "failed to init instance checker,
instance_id="
<< instance.instance_id();
@@ -519,16 +531,156 @@ int key_exist(TxnKv* txn_kv, std::string_view key) {
}
}
-InstanceChecker::InstanceChecker(std::shared_ptr<TxnKv> txn_kv, const
std::string& instance_id)
- : txn_kv_(txn_kv), instance_id_(instance_id) {
+SharedStorageIndex::SharedStorageIndex(std::shared_ptr<TxnKv> txn_kv,
+
std::optional<std::vector<InstanceInfoPB>> instances)
+ : txn_kv_(std::move(txn_kv)),
preloaded_instances_(std::move(instances)) {}
+
+std::optional<SharedStorageIndex::StorageLocation>
SharedStorageIndex::s3_location(
+ const ObjectStoreInfoPB& obj_info) {
+ // Missing connection info must not make unrelated resources match.
+ if (obj_info.endpoint().empty() || obj_info.bucket().empty()) {
+ return std::nullopt;
+ }
+ return StorageLocation {"s3", obj_info.endpoint(), obj_info.bucket(),
obj_info.prefix()};
+}
+
+std::optional<SharedStorageIndex::StorageLocation>
SharedStorageIndex::hdfs_location(
+ const HdfsVaultInfo& hdfs_info) {
+ if (hdfs_info.build_conf().fs_name().empty()) {
+ return std::nullopt;
+ }
+ return StorageLocation {"hdfs", hdfs_info.build_conf().fs_name(), "",
hdfs_info.prefix()};
+}
+
+void SharedStorageIndex::ensure_loaded() {
+ std::call_once(load_once_, [this] {
+ load_status_ = load_all_instances();
+ if (load_status_ != 0) {
+ // Never expose a partially built index.
+ instance_resource_locations_.clear();
+ location_to_owners_.clear();
+ }
+ });
+}
+
+int SharedStorageIndex::load_all_instances() {
+ std::vector<InstanceInfoPB> instances;
+ if (preloaded_instances_.has_value()) {
+ instances = std::move(*preloaded_instances_);
+ preloaded_instances_.reset();
+ } else if (get_all_instances(txn_kv_.get(), instances) != 0) {
+ return -1;
+ }
+ // Load all instances, even those not scheduled for checking: decoupled
+ // instances can still share storage with the checked one.
+ for (const auto& instance : instances) {
+ if (load_instance(instance) != 0) {
+ return -1;
+ }
+ }
+ return 0;
+}
+
+int SharedStorageIndex::load_instance(const InstanceInfoPB& instance) {
+ const std::string& instance_id = instance.instance_id();
+ // Register the instance even if it has no resources, so that it can be
told
+ // apart from an unknown instance.
+ instance_resource_locations_[instance_id];
+ // Locations of the resources in `resource_ids` are filled by the vault
scan below.
+ for (const auto& resource_id : instance.resource_ids()) {
+ add_resource(instance_id, resource_id, std::nullopt);
+ }
+ for (const auto& obj_info : instance.obj_info()) {
+ add_resource(instance_id, obj_info.id(), s3_location(obj_info));
+ }
+ if (instance.resource_ids().empty()) {
+ return 0;
+ }
+ return load_storage_vaults(instance_id);
+}
+
+int SharedStorageIndex::load_storage_vaults(const std::string& instance_id) {
+ std::unique_ptr<Transaction> txn;
+ if (txn_kv_->create_txn(&txn) != TxnErrorCode::TXN_OK) {
+ return -1;
+ }
+ auto iter = txn->full_range_get(storage_vault_key({instance_id, ""}),
+ storage_vault_key({instance_id, "\xff"}));
+ for (auto entry = iter->next(); entry.has_value(); entry = iter->next()) {
+ StorageVaultPB vault;
+ if (!vault.ParseFromArray(entry->second.data(), entry->second.size()))
{
+ LOG_WARNING("malformed storage vault").tag("instance_id",
instance_id);
+ return -1;
+ }
+ std::optional<StorageLocation> location;
+ if (vault.has_obj_info()) {
+ location = s3_location(vault.obj_info());
+ } else if (vault.has_hdfs_info()) {
+ location = hdfs_location(vault.hdfs_info());
+ }
+ add_resource(instance_id, vault.id(), std::move(location));
+ }
+ return iter->is_valid() ? 0 : -1;
+}
+
+void SharedStorageIndex::add_resource(const std::string& instance_id,
+ const std::string& resource_id,
+ std::optional<StorageLocation> location)
{
+ auto& resource_location =
instance_resource_locations_[instance_id][resource_id];
+ if (!location.has_value()) {
+ return;
+ }
+ if (!resource_location.has_value()) {
+ resource_location = location;
+ }
+ location_to_owners_[std::move(*location)].push_back({instance_id,
resource_id});
+}
+
+int SharedStorageIndex::get_shared_resource_owners(std::string_view
instance_id,
+ SharedResourceOwners*
owners) {
+ ensure_loaded();
+ owners->clear();
+ if (load_status_ != 0) {
+ return -1;
+ }
+ auto instance =
instance_resource_locations_.find(std::string(instance_id));
+ if (instance == instance_resource_locations_.end()) {
+ LOG_WARNING("instance not found while checking shared resources")
+ .tag("instance_id", instance_id);
+ return -1;
+ }
+ for (const auto& [resource_id, location] : instance->second) {
+ if (location.has_value()) {
+ (*owners)[resource_id] = location_to_owners_.at(*location);
+ } else {
+ // Without a known location, the resource is only owned by its own
instance.
+ (*owners)[resource_id] = {{instance->first, resource_id}};
+ }
+ }
+ return 0;
+}
+
+InstanceChecker::InstanceChecker(std::shared_ptr<TxnKv> txn_kv, const
std::string& instance_id,
+ std::shared_ptr<ResourceManager> resource_mgr,
+ std::shared_ptr<SharedStorageIndex>
shared_storage_index)
+ : txn_kv_(txn_kv),
+ instance_id_(instance_id),
+ resource_mgr_(std::move(resource_mgr)),
+ shared_storage_index_(std::move(shared_storage_index)) {
snapshot_manager_ = create_snapshot_manager(txn_kv);
- resource_mgr_ = std::make_shared<ResourceManager>(std::move(txn_kv));
- resource_mgr_->init();
+ if (!shared_storage_index_) {
+ shared_storage_index_ = std::make_shared<SharedStorageIndex>(txn_kv);
+ }
+ if (!resource_mgr_) {
+ resource_mgr_ = std::make_shared<ResourceManager>(std::move(txn_kv));
+ resource_mgr_->init();
+ }
}
int InstanceChecker::init(const InstanceInfoPB& instance) {
table_stream_versioned_write_ = instance.multi_version_status() ==
MULTI_VERSION_WRITE_ONLY ||
instance.multi_version_status() ==
MULTI_VERSION_READ_WRITE;
+ resource_mgr_->refresh_instance(instance_id_, instance);
int ret = init_obj_store_accessors(instance);
if (ret != 0) {
@@ -655,6 +807,7 @@ int InstanceChecker::do_check() {
struct TabletFiles {
int64_t tablet_id {0};
+ std::string resource_id;
std::unordered_set<std::string> files;
};
TabletFiles tablet_files_cache;
@@ -678,7 +831,8 @@ int InstanceChecker::do_check() {
};
++num_scanned_with_segment;
- if (tablet_files_cache.tablet_id != rs_meta.tablet_id()) {
+ if (tablet_files_cache.tablet_id != rs_meta.tablet_id() ||
+ tablet_files_cache.resource_id != rs_meta.resource_id()) {
long tablet_volume = 0;
// Clear cache
tablet_files_cache.tablet_id = 0;
@@ -706,7 +860,12 @@ int InstanceChecker::do_check() {
tablet_files_cache.files.insert(std::move(file->path));
tablet_volume += file->size;
}
+ if (!list_iter->is_valid()) {
+ check_ret = -1;
+ return;
+ }
tablet_files_cache.tablet_id = rs_meta.tablet_id();
+ tablet_files_cache.resource_id = rs_meta.resource_id();
instance_volume += tablet_volume;
}
@@ -744,7 +903,13 @@ int InstanceChecker::do_check() {
}
TabletIndexPB tablet_index;
- if (get_tablet_idx(txn_kv_.get(), instance_id_, rs_meta.tablet_id(),
tablet_index) == -1) {
+ CloneChainReader reader(instance_id_, resource_mgr_.get());
+ int index_ret =
+ get_tablet_idx(txn_kv_.get(), instance_id_,
rs_meta.tablet_id(), tablet_index);
+ if (index_ret == 1) {
+ err = reader.get_tablet_index(txn.get(), rs_meta.tablet_id(),
&tablet_index);
+ }
+ if (index_ret < 0 || err != TxnErrorCode::TXN_OK) {
LOG(WARNING) << "failed to get tablet index, tablet_id= " <<
rs_meta.tablet_id();
check_ret = -1;
return;
@@ -754,16 +919,20 @@ int InstanceChecker::do_check() {
meta_schema_key({instance_id_, tablet_index.index_id(),
rs_meta.schema_version()});
ValueBuf tablet_schema_val;
err = cloud::blob_get(txn.get(), tablet_schema_key,
&tablet_schema_val);
-
- if (err != TxnErrorCode::TXN_OK) {
+ auto* schema = rs_meta.mutable_tablet_schema();
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ // Clone/rollback rowsets may reference a schema that only exists
in
+ // versioned metadata, possibly in an ancestor of the current
instance.
+ err = reader.get_tablet_schema(txn.get(), tablet_index.index_id(),
+ rs_meta.schema_version(), schema);
+ } else if (err == TxnErrorCode::TXN_OK &&
!parse_schema_value(tablet_schema_val, schema)) {
check_ret = -1;
- LOG(WARNING) << "failed to get schema, err=" << err;
+ LOG(WARNING) << "malformed schema value, key=" <<
hex(tablet_schema_key);
return;
}
-
- auto* schema = rs_meta.mutable_tablet_schema();
- if (!parse_schema_value(tablet_schema_val, schema)) {
- LOG(WARNING) << "malformed schema value, key=" <<
hex(tablet_schema_key);
+ if (err != TxnErrorCode::TXN_OK) {
+ check_ret = -1;
+ LOG(WARNING) << "failed to get schema, err=" << err;
return;
}
diff --git a/cloud/src/recycler/checker.h b/cloud/src/recycler/checker.h
index 9a643f2f382..630d1526d90 100644
--- a/cloud/src/recycler/checker.h
+++ b/cloud/src/recycler/checker.h
@@ -26,9 +26,12 @@
#include <cstdint>
#include <deque>
#include <functional>
-#include <memory>
+#include <map>
+#include <mutex>
+#include <optional>
#include <string_view>
#include <thread>
+#include <tuple>
#include <unordered_map>
#include <unordered_set>
@@ -46,6 +49,8 @@ class StorageVaultAccessor;
class InstanceChecker;
class TxnKv;
class InstanceInfoPB;
+class ObjectStoreInfoPB;
+class HdfsVaultInfo;
struct PendingTableStreamDrop {
int64_t base_db_id;
@@ -63,6 +68,77 @@ TxnErrorCode collect_pending_table_stream_drops(
const std::shared_ptr<TxnKv>& txn_kv, std::string_view instance_id,
std::unordered_map<int64_t, PendingTableStreamDrop>* pending_drops);
+struct SharedResourceOwner {
+ std::string instance_id;
+ std::string resource_id;
+};
+
+// resource id -> all instance/resource pairs that use the same physical
storage.
+using SharedResourceOwners = std::unordered_map<std::string,
std::vector<SharedResourceOwner>>;
+
+// Tells which instance/resource pairs write into the same physical storage.
+// Resource IDs are local to an instance, so the physical storage location is
+// matched instead: descendants can write into inherited resources even after
+// decoupling removes their source_instance_id.
+//
+// One index is shared by all checkers of a scheduling round. It is loaded
+// lazily on the first query, and the result (success or failure) is cached
+// until the round ends. A new round must create a new index to see changes.
+class SharedStorageIndex {
+public:
+ // If `instances` is given, it is used as the full instance list instead of
+ // scanning all instances from the kv store.
+ explicit SharedStorageIndex(
+ std::shared_ptr<TxnKv> txn_kv,
+ std::optional<std::vector<InstanceInfoPB>> instances =
std::nullopt);
+
+ // For every resource of `instance_id`, fill in all owners sharing its
physical storage.
+ // A resource without identifiable storage location is only owned by
itself.
+ // Return 0 on success, -1 if the index failed to load or the instance is
unknown.
+ // Never fills partial owners on failure.
+ int get_shared_resource_owners(std::string_view instance_id,
SharedResourceOwners* owners);
+
+private:
+ // Physical storage location. All fields must match exactly, overlapping
+ // prefixes are not merged. Credentials are deliberately not part of it.
+ struct StorageLocation {
+ std::string type; // "s3" or "hdfs"
+ std::string endpoint; // S3 endpoint, or HDFS fs_name
+ std::string bucket; // empty for HDFS
+ std::string prefix;
+
+ bool operator<(const StorageLocation& other) const {
+ return std::tie(type, endpoint, bucket, prefix) <
+ std::tie(other.type, other.endpoint, other.bucket,
other.prefix);
+ }
+ };
+
+ // Resources without connection info have no location, so that they never
+ // match unrelated resources.
+ using ResourceLocations = std::map<std::string,
std::optional<StorageLocation>>;
+
+ static std::optional<StorageLocation> s3_location(const ObjectStoreInfoPB&
obj_info);
+ static std::optional<StorageLocation> hdfs_location(const HdfsVaultInfo&
hdfs_info);
+
+ void ensure_loaded();
+ int load_all_instances();
+ int load_instance(const InstanceInfoPB& instance);
+ int load_storage_vaults(const std::string& instance_id);
+ void add_resource(const std::string& instance_id, const std::string&
resource_id,
+ std::optional<StorageLocation> location);
+
+ std::shared_ptr<TxnKv> txn_kv_;
+ // Consumed by the first load.
+ std::optional<std::vector<InstanceInfoPB>> preloaded_instances_;
+
+ std::once_flag load_once_;
+ int load_status_ = -1;
+ // instance_id -> (resource_id -> location)
+ std::unordered_map<std::string, ResourceLocations>
instance_resource_locations_;
+ // location -> all instance/resource pairs located there
+ std::map<StorageLocation, std::vector<SharedResourceOwner>>
location_to_owners_;
+};
+
class Checker {
public:
explicit Checker(std::shared_ptr<TxnKv> txn_kv);
@@ -82,6 +158,7 @@ private:
friend class RecyclerServiceImpl;
std::shared_ptr<TxnKv> txn_kv_;
+ std::shared_ptr<ResourceManager> resource_mgr_;
std::atomic_bool stopped_ {false};
std::string ip_port_;
std::vector<std::thread> workers_;
@@ -89,7 +166,11 @@ private:
std::mutex mtx_;
// notify check workers
std::condition_variable pending_instance_cond_;
- std::deque<InstanceInfoPB> pending_instance_queue_;
+ struct PendingInstance {
+ InstanceInfoPB instance;
+ std::shared_ptr<SharedStorageIndex> shared_storage_index;
+ };
+ std::deque<PendingInstance> pending_instance_queue_;
// instance_id -> enqueue_timestamp
std::unordered_map<std::string, long> pending_instance_map_;
std::unordered_map<std::string, std::shared_ptr<InstanceChecker>>
working_instance_map_;
@@ -101,7 +182,9 @@ private:
class InstanceChecker {
public:
- explicit InstanceChecker(std::shared_ptr<TxnKv> txn_kv, const std::string&
instance_id);
+ explicit InstanceChecker(std::shared_ptr<TxnKv> txn_kv, const std::string&
instance_id,
+ std::shared_ptr<ResourceManager> resource_mgr =
nullptr,
+ std::shared_ptr<SharedStorageIndex>
shared_storage_index = nullptr);
// Return 0 if success, otherwise error
int init(const InstanceInfoPB& instance);
// Return 0 if success.
@@ -169,6 +252,8 @@ public:
ResourceManager* resource_mgr() const { return resource_mgr_.get(); }
+ SharedStorageIndex* shared_storage_index() const { return
shared_storage_index_.get(); }
+
void get_all_accessor(std::vector<StorageVaultAccessor*>* accessors);
std::string_view instance_id() const { return instance_id_; }
@@ -280,6 +365,7 @@ private:
std::shared_ptr<SnapshotManager> snapshot_manager_;
std::shared_ptr<ResourceManager> resource_mgr_;
bool table_stream_versioned_write_ {false};
+ std::shared_ptr<SharedStorageIndex> shared_storage_index_;
};
} // namespace doris::cloud
diff --git a/cloud/src/recycler/recycler_service.cpp
b/cloud/src/recycler/recycler_service.cpp
index c9c68004f8a..e6559c4e8bc 100644
--- a/cloud/src/recycler/recycler_service.cpp
+++ b/cloud/src/recycler/recycler_service.cpp
@@ -393,7 +393,7 @@ void RecyclerServiceImpl::check_instance(const std::string&
instance_id, MetaSer
// skip instance already in pending queue
if (success) {
// TODO(plat1ko): Support high priority
- checker_->pending_instance_queue_.push_back(std::move(instance));
+ checker_->pending_instance_queue_.push_back({std::move(instance),
nullptr});
}
checker_->pending_instance_cond_.notify_all();
}
diff --git a/regression-test/plugins/snapshot_recycler_plugin.groovy
b/regression-test/plugins/snapshot_recycler_plugin.groovy
index 702e29043cc..4360291b7a9 100644
--- a/regression-test/plugins/snapshot_recycler_plugin.groovy
+++ b/regression-test/plugins/snapshot_recycler_plugin.groovy
@@ -40,8 +40,7 @@ def recyclerRequest = { Suite suite, String address, String
method, String path,
// The snapshot Docker suites set both scheduling intervals to 10 seconds.
Suite.metaClass.recycleAndCheckSnapshot = { String recyclerAddress, String
instanceId,
- int maxRounds = 10, int maxPolls =
60,
- boolean requireCheck = true ->
+ int maxRounds = 10, int maxPolls =
60 ->
Suite suite = delegate as Suite
assert maxRounds > 0 && maxPolls > 0
String token = "greedisgood9999"
@@ -87,11 +86,9 @@ Suite.metaClass.recycleAndCheckSnapshot = { String
recyclerAddress, String insta
for (int round = 1; round <= maxRounds; round++) {
suite.getLogger().info("Starting snapshot recycle/check:
instance=${instanceId}, round=${round}/${maxRounds}")
boolean recycled = runJob('recycle')
- // Check cloned instances through SQL below: their inherited snapshot
files do not
- // have snapshot keys in the derived instance, so the recycler checker
rejects them.
- boolean checked = !requireCheck || runJob('check')
+ boolean checked = runJob('check')
suite.getLogger().info("Snapshot recycle/check:
instance=${instanceId}, round=${round}/${maxRounds}, "
- + "recycled=${recycled}, checked=${checked},
checkRequired=${requireCheck}, jobs=${lastJobs}")
+ + "recycled=${recycled}, checked=${checked}, jobs=${lastJobs}")
if (recycled && checked) return
}
assert false : "Recycle/check failed after ${maxRounds} rounds:
instance=${instanceId}, jobs=${lastJobs}"
@@ -152,11 +149,7 @@ Suite.metaClass.checkSnapshotRecycle = { SuiteCluster
recyclerCluster, Map insta
}
}
instanceClusters.each { instanceId, dataCluster ->
- if (dataCluster != null && dataCluster != recyclerCluster) {
- suite.recycleAndCheckSnapshot(recyclerAddress,
instanceId.toString(), 10, 60, false)
- } else {
- suite.recycleAndCheckSnapshot(recyclerAddress,
instanceId.toString())
- }
+ suite.recycleAndCheckSnapshot(recyclerAddress, instanceId.toString())
}
instanceClusters.each { instanceId, dataCluster ->
if (dataCluster != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
