This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ce85663a9e5 [improvement](cloud) Enable batch deletion for GCS objects
in recycler (#68967)
ce85663a9e5 is described below
commit ce85663a9e5bec44533014b39eab6d7aa5fd5d85
Author: Yixuan Wang <[email protected]>
AuthorDate: Tue Oct 13 12:36:10 2026 +0800
[improvement](cloud) Enable batch deletion for GCS objects in recycler
(#68967)
Problem Summary: The GCS-specific accessor deleted files and objects
under a prefix one at a time. Remove that accessor and route GCS through
the shared S3Accessor deletion paths, which use batch deletion for file
lists and recursive deletion.
---
cloud/src/recycler/s3_accessor.cpp | 89 +-------------------------------------
cloud/src/recycler/s3_accessor.h | 11 -----
2 files changed, 1 insertion(+), 99 deletions(-)
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 50bf53703aa..fe8455b4907 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -325,15 +325,7 @@ std::string S3Accessor::to_uri(const std::string&
relative_path) const {
int S3Accessor::create(S3Conf conf, std::shared_ptr<S3Accessor>* accessor) {
TEST_SYNC_POINT_RETURN_WITH_VALUE("S3Accessor::init.s3_init_failed",
(int)-1);
- switch (conf.provider) {
- case S3Conf::GCS:
- *accessor = std::make_shared<GcsAccessor>(conf);
- break;
- default:
- *accessor = std::make_shared<S3Accessor>(conf);
- break;
- }
-
+ *accessor = std::make_shared<S3Accessor>(conf);
return (*accessor)->init();
}
@@ -611,83 +603,4 @@ int S3Accessor::check_versioning() {
return obj_client_->check_versioning(conf_.bucket).status.code;
}
-int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t
expiration_time) {
- LOG_INFO("begin delete prefix").tag("uri", to_uri(path_prefix));
-
- int ret = 0;
- int cnt = 0;
- int skip = 0;
- int64_t del_nonexisted = 0;
- int del = 0;
- auto iter = obj_client_->list_objects({.bucket = conf_.bucket, .prefix =
get_key(path_prefix)});
- for (;;) {
- auto result = iter->next();
- if (!result.object.has_value()) {
- if (result.resp.status.code != ObjStorageStatus::NOT_FOUND &&
- result.resp.status.code != ObjStorageStatus::OK) {
- ret = result.resp.status.code;
- }
- break;
- }
- auto& obj = *result.object;
- if (!(++cnt % 100)) {
- LOG_INFO("loop delete prefix")
- .tag("uri", to_uri(path_prefix))
- .tag("total_obj_cnt", cnt)
- .tag("deleted", del)
- .tag("del_nonexisted", del_nonexisted)
- .tag("skipped", skip);
- }
- if (expiration_time > 0 && obj.mtime_s > expiration_time) {
- skip++;
- continue;
- }
- del++;
-
- // FIXME(plat1ko): Delete objects by batch with genuine GCS client
- int del_ret =
- obj_client_->delete_object({.bucket = conf_.bucket, .key =
obj.key}).status.code;
- del_nonexisted += (del_ret == ObjStorageStatus::NOT_FOUND);
- static_assert(ObjStorageStatus::OK == 0);
- if (del_ret != ObjStorageStatus::OK && del_ret !=
ObjStorageStatus::NOT_FOUND) {
- ret = del_ret;
- }
- }
-
- LOG_INFO("finish delete prefix")
- .tag("uri", to_uri(path_prefix))
- .tag("total_obj_cnt", cnt)
- .tag("deleted", del)
- .tag("del_nonexisted", del_nonexisted)
- .tag("skipped", skip);
-
- if (!iter->is_valid()) {
- return -1;
- }
-
- return ret;
-}
-
-int GcsAccessor::delete_files(const std::vector<std::string>& paths) {
- std::vector<int> delete_rets(paths.size());
-#ifdef USE_LIBCPP
- std::transform(paths.begin(), paths.end(), delete_rets.begin(),
-#else
- std::transform(std::execution::par, paths.begin(), paths.end(),
delete_rets.begin(),
-#endif
- [this](const std::string& path) {
- LOG_INFO("delete file").tag("uri", to_uri(path));
- return delete_file(path);
- });
-
- int ret = 0;
- for (int delete_ret : delete_rets) {
- if (delete_ret != 0) {
- ret = delete_ret;
- break;
- }
- }
- return ret;
-}
-
} // namespace doris::cloud
diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h
index deded63abe1..922d2fdfdae 100644
--- a/cloud/src/recycler/s3_accessor.h
+++ b/cloud/src/recycler/s3_accessor.h
@@ -157,16 +157,5 @@ protected:
std::string _ca_cert_file_path;
};
-class GcsAccessor final : public S3Accessor {
-public:
- explicit GcsAccessor(S3Conf conf) : S3Accessor(std::move(conf)) {}
- ~GcsAccessor() override = default;
-
- int delete_files(const std::vector<std::string>& paths) override;
-
-private:
- int delete_prefix_impl(const std::string& path_prefix, int64_t
expiration_time = 0) override;
-};
-
} // namespace cloud
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
