This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 81aa4968b77 [feature](Azure) Consider delete non-existent file always
successful on Azure (#36932)
81aa4968b77 is described below
commit 81aa4968b77e671cdca7d82ff109333f5446a137
Author: AlexYue <[email protected]>
AuthorDate: Fri Jun 28 12:58:50 2024 +0800
[feature](Azure) Consider delete non-existent file always successful on
Azure (#36932)
When deleting non-existent files on object storage like S3, the result
would be considered as always true. This pr makes azure client acts the
same way.
As for the added compare string, you can refer to this
[doc](https://github.com/microsoft/api-guidelines/blob/vNext/azure/ConsiderationsForServiceDesign.md)
---
be/src/io/fs/azure_obj_storage_client.cpp | 34 ++++++++++++++++++++++---------
cloud/src/recycler/azure_obj_client.cpp | 32 +++++++++++++++++++----------
2 files changed, 45 insertions(+), 21 deletions(-)
diff --git a/be/src/io/fs/azure_obj_storage_client.cpp
b/be/src/io/fs/azure_obj_storage_client.cpp
index b7ff0a88887..043886672a2 100644
--- a/be/src/io/fs/azure_obj_storage_client.cpp
+++ b/be/src/io/fs/azure_obj_storage_client.cpp
@@ -58,6 +58,7 @@ auto base64_encode_part_num(int part_num) {
}
constexpr char SAS_TOKEN_URL_TEMPLATE[] =
"https://{}.blob.core.windows.net/{}/{}{}";;
+constexpr char BlobNotFound[] = "BlobNotFound";
} // namespace
namespace doris::io {
@@ -105,17 +106,30 @@ struct AzureBatchDeleter {
return resp;
}
- auto get_defer_response = [](const auto& defer) {
- // DeferredResponse<Models::DeleteBlobResult> might throw exception
- if (!defer.GetResponse().Value.Deleted) {
- throw Exception(Status::IOError<false>("Batch delete failed"));
- }
- };
for (auto&& defer_response : deferred_resps) {
- auto response =
- do_azure_client_call([&]() {
get_defer_response(defer_response); }, _opts);
- if (response.status.code != ErrorCode::OK) {
- return response;
+ try {
+ auto r = defer_response.GetResponse();
+ if (!r.Value.Deleted) {
+ auto msg = fmt::format("Azure batch delete failed, path
msg {}",
+
wrap_object_storage_path_msg(_opts));
+ LOG_WARNING(msg);
+ return {.status = convert_to_obj_response(
+
Status::InternalError<false>(std::move(msg)))};
+ }
+ } catch (Azure::Core::RequestFailedException& e) {
+ if (Azure::Core::Http::HttpStatusCode::NotFound ==
e.StatusCode &&
+ 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) {
+ continue;
+ }
+ auto msg = fmt::format(
+ "Azure request failed because {}, error msg {}, http
code {}, path msg {}",
+ e.what(), e.Message, static_cast<int>(e.StatusCode),
+ wrap_object_storage_path_msg(_opts));
+ LOG_WARNING(msg);
+ return {.status = convert_to_obj_response(
+ Status::InternalError<false>(std::move(msg))),
+ .http_code = static_cast<int>(e.StatusCode),
+ .request_id = std::move(e.RequestId)};
}
}
diff --git a/cloud/src/recycler/azure_obj_client.cpp
b/cloud/src/recycler/azure_obj_client.cpp
index b5cd3e76d8e..163c9b43559 100644
--- a/cloud/src/recycler/azure_obj_client.cpp
+++ b/cloud/src/recycler/azure_obj_client.cpp
@@ -39,6 +39,7 @@ using namespace Azure::Storage::Blobs;
namespace doris::cloud {
constexpr size_t BlobBatchMaxOperations = 256;
+constexpr char BlobNotFound[] = "BlobNotFound";
template <typename Func>
ObjectStorageResponse do_azure_client_call(Func f, const
ObjectStoragePathOptions& opts) {
@@ -78,18 +79,27 @@ struct AzureBatchDeleter {
return resp;
}
- auto get_defer_response = [](const auto& defer) {
- // DeferredResponse<Models::DeleteBlobResult> might throw exception
- if (!defer.GetResponse().Value.Deleted) {
- throw std::runtime_error("Batch delete blobs failed");
- }
- };
-
for (auto&& defer_response : deferred_resps) {
- auto response =
- do_azure_client_call([&]() {
get_defer_response(defer_response); }, _opts);
- if (response.ret != 0) {
- return response;
+ try {
+ auto r = defer_response.GetResponse();
+ if (!r.Value.Deleted) {
+ LOG_INFO("Azure batch delete failed, bucket {}, key {},
prefix {}, endpoint {}",
+ _opts.bucket, _opts.key, _opts.prefix,
_opts.endpoint);
+ return {-1};
+ }
+ } catch (Azure::Storage::StorageException& e) {
+ if (Azure::Core::Http::HttpStatusCode::NotFound ==
e.StatusCode &&
+ 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) {
+ continue;
+ }
+ auto msg = fmt::format(
+ "Azure request failed because {}, http code {},
request id {}, bucket {}, "
+ "key {}, "
+ "prefix {}, endpoint {}",
+ e.Message, static_cast<int>(e.StatusCode),
e.RequestId, _opts.bucket,
+ _opts.key, _opts.prefix, _opts.endpoint);
+ LOG_WARNING(msg);
+ return {-1, std::move(msg)};
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
