This is an automated email from the ASF dual-hosted git repository.
liaoxin01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f421e9df50d [chore](s3) Adjust S3 SlowDown retry configurable (#65508)
f421e9df50d is described below
commit f421e9df50d65d445e596e5784984dc855a17f29
Author: Yixuan Wang <[email protected]>
AuthorDate: Wed Jul 15 16:23:36 2026 +0800
[chore](s3) Adjust S3 SlowDown retry configurable (#65508)
### What problem does this PR solve?
Allow rate-limited retries on the client side (BE), but disable
rate-limited retries on the recycler side.
The decision to no longer use config to manage whether to limit retry
speed is a clear action.
fix: https://github.com/apache/doris/pull/63776/
---
be/src/util/s3_util.cpp | 3 ++-
cloud/src/common/config.h | 2 --
cloud/src/recycler/s3_accessor.cpp | 6 ++----
3 files changed, 4 insertions(+), 7 deletions(-)
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index fc4c81d4bd9..2021b80fda6 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -349,6 +349,7 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_azure_client(
}
Azure::Storage::Blobs::BlobClientOptions options;
+
options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests);
options.Retry.MaxRetries = config::max_s3_client_retry;
options.PerRetryPolicies.emplace_back(std::make_unique<AzureRetryRecordPolicy>());
if (_ca_cert_file_path.empty()) {
@@ -525,7 +526,7 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_s3_client(
}
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
- config::max_s3_client_retry /*scaleFactor = 25*/,
/*retry_slow_down=*/false);
+ config::max_s3_client_retry /*scaleFactor = 25*/,
/*retry_slow_down=*/true);
std::shared_ptr<Aws::S3::S3Client> new_client =
std::make_shared<Aws::S3::S3Client>(
get_aws_credentials_provider(s3_conf), std::move(aws_config),
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 29e795ca078..926a73a2c4c 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -332,8 +332,6 @@ CONF_Validator(s3_client_http_scheme, [](const std::string&
config) -> bool {
// Max retry times for object storage request
CONF_mInt64(max_s3_client_retry, "10");
-// Whether to retry on S3 SlowDown (429/503) errors
-CONF_Bool(s3_client_retry_slow_down, "false");
// Max byte getting delete bitmap can return, default is 1GB
CONF_mInt64(max_get_delete_bitmap_byte, "1073741824");
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 51cc9023904..8ff29694bd7 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -394,9 +394,6 @@ int S3Accessor::init() {
case S3Conf::AZURE: {
#ifdef USE_AZURE
Azure::Storage::Blobs::BlobClientOptions options;
- if (config::s3_client_retry_slow_down) {
-
options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests);
- }
options.Retry.MaxRetries = config::max_s3_client_retry;
auto cred =
std::make_shared<Azure::Storage::StorageSharedKeyCredential>(conf_.ak,
conf_.sk);
@@ -441,8 +438,9 @@ int S3Accessor::init() {
if (config::s3_client_http_scheme == "http") {
aws_config.scheme = Aws::Http::Scheme::HTTP;
}
+ // Recycler should fail fast on S3 SlowDown instead of retrying and
blocking worker threads.
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
- config::max_s3_client_retry,
config::s3_client_retry_slow_down);
+ config::max_s3_client_retry, /*retry_slow_down=*/false);
if (_ca_cert_file_path.empty()) {
_ca_cert_file_path =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
