This is an automated email from the ASF dual-hosted git repository.
jtuglu1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new d06aa83cd80 fix: retry transient AWS credential resolution failures
(#19558)
d06aa83cd80 is described below
commit d06aa83cd809f6cedec15ba8f8e8e12de26e1587
Author: jtuglu1 <[email protected]>
AuthorDate: Fri Jun 5 15:18:41 2026 -0700
fix: retry transient AWS credential resolution failures (#19558)
S3 segment pushes that use the AWS SDK v2 transfer manager can resolve
credentials on the async upload path. If a file-session credential refresh,
container credential lookup, or IMDS lookup is temporarily unavailable, the SDK
reports an SdkClientException such as 'Unable to load credentials from any of
the providers in the chain'.
Druid's S3 push path already wraps uploads in retryS3Operation, but these
credential-provider failures were not classified as recoverable after the SDK
v2 migration. That made an intermittent credential miss fail the task
immediately instead of using the existing retry budget.
---
.../org/apache/druid/common/aws/AWSClientUtil.java | 14 +++++++++--
.../apache/druid/common/aws/AWSClientUtilTest.java | 27 ++++++++++++++++++++++
.../org/apache/druid/storage/s3/S3UtilsTest.java | 26 +++++++++++++++++++++
3 files changed, 65 insertions(+), 2 deletions(-)
diff --git
a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientUtil.java
b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientUtil.java
index 68fa9e25ca4..4f138e3a750 100644
---
a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientUtil.java
+++
b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientUtil.java
@@ -62,6 +62,12 @@ public class AWSClientUtil
"Throttling"
);
+ private static final String UNABLE_TO_LOAD_CREDENTIALS_FROM_PROVIDER_CHAIN =
+ "Unable to load credentials from any of the providers in the chain";
+ private static final String FAILED_TO_LOAD_CREDENTIALS_FROM_IMDS = "Failed
to load credentials from IMDS";
+ private static final String CANNOT_REFRESH_AWS_CREDENTIALS = "cannot refresh
AWS credentials";
+ private static final String CANNOT_FETCH_CREDENTIALS_FROM_CONTAINER =
"Cannot fetch credentials from container";
+
/**
* Checks whether an exception can be retried or not for AWS SDK v2.
*/
@@ -101,11 +107,15 @@ public class AWSClientUtil
// Check for SdkClientException specific messages
if (exception instanceof SdkClientException) {
- String message = exception.getMessage();
+ final String message = exception.getMessage();
if (message != null) {
if (message.contains("Unable to execute HTTP request") ||
message.contains("Data read has a different length than the
expected") ||
- message.contains("Unable to find a region")) {
+ message.contains("Unable to find a region") ||
+ message.contains(UNABLE_TO_LOAD_CREDENTIALS_FROM_PROVIDER_CHAIN) ||
+ message.contains(FAILED_TO_LOAD_CREDENTIALS_FROM_IMDS) ||
+ message.contains(CANNOT_REFRESH_AWS_CREDENTIALS) ||
+ message.contains(CANNOT_FETCH_CREDENTIALS_FROM_CONTAINER)) {
return true;
}
}
diff --git
a/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSClientUtilTest.java
b/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSClientUtilTest.java
index 6f47e78471c..535bfcc8b39 100644
---
a/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSClientUtilTest.java
+++
b/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSClientUtilTest.java
@@ -89,6 +89,33 @@ public class AWSClientUtilTest
Assert.assertTrue(AWSClientUtil.isClientExceptionRecoverable(ex));
}
+ @Test
+ public void testRecoverableException_CredentialsProviderChain()
+ {
+ final SdkClientException ex = SdkClientException.builder()
+ .message("Unable to load credentials from any of the providers in the
chain AwsCredentialsProviderChain")
+ .build();
+ Assert.assertTrue(AWSClientUtil.isClientExceptionRecoverable(ex));
+ }
+
+ @Test
+ public void testRecoverableException_FileSessionCredentialsRefresh()
+ {
+ final SdkClientException ex = SdkClientException.builder()
+ .message("LazyFileSessionCredentialsProvider(): cannot refresh AWS
credentials")
+ .build();
+ Assert.assertTrue(AWSClientUtil.isClientExceptionRecoverable(ex));
+ }
+
+ @Test
+ public void testRecoverableException_InstanceProfileCredentials()
+ {
+ final SdkClientException ex = SdkClientException.builder()
+ .message("InstanceProfileCredentialsProvider(): Failed to load
credentials from IMDS.")
+ .build();
+ Assert.assertTrue(AWSClientUtil.isClientExceptionRecoverable(ex));
+ }
+
@Test
public void testRecoverableException_ClockSkewedError()
{
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3UtilsTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3UtilsTest.java
index 16b8c20d0f2..dcf7b3f8d0d 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3UtilsTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3UtilsTest.java
@@ -36,6 +36,7 @@ import software.amazon.awssdk.services.s3.model.S3Exception;
import java.io.IOException;
import java.util.List;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@@ -151,6 +152,31 @@ public class S3UtilsTest
Assert.assertEquals(maxRetries, count.get());
}
+ @Test
+ public void testRetryWithAsyncCredentialProviderChainException() throws
Exception
+ {
+ final int maxRetries = 3;
+ final AtomicInteger count = new AtomicInteger();
+ S3Utils.retryS3Operation(
+ () -> {
+ if (count.incrementAndGet() >= maxRetries) {
+ return "hey";
+ } else {
+ throw new CompletionException(
+ SdkClientException.builder()
+ .message(
+ "Unable to load credentials from any of
the providers in the chain "
+ + "AwsCredentialsProviderChain"
+ )
+ .build()
+ );
+ }
+ },
+ maxRetries
+ );
+ Assert.assertEquals(maxRetries, count.get());
+ }
+
@Test
public void testRetryWithS3InternalError() throws Exception
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
