This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new e38d2934109 Support AWS InternalError code retries (#18720)
e38d2934109 is described below
commit e38d29341090a10c8758950ce49a4d209b64dea9
Author: Uddeshya Singh <[email protected]>
AuthorDate: Thu Nov 6 16:28:30 2025 +0530
Support AWS InternalError code retries (#18720)
* Support InternalError code retries
* Accommodate review comments
---
.../java/org/apache/druid/storage/s3/S3Utils.java | 5 ++
.../org/apache/druid/storage/s3/S3UtilsTest.java | 97 ++++++++++++++++++++++
2 files changed, 102 insertions(+)
diff --git
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
index 2a7d4c58179..ddfb6599ddd 100644
---
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
+++
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
@@ -96,6 +96,11 @@ public class S3Utils
// This can happen sometimes when AWS isn't able to obtain the
credentials for some service:
// https://github.com/aws/aws-sdk-java/issues/2285
return true;
+ } else if (e instanceof AmazonS3Exception && ((AmazonS3Exception)
e).getStatusCode() == 200 &&
+ (e.getMessage().contains("InternalError") ||
e.getMessage().contains("SlowDown"))) {
+ // This can happen sometimes when AWS returns a 200 response with
internal error message
+ // https://repost.aws/knowledge-center/s3-resolve-200-internalerror
+ return true;
} else if (e instanceof InterruptedException) {
Thread.interrupted(); // Clear interrupted state and not retry
return false;
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3UtilsTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3UtilsTest.java
index 0c5e5840efe..5b8a4dcbf0b 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3UtilsTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3UtilsTest.java
@@ -130,4 +130,101 @@ public class S3UtilsTest
);
Assert.assertEquals(maxRetries, count.get());
}
+
+ @Test
+ public void testRetryWithAmazonS3InternalError() throws Exception
+ {
+ final int maxRetries = 3;
+ final AtomicInteger count = new AtomicInteger();
+ S3Utils.retryS3Operation(
+ () -> {
+ if (count.incrementAndGet() >= maxRetries) {
+ return "donezo";
+ } else {
+ AmazonS3Exception s3Exception = new AmazonS3Exception("We
encountered an internal error. Please try again. (Service: Amazon S3; Status
Code: 200; Error Code: InternalError; Request ID: some-id)");
+ s3Exception.setStatusCode(200);
+ throw s3Exception;
+ }
+ },
+ maxRetries
+ );
+ Assert.assertEquals(maxRetries, count.get());
+ }
+
+ @Test
+ public void testRetryWithAmazonS3SlowDown() throws Exception
+ {
+ final int maxRetries = 3;
+ final AtomicInteger count = new AtomicInteger();
+ S3Utils.retryS3Operation(
+ () -> {
+ if (count.incrementAndGet() >= maxRetries) {
+ return "success";
+ } else {
+ AmazonS3Exception s3Exception = new AmazonS3Exception("Please
reduce your request rate. SlowDown");
+ s3Exception.setStatusCode(200);
+ throw s3Exception;
+ }
+ },
+ maxRetries
+ );
+ Assert.assertEquals(maxRetries, count.get());
+ }
+
+ @Test
+ public void testNoRetryWithAmazonS3InternalErrorNon200Status()
+ {
+ final AtomicInteger count = new AtomicInteger();
+ Assert.assertThrows(
+ Exception.class,
+ () -> S3Utils.retryS3Operation(
+ () -> {
+ count.incrementAndGet();
+ AmazonS3Exception s3Exception = new
AmazonS3Exception("InternalError occurred");
+ s3Exception.setStatusCode(403);
+ throw s3Exception;
+ },
+ 3
+ )
+ );
+ Assert.assertEquals(1, count.get());
+ }
+
+ @Test
+ public void testNoRetryWithAmazonS3SlowDownNon200Status()
+ {
+ final AtomicInteger count = new AtomicInteger();
+ Assert.assertThrows(
+ Exception.class,
+ () -> S3Utils.retryS3Operation(
+ () -> {
+ count.incrementAndGet();
+ AmazonS3Exception s3Exception = new AmazonS3Exception("SlowDown
message");
+ s3Exception.setStatusCode(404);
+ throw s3Exception;
+ },
+ 3
+ )
+ );
+ Assert.assertEquals(1, count.get());
+ }
+
+ @Test
+ public void testRetryWithAmazonS3Status200ButDifferentError()
+ {
+ final AtomicInteger count = new AtomicInteger();
+ Assert.assertThrows(
+ Exception.class,
+ () -> S3Utils.retryS3Operation(
+ () -> {
+ count.incrementAndGet();
+ AmazonS3Exception s3Exception = new AmazonS3Exception("Some
other error message");
+ s3Exception.setStatusCode(200);
+ throw s3Exception;
+ },
+ 3
+ )
+ );
+ Assert.assertEquals(1, count.get());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
