This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 745e819f37 AWS: Make sure overridden configurations are applied
(#11274)
745e819f37 is described below
commit 745e819f372fe9ba6bebf2f7edaa27197cd0dd0b
Author: hsiang-c <[email protected]>
AuthorDate: Mon Oct 7 14:48:53 2024 +0800
AWS: Make sure overridden configurations are applied (#11274)
---
.../java/org/apache/iceberg/aws/s3/S3FileIOProperties.java | 12 +++++++++---
.../java/org/apache/iceberg/aws/TestS3FileIOProperties.java | 7 ++++++-
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git
a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
index 6813913a4d..4ab1514a22 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
@@ -899,9 +899,14 @@ public class S3FileIOProperties implements Serializable {
* </pre>
*/
public <T extends S3ClientBuilder> void applyRetryConfigurations(T builder) {
+ ClientOverrideConfiguration.Builder configBuilder =
+ null != builder.overrideConfiguration()
+ ? builder.overrideConfiguration().toBuilder()
+ : ClientOverrideConfiguration.builder();
+
builder.overrideConfiguration(
- config ->
- config.retryPolicy(
+ configBuilder
+ .retryPolicy(
// Use a retry strategy which will persistently retry
throttled exceptions with
// exponential backoff, to give S3 a chance to autoscale.
// LEGACY mode works best here, as it will allow throttled
exceptions to use all of
@@ -945,7 +950,8 @@ public class S3FileIOProperties implements Serializable {
return 5;
})
.build())
- .build()));
+ .build())
+ .build());
}
/**
diff --git
a/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java
b/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java
index 92ed3648bc..e2499e9476 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java
@@ -36,6 +36,7 @@ import
software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
+import software.amazon.awssdk.core.retry.RetryPolicy;
import software.amazon.awssdk.core.signer.Signer;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
@@ -239,7 +240,7 @@ public class TestS3FileIOProperties {
}
@Test
- public void s3RemoteSigningEnabledWithUserAgent() {
+ public void s3RemoteSigningEnabledWithUserAgentAndRetryPolicy() {
String uri = "http://localhost:12345";;
Map<String, String> properties =
ImmutableMap.of(
@@ -249,6 +250,7 @@ public class TestS3FileIOProperties {
s3Properties.applySignerConfiguration(builder);
s3Properties.applyUserAgentConfigurations(builder);
+ s3Properties.applyRetryConfigurations(builder);
Optional<String> userAgent =
builder.overrideConfiguration().advancedOption(SdkAdvancedClientOption.USER_AGENT_PREFIX);
@@ -260,6 +262,9 @@ public class TestS3FileIOProperties {
S3V4RestSignerClient signerClient = (S3V4RestSignerClient) signer.get();
assertThat(signerClient.baseSignerUri()).isEqualTo(uri);
assertThat(signerClient.properties()).isEqualTo(properties);
+
+ Optional<RetryPolicy> retryPolicy =
builder.overrideConfiguration().retryPolicy();
+ assertThat(retryPolicy).isPresent().get().isInstanceOf(RetryPolicy.class);
}
@Test
