This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 9a98de0410 AWS: Don't fetch credential from endpoint if properties
contain a valid credential (#12504)
9a98de0410 is described below
commit 9a98de04103755ad4833ac6c3918c39022d54a99
Author: Eduard Tudenhoefner <[email protected]>
AuthorDate: Thu Mar 13 06:23:56 2025 +0100
AWS: Don't fetch credential from endpoint if properties contain a valid
credential (#12504)
When the `VendedCredentialProvider` is created, the `properties` typically
already contain a valid credential from the first time a table is loaded.
This PR uses the credential from the properties and otherwise falls back to
loading a valid credential from the refresh endpoint in case the credential in
`properties` is incomplete or expired.
---
.../iceberg/aws/s3/VendedCredentialsProvider.java | 37 ++++-
.../aws/s3/TestVendedCredentialsProvider.java | 170 +++++++++++++++++++++
2 files changed, 206 insertions(+), 1 deletion(-)
diff --git
a/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
b/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
index 792a6ff27b..769d187875 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
@@ -22,8 +22,10 @@ import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
import org.apache.iceberg.rest.ErrorHandlers;
import org.apache.iceberg.rest.HTTPClient;
import org.apache.iceberg.rest.HTTPHeaders;
@@ -52,7 +54,7 @@ public class VendedCredentialsProvider implements
AwsCredentialsProvider, SdkAut
Preconditions.checkArgument(null != properties.get(URI), "Invalid URI:
null");
this.properties = properties;
this.credentialCache =
- CachedSupplier.builder(this::refreshCredential)
+ CachedSupplier.builder(() ->
credentialFromProperties().orElseGet(this::refreshCredential))
.cachedValueName(VendedCredentialsProvider.class.getName())
.build();
}
@@ -101,6 +103,39 @@ public class VendedCredentialsProvider implements
AwsCredentialsProvider, SdkAut
ErrorHandlers.defaultErrorHandler());
}
+ private Optional<RefreshResult<AwsCredentials>> credentialFromProperties() {
+ String accessKeyId = properties.get(S3FileIOProperties.ACCESS_KEY_ID);
+ String secretAccessKey =
properties.get(S3FileIOProperties.SECRET_ACCESS_KEY);
+ String sessionToken = properties.get(S3FileIOProperties.SESSION_TOKEN);
+ String tokenExpiresAtMillis =
properties.get(S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS);
+ if (Strings.isNullOrEmpty(accessKeyId)
+ || Strings.isNullOrEmpty(secretAccessKey)
+ || Strings.isNullOrEmpty(sessionToken)
+ || Strings.isNullOrEmpty(tokenExpiresAtMillis)) {
+ return Optional.empty();
+ }
+
+ Instant expiresAt =
Instant.ofEpochMilli(Long.parseLong(tokenExpiresAtMillis));
+ Instant prefetchAt = expiresAt.minus(5, ChronoUnit.MINUTES);
+
+ if (Instant.now().isAfter(prefetchAt)) {
+ return Optional.empty();
+ }
+
+ return Optional.of(
+ RefreshResult.builder(
+ (AwsCredentials)
+ AwsSessionCredentials.builder()
+ .accessKeyId(accessKeyId)
+ .secretAccessKey(secretAccessKey)
+ .sessionToken(sessionToken)
+ .expirationTime(expiresAt)
+ .build())
+ .staleTime(expiresAt)
+ .prefetchTime(prefetchAt)
+ .build());
+ }
+
private RefreshResult<AwsCredentials> refreshCredential() {
LoadCredentialsResponse response = fetchCredentials();
diff --git
a/aws/src/test/java/org/apache/iceberg/aws/s3/TestVendedCredentialsProvider.java
b/aws/src/test/java/org/apache/iceberg/aws/s3/TestVendedCredentialsProvider.java
index 67cd1cb552..51aca88943 100644
---
a/aws/src/test/java/org/apache/iceberg/aws/s3/TestVendedCredentialsProvider.java
+++
b/aws/src/test/java/org/apache/iceberg/aws/s3/TestVendedCredentialsProvider.java
@@ -302,6 +302,176 @@ public class TestVendedCredentialsProvider {
}
}
+ @Test
+ public void nonExpiredTokenInProperties() {
+ HttpRequest mockRequest =
request("/v1/credentials").withMethod(HttpMethod.GET.name());
+ String expiresAt = Long.toString(Instant.now().plus(10,
ChronoUnit.HOURS).toEpochMilli());
+ Credential credentialFromProperties =
+ ImmutableCredential.builder()
+ .prefix("s3")
+ .config(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKeyFromProperties",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKeyFromProperties",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionTokenFromProperties",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ expiresAt))
+ .build();
+
+ Credential credential =
+ ImmutableCredential.builder()
+ .prefix("s3")
+ .config(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKey",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKey",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionToken",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ Long.toString(Instant.now().plus(1,
ChronoUnit.HOURS).toEpochMilli())))
+ .build();
+ LoadCredentialsResponse response =
+
ImmutableLoadCredentialsResponse.builder().addCredentials(credential).build();
+
+ HttpResponse mockResponse =
+
response(LoadCredentialsResponseParser.toJson(response)).withStatusCode(200);
+ mockServer.when(mockRequest).respond(mockResponse);
+
+ try (VendedCredentialsProvider provider =
+ VendedCredentialsProvider.create(
+ ImmutableMap.of(
+ VendedCredentialsProvider.URI,
+ URI,
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKeyFromProperties",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKeyFromProperties",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionTokenFromProperties",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ expiresAt))) {
+ AwsCredentials awsCredentials = provider.resolveCredentials();
+
+ verifyCredentials(awsCredentials, credentialFromProperties);
+
+ for (int i = 0; i < 5; i++) {
+ // resolving credentials multiple times should not hit the credentials
endpoint again
+ assertThat(provider.resolveCredentials()).isSameAs(awsCredentials);
+ }
+ }
+
+ // token endpoint isn't hit, because the credentials are extracted from
the properties
+ mockServer.verify(mockRequest, VerificationTimes.never());
+ }
+
+ @Test
+ public void expiredTokenInProperties() {
+ HttpRequest mockRequest =
request("/v1/credentials").withMethod(HttpMethod.GET.name());
+
+ Credential credential =
+ ImmutableCredential.builder()
+ .prefix("s3")
+ .config(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKey",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKey",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionToken",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ Long.toString(Instant.now().plus(1,
ChronoUnit.HOURS).toEpochMilli())))
+ .build();
+ LoadCredentialsResponse response =
+
ImmutableLoadCredentialsResponse.builder().addCredentials(credential).build();
+
+ HttpResponse mockResponse =
+
response(LoadCredentialsResponseParser.toJson(response)).withStatusCode(200);
+ mockServer.when(mockRequest).respond(mockResponse);
+
+ try (VendedCredentialsProvider provider =
+ VendedCredentialsProvider.create(
+ ImmutableMap.of(
+ VendedCredentialsProvider.URI,
+ URI,
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKeyFromProperties",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKeyFromProperties",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionTokenFromProperties",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ Long.toString(Instant.now().minus(1,
ChronoUnit.HOURS).toEpochMilli())))) {
+ AwsCredentials awsCredentials = provider.resolveCredentials();
+
+ verifyCredentials(awsCredentials, credential);
+
+ for (int i = 0; i < 5; i++) {
+ // resolving credentials multiple times should not hit the credentials
endpoint again
+ assertThat(provider.resolveCredentials()).isSameAs(awsCredentials);
+ }
+ }
+
+ // token endpoint is hit once due to the properties containing an expired
token
+ mockServer.verify(mockRequest, VerificationTimes.once());
+ }
+
+ @Test
+ public void invalidTokenInProperties() {
+ HttpRequest mockRequest =
request("/v1/credentials").withMethod(HttpMethod.GET.name());
+
+ Credential credential =
+ ImmutableCredential.builder()
+ .prefix("s3")
+ .config(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKey",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKey",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionToken",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ Long.toString(Instant.now().plus(1,
ChronoUnit.HOURS).toEpochMilli())))
+ .build();
+ LoadCredentialsResponse response =
+
ImmutableLoadCredentialsResponse.builder().addCredentials(credential).build();
+
+ HttpResponse mockResponse =
+
response(LoadCredentialsResponseParser.toJson(response)).withStatusCode(200);
+ mockServer.when(mockRequest).respond(mockResponse);
+
+ // token expiration is missing from the properties
+ try (VendedCredentialsProvider provider =
+ VendedCredentialsProvider.create(
+ ImmutableMap.of(
+ VendedCredentialsProvider.URI,
+ URI,
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKeyFromProperties",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKeyFromProperties",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionTokenFromProperties"))) {
+ AwsCredentials awsCredentials = provider.resolveCredentials();
+
+ verifyCredentials(awsCredentials, credential);
+
+ for (int i = 0; i < 5; i++) {
+ // resolving credentials multiple times should not hit the credentials
endpoint again
+ assertThat(provider.resolveCredentials()).isSameAs(awsCredentials);
+ }
+ }
+
+ // token endpoint is hit once due to the properties not containing the
token's expiration
+ mockServer.verify(mockRequest, VerificationTimes.once());
+ }
+
private void verifyCredentials(AwsCredentials awsCredentials, Credential
credential) {
assertThat(awsCredentials).isInstanceOf(AwsSessionCredentials.class);
AwsSessionCredentials creds = (AwsSessionCredentials) awsCredentials;
