This is an automated email from the ASF dual-hosted git repository.
dimas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 7b04abe04 Push AccessConfig creation to PolarisStorageIntegration
(#2171)
7b04abe04 is described below
commit 7b04abe044995c6d7f5762acc959c007bcae5a44
Author: Dmitri Bourlatchkov <dmitri.bourlatch...@gmail.com>
AuthorDate: Fri Jul 25 15:55:06 2025 -0400
Push AccessConfig creation to PolarisStorageIntegration (#2171)
This refactoring does not change Polaris behaviour.
* Move storage-specific access properties processing logic from
core code to storage integration implementations.
* Add `isExpirationTimestamp` flag to `StorageAccessProperty` to
allow them to be processed uniformly.
* Prepare for supporting access config properties that may have
different values in Polaris Servers and Clients. This enables
future enhancements to support different S3 endpoint DNS names
in servers and clients for #1530
---
.../AtomicOperationMetaStoreManager.java | 7 +-
.../dao/entity/ScopedCredentialsResult.java | 32 ++---
.../TransactionalMetaStoreManagerImpl.java | 7 +-
.../apache/polaris/core/storage/AccessConfig.java | 50 +++++++-
.../core/storage/PolarisStorageIntegration.java | 3 +-
.../core/storage/StorageAccessProperty.java | 33 +++--
.../aws/AwsCredentialsStorageIntegration.java | 34 +++---
.../azure/AzureCredentialsStorageIntegration.java | 48 ++++++--
.../core/storage/cache/StorageCredentialCache.java | 3 +-
.../storage/cache/StorageCredentialCacheEntry.java | 74 +----------
.../gcp/GcpCredentialsStorageIntegration.java | 12 +-
.../polaris/core/storage/AccessConfigTest.java | 93 ++++++++++++++
.../storage/InMemoryStorageIntegrationTest.java | 3 +-
.../AzureCredentialsStorageIntegrationTest.java | 55 +++++++++
.../storage/cache/StorageCredentialCacheTest.java | 135 +++------------------
.../aws/AwsCredentialsStorageIntegrationTest.java | 93 +++++++-------
.../AzureCredentialStorageIntegrationTest.java | 41 +++----
.../gcp/GcpCredentialsStorageIntegrationTest.java | 31 +++--
.../catalog/AbstractIcebergCatalogTest.java | 12 +-
.../service/catalog/io/DefaultFileIOFactory.java | 1 +
.../PolarisStorageIntegrationProviderImpl.java | 7 +-
21 files changed, 413 insertions(+), 361 deletions(-)
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java
b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java
index e4e77c155..35d0098b3 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java
@@ -21,7 +21,6 @@ package org.apache.polaris.core.persistence;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import java.util.ArrayList;
-import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -71,9 +70,9 @@ import
org.apache.polaris.core.policy.PolarisPolicyMappingRecord;
import org.apache.polaris.core.policy.PolicyEntity;
import org.apache.polaris.core.policy.PolicyMappingUtil;
import org.apache.polaris.core.policy.PolicyType;
+import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
import org.apache.polaris.core.storage.PolarisStorageIntegration;
-import org.apache.polaris.core.storage.StorageAccessProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1614,14 +1613,14 @@ public class AtomicOperationMetaStoreManager extends
BaseMetaStoreManager {
PolarisStorageConfigurationInfo storageConfigurationInfo =
BaseMetaStoreManager.extractStorageConfiguration(callCtx,
reloadedEntity.getEntity());
try {
- EnumMap<StorageAccessProperty, String> creds =
+ AccessConfig accessConfig =
storageIntegration.getSubscopedCreds(
callCtx,
storageConfigurationInfo,
allowListOperation,
allowedReadLocations,
allowedWriteLocations);
- return new ScopedCredentialsResult(creds);
+ return new ScopedCredentialsResult(accessConfig);
} catch (Exception ex) {
return new ScopedCredentialsResult(
BaseResult.ReturnStatus.SUBSCOPE_CREDS_ERROR, ex.getMessage());
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ScopedCredentialsResult.java
b/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ScopedCredentialsResult.java
index 60e1cee01..76526a863 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ScopedCredentialsResult.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ScopedCredentialsResult.java
@@ -18,19 +18,15 @@
*/
package org.apache.polaris.core.persistence.dao.entity;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
-import java.util.EnumMap;
-import java.util.Map;
-import org.apache.polaris.core.storage.StorageAccessProperty;
+import org.apache.polaris.core.storage.AccessConfig;
/** Result of a getSubscopedCredsForEntity() call */
public class ScopedCredentialsResult extends BaseResult {
// null if not success. Else, set of name/value pairs for the credentials
- private final EnumMap<StorageAccessProperty, String> credentials;
+ private final AccessConfig accessConfig;
/**
* Constructor for an error
@@ -41,32 +37,20 @@ public class ScopedCredentialsResult extends BaseResult {
public ScopedCredentialsResult(
@Nonnull ReturnStatus errorCode, @Nullable String extraInformation) {
super(errorCode, extraInformation);
- this.credentials = null;
+ this.accessConfig = null;
}
/**
* Constructor for success
*
- * @param credentials credentials
+ * @param accessConfig credentials
*/
- public ScopedCredentialsResult(@Nonnull EnumMap<StorageAccessProperty,
String> credentials) {
+ public ScopedCredentialsResult(AccessConfig accessConfig) {
super(ReturnStatus.SUCCESS);
- this.credentials = credentials;
+ this.accessConfig = accessConfig;
}
- @JsonCreator
- private ScopedCredentialsResult(
- @JsonProperty("returnStatus") @Nonnull ReturnStatus returnStatus,
- @JsonProperty("extraInformation") String extraInformation,
- @JsonProperty("credentials") Map<String, String> credentials) {
- super(returnStatus, extraInformation);
- this.credentials = new EnumMap<>(StorageAccessProperty.class);
- if (credentials != null) {
- credentials.forEach((k, v) ->
this.credentials.put(StorageAccessProperty.valueOf(k), v));
- }
- }
-
- public EnumMap<StorageAccessProperty, String> getCredentials() {
- return credentials;
+ public AccessConfig getAccessConfig() {
+ return accessConfig;
}
}
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java
b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java
index 8c8e26eb8..53d57ceb2 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java
@@ -22,7 +22,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import java.util.ArrayList;
-import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -76,9 +75,9 @@ import
org.apache.polaris.core.policy.PolarisPolicyMappingRecord;
import org.apache.polaris.core.policy.PolicyEntity;
import org.apache.polaris.core.policy.PolicyMappingUtil;
import org.apache.polaris.core.policy.PolicyType;
+import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
import org.apache.polaris.core.storage.PolarisStorageIntegration;
-import org.apache.polaris.core.storage.StorageAccessProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2062,14 +2061,14 @@ public class TransactionalMetaStoreManagerImpl extends
BaseMetaStoreManager {
PolarisStorageConfigurationInfo storageConfigurationInfo =
BaseMetaStoreManager.extractStorageConfiguration(callCtx,
reloadedEntity.getEntity());
try {
- EnumMap<StorageAccessProperty, String> creds =
+ AccessConfig accessConfig =
storageIntegration.getSubscopedCreds(
callCtx,
storageConfigurationInfo,
allowListOperation,
allowedReadLocations,
allowedWriteLocations);
- return new ScopedCredentialsResult(creds);
+ return new ScopedCredentialsResult(accessConfig);
} catch (Exception ex) {
return new ScopedCredentialsResult(
BaseResult.ReturnStatus.SUBSCOPE_CREDS_ERROR, ex.getMessage());
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java
index 61a754ecc..e15fd2e91 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java
@@ -18,7 +18,10 @@
*/
package org.apache.polaris.core.storage;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.time.Instant;
import java.util.Map;
+import java.util.Optional;
import org.apache.polaris.immutables.PolarisImmutable;
@PolarisImmutable
@@ -27,7 +30,52 @@ public interface AccessConfig {
Map<String, String> extraProperties();
- static ImmutableAccessConfig.Builder builder() {
+ /**
+ * Configuration properties that are relevant only to the Polaris Server,
but not to clients.
+ * These properties override corresponding entries from {@link
#extraProperties()}.
+ */
+ Map<String, String> internalProperties();
+
+ Optional<Instant> expiresAt();
+
+ default String get(StorageAccessProperty key) {
+ if (key.isCredential()) {
+ return credentials().get(key.getPropertyName());
+ } else {
+ String value = internalProperties().get(key.getPropertyName());
+ return value != null ? value :
extraProperties().get(key.getPropertyName());
+ }
+ }
+
+ static AccessConfig.Builder builder() {
return ImmutableAccessConfig.builder();
}
+
+ interface Builder {
+ @CanIgnoreReturnValue
+ Builder putCredential(String key, String value);
+
+ @CanIgnoreReturnValue
+ Builder putExtraProperty(String key, String value);
+
+ @CanIgnoreReturnValue
+ Builder putInternalProperty(String key, String value);
+
+ @CanIgnoreReturnValue
+ Builder expiresAt(Instant expiresAt);
+
+ default Builder put(StorageAccessProperty key, String value) {
+ if (key.isExpirationTimestamp()) {
+ expiresAt(Instant.ofEpochMilli(Long.parseLong(value)));
+ }
+
+ if (key.isCredential()) {
+ return putCredential(key.getPropertyName(), value);
+ } else {
+ return putExtraProperty(key.getPropertyName(), value);
+ }
+ }
+
+ AccessConfig build();
+ }
}
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java
index 147ec5c66..c7f060a12 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java
@@ -19,7 +19,6 @@
package org.apache.polaris.core.storage;
import jakarta.annotation.Nonnull;
-import java.util.EnumMap;
import java.util.Map;
import java.util.Set;
import org.apache.polaris.core.context.CallContext;
@@ -53,7 +52,7 @@ public abstract class PolarisStorageIntegration<T extends
PolarisStorageConfigur
* @param allowedWriteLocations a set of allowed to write locations
* @return An enum map including the scoped credentials
*/
- public abstract EnumMap<StorageAccessProperty, String> getSubscopedCreds(
+ public abstract AccessConfig getSubscopedCreds(
@Nonnull CallContext callContext,
@Nonnull T storageConfig,
boolean allowListOperation,
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java
index bfd8c934d..33526d2e2 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java
@@ -31,7 +31,9 @@ public enum StorageAccessProperty {
AWS_SESSION_TOKEN_EXPIRES_AT_MS(
String.class,
"s3.session-token-expires-at-ms",
- "the time the aws session token expires, in milliseconds"),
+ "the time the aws session token expires, in milliseconds",
+ true,
+ true),
AWS_ENDPOINT(String.class, "s3.endpoint", "the S3 endpoint to use for
requests", false),
AWS_PATH_STYLE_ACCESS(
Boolean.class, "s3.path-style-access", "whether to use S3 path style
access", false),
@@ -42,23 +44,26 @@ public enum StorageAccessProperty {
GCS_ACCESS_TOKEN_EXPIRES_AT(
String.class,
"gcs.oauth2.token-expires-at",
- "the time the gcs access token expires, in milliseconds"),
+ "the time the gcs access token expires, in milliseconds",
+ true,
+ true),
// Currently not using ACCESS TOKEN as the ResolvingFileIO is using
ADLSFileIO for azure case and
// it expects for SAS
AZURE_ACCESS_TOKEN(String.class, "", "the azure scoped access token"),
AZURE_SAS_TOKEN(String.class, "adls.sas-token.", "an azure shared access
signature token"),
- AZURE_ACCOUNT_HOST(
- String.class,
- "the azure storage account host",
- "the azure account name + endpoint that will append to the
ADLS_SAS_TOKEN_PREFIX"),
EXPIRATION_TIME(
- Long.class, "expiration-time", "the expiration time for the access
token, in milliseconds");
+ Long.class,
+ "expiration-time",
+ "the expiration time for the access token, in milliseconds",
+ true,
+ true);
private final Class valueType;
private final String propertyName;
private final String description;
private final boolean isCredential;
+ private final boolean isExpirationTimestamp;
/*
s3.access-key-id`: id for for credentials that provide access to the data in
S3
@@ -71,10 +76,20 @@ public enum StorageAccessProperty {
StorageAccessProperty(
Class valueType, String propertyName, String description, boolean
isCredential) {
+ this(valueType, propertyName, description, isCredential, false);
+ }
+
+ StorageAccessProperty(
+ Class valueType,
+ String propertyName,
+ String description,
+ boolean isCredential,
+ boolean isExpirationTimestamp) {
this.valueType = valueType;
this.propertyName = propertyName;
this.description = description;
this.isCredential = isCredential;
+ this.isExpirationTimestamp = isExpirationTimestamp;
}
public String getPropertyName() {
@@ -84,4 +99,8 @@ public enum StorageAccessProperty {
public boolean isCredential() {
return isCredential;
}
+
+ public boolean isExpirationTimestamp() {
+ return isExpirationTimestamp;
+ }
}
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java
index 338f60c4e..f619a9a13 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java
@@ -22,13 +22,13 @@ import static
org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDEN
import jakarta.annotation.Nonnull;
import java.net.URI;
-import java.util.EnumMap;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.InMemoryStorageIntegration;
import org.apache.polaris.core.storage.StorageAccessProperty;
import org.apache.polaris.core.storage.StorageUtil;
@@ -66,7 +66,7 @@ public class AwsCredentialsStorageIntegration
/** {@inheritDoc} */
@Override
- public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
+ public AccessConfig getSubscopedCreds(
@Nonnull CallContext callContext,
@Nonnull AwsStorageConfigurationInfo storageConfig,
boolean allowListOperation,
@@ -90,50 +90,48 @@ public class AwsCredentialsStorageIntegration
credentialsProvider.ifPresent(
cp -> request.overrideConfiguration(b -> b.credentialsProvider(cp)));
+ String region = storageConfig.getRegion();
@SuppressWarnings("resource")
// Note: stsClientProvider returns "thin" clients that do not need closing
StsClient stsClient =
- stsClientProvider.stsClient(
- StsDestination.of(storageConfig.getStsEndpointUri(),
storageConfig.getRegion()));
+
stsClientProvider.stsClient(StsDestination.of(storageConfig.getStsEndpointUri(),
region));
AssumeRoleResponse response = stsClient.assumeRole(request.build());
- EnumMap<StorageAccessProperty, String> credentialMap =
- new EnumMap<>(StorageAccessProperty.class);
- credentialMap.put(StorageAccessProperty.AWS_KEY_ID,
response.credentials().accessKeyId());
- credentialMap.put(
+ AccessConfig.Builder accessConfig = AccessConfig.builder();
+ accessConfig.put(StorageAccessProperty.AWS_KEY_ID,
response.credentials().accessKeyId());
+ accessConfig.put(
StorageAccessProperty.AWS_SECRET_KEY,
response.credentials().secretAccessKey());
- credentialMap.put(StorageAccessProperty.AWS_TOKEN,
response.credentials().sessionToken());
+ accessConfig.put(StorageAccessProperty.AWS_TOKEN,
response.credentials().sessionToken());
Optional.ofNullable(response.credentials().expiration())
.ifPresent(
i -> {
- credentialMap.put(
+ accessConfig.put(
StorageAccessProperty.EXPIRATION_TIME,
String.valueOf(i.toEpochMilli()));
- credentialMap.put(
+ accessConfig.put(
StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS,
String.valueOf(i.toEpochMilli()));
});
- if (storageConfig.getRegion() != null) {
- credentialMap.put(StorageAccessProperty.CLIENT_REGION,
storageConfig.getRegion());
+ if (region != null) {
+ accessConfig.put(StorageAccessProperty.CLIENT_REGION, region);
}
URI endpointUri = storageConfig.getEndpointUri();
if (endpointUri != null) {
- credentialMap.put(StorageAccessProperty.AWS_ENDPOINT,
endpointUri.toString());
+ accessConfig.put(StorageAccessProperty.AWS_ENDPOINT,
endpointUri.toString());
}
if (Boolean.TRUE.equals(storageConfig.getPathStyleAccess())) {
- credentialMap.put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS,
Boolean.TRUE.toString());
+ accessConfig.put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS,
Boolean.TRUE.toString());
}
- if (storageConfig.getAwsPartition().equals("aws-us-gov")
- && credentialMap.get(StorageAccessProperty.CLIENT_REGION) == null) {
+ if (storageConfig.getAwsPartition().equals("aws-us-gov") && region ==
null) {
throw new IllegalArgumentException(
String.format(
"AWS region must be set when using partition %s",
storageConfig.getAwsPartition()));
}
- return credentialMap;
+ return accessConfig.build();
}
/**
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java
index 232994a37..cc6ccfdb3 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java
@@ -37,17 +37,18 @@ import
com.azure.storage.file.datalake.DataLakeServiceClientBuilder;
import com.azure.storage.file.datalake.models.DataLakeStorageException;
import com.azure.storage.file.datalake.sas.DataLakeServiceSasSignatureValues;
import com.azure.storage.file.datalake.sas.PathSasPermission;
+import com.google.common.annotations.VisibleForTesting;
import jakarta.annotation.Nonnull;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.Period;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
-import java.util.EnumMap;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.InMemoryStorageIntegration;
import org.apache.polaris.core.storage.StorageAccessProperty;
import org.slf4j.Logger;
@@ -71,14 +72,12 @@ public class AzureCredentialsStorageIntegration
}
@Override
- public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
+ public AccessConfig getSubscopedCreds(
@Nonnull CallContext callContext,
@Nonnull AzureStorageConfigurationInfo storageConfig,
boolean allowListOperation,
@Nonnull Set<String> allowedReadLocations,
@Nonnull Set<String> allowedWriteLocations) {
- EnumMap<StorageAccessProperty, String> credentialMap =
- new EnumMap<>(StorageAccessProperty.class);
String loc =
!allowedWriteLocations.isEmpty()
? allowedWriteLocations.stream().findAny().orElse(null)
@@ -171,12 +170,41 @@ public class AzureCredentialsStorageIntegration
throw new RuntimeException(
String.format("Endpoint %s not supported", location.getEndpoint()));
}
- credentialMap.put(StorageAccessProperty.AZURE_SAS_TOKEN, sasToken);
- credentialMap.put(StorageAccessProperty.AZURE_ACCOUNT_HOST,
storageDnsName);
- credentialMap.put(
- StorageAccessProperty.EXPIRATION_TIME,
- String.valueOf(sanitizedEndTime.toInstant().toEpochMilli()));
- return credentialMap;
+
+ return toAccessConfig(sasToken, storageDnsName,
sanitizedEndTime.toInstant());
+ }
+
+ @VisibleForTesting
+ static AccessConfig toAccessConfig(String sasToken, String storageDnsName,
Instant expiresAt) {
+ AccessConfig.Builder accessConfig = AccessConfig.builder();
+ handleAzureCredential(accessConfig, sasToken, storageDnsName);
+ accessConfig.put(
+ StorageAccessProperty.EXPIRATION_TIME,
String.valueOf(expiresAt.toEpochMilli()));
+ return accessConfig.build();
+ }
+
+ private static void handleAzureCredential(
+ AccessConfig.Builder config, String sasToken, String host) {
+
config.putCredential(StorageAccessProperty.AZURE_SAS_TOKEN.getPropertyName() +
host, sasToken);
+
+ // Iceberg 1.7.x may expect the credential key to _not_ be suffixed with
endpoint
+ if (host.endsWith(AzureLocation.ADLS_ENDPOINT)) {
+ int suffixIndex = host.lastIndexOf(AzureLocation.ADLS_ENDPOINT) - 1;
+ if (suffixIndex > 0) {
+ String withSuffixStripped = host.substring(0, suffixIndex);
+ config.putCredential(
+ StorageAccessProperty.AZURE_SAS_TOKEN.getPropertyName() +
withSuffixStripped, sasToken);
+ }
+ }
+
+ if (host.endsWith(AzureLocation.BLOB_ENDPOINT)) {
+ int suffixIndex = host.lastIndexOf(AzureLocation.BLOB_ENDPOINT) - 1;
+ if (suffixIndex > 0) {
+ String withSuffixStripped = host.substring(0, suffixIndex);
+ config.putCredential(
+ StorageAccessProperty.AZURE_SAS_TOKEN.getPropertyName() +
withSuffixStripped, sasToken);
+ }
+ }
}
private String getBlobUserDelegationSas(
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java
index 49dc858d3..d8d88edc6 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java
@@ -133,7 +133,8 @@ public class StorageCredentialCache {
k.allowedWriteLocations());
if (scopedCredentialsResult.isSuccess()) {
long maxCacheDurationMs =
maxCacheDurationMs(callCtx.getRealmConfig());
- return new StorageCredentialCacheEntry(scopedCredentialsResult,
maxCacheDurationMs);
+ return new StorageCredentialCacheEntry(
+ scopedCredentialsResult.getAccessConfig(), maxCacheDurationMs);
}
LOGGER
.atDebug()
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheEntry.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheEntry.java
index cef5907f2..7f5789ecb 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheEntry.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheEntry.java
@@ -18,27 +18,19 @@
*/
package org.apache.polaris.core.storage.cache;
-import java.util.EnumMap;
-import java.util.function.BiConsumer;
-import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult;
+import java.time.Instant;
import org.apache.polaris.core.storage.AccessConfig;
-import org.apache.polaris.core.storage.ImmutableAccessConfig;
-import org.apache.polaris.core.storage.StorageAccessProperty;
-import org.apache.polaris.core.storage.azure.AzureLocation;
/** A storage credential cached entry. */
public class StorageCredentialCacheEntry {
/** The scoped creds map that is fetched from a creds vending service */
- public final EnumMap<StorageAccessProperty, String> credsMap;
+ public final AccessConfig accessConfig;
- private final ScopedCredentialsResult scopedCredentialsResult;
private final long maxCacheDurationMs;
- public StorageCredentialCacheEntry(
- ScopedCredentialsResult scopedCredentialsResult, long
maxCacheDurationMs) {
- this.scopedCredentialsResult = scopedCredentialsResult;
+ public StorageCredentialCacheEntry(AccessConfig accessConfig, long
maxCacheDurationMs) {
+ this.accessConfig = accessConfig;
this.maxCacheDurationMs = maxCacheDurationMs;
- this.credsMap = scopedCredentialsResult.getCredentials();
}
public long getMaxCacheDurationMs() {
@@ -47,45 +39,7 @@ public class StorageCredentialCacheEntry {
/** Get the expiration time in millisecond for the cached entry */
public long getExpirationTime() {
- if
(credsMap.containsKey(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT)) {
- return
Long.parseLong(credsMap.get(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT));
- }
- if
(credsMap.containsKey(StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS)) {
- return
Long.parseLong(credsMap.get(StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS));
- }
- if (credsMap.containsKey(StorageAccessProperty.EXPIRATION_TIME)) {
- return
Long.parseLong(credsMap.get(StorageAccessProperty.EXPIRATION_TIME));
- }
- return Long.MAX_VALUE;
- }
-
- /**
- * Azure needs special handling, the credential key is dynamically generated
based on the storage
- * account endpoint
- */
- private void handleAzureCredential(
- BiConsumer<String, String> results, StorageAccessProperty
credentialProperty, String value) {
- if (credentialProperty.equals(StorageAccessProperty.AZURE_SAS_TOKEN)) {
- String host = credsMap.get(StorageAccessProperty.AZURE_ACCOUNT_HOST);
- results.accept(credentialProperty.getPropertyName() + host, value);
-
- // Iceberg 1.7.x may expect the credential key to _not_ be suffixed with
endpoint
- if (host.endsWith(AzureLocation.ADLS_ENDPOINT)) {
- int suffixIndex = host.lastIndexOf(AzureLocation.ADLS_ENDPOINT) - 1;
- if (suffixIndex > 0) {
- String withSuffixStripped = host.substring(0, suffixIndex);
- results.accept(credentialProperty.getPropertyName() +
withSuffixStripped, value);
- }
- }
-
- if (host.endsWith(AzureLocation.BLOB_ENDPOINT)) {
- int suffixIndex = host.lastIndexOf(AzureLocation.BLOB_ENDPOINT) - 1;
- if (suffixIndex > 0) {
- String withSuffixStripped = host.substring(0, suffixIndex);
- results.accept(credentialProperty.getPropertyName() +
withSuffixStripped, value);
- }
- }
- }
+ return
accessConfig.expiresAt().map(Instant::toEpochMilli).orElse(Long.MAX_VALUE);
}
/**
@@ -94,22 +48,6 @@ public class StorageCredentialCacheEntry {
* @return a map of string representing the subscoped creds info.
*/
AccessConfig toAccessConfig() {
- ImmutableAccessConfig.Builder config = AccessConfig.builder();
- if (!credsMap.isEmpty()) {
- credsMap.forEach(
- (key, value) -> {
- if (!key.isCredential()) {
- config.putExtraProperty(key.getPropertyName(), value);
- return;
- }
-
- if (key.equals(StorageAccessProperty.AZURE_SAS_TOKEN)) {
- handleAzureCredential(config::putCredential, key, value);
- } else if (!key.equals(StorageAccessProperty.AZURE_ACCOUNT_HOST)) {
- config.putCredential(key.getPropertyName(), value);
- }
- });
- }
- return config.build();
+ return accessConfig;
}
}
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java
b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java
index 2fc1438fe..1dff89ca1 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java
@@ -30,7 +30,6 @@ import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
-import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -39,6 +38,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Stream;
import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.InMemoryStorageIntegration;
import org.apache.polaris.core.storage.PolarisStorageIntegration;
import org.apache.polaris.core.storage.StorageAccessProperty;
@@ -69,7 +69,7 @@ public class GcpCredentialsStorageIntegration
}
@Override
- public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
+ public AccessConfig getSubscopedCreds(
@Nonnull CallContext callContext,
@Nonnull GcpStorageConfigurationInfo storageConfig,
boolean allowListOperation,
@@ -106,12 +106,12 @@ public class GcpCredentialsStorageIntegration
// If expires_in missing, use source credential's expire time, which
require another api call to
// get.
- EnumMap<StorageAccessProperty, String> propertyMap = new
EnumMap<>(StorageAccessProperty.class);
- propertyMap.put(StorageAccessProperty.GCS_ACCESS_TOKEN,
token.getTokenValue());
- propertyMap.put(
+ AccessConfig.Builder accessConfig = AccessConfig.builder();
+ accessConfig.put(StorageAccessProperty.GCS_ACCESS_TOKEN,
token.getTokenValue());
+ accessConfig.put(
StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT,
String.valueOf(token.getExpirationTime().getTime()));
- return propertyMap;
+ return accessConfig.build();
}
private String convertToString(CredentialAccessBoundary accessBoundary) {
diff --git
a/polaris-core/src/test/java/org/apache/polaris/core/storage/AccessConfigTest.java
b/polaris-core/src/test/java/org/apache/polaris/core/storage/AccessConfigTest.java
new file mode 100644
index 000000000..57e1f1465
--- /dev/null
+++
b/polaris-core/src/test/java/org/apache/polaris/core/storage/AccessConfigTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.core.storage;
+
+import static
org.apache.polaris.core.storage.StorageAccessProperty.AWS_ENDPOINT;
+import static
org.apache.polaris.core.storage.StorageAccessProperty.AWS_SECRET_KEY;
+import static
org.apache.polaris.core.storage.StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS;
+import static
org.apache.polaris.core.storage.StorageAccessProperty.EXPIRATION_TIME;
+import static
org.apache.polaris.core.storage.StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.Instant;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+
+public class AccessConfigTest {
+
+ @Test
+ public void testPutGet() {
+ AccessConfig.Builder b = AccessConfig.builder();
+ b.put(AWS_ENDPOINT, "ep1");
+ b.put(AWS_SECRET_KEY, "sk2");
+ AccessConfig c = b.build();
+
assertThat(c.credentials()).isEqualTo(Map.of(AWS_SECRET_KEY.getPropertyName(),
"sk2"));
+
assertThat(c.extraProperties()).isEqualTo(Map.of(AWS_ENDPOINT.getPropertyName(),
"ep1"));
+ assertThat(c.get(AWS_SECRET_KEY)).isEqualTo("sk2");
+ assertThat(c.get(AWS_ENDPOINT)).isEqualTo("ep1");
+ }
+
+ @Test
+ public void testGetExtraProperty() {
+ AccessConfig.Builder b = AccessConfig.builder();
+ b.putExtraProperty(AWS_ENDPOINT.getPropertyName(), "extra");
+ AccessConfig c = b.build();
+
assertThat(c.extraProperties()).isEqualTo(Map.of(AWS_ENDPOINT.getPropertyName(),
"extra"));
+ assertThat(c.get(AWS_ENDPOINT)).isEqualTo("extra");
+ }
+
+ @Test
+ public void testGetInternalProperty() {
+ AccessConfig.Builder b = AccessConfig.builder();
+ b.putExtraProperty(AWS_ENDPOINT.getPropertyName(), "extra");
+ b.putInternalProperty(AWS_ENDPOINT.getPropertyName(), "ep1");
+ AccessConfig c = b.build();
+
assertThat(c.extraProperties()).isEqualTo(Map.of(AWS_ENDPOINT.getPropertyName(),
"extra"));
+
assertThat(c.internalProperties()).isEqualTo(Map.of(AWS_ENDPOINT.getPropertyName(),
"ep1"));
+ assertThat(c.get(AWS_ENDPOINT)).isEqualTo("ep1");
+ }
+
+ @Test
+ public void testNoCredentialOverride() {
+ AccessConfig.Builder b = AccessConfig.builder();
+ b.put(AWS_SECRET_KEY, "sk-test");
+ b.putExtraProperty(AWS_SECRET_KEY.getPropertyName(), "sk-extra");
+ b.putInternalProperty(AWS_SECRET_KEY.getPropertyName(), "sk-internal");
+ AccessConfig c = b.build();
+ assertThat(c.get(AWS_SECRET_KEY)).isEqualTo("sk-test");
+
assertThat(c.extraProperties()).isEqualTo(Map.of(AWS_SECRET_KEY.getPropertyName(),
"sk-extra"));
+ assertThat(c.internalProperties())
+ .isEqualTo(Map.of(AWS_SECRET_KEY.getPropertyName(), "sk-internal"));
+ }
+
+ @Test
+ public void testExpiresAt() {
+ AccessConfig.Builder b = AccessConfig.builder();
+ assertThat(b.build().expiresAt()).isEmpty();
+ b.put(GCS_ACCESS_TOKEN_EXPIRES_AT, "111");
+ assertThat(b.build().expiresAt()).hasValue(Instant.ofEpochMilli(111));
+ b.put(AWS_SESSION_TOKEN_EXPIRES_AT_MS, "222");
+ assertThat(b.build().expiresAt()).hasValue(Instant.ofEpochMilli(222));
+ b.put(EXPIRATION_TIME, "333");
+ assertThat(b.build().expiresAt()).hasValue(Instant.ofEpochMilli(333));
+ b.expiresAt(Instant.ofEpochMilli(444));
+ assertThat(b.build().expiresAt()).hasValue(Instant.ofEpochMilli(444));
+ }
+}
diff --git
a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java
b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java
index d0bf9de8a..0842a132b 100644
---
a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java
+++
b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java
@@ -21,7 +21,6 @@ package org.apache.polaris.core.storage;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import java.time.Clock;
-import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -212,7 +211,7 @@ class InMemoryStorageIntegrationTest {
}
@Override
- public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
+ public AccessConfig getSubscopedCreds(
@Nonnull CallContext callContext,
@Nonnull PolarisStorageConfigurationInfo storageConfig,
boolean allowListOperation,
diff --git
a/polaris-core/src/test/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegrationTest.java
b/polaris-core/src/test/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegrationTest.java
new file mode 100644
index 000000000..89b60dba5
--- /dev/null
+++
b/polaris-core/src/test/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegrationTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.core.storage.azure;
+
+import static
org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration.toAccessConfig;
+
+import java.time.Instant;
+import org.apache.polaris.core.storage.AccessConfig;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class AzureCredentialsStorageIntegrationTest {
+
+ @Test
+ public void testAzureCredentialFormatting() {
+ Instant expiresAt = Instant.ofEpochMilli(Long.MAX_VALUE);
+
+ AccessConfig noSuffixResult = toAccessConfig("sasToken", "some_account",
expiresAt);
+ Assertions.assertThat(noSuffixResult.credentials()).hasSize(2);
+
Assertions.assertThat(noSuffixResult.credentials()).containsKey("adls.sas-token.some_account");
+
+ AccessConfig adlsSuffixResult =
+ toAccessConfig("sasToken", "some_account." +
AzureLocation.ADLS_ENDPOINT, expiresAt);
+ Assertions.assertThat(adlsSuffixResult.credentials()).hasSize(3);
+ Assertions.assertThat(adlsSuffixResult.credentials())
+ .containsKey("adls.sas-token.some_account");
+ Assertions.assertThat(adlsSuffixResult.credentials())
+ .containsKey("adls.sas-token.some_account." +
AzureLocation.ADLS_ENDPOINT);
+
+ AccessConfig blobSuffixResult =
+ toAccessConfig("sasToken", "some_account." +
AzureLocation.BLOB_ENDPOINT, expiresAt);
+ Assertions.assertThat(blobSuffixResult.credentials()).hasSize(3);
+ Assertions.assertThat(blobSuffixResult.credentials())
+ .containsKey("adls.sas-token.some_account");
+ Assertions.assertThat(blobSuffixResult.credentials())
+ .containsKey("adls.sas-token.some_account." +
AzureLocation.BLOB_ENDPOINT);
+ }
+}
diff --git
a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java
b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java
index bf1cf1fcf..39095f08b 100644
---
a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java
+++
b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java
@@ -20,11 +20,9 @@ package org.apache.polaris.core.storage.cache;
import static
org.apache.polaris.core.persistence.PrincipalSecretsGenerator.RANDOM_SECRETS;
-import com.google.common.collect.ImmutableMap;
import jakarta.annotation.Nonnull;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -46,7 +44,6 @@ import
org.apache.polaris.core.persistence.transactional.TreeMapMetaStore;
import
org.apache.polaris.core.persistence.transactional.TreeMapTransactionalPersistenceImpl;
import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.StorageAccessProperty;
-import org.apache.polaris.core.storage.azure.AzureLocation;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
@@ -398,30 +395,26 @@ public class StorageCredentialCacheTest {
: String.valueOf(Long.MAX_VALUE);
res.add(
new ScopedCredentialsResult(
- new EnumMap<>(
- ImmutableMap.<StorageAccessProperty, String>builder()
- .put(StorageAccessProperty.AWS_KEY_ID, "key_id_" +
finalI)
- .put(StorageAccessProperty.AWS_SECRET_KEY, "key_secret_"
+ finalI)
-
.put(StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, expireTime)
- .put(StorageAccessProperty.EXPIRATION_TIME, expireTime)
- .buildOrThrow())));
+ AccessConfig.builder()
+ .put(StorageAccessProperty.AWS_KEY_ID, "key_id_" + finalI)
+ .put(StorageAccessProperty.AWS_SECRET_KEY, "key_secret_" +
finalI)
+ .put(StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS,
expireTime)
+ .put(StorageAccessProperty.EXPIRATION_TIME, expireTime)
+ .build()));
if (res.size() == number) return res;
res.add(
new ScopedCredentialsResult(
- new EnumMap<>(
- ImmutableMap.<StorageAccessProperty, String>builder()
- .put(StorageAccessProperty.AZURE_SAS_TOKEN, "sas_token_"
+ finalI)
- .put(StorageAccessProperty.AZURE_ACCOUNT_HOST,
"account_host")
- .put(StorageAccessProperty.EXPIRATION_TIME, expireTime)
- .buildOrThrow())));
+ AccessConfig.builder()
+ .put(StorageAccessProperty.AZURE_SAS_TOKEN, "sas_token_" +
finalI)
+ .put(StorageAccessProperty.EXPIRATION_TIME, expireTime)
+ .build()));
if (res.size() == number) return res;
res.add(
new ScopedCredentialsResult(
- new EnumMap<>(
- ImmutableMap.<StorageAccessProperty, String>builder()
- .put(StorageAccessProperty.GCS_ACCESS_TOKEN,
"gcs_token_" + finalI)
- .put(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT,
expireTime)
- .buildOrThrow())));
+ AccessConfig.builder()
+ .put(StorageAccessProperty.GCS_ACCESS_TOKEN, "gcs_token_" +
finalI)
+ .put(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT,
expireTime)
+ .build()));
}
return res;
}
@@ -444,106 +437,16 @@ public class StorageCredentialCacheTest {
return Arrays.asList(polarisEntity1, polarisEntity2, polarisEntity3);
}
- @Test
- public void testAzureCredentialFormatting() {
- storageCredentialCache = newStorageCredentialCache();
- List<ScopedCredentialsResult> mockedScopedCreds =
- List.of(
- new ScopedCredentialsResult(
- new EnumMap<>(
- ImmutableMap.<StorageAccessProperty, String>builder()
- .put(StorageAccessProperty.AZURE_SAS_TOKEN,
"sas_token_azure_1")
- .put(StorageAccessProperty.AZURE_ACCOUNT_HOST,
"some_account")
- .put(StorageAccessProperty.EXPIRATION_TIME,
String.valueOf(Long.MAX_VALUE))
- .buildOrThrow())),
- new ScopedCredentialsResult(
- new EnumMap<>(
- ImmutableMap.<StorageAccessProperty, String>builder()
- .put(StorageAccessProperty.AZURE_SAS_TOKEN,
"sas_token_azure_2")
- .put(
- StorageAccessProperty.AZURE_ACCOUNT_HOST,
- "some_account." + AzureLocation.ADLS_ENDPOINT)
- .put(StorageAccessProperty.EXPIRATION_TIME,
String.valueOf(Long.MAX_VALUE))
- .buildOrThrow())),
- new ScopedCredentialsResult(
- new EnumMap<>(
- ImmutableMap.<StorageAccessProperty, String>builder()
- .put(StorageAccessProperty.AZURE_SAS_TOKEN,
"sas_token_azure_3")
- .put(
- StorageAccessProperty.AZURE_ACCOUNT_HOST,
- "some_account." + AzureLocation.BLOB_ENDPOINT)
- .put(StorageAccessProperty.EXPIRATION_TIME,
String.valueOf(Long.MAX_VALUE))
- .buildOrThrow())));
-
- Mockito.when(
- metaStoreManager.getSubscopedCredsForEntity(
- Mockito.any(),
- Mockito.anyLong(),
- Mockito.anyLong(),
- Mockito.any(),
- Mockito.anyBoolean(),
- Mockito.anySet(),
- Mockito.anySet()))
- .thenReturn(mockedScopedCreds.get(0))
- .thenReturn(mockedScopedCreds.get(1))
- .thenReturn(mockedScopedCreds.get(2));
- List<PolarisEntity> entityList = getPolarisEntities();
-
- Map<String, String> noSuffixResult =
- storageCredentialCache
- .getOrGenerateSubScopeCreds(
- metaStoreManager,
- callCtx,
- entityList.get(0),
- true,
- Set.of("s3://bucket1/path", "s3://bucket2/path"),
- Set.of("s3://bucket3/path", "s3://bucket4/path"))
- .credentials();
- Assertions.assertThat(noSuffixResult.size()).isEqualTo(2);
-
Assertions.assertThat(noSuffixResult).containsKey("adls.sas-token.some_account");
-
- Map<String, String> adlsSuffixResult =
- storageCredentialCache
- .getOrGenerateSubScopeCreds(
- metaStoreManager,
- callCtx,
- entityList.get(1),
- true,
- Set.of("s3://bucket1/path", "s3://bucket2/path"),
- Set.of("s3://bucket3/path", "s3://bucket4/path"))
- .credentials();
- Assertions.assertThat(adlsSuffixResult.size()).isEqualTo(3);
-
Assertions.assertThat(adlsSuffixResult).containsKey("adls.sas-token.some_account");
- Assertions.assertThat(adlsSuffixResult)
- .containsKey("adls.sas-token.some_account." +
AzureLocation.ADLS_ENDPOINT);
-
- Map<String, String> blobSuffixResult =
- storageCredentialCache
- .getOrGenerateSubScopeCreds(
- metaStoreManager,
- callCtx,
- entityList.get(2),
- true,
- Set.of("s3://bucket1/path", "s3://bucket2/path"),
- Set.of("s3://bucket3/path", "s3://bucket4/path"))
- .credentials();
- Assertions.assertThat(blobSuffixResult.size()).isEqualTo(3);
-
Assertions.assertThat(blobSuffixResult).containsKey("adls.sas-token.some_account");
- Assertions.assertThat(blobSuffixResult)
- .containsKey("adls.sas-token.some_account." +
AzureLocation.BLOB_ENDPOINT);
- }
-
@Test
public void testExtraProperties() {
storageCredentialCache = newStorageCredentialCache();
ScopedCredentialsResult properties =
new ScopedCredentialsResult(
- new EnumMap<>(
- ImmutableMap.<StorageAccessProperty, String>builder()
- .put(StorageAccessProperty.AWS_SECRET_KEY,
"super-secret-123")
- .put(StorageAccessProperty.AWS_ENDPOINT, "test-endpoint1")
- .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS, "true")
- .buildOrThrow()));
+ AccessConfig.builder()
+ .put(StorageAccessProperty.AWS_SECRET_KEY, "super-secret-123")
+ .put(StorageAccessProperty.AWS_ENDPOINT, "test-endpoint1")
+ .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS, "true")
+ .build());
Mockito.when(
metaStoreManager.getSubscopedCredsForEntity(
Mockito.any(),
diff --git
a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java
b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java
index 7dee37ebb..cd7729e90 100644
---
a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java
+++
b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java
@@ -18,13 +18,14 @@
*/
package org.apache.polaris.service.storage.aws;
+import static
org.apache.polaris.core.storage.PolarisStorageConfigurationInfo.StorageType.S3;
import static org.assertj.core.api.Assertions.assertThat;
import jakarta.annotation.Nonnull;
import java.time.Instant;
-import java.util.EnumMap;
import java.util.List;
import java.util.Set;
+import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.BaseStorageIntegrationTest;
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
import org.apache.polaris.core.storage.StorageAccessProperty;
@@ -84,26 +85,22 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
return ASSUME_ROLE_RESPONSE;
});
String warehouseDir = scheme + "://bucket/path/to/warehouse";
- EnumMap<StorageAccessProperty, String> credentials =
+ AccessConfig accessConfig =
new AwsCredentialsStorageIntegration(stsClient)
.getSubscopedCreds(
newCallContext(),
new AwsStorageConfigurationInfo(
- PolarisStorageConfigurationInfo.StorageType.S3,
- List.of(warehouseDir),
- roleARN,
- externalId,
- null),
+ S3, List.of(warehouseDir), roleARN, externalId, null),
true,
Set.of(warehouseDir + "/namespace/table"),
Set.of(warehouseDir + "/namespace/table"));
- assertThat(credentials)
+ assertThat(accessConfig.credentials())
.isNotEmpty()
- .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess")
- .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey")
- .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey")
+ .containsEntry(StorageAccessProperty.AWS_TOKEN.getPropertyName(),
"sess")
+ .containsEntry(StorageAccessProperty.AWS_KEY_ID.getPropertyName(),
"accessKey")
+ .containsEntry(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(),
"secretKey")
.containsEntry(
- StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS,
+
StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS.getPropertyName(),
String.valueOf(EXPIRE_TIME.toEpochMilli()));
}
@@ -249,7 +246,7 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
break;
case AWS_PARTITION:
case "aws-us-gov":
- EnumMap<StorageAccessProperty, String> credentials =
+ AccessConfig accessConfig =
new AwsCredentialsStorageIntegration(stsClient)
.getSubscopedCreds(
newCallContext(),
@@ -262,13 +259,13 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
true,
Set.of(s3Path(bucket, firstPath), s3Path(bucket,
secondPath)),
Set.of(s3Path(bucket, firstPath)));
- assertThat(credentials)
+ assertThat(accessConfig.credentials())
.isNotEmpty()
- .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess")
- .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey")
- .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey")
+ .containsEntry(StorageAccessProperty.AWS_TOKEN.getPropertyName(),
"sess")
+ .containsEntry(StorageAccessProperty.AWS_KEY_ID.getPropertyName(),
"accessKey")
+
.containsEntry(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(),
"secretKey")
.containsEntry(
- StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS,
+
StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS.getPropertyName(),
String.valueOf(EXPIRE_TIME.toEpochMilli()));
break;
default:
@@ -350,12 +347,12 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
});
PolarisStorageConfigurationInfo.StorageType storageType =
PolarisStorageConfigurationInfo.StorageType.S3;
- EnumMap<StorageAccessProperty, String> credentials =
+ AccessConfig accessConfig =
new AwsCredentialsStorageIntegration(stsClient)
.getSubscopedCreds(
newCallContext(),
new AwsStorageConfigurationInfo(
- PolarisStorageConfigurationInfo.StorageType.S3,
+ S3,
List.of(s3Path(bucket, warehouseKeyPrefix)),
roleARN,
externalId,
@@ -363,13 +360,13 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
false, /* allowList = false*/
Set.of(s3Path(bucket, firstPath), s3Path(bucket, secondPath)),
Set.of(s3Path(bucket, firstPath)));
- assertThat(credentials)
+ assertThat(accessConfig.credentials())
.isNotEmpty()
- .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess")
- .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey")
- .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey")
+ .containsEntry(StorageAccessProperty.AWS_TOKEN.getPropertyName(),
"sess")
+ .containsEntry(StorageAccessProperty.AWS_KEY_ID.getPropertyName(),
"accessKey")
+ .containsEntry(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(),
"secretKey")
.containsEntry(
- StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS,
+
StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS.getPropertyName(),
String.valueOf(EXPIRE_TIME.toEpochMilli()));
}
@@ -445,7 +442,7 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
});
PolarisStorageConfigurationInfo.StorageType storageType =
PolarisStorageConfigurationInfo.StorageType.S3;
- EnumMap<StorageAccessProperty, String> credentials =
+ AccessConfig accessConfig =
new AwsCredentialsStorageIntegration(stsClient)
.getSubscopedCreds(
newCallContext(),
@@ -458,13 +455,13 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
true, /* allowList = true */
Set.of(s3Path(bucket, firstPath), s3Path(bucket, secondPath)),
Set.of());
- assertThat(credentials)
+ assertThat(accessConfig.credentials())
.isNotEmpty()
- .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess")
- .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey")
- .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey")
+ .containsEntry(StorageAccessProperty.AWS_TOKEN.getPropertyName(),
"sess")
+ .containsEntry(StorageAccessProperty.AWS_KEY_ID.getPropertyName(),
"accessKey")
+ .containsEntry(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(),
"secretKey")
.containsEntry(
- StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS,
+
StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS.getPropertyName(),
String.valueOf(EXPIRE_TIME.toEpochMilli()));
}
@@ -510,12 +507,12 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
});
return ASSUME_ROLE_RESPONSE;
});
- EnumMap<StorageAccessProperty, String> credentials =
+ AccessConfig accessConfig =
new AwsCredentialsStorageIntegration(stsClient)
.getSubscopedCreds(
newCallContext(),
new AwsStorageConfigurationInfo(
- PolarisStorageConfigurationInfo.StorageType.S3,
+ S3,
List.of(s3Path(bucket, warehouseKeyPrefix)),
roleARN,
externalId,
@@ -523,13 +520,13 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
true, /* allowList = true */
Set.of(),
Set.of());
- assertThat(credentials)
+ assertThat(accessConfig.credentials())
.isNotEmpty()
- .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess")
- .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey")
- .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey")
+ .containsEntry(StorageAccessProperty.AWS_TOKEN.getPropertyName(),
"sess")
+ .containsEntry(StorageAccessProperty.AWS_KEY_ID.getPropertyName(),
"accessKey")
+ .containsEntry(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(),
"secretKey")
.containsEntry(
- StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS,
+
StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS.getPropertyName(),
String.valueOf(EXPIRE_TIME.toEpochMilli()));
}
@@ -567,12 +564,12 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
break;
case AWS_PARTITION:
case "aws-us-gov":
- EnumMap<StorageAccessProperty, String> credentials =
+ AccessConfig accessConfig =
new AwsCredentialsStorageIntegration(stsClient)
.getSubscopedCreds(
newCallContext(),
new AwsStorageConfigurationInfo(
- PolarisStorageConfigurationInfo.StorageType.S3,
+ S3,
List.of(s3Path(bucket, warehouseKeyPrefix)),
roleARN,
externalId,
@@ -580,9 +577,9 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
true, /* allowList = true */
Set.of(),
Set.of());
- assertThat(credentials)
+ assertThat(accessConfig.credentials())
.isNotEmpty()
- .containsEntry(StorageAccessProperty.CLIENT_REGION, clientRegion);
+
.containsEntry(StorageAccessProperty.CLIENT_REGION.getPropertyName(),
clientRegion);
break;
default:
throw new IllegalArgumentException("Unknown aws partition: " +
awsPartition);
@@ -605,20 +602,18 @@ class AwsCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
});
switch (awsPartition) {
case AWS_PARTITION:
- EnumMap<StorageAccessProperty, String> credentials =
+ AccessConfig accessConfig =
new AwsCredentialsStorageIntegration(stsClient)
.getSubscopedCreds(
newCallContext(),
new AwsStorageConfigurationInfo(
- PolarisStorageConfigurationInfo.StorageType.S3,
- List.of(s3Path(bucket, warehouseKeyPrefix)),
- roleARN,
- externalId,
- null),
+ S3, List.of(s3Path(bucket, warehouseKeyPrefix)),
roleARN, externalId, null),
true, /* allowList = true */
Set.of(),
Set.of());
-
assertThat(credentials).isNotEmpty().doesNotContainKey(StorageAccessProperty.CLIENT_REGION);
+ assertThat(accessConfig.credentials())
+ .isNotEmpty()
+
.doesNotContainKey(StorageAccessProperty.CLIENT_REGION.getPropertyName());
break;
case "aws-cn":
case "aws-us-gov":
diff --git
a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java
b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java
index 9f43d42bd..14bfcac1c 100644
---
a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java
+++
b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java
@@ -42,11 +42,10 @@ import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.EnumMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.stream.Stream;
+import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.BaseStorageIntegrationTest;
import org.apache.polaris.core.storage.StorageAccessProperty;
import
org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration;
@@ -120,13 +119,13 @@ public class AzureCredentialStorageIntegrationTest
extends BaseStorageIntegratio
String.format(
"abfss://container@icebergdfsstorageacct.%s.core.windows.net/polaris-test/",
service));
- Map<StorageAccessProperty, String> credsMap =
+ AccessConfig accessConfig =
subscopedCredsForOperations(
/* allowedReadLoc= */ allowedLoc,
/* allowedWriteLoc= */ new ArrayList<>(),
allowListAction);
- Assertions.assertThat(credsMap).hasSize(2);
- String sasToken = credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN);
+ Assertions.assertThat(accessConfig.credentials()).hasSize(2);
+ String sasToken = accessConfig.get(StorageAccessProperty.AZURE_SAS_TOKEN);
Assertions.assertThat(sasToken).isNotNull();
String serviceEndpoint =
String.format("https://icebergdfsstorageacct.%s.core.windows.net";,
service);
@@ -191,7 +190,7 @@ public class AzureCredentialStorageIntegrationTest extends
BaseStorageIntegratio
String.format(
"abfss://container@icebergdfsstorageacct.%s.core.windows.net/%s",
service, allowedPrefix));
- Map<StorageAccessProperty, String> credsMap =
+ AccessConfig accessConfig =
subscopedCredsForOperations(
/* allowedReadLoc= */ allowedLoc,
/* allowedWriteLoc= */ new ArrayList<>(),
@@ -199,7 +198,7 @@ public class AzureCredentialStorageIntegrationTest extends
BaseStorageIntegratio
BlobClient blobClient =
createBlobClient(
- credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN),
+ accessConfig.get(StorageAccessProperty.AZURE_SAS_TOKEN),
"https://icebergdfsstorageacct.dfs.core.windows.net";,
"container",
allowedPrefix);
@@ -230,7 +229,7 @@ public class AzureCredentialStorageIntegrationTest extends
BaseStorageIntegratio
// read fail because container is blocked
BlobClient blobClientReadFail =
createBlobClient(
- credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN),
+ accessConfig.get(StorageAccessProperty.AZURE_SAS_TOKEN),
String.format("https://icebergdfsstorageacct.%s.core.windows.net";,
service),
"regtest",
blockedPrefix);
@@ -261,7 +260,7 @@ public class AzureCredentialStorageIntegrationTest extends
BaseStorageIntegratio
String.format(
"abfss://container@icebergdfsstorageacct.%s.core.windows.net/%s",
service, allowedPrefix));
- Map<StorageAccessProperty, String> credsMap =
+ AccessConfig accessConfig =
subscopedCredsForOperations(
/* allowedReadLoc= */ new ArrayList<>(),
/* allowedWriteLoc= */ allowedLoc,
@@ -270,13 +269,13 @@ public class AzureCredentialStorageIntegrationTest
extends BaseStorageIntegratio
String.format("https://icebergdfsstorageacct.%s.core.windows.net";,
service);
BlobClient blobClient =
createBlobClient(
- credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN),
+ accessConfig.get(StorageAccessProperty.AZURE_SAS_TOKEN),
serviceEndpoint,
"container",
allowedPrefix +
"metadata/00000-65ffa17b-fe64-4c38-bcb9-06f9bd12aa2a.metadata.json");
DataLakeFileClient fileClient =
createDatalakeFileClient(
- credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN),
+ accessConfig.get(StorageAccessProperty.AZURE_SAS_TOKEN),
serviceEndpoint,
"container",
"polaris-test/scopedcreds/metadata",
@@ -311,13 +310,13 @@ public class AzureCredentialStorageIntegrationTest
extends BaseStorageIntegratio
String blockedContainer = "regtest";
BlobClient blobClientWriteFail =
createBlobClient(
- credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN),
+ accessConfig.get(StorageAccessProperty.AZURE_SAS_TOKEN),
serviceEndpoint,
blockedContainer,
blockedPrefix);
DataLakeFileClient fileClientFail =
createDatalakeFileClient(
- credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN),
+ accessConfig.get(StorageAccessProperty.AZURE_SAS_TOKEN),
serviceEndpoint,
blockedContainer,
"polaris-test/scopedcreds/metadata",
@@ -338,7 +337,7 @@ public class AzureCredentialStorageIntegrationTest extends
BaseStorageIntegratio
}
}
- private Map<StorageAccessProperty, String> subscopedCredsForOperations(
+ private AccessConfig subscopedCredsForOperations(
List<String> allowedReadLoc, List<String> allowedWriteLoc, boolean
allowListAction) {
List<String> allowedLoc = new ArrayList<>();
allowedLoc.addAll(allowedReadLoc);
@@ -347,14 +346,12 @@ public class AzureCredentialStorageIntegrationTest
extends BaseStorageIntegratio
new AzureStorageConfigurationInfo(allowedLoc, tenantId);
AzureCredentialsStorageIntegration azureCredsIntegration =
new AzureCredentialsStorageIntegration();
- EnumMap<StorageAccessProperty, String> credsMap =
- azureCredsIntegration.getSubscopedCreds(
- newCallContext(),
- azureConfig,
- allowListAction,
- new HashSet<>(allowedReadLoc),
- new HashSet<>(allowedWriteLoc));
- return credsMap;
+ return azureCredsIntegration.getSubscopedCreds(
+ newCallContext(),
+ azureConfig,
+ allowListAction,
+ new HashSet<>(allowedReadLoc),
+ new HashSet<>(allowedWriteLoc));
}
private BlobContainerClient createContainerClient(
diff --git
a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java
b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java
index 861ad43ac..d3d3daac1 100644
---
a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java
+++
b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java
@@ -43,11 +43,10 @@ import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
-import java.util.EnumMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
+import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.BaseStorageIntegrationTest;
import org.apache.polaris.core.storage.StorageAccessProperty;
import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration;
@@ -136,9 +135,8 @@ class GcpCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
private Storage setupStorageClient(
List<String> allowedReadLoc, List<String> allowedWriteLoc, boolean
allowListAction)
throws IOException {
- Map<StorageAccessProperty, String> credsMap =
- subscopedCredsForOperations(allowedReadLoc, allowedWriteLoc,
allowListAction);
- return createStorageClient(credsMap);
+ return createStorageClient(
+ subscopedCredsForOperations(allowedReadLoc, allowedWriteLoc,
allowListAction));
}
BlobInfo createStorageBlob(String bucket, String prefix, String fileName) {
@@ -146,19 +144,20 @@ class GcpCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
return BlobInfo.newBuilder(blobId).build();
}
- private Storage createStorageClient(Map<StorageAccessProperty, String>
credsMap) {
+ private Storage createStorageClient(AccessConfig accessConfig) {
AccessToken accessToken =
new AccessToken(
- credsMap.get(StorageAccessProperty.GCS_ACCESS_TOKEN),
+ accessConfig.get(StorageAccessProperty.GCS_ACCESS_TOKEN),
new Date(
-
Long.parseLong(credsMap.get(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT))));
+ Long.parseLong(
+
accessConfig.get(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT))));
return StorageOptions.newBuilder()
.setCredentials(GoogleCredentials.create(accessToken))
.build()
.getService();
}
- private Map<StorageAccessProperty, String> subscopedCredsForOperations(
+ private AccessConfig subscopedCredsForOperations(
List<String> allowedReadLoc, List<String> allowedWriteLoc, boolean
allowListAction)
throws IOException {
List<String> allowedLoc = new ArrayList<>();
@@ -169,14 +168,12 @@ class GcpCredentialsStorageIntegrationTest extends
BaseStorageIntegrationTest {
new GcpCredentialsStorageIntegration(
GoogleCredentials.getApplicationDefault(),
ServiceOptions.getFromServiceLoader(HttpTransportFactory.class,
NetHttpTransport::new));
- EnumMap<StorageAccessProperty, String> credsMap =
- gcpCredsIntegration.getSubscopedCreds(
- newCallContext(),
- gcpConfig,
- allowListAction,
- new HashSet<>(allowedReadLoc),
- new HashSet<>(allowedWriteLoc));
- return credsMap;
+ return gcpCredsIntegration.getSubscopedCreds(
+ newCallContext(),
+ gcpConfig,
+ allowListAction,
+ new HashSet<>(allowedReadLoc),
+ new HashSet<>(allowedWriteLoc));
}
@Test
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/quarkus/catalog/AbstractIcebergCatalogTest.java
b/runtime/service/src/test/java/org/apache/polaris/service/quarkus/catalog/AbstractIcebergCatalogTest.java
index 4b60434d8..d1d3868ad 100644
---
a/runtime/service/src/test/java/org/apache/polaris/service/quarkus/catalog/AbstractIcebergCatalogTest.java
+++
b/runtime/service/src/test/java/org/apache/polaris/service/quarkus/catalog/AbstractIcebergCatalogTest.java
@@ -43,7 +43,6 @@ import java.io.UncheckedIOException;
import java.lang.reflect.Method;
import java.time.Clock;
import java.util.Arrays;
-import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -1833,7 +1832,7 @@ public abstract class AbstractIcebergCatalogTest extends
CatalogTests<IcebergCat
.getEntities();
Assertions.assertThat(tasks).hasSize(1);
TaskEntity taskEntity = TaskEntity.of(tasks.get(0));
- EnumMap<StorageAccessProperty, String> credentials =
+ Map<String, String> credentials =
metaStoreManager
.getSubscopedCredsForEntity(
polarisContext,
@@ -1843,13 +1842,14 @@ public abstract class AbstractIcebergCatalogTest
extends CatalogTests<IcebergCat
true,
Set.of(tableMetadata.location()),
Set.of(tableMetadata.location()))
- .getCredentials();
+ .getAccessConfig()
+ .credentials();
Assertions.assertThat(credentials)
.isNotNull()
.isNotEmpty()
- .containsEntry(StorageAccessProperty.AWS_KEY_ID, TEST_ACCESS_KEY)
- .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, SECRET_ACCESS_KEY)
- .containsEntry(StorageAccessProperty.AWS_TOKEN, SESSION_TOKEN);
+ .containsEntry(StorageAccessProperty.AWS_KEY_ID.getPropertyName(),
TEST_ACCESS_KEY)
+ .containsEntry(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(),
SECRET_ACCESS_KEY)
+ .containsEntry(StorageAccessProperty.AWS_TOKEN.getPropertyName(),
SESSION_TOKEN);
FileIO fileIO =
new TaskFileIOSupplier(
new DefaultFileIOFactory(storageCredentialCache,
metaStoreManagerFactory))
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java
index 9f530d4e7..d2c73e268 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java
@@ -100,6 +100,7 @@ public class DefaultFileIOFactory implements FileIOFactory {
if (accessConfig.isPresent()) {
properties.putAll(accessConfig.get().credentials());
properties.putAll(accessConfig.get().extraProperties());
+ properties.putAll(accessConfig.get().internalProperties());
}
return loadFileIOInternal(ioImplClassName, properties);
diff --git
a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java
b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java
index 758da2874..6bf3a08c8 100644
---
a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java
+++
b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java
@@ -26,17 +26,16 @@ import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
-import java.util.EnumMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.PolarisStorageActions;
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
import org.apache.polaris.core.storage.PolarisStorageIntegration;
import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider;
-import org.apache.polaris.core.storage.StorageAccessProperty;
import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration;
import org.apache.polaris.core.storage.aws.StsClientProvider;
import
org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration;
@@ -99,13 +98,13 @@ public class PolarisStorageIntegrationProviderImpl
implements PolarisStorageInte
storageIntegration =
new PolarisStorageIntegration<>("file") {
@Override
- public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
+ public AccessConfig getSubscopedCreds(
@Nonnull CallContext callContext,
@Nonnull T storageConfig,
boolean allowListOperation,
@Nonnull Set<String> allowedReadLocations,
@Nonnull Set<String> allowedWriteLocations) {
- return new EnumMap<>(StorageAccessProperty.class);
+ return AccessConfig.builder().build();
}
@Override
