This is an automated email from the ASF dual-hosted git repository. singhpk234 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push: new 8942f6813 Remove PolarisConfiguration.loadConfig (v2) (#1858) 8942f6813 is described below commit 8942f6813aa6921b4bb2eec6081031b8cf2a7ab1 Author: Eric Maynard <eric.maynard+...@snowflake.com> AuthorDate: Wed Jun 11 14:53:17 2025 -0700 Remove PolarisConfiguration.loadConfig (v2) (#1858) --- .../EclipseLinkPolarisMetaStoreManagerFactory.java | 8 ++++--- .../jdbc/JdbcMetaStoreManagerFactory.java | 8 +++++-- .../polaris/core/config/PolarisConfiguration.java | 20 ---------------- .../apache/polaris/core/entity/CatalogEntity.java | 20 ++++++++++------ .../AtomicOperationMetaStoreManager.java | 2 +- .../LocalPolarisMetaStoreManagerFactory.java | 13 +++++++--- .../persistence/cache/InMemoryEntityCache.java | 16 +++++++++---- .../TransactionalMetaStoreManagerImpl.java | 2 +- .../core/storage/PolarisStorageIntegration.java | 6 ++--- .../aws/AwsCredentialsStorageIntegration.java | 12 ++++++---- .../azure/AzureCredentialsStorageIntegration.java | 12 ++++++---- .../core/storage/cache/StorageCredentialCache.java | 22 +++++++++++------ .../gcp/GcpCredentialsStorageIntegration.java | 4 ++-- .../persistence/cache/InMemoryEntityCacheTest.java | 3 ++- .../core/storage/BaseStorageIntegrationTest.java | 23 ++++++++---------- .../storage/InMemoryStorageIntegrationTest.java | 3 +-- .../storage/cache/StorageCredentialCacheTest.java | 20 +++++++++------- .../aws/AwsCredentialsStorageIntegrationTest.java | 24 +++++++++---------- .../AzureCredentialStorageIntegrationTest.java | 6 ++--- .../gcp/GcpCredentialsStorageIntegrationTest.java | 6 ++--- .../polaris/core/persistence/BaseResolverTest.java | 4 +++- .../service/quarkus/config/QuarkusProducers.java | 5 ++-- .../quarkus/admin/PolarisAuthzTestBase.java | 2 +- .../catalog/IcebergCatalogHandlerAuthzTest.java | 2 +- .../quarkus/catalog/IcebergCatalogTest.java | 18 ++++++++++---- .../quarkus/catalog/IcebergCatalogViewTest.java | 5 ++-- .../catalog/PolarisCatalogWithEntityCacheTest.java | 2 +- .../catalog/PolarisGenericTableCatalogTest.java | 11 +++++---- .../service/quarkus/catalog/PolicyCatalogTest.java | 11 +++++---- .../service/quarkus/entity/CatalogEntityTest.java | 28 +++++++++++++--------- .../polaris/service/admin/PolarisAdminService.java | 4 ++-- ...moryAtomicOperationMetaStoreManagerFactory.java | 9 ++++--- .../InMemoryPolarisMetaStoreManagerFactory.java | 9 ++++--- .../PolarisStorageIntegrationProviderImpl.java | 4 ++-- .../org/apache/polaris/service/TestServices.java | 2 +- 35 files changed, 198 insertions(+), 148 deletions(-) diff --git a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index 492f61668..9f7af1a26 100644 --- a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -25,6 +25,7 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.nio.file.Path; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -46,12 +47,13 @@ public class EclipseLinkPolarisMetaStoreManagerFactory @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; protected EclipseLinkPolarisMetaStoreManagerFactory() { - this(null); + this(null, null); } @Inject - protected EclipseLinkPolarisMetaStoreManagerFactory(PolarisDiagnostics diagnostics) { - super(diagnostics); + protected EclipseLinkPolarisMetaStoreManagerFactory( + PolarisDiagnostics diagnostics, PolarisConfigurationStore configurationStore) { + super(diagnostics, configurationStore); } @Override diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index f94465414..4b369953d 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -32,6 +32,7 @@ import javax.sql.DataSource; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; @@ -74,6 +75,7 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject Instance<DataSource> dataSource; @Inject RelationalJdbcConfiguration relationalJdbcConfiguration; + @Inject PolarisConfigurationStore configurationStore; protected JdbcMetaStoreManagerFactory() {} @@ -205,7 +207,8 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { RealmContext realmContext) { if (!storageCredentialCacheMap.containsKey(realmContext.getRealmIdentifier())) { storageCredentialCacheMap.put( - realmContext.getRealmIdentifier(), new StorageCredentialCache()); + realmContext.getRealmIdentifier(), + new StorageCredentialCache(realmContext, configurationStore)); } return storageCredentialCacheMap.get(realmContext.getRealmIdentifier()); @@ -216,7 +219,8 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { if (!entityCacheMap.containsKey(realmContext.getRealmIdentifier())) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); entityCacheMap.put( - realmContext.getRealmIdentifier(), new InMemoryEntityCache(metaStoreManager)); + realmContext.getRealmIdentifier(), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); } return entityCacheMap.get(realmContext.getRealmIdentifier()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java index adb66daf5..31ae18795 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java @@ -24,7 +24,6 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -210,25 +209,6 @@ public abstract class PolarisConfiguration<T> { } } - /** - * Returns the value of a `PolarisConfiguration`, or the default if it cannot be loaded. This - * method does not need to be used when a `CallContext` is already available - */ - public static <T> T loadConfig(PolarisConfiguration<T> configuration) { - var callContext = CallContext.getCurrentContext(); - if (callContext == null) { - LOGGER.warn( - String.format( - "Unable to load current call context; using %s = %s", - configuration.key, configuration.defaultValue)); - return configuration.defaultValue; - } - return callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration(callContext.getRealmContext(), configuration); - } - public static <T> Builder<T> builder() { return new Builder<>(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index a99588330..f975ae739 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -43,6 +43,7 @@ import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.config.BehaviorChangeConfiguration; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.storage.FileStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; @@ -80,7 +81,7 @@ public class CatalogEntity extends PolarisEntity { return null; } - public static CatalogEntity fromCatalog(Catalog catalog) { + public static CatalogEntity fromCatalog(CallContext callContext, Catalog catalog) { Builder builder = new Builder() .setName(catalog.getName()) @@ -90,7 +91,7 @@ public class CatalogEntity extends PolarisEntity { internalProperties.put(CATALOG_TYPE_PROPERTY, catalog.getType().name()); builder.setInternalProperties(internalProperties); builder.setStorageConfigurationInfo( - catalog.getStorageConfigInfo(), getDefaultBaseLocation(catalog)); + callContext, catalog.getStorageConfigInfo(), getDefaultBaseLocation(catalog)); return builder.build(); } @@ -247,7 +248,7 @@ public class CatalogEntity extends PolarisEntity { } public Builder setStorageConfigurationInfo( - StorageConfigInfo storageConfigModel, String defaultBaseLocation) { + CallContext callContext, StorageConfigInfo storageConfigModel, String defaultBaseLocation) { if (storageConfigModel != null) { PolarisStorageConfigurationInfo config; Set<String> allowedLocations = new HashSet<>(storageConfigModel.getAllowedLocations()); @@ -261,7 +262,7 @@ public class CatalogEntity extends PolarisEntity { throw new BadRequestException("Must specify default base location"); } allowedLocations.add(defaultBaseLocation); - validateMaxAllowedLocations(allowedLocations); + validateMaxAllowedLocations(callContext, allowedLocations); switch (storageConfigModel.getStorageType()) { case S3: AwsStorageConfigInfo awsConfigModel = (AwsStorageConfigInfo) storageConfigModel; @@ -305,10 +306,15 @@ public class CatalogEntity extends PolarisEntity { } /** Validate the number of allowed locations not exceeding the max value. */ - private void validateMaxAllowedLocations(Collection<String> allowedLocations) { + private void validateMaxAllowedLocations( + CallContext callContext, Collection<String> allowedLocations) { int maxAllowedLocations = - BehaviorChangeConfiguration.loadConfig( - BehaviorChangeConfiguration.STORAGE_CONFIGURATION_MAX_LOCATIONS); + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getRealmContext(), + BehaviorChangeConfiguration.STORAGE_CONFIGURATION_MAX_LOCATIONS); if (maxAllowedLocations != -1 && allowedLocations.size() > maxAllowedLocations) { throw new IllegalArgumentException( String.format( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java index 060a908e1..4f5a98ce3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java @@ -1619,7 +1619,7 @@ public class AtomicOperationMetaStoreManager extends BaseMetaStoreManager { try { EnumMap<StorageAccessProperty, String> creds = storageIntegration.getSubscopedCreds( - callCtx.getDiagServices(), + callCtx, storageConfigurationInfo, allowListOperation, allowedReadLocations, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 1cfa89d0f..747991636 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -26,6 +26,7 @@ import java.util.function.Supplier; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; @@ -63,10 +64,14 @@ public abstract class LocalPolarisMetaStoreManagerFactory<StoreType> LoggerFactory.getLogger(LocalPolarisMetaStoreManagerFactory.class); private final PolarisDiagnostics diagnostics; + private final PolarisConfigurationStore configurationStore; private boolean bootstrap; - protected LocalPolarisMetaStoreManagerFactory(@Nonnull PolarisDiagnostics diagnostics) { + protected LocalPolarisMetaStoreManagerFactory( + @Nonnull PolarisDiagnostics diagnostics, + @Nonnull PolarisConfigurationStore configurationStore) { this.diagnostics = diagnostics; + this.configurationStore = configurationStore; } protected abstract StoreType createBackingStore(@Nonnull PolarisDiagnostics diagnostics); @@ -177,7 +182,8 @@ public abstract class LocalPolarisMetaStoreManagerFactory<StoreType> RealmContext realmContext) { if (!storageCredentialCacheMap.containsKey(realmContext.getRealmIdentifier())) { storageCredentialCacheMap.put( - realmContext.getRealmIdentifier(), new StorageCredentialCache()); + realmContext.getRealmIdentifier(), + new StorageCredentialCache(realmContext, configurationStore)); } return storageCredentialCacheMap.get(realmContext.getRealmIdentifier()); @@ -188,7 +194,8 @@ public abstract class LocalPolarisMetaStoreManagerFactory<StoreType> if (!entityCacheMap.containsKey(realmContext.getRealmIdentifier())) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); entityCacheMap.put( - realmContext.getRealmIdentifier(), new InMemoryEntityCache(metaStoreManager)); + realmContext.getRealmIdentifier(), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); } return entityCacheMap.get(realmContext.getRealmIdentifier()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java index 4599f4aed..3e1fc3ec6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java @@ -30,7 +30,8 @@ import java.util.concurrent.TimeUnit; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.BehaviorChangeConfiguration; import org.apache.polaris.core.config.FeatureConfiguration; -import org.apache.polaris.core.config.PolarisConfiguration; +import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; @@ -58,7 +59,10 @@ public class InMemoryEntityCache implements EntityCache { * * @param polarisMetaStoreManager the meta store manager implementation */ - public InMemoryEntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreManager) { + public InMemoryEntityCache( + @Nonnull RealmContext realmContext, + @Nonnull PolarisConfigurationStore configurationStore, + @Nonnull PolarisMetaStoreManager polarisMetaStoreManager) { // by name cache this.byName = new ConcurrentHashMap<>(); @@ -76,7 +80,8 @@ public class InMemoryEntityCache implements EntityCache { }; long weigherTarget = - PolarisConfiguration.loadConfig(FeatureConfiguration.ENTITY_CACHE_WEIGHER_TARGET); + configurationStore.getConfiguration( + realmContext, FeatureConfiguration.ENTITY_CACHE_WEIGHER_TARGET); Caffeine<Long, ResolvedPolarisEntity> byIdBuilder = Caffeine.newBuilder() .maximumWeight(weigherTarget) @@ -84,7 +89,10 @@ public class InMemoryEntityCache implements EntityCache { .expireAfterAccess(1, TimeUnit.HOURS) // Expire entries after 1 hour of no access .removalListener(removalListener); // Set the removal listener - if (PolarisConfiguration.loadConfig(BehaviorChangeConfiguration.ENTITY_CACHE_SOFT_VALUES)) { + boolean useSoftValues = + configurationStore.getConfiguration( + realmContext, BehaviorChangeConfiguration.ENTITY_CACHE_SOFT_VALUES); + if (useSoftValues) { byIdBuilder.softValues(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index 0d6111d54..71686165d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -2053,7 +2053,7 @@ public class TransactionalMetaStoreManagerImpl extends BaseMetaStoreManager { try { EnumMap<StorageAccessProperty, String> creds = storageIntegration.getSubscopedCreds( - callCtx.getDiagServices(), + callCtx, storageConfigurationInfo, allowListOperation, allowedReadLocations, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java index e549dd1f8..147ec5c66 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java @@ -22,7 +22,7 @@ import jakarta.annotation.Nonnull; import java.util.EnumMap; import java.util.Map; import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; /** * Abstract of Polaris Storage Integration. It holds the reference to an object that having the @@ -45,7 +45,7 @@ public abstract class PolarisStorageIntegration<T extends PolarisStorageConfigur /** * Subscope the creds against the allowed read and write locations. * - * @param diagnostics the diagnostics service + * @param callContext the call context * @param storageConfig storage configuration * @param allowListOperation whether to allow LIST on all the provided allowed read/write * locations @@ -54,7 +54,7 @@ public abstract class PolarisStorageIntegration<T extends PolarisStorageConfigur * @return An enum map including the scoped credentials */ public abstract EnumMap<StorageAccessProperty, String> getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull T storageConfig, boolean allowListOperation, @Nonnull Set<String> allowedReadLocations, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index bc8f5e33c..04c1970dd 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -19,7 +19,6 @@ package org.apache.polaris.core.storage.aws; import static org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS; -import static org.apache.polaris.core.config.PolarisConfiguration.loadConfig; import jakarta.annotation.Nonnull; import java.net.URI; @@ -29,7 +28,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.StorageUtil; @@ -63,11 +62,16 @@ public class AwsCredentialsStorageIntegration /** {@inheritDoc} */ @Override public EnumMap<StorageAccessProperty, String> getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull AwsStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set<String> allowedReadLocations, @Nonnull Set<String> allowedWriteLocations) { + int storageCredentialDurationSeconds = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration(callContext.getRealmContext(), STORAGE_CREDENTIAL_DURATION_SECONDS); AssumeRoleRequest.Builder request = AssumeRoleRequest.builder() .externalId(storageConfig.getExternalId()) @@ -80,7 +84,7 @@ public class AwsCredentialsStorageIntegration allowedReadLocations, allowedWriteLocations) .toJson()) - .durationSeconds(loadConfig(STORAGE_CREDENTIAL_DURATION_SECONDS)); + .durationSeconds(storageCredentialDurationSeconds); credentialsProvider.ifPresent( cp -> request.overrideConfiguration(b -> b.credentialsProvider(cp))); AssumeRoleResponse response = stsClient.assumeRole(request.build()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java index bcbc91a5c..39bd363d4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.core.storage.azure; +import static org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS; + import com.azure.core.credential.AccessToken; import com.azure.core.credential.TokenRequestContext; import com.azure.identity.DefaultAzureCredential; @@ -45,8 +47,7 @@ import java.util.EnumMap; import java.util.HashSet; import java.util.Objects; import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.StorageAccessProperty; import org.slf4j.Logger; @@ -71,7 +72,7 @@ public class AzureCredentialsStorageIntegration @Override public EnumMap<StorageAccessProperty, String> getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull AzureStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set<String> allowedReadLocations, @@ -126,7 +127,10 @@ public class AzureCredentialsStorageIntegration // clock skew between the client and server, OffsetDateTime startTime = start.truncatedTo(ChronoUnit.SECONDS).atOffset(ZoneOffset.UTC); int intendedDurationSeconds = - FeatureConfiguration.loadConfig(FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration(callContext.getRealmContext(), STORAGE_CREDENTIAL_DURATION_SECONDS); OffsetDateTime intendedEndTime = start.plusSeconds(intendedDurationSeconds).atOffset(ZoneOffset.UTC); OffsetDateTime maxAllowedEndTime = diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java index ef9297360..36f666db0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java @@ -32,7 +32,8 @@ import java.util.function.Function; import org.apache.iceberg.exceptions.UnprocessableEntityException; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.FeatureConfiguration; -import org.apache.polaris.core.config.PolarisConfiguration; +import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult; @@ -47,10 +48,16 @@ public class StorageCredentialCache { private static final Logger LOGGER = LoggerFactory.getLogger(StorageCredentialCache.class); private static final long CACHE_MAX_NUMBER_OF_ENTRIES = 10_000L; + private final LoadingCache<StorageCredentialCacheKey, StorageCredentialCacheEntry> cache; + private final RealmContext realmContext; + private final PolarisConfigurationStore configurationStore; /** Initialize the creds cache */ - public StorageCredentialCache() { + public StorageCredentialCache( + RealmContext realmContext, PolarisConfigurationStore configurationStore) { + this.realmContext = realmContext; + this.configurationStore = configurationStore; cache = Caffeine.newBuilder() .maximumSize(CACHE_MAX_NUMBER_OF_ENTRIES) @@ -62,7 +69,7 @@ public class StorageCredentialCache { 0, Math.min( (entry.getExpirationTime() - System.currentTimeMillis()) / 2, - maxCacheDurationMs())); + this.maxCacheDurationMs())); return Duration.ofMillis(expireAfterMillis); })) .build( @@ -73,12 +80,13 @@ public class StorageCredentialCache { } /** How long credentials should remain in the cache. */ - private static long maxCacheDurationMs() { + private long maxCacheDurationMs() { var cacheDurationSeconds = - PolarisConfiguration.loadConfig( - FeatureConfiguration.STORAGE_CREDENTIAL_CACHE_DURATION_SECONDS); + configurationStore.getConfiguration( + realmContext, FeatureConfiguration.STORAGE_CREDENTIAL_CACHE_DURATION_SECONDS); var credentialDurationSeconds = - PolarisConfiguration.loadConfig(FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); + configurationStore.getConfiguration( + realmContext, FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); if (cacheDurationSeconds >= credentialDurationSeconds) { throw new IllegalArgumentException( String.format( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java index 3a7c85780..2fc1438fe 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java @@ -38,7 +38,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Stream; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.StorageAccessProperty; @@ -70,7 +70,7 @@ public class GcpCredentialsStorageIntegration @Override public EnumMap<StorageAccessProperty, String> getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull GcpStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set<String> allowedReadLocations, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java index 1d0564be9..945f1ccb6 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java @@ -103,7 +103,8 @@ public class InMemoryEntityCacheTest { * @return new cache for the entity store */ InMemoryEntityCache allocateNewCache() { - return new InMemoryEntityCache(this.metaStoreManager); + return new InMemoryEntityCache( + callCtx.getRealmContext(), callCtx.getConfigurationStore(), this.metaStoreManager); } @Test diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java similarity index 56% copy from quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java copy to polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java index 079b02bef..e008abf74 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java @@ -16,21 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.quarkus.catalog; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; -import jakarta.annotation.Nullable; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.cache.InMemoryEntityCache; +package org.apache.polaris.core.storage; -@QuarkusTest -@TestProfile(IcebergCatalogTest.Profile.class) -public class PolarisCatalogWithEntityCacheTest extends IcebergCatalogTest { +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.persistence.BasePersistence; +import org.mockito.Mockito; - @Nullable - @Override - protected InMemoryEntityCache createEntityCache(PolarisMetaStoreManager metaStoreManager) { - return new InMemoryEntityCache(metaStoreManager); +public abstract class BaseStorageIntegrationTest { + protected CallContext newCallContext() { + return new PolarisCallContext( + () -> "realm", Mockito.mock(BasePersistence.class), Mockito.mock(PolarisDiagnostics.class)); } } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java index a4e58860d..e679d3e32 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Set; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; @@ -209,7 +208,7 @@ class InMemoryStorageIntegrationTest { @Override public EnumMap<StorageAccessProperty, String> getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull PolarisStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set<String> allowedReadLocations, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index b1d1789da..3d889b765 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -72,12 +72,16 @@ public class StorageCredentialCacheTest { new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS); callCtx = new PolarisCallContext(() -> "testRealm", metaStore, diagServices); metaStoreManager = Mockito.mock(PolarisMetaStoreManager.class); - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); + } + + private StorageCredentialCache newStorageCredentialCache() { + return new StorageCredentialCache(callCtx.getRealmContext(), callCtx.getConfigurationStore()); } @Test public void testBadResult() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); ScopedCredentialsResult badResult = new ScopedCredentialsResult( BaseResult.ReturnStatus.SUBSCOPE_CREDS_ERROR, "extra_error_info"); @@ -110,7 +114,7 @@ public class StorageCredentialCacheTest { @Test public void testCacheHit() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List<ScopedCredentialsResult> mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ false); Mockito.when( @@ -153,7 +157,7 @@ public class StorageCredentialCacheTest { @RepeatedTest(10) public void testCacheEvict() throws InterruptedException { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List<ScopedCredentialsResult> mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ true); Mockito.when( @@ -211,7 +215,7 @@ public class StorageCredentialCacheTest { @Test public void testCacheGenerateNewEntries() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List<ScopedCredentialsResult> mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ false); Mockito.when( @@ -298,7 +302,7 @@ public class StorageCredentialCacheTest { @Test public void testCacheNotAffectedBy() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List<ScopedCredentialsResult> mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ false); @@ -443,7 +447,7 @@ public class StorageCredentialCacheTest { @Test public void testAzureCredentialFormatting() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List<ScopedCredentialsResult> mockedScopedCreds = List.of( new ScopedCredentialsResult( @@ -532,7 +536,7 @@ public class StorageCredentialCacheTest { @Test public void testExtraProperties() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); ScopedCredentialsResult properties = new ScopedCredentialsResult( new EnumMap<>( diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java index b837033e1..93f96b358 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java @@ -25,7 +25,7 @@ import java.time.Instant; import java.util.EnumMap; import java.util.List; import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.storage.BaseStorageIntegrationTest; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; @@ -48,7 +48,7 @@ import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; import software.amazon.awssdk.services.sts.model.Credentials; -class AwsCredentialsStorageIntegrationTest { +class AwsCredentialsStorageIntegrationTest extends BaseStorageIntegrationTest { public static final Instant EXPIRE_TIME = Instant.now().plusMillis(3600_000); @@ -83,7 +83,7 @@ class AwsCredentialsStorageIntegrationTest { EnumMap<StorageAccessProperty, String> credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(warehouseDir), @@ -231,7 +231,7 @@ class AwsCredentialsStorageIntegrationTest { () -> new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( storageType, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -248,7 +248,7 @@ class AwsCredentialsStorageIntegrationTest { EnumMap<StorageAccessProperty, String> credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( storageType, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -349,7 +349,7 @@ class AwsCredentialsStorageIntegrationTest { EnumMap<StorageAccessProperty, String> credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -444,7 +444,7 @@ class AwsCredentialsStorageIntegrationTest { EnumMap<StorageAccessProperty, String> credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( storageType, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -509,7 +509,7 @@ class AwsCredentialsStorageIntegrationTest { EnumMap<StorageAccessProperty, String> credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -549,7 +549,7 @@ class AwsCredentialsStorageIntegrationTest { () -> new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -566,7 +566,7 @@ class AwsCredentialsStorageIntegrationTest { EnumMap<StorageAccessProperty, String> credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -604,7 +604,7 @@ class AwsCredentialsStorageIntegrationTest { EnumMap<StorageAccessProperty, String> credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -622,7 +622,7 @@ class AwsCredentialsStorageIntegrationTest { () -> new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java index f4738d73e..9f43d42bd 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java @@ -47,7 +47,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Stream; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.storage.BaseStorageIntegrationTest; import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration; import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; @@ -61,7 +61,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; -public class AzureCredentialStorageIntegrationTest { +public class AzureCredentialStorageIntegrationTest extends BaseStorageIntegrationTest { private final String clientId = System.getenv("AZURE_CLIENT_ID"); private final String clientSecret = System.getenv("AZURE_CLIENT_SECRET"); @@ -349,7 +349,7 @@ public class AzureCredentialStorageIntegrationTest { new AzureCredentialsStorageIntegration(); EnumMap<StorageAccessProperty, String> credsMap = azureCredsIntegration.getSubscopedCreds( - new PolarisDefaultDiagServiceImpl(), + newCallContext(), azureConfig, allowListAction, new HashSet<>(allowedReadLoc), diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java index fc88bd817..861ad43ac 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java @@ -48,7 +48,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.storage.BaseStorageIntegrationTest; import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo; @@ -59,7 +59,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -class GcpCredentialsStorageIntegrationTest { +class GcpCredentialsStorageIntegrationTest extends BaseStorageIntegrationTest { private final String gcsServiceKeyJsonFileLocation = System.getenv("GOOGLE_APPLICATION_CREDENTIALS"); @@ -171,7 +171,7 @@ class GcpCredentialsStorageIntegrationTest { ServiceOptions.getFromServiceLoader(HttpTransportFactory.class, NetHttpTransport::new)); EnumMap<StorageAccessProperty, String> credsMap = gcpCredsIntegration.getSubscopedCreds( - new PolarisDefaultDiagServiceImpl(), + newCallContext(), gcpConfig, allowListAction, new HashSet<>(allowedReadLoc), diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java index 728187a38..1a7a82731 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java @@ -468,7 +468,9 @@ public abstract class BaseResolverTest { // create a new cache if needs be if (cache == null) { - this.cache = new InMemoryEntityCache(metaStoreManager()); + this.cache = + new InMemoryEntityCache( + callCtx().getRealmContext(), callCtx().getConfigurationStore(), metaStoreManager()); } boolean allRoles = principalRolesScope == null; Optional<List<PrincipalRoleEntity>> roleEntities = diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 593853c40..e1b197005 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -95,8 +95,9 @@ public class QuarkusProducers { @Produces @ApplicationScoped - public StorageCredentialCache storageCredentialCache() { - return new StorageCredentialCache(); + public StorageCredentialCache storageCredentialCache( + RealmContext realmContext, PolarisConfigurationStore configurationStore) { + return new StorageCredentialCache(realmContext, configurationStore); } @Produces diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 4e8748de5..1c232bf54 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -284,7 +284,7 @@ public abstract class PolarisAuthzTestBase { .setName(CATALOG_NAME) .setCatalogType("INTERNAL") .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo(callContext, storageConfigModel, storageLocation) .build() .asCatalog())); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 5c488809d..4d89d5523 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -1726,7 +1726,7 @@ public class IcebergCatalogHandlerAuthzTest extends PolarisAuthzTestBase { new CatalogEntity.Builder() .setName(externalCatalog) .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo(callContext, storageConfigModel, storageLocation) .setCatalogType("EXTERNAL") .build() .asCatalog())); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 2b4c4205c..614fab725 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -225,6 +225,10 @@ public abstract class IcebergCatalogTest extends CatalogTests<IcebergCatalog> { private TestPolarisEventListener testPolarisEventListener; private ReservedProperties reservedProperties; + protected String getRealmName() { + return realmName; + } + @BeforeAll public static void setUpMocks() { PolarisStorageIntegrationProviderImpl mock = @@ -257,7 +261,9 @@ public abstract class IcebergCatalogTest extends CatalogTests<IcebergCatalog> { entityManager = new PolarisEntityManager( - metaStoreManager, new StorageCredentialCache(), createEntityCache(metaStoreManager)); + metaStoreManager, + new StorageCredentialCache(realmContext, configurationStore), + createEntityCache(metaStoreManager)); PrincipalEntity rootEntity = new PrincipalEntity( @@ -313,7 +319,8 @@ public abstract class IcebergCatalogTest extends CatalogTests<IcebergCatalog> { "true") .addProperty( FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + polarisContext, storageConfigModel, storageLocation) .build() .asCatalog())); @@ -423,12 +430,12 @@ public abstract class IcebergCatalogTest extends CatalogTests<IcebergCatalog> { @Override public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); + return new StorageCredentialCache(realmContext, configurationStore); } @Override public InMemoryEntityCache getOrCreateEntityCache(RealmContext realmContext) { - return new InMemoryEntityCache(metaStoreManager); + return new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager); } @Override @@ -1590,7 +1597,8 @@ public abstract class IcebergCatalogTest extends CatalogTests<IcebergCatalog> { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") .addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") - .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) + .setStorageConfigurationInfo( + polarisContext, noPurgeStorageConfigModel, storageLocation) .build() .asCatalog())); PolarisPassthroughResolutionView passthroughView = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 0f06a0225..6ea276cc0 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -179,8 +179,8 @@ public class IcebergCatalogViewTest extends ViewCatalogTests<IcebergCatalog> { PolarisEntityManager entityManager = new PolarisEntityManager( metaStoreManager, - new StorageCredentialCache(), - new InMemoryEntityCache(metaStoreManager)); + new StorageCredentialCache(realmContext, configurationStore), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); CallContext.setCurrentContext(polarisContext); @@ -224,6 +224,7 @@ public class IcebergCatalogViewTest extends ViewCatalogTests<IcebergCatalog> { .addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "true") .setDefaultBaseLocation("file://tmp") .setStorageConfigurationInfo( + polarisContext, new FileStorageConfigInfo( StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://", "/", "*")), "file://tmp") diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java index 079b02bef..3c59dff18 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java @@ -31,6 +31,6 @@ public class PolarisCatalogWithEntityCacheTest extends IcebergCatalogTest { @Nullable @Override protected InMemoryEntityCache createEntityCache(PolarisMetaStoreManager metaStoreManager) { - return new InMemoryEntityCache(metaStoreManager); + return new InMemoryEntityCache(() -> getRealmName(), configurationStore, metaStoreManager); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java index 2aba1773b..c9196f175 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java @@ -177,8 +177,8 @@ public class PolarisGenericTableCatalogTest { entityManager = new PolarisEntityManager( metaStoreManager, - new StorageCredentialCache(), - new InMemoryEntityCache(metaStoreManager)); + new StorageCredentialCache(realmContext, configurationStore), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); PrincipalEntity rootEntity = new PrincipalEntity( @@ -233,7 +233,8 @@ public class PolarisGenericTableCatalogTest { "true") .addProperty( FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + polarisContext, storageConfigModel, storageLocation) .build() .asCatalog())); @@ -302,12 +303,12 @@ public class PolarisGenericTableCatalogTest { @Override public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); + return new StorageCredentialCache(realmContext, configurationStore); } @Override public InMemoryEntityCache getOrCreateEntityCache(RealmContext realmContext) { - return new InMemoryEntityCache(metaStoreManager); + return new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager); } @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java index 93ee06072..c0d4b8b46 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java @@ -203,8 +203,8 @@ public class PolicyCatalogTest { entityManager = new PolarisEntityManager( metaStoreManager, - new StorageCredentialCache(), - new InMemoryEntityCache(metaStoreManager)); + new StorageCredentialCache(realmContext, configurationStore), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); callContext = polarisContext; @@ -259,7 +259,8 @@ public class PolicyCatalogTest { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + polarisContext, storageConfigModel, storageLocation) .build() .asCatalog())); @@ -326,12 +327,12 @@ public class PolicyCatalogTest { @Override public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); + return new StorageCredentialCache(realmContext, configurationStore); } @Override public InMemoryEntityCache getOrCreateEntityCache(RealmContext realmContext) { - return new InMemoryEntityCache(metaStoreManager); + return new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager); } @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java index 0a6dea8fe..ed44510a9 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java @@ -34,15 +34,17 @@ import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; public class CatalogEntityTest { - @BeforeAll - public static void setup() { + private CallContext callContext; + + @BeforeEach + public void setup() { MetaStoreManagerFactory metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); RealmContext realmContext = () -> "realm"; PolarisCallContext polarisCallContext = @@ -50,6 +52,7 @@ public class CatalogEntityTest { realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(() -> "realm").get(), new PolarisDefaultDiagServiceImpl()); + this.callContext = polarisCallContext; CallContext.setCurrentContext(polarisCallContext); } @@ -72,7 +75,7 @@ public class CatalogEntityTest { .setProperties(prop) .setStorageConfigInfo(awsStorageConfigModel) .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(callContext, awsCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( "Location prefix not allowed: 'unsupportPrefix://mybucket/path', expected prefixes"); @@ -93,7 +96,7 @@ public class CatalogEntityTest { new CatalogProperties("abfs://contai...@storageaccount.blob.windows.net/path")) .setStorageConfigInfo(azureStorageConfigModel) .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(azureCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(callContext, azureCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Invalid azure location uri unsupportPrefix://mybucket/path"); @@ -110,7 +113,7 @@ public class CatalogEntityTest { .setProperties(new CatalogProperties("gs://externally-owned-bucket")) .setStorageConfigInfo(gcpStorageConfigModel) .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(gcpCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(callContext, gcpCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( "Location prefix not allowed: 'unsupportPrefix://mybucket/path', expected prefixes"); @@ -142,7 +145,7 @@ public class CatalogEntityTest { .setProperties(prop) .setStorageConfigInfo(awsStorageConfigModel) .build(); - Assertions.assertThatCode(() -> CatalogEntity.fromCatalog(awsCatalog)) + Assertions.assertThatCode(() -> CatalogEntity.fromCatalog(callContext, awsCatalog)) .doesNotThrowAnyException(); } @@ -166,7 +169,8 @@ public class CatalogEntityTest { .setProperties(prop) .setStorageConfigInfo(awsStorageConfigModel) .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)); + Assertions.assertThatNoException() + .isThrownBy(() -> CatalogEntity.fromCatalog(callContext, awsCatalog)); basedLocation = "abfs://contai...@storageaccount.blob.windows.net/path"; prop.put(CatalogEntity.DEFAULT_BASE_LOCATION_KEY, basedLocation); @@ -183,7 +187,8 @@ public class CatalogEntityTest { .setProperties(new CatalogProperties(basedLocation)) .setStorageConfigInfo(azureStorageConfigModel) .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(azureCatalog)); + Assertions.assertThatNoException() + .isThrownBy(() -> CatalogEntity.fromCatalog(callContext, azureCatalog)); basedLocation = "gs://externally-owned-bucket"; prop.put(CatalogEntity.DEFAULT_BASE_LOCATION_KEY, basedLocation); @@ -199,7 +204,8 @@ public class CatalogEntityTest { .setProperties(new CatalogProperties(basedLocation)) .setStorageConfigInfo(gcpStorageConfigModel) .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(gcpCatalog)); + Assertions.assertThatNoException() + .isThrownBy(() -> CatalogEntity.fromCatalog(callContext, gcpCatalog)); } @ParameterizedTest @@ -234,7 +240,7 @@ public class CatalogEntityTest { expectedMessage = "Invalid role ARN format"; } ; - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(callContext, awsCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessage(expectedMessage); } diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 4495d96fd..1a1914ad1 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -716,7 +716,7 @@ public class PolarisAdminService { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_CATALOG; authorizeBasicRootOperationOrThrow(op); - CatalogEntity entity = CatalogEntity.fromCatalog(catalogRequest.getCatalog()); + CatalogEntity entity = CatalogEntity.fromCatalog(callContext, catalogRequest.getCatalog()); checkArgument(entity.getId() == -1, "Entity to be created must have no ID assigned"); @@ -898,7 +898,7 @@ public class PolarisAdminService { } if (updateRequest.getStorageConfigInfo() != null) { updateBuilder.setStorageConfigurationInfo( - updateRequest.getStorageConfigInfo(), defaultBaseLocation); + callContext, updateRequest.getStorageConfigInfo(), defaultBaseLocation); } CatalogEntity updatedEntity = updateBuilder.build(); diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java index 3f0b1a355..83908f017 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java @@ -22,6 +22,7 @@ import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -36,13 +37,15 @@ public class InMemoryAtomicOperationMetaStoreManagerFactory extends InMemoryPolarisMetaStoreManagerFactory { public InMemoryAtomicOperationMetaStoreManagerFactory() { - super(null, null); + super(null, null, null); } @Inject public InMemoryAtomicOperationMetaStoreManagerFactory( - PolarisStorageIntegrationProvider storageIntegration, PolarisDiagnostics diagnostics) { - super(storageIntegration, diagnostics); + PolarisStorageIntegrationProvider storageIntegration, + PolarisDiagnostics diagnostics, + PolarisConfigurationStore configurationStore) { + super(storageIntegration, diagnostics, configurationStore); } @Override diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 895e5b51b..367a57de6 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Supplier; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -48,13 +49,15 @@ public class InMemoryPolarisMetaStoreManagerFactory private final Set<String> bootstrappedRealms = new HashSet<>(); public InMemoryPolarisMetaStoreManagerFactory() { - this(null, null); + this(null, null, null); } @Inject public InMemoryPolarisMetaStoreManagerFactory( - PolarisStorageIntegrationProvider storageIntegration, PolarisDiagnostics diagnostics) { - super(diagnostics); + PolarisStorageIntegrationProvider storageIntegration, + PolarisDiagnostics diagnostics, + PolarisConfigurationStore configurationStore) { + super(diagnostics, configurationStore); this.storageIntegration = storageIntegration; } diff --git a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index 2d84e3747..253ab5d03 100644 --- a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -31,7 +31,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -99,7 +99,7 @@ public class PolarisStorageIntegrationProviderImpl implements PolarisStorageInte new PolarisStorageIntegration<>("file") { @Override public EnumMap<StorageAccessProperty, String> getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull T storageConfig, boolean allowListOperation, @Nonnull Set<String> allowedReadLocations, diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index d3622126f..8f63ecfb8 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -154,7 +154,7 @@ public record TestServices( () -> GoogleCredentials.create(new AccessToken(GCP_ACCESS_TOKEN, new Date()))); InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory( - storageIntegrationProvider, polarisDiagnostics); + storageIntegrationProvider, polarisDiagnostics, configurationStore); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(metaStoreManagerFactory) {}; UserSecretsManagerFactory userSecretsManagerFactory =