This is an automated email from the ASF dual-hosted git repository.
collado pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 91c69cd5 Fix GCP and Azure storage configuration to store missing
properties (#370)
91c69cd5 is described below
commit 91c69cd5c5e0265be4285993ba8b893f836f3656
Author: Michael Collado <[email protected]>
AuthorDate: Wed Oct 16 09:33:05 2024 -0700
Fix GCP and Azure storage configuration to store missing properties (#370)
* Fix GCP and Azure storage configuration to store missing properties
* Move aws config to its own variable
* Change gcp token creation to use Optional
---
.../apache/polaris/core/entity/CatalogEntity.java | 16 +++--
.../apache/polaris/service/PolarisApplication.java | 3 +-
.../service/config/PolarisApplicationConfig.java | 64 +++++++++++++++++++
.../PolarisStorageIntegrationProviderImpl.java | 23 +++----
.../service/admin/PolarisAuthzTestBase.java | 6 +-
.../admin/PolarisServiceImplIntegrationTest.java | 72 +++++++++++++++++++++-
.../catalog/BasePolarisCatalogViewTest.java | 6 +-
7 files changed, 169 insertions(+), 21 deletions(-)
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
index 924b2077..fea6b746 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
@@ -239,22 +239,30 @@ public class CatalogEntity extends PolarisEntity {
switch (storageConfigModel.getStorageType()) {
case S3:
AwsStorageConfigInfo awsConfigModel = (AwsStorageConfigInfo)
storageConfigModel;
- config =
+ AwsStorageConfigurationInfo awsConfig =
new AwsStorageConfigurationInfo(
PolarisStorageConfigurationInfo.StorageType.S3,
new ArrayList<>(allowedLocations),
awsConfigModel.getRoleArn(),
awsConfigModel.getExternalId());
- ((AwsStorageConfigurationInfo)
config).validateArn(awsConfigModel.getRoleArn());
+ awsConfig.validateArn(awsConfigModel.getRoleArn());
+ config = awsConfig;
break;
case AZURE:
AzureStorageConfigInfo azureConfigModel = (AzureStorageConfigInfo)
storageConfigModel;
- config =
+ AzureStorageConfigurationInfo azureconfigInfo =
new AzureStorageConfigurationInfo(
new ArrayList<>(allowedLocations),
azureConfigModel.getTenantId());
+
azureconfigInfo.setMultiTenantAppName(azureConfigModel.getMultiTenantAppName());
+ azureconfigInfo.setConsentUrl(azureConfigModel.getConsentUrl());
+ config = azureconfigInfo;
break;
case GCS:
- config = new GcpStorageConfigurationInfo(new
ArrayList<>(allowedLocations));
+ GcpStorageConfigurationInfo gcpConfig =
+ new GcpStorageConfigurationInfo(new
ArrayList<>(allowedLocations));
+ gcpConfig.setGcpServiceAccount(
+ ((GcpStorageConfigInfo)
storageConfigModel).getGcsServiceAccount());
+ config = gcpConfig;
break;
case FILE:
config = new FileStorageConfigurationInfo(new
ArrayList<>(allowedLocations));
diff --git
a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
index dd0597b8..143d3dae 100644
---
a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
+++
b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
@@ -165,7 +165,8 @@ public class PolarisApplication extends
Application<PolarisApplicationConfig> {
stsClientBuilder.credentialsProvider(awsCredentialsProvider);
}
return stsClientBuilder.build();
- }));
+ },
+ configuration.getGcpCredentialsProvider()));
PolarisMetricRegistry polarisMetricRegistry =
new PolarisMetricRegistry(new
PrometheusMeterRegistry(PrometheusConfig.DEFAULT));
diff --git
a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java
b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java
index e623f253..065b6d8f 100644
---
a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java
+++
b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java
@@ -19,11 +19,17 @@
package org.apache.polaris.service.config;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
import com.google.common.base.Preconditions;
import io.dropwizard.core.Configuration;
+import java.io.IOException;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.apache.polaris.core.PolarisConfigurationStore;
@@ -59,6 +65,8 @@ public class PolarisApplicationConfig extends Configuration {
private FileIOFactory fileIOFactory;
private RateLimiter rateLimiter;
+ private AccessToken gcpAccessToken;
+
public static final long REQUEST_BODY_BYTES_NO_LIMIT = -1;
private long maxRequestBodyBytes = REQUEST_BODY_BYTES_NO_LIMIT;
@@ -208,4 +216,60 @@ public class PolarisApplicationConfig extends
Configuration {
public void setDefaultRealms(List<String> defaultRealms) {
this.defaultRealms = defaultRealms;
}
+
+ public Supplier<GoogleCredentials> getGcpCredentialsProvider() {
+ return () ->
+ Optional.ofNullable(gcpAccessToken)
+ .map(GoogleCredentials::create)
+ .orElseGet(
+ () -> {
+ try {
+ return GoogleCredentials.getApplicationDefault();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to get GCP
credentials", e);
+ }
+ });
+ }
+
+ @JsonProperty("gcp_credentials")
+ void setGcpCredentials(GcpAccessToken token) {
+ this.gcpAccessToken =
+ new AccessToken(
+ token.getAccessToken(),
+ new Date(System.currentTimeMillis() + token.getExpiresIn() *
1000));
+ }
+
+ /**
+ * A static AccessToken representation used to store a static token and
expiration date. This
+ * should strictly be used for testing.
+ */
+ static class GcpAccessToken {
+ private String accessToken;
+ private long expiresIn;
+
+ public GcpAccessToken() {}
+
+ public GcpAccessToken(String accessToken, long expiresIn) {
+ this.accessToken = accessToken;
+ this.expiresIn = expiresIn;
+ }
+
+ public String getAccessToken() {
+ return accessToken;
+ }
+
+ @JsonProperty("access_token")
+ public void setAccessToken(String accessToken) {
+ this.accessToken = accessToken;
+ }
+
+ public long getExpiresIn() {
+ return expiresIn;
+ }
+
+ @JsonProperty("expires_in")
+ public void setExpiresIn(long expiresIn) {
+ this.expiresIn = expiresIn;
+ }
+ }
}
diff --git
a/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java
b/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java
index 3fb03413..579ef033 100644
---
a/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java
+++
b/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java
@@ -22,7 +22,6 @@ import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.auth.http.HttpTransportFactory;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.ServiceOptions;
-import java.io.IOException;
import java.util.EnumMap;
import java.util.Map;
import java.util.Set;
@@ -43,9 +42,12 @@ import software.amazon.awssdk.services.sts.StsClient;
public class PolarisStorageIntegrationProviderImpl implements
PolarisStorageIntegrationProvider {
private final Supplier<StsClient> stsClientSupplier;
+ private final Supplier<GoogleCredentials> gcpCredsProvider;
- public PolarisStorageIntegrationProviderImpl(Supplier<StsClient>
stsClientSupplier) {
+ public PolarisStorageIntegrationProviderImpl(
+ Supplier<StsClient> stsClientSupplier, Supplier<GoogleCredentials>
gcpCredsProvider) {
this.stsClientSupplier = stsClientSupplier;
+ this.gcpCredsProvider = gcpCredsProvider;
}
@Override
@@ -64,17 +66,12 @@ public class PolarisStorageIntegrationProviderImpl
implements PolarisStorageInte
new AwsCredentialsStorageIntegration(stsClientSupplier.get());
break;
case GCS:
- try {
- storageIntegration =
- (PolarisStorageIntegration<T>)
- new GcpCredentialsStorageIntegration(
- GoogleCredentials.getApplicationDefault(),
- ServiceOptions.getFromServiceLoader(
- HttpTransportFactory.class, NetHttpTransport::new));
- } catch (IOException e) {
- throw new RuntimeException(
- "Error initializing default google credentials. " +
e.getMessage());
- }
+ storageIntegration =
+ (PolarisStorageIntegration<T>)
+ new GcpCredentialsStorageIntegration(
+ gcpCredsProvider.get(),
+ ServiceOptions.getFromServiceLoader(
+ HttpTransportFactory.class, NetHttpTransport::new));
break;
case AZURE:
storageIntegration =
diff --git
a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java
b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java
index 702acf41..88b19303 100644
---
a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java
+++
b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java
@@ -20,9 +20,12 @@ package org.apache.polaris.service.admin;
import static org.apache.iceberg.types.Types.NestedField.required;
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.time.Clock;
+import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -148,7 +151,8 @@ public abstract class PolarisAuthzTestBase {
InMemoryPolarisMetaStoreManagerFactory managerFactory =
new InMemoryPolarisMetaStoreManagerFactory();
managerFactory.setStorageIntegrationProvider(
- new PolarisStorageIntegrationProviderImpl(Mockito::mock));
+ new PolarisStorageIntegrationProviderImpl(
+ Mockito::mock, () -> GoogleCredentials.create(new
AccessToken("abc", new Date()))));
RealmContext realmContext = () -> "realm";
PolarisMetaStoreManager metaStoreManager =
managerFactory.getOrCreateMetaStoreManager(realmContext);
diff --git
a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java
b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java
index dbe95622..2eb5a452 100644
---
a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java
+++
b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java
@@ -59,6 +59,7 @@ import
org.apache.polaris.core.admin.model.CreatePrincipalRequest;
import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest;
import org.apache.polaris.core.admin.model.ExternalCatalog;
import org.apache.polaris.core.admin.model.FileStorageConfigInfo;
+import org.apache.polaris.core.admin.model.GcpStorageConfigInfo;
import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest;
import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest;
import org.apache.polaris.core.admin.model.GrantResource;
@@ -107,7 +108,9 @@ public class PolarisServiceImplIntegrationTest {
// disallow FILE urls for the sake of tests below
ConfigOverride.config(
- "featureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES",
"S3,GCS,AZURE"));
+ "featureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES",
"S3,GCS,AZURE"),
+ ConfigOverride.config("gcp_credentials.access_token", "abc"),
+ ConfigOverride.config("gcp_credentials.expires_in", "12345"));
private static String userToken;
private static String realm;
@@ -379,6 +382,73 @@ public class PolarisServiceImplIntegrationTest {
}
}
+ @Test
+ public void testCreateCatalogWithAzureStorageConfig() {
+ AzureStorageConfigInfo azureConfigInfo =
+ AzureStorageConfigInfo.builder()
+ .setConsentUrl("https://consent.url";)
+ .setMultiTenantAppName("myappname")
+ .setTenantId("tenantId")
+ .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE)
+ .build();
+ Catalog catalog =
+ PolarisCatalog.builder()
+ .setType(Catalog.TypeEnum.INTERNAL)
+ .setName("my-catalog")
+ .setProperties(
+ new CatalogProperties(
+
"abfss://[email protected]/path/to/my/data/"))
+ .setStorageConfigInfo(azureConfigInfo)
+ .build();
+ try (Response response =
+ newRequest("http://localhost:%d/api/management/v1/catalogs";)
+ .post(Entity.json(new CreateCatalogRequest(catalog)))) {
+ assertThat(response).returns(Response.Status.CREATED.getStatusCode(),
Response::getStatus);
+ }
+ try (Response response =
+
newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog";).get()) {
+ assertThat(response).returns(Response.Status.OK.getStatusCode(),
Response::getStatus);
+ Catalog catResponse = response.readEntity(Catalog.class);
+ assertThat(catResponse.getStorageConfigInfo())
+ .isInstanceOf(AzureStorageConfigInfo.class)
+ .hasFieldOrPropertyWithValue("consentUrl", "https://consent.url";)
+ .hasFieldOrPropertyWithValue("multiTenantAppName", "myappname")
+ .hasFieldOrPropertyWithValue(
+ "allowedLocations",
+
List.of("abfss://[email protected]/path/to/my/data/"));
+ }
+ }
+
+ @Test
+ public void testCreateCatalogWithGcpStorageConfig() {
+ GcpStorageConfigInfo gcpConfigModel =
+ GcpStorageConfigInfo.builder()
+ .setGcsServiceAccount("my-sa")
+ .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS)
+ .build();
+ Catalog catalog =
+ PolarisCatalog.builder()
+ .setType(Catalog.TypeEnum.INTERNAL)
+ .setName("my-catalog")
+ .setProperties(new
CatalogProperties("gs://my-bucket/path/to/data"))
+ .setStorageConfigInfo(gcpConfigModel)
+ .build();
+ try (Response response =
+ newRequest("http://localhost:%d/api/management/v1/catalogs";)
+ .post(Entity.json(new CreateCatalogRequest(catalog)))) {
+ assertThat(response).returns(Response.Status.CREATED.getStatusCode(),
Response::getStatus);
+ }
+ try (Response response =
+
newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog";).get()) {
+ assertThat(response).returns(Response.Status.OK.getStatusCode(),
Response::getStatus);
+ Catalog catResponse = response.readEntity(Catalog.class);
+ assertThat(catResponse.getStorageConfigInfo())
+ .isInstanceOf(GcpStorageConfigInfo.class)
+ .hasFieldOrPropertyWithValue("gcsServiceAccount", "my-sa")
+ .hasFieldOrPropertyWithValue("allowedLocations",
List.of("gs://my-bucket/path/to/data"));
+ }
+ }
+
@Test
public void testCreateCatalogWithNullBaseLocation() {
AwsStorageConfigInfo awsConfigModel =
diff --git
a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java
b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java
index d64ac3bb..4f6a2753 100644
---
a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java
+++
b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java
@@ -18,8 +18,11 @@
*/
package org.apache.polaris.service.catalog;
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
import com.google.common.collect.ImmutableMap;
import java.time.Clock;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -66,7 +69,8 @@ public class BasePolarisCatalogViewTest extends
ViewCatalogTests<BasePolarisCata
InMemoryPolarisMetaStoreManagerFactory managerFactory =
new InMemoryPolarisMetaStoreManagerFactory();
managerFactory.setStorageIntegrationProvider(
- new PolarisStorageIntegrationProviderImpl(Mockito::mock));
+ new PolarisStorageIntegrationProviderImpl(
+ Mockito::mock, () -> GoogleCredentials.create(new
AccessToken("abc", new Date()))));
PolarisMetaStoreManager metaStoreManager =
managerFactory.getOrCreateMetaStoreManager(realmContext);
Map<String, Object> configMap = new HashMap<>();
