flyrain commented on code in PR #389: URL: https://github.com/apache/polaris/pull/389#discussion_r1948001849
########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Polaris Storage Configuration information for an S3 Compatible solution, MinIO, Ceph, Dell ECS... + */ +public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigurationInfo { + + // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required + // for allowed read and write locations for subscoping creds. + @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; + private @NotNull String s3Endpoint; + private @Nullable String s3CredentialsCatalogAccessKeyId; + private @Nullable String s3CredentialsCatalogSecretAccessKey; + private @Nullable Boolean s3PathStyleAccess; + private @NotNull Boolean skipCredentialSubscopingIndirection; + private @Nullable String s3CredentialsClientAccessKeyId; + private @Nullable String s3CredentialsClientSecretAccessKey; + private @Nullable String s3Region; + private @Nullable String s3RoleArn; + + // Constructor Review Comment: Nit: remove this comment? ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Polaris Storage Configuration information for an S3 Compatible solution, MinIO, Ceph, Dell ECS... + */ +public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigurationInfo { + + // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required + // for allowed read and write locations for subscoping creds. + @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; + private @NotNull String s3Endpoint; + private @Nullable String s3CredentialsCatalogAccessKeyId; + private @Nullable String s3CredentialsCatalogSecretAccessKey; + private @Nullable Boolean s3PathStyleAccess; + private @NotNull Boolean skipCredentialSubscopingIndirection; + private @Nullable String s3CredentialsClientAccessKeyId; + private @Nullable String s3CredentialsClientSecretAccessKey; + private @Nullable String s3Region; + private @Nullable String s3RoleArn; Review Comment: Minor: can we mark them `final` ? ########## polaris-server.yml: ########## @@ -69,6 +69,7 @@ featureConfiguration: ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false SUPPORTED_CATALOG_STORAGE_TYPES: - S3 + - S3_COMPATIBLE Review Comment: This will need rebase as Quakus has changed the configurations. You can find it here, https://polaris.apache.org/in-dev/unreleased/configuration/ ``` polaris.features.defaults.SUPPORTED_CATALOG_STORAGE_TYPES=["S3","GCS","AZURE"]. Note: this excludes the default FILE storage type, which is not meaningful in a distributed deployment. ``` ########## regtests/minio/Readme.md: ########## @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# MiniIO Secured +## Minio and secured buckets with TLS self-signed / custom AC Review Comment: Can we give a full name for `AC`? ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java: ########## @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import jakarta.annotation.Nonnull; +import java.net.URI; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.storage.InMemoryStorageIntegration; +import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageUtil; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.policybuilder.iam.IamStatement; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; + +/** S3 compatible implementation of PolarisStorageIntegration */ +public class S3CompatibleCredentialsStorageIntegration + extends InMemoryStorageIntegration<S3CompatibleStorageConfigurationInfo> { + private static final Logger LOGGER = + LoggerFactory.getLogger(S3CompatibleCredentialsStorageIntegration.class); + private StsClient stsClient; + private String caI, caS; // catalogAccessKeyId & catalogSecretAccessKey + private String clI, clS; // clientAccessKeyId & clientSecretAccessKey + + public S3CompatibleCredentialsStorageIntegration() { + super(S3CompatibleCredentialsStorageIntegration.class.getName()); + } + + /** {@inheritDoc} */ + @Override + public EnumMap<PolarisCredentialProperty, String> getSubscopedCreds( + @NotNull PolarisDiagnostics diagnostics, + @NotNull S3CompatibleStorageConfigurationInfo storageConfig, + boolean allowListOperation, + @NotNull Set<String> allowedReadLocations, + @NotNull Set<String> allowedWriteLocations) { + + caI = System.getenv(storageConfig.getS3CredentialsCatalogAccessKeyId()); + caS = System.getenv(storageConfig.getS3CredentialsCatalogSecretAccessKey()); + + EnumMap<PolarisCredentialProperty, String> propertiesMap = + new EnumMap<>(PolarisCredentialProperty.class); + propertiesMap.put(PolarisCredentialProperty.AWS_ENDPOINT, storageConfig.getS3Endpoint()); + propertiesMap.put( + PolarisCredentialProperty.AWS_PATH_STYLE_ACCESS, + storageConfig.getS3PathStyleAccess().toString()); + if (storageConfig.getS3Region() != null) { + propertiesMap.put(PolarisCredentialProperty.CLIENT_REGION, storageConfig.getS3Region()); + } + + if (storageConfig.getSkipCredentialSubscopingIndirection() == true) { + LOGGER.debug("S3Compatible - skipCredentialSubscopingIndirection !"); + clI = System.getenv(storageConfig.getS3CredentialsClientAccessKeyId()); + clS = System.getenv(storageConfig.getS3CredentialsClientSecretAccessKey()); + if (clI != null && clS != null) { + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, clI); + propertiesMap.put(PolarisCredentialProperty.AWS_SECRET_KEY, clS); + } else { + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, caI); + propertiesMap.put(PolarisCredentialProperty.AWS_SECRET_KEY, caS); + } + } else { + LOGGER.debug("S3Compatible - assumeRole !"); + createStsClient(storageConfig); + AssumeRoleResponse response = + stsClient.assumeRole( + AssumeRoleRequest.builder() + .roleSessionName("PolarisCredentialsSTS") + .roleArn( + (storageConfig.getS3RoleArn() == null) ? "" : storageConfig.getS3RoleArn()) + .policy( + policyString(allowListOperation, allowedReadLocations, allowedWriteLocations) + .toJson()) + .build()); + + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, response.credentials().accessKeyId()); + propertiesMap.put( + PolarisCredentialProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); + propertiesMap.put(PolarisCredentialProperty.AWS_TOKEN, response.credentials().sessionToken()); + LOGGER.debug( + "S3Compatible - assumeRole - Token Expiration at : {}", + response.credentials().expiration().toString()); + } + + return propertiesMap; + } + + public void createStsClient(S3CompatibleStorageConfigurationInfo storageConfig) { + LOGGER.debug("S3Compatible - createStsClient()"); + try { + StsClientBuilder stsBuilder = software.amazon.awssdk.services.sts.StsClient.builder(); + stsBuilder.endpointOverride(URI.create(storageConfig.getS3Endpoint())); + if (caI != null && caS != null) { + // else default provider build credentials from profile or standard AWS env var + stsBuilder.credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(caI, caS))); + } + this.stsClient = stsBuilder.build(); + LOGGER.debug("S3Compatible - stsClient successfully built"); + + } catch (Exception e) { + System.err.println("S3Compatible - stsClient - build failure : " + e.getMessage()); + } + } + + /* + * function from AwsCredentialsStorageIntegration but without roleArn parameter + */ + private IamPolicy policyString( Review Comment: Same here, I'd avoid duplicating such a long method. Can we refactor it a bit? ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java: ########## @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import jakarta.annotation.Nonnull; +import java.net.URI; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.storage.InMemoryStorageIntegration; +import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageUtil; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.policybuilder.iam.IamStatement; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; + +/** S3 compatible implementation of PolarisStorageIntegration */ +public class S3CompatibleCredentialsStorageIntegration + extends InMemoryStorageIntegration<S3CompatibleStorageConfigurationInfo> { + private static final Logger LOGGER = + LoggerFactory.getLogger(S3CompatibleCredentialsStorageIntegration.class); + private StsClient stsClient; + private String caI, caS; // catalogAccessKeyId & catalogSecretAccessKey + private String clI, clS; // clientAccessKeyId & clientSecretAccessKey + + public S3CompatibleCredentialsStorageIntegration() { + super(S3CompatibleCredentialsStorageIntegration.class.getName()); + } + + /** {@inheritDoc} */ + @Override + public EnumMap<PolarisCredentialProperty, String> getSubscopedCreds( + @NotNull PolarisDiagnostics diagnostics, + @NotNull S3CompatibleStorageConfigurationInfo storageConfig, + boolean allowListOperation, + @NotNull Set<String> allowedReadLocations, + @NotNull Set<String> allowedWriteLocations) { + + caI = System.getenv(storageConfig.getS3CredentialsCatalogAccessKeyId()); + caS = System.getenv(storageConfig.getS3CredentialsCatalogSecretAccessKey()); + + EnumMap<PolarisCredentialProperty, String> propertiesMap = + new EnumMap<>(PolarisCredentialProperty.class); + propertiesMap.put(PolarisCredentialProperty.AWS_ENDPOINT, storageConfig.getS3Endpoint()); + propertiesMap.put( + PolarisCredentialProperty.AWS_PATH_STYLE_ACCESS, + storageConfig.getS3PathStyleAccess().toString()); + if (storageConfig.getS3Region() != null) { + propertiesMap.put(PolarisCredentialProperty.CLIENT_REGION, storageConfig.getS3Region()); + } + + if (storageConfig.getSkipCredentialSubscopingIndirection() == true) { + LOGGER.debug("S3Compatible - skipCredentialSubscopingIndirection !"); + clI = System.getenv(storageConfig.getS3CredentialsClientAccessKeyId()); + clS = System.getenv(storageConfig.getS3CredentialsClientSecretAccessKey()); + if (clI != null && clS != null) { + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, clI); + propertiesMap.put(PolarisCredentialProperty.AWS_SECRET_KEY, clS); + } else { + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, caI); + propertiesMap.put(PolarisCredentialProperty.AWS_SECRET_KEY, caS); + } + } else { + LOGGER.debug("S3Compatible - assumeRole !"); + createStsClient(storageConfig); + AssumeRoleResponse response = + stsClient.assumeRole( + AssumeRoleRequest.builder() + .roleSessionName("PolarisCredentialsSTS") + .roleArn( + (storageConfig.getS3RoleArn() == null) ? "" : storageConfig.getS3RoleArn()) + .policy( + policyString(allowListOperation, allowedReadLocations, allowedWriteLocations) + .toJson()) + .build()); + + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, response.credentials().accessKeyId()); + propertiesMap.put( + PolarisCredentialProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); + propertiesMap.put(PolarisCredentialProperty.AWS_TOKEN, response.credentials().sessionToken()); + LOGGER.debug( + "S3Compatible - assumeRole - Token Expiration at : {}", + response.credentials().expiration().toString()); + } + + return propertiesMap; + } + + public void createStsClient(S3CompatibleStorageConfigurationInfo storageConfig) { Review Comment: Can we make it `private` as it is invoked only within this class? ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Polaris Storage Configuration information for an S3 Compatible solution, MinIO, Ceph, Dell ECS... + */ +public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigurationInfo { + + // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required + // for allowed read and write locations for subscoping creds. + @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; + private @NotNull String s3Endpoint; + private @Nullable String s3CredentialsCatalogAccessKeyId; + private @Nullable String s3CredentialsCatalogSecretAccessKey; + private @Nullable Boolean s3PathStyleAccess; + private @NotNull Boolean skipCredentialSubscopingIndirection; + private @Nullable String s3CredentialsClientAccessKeyId; + private @Nullable String s3CredentialsClientSecretAccessKey; + private @Nullable String s3Region; + private @Nullable String s3RoleArn; + + // Constructor + @JsonCreator + public S3CompatibleStorageConfigurationInfo( + @JsonProperty(value = "storageType", required = true) @NotNull StorageType storageType, + @JsonProperty(value = "s3Endpoint", required = true) @NotNull String s3Endpoint, + @JsonProperty(value = "s3CredentialsCatalogAccessKeyId", required = true) @NotNull + String s3CredentialsCatalogAccessKeyId, + @JsonProperty(value = "s3CredentialsCatalogSecretAccessKey", required = true) @NotNull + String s3CredentialsCatalogSecretAccessKey, + @JsonProperty(value = "SkipCredentialSubscopingIndirection", required = false) @Nullable + Boolean skipCredentialSubscopingIndirection, + @JsonProperty(value = "s3CredentialsClientAccessKeyId", required = false) @Nullable + String s3CredentialsClientAccessKeyId, + @JsonProperty(value = "s3CredentialsClientSecretAccessKey", required = false) @Nullable + String s3CredentialsClientSecretAccessKey, + @JsonProperty(value = "s3PathStyleAccess", required = false) @Nullable + Boolean s3PathStyleAccess, + @JsonProperty(value = "s3Region", required = false) @Nullable String s3Region, + @JsonProperty(value = "s3RoleArn", required = false) @Nullable String s3RoleArn, + @JsonProperty(value = "allowedLocations", required = true) @Nullable + List<String> allowedLocations) { + + // storing properties + super(storageType, allowedLocations); + validateMaxAllowedLocations(MAX_ALLOWED_LOCATIONS); + this.s3PathStyleAccess = s3PathStyleAccess; + this.s3Endpoint = s3Endpoint; + this.s3CredentialsCatalogAccessKeyId = s3CredentialsCatalogAccessKeyId; + this.s3CredentialsCatalogSecretAccessKey = s3CredentialsCatalogSecretAccessKey; + this.s3CredentialsClientAccessKeyId = s3CredentialsClientAccessKeyId; + this.s3CredentialsClientSecretAccessKey = s3CredentialsClientSecretAccessKey; + this.skipCredentialSubscopingIndirection = skipCredentialSubscopingIndirection; + this.s3Region = s3Region; + this.s3RoleArn = s3RoleArn; + } + + public @NotNull String getS3Endpoint() { + return this.s3Endpoint; + } + + public @NotNull Boolean getS3PathStyleAccess() { + return this.s3PathStyleAccess; + } + + public @NotNull String getS3CredentialsCatalogAccessKeyId() { + return (this.s3CredentialsCatalogAccessKeyId == null) + ? "" + : this.s3CredentialsCatalogAccessKeyId; + } + + public @NotNull String getS3CredentialsCatalogSecretAccessKey() { + return (this.s3CredentialsCatalogSecretAccessKey == null) + ? "" + : this.s3CredentialsCatalogSecretAccessKey; Review Comment: Can we move these logic to constructor? I also suggest builder style, which seems a better fit here. ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Polaris Storage Configuration information for an S3 Compatible solution, MinIO, Ceph, Dell ECS... + */ +public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigurationInfo { + + // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required + // for allowed read and write locations for subscoping creds. + @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; + private @NotNull String s3Endpoint; + private @Nullable String s3CredentialsCatalogAccessKeyId; + private @Nullable String s3CredentialsCatalogSecretAccessKey; + private @Nullable Boolean s3PathStyleAccess; + private @NotNull Boolean skipCredentialSubscopingIndirection; + private @Nullable String s3CredentialsClientAccessKeyId; + private @Nullable String s3CredentialsClientSecretAccessKey; + private @Nullable String s3Region; + private @Nullable String s3RoleArn; + + // Constructor + @JsonCreator + public S3CompatibleStorageConfigurationInfo( + @JsonProperty(value = "storageType", required = true) @NotNull StorageType storageType, + @JsonProperty(value = "s3Endpoint", required = true) @NotNull String s3Endpoint, + @JsonProperty(value = "s3CredentialsCatalogAccessKeyId", required = true) @NotNull + String s3CredentialsCatalogAccessKeyId, + @JsonProperty(value = "s3CredentialsCatalogSecretAccessKey", required = true) @NotNull + String s3CredentialsCatalogSecretAccessKey, + @JsonProperty(value = "SkipCredentialSubscopingIndirection", required = false) @Nullable + Boolean skipCredentialSubscopingIndirection, + @JsonProperty(value = "s3CredentialsClientAccessKeyId", required = false) @Nullable + String s3CredentialsClientAccessKeyId, + @JsonProperty(value = "s3CredentialsClientSecretAccessKey", required = false) @Nullable + String s3CredentialsClientSecretAccessKey, + @JsonProperty(value = "s3PathStyleAccess", required = false) @Nullable + Boolean s3PathStyleAccess, + @JsonProperty(value = "s3Region", required = false) @Nullable String s3Region, + @JsonProperty(value = "s3RoleArn", required = false) @Nullable String s3RoleArn, + @JsonProperty(value = "allowedLocations", required = true) @Nullable + List<String> allowedLocations) { + + // storing properties + super(storageType, allowedLocations); Review Comment: Can we use the new type directly here? I think we don't need to pass it from the constructor. Do we? ```suggestion super(StorageType.S3_COMPATIBLE, allowedLocations); ``` ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Polaris Storage Configuration information for an S3 Compatible solution, MinIO, Ceph, Dell ECS... + */ +public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigurationInfo { + + // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required + // for allowed read and write locations for subscoping creds. + @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; + private @NotNull String s3Endpoint; + private @Nullable String s3CredentialsCatalogAccessKeyId; + private @Nullable String s3CredentialsCatalogSecretAccessKey; + private @Nullable Boolean s3PathStyleAccess; + private @NotNull Boolean skipCredentialSubscopingIndirection; + private @Nullable String s3CredentialsClientAccessKeyId; + private @Nullable String s3CredentialsClientSecretAccessKey; + private @Nullable String s3Region; + private @Nullable String s3RoleArn; + + // Constructor + @JsonCreator + public S3CompatibleStorageConfigurationInfo( + @JsonProperty(value = "storageType", required = true) @NotNull StorageType storageType, + @JsonProperty(value = "s3Endpoint", required = true) @NotNull String s3Endpoint, + @JsonProperty(value = "s3CredentialsCatalogAccessKeyId", required = true) @NotNull + String s3CredentialsCatalogAccessKeyId, + @JsonProperty(value = "s3CredentialsCatalogSecretAccessKey", required = true) @NotNull + String s3CredentialsCatalogSecretAccessKey, + @JsonProperty(value = "SkipCredentialSubscopingIndirection", required = false) @Nullable + Boolean skipCredentialSubscopingIndirection, + @JsonProperty(value = "s3CredentialsClientAccessKeyId", required = false) @Nullable + String s3CredentialsClientAccessKeyId, + @JsonProperty(value = "s3CredentialsClientSecretAccessKey", required = false) @Nullable + String s3CredentialsClientSecretAccessKey, + @JsonProperty(value = "s3PathStyleAccess", required = false) @Nullable + Boolean s3PathStyleAccess, + @JsonProperty(value = "s3Region", required = false) @Nullable String s3Region, + @JsonProperty(value = "s3RoleArn", required = false) @Nullable String s3RoleArn, + @JsonProperty(value = "allowedLocations", required = true) @Nullable + List<String> allowedLocations) { + + // storing properties + super(storageType, allowedLocations); + validateMaxAllowedLocations(MAX_ALLOWED_LOCATIONS); + this.s3PathStyleAccess = s3PathStyleAccess; + this.s3Endpoint = s3Endpoint; + this.s3CredentialsCatalogAccessKeyId = s3CredentialsCatalogAccessKeyId; + this.s3CredentialsCatalogSecretAccessKey = s3CredentialsCatalogSecretAccessKey; + this.s3CredentialsClientAccessKeyId = s3CredentialsClientAccessKeyId; + this.s3CredentialsClientSecretAccessKey = s3CredentialsClientSecretAccessKey; + this.skipCredentialSubscopingIndirection = skipCredentialSubscopingIndirection; + this.s3Region = s3Region; + this.s3RoleArn = s3RoleArn; + } + + public @NotNull String getS3Endpoint() { + return this.s3Endpoint; + } + + public @NotNull Boolean getS3PathStyleAccess() { + return this.s3PathStyleAccess; + } + + public @NotNull String getS3CredentialsCatalogAccessKeyId() { + return (this.s3CredentialsCatalogAccessKeyId == null) + ? "" + : this.s3CredentialsCatalogAccessKeyId; + } + + public @NotNull String getS3CredentialsCatalogSecretAccessKey() { + return (this.s3CredentialsCatalogSecretAccessKey == null) + ? "" + : this.s3CredentialsCatalogSecretAccessKey; + } + + public @NotNull String getS3CredentialsClientAccessKeyId() { + return (this.s3CredentialsClientAccessKeyId == null) ? "" : this.s3CredentialsClientAccessKeyId; + } + + public @NotNull String getS3CredentialsClientSecretAccessKey() { + return (this.s3CredentialsClientSecretAccessKey == null) + ? "" + : this.s3CredentialsClientSecretAccessKey; + } + + public @NotNull String getS3RoleArn() { + return this.s3RoleArn; + } + + public @Nullable String getS3Region() { + return this.s3Region; + } + + public @Nullable Boolean getSkipCredentialSubscopingIndirection() { + return this.skipCredentialSubscopingIndirection; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("storageType", getStorageType().name()) + .add("allowedLocation", getAllowedLocations()) + .toString(); Review Comment: Can we include all fields? ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Polaris Storage Configuration information for an S3 Compatible solution, MinIO, Ceph, Dell ECS... + */ +public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigurationInfo { + + // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required + // for allowed read and write locations for subscoping creds. + @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; + private @NotNull String s3Endpoint; + private @Nullable String s3CredentialsCatalogAccessKeyId; + private @Nullable String s3CredentialsCatalogSecretAccessKey; + private @Nullable Boolean s3PathStyleAccess; + private @NotNull Boolean skipCredentialSubscopingIndirection; + private @Nullable String s3CredentialsClientAccessKeyId; + private @Nullable String s3CredentialsClientSecretAccessKey; + private @Nullable String s3Region; + private @Nullable String s3RoleArn; + + // Constructor + @JsonCreator + public S3CompatibleStorageConfigurationInfo( + @JsonProperty(value = "storageType", required = true) @NotNull StorageType storageType, + @JsonProperty(value = "s3Endpoint", required = true) @NotNull String s3Endpoint, + @JsonProperty(value = "s3CredentialsCatalogAccessKeyId", required = true) @NotNull + String s3CredentialsCatalogAccessKeyId, + @JsonProperty(value = "s3CredentialsCatalogSecretAccessKey", required = true) @NotNull + String s3CredentialsCatalogSecretAccessKey, + @JsonProperty(value = "SkipCredentialSubscopingIndirection", required = false) @Nullable + Boolean skipCredentialSubscopingIndirection, + @JsonProperty(value = "s3CredentialsClientAccessKeyId", required = false) @Nullable + String s3CredentialsClientAccessKeyId, + @JsonProperty(value = "s3CredentialsClientSecretAccessKey", required = false) @Nullable + String s3CredentialsClientSecretAccessKey, + @JsonProperty(value = "s3PathStyleAccess", required = false) @Nullable + Boolean s3PathStyleAccess, + @JsonProperty(value = "s3Region", required = false) @Nullable String s3Region, + @JsonProperty(value = "s3RoleArn", required = false) @Nullable String s3RoleArn, + @JsonProperty(value = "allowedLocations", required = true) @Nullable + List<String> allowedLocations) { + + // storing properties Review Comment: Nit: can we remove the comment? ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Polaris Storage Configuration information for an S3 Compatible solution, MinIO, Ceph, Dell ECS... Review Comment: ```suggestion * S3-Compatible Storage Configuration. This class holds the parameters needed to connect to S3-compatible storage services such as MinIO, Ceph, Dell ECS, etc. ``` ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Polaris Storage Configuration information for an S3 Compatible solution, MinIO, Ceph, Dell ECS... + */ +public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigurationInfo { + + // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required + // for allowed read and write locations for subscoping creds. + @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; + private @NotNull String s3Endpoint; + private @Nullable String s3CredentialsCatalogAccessKeyId; + private @Nullable String s3CredentialsCatalogSecretAccessKey; + private @Nullable Boolean s3PathStyleAccess; + private @NotNull Boolean skipCredentialSubscopingIndirection; + private @Nullable String s3CredentialsClientAccessKeyId; + private @Nullable String s3CredentialsClientSecretAccessKey; + private @Nullable String s3Region; + private @Nullable String s3RoleArn; + + // Constructor + @JsonCreator + public S3CompatibleStorageConfigurationInfo( + @JsonProperty(value = "storageType", required = true) @NotNull StorageType storageType, + @JsonProperty(value = "s3Endpoint", required = true) @NotNull String s3Endpoint, + @JsonProperty(value = "s3CredentialsCatalogAccessKeyId", required = true) @NotNull + String s3CredentialsCatalogAccessKeyId, + @JsonProperty(value = "s3CredentialsCatalogSecretAccessKey", required = true) @NotNull + String s3CredentialsCatalogSecretAccessKey, + @JsonProperty(value = "SkipCredentialSubscopingIndirection", required = false) @Nullable + Boolean skipCredentialSubscopingIndirection, + @JsonProperty(value = "s3CredentialsClientAccessKeyId", required = false) @Nullable + String s3CredentialsClientAccessKeyId, + @JsonProperty(value = "s3CredentialsClientSecretAccessKey", required = false) @Nullable + String s3CredentialsClientSecretAccessKey, + @JsonProperty(value = "s3PathStyleAccess", required = false) @Nullable + Boolean s3PathStyleAccess, + @JsonProperty(value = "s3Region", required = false) @Nullable String s3Region, + @JsonProperty(value = "s3RoleArn", required = false) @Nullable String s3RoleArn, + @JsonProperty(value = "allowedLocations", required = true) @Nullable + List<String> allowedLocations) { + + // storing properties + super(storageType, allowedLocations); + validateMaxAllowedLocations(MAX_ALLOWED_LOCATIONS); + this.s3PathStyleAccess = s3PathStyleAccess; Review Comment: Do we need to set a default value, like `false` when s3PathStyleAccess is null? ########## regtests/minio/Readme.md: ########## @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# MiniIO Secured +## Minio and secured buckets with TLS self-signed / custom AC + +To be able to test Polaris with buckets in TLS under custom AC or self-signed certificate + +## MiniIO generate self-signed certificates designed for docker-compose setup + +- Download minio certificate generator : https://github.com/minio/certgen +- ```./certgen -host "localhost,minio,*"``` Review Comment: ```suggestion - Generate certifications: ```./certgen -host "localhost,minio,*"``` ``` ########## regtests/minio/Readme.md: ########## @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# MiniIO Secured +## Minio and secured buckets with TLS self-signed / custom AC + +To be able to test Polaris with buckets in TLS under custom AC or self-signed certificate Review Comment: ```suggestion Here are steps to test Polaris with buckets in TLS under custom AC or self-signed certificate ``` ########## spec/polaris-management-service.yml: ########## @@ -905,6 +907,51 @@ components: required: - roleArn + S3CompatibleStorageConfigInfo: + type: object + description: S3 compatible storage configuration info (MinIO, Ceph, Dell ECS, Netapp StorageGRID, ...) Review Comment: ```suggestion description: s3-compatible storage configuration info (MinIO, Ceph, Dell ECS, Netapp StorageGRID, ...) ``` ########## spec/polaris-management-service.yml: ########## @@ -905,6 +907,51 @@ components: required: - roleArn + S3CompatibleStorageConfigInfo: + type: object + description: S3 compatible storage configuration info (MinIO, Ceph, Dell ECS, Netapp StorageGRID, ...) + allOf: + - $ref: '#/components/schemas/StorageConfigInfo' + properties: + s3.endpoint: + type: string + description: the S3 endpoint, will also be used as STS endpoint Review Comment: I guess we don't have to mention `STS endpoint`, since there is no separated configure for that, and `s3.endpoint` will always be used as the STS endpoint. Can you confirm? ########## regtests/minio/Readme.md: ########## @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# MiniIO Secured +## Minio and secured buckets with TLS self-signed / custom AC + +To be able to test Polaris with buckets in TLS under custom AC or self-signed certificate + +## MiniIO generate self-signed certificates designed for docker-compose setup + +- Download minio certificate generator : https://github.com/minio/certgen +- ```./certgen -host "localhost,minio,*"``` +- put them in ./certs and ./certs/CAs +- they will be mounted in default minio container placeholder Review Comment: ```suggestion - Put them in ./certs and ./certs/CAs. They will be mounted in the default MinIO container placeholder. ``` ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java: ########## @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import jakarta.annotation.Nonnull; +import java.net.URI; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.storage.InMemoryStorageIntegration; +import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageUtil; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.policybuilder.iam.IamStatement; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; + +/** S3 compatible implementation of PolarisStorageIntegration */ +public class S3CompatibleCredentialsStorageIntegration + extends InMemoryStorageIntegration<S3CompatibleStorageConfigurationInfo> { + private static final Logger LOGGER = + LoggerFactory.getLogger(S3CompatibleCredentialsStorageIntegration.class); + private StsClient stsClient; + private String caI, caS; // catalogAccessKeyId & catalogSecretAccessKey + private String clI, clS; // clientAccessKeyId & clientSecretAccessKey + + public S3CompatibleCredentialsStorageIntegration() { + super(S3CompatibleCredentialsStorageIntegration.class.getName()); + } + + /** {@inheritDoc} */ + @Override + public EnumMap<PolarisCredentialProperty, String> getSubscopedCreds( + @NotNull PolarisDiagnostics diagnostics, + @NotNull S3CompatibleStorageConfigurationInfo storageConfig, + boolean allowListOperation, + @NotNull Set<String> allowedReadLocations, + @NotNull Set<String> allowedWriteLocations) { + + caI = System.getenv(storageConfig.getS3CredentialsCatalogAccessKeyId()); + caS = System.getenv(storageConfig.getS3CredentialsCatalogSecretAccessKey()); + + EnumMap<PolarisCredentialProperty, String> propertiesMap = + new EnumMap<>(PolarisCredentialProperty.class); + propertiesMap.put(PolarisCredentialProperty.AWS_ENDPOINT, storageConfig.getS3Endpoint()); + propertiesMap.put( + PolarisCredentialProperty.AWS_PATH_STYLE_ACCESS, + storageConfig.getS3PathStyleAccess().toString()); + if (storageConfig.getS3Region() != null) { + propertiesMap.put(PolarisCredentialProperty.CLIENT_REGION, storageConfig.getS3Region()); + } + + if (storageConfig.getSkipCredentialSubscopingIndirection() == true) { + LOGGER.debug("S3Compatible - skipCredentialSubscopingIndirection !"); + clI = System.getenv(storageConfig.getS3CredentialsClientAccessKeyId()); + clS = System.getenv(storageConfig.getS3CredentialsClientSecretAccessKey()); + if (clI != null && clS != null) { + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, clI); + propertiesMap.put(PolarisCredentialProperty.AWS_SECRET_KEY, clS); + } else { + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, caI); + propertiesMap.put(PolarisCredentialProperty.AWS_SECRET_KEY, caS); + } + } else { + LOGGER.debug("S3Compatible - assumeRole !"); + createStsClient(storageConfig); + AssumeRoleResponse response = + stsClient.assumeRole( + AssumeRoleRequest.builder() + .roleSessionName("PolarisCredentialsSTS") + .roleArn( + (storageConfig.getS3RoleArn() == null) ? "" : storageConfig.getS3RoleArn()) + .policy( + policyString(allowListOperation, allowedReadLocations, allowedWriteLocations) + .toJson()) + .build()); + + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, response.credentials().accessKeyId()); + propertiesMap.put( + PolarisCredentialProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); + propertiesMap.put(PolarisCredentialProperty.AWS_TOKEN, response.credentials().sessionToken()); + LOGGER.debug( + "S3Compatible - assumeRole - Token Expiration at : {}", + response.credentials().expiration().toString()); + } + + return propertiesMap; + } + + public void createStsClient(S3CompatibleStorageConfigurationInfo storageConfig) { + LOGGER.debug("S3Compatible - createStsClient()"); + try { + StsClientBuilder stsBuilder = software.amazon.awssdk.services.sts.StsClient.builder(); + stsBuilder.endpointOverride(URI.create(storageConfig.getS3Endpoint())); + if (caI != null && caS != null) { + // else default provider build credentials from profile or standard AWS env var + stsBuilder.credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(caI, caS))); + } + this.stsClient = stsBuilder.build(); + LOGGER.debug("S3Compatible - stsClient successfully built"); + + } catch (Exception e) { + System.err.println("S3Compatible - stsClient - build failure : " + e.getMessage()); + } + } + + /* + * function from AwsCredentialsStorageIntegration but without roleArn parameter + */ + private IamPolicy policyString( + boolean allowList, Set<String> readLocations, Set<String> writeLocations) { + IamPolicy.Builder policyBuilder = IamPolicy.builder(); + IamStatement.Builder allowGetObjectStatementBuilder = + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:GetObject") + .addAction("s3:GetObjectVersion"); + Map<String, IamStatement.Builder> bucketListStatementBuilder = new HashMap<>(); + Map<String, IamStatement.Builder> bucketGetLocationStatementBuilder = new HashMap<>(); + + String arnPrefix = "arn:aws:s3:::"; + Stream.concat(readLocations.stream(), writeLocations.stream()) + .distinct() + .forEach( + location -> { + URI uri = URI.create(location); + allowGetObjectStatementBuilder.addResource( + IamResource.create( + arnPrefix + StorageUtil.concatFilePrefixes(parseS3Path(uri), "*", "/"))); + final var bucket = arnPrefix + StorageUtil.getBucket(uri); + if (allowList) { + bucketListStatementBuilder + .computeIfAbsent( + bucket, + (String key) -> + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:ListBucket") + .addResource(key)) + .addCondition( + IamConditionOperator.STRING_LIKE, + "s3:prefix", + StorageUtil.concatFilePrefixes(trimLeadingSlash(uri.getPath()), "*", "/")); + } + bucketGetLocationStatementBuilder.computeIfAbsent( + bucket, + key -> + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:GetBucketLocation") + .addResource(key)); + }); + + if (!writeLocations.isEmpty()) { + IamStatement.Builder allowPutObjectStatementBuilder = + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:PutObject") + .addAction("s3:DeleteObject"); + writeLocations.forEach( + location -> { + URI uri = URI.create(location); + allowPutObjectStatementBuilder.addResource( + IamResource.create( + arnPrefix + StorageUtil.concatFilePrefixes(parseS3Path(uri), "*", "/"))); + }); + policyBuilder.addStatement(allowPutObjectStatementBuilder.build()); + } + if (!bucketListStatementBuilder.isEmpty()) { + bucketListStatementBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + } else if (allowList) { + // add list privilege with 0 resources + policyBuilder.addStatement( + IamStatement.builder().effect(IamEffect.ALLOW).addAction("s3:ListBucket").build()); + } + + bucketGetLocationStatementBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + return policyBuilder.addStatement(allowGetObjectStatementBuilder.build()).build(); + } + + /* function from AwsCredentialsStorageIntegration */ + private static @Nonnull String parseS3Path(URI uri) { + String bucket = StorageUtil.getBucket(uri); + String path = trimLeadingSlash(uri.getPath()); + return String.join("/", bucket, path); + } + + /* function from AwsCredentialsStorageIntegration */ + private static @Nonnull String trimLeadingSlash(String path) { + if (path.startsWith("/")) { + path = path.substring(1); + } + return path; + } Review Comment: Here are options to avoid duplication: 1. Make the method public in the original class(`AwsCredentialsStorageIntegration`), and invoke them here. 2. Move them from `AwsCredentialsStorageIntegration` to a util class like `StorageUtil` ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java: ########## @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import jakarta.annotation.Nonnull; +import java.net.URI; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.storage.InMemoryStorageIntegration; +import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageUtil; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.policybuilder.iam.IamStatement; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; + +/** S3 compatible implementation of PolarisStorageIntegration */ +public class S3CompatibleCredentialsStorageIntegration + extends InMemoryStorageIntegration<S3CompatibleStorageConfigurationInfo> { + private static final Logger LOGGER = + LoggerFactory.getLogger(S3CompatibleCredentialsStorageIntegration.class); + private StsClient stsClient; + private String caI, caS; // catalogAccessKeyId & catalogSecretAccessKey + private String clI, clS; // clientAccessKeyId & clientSecretAccessKey + + public S3CompatibleCredentialsStorageIntegration() { + super(S3CompatibleCredentialsStorageIntegration.class.getName()); + } + + /** {@inheritDoc} */ + @Override + public EnumMap<PolarisCredentialProperty, String> getSubscopedCreds( + @NotNull PolarisDiagnostics diagnostics, + @NotNull S3CompatibleStorageConfigurationInfo storageConfig, + boolean allowListOperation, + @NotNull Set<String> allowedReadLocations, + @NotNull Set<String> allowedWriteLocations) { + + caI = System.getenv(storageConfig.getS3CredentialsCatalogAccessKeyId()); + caS = System.getenv(storageConfig.getS3CredentialsCatalogSecretAccessKey()); + + EnumMap<PolarisCredentialProperty, String> propertiesMap = + new EnumMap<>(PolarisCredentialProperty.class); + propertiesMap.put(PolarisCredentialProperty.AWS_ENDPOINT, storageConfig.getS3Endpoint()); + propertiesMap.put( + PolarisCredentialProperty.AWS_PATH_STYLE_ACCESS, + storageConfig.getS3PathStyleAccess().toString()); + if (storageConfig.getS3Region() != null) { + propertiesMap.put(PolarisCredentialProperty.CLIENT_REGION, storageConfig.getS3Region()); + } + + if (storageConfig.getSkipCredentialSubscopingIndirection() == true) { + LOGGER.debug("S3Compatible - skipCredentialSubscopingIndirection !"); + clI = System.getenv(storageConfig.getS3CredentialsClientAccessKeyId()); + clS = System.getenv(storageConfig.getS3CredentialsClientSecretAccessKey()); + if (clI != null && clS != null) { + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, clI); + propertiesMap.put(PolarisCredentialProperty.AWS_SECRET_KEY, clS); + } else { + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, caI); + propertiesMap.put(PolarisCredentialProperty.AWS_SECRET_KEY, caS); + } + } else { + LOGGER.debug("S3Compatible - assumeRole !"); + createStsClient(storageConfig); + AssumeRoleResponse response = + stsClient.assumeRole( + AssumeRoleRequest.builder() + .roleSessionName("PolarisCredentialsSTS") + .roleArn( + (storageConfig.getS3RoleArn() == null) ? "" : storageConfig.getS3RoleArn()) + .policy( + policyString(allowListOperation, allowedReadLocations, allowedWriteLocations) + .toJson()) + .build()); + + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, response.credentials().accessKeyId()); + propertiesMap.put( + PolarisCredentialProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); + propertiesMap.put(PolarisCredentialProperty.AWS_TOKEN, response.credentials().sessionToken()); + LOGGER.debug( + "S3Compatible - assumeRole - Token Expiration at : {}", + response.credentials().expiration().toString()); + } + + return propertiesMap; + } + + public void createStsClient(S3CompatibleStorageConfigurationInfo storageConfig) { + LOGGER.debug("S3Compatible - createStsClient()"); + try { + StsClientBuilder stsBuilder = software.amazon.awssdk.services.sts.StsClient.builder(); + stsBuilder.endpointOverride(URI.create(storageConfig.getS3Endpoint())); + if (caI != null && caS != null) { + // else default provider build credentials from profile or standard AWS env var + stsBuilder.credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(caI, caS))); + } + this.stsClient = stsBuilder.build(); + LOGGER.debug("S3Compatible - stsClient successfully built"); + + } catch (Exception e) { + System.err.println("S3Compatible - stsClient - build failure : " + e.getMessage()); Review Comment: If creating sts client fails, should we propagate the error to the caller? I believe we should. We can either let the exception bubble up naturally or catch it here and wrap it in a custom runtime exception with a specific, informative error message to provide better context. ########## spec/polaris-management-service.yml: ########## @@ -905,6 +907,51 @@ components: required: - roleArn + S3CompatibleStorageConfigInfo: + type: object + description: S3 compatible storage configuration info (MinIO, Ceph, Dell ECS, Netapp StorageGRID, ...) + allOf: + - $ref: '#/components/schemas/StorageConfigInfo' + properties: + s3.endpoint: + type: string + description: the S3 endpoint, will also be used as STS endpoint + example: "http[s]://host:port" + s3.credentials.catalog.accessKeyId: Review Comment: Can we use a name like `s3.credentials.catalog.accessKeyEnvVar` to explicitly state it's for an environment variable? Same for others(`s3.credentials.catalog.secretAccessKey`, etc). ########## polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java: ########## @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import jakarta.annotation.Nonnull; +import java.net.URI; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.storage.InMemoryStorageIntegration; +import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageUtil; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.policybuilder.iam.IamStatement; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; + +/** S3 compatible implementation of PolarisStorageIntegration */ +public class S3CompatibleCredentialsStorageIntegration + extends InMemoryStorageIntegration<S3CompatibleStorageConfigurationInfo> { + private static final Logger LOGGER = + LoggerFactory.getLogger(S3CompatibleCredentialsStorageIntegration.class); + private StsClient stsClient; Review Comment: Can we move it to the method `getSubscopedCreds()` if a new client is created every time `getSubscopedCreds()` is called? ########## spec/polaris-management-service.yml: ########## @@ -905,6 +907,51 @@ components: required: - roleArn + S3CompatibleStorageConfigInfo: + type: object + description: S3 compatible storage configuration info (MinIO, Ceph, Dell ECS, Netapp StorageGRID, ...) + allOf: + - $ref: '#/components/schemas/StorageConfigInfo' + properties: + s3.endpoint: + type: string + description: the S3 endpoint, will also be used as STS endpoint + example: "http[s]://host:port" + s3.credentials.catalog.accessKeyId: + type: string + description: Default to AWS credentials, otherwise set the environement variable name for the 'ACCESS_KEY_ID' used by the catalog to communicate with S3 + example: "CATALOG_1_ACCESS_KEY_ENV_VARIABLE_NAME or AWS_ACCESS_KEY_ID" Review Comment: Does this means every time a Polaris client creates a new catalog, it has to know env variables in the server side? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
