dimas-b commented on code in PR #2153:
URL: https://github.com/apache/polaris/pull/2153#discussion_r2221006276
##########
service/common/src/main/java/org/apache/polaris/service/catalog/validation/StorageTypeFileIO.java:
##########
@@ -32,6 +32,8 @@ enum StorageTypeFileIO {
FILE("org.apache.iceberg.hadoop.HadoopFileIO", false),
+ OSS("org.apache.iceberg.aliyun.oss.OSSFileIO", true),
Review Comment:
`OSS` is too confusing WRT "Open Source Software". How about `ALIYUN_OSS`
(per README in https://github.com/aliyun/aliyun-oss-java-sdk)?
##########
polaris-core/src/main/java/org/apache/polaris/core/storage/oss/OssStsClientProvider.java:
##########
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.storage.oss;
+
+import com.aliyuncs.DefaultAcsClient;
+import com.aliyuncs.IAcsClient;
+import com.aliyuncs.auth.sts.AssumeRoleRequest;
+import com.aliyuncs.auth.sts.AssumeRoleResponse;
+import com.aliyuncs.exceptions.ClientException;
+import com.aliyuncs.http.MethodType;
+import com.aliyuncs.profile.DefaultProfile;
+import com.aliyuncs.profile.IClientProfile;
+import jakarta.annotation.Nonnull;
+import jakarta.annotation.Nullable;
+import java.net.URI;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Provider for OSS STS clients with caching support using direct STS
AssumeRole API
+ */
+public class OssStsClientProvider {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(OssStsClientProvider.class);
+
+ private final ConcurrentMap<StsDestination, IAcsClient> clientCache = new
ConcurrentHashMap<>();
+ private final int maxCacheSize;
+
+ public OssStsClientProvider() {
+ this(50); // Default cache size
+ }
+
+ public OssStsClientProvider(int maxCacheSize) {
+ this.maxCacheSize = maxCacheSize;
+ }
+
+ /**
+ * Perform STS AssumeRole operation to get temporary credentials
+ */
+ public AssumeRoleResponse assumeRole(
+ String roleArn, String roleSessionName, String externalId, String region,
+ URI stsEndpoint, String policy, Long durationSeconds) throws
ClientException {
+
+ // Get environment variables for authentication
+ String accessKeyId = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID");
Review Comment:
Please make credentials injectable.
##########
polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java:
##########
@@ -61,10 +62,11 @@
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME)
@JsonSubTypes({
- @JsonSubTypes.Type(value = AwsStorageConfigurationInfo.class),
- @JsonSubTypes.Type(value = AzureStorageConfigurationInfo.class),
- @JsonSubTypes.Type(value = GcpStorageConfigurationInfo.class),
- @JsonSubTypes.Type(value = FileStorageConfigurationInfo.class),
+ @JsonSubTypes.Type(value = AwsStorageConfigurationInfo.class),
Review Comment:
It looks like a spurious indentation change.
##########
polaris-core/src/main/java/org/apache/polaris/core/storage/oss/OssCredentialsStorageIntegration.java:
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.storage.oss;
+
+import static
org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS;
+
+import com.aliyuncs.AcsRequest;
+import com.aliyuncs.AcsResponse;
+import com.aliyuncs.DefaultAcsClient;
+import com.aliyuncs.IAcsClient;
+import com.aliyuncs.profile.DefaultProfile;
+import com.aliyuncs.auth.sts.AssumeRoleRequest;
+import com.aliyuncs.auth.sts.AssumeRoleResponse;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.annotation.Nonnull;
+import java.net.URI;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.InMemoryStorageIntegration;
+import org.apache.polaris.core.storage.PolarisStorageActions;
+import org.apache.polaris.core.storage.StorageAccessProperty;
+import org.apache.polaris.core.storage.StorageUtil;
+import org.apache.polaris.core.storage.oss.OssStsClientProvider.StsDestination;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** OSS credential vendor that supports generating temporary credentials using
Alibaba Cloud STS */
+public class OssCredentialsStorageIntegration
+ extends InMemoryStorageIntegration<OssStorageConfigurationInfo> {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(OssCredentialsStorageIntegration.class);
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private final OssStsClientProvider stsClientProvider;
+
+ public OssCredentialsStorageIntegration() {
+ this(new OssStsClientProvider());
+ }
+
+
+ public OssCredentialsStorageIntegration(OssStsClientProvider
stsClientProvider) {
+ super(OssCredentialsStorageIntegration.class.getName());
+ this.stsClientProvider = stsClientProvider;
Review Comment:
Is it possible to make credentials injectable as in
`AwsCredentialsStorageIntegration`?
##########
polaris-core/build.gradle.kts:
##########
@@ -101,9 +101,15 @@ dependencies {
implementation(platform(libs.google.cloud.storage.bom))
implementation("com.google.cloud:google-cloud-storage")
+ implementation("org.apache.iceberg:iceberg-aliyun")
+ implementation("com.aliyun.oss:aliyun-sdk-oss:3.17.4")
+ implementation("com.aliyun:aliyun-java-sdk-sts:3.1.2")
+ implementation("com.aliyun:aliyun-java-sdk-core:4.6.4")
Review Comment:
Please use the `toml` file for defining versions.
##########
spec/polaris-management-service.yml:
##########
@@ -1101,6 +1104,37 @@ components:
allOf:
- $ref: '#/components/schemas/StorageConfigInfo'
+ #新增oss
+ OssStorageConfigInfo:
Review Comment:
How is it different from `AwsStorageConfigInfo`? Does it warrant a common
sub-type?
##########
polaris-core/src/main/java/org/apache/polaris/core/storage/oss/OssCredentialsStorageIntegration.java:
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.storage.oss;
+
+import static
org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS;
+
+import com.aliyuncs.AcsRequest;
+import com.aliyuncs.AcsResponse;
+import com.aliyuncs.DefaultAcsClient;
+import com.aliyuncs.IAcsClient;
+import com.aliyuncs.profile.DefaultProfile;
+import com.aliyuncs.auth.sts.AssumeRoleRequest;
+import com.aliyuncs.auth.sts.AssumeRoleResponse;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.annotation.Nonnull;
+import java.net.URI;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.InMemoryStorageIntegration;
+import org.apache.polaris.core.storage.PolarisStorageActions;
+import org.apache.polaris.core.storage.StorageAccessProperty;
+import org.apache.polaris.core.storage.StorageUtil;
+import org.apache.polaris.core.storage.oss.OssStsClientProvider.StsDestination;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** OSS credential vendor that supports generating temporary credentials using
Alibaba Cloud STS */
+public class OssCredentialsStorageIntegration
+ extends InMemoryStorageIntegration<OssStorageConfigurationInfo> {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(OssCredentialsStorageIntegration.class);
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private final OssStsClientProvider stsClientProvider;
+
+ public OssCredentialsStorageIntegration() {
+ this(new OssStsClientProvider());
+ }
+
+
+ public OssCredentialsStorageIntegration(OssStsClientProvider
stsClientProvider) {
+ super(OssCredentialsStorageIntegration.class.getName());
+ this.stsClientProvider = stsClientProvider;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
+ @Nonnull CallContext callContext,
+ @Nonnull OssStorageConfigurationInfo storageConfig,
+ boolean allowListOperation,
+ @Nonnull Set<String> allowedReadLocations,
+ @Nonnull Set<String> allowedWriteLocations) {
+
+ int storageCredentialDurationSeconds =
+
callContext.getRealmConfig().getConfig(STORAGE_CREDENTIAL_DURATION_SECONDS);
+
+ EnumMap<StorageAccessProperty, String> credentialMap =
+ new EnumMap<>(StorageAccessProperty.class);
+
+ try {
+ // Generate role session name
+ String roleSessionName = "PolarisOssCredentialsStorageIntegration-" +
System.currentTimeMillis();
+
+ // Generate RAM policy for scoped access
+ String policy = generateOssPolicy(allowListOperation,
allowedReadLocations, allowedWriteLocations);
+
+ // Use STS client provider to assume role directly
+ AssumeRoleResponse response = stsClientProvider.assumeRole(
+ storageConfig.getRoleArn(),
+ roleSessionName,
+ storageConfig.getExternalId(),
+ storageConfig.getRegion(),
+ storageConfig.getStsEndpointUri(),
+ policy,
+ (long) storageCredentialDurationSeconds);
+
+ AssumeRoleResponse.Credentials credentials = response.getCredentials();
+
+ // Populate credential map with STS response
+ credentialMap.put(StorageAccessProperty.OSS_ACCESS_KEY_ID,
credentials.getAccessKeyId());
+ credentialMap.put(StorageAccessProperty.OSS_ACCESS_KEY_SECRET,
credentials.getAccessKeySecret());
+ credentialMap.put(StorageAccessProperty.OSS_SECURITY_TOKEN,
credentials.getSecurityToken());
+
+ // 设置为OSS模式的特殊属性，帮助Apache Iceberg识别这是OSS凭证
+ credentialMap.put(StorageAccessProperty.CLIENT_REGION,
storageConfig.getRegion() != null ? storageConfig.getRegion() : "cn-hangzhou");
+
+ // Set expiration time - convert ISO format to milliseconds timestamp
+ String expirationStr = credentials.getExpiration();
+ try {
+ Instant expirationInstant = Instant.parse(expirationStr);
+ long expirationMillis = expirationInstant.toEpochMilli();
+ credentialMap.put(StorageAccessProperty.EXPIRATION_TIME,
String.valueOf(expirationMillis));
+ } catch (Exception e) {
+ LOGGER.warn("Failed to parse expiration time '{}', using current time
+ 1 hour", expirationStr, e);
+ // Fallback to current time + 1 hour if parsing fails
+ long fallbackExpiration = System.currentTimeMillis() + 3600_000;
+ // 1 hour
+ credentialMap.put(StorageAccessProperty.EXPIRATION_TIME,
String.valueOf(fallbackExpiration));
+ }
+
+ // Set region if provided
+ if (storageConfig.getRegion() != null) {
+ credentialMap.put(StorageAccessProperty.OSS_REGION,
storageConfig.getRegion());
+ }
+
+ // Set endpoint if provided
+ URI endpointUri = storageConfig.getEndpointUri();
+ if (endpointUri != null) {
+ credentialMap.put(StorageAccessProperty.OSS_ENDPOINT,
endpointUri.toString());
+ }
+
+ LOGGER.debug(
+ "Generated OSS subscoped credentials for roleArn={}, region={},
duration={}s",
+ storageConfig.getRoleArn(),
+ storageConfig.getRegion(),
+ storageCredentialDurationSeconds);
+
+ } catch (Exception e) {
+ LOGGER.error("Failed to assume OSS role: {}", e.getMessage(), e);
+ throw new RuntimeException("Failed to assume OSS role", e);
+ }
+
+ return credentialMap;
+ }
+
+ /**
+ * Generate OSS RAM policy based on allowed operations and locations
+ * This follows Alibaba Cloud RAM policy format
+ */
+ private String generateOssPolicy(boolean allowList, Set<String>
readLocations, Set<String> writeLocations) {
Review Comment:
Could you add unit test for policy generation?
##########
polaris-core/src/main/java/org/apache/polaris/core/storage/oss/OssCredentialsStorageIntegration.java:
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.storage.oss;
+
+import static
org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS;
+
+import com.aliyuncs.AcsRequest;
+import com.aliyuncs.AcsResponse;
+import com.aliyuncs.DefaultAcsClient;
+import com.aliyuncs.IAcsClient;
+import com.aliyuncs.profile.DefaultProfile;
+import com.aliyuncs.auth.sts.AssumeRoleRequest;
+import com.aliyuncs.auth.sts.AssumeRoleResponse;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.annotation.Nonnull;
+import java.net.URI;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.InMemoryStorageIntegration;
+import org.apache.polaris.core.storage.PolarisStorageActions;
+import org.apache.polaris.core.storage.StorageAccessProperty;
+import org.apache.polaris.core.storage.StorageUtil;
+import org.apache.polaris.core.storage.oss.OssStsClientProvider.StsDestination;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** OSS credential vendor that supports generating temporary credentials using
Alibaba Cloud STS */
+public class OssCredentialsStorageIntegration
+ extends InMemoryStorageIntegration<OssStorageConfigurationInfo> {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(OssCredentialsStorageIntegration.class);
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private final OssStsClientProvider stsClientProvider;
+
+ public OssCredentialsStorageIntegration() {
+ this(new OssStsClientProvider());
+ }
+
+
+ public OssCredentialsStorageIntegration(OssStsClientProvider
stsClientProvider) {
+ super(OssCredentialsStorageIntegration.class.getName());
+ this.stsClientProvider = stsClientProvider;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
+ @Nonnull CallContext callContext,
+ @Nonnull OssStorageConfigurationInfo storageConfig,
+ boolean allowListOperation,
+ @Nonnull Set<String> allowedReadLocations,
+ @Nonnull Set<String> allowedWriteLocations) {
+
+ int storageCredentialDurationSeconds =
+
callContext.getRealmConfig().getConfig(STORAGE_CREDENTIAL_DURATION_SECONDS);
+
+ EnumMap<StorageAccessProperty, String> credentialMap =
+ new EnumMap<>(StorageAccessProperty.class);
+
+ try {
+ // Generate role session name
+ String roleSessionName = "PolarisOssCredentialsStorageIntegration-" +
System.currentTimeMillis();
+
+ // Generate RAM policy for scoped access
+ String policy = generateOssPolicy(allowListOperation,
allowedReadLocations, allowedWriteLocations);
+
+ // Use STS client provider to assume role directly
+ AssumeRoleResponse response = stsClientProvider.assumeRole(
+ storageConfig.getRoleArn(),
+ roleSessionName,
+ storageConfig.getExternalId(),
+ storageConfig.getRegion(),
+ storageConfig.getStsEndpointUri(),
+ policy,
+ (long) storageCredentialDurationSeconds);
+
+ AssumeRoleResponse.Credentials credentials = response.getCredentials();
+
+ // Populate credential map with STS response
+ credentialMap.put(StorageAccessProperty.OSS_ACCESS_KEY_ID,
credentials.getAccessKeyId());
+ credentialMap.put(StorageAccessProperty.OSS_ACCESS_KEY_SECRET,
credentials.getAccessKeySecret());
+ credentialMap.put(StorageAccessProperty.OSS_SECURITY_TOKEN,
credentials.getSecurityToken());
+
+ // 设置为OSS模式的特殊属性，帮助Apache Iceberg识别这是OSS凭证
Review Comment:
Would you mind converting comments to English?
##########
polaris-core/build.gradle.kts:
##########
@@ -101,9 +101,15 @@ dependencies {
implementation(platform(libs.google.cloud.storage.bom))
implementation("com.google.cloud:google-cloud-storage")
+ implementation("org.apache.iceberg:iceberg-aliyun")
+ implementation("com.aliyun.oss:aliyun-sdk-oss:3.17.4")
Review Comment:
Where are the sources located? How is the SDK project maintained?
https://github.com/aliyun/aliyun-oss-java-sdk (linked from Maven Central)
shows the latest version as `3.17.2`, but in Maven the latest version is 3.18.3?
##########
polaris-core/src/main/java/org/apache/polaris/core/storage/oss/OssCredentialsStorageIntegration.java:
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.storage.oss;
+
+import static
org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS;
+
+import com.aliyuncs.AcsRequest;
+import com.aliyuncs.AcsResponse;
+import com.aliyuncs.DefaultAcsClient;
+import com.aliyuncs.IAcsClient;
+import com.aliyuncs.profile.DefaultProfile;
+import com.aliyuncs.auth.sts.AssumeRoleRequest;
+import com.aliyuncs.auth.sts.AssumeRoleResponse;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.annotation.Nonnull;
+import java.net.URI;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.InMemoryStorageIntegration;
+import org.apache.polaris.core.storage.PolarisStorageActions;
+import org.apache.polaris.core.storage.StorageAccessProperty;
+import org.apache.polaris.core.storage.StorageUtil;
+import org.apache.polaris.core.storage.oss.OssStsClientProvider.StsDestination;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** OSS credential vendor that supports generating temporary credentials using
Alibaba Cloud STS */
+public class OssCredentialsStorageIntegration
+ extends InMemoryStorageIntegration<OssStorageConfigurationInfo> {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(OssCredentialsStorageIntegration.class);
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private final OssStsClientProvider stsClientProvider;
+
+ public OssCredentialsStorageIntegration() {
+ this(new OssStsClientProvider());
+ }
+
+
+ public OssCredentialsStorageIntegration(OssStsClientProvider
stsClientProvider) {
+ super(OssCredentialsStorageIntegration.class.getName());
+ this.stsClientProvider = stsClientProvider;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
Review Comment:
Is there an emulator for Aliyun OSS services to be used in CI?
##########
polaris-core/src/main/java/org/apache/polaris/core/storage/oss/OssCredentialsStorageIntegration.java:
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.storage.oss;
+
+import static
org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS;
+
+import com.aliyuncs.AcsRequest;
+import com.aliyuncs.AcsResponse;
+import com.aliyuncs.DefaultAcsClient;
+import com.aliyuncs.IAcsClient;
+import com.aliyuncs.profile.DefaultProfile;
+import com.aliyuncs.auth.sts.AssumeRoleRequest;
+import com.aliyuncs.auth.sts.AssumeRoleResponse;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.annotation.Nonnull;
+import java.net.URI;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.storage.InMemoryStorageIntegration;
+import org.apache.polaris.core.storage.PolarisStorageActions;
+import org.apache.polaris.core.storage.StorageAccessProperty;
+import org.apache.polaris.core.storage.StorageUtil;
+import org.apache.polaris.core.storage.oss.OssStsClientProvider.StsDestination;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** OSS credential vendor that supports generating temporary credentials using
Alibaba Cloud STS */
+public class OssCredentialsStorageIntegration
+ extends InMemoryStorageIntegration<OssStorageConfigurationInfo> {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(OssCredentialsStorageIntegration.class);
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private final OssStsClientProvider stsClientProvider;
+
+ public OssCredentialsStorageIntegration() {
+ this(new OssStsClientProvider());
+ }
+
+
+ public OssCredentialsStorageIntegration(OssStsClientProvider
stsClientProvider) {
+ super(OssCredentialsStorageIntegration.class.getName());
+ this.stsClientProvider = stsClientProvider;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
+ @Nonnull CallContext callContext,
+ @Nonnull OssStorageConfigurationInfo storageConfig,
+ boolean allowListOperation,
+ @Nonnull Set<String> allowedReadLocations,
+ @Nonnull Set<String> allowedWriteLocations) {
+
+ int storageCredentialDurationSeconds =
+
callContext.getRealmConfig().getConfig(STORAGE_CREDENTIAL_DURATION_SECONDS);
+
+ EnumMap<StorageAccessProperty, String> credentialMap =
+ new EnumMap<>(StorageAccessProperty.class);
+
+ try {
+ // Generate role session name
+ String roleSessionName = "PolarisOssCredentialsStorageIntegration-" +
System.currentTimeMillis();
+
+ // Generate RAM policy for scoped access
+ String policy = generateOssPolicy(allowListOperation,
allowedReadLocations, allowedWriteLocations);
+
+ // Use STS client provider to assume role directly
+ AssumeRoleResponse response = stsClientProvider.assumeRole(
+ storageConfig.getRoleArn(),
+ roleSessionName,
+ storageConfig.getExternalId(),
+ storageConfig.getRegion(),
+ storageConfig.getStsEndpointUri(),
+ policy,
+ (long) storageCredentialDurationSeconds);
+
+ AssumeRoleResponse.Credentials credentials = response.getCredentials();
+
+ // Populate credential map with STS response
+ credentialMap.put(StorageAccessProperty.OSS_ACCESS_KEY_ID,
credentials.getAccessKeyId());
+ credentialMap.put(StorageAccessProperty.OSS_ACCESS_KEY_SECRET,
credentials.getAccessKeySecret());
+ credentialMap.put(StorageAccessProperty.OSS_SECURITY_TOKEN,
credentials.getSecurityToken());
+
+ // 设置为OSS模式的特殊属性，帮助Apache Iceberg识别这是OSS凭证
+ credentialMap.put(StorageAccessProperty.CLIENT_REGION,
storageConfig.getRegion() != null ? storageConfig.getRegion() : "cn-hangzhou");
Review Comment:
`cn-hangzhou` is used as a default in multiple places. Could you make it a
constant + a utility method for resolving regions with defaults?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@polaris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
