This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new e65eb07ee [ #5625] feat(bundles) support Aliyun oss credential
provider (#5701)
e65eb07ee is described below
commit e65eb07eefd503e16287497bdb670612e7993a8d
Author: Xiaojian Sun <[email protected]>
AuthorDate: Thu Dec 5 15:52:21 2024 +0800
[ #5625] feat(bundles) support Aliyun oss credential provider (#5701)
### What changes were proposed in this pull request?
support Aliyun oss credential provider
### Why are the changes needed?
Fix: [# (5625)](https://github.com/apache/gravitino/issues/5625)
### How was this patch tested?
IcebergRESTOSSIT
---
.../gravitino/credential/OSSTokenCredential.java | 112 +++++++++
bundles/aliyun-bundle/build.gradle.kts | 10 +
.../gravitino/oss/credential/OSSTokenProvider.java | 259 +++++++++++++++++++++
.../gravitino/oss/credential/policy/Condition.java | 44 +++-
.../gravitino/oss/credential/policy/Effect.java | 14 +-
.../gravitino/oss/credential/policy/Policy.java | 73 ++++++
.../gravitino/oss/credential/policy/Statement.java | 103 ++++++++
.../oss/credential/policy/StringLike.java | 58 +++++
....apache.gravitino.credential.CredentialProvider | 19 ++
.../gravitino/credential/CredentialConstants.java | 3 +
.../apache/gravitino/storage/OSSProperties.java | 11 +-
.../credential/CredentialPropertyUtils.java | 15 +-
.../credential/TestCredentialPropertiesUtils.java | 17 ++
.../credential/config/OSSCredentialConfig.java | 110 +++++++++
docs/iceberg-rest-service.md | 22 +-
gradle/libs.versions.toml | 3 +
iceberg/iceberg-rest-server/build.gradle.kts | 1 +
.../iceberg/integration/test/IcebergRESTOSSIT.java | 135 +++++++++++
18 files changed, 977 insertions(+), 32 deletions(-)
diff --git
a/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java
b/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java
new file mode 100644
index 000000000..308d76a8d
--- /dev/null
+++ b/api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+
+/** OSS token credential. */
+public class OSSTokenCredential implements Credential {
+
+ /** OSS token credential type. */
+ public static final String OSS_TOKEN_CREDENTIAL_TYPE = "oss-token";
+ /** OSS access key ID used to access OSS data. */
+ public static final String GRAVITINO_OSS_SESSION_ACCESS_KEY_ID =
"oss-access-key-id";
+ /** OSS secret access key used to access OSS data. */
+ public static final String GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY =
"oss-secret-access-key";
+ /** OSS security token. */
+ public static final String GRAVITINO_OSS_TOKEN = "oss-security-token";
+
+ private final String accessKeyId;
+ private final String secretAccessKey;
+ private final String securityToken;
+ private final long expireTimeInMS;
+
+ /**
+ * Constructs an instance of {@link OSSTokenCredential} with secret key and
token.
+ *
+ * @param accessKeyId The oss access key ID.
+ * @param secretAccessKey The oss secret access key.
+ * @param securityToken The oss security token.
+ * @param expireTimeInMS The oss token expire time in ms.
+ */
+ public OSSTokenCredential(
+ String accessKeyId, String secretAccessKey, String securityToken, long
expireTimeInMS) {
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(accessKeyId), "OSS access key Id should not be
empty");
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(secretAccessKey), "OSS access key secret should
not be empty");
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(securityToken), "OSS security token should not
be empty");
+
+ this.accessKeyId = accessKeyId;
+ this.secretAccessKey = secretAccessKey;
+ this.securityToken = securityToken;
+ this.expireTimeInMS = expireTimeInMS;
+ }
+
+ @Override
+ public String credentialType() {
+ return OSS_TOKEN_CREDENTIAL_TYPE;
+ }
+
+ @Override
+ public long expireTimeInMs() {
+ return expireTimeInMS;
+ }
+
+ @Override
+ public Map<String, String> credentialInfo() {
+ return (new ImmutableMap.Builder<String, String>())
+ .put(GRAVITINO_OSS_SESSION_ACCESS_KEY_ID, accessKeyId)
+ .put(GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY, secretAccessKey)
+ .put(GRAVITINO_OSS_TOKEN, securityToken)
+ .build();
+ }
+
+ /**
+ * Get oss access key ID.
+ *
+ * @return The oss access key ID.
+ */
+ public String accessKeyId() {
+ return accessKeyId;
+ }
+
+ /**
+ * Get oss secret access key.
+ *
+ * @return The oss secret access key.
+ */
+ public String secretAccessKey() {
+ return secretAccessKey;
+ }
+
+ /**
+ * Get oss security token.
+ *
+ * @return The oss security token.
+ */
+ public String securityToken() {
+ return securityToken;
+ }
+}
diff --git a/bundles/aliyun-bundle/build.gradle.kts
b/bundles/aliyun-bundle/build.gradle.kts
index e0d8e9e15..e803c517b 100644
--- a/bundles/aliyun-bundle/build.gradle.kts
+++ b/bundles/aliyun-bundle/build.gradle.kts
@@ -25,16 +25,26 @@ plugins {
}
dependencies {
+ compileOnly(project(":api"))
+ compileOnly(project(":core"))
+ compileOnly(project(":catalogs:catalog-common"))
compileOnly(project(":catalogs:catalog-hadoop"))
compileOnly(libs.hadoop3.common)
+
+ implementation(libs.aliyun.credentials.sdk)
implementation(libs.hadoop3.oss)
+ // Aliyun oss SDK depends on this package, and JDK >= 9 requires manual add
+ //
https://www.alibabacloud.com/help/en/oss/developer-reference/java-installation?spm=a2c63.p38356.0.i1
+ implementation(libs.sun.activation)
+
// oss needs StringUtils from commons-lang3 or the following error will
occur in 3.3.0
// java.lang.NoClassDefFoundError: org/apache/commons/lang3/StringUtils
//
org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111)
//
org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323)
// org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611)
implementation(libs.commons.lang3)
+
implementation(project(":catalogs:catalog-common")) {
exclude("*")
}
diff --git
a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java
new file mode 100644
index 000000000..04ef0022a
--- /dev/null
+++
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.oss.credential;
+
+import com.aliyun.credentials.Client;
+import com.aliyun.credentials.models.Config;
+import com.aliyun.credentials.models.CredentialModel;
+import com.aliyun.credentials.utils.AuthConstant;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.credential.Credential;
+import org.apache.gravitino.credential.CredentialContext;
+import org.apache.gravitino.credential.CredentialProvider;
+import org.apache.gravitino.credential.OSSTokenCredential;
+import org.apache.gravitino.credential.PathBasedCredentialContext;
+import org.apache.gravitino.credential.config.OSSCredentialConfig;
+import org.apache.gravitino.oss.credential.policy.Condition;
+import org.apache.gravitino.oss.credential.policy.Effect;
+import org.apache.gravitino.oss.credential.policy.Policy;
+import org.apache.gravitino.oss.credential.policy.Statement;
+import org.apache.gravitino.oss.credential.policy.StringLike;
+
+/** Generates OSS token to access OSS data. */
+public class OSSTokenProvider implements CredentialProvider {
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private String accessKeyId;
+ private String secretAccessKey;
+ private String roleArn;
+ private String externalID;
+ private int tokenExpireSecs;
+ private String region;
+
+ /**
+ * Initializes the credential provider with catalog properties.
+ *
+ * @param properties catalog properties that can be used to configure the
provider. The specific
+ * properties required vary by implementation.
+ */
+ @Override
+ public void initialize(Map<String, String> properties) {
+ OSSCredentialConfig credentialConfig = new OSSCredentialConfig(properties);
+ this.roleArn = credentialConfig.ossRoleArn();
+ this.externalID = credentialConfig.externalID();
+ this.tokenExpireSecs = credentialConfig.tokenExpireInSecs();
+ this.accessKeyId = credentialConfig.accessKeyID();
+ this.secretAccessKey = credentialConfig.secretAccessKey();
+ this.region = credentialConfig.region();
+ }
+
+ /**
+ * Returns the type of credential, it should be identical in Gravitino.
+ *
+ * @return A string identifying the type of credentials.
+ */
+ @Override
+ public String credentialType() {
+ return OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE;
+ }
+
+ /**
+ * Obtains a credential based on the provided context information.
+ *
+ * @param context A context object providing necessary information for
retrieving credentials.
+ * @return A Credential object containing the authentication information
needed to access a system
+ * or resource. Null will be returned if no credential is available.
+ */
+ @Nullable
+ @Override
+ public Credential getCredential(CredentialContext context) {
+ if (!(context instanceof PathBasedCredentialContext)) {
+ return null;
+ }
+ PathBasedCredentialContext pathBasedCredentialContext =
(PathBasedCredentialContext) context;
+ CredentialModel credentialModel =
+ createOSSCredentialModel(
+ roleArn,
+ pathBasedCredentialContext.getReadPaths(),
+ pathBasedCredentialContext.getWritePaths(),
+ pathBasedCredentialContext.getUserName());
+ return new OSSTokenCredential(
+ credentialModel.accessKeyId,
+ credentialModel.accessKeySecret,
+ credentialModel.securityToken,
+ credentialModel.expiration);
+ }
+
+ private CredentialModel createOSSCredentialModel(
+ String roleArn, Set<String> readLocations, Set<String> writeLocations,
String userName) {
+ Config config = new Config();
+ config.setAccessKeyId(accessKeyId);
+ config.setAccessKeySecret(secretAccessKey);
+ config.setType(AuthConstant.RAM_ROLE_ARN);
+ config.setRoleArn(roleArn);
+ config.setRoleSessionName(getRoleName(userName));
+ if (StringUtils.isNotBlank(externalID)) {
+ config.setExternalId(externalID);
+ }
+ config.setRoleSessionExpiration(tokenExpireSecs);
+ config.setPolicy(createPolicy(readLocations, writeLocations));
+ // Local object and client is a simple proxy that does not require manual
release
+ Client client = new Client(config);
+ return client.getCredential();
+ }
+
+ // reference:
+ //
https://www.alibabacloud.com/help/en/oss/user-guide/tutorial-use-ram-policies-to-control-access-to-oss?spm=a2c63.p38356.help-menu-31815.d_2_4_5_1.5536471b56XPRQ
+ private String createPolicy(Set<String> readLocations, Set<String>
writeLocations) {
+ Policy.Builder policyBuilder = Policy.builder().version("1");
+
+ // Allow read and write access to the specified locations
+ Statement.Builder allowGetObjectStatementBuilder =
+ Statement.builder()
+ .effect(Effect.ALLOW)
+ .addAction("oss:GetObject")
+ .addAction("oss:GetObjectVersion");
+ // Add support for bucket-level policies
+ Map<String, Statement.Builder> bucketListStatementBuilder = new
HashMap<>();
+ Map<String, Statement.Builder> bucketGetLocationStatementBuilder = new
HashMap<>();
+
+ String arnPrefix = getArnPrefix();
+ Stream.concat(readLocations.stream(), writeLocations.stream())
+ .distinct()
+ .forEach(
+ location -> {
+ URI uri = URI.create(location);
+
allowGetObjectStatementBuilder.addResource(getOssUriWithArn(arnPrefix, uri));
+ String bucketArn = arnPrefix + getBucketName(uri);
+ // ListBucket
+ bucketListStatementBuilder.computeIfAbsent(
+ bucketArn,
+ key ->
+ Statement.builder()
+ .effect(Effect.ALLOW)
+ .addAction("oss:ListBucket")
+ .addResource(key)
+ .condition(getCondition(uri)));
+ // GetBucketLocation
+ bucketGetLocationStatementBuilder.computeIfAbsent(
+ bucketArn,
+ key ->
+ Statement.builder()
+ .effect(Effect.ALLOW)
+ .addAction("oss:GetBucketLocation")
+ .addResource(key));
+ });
+
+ if (!writeLocations.isEmpty()) {
+ Statement.Builder allowPutObjectStatementBuilder =
+ Statement.builder()
+ .effect(Effect.ALLOW)
+ .addAction("oss:PutObject")
+ .addAction("oss:DeleteObject");
+ writeLocations.forEach(
+ location -> {
+ URI uri = URI.create(location);
+
allowPutObjectStatementBuilder.addResource(getOssUriWithArn(arnPrefix, uri));
+ });
+ policyBuilder.addStatement(allowPutObjectStatementBuilder.build());
+ }
+
+ if (!bucketListStatementBuilder.isEmpty()) {
+ bucketListStatementBuilder
+ .values()
+ .forEach(statementBuilder ->
policyBuilder.addStatement(statementBuilder.build()));
+ } else {
+ // add list privilege with 0 resources
+ policyBuilder.addStatement(
+
Statement.builder().effect(Effect.ALLOW).addAction("oss:ListBucket").build());
+ }
+ bucketGetLocationStatementBuilder
+ .values()
+ .forEach(statementBuilder ->
policyBuilder.addStatement(statementBuilder.build()));
+
+ policyBuilder.addStatement(allowGetObjectStatementBuilder.build());
+ try {
+ return objectMapper.writeValueAsString(policyBuilder.build());
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Condition getCondition(URI uri) {
+ return Condition.builder()
+ .stringLike(
+ StringLike.builder()
+ .addPrefix(concatPathWithSep(trimLeadingSlash(uri.getPath()),
"*", "/"))
+ .build())
+ .build();
+ }
+
+ private String getArnPrefix() {
+ if (StringUtils.isNotEmpty(region)) {
+ return "acs:oss:" + region + ":*:";
+ }
+ return "acs:oss:*:*:";
+ }
+
+ private String getBucketName(URI uri) {
+ return uri.getHost();
+ }
+
+ private String getOssUriWithArn(String arnPrefix, URI uri) {
+ return arnPrefix + concatPathWithSep(removeSchemaFromOSSUri(uri), "*",
"/");
+ }
+
+ private static String concatPathWithSep(String leftPath, String rightPath,
String fileSep) {
+ if (leftPath.endsWith(fileSep) && rightPath.startsWith(fileSep)) {
+ return leftPath + rightPath.substring(1);
+ } else if (!leftPath.endsWith(fileSep) && !rightPath.startsWith(fileSep)) {
+ return leftPath + fileSep + rightPath;
+ } else {
+ return leftPath + rightPath;
+ }
+ }
+
+ // Transform 'oss://bucket/path' to /bucket/path
+ private String removeSchemaFromOSSUri(URI uri) {
+ String bucket = uri.getHost();
+ String path = trimLeadingSlash(uri.getPath());
+ return String.join(
+ "/", Stream.of(bucket,
path).filter(Objects::nonNull).toArray(String[]::new));
+ }
+
+ private String trimLeadingSlash(String path) {
+ return path.startsWith("/") ? path.substring(1) : path;
+ }
+
+ private String getRoleName(String userName) {
+ return "gravitino_" + userName;
+ }
+
+ @Override
+ public void close() throws IOException {}
+}
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java
similarity index 50%
copy from
catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
copy to
bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java
index 3885eb360..a2b20612c 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
+++
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java
@@ -16,17 +16,41 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.gravitino.storage;
-// Defines the unified OSS properties for different catalogs and connectors.
-public class OSSProperties {
+package org.apache.gravitino.oss.credential.policy;
- // The endpoint of Aliyun OSS service.
- public static final String GRAVITINO_OSS_ENDPOINT = "oss-endpoint";
- // The static access key ID used to access OSS data.
- public static final String GRAVITINO_OSS_ACCESS_KEY_ID = "oss-access-key-id";
- // The static access key secret used to access OSS data.
- public static final String GRAVITINO_OSS_ACCESS_KEY_SECRET =
"oss-secret-access-key";
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
- private OSSProperties() {}
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Condition {
+
+ @JsonProperty("StringLike")
+ private StringLike stringLike;
+
+ private Condition(Builder builder) {
+ this.stringLike = builder.stringLike;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private StringLike stringLike;
+
+ public Builder stringLike(StringLike stringLike) {
+ this.stringLike = stringLike;
+ return this;
+ }
+
+ public Condition build() {
+ return new Condition(this);
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public StringLike getStringLike() {
+ return stringLike;
+ }
}
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java
similarity index 58%
copy from
catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
copy to
bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java
index 3885eb360..5906cb13c 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
+++
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java
@@ -16,17 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.gravitino.storage;
-// Defines the unified OSS properties for different catalogs and connectors.
-public class OSSProperties {
+package org.apache.gravitino.oss.credential.policy;
- // The endpoint of Aliyun OSS service.
- public static final String GRAVITINO_OSS_ENDPOINT = "oss-endpoint";
- // The static access key ID used to access OSS data.
- public static final String GRAVITINO_OSS_ACCESS_KEY_ID = "oss-access-key-id";
- // The static access key secret used to access OSS data.
- public static final String GRAVITINO_OSS_ACCESS_KEY_SECRET =
"oss-secret-access-key";
-
- private OSSProperties() {}
+public class Effect {
+ public static final String ALLOW = "Allow";
}
diff --git
a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java
new file mode 100644
index 000000000..4c8c0f5ba
--- /dev/null
+++
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.oss.credential.policy;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.ArrayList;
+import java.util.List;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Policy {
+
+ @JsonProperty("Version")
+ private String version;
+
+ @JsonProperty("Statement")
+ private List<Statement> statements;
+
+ private Policy(Builder builder) {
+ this.version = builder.version;
+ this.statements = builder.statements;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private String version;
+ private List<Statement> statements = new ArrayList<>();
+
+ public Builder version(String version) {
+ this.version = version;
+ return this;
+ }
+
+ public Builder addStatement(Statement statement) {
+ this.statements.add(statement);
+ return this;
+ }
+
+ public Policy build() {
+ return new Policy(this);
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public String getVersion() {
+ return version;
+ }
+
+ @SuppressWarnings("unused")
+ public List<Statement> getStatements() {
+ return statements;
+ }
+}
diff --git
a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java
new file mode 100644
index 000000000..2dca6e74f
--- /dev/null
+++
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.oss.credential.policy;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.ArrayList;
+import java.util.List;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Statement {
+
+ @JsonProperty("Effect")
+ private String effect;
+
+ @JsonProperty("Action")
+ private List<String> actions;
+
+ @JsonProperty("Resource")
+ private List<String> resources;
+
+ @JsonProperty("Condition")
+ private Condition condition;
+
+ private Statement(Builder builder) {
+ this.effect = builder.effect;
+ this.actions = builder.actions;
+ this.resources = builder.resources;
+ this.condition = builder.condition;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private String effect;
+ private List<String> actions = new ArrayList<>();
+ private List<String> resources = new ArrayList<>();
+ private Condition condition;
+
+ public Builder effect(String effect) {
+ this.effect = effect;
+ return this;
+ }
+
+ public Builder addAction(String action) {
+ this.actions.add(action);
+ return this;
+ }
+
+ public Builder addResource(String resource) {
+ this.resources.add(resource);
+ return this;
+ }
+
+ public Builder condition(Condition condition) {
+ this.condition = condition;
+ return this;
+ }
+
+ public Statement build() {
+ return new Statement(this);
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public String getEffect() {
+ return effect;
+ }
+
+ @SuppressWarnings("unused")
+ public List<String> getActions() {
+ return actions;
+ }
+
+ @SuppressWarnings("unused")
+ public List<String> getResources() {
+ return resources;
+ }
+
+ @SuppressWarnings("unused")
+ public Condition getCondition() {
+ return condition;
+ }
+}
diff --git
a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java
new file mode 100644
index 000000000..7438907bd
--- /dev/null
+++
b/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.oss.credential.policy;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.ArrayList;
+import java.util.List;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class StringLike {
+
+ @JsonProperty("oss:Prefix")
+ private List<String> prefix;
+
+ private StringLike(Builder builder) {
+ this.prefix = builder.prefix;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private List<String> prefix = new ArrayList<>();
+
+ public Builder addPrefix(String prefix) {
+ this.prefix.add(prefix);
+ return this;
+ }
+
+ public StringLike build() {
+ return new StringLike(this);
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public List<String> getPrefix() {
+ return prefix;
+ }
+}
diff --git
a/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
b/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
new file mode 100644
index 000000000..d2f4be51b
--- /dev/null
+++
b/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+org.apache.gravitino.oss.credential.OSSTokenProvider
\ No newline at end of file
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
index 665a78ee7..739cd0139 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
@@ -26,5 +26,8 @@ public class CredentialConstants {
public static final String GCS_TOKEN_CREDENTIAL_PROVIDER_TYPE = "gcs-token";
+ public static final String OSS_TOKEN_CREDENTIAL_PROVIDER = "oss-token";
+ public static final String OSS_TOKEN_EXPIRE_IN_SECS =
"oss-token-expire-in-secs";
+
private CredentialConstants() {}
}
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
index 3885eb360..8471682ae 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
@@ -18,15 +18,22 @@
*/
package org.apache.gravitino.storage;
-// Defines the unified OSS properties for different catalogs and connectors.
+// Properties for OSS.
public class OSSProperties {
- // The endpoint of Aliyun OSS service.
+ // The region of Aliyun OSS.
+ public static final String GRAVITINO_OSS_REGION = "oss-region";
+ // The endpoint of Aliyun OSS.
public static final String GRAVITINO_OSS_ENDPOINT = "oss-endpoint";
// The static access key ID used to access OSS data.
public static final String GRAVITINO_OSS_ACCESS_KEY_ID = "oss-access-key-id";
// The static access key secret used to access OSS data.
public static final String GRAVITINO_OSS_ACCESS_KEY_SECRET =
"oss-secret-access-key";
+ // OSS role arn
+ public static final String GRAVITINO_OSS_ROLE_ARN = "oss-role-arn";
+ // OSS external id
+ public static final String GRAVITINO_OSS_EXTERNAL_ID = "oss-external-id";
+
private OSSProperties() {}
}
diff --git
a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
index 811f04c53..8f56f802d 100644
---
a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
+++
b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
@@ -34,6 +34,10 @@ public class CredentialPropertyUtils {
@VisibleForTesting static final String ICEBERG_S3_TOKEN = "s3.session-token";
@VisibleForTesting static final String ICEBERG_GCS_TOKEN =
"gcs.oauth2.token";
+ @VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_ID =
"client.access-key-id";
+ @VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_SECRET =
"client.access-key-secret";
+ @VisibleForTesting static final String ICEBERG_OSS_SECURITY_TOKEN =
"client.security-token";
+
private static Map<String, String> icebergCredentialPropertyMap =
ImmutableMap.of(
GCSTokenCredential.GCS_TOKEN_NAME,
@@ -43,7 +47,13 @@ public class CredentialPropertyUtils {
S3SecretKeyCredential.GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY,
ICEBERG_S3_SECRET_ACCESS_KEY,
S3TokenCredential.GRAVITINO_S3_TOKEN,
- ICEBERG_S3_TOKEN);
+ ICEBERG_S3_TOKEN,
+ OSSTokenCredential.GRAVITINO_OSS_TOKEN,
+ ICEBERG_OSS_SECURITY_TOKEN,
+ OSSTokenCredential.GRAVITINO_OSS_SESSION_ACCESS_KEY_ID,
+ ICEBERG_OSS_ACCESS_KEY_ID,
+ OSSTokenCredential.GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY,
+ ICEBERG_OSS_ACCESS_KEY_SECRET);
/**
* Transforms a specific credential into a map of Iceberg properties.
@@ -62,6 +72,9 @@ public class CredentialPropertyUtils {
if (credential instanceof S3TokenCredential || credential instanceof
S3SecretKeyCredential) {
return transformProperties(credential.credentialInfo(),
icebergCredentialPropertyMap);
}
+ if (credential instanceof OSSTokenCredential) {
+ return transformProperties(credential.credentialInfo(),
icebergCredentialPropertyMap);
+ }
return credential.toProperties();
}
diff --git
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
index 4d3ad698b..3b4571e01 100644
---
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
+++
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
@@ -51,4 +51,21 @@ public class TestCredentialPropertiesUtils {
"secret");
Assertions.assertEquals(expectedProperties, icebergProperties);
}
+
+ @Test
+ void testToIcebergPropertiesForOSS() {
+ OSSTokenCredential ossTokenCredential =
+ new OSSTokenCredential("key", "secret", "security-token", 0);
+ Map<String, String> icebergProperties =
+ CredentialPropertyUtils.toIcebergProperties(ossTokenCredential);
+ Map<String, String> expectedProperties =
+ ImmutableMap.of(
+ CredentialPropertyUtils.ICEBERG_OSS_ACCESS_KEY_ID,
+ "key",
+ CredentialPropertyUtils.ICEBERG_OSS_ACCESS_KEY_SECRET,
+ "secret",
+ CredentialPropertyUtils.ICEBERG_OSS_SECURITY_TOKEN,
+ "security-token");
+ Assertions.assertEquals(expectedProperties, icebergProperties);
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/config/OSSCredentialConfig.java
b/core/src/main/java/org/apache/gravitino/credential/config/OSSCredentialConfig.java
new file mode 100644
index 000000000..0422bb3b7
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/credential/config/OSSCredentialConfig.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential.config;
+
+import java.util.Map;
+import javax.validation.constraints.NotNull;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.Config;
+import org.apache.gravitino.config.ConfigBuilder;
+import org.apache.gravitino.config.ConfigConstants;
+import org.apache.gravitino.config.ConfigEntry;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.storage.OSSProperties;
+
+public class OSSCredentialConfig extends Config {
+
+ public static final ConfigEntry<String> OSS_REGION =
+ new ConfigBuilder(OSSProperties.GRAVITINO_OSS_REGION)
+ .doc("The region of the OSS service")
+ .version(ConfigConstants.VERSION_0_8_0)
+ .stringConf()
+ .create();
+
+ public static final ConfigEntry<String> OSS_ACCESS_KEY_ID =
+ new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID)
+ .doc("The static access key ID used to access OSS data")
+ .version(ConfigConstants.VERSION_0_8_0)
+ .stringConf()
+ .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
+ .create();
+
+ public static final ConfigEntry<String> OSS_SECRET_ACCESS_KEY =
+ new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET)
+ .doc("The static secret access key used to access OSS data")
+ .version(ConfigConstants.VERSION_0_8_0)
+ .stringConf()
+ .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
+ .create();
+
+ public static final ConfigEntry<String> OSS_ROLE_ARN =
+ new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ROLE_ARN)
+ .doc("OSS role arn")
+ .version(ConfigConstants.VERSION_0_8_0)
+ .stringConf()
+ .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
+ .create();
+
+ public static final ConfigEntry<String> OSS_EXTERNAL_ID =
+ new ConfigBuilder(OSSProperties.GRAVITINO_OSS_EXTERNAL_ID)
+ .doc("OSS external ID")
+ .version(ConfigConstants.VERSION_0_8_0)
+ .stringConf()
+ .create();
+
+ public static final ConfigEntry<Integer> OSS_TOKEN_EXPIRE_IN_SECS =
+ new ConfigBuilder(CredentialConstants.OSS_TOKEN_EXPIRE_IN_SECS)
+ .doc("OSS token expire in seconds")
+ .version(ConfigConstants.VERSION_0_8_0)
+ .intConf()
+ .createWithDefault(3600);
+
+ public OSSCredentialConfig(Map<String, String> properties) {
+ super(false);
+ loadFromMap(properties, k -> true);
+ }
+
+ public String region() {
+ return this.get(OSS_REGION);
+ }
+
+ @NotNull
+ public String ossRoleArn() {
+ return this.get(OSS_ROLE_ARN);
+ }
+
+ @NotNull
+ public String accessKeyID() {
+ return this.get(OSS_ACCESS_KEY_ID);
+ }
+
+ @NotNull
+ public String secretAccessKey() {
+ return this.get(OSS_SECRET_ACCESS_KEY);
+ }
+
+ public String externalID() {
+ return this.get(OSS_EXTERNAL_ID);
+ }
+
+ public Integer tokenExpireInSecs() {
+ return this.get(OSS_TOKEN_EXPIRE_IN_SECS);
+ }
+}
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index 20fc4221d..4f626d314 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -17,7 +17,7 @@ The Apache Gravitino Iceberg REST Server follows the [Apache
Iceberg REST API sp
- multi table transaction
- pagination
- Works as a catalog proxy, supporting `Hive` and `JDBC` as catalog backend.
-- Supports credential vending for `S3` and `GCS`.
+- Supports credential vending for `S3`、`GCS` and `OSS`.
- Supports different storages like `S3`, `HDFS`, `OSS`, `GCS` and provides the
capability to support other storages.
- Supports event listener.
- Supports Audit log.
@@ -128,17 +128,23 @@ To configure the JDBC catalog backend, set the
`gravitino.iceberg-rest.warehouse
#### OSS configuration
-Gravitino Iceberg REST service supports using static access-key-id and
secret-access-key to access OSS data.
+Gravitino Iceberg REST service supports using static access-key-id and
secret-access-key or generating temporary token to access OSS data.
-| Configuration item | Description
|
Default value | Required | Since Version |
-|------------------------------------------------|-------------------------------------------------------------------------------------------------------|---------------|----------|------------------|
-| `gravitino.iceberg-rest.io-impl` | The IO implementation for
`FileIO` in Iceberg, use `org.apache.iceberg.aliyun.oss.OSSFileIO` for OSS. |
(none) | No | 0.6.0-incubating |
-| `gravitino.iceberg-rest.oss-access-key-id` | The static access key ID
used to access OSS data. |
(none) | No | 0.7.0-incubating |
-| `gravitino.iceberg-rest.oss-secret-access-key` | The static secret access
key used to access OSS data. |
(none) | No | 0.7.0-incubating |
-| `gravitino.iceberg-rest.oss-endpoint` | The endpoint of Aliyun OSS
service. |
(none) | No | 0.7.0-incubating |
+| Configuration item | Description
| Default value | Required | Since Version |
+|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------|-----------------|----------|------------------|
+| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.aliyun.oss.OSSFileIO` for OSS.
| (none) | No | 0.6.0-incubating |
+| `gravitino.iceberg-rest.oss-access-key-id` | The static access key ID
used to access OSS data.
| (none) | No | 0.7.0-incubating |
+| `gravitino.iceberg-rest.oss-secret-access-key` | The static secret access
key used to access OSS data.
| (none) | No | 0.7.0-incubating |
+| `gravitino.iceberg-rest.oss-endpoint` | The endpoint of Aliyun
OSS service.
| (none) | No | 0.7.0-incubating |
+| `gravitino.iceberg-rest.oss-region` | The region of the OSS
service, like `oss-cn-hangzhou`, only used when `credential-provider-type` is
`oss-token`. | (none) | No | 0.8.0-incubating |
+| `gravitino.iceberg-rest.oss-role-arn` | The ARN of the role to
access the OSS data, only used when `credential-provider-type` is `oss-token`.
| (none) | No | 0.8.0-incubating |
+| `gravitino.iceberg-rest.oss-external-id` | The OSS external id to
generate token, only used when `credential-provider-type` is `oss-token`.
| (none) | No | 0.8.0-incubating |
+| `gravitino.iceberg-rest.oss-token-expire-in-secs` | The OSS security token
expire time in secs, only used when `credential-provider-type` is `oss-token`.
| 3600 | No | 0.8.0-incubating |
For other Iceberg OSS properties not managed by Gravitino like
`client.security-token`, you could config it directly by
`gravitino.iceberg-rest.client.security-token`.
+If you set `credential-provider-type` explicitly, please downloading
[Gravitino Aliyun bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/aliyun-bundle),
and place it to the classpath of Iceberg REST server.
+
:::info
Please set the `gravitino.iceberg-rest.warehouse` parameter to
`oss://{bucket_name}/${prefix_name}`. Additionally, download the [Aliyun OSS
SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and
copy `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` in
the classpath of Iceberg REST server, `iceberg-rest-server/libs` for the
auxiliary server, `libs` for the standalone server.
:::
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index f1f29cf18..88636c5a5 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -113,6 +113,7 @@ datanucleus-rdbms = "4.1.19"
datanucleus-jdo = "3.2.0-m3"
hudi = "0.15.0"
google-auth = "1.28.0"
+aliyun-credentials = "0.3.12"
[libraries]
aws-iam = { group = "software.amazon.awssdk", name = "iam", version.ref =
"awssdk" }
@@ -266,6 +267,8 @@ jettison = { group = "org.codehaus.jettison", name =
"jettison", version.ref = "
google-auth-http = { group = "com.google.auth", name =
"google-auth-library-oauth2-http", version.ref = "google-auth" }
google-auth-credentials = { group = "com.google.auth", name =
"google-auth-library-credentials", version.ref = "google-auth" }
+aliyun-credentials-sdk = { group='com.aliyun', name='credentials-java',
version.ref='aliyun-credentials' }
+
[bundles]
log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core",
"log4j-12-api", "log4j-layout-template-json"]
jetty = ["jetty-server", "jetty-servlet", "jetty-webapp", "jetty-servlets"]
diff --git a/iceberg/iceberg-rest-server/build.gradle.kts
b/iceberg/iceberg-rest-server/build.gradle.kts
index 5d84ef778..e46193c97 100644
--- a/iceberg/iceberg-rest-server/build.gradle.kts
+++ b/iceberg/iceberg-rest-server/build.gradle.kts
@@ -62,6 +62,7 @@ dependencies {
annotationProcessor(libs.lombok)
compileOnly(libs.lombok)
+ testImplementation(project(":bundles:aliyun-bundle"))
testImplementation(project(":bundles:aws-bundle"))
testImplementation(project(":bundles:gcp-bundle", configuration = "shadow"))
testImplementation(project(":integration-test-common", "testArtifacts"))
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
new file mode 100644
index 000000000..fb6bac65b
--- /dev/null
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.iceberg.integration.test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.iceberg.common.IcebergConfig;
+import org.apache.gravitino.integration.test.util.BaseIT;
+import org.apache.gravitino.integration.test.util.DownloaderUtils;
+import org.apache.gravitino.integration.test.util.ITUtils;
+import org.apache.gravitino.storage.OSSProperties;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import org.junit.platform.commons.util.StringUtils;
+
+@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches =
"true")
+public class IcebergRESTOSSIT extends IcebergRESTJdbcCatalogIT {
+
+ private String warehouse;
+ private String accessKey;
+ private String secretKey;
+ private String endpoint;
+ private String roleArn;
+ private String externalId;
+ private String region;
+
+ @Override
+ void initEnv() {
+ this.warehouse =
+ String.format(
+ "oss://%s/gravitino-test",
+ getFromEnvOrDefault("GRAVITINO_OSS_BUCKET", "{BUCKET_NAME}"));
+ this.accessKey = getFromEnvOrDefault("GRAVITINO_OSS_ACCESS_KEY",
"{ACCESS_KEY}");
+ this.secretKey = getFromEnvOrDefault("GRAVITINO_OSS_SECRET_KEY",
"{SECRET_KEY}");
+ this.endpoint = getFromEnvOrDefault("GRAVITINO_OSS_ENDPOINT",
"{GRAVITINO_OSS_ENDPOINT}");
+ this.region = getFromEnvOrDefault("GRAVITINO_OSS_REGION",
"oss-cn-hangzhou");
+ this.roleArn = getFromEnvOrDefault("GRAVITINO_OSS_ROLE_ARN", "{ROLE_ARN}");
+ this.externalId = getFromEnvOrDefault("GRAVITINO_OSS_EXTERNAL_ID", "");
+
+ if (ITUtils.isEmbedded()) {
+ return;
+ }
+ try {
+ downloadIcebergForAliyunJar();
+ } catch (IOException e) {
+ LOG.warn("Download Iceberg Aliyun bundle jar failed,", e);
+ throw new RuntimeException(e);
+ }
+ copyAliyunOSSJar();
+ }
+
+ @Override
+ public Map<String, String> getCatalogConfig() {
+ HashMap m = new HashMap<String, String>();
+ m.putAll(getCatalogJdbcConfig());
+ m.putAll(getOSSConfig());
+ return m;
+ }
+
+ public boolean supportsCredentialVending() {
+ return true;
+ }
+
+ private Map<String, String> getOSSConfig() {
+ Map configMap = new HashMap<String, String>();
+
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
+ CredentialConstants.OSS_TOKEN_CREDENTIAL_PROVIDER);
+ configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX +
OSSProperties.GRAVITINO_OSS_REGION, region);
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
OSSProperties.GRAVITINO_OSS_ENDPOINT, endpoint);
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, accessKey);
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET,
+ secretKey);
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
OSSProperties.GRAVITINO_OSS_ROLE_ARN, roleArn);
+ if (StringUtils.isNotBlank(externalId)) {
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
OSSProperties.GRAVITINO_OSS_EXTERNAL_ID,
+ externalId);
+ }
+
+ configMap.put(
+ IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.IO_IMPL,
+ "org.apache.iceberg.aliyun.oss.OSSFileIO");
+ configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX +
IcebergConstants.WAREHOUSE, warehouse);
+
+ return configMap;
+ }
+
+ private void downloadIcebergForAliyunJar() throws IOException {
+ String icebergBundleJarName = "iceberg-aliyun-1.5.2.jar";
+ String icebergBundleJarUri =
+ "https://repo1.maven.org/maven2/org/apache/iceberg/";
+ + "iceberg-aliyun/1.5.2/"
+ + icebergBundleJarName;
+ String gravitinoHome = System.getenv("GRAVITINO_HOME");
+ String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
+ DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
+ }
+
+ private void copyAliyunOSSJar() {
+ String gravitinoHome = System.getenv("GRAVITINO_HOME");
+ String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
+ BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir);
+ }
+
+ private String getFromEnvOrDefault(String envVar, String defaultValue) {
+ String envValue = System.getenv(envVar);
+ return Optional.ofNullable(envValue).orElse(defaultValue);
+ }
+}
