This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 528d0eacac [#6024] feat(iceberg): refactor Iceberg credential code to
reuse credential component in Gravitino server (#6021)
528d0eacac is described below
commit 528d0eacac536569ff9109f70c4ca3af0284bccd
Author: FANNG <[email protected]>
AuthorDate: Mon Dec 30 14:08:26 2024 +0800
[#6024] feat(iceberg): refactor Iceberg credential code to reuse credential
component in Gravitino server (#6021)
### What changes were proposed in this pull request?
1. reuse `CatalogCredentialManager` to manage the credentials in Iceberg
catalog.
2. use `CatalogWrapperForREST` to manage some REST specific operations
like credential vending.
3. depracate `catalog-provider-type` , use `catalog-providers` instead.
### Why are the changes needed?
Fix: #6024
### Does this PR introduce _any_ user-facing change?
yes, depracate `catalog-provider-type` , use `catalog-providers`, do
some compatibility work.
### How was this patch tested?
run with s3 token
---
.../gravitino/credential/CredentialConstants.java | 2 +-
.../test/FilesetCatalogCredentialIT.java | 3 -
.../credential/CatalogCredentialManager.java | 16 +++
.../credential/CredentialProviderManager.java | 73 ----------
.../gravitino/credential/CredentialUtils.java | 28 +---
.../credential/config/CredentialConfig.java | 10 ++
dev/docker/iceberg-rest-server/rewrite_config.py | 3 +-
docs/iceberg-rest-service.md | 99 +++++++------
.../gravitino/iceberg/common/IcebergConfig.java | 6 +-
.../iceberg/common/ops/IcebergCatalogWrapper.java | 28 +---
.../iceberg/service/CatalogWrapperForREST.java | 157 +++++++++++++++++++++
.../service/IcebergCatalogWrapperManager.java | 48 ++-----
.../dispatcher/IcebergTableOperationExecutor.java | 4 +-
.../service/rest/IcebergTableOperations.java | 77 +---------
.../listener/api/event/IcebergRequestContext.java | 25 +++-
.../integration/test/IcebergRESTADLSTokenIT.java | 2 +-
.../test/IcebergRESTAzureAccountKeyIT.java | 2 +-
.../iceberg/integration/test/IcebergRESTGCSIT.java | 2 +-
.../iceberg/integration/test/IcebergRESTOSSIT.java | 2 +-
.../integration/test/IcebergRESTOSSSecretIT.java | 2 +-
.../iceberg/integration/test/IcebergRESTS3IT.java | 2 +-
...> TestIcebergCatalogWrapperManagerForREST.java} | 2 +-
...pperForTest.java => CatalogWrapperForTest.java} | 8 +-
.../rest/IcebergCatalogWrapperManagerForTest.java | 7 +-
.../iceberg/service/rest/IcebergRestTestUtil.java | 3 +-
.../service/rest/MockIcebergTableOperations.java | 4 +-
26 files changed, 318 insertions(+), 297 deletions(-)
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
index 7d552deb6b..a6e0d54bfa 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
@@ -20,7 +20,7 @@
package org.apache.gravitino.credential;
public class CredentialConstants {
- public static final String CREDENTIAL_PROVIDER_TYPE =
"credential-provider-type";
+ @Deprecated public static final String CREDENTIAL_PROVIDER_TYPE =
"credential-provider-type";
public static final String CREDENTIAL_PROVIDERS = "credential-providers";
public static final String CREDENTIAL_CACHE_EXPIRE_RATIO =
"credential-cache-expire-ratio";
public static final String CREDENTIAL_CACHE_MAX_SIZE =
"credential-cache-max-size";
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java
index 94239fef28..3dc3ad82ae 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java
@@ -86,9 +86,6 @@ public class FilesetCatalogCredentialIT extends BaseIT {
S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE
+ ","
+ S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE);
- properties.put(
- CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
- S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE);
properties.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, S3_ACCESS_KEY);
properties.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, S3_SECRET_KEY);
properties.put(S3Properties.GRAVITINO_S3_ENDPOINT,
"s3.ap-southeast-2.amazonaws.com");
diff --git
a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java
b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java
index 0e407a399b..7fbead57a5 100644
---
a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java
+++
b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java
@@ -51,6 +51,17 @@ public class CatalogCredentialManager implements Closeable {
return credentialCache.getCredential(credentialCacheKey, cacheKey ->
doGetCredential(cacheKey));
}
+ // Get credential with only one credential provider.
+ public Credential getCredential(CredentialContext context) {
+ if (credentialProviders.size() == 0) {
+ throw new IllegalArgumentException("There are no credential provider for
the catalog.");
+ } else if (credentialProviders.size() > 1) {
+ throw new UnsupportedOperationException(
+ "There are multiple credential providers for the catalog.");
+ }
+ return getCredential(credentialProviders.keySet().iterator().next(),
context);
+ }
+
@Override
public void close() {
credentialProviders
@@ -67,6 +78,11 @@ public class CatalogCredentialManager implements Closeable {
e);
}
});
+ try {
+ credentialCache.close();
+ } catch (IOException e) {
+ LOG.warn("Close credential cache failed, catalog: {}", catalogName, e);
+ }
}
private Credential doGetCredential(CredentialCacheKey credentialCacheKey) {
diff --git
a/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java
b/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java
deleted file mode 100644
index b583bedcfd..0000000000
---
a/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.gravitino.credential;
-
-import com.google.common.base.Preconditions;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import javax.annotation.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CredentialProviderManager {
-
- private static final Logger LOG =
LoggerFactory.getLogger(CredentialProviderManager.class);
- private Map<String, CredentialProvider> credentialProviders;
-
- public CredentialProviderManager() {
- this.credentialProviders = new ConcurrentHashMap<>();
- }
-
- public void registerCredentialProvider(
- String catalogName, CredentialProvider credentialProvider) {
- CredentialProvider current = credentialProviders.putIfAbsent(catalogName,
credentialProvider);
- Preconditions.checkState(
- !credentialProvider.equals(current),
- String.format(
- "Should not register multiple times to CredentialProviderManager,
catalog: %s, "
- + "credential provider: %s",
- catalogName, credentialProvider.credentialType()));
- LOG.info(
- "Register catalog:%s credential provider:%s to
CredentialProviderManager",
- catalogName, credentialProvider.credentialType());
- }
-
- public void unregisterCredentialProvider(String catalogName) {
- CredentialProvider credentialProvider =
credentialProviders.remove(catalogName);
- // Not all catalog has credential provider
- if (credentialProvider != null) {
- LOG.info(
- "Unregister catalog:{} credential provider:{} to
CredentialProviderManager",
- catalogName,
- credentialProvider.credentialType());
- try {
- credentialProvider.close();
- } catch (IOException e) {
- LOG.warn("Close credential provider failed", e);
- }
- }
- }
-
- @Nullable
- public CredentialProvider getCredentialProvider(String catalogName) {
- return credentialProviders.get(catalogName);
- }
-}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java
b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java
index 9a202ec974..9d2ea43e66 100644
--- a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java
+++ b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java
@@ -19,30 +19,20 @@
package org.apache.gravitino.credential;
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableSet;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
-import org.apache.gravitino.utils.PrincipalUtils;
+import org.apache.gravitino.credential.config.CredentialConfig;
public class CredentialUtils {
- private static final Splitter splitter = Splitter.on(",");
-
- public static Credential vendCredential(CredentialProvider
credentialProvider, String[] path) {
- PathBasedCredentialContext pathBasedCredentialContext =
- new PathBasedCredentialContext(
- PrincipalUtils.getCurrentUserName(), ImmutableSet.copyOf(path),
Collections.emptySet());
- return credentialProvider.getCredential(pathBasedCredentialContext);
- }
-
public static Map<String, CredentialProvider> loadCredentialProviders(
Map<String, String> catalogProperties) {
- Set<String> credentialProviders =
- CredentialUtils.getCredentialProvidersByOrder(() -> catalogProperties);
+ CredentialConfig credentialConfig = new
CredentialConfig(catalogProperties);
+ List<String> credentialProviders =
credentialConfig.get(CredentialConfig.CREDENTIAL_PROVIDERS);
return credentialProviders.stream()
.collect(
@@ -80,14 +70,8 @@ public class CredentialUtils {
return Collections.emptySet();
}
- String providers =
properties.get(CredentialConstants.CREDENTIAL_PROVIDERS);
- if (providers == null) {
- return Collections.emptySet();
- }
- return splitter
- .trimResults()
- .omitEmptyStrings()
- .splitToStream(providers)
+ CredentialConfig credentialConfig = new CredentialConfig(properties);
+ return credentialConfig.get(CredentialConfig.CREDENTIAL_PROVIDERS).stream()
.collect(Collectors.toSet());
}
}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java
b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java
index 31a5183cc2..b08bda8569 100644
---
a/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java
+++
b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java
@@ -20,6 +20,8 @@
package org.apache.gravitino.credential.config;
import com.google.common.collect.ImmutableMap;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import org.apache.gravitino.Config;
import org.apache.gravitino.config.ConfigBuilder;
@@ -67,6 +69,14 @@ public class CredentialConfig extends Config {
false /* reserved */))
.build();
+ public static final ConfigEntry<List<String>> CREDENTIAL_PROVIDERS =
+ new ConfigBuilder(CredentialConstants.CREDENTIAL_PROVIDERS)
+ .doc("Credential providers, separated by comma.")
+ .version(ConfigConstants.VERSION_0_8_0)
+ .stringConf()
+ .toSequence()
+ .createWithDefault(Collections.emptyList());
+
public static final ConfigEntry<Double> CREDENTIAL_CACHE_EXPIRE_RATIO =
new ConfigBuilder(CredentialConstants.CREDENTIAL_CACHE_EXPIRE_RATIO)
.doc(
diff --git a/dev/docker/iceberg-rest-server/rewrite_config.py
b/dev/docker/iceberg-rest-server/rewrite_config.py
index 624c67750c..d607eb6ab4 100755
--- a/dev/docker/iceberg-rest-server/rewrite_config.py
+++ b/dev/docker/iceberg-rest-server/rewrite_config.py
@@ -22,7 +22,8 @@ env_map = {
"GRAVITINO_IO_IMPL" : "io-impl",
"GRAVITINO_URI" : "uri",
"GRAVITINO_WAREHOUSE" : "warehouse",
- "GRAVITINO_CREDENTIAL_PROVIDER_TYPE" : "credential-provider-type",
+ "GRAVITINO_CREDENTIAL_PROVIDER_TYPE" : "credential-providers",
+ "GRAVITINO_CREDENTIAL_PROVIDERS" : "credential-providers",
"GRAVITINO_GCS_CREDENTIAL_FILE_PATH" : "gcs-credential-file-path",
"GRAVITINO_S3_ACCESS_KEY" : "s3-access-key-id",
"GRAVITINO_S3_SECRET_KEY" : "s3-secret-access-key",
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index f31aa13685..3c2f27a3d1 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -106,22 +106,23 @@ The detailed configuration items are as follows:
Gravitino Iceberg REST service supports using static S3 secret key or
generating temporary token to access S3 data.
-| Configuration item | Description
| Default value | Required
| Since Version |
-|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------|------------------|
-| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3.
| (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.credential-provider-type` | Supports `s3-token` and
`s3-secret-key` for S3. `s3-token` generates a temporary token according to the
query data path while `s3-secret-key` using the s3 secret access key to access
S3 data. | (none) | No
| 0.7.0-incubating |
-| `gravitino.iceberg-rest.s3-access-key-id` | The static access key
ID used to access S3 data.
| (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret
access key used to access S3 data.
| (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint
of the S3 service, This could be used for S3FileIO with any s3-compatible
object storage service that has a different endpoint, or access a private S3
endpoint in a virtual private cloud. | (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-region` | The region of the S3
service, like `us-west-2`.
| (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to
access the S3 data.
| (none) | Yes, when
`credential-provider-type` is `s3-token` | 0.7.0-incubating |
-| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to
generate token, only used when `credential-provider-type` is `s3-token`.
| (none) | No
| 0.7.0-incubating |
-| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token
expire time in secs, it couldn't exceed the max session time of the assumed
role, only used when `credential-provider-type` is `s3-token`.
| 3600 | No
| 0.7.0-incubating |
-| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative endpoint
of the S3 token service, This could be used with s3-compatible object storage
service like MINIO that has a different STS endpoint.
| (none) | No
| 0.8.0-incubating |
+| Configuration item | Description
| Default value | Required
| Since Version |
+|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|------------------------------------------------|------------------|
+| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3.
| (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.credential-provider-type` | Deprecated, please use
`gravitino.iceberg-rest.credential-providers` instead.
| (none) | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.credential-providers` | Supports `s3-token` and
`s3-secret-key` for S3. `s3-token` generates a temporary token according to the
query data path while `s3-secret-key` using the s3 secret access key to access
S3 data. | (none) | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.s3-access-key-id` | The static access key
ID used to access S3 data.
| (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret
access key used to access S3 data.
| (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint
of the S3 service, This could be used for S3FileIO with any s3-compatible
object storage service that has a different endpoint, or access a private S3
endpoint in a virtual private cloud. | (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-region` | The region of the S3
service, like `us-west-2`.
| (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to
access the S3 data.
| (none) | Yes, when `credential-providers`
is `s3-token` | 0.7.0-incubating |
+| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to
generate token, only used when `credential-providers` is `s3-token`.
| (none) | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token
expire time in secs, it couldn't exceed the max session time of the assumed
role, only used when `credential-providers` is `s3-token`.
| 3600 | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative endpoint
of the S3 token service, This could be used with s3-compatible object storage
service like MINIO that has a different STS endpoint.
| (none) | No
| 0.8.0-incubating |
For other Iceberg s3 properties not managed by Gravitino like `s3.sse.type`,
you could config it directly by `gravitino.iceberg-rest.s3.sse.type`.
-If you set `credential-provider-type` explicitly, please downloading
[Gravitino AWS bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/aws-bundle), and
place it to the classpath of Iceberg REST server.
+If you set `credential-providers` explicitly, please downloading [Gravitino
AWS bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/aws-bundle), and
place it to the classpath of Iceberg REST server.
:::info
To configure the JDBC catalog backend, set the
`gravitino.iceberg-rest.warehouse` parameter to
`s3://{bucket_name}/${prefix_name}`. For the Hive catalog backend, set
`gravitino.iceberg-rest.warehouse` to `s3a://{bucket_name}/${prefix_name}`.
Additionally, download the [Iceberg AWS
bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-aws-bundle)
and place it in the classpath of Iceberg REST server.
@@ -134,18 +135,19 @@ Gravitino Iceberg REST service supports using static
access-key-id and secret-ac
| Configuration item | Description
| Default value | Required
| Since Version |
|---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|------------------------------------------------------|------------------|
| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.aliyun.oss.OSSFileIO` for OSS.
| (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.credential-provider-type` | Supports `oss-token` and
`oss-secret-key` for OSS. `oss-token` generates a temporary token according to
the query data path while `oss-secret-key` using the oss secret access key to
access S3 data. | (none) | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.credential-provider-type` | Deprecated, please use
`gravitino.iceberg-rest.credential-providers` instead.
| (none) | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.credential-providers` | Supports `oss-token`
and `oss-secret-key` for OSS. `oss-token` generates a temporary token according
to the query data path while `oss-secret-key` using the oss secret access key
to access S3 data. | (none) | No
| 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-access-key-id` | The static access key ID
used to access OSS data.
| (none) | No
| 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-secret-access-key` | The static secret access
key used to access OSS data.
| (none) | No
| 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-endpoint` | The endpoint of Aliyun
OSS service.
| (none) | No
| 0.7.0-incubating |
-| `gravitino.iceberg-rest.oss-region` | The region of the OSS
service, like `oss-cn-hangzhou`, only used when `credential-provider-type` is
`oss-token`.
| (none) | No
| 0.8.0-incubating |
-| `gravitino.iceberg-rest.oss-role-arn` | The ARN of the role to
access the OSS data, only used when `credential-provider-type` is `oss-token`.
| (none) | Yes, when `credential-provider-type` is
`oss-token`. | 0.8.0-incubating |
-| `gravitino.iceberg-rest.oss-external-id` | The OSS external id to
generate token, only used when `credential-provider-type` is `oss-token`.
| (none) | No
| 0.8.0-incubating |
-| `gravitino.iceberg-rest.oss-token-expire-in-secs` | The OSS security token
expire time in secs, only used when `credential-provider-type` is `oss-token`.
| 3600 | No
| 0.8.0-incubating |
+| `gravitino.iceberg-rest.oss-region` | The region of the OSS
service, like `oss-cn-hangzhou`, only used when `credential-providers` is
`oss-token`.
| (none) | No
| 0.8.0-incubating |
+| `gravitino.iceberg-rest.oss-role-arn` | The ARN of the role to
access the OSS data, only used when `credential-providers` is `oss-token`.
| (none) | Yes, when `credential-provider-type` is
`oss-token`. | 0.8.0-incubating |
+| `gravitino.iceberg-rest.oss-external-id` | The OSS external id to
generate token, only used when `credential-providers` is `oss-token`.
| (none) | No
| 0.8.0-incubating |
+| `gravitino.iceberg-rest.oss-token-expire-in-secs` | The OSS security token
expire time in secs, only used when `credential-providers` is `oss-token`.
| 3600 | No
| 0.8.0-incubating |
For other Iceberg OSS properties not managed by Gravitino like
`client.security-token`, you could config it directly by
`gravitino.iceberg-rest.client.security-token`.
-If you set `credential-provider-type` explicitly, please downloading
[Gravitino Aliyun bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/aliyun-bundle),
and place it to the classpath of Iceberg REST server.
+If you set `credential-providers` explicitly, please downloading [Gravitino
Aliyun bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/aliyun-bundle),
and place it to the classpath of Iceberg REST server.
:::info
Please set the `gravitino.iceberg-rest.warehouse` parameter to
`oss://{bucket_name}/${prefix_name}`. Additionally, download the [Aliyun OSS
SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and
copy `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` in
the classpath of Iceberg REST server, `iceberg-rest-server/libs` for the
auxiliary server, `libs` for the standalone server.
@@ -158,12 +160,13 @@ Supports using static GCS credential file or generating
GCS token to access GCS
| Configuration item | Description
|
Default value | Required | Since Version |
|---------------------------------------------------|----------------------------------------------------------------------------------------------------|---------------|----------|------------------|
| `gravitino.iceberg-rest.io-impl` | The io implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.gcp.gcs.GCSFileIO` for GCS. |
(none) | No | 0.6.0-incubating |
-| `gravitino.iceberg-rest.credential-provider-type` | Supports `gcs-token`,
generates a temporary token according to the query data path. |
(none) | No | 0.7.0-incubating |
-| `gravitino.iceberg-rest.gcs-credential-file-path` | The location of GCS
credential file, only used when `credential-provider-type` is `gcs-token`.
| (none) | No | 0.7.0-incubating |
+| `gravitino.iceberg-rest.credential-provider-type` | Deprecated, please use
`gravitino.iceberg-rest.credential-providers` instead. |
(none) | No |
0.7.0-incubating |
+| `gravitino.iceberg-rest.credential-providers` | Supports `gcs-token`,
generates a temporary token according to the query data path. |
(none) | No | 0.7.0-incubating |
+| `gravitino.iceberg-rest.gcs-credential-file-path` | The location of GCS
credential file, only used when `credential-providers` is `gcs-token`.
| (none) | No | 0.7.0-incubating |
For other Iceberg GCS properties not managed by Gravitino like
`gcs.project-id`, you could config it directly by
`gravitino.iceberg-rest.gcs.project-id`.
-If you set `credential-provider-type` explicitly, please downloading
[Gravitino GCP bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/gcp-bundle), and
place it to the classpath of Iceberg REST server.
+If you set `credential-providers` explicitly, please downloading [Gravitino
GCP bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/gcp-bundle), and
place it to the classpath of Iceberg REST server.
Please make sure the credential file is accessible by Gravitino, like using
`export
GOOGLE_APPLICATION_CREDENTIALS=/xx/application_default_credentials.json` before
Gravitino Iceberg REST server is started.
@@ -178,17 +181,18 @@ Gravitino Iceberg REST service supports generating SAS
token to access ADLS data
| Configuration item | Description
| Default value | Required | Since Version |
|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|------------------|
| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for
ADLS.
| (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.credential-provider-type` | Supports `adls-token`
and `azure-account-key`. `adls-token` generates a temporary token according to
the query data path while `azure-account-key` uses a storage account key to
access ADLS data. | (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.credential-provider-type` | Deprecated, please use
`gravitino.iceberg-rest.credential-providers` instead.
| (none) | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.credential-providers` | Supports `adls-token`
and `azure-account-key`. `adls-token` generates a temporary token according to
the query data path while `azure-account-key` uses a storage account key to
access ADLS data. | (none) | Yes | 0.8.0-incubating |
| `gravitino.iceberg-rest.azure-storage-account-name` | The static storage
account name used to access ADLS data.
| (none) | Yes | 0.8.0-incubating |
| `gravitino.iceberg-rest.azure-storage-account-key` | The static storage
account key used to access ADLS data.
| (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory
(AAD) tenant ID, only used when `credential-provider-type` is `adls-token`.
| (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory
(AAD) client ID used for authentication, only used when
`credential-provider-type` is `adls-token`.
| (none) | Yes |
0.8.0-incubating |
-| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory
(AAD) client secret used for authentication, only used when
`credential-provider-type` is `adls-token`.
| (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.adls-token-expire-in-secs` | The ADLS SAS token
expire time in secs, only used when `credential-provider-type` is
`adls-token`.
| 3600 | No | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory
(AAD) tenant ID, only used when `credential-providers` is `adls-token`.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory
(AAD) client ID used for authentication, only used when `credential-providers`
is `adls-token`.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory
(AAD) client secret used for authentication, only used when
`credential-providers` is `adls-token`.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.adls-token-expire-in-secs` | The ADLS SAS token
expire time in secs, only used when `credential-providers` is `adls-token`.
| 3600 | No | 0.8.0-incubating |
For other Iceberg ADLS properties not managed by Gravitino like
`adls.read.block-size-bytes`, you could config it directly by
`gravitino.iceberg-rest.adls.read.block-size-bytes`.
-If you set `credential-provider-type` explicitly, please downloading
[Gravitino Azure bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/azure-bundle), and
place it to the classpath of Iceberg REST server.
+If you set `credential-providers` explicitly, please downloading [Gravitino
Azure bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/azure-bundle), and
place it to the classpath of Iceberg REST server.
:::info
Please set `gravitino.iceberg-rest.warehouse` to
`abfs[s]://{container-name}@{storage-account-name}.dfs.core.windows.net/{path}`,
and download the [Iceberg Azure
bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-azure-bundle)
and place it in the classpath of Iceberg REST server.
@@ -415,7 +419,7 @@ For example, we can configure Spark catalog options to use
Gravitino Iceberg RES
--conf spark.sql.catalog.rest.uri=http://127.0.0.1:9001/iceberg/
```
-You may need to adjust the Iceberg Spark runtime jar file name according to
the real version number in your environment. If you want to access the data
stored in cloud, you need to download corresponding jars (please refer to the
cloud storage part) and place it in the classpath of Spark. If you want to
enable credential vending, please set `credential-provider-type` to a proper
value in the server side, set
`spark.sql.catalog.rest.header.X-Iceberg-Access-Delegation` =
`vended-credential [...]
+You may need to adjust the Iceberg Spark runtime jar file name according to
the real version number in your environment. If you want to access the data
stored in cloud, you need to download corresponding jars (please refer to the
cloud storage part) and place it in the classpath of Spark. If you want to
enable credential vending, please set `credential-providers` to a proper value
in the server side, set
`spark.sql.catalog.rest.header.X-Iceberg-Access-Delegation` =
`vended-credentials` i [...]
For other storages not managed by Gravitino, the properties wouldn't transfer
from the server to client automatically, if you want to pass custom properties
to initialize `FileIO`, you could add it by
`spark.sql.catalog.${iceberg_catalog_name}.${configuration_key}` =
`{property_value}`.
@@ -441,24 +445,25 @@ docker run -d -p 9001:9001
apache/gravitino-iceberg-rest:0.7.0-incubating
Gravitino Iceberg REST server in docker image could access local storage by
default, you could set the following environment variables if the storage is
cloud/remote storage like S3, please refer to [storage section](#storage) for
more details.
-| Environment variables | Configuration items
| Since version |
-|-----------------------------------------|-----------------------------------------------------|-------------------|
-| `GRAVITINO_IO_IMPL` | `gravitino.iceberg-rest.io-impl`
| 0.7.0-incubating |
-| `GRAVITINO_URI` | `gravitino.iceberg-rest.uri`
| 0.7.0-incubating |
-| `GRAVITINO_WAREHOUSE` | `gravitino.iceberg-rest.warehouse`
| 0.7.0-incubating |
-| `GRAVITINO_CREDENTIAL_PROVIDER_TYPE` |
`gravitino.iceberg-rest.credential-provider-type` | 0.7.0-incubating |
-| `GRAVITINO_GCS_CREDENTIAL_FILE_PATH` |
`gravitino.iceberg-rest.gcs-credential-file-path` | 0.7.0-incubating |
-| `GRAVITINO_S3_ACCESS_KEY` |
`gravitino.iceberg-rest.s3-access-key-id` | 0.7.0-incubating |
-| `GRAVITINO_S3_SECRET_KEY` |
`gravitino.iceberg-rest.s3-secret-access-key` | 0.7.0-incubating |
-| `GRAVITINO_S3_REGION` | `gravitino.iceberg-rest.s3-region`
| 0.7.0-incubating |
-| `GRAVITINO_S3_ROLE_ARN` |
`gravitino.iceberg-rest.s3-role-arn` | 0.7.0-incubating |
-| `GRAVITINO_S3_EXTERNAL_ID` |
`gravitino.iceberg-rest.s3-external-id` | 0.7.0-incubating |
-| `GRAVITINO_S3_TOKEN_SERVICE_ENDPOINT` |
`gravitino.iceberg-rest.s3-token-service-endpoint` | 0.8.0-incubating |
-| `GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME` |
`gravitino.iceberg-rest.azure-storage-account-name` | 0.8.0-incubating |
-| `GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY` |
`gravitino.iceberg-rest.azure-storage-account-key` | 0.8.0-incubating |
-| `GRAVITINO_AZURE_TENANT_ID` |
`gravitino.iceberg-rest.azure-tenant-id` | 0.8.0-incubating |
-| `GRAVITINO_AZURE_CLIENT_ID` |
`gravitino.iceberg-rest.azure-client-id` | 0.8.0-incubating |
-| `GRAVITINO_AZURE_CLIENT_SECRET` |
`gravitino.iceberg-rest.azure-client-secret` | 0.8.0-incubating |
+| Environment variables | Configuration items
| Since version |
+|----------------------------------------|-----------------------------------------------------|------------------|
+| `GRAVITINO_IO_IMPL` | `gravitino.iceberg-rest.io-impl`
| 0.7.0-incubating |
+| `GRAVITINO_URI` | `gravitino.iceberg-rest.uri`
| 0.7.0-incubating |
+| `GRAVITINO_WAREHOUSE` | `gravitino.iceberg-rest.warehouse`
| 0.7.0-incubating |
+| `GRAVITINO_CREDENTIAL_PROVIDER_TYPE` |
`gravitino.iceberg-rest.credential-providers` | 0.8.0-incubating |
+| `GRAVITINO_CREDENTIAL_PROVIDERS` |
`gravitino.iceberg-rest.credential-providers` | 0.8.0-incubating |
+| `GRAVITINO_GCS_CREDENTIAL_FILE_PATH` |
`gravitino.iceberg-rest.gcs-credential-file-path` | 0.7.0-incubating |
+| `GRAVITINO_S3_ACCESS_KEY` |
`gravitino.iceberg-rest.s3-access-key-id` | 0.7.0-incubating |
+| `GRAVITINO_S3_SECRET_KEY` |
`gravitino.iceberg-rest.s3-secret-access-key` | 0.7.0-incubating |
+| `GRAVITINO_S3_REGION` | `gravitino.iceberg-rest.s3-region`
| 0.7.0-incubating |
+| `GRAVITINO_S3_ROLE_ARN` |
`gravitino.iceberg-rest.s3-role-arn` | 0.7.0-incubating |
+| `GRAVITINO_S3_EXTERNAL_ID` |
`gravitino.iceberg-rest.s3-external-id` | 0.7.0-incubating |
+| `GRAVITINO_S3_TOKEN_SERVICE_ENDPOINT` |
`gravitino.iceberg-rest.s3-token-service-endpoint` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME` |
`gravitino.iceberg-rest.azure-storage-account-name` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY` |
`gravitino.iceberg-rest.azure-storage-account-key` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_TENANT_ID` |
`gravitino.iceberg-rest.azure-tenant-id` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_CLIENT_ID` |
`gravitino.iceberg-rest.azure-client-id` | 0.8.0-incubating |
+| `GRAVITINO_AZURE_CLIENT_SECRET` |
`gravitino.iceberg-rest.azure-client-secret` | 0.8.0-incubating |
Or build it manually to add custom configuration or logics:
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
index 60a7491b85..638d0c6d31 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
@@ -239,9 +239,13 @@ public class IcebergConfig extends Config implements
OverwriteDefaultConfig {
.toSequence()
.createWithDefault(Collections.emptyList());
+ @Deprecated
public static final ConfigEntry<String> CREDENTIAL_PROVIDER_TYPE =
new ConfigBuilder(CredentialConstants.CREDENTIAL_PROVIDER_TYPE)
- .doc("The credential provider type for Iceberg")
+ .doc(
+ String.format(
+ "Deprecated, please use %s instead, The credential provider
type for Iceberg",
+ CredentialConstants.CREDENTIAL_PROVIDERS))
.version(ConfigConstants.VERSION_0_7_0)
.stringConf()
.create();
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
index 0ed62b26f7..d444c55a75 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
@@ -19,23 +19,19 @@
package org.apache.gravitino.iceberg.common.ops;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
import java.sql.Driver;
import java.sql.DriverManager;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.function.Supplier;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
-import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
import org.apache.gravitino.iceberg.common.IcebergCatalogBackend;
import org.apache.gravitino.iceberg.common.IcebergConfig;
import org.apache.gravitino.iceberg.common.utils.IcebergCatalogUtil;
import org.apache.gravitino.utils.IsolatedClassLoader;
-import org.apache.gravitino.utils.MapUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.iceberg.Transaction;
@@ -62,6 +58,10 @@ import
org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * A wrapper for Iceberg catalog backend, provides the common interface for
Iceberg REST server and
+ * Gravitino Iceberg catalog.
+ */
public class IcebergCatalogWrapper implements AutoCloseable {
public static final Logger LOG =
LoggerFactory.getLogger(IcebergCatalogWrapper.class);
@@ -70,14 +70,7 @@ public class IcebergCatalogWrapper implements AutoCloseable {
private SupportsNamespaces asNamespaceCatalog;
private final IcebergCatalogBackend catalogBackend;
private String catalogUri = null;
- private Map<String, String> catalogConfigToClients;
private Map<String, String> catalogPropertiesMap;
- private static final Set<String> catalogPropertiesToClientKeys =
- ImmutableSet.of(
- IcebergConstants.IO_IMPL,
- IcebergConstants.AWS_S3_REGION,
- IcebergConstants.ICEBERG_S3_ENDPOINT,
- IcebergConstants.ICEBERG_OSS_ENDPOINT);
public IcebergCatalogWrapper(IcebergConfig icebergConfig) {
this.catalogBackend =
@@ -97,10 +90,6 @@ public class IcebergCatalogWrapper implements AutoCloseable {
if (catalog instanceof SupportsNamespaces) {
this.asNamespaceCatalog = (SupportsNamespaces) catalog;
}
- this.catalogConfigToClients =
- MapUtils.getFilteredMap(
- icebergConfig.getIcebergCatalogProperties(),
- key -> catalogPropertiesToClientKeys.contains(key));
this.catalogPropertiesMap = icebergConfig.getIcebergCatalogProperties();
}
@@ -307,14 +296,7 @@ public class IcebergCatalogWrapper implements
AutoCloseable {
// Some io and security configuration should pass to Iceberg REST client
private LoadTableResponse injectTableConfig(Supplier<LoadTableResponse>
supplier) {
LoadTableResponse loadTableResponse = supplier.get();
- return LoadTableResponse.builder()
- .withTableMetadata(loadTableResponse.tableMetadata())
- .addAllConfig(getCatalogConfigToClient())
- .build();
- }
-
- private Map<String, String> getCatalogConfigToClient() {
- return catalogConfigToClients;
+ return
LoadTableResponse.builder().withTableMetadata(loadTableResponse.tableMetadata()).build();
}
@Getter
diff --git
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java
new file mode 100644
index 0000000000..8ae7bd66dd
--- /dev/null
+++
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.iceberg.service;
+
+import com.google.common.collect.ImmutableSet;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.credential.CatalogCredentialManager;
+import org.apache.gravitino.credential.Credential;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.credential.CredentialPropertyUtils;
+import org.apache.gravitino.credential.PathBasedCredentialContext;
+import org.apache.gravitino.iceberg.common.IcebergConfig;
+import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
+import org.apache.gravitino.utils.MapUtils;
+import org.apache.gravitino.utils.PrincipalUtils;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.rest.requests.CreateTableRequest;
+import org.apache.iceberg.rest.responses.LoadTableResponse;
+
+/** Process Iceberg REST specific operations, like credential vending. */
+public class CatalogWrapperForREST extends IcebergCatalogWrapper {
+
+ private final CatalogCredentialManager catalogCredentialManager;
+
+ private final Map<String, String> catalogConfigToClients;
+
+ private static final Set<String> catalogPropertiesToClientKeys =
+ ImmutableSet.of(
+ IcebergConstants.IO_IMPL,
+ IcebergConstants.AWS_S3_REGION,
+ IcebergConstants.ICEBERG_S3_ENDPOINT,
+ IcebergConstants.ICEBERG_OSS_ENDPOINT);
+
+ public CatalogWrapperForREST(String catalogName, IcebergConfig config) {
+ super(config);
+ this.catalogConfigToClients =
+ MapUtils.getFilteredMap(
+ config.getIcebergCatalogProperties(),
+ key -> catalogPropertiesToClientKeys.contains(key));
+ // To be compatible with old properties
+ Map<String, String> catalogProperties =
checkForCompatibility(config.getAllConfig());
+ this.catalogCredentialManager = new CatalogCredentialManager(catalogName,
catalogProperties);
+ }
+
+ public LoadTableResponse createTable(
+ Namespace namespace, CreateTableRequest request, boolean
requestCredential) {
+ LoadTableResponse loadTableResponse = super.createTable(namespace,
request);
+ if (requestCredential) {
+ return injectCredentialConfig(
+ TableIdentifier.of(namespace, request.name()), loadTableResponse);
+ }
+ return loadTableResponse;
+ }
+
+ public LoadTableResponse loadTable(TableIdentifier identifier, boolean
requestCredential) {
+ LoadTableResponse loadTableResponse = super.loadTable(identifier);
+ if (requestCredential) {
+ return injectCredentialConfig(identifier, loadTableResponse);
+ }
+ return loadTableResponse;
+ }
+
+ @Override
+ public void close() {
+ if (catalogCredentialManager != null) {
+ catalogCredentialManager.close();
+ }
+ }
+
+ private Map<String, String> getCatalogConfigToClient() {
+ return catalogConfigToClients;
+ }
+
+ private LoadTableResponse injectCredentialConfig(
+ TableIdentifier tableIdentifier, LoadTableResponse loadTableResponse) {
+ TableMetadata tableMetadata = loadTableResponse.tableMetadata();
+ String[] path =
+ Stream.of(
+ tableMetadata.location(),
+ tableMetadata.property(TableProperties.WRITE_DATA_LOCATION,
""),
+
tableMetadata.property(TableProperties.WRITE_METADATA_LOCATION, ""))
+ .filter(StringUtils::isNotBlank)
+ .toArray(String[]::new);
+
+ PathBasedCredentialContext context =
+ new PathBasedCredentialContext(
+ PrincipalUtils.getCurrentUserName(), ImmutableSet.copyOf(path),
Collections.emptySet());
+ Credential credential = catalogCredentialManager.getCredential(context);
+ if (credential == null) {
+ throw new ServiceUnavailableException("Couldn't generate credential,
%s", context);
+ }
+
+ LOG.info(
+ "Generate credential: {} for Iceberg table: {}",
+ credential.credentialType(),
+ tableIdentifier);
+
+ Map<String, String> credentialConfig =
CredentialPropertyUtils.toIcebergProperties(credential);
+ return LoadTableResponse.builder()
+ .withTableMetadata(loadTableResponse.tableMetadata())
+ .addAllConfig(loadTableResponse.config())
+ .addAllConfig(getCatalogConfigToClient())
+ .addAllConfig(credentialConfig)
+ .build();
+ }
+
+ @SuppressWarnings("deprecation")
+ private Map<String, String> checkForCompatibility(Map<String, String>
properties) {
+ HashMap<String, String> normalizedProperties = new HashMap<>(properties);
+ String credentialProviderType =
properties.get(CredentialConstants.CREDENTIAL_PROVIDER_TYPE);
+ String credentialProviders =
properties.get(CredentialConstants.CREDENTIAL_PROVIDERS);
+ if (StringUtils.isNotBlank(credentialProviders)
+ && StringUtils.isNotBlank(credentialProviderType)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Should not set both %s and %s",
+ CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
+ CredentialConstants.CREDENTIAL_PROVIDERS));
+ }
+
+ if (StringUtils.isNotBlank(credentialProviderType)) {
+ LOG.warn(
+ "%s is deprecated, please use %s instead.",
+ CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
CredentialConstants.CREDENTIAL_PROVIDERS);
+ normalizedProperties.put(CredentialConstants.CREDENTIAL_PROVIDERS,
credentialProviderType);
+ }
+
+ return normalizedProperties;
+ }
+}
diff --git
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java
index 6e25ceec42..7b3e18109f 100644
---
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java
+++
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java
@@ -27,10 +27,6 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.gravitino.credential.CredentialProvider;
-import org.apache.gravitino.credential.CredentialProviderFactory;
-import org.apache.gravitino.credential.CredentialProviderManager;
import org.apache.gravitino.iceberg.common.IcebergConfig;
import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
import org.apache.gravitino.iceberg.service.provider.IcebergConfigProvider;
@@ -38,17 +34,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class IcebergCatalogWrapperManager implements AutoCloseable {
+
public static final Logger LOG =
LoggerFactory.getLogger(IcebergCatalogWrapperManager.class);
- private final Cache<String, IcebergCatalogWrapper>
icebergCatalogWrapperCache;
+ private final Cache<String, CatalogWrapperForREST>
icebergCatalogWrapperCache;
private final IcebergConfigProvider configProvider;
- private CredentialProviderManager credentialProviderManager;
-
public IcebergCatalogWrapperManager(
Map<String, String> properties, IcebergConfigProvider configProvider) {
- this.credentialProviderManager = new CredentialProviderManager();
this.configProvider = configProvider;
this.icebergCatalogWrapperCache =
Caffeine.newBuilder()
@@ -61,7 +55,6 @@ public class IcebergCatalogWrapperManager implements
AutoCloseable {
String catalogName = (String) k;
LOG.info("Remove IcebergCatalogWrapper cache {}.",
catalogName);
closeIcebergCatalogWrapper((IcebergCatalogWrapper) v);
-
credentialProviderManager.unregisterCredentialProvider(catalogName);
})
.scheduler(
Scheduler.forScheduledExecutorService(
@@ -79,44 +72,33 @@ public class IcebergCatalogWrapperManager implements
AutoCloseable {
* ([^/]*\/), end with /
* @return the instance of IcebergCatalogWrapper.
*/
- public IcebergCatalogWrapper getOps(String rawPrefix) {
+ public CatalogWrapperForREST getOps(String rawPrefix) {
String catalogName = IcebergRestUtils.getCatalogName(rawPrefix);
return getCatalogWrapper(catalogName);
}
- public IcebergCatalogWrapper getCatalogWrapper(String catalogName) {
- IcebergCatalogWrapper catalogWrapper =
+ public CatalogWrapperForREST getCatalogWrapper(String catalogName) {
+ CatalogWrapperForREST catalogWrapperForREST =
icebergCatalogWrapperCache.get(catalogName, k ->
createCatalogWrapper(catalogName));
// Reload conf to reset UserGroupInformation or icebergTableOps will
always use
// Simple auth.
- catalogWrapper.reloadHadoopConf();
- return catalogWrapper;
- }
-
- public CredentialProvider getCredentialProvider(String catalogName) {
- return credentialProviderManager.getCredentialProvider(catalogName);
+ catalogWrapperForREST.reloadHadoopConf();
+ return catalogWrapperForREST;
}
- @VisibleForTesting
- protected IcebergCatalogWrapper createIcebergCatalogWrapper(IcebergConfig
icebergConfig) {
- return new IcebergCatalogWrapper(icebergConfig);
- }
-
- private IcebergCatalogWrapper createCatalogWrapper(String catalogName) {
+ private CatalogWrapperForREST createCatalogWrapper(String catalogName) {
Optional<IcebergConfig> icebergConfig =
configProvider.getIcebergCatalogConfig(catalogName);
if (!icebergConfig.isPresent()) {
throw new RuntimeException("Couldn't find Iceberg configuration for " +
catalogName);
}
+ return createCatalogWrapper(catalogName, icebergConfig.get());
+ }
- IcebergConfig config = icebergConfig.get();
- String credentialProviderType =
config.get(IcebergConfig.CREDENTIAL_PROVIDER_TYPE);
- if (StringUtils.isNotBlank(credentialProviderType)) {
- CredentialProvider credentialProvider =
- CredentialProviderFactory.create(credentialProviderType,
config.getAllConfig());
- credentialProviderManager.registerCredentialProvider(catalogName,
credentialProvider);
- }
-
- return createIcebergCatalogWrapper(icebergConfig.get());
+ // Overriding this method to create a new CatalogWrapperForREST for test;
+ @VisibleForTesting
+ protected CatalogWrapperForREST createCatalogWrapper(
+ String catalogName, IcebergConfig icebergConfig) {
+ return new CatalogWrapperForREST(catalogName, icebergConfig);
}
private void closeIcebergCatalogWrapper(IcebergCatalogWrapper
catalogWrapper) {
diff --git
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergTableOperationExecutor.java
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergTableOperationExecutor.java
index e6385bfdc6..31e94ab9f6 100644
---
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergTableOperationExecutor.java
+++
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergTableOperationExecutor.java
@@ -42,7 +42,7 @@ public class IcebergTableOperationExecutor implements
IcebergTableOperationDispa
IcebergRequestContext context, Namespace namespace, CreateTableRequest
createTableRequest) {
return icebergCatalogWrapperManager
.getCatalogWrapper(context.catalogName())
- .createTable(namespace, createTableRequest);
+ .createTable(namespace, createTableRequest,
context.requestCredentialVending());
}
@Override
@@ -74,7 +74,7 @@ public class IcebergTableOperationExecutor implements
IcebergTableOperationDispa
IcebergRequestContext context, TableIdentifier tableIdentifier) {
return icebergCatalogWrapperManager
.getCatalogWrapper(context.catalogName())
- .loadTable(tableIdentifier);
+ .loadTable(tableIdentifier, context.requestCredentialVending());
}
@Override
diff --git
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java
index 12f9c5055b..96fc98921f 100644
---
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java
+++
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java
@@ -23,8 +23,6 @@ import com.codahale.metrics.annotation.Timed;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
-import java.util.Map;
-import java.util.stream.Stream;
import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes;
@@ -33,7 +31,6 @@ import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.HEAD;
import javax.ws.rs.HeaderParam;
-import javax.ws.rs.NotSupportedException;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -43,23 +40,14 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
-import org.apache.gravitino.credential.Credential;
-import org.apache.gravitino.credential.CredentialConstants;
-import org.apache.gravitino.credential.CredentialPropertyUtils;
-import org.apache.gravitino.credential.CredentialProvider;
-import org.apache.gravitino.credential.CredentialUtils;
-import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager;
import org.apache.gravitino.iceberg.service.IcebergObjectMapper;
import org.apache.gravitino.iceberg.service.IcebergRestUtils;
import
org.apache.gravitino.iceberg.service.dispatcher.IcebergTableOperationDispatcher;
import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager;
import org.apache.gravitino.listener.api.event.IcebergRequestContext;
import org.apache.gravitino.metrics.MetricNames;
-import org.apache.iceberg.TableMetadata;
-import org.apache.iceberg.TableProperties;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
-import org.apache.iceberg.exceptions.ServiceUnavailableException;
import org.apache.iceberg.rest.RESTUtil;
import org.apache.iceberg.rest.requests.CreateTableRequest;
import org.apache.iceberg.rest.requests.ReportMetricsRequest;
@@ -79,7 +67,6 @@ public class IcebergTableOperations {
@VisibleForTesting
public static final String X_ICEBERG_ACCESS_DELEGATION =
"X-Iceberg-Access-Delegation";
- private IcebergCatalogWrapperManager icebergCatalogWrapperManager;
private IcebergMetricsManager icebergMetricsManager;
private ObjectMapper icebergObjectMapper;
@@ -89,10 +76,8 @@ public class IcebergTableOperations {
@Inject
public IcebergTableOperations(
- IcebergCatalogWrapperManager icebergCatalogWrapperManager,
IcebergMetricsManager icebergMetricsManager,
IcebergTableOperationDispatcher tableOperationDispatcher) {
- this.icebergCatalogWrapperManager = icebergCatalogWrapperManager;
this.icebergMetricsManager = icebergMetricsManager;
this.tableOperationDispatcher = tableOperationDispatcher;
this.icebergObjectMapper = IcebergObjectMapper.getInstance();
@@ -132,18 +117,11 @@ public class IcebergTableOperations {
createTableRequest,
accessDelegation,
isCredentialVending);
- IcebergRequestContext context = new
IcebergRequestContext(httpServletRequest(), catalogName);
+ IcebergRequestContext context =
+ new IcebergRequestContext(httpServletRequest(), catalogName,
isCredentialVending);
LoadTableResponse loadTableResponse =
tableOperationDispatcher.createTable(context, icebergNS,
createTableRequest);
- if (isCredentialVending) {
- return IcebergRestUtils.ok(
- injectCredentialConfig(
- catalogName,
- TableIdentifier.of(icebergNS, createTableRequest.name()),
- loadTableResponse));
- } else {
- return IcebergRestUtils.ok(loadTableResponse);
- }
+ return IcebergRestUtils.ok(loadTableResponse);
}
@POST
@@ -221,15 +199,11 @@ public class IcebergTableOperations {
isCredentialVending);
// todo support snapshots
TableIdentifier tableIdentifier = TableIdentifier.of(icebergNS, table);
- IcebergRequestContext context = new
IcebergRequestContext(httpServletRequest(), catalogName);
+ IcebergRequestContext context =
+ new IcebergRequestContext(httpServletRequest(), catalogName,
isCredentialVending);
LoadTableResponse loadTableResponse =
tableOperationDispatcher.loadTable(context, tableIdentifier);
- if (isCredentialVending) {
- return IcebergRestUtils.ok(
- injectCredentialConfig(catalogName, tableIdentifier,
loadTableResponse));
- } else {
- return IcebergRestUtils.ok(loadTableResponse);
- }
+ return IcebergRestUtils.ok(loadTableResponse);
}
@HEAD
@@ -287,45 +261,6 @@ public class IcebergTableOperations {
}
}
- private LoadTableResponse injectCredentialConfig(
- String catalogName, TableIdentifier tableIdentifier, LoadTableResponse
loadTableResponse) {
- CredentialProvider credentialProvider =
- icebergCatalogWrapperManager.getCredentialProvider(catalogName);
- if (credentialProvider == null) {
- throw new NotSupportedException(
- "Doesn't support credential vending, please add "
- + CredentialConstants.CREDENTIAL_PROVIDER_TYPE
- + " to the catalog configurations");
- }
-
- TableMetadata tableMetadata = loadTableResponse.tableMetadata();
- String[] path =
- Stream.of(
- tableMetadata.location(),
- tableMetadata.property(TableProperties.WRITE_DATA_LOCATION,
""),
-
tableMetadata.property(TableProperties.WRITE_METADATA_LOCATION, ""))
- .filter(StringUtils::isNotBlank)
- .toArray(String[]::new);
-
- Credential credential = CredentialUtils.vendCredential(credentialProvider,
path);
- if (credential == null) {
- throw new ServiceUnavailableException(
- "Couldn't generate credential for %s",
credentialProvider.credentialType());
- }
-
- LOG.info(
- "Generate credential: {} for Iceberg table: {}",
- credential.credentialType(),
- tableIdentifier);
-
- Map<String, String> credentialConfig =
CredentialPropertyUtils.toIcebergProperties(credential);
- return LoadTableResponse.builder()
- .withTableMetadata(loadTableResponse.tableMetadata())
- .addAllConfig(loadTableResponse.config())
- .addAllConfig(credentialConfig)
- .build();
- }
-
private boolean isCredentialVending(String accessDelegation) {
if (StringUtils.isBlank(accessDelegation)) {
return false;
diff --git
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/listener/api/event/IcebergRequestContext.java
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/listener/api/event/IcebergRequestContext.java
index c0849a117a..c46fcdfd09 100644
---
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/listener/api/event/IcebergRequestContext.java
+++
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/listener/api/event/IcebergRequestContext.java
@@ -33,19 +33,33 @@ public class IcebergRequestContext {
private final String userName;
private final String remoteHostName;
private final Map<String, String> httpHeaders;
+ private final boolean requestCredentialVending;
/**
- * Constructs a new {@code IcebergRequestContext} with specified HTTP
request and catalog name.
+ * Constructs a new {@code IcebergRequestContext} instance.
*
* @param httpRequest The HttpServletRequest object containing request
details.
* @param catalogName The name of the catalog to be accessed in the request.
*/
public IcebergRequestContext(HttpServletRequest httpRequest, String
catalogName) {
+ this(httpRequest, catalogName, false);
+ }
+
+ /**
+ * Constructs a new {@code IcebergRequestContext} instance.
+ *
+ * @param httpRequest The HttpServletRequest object containing request
details.
+ * @param catalogName The name of the catalog to be accessed in the request.
+ * @param requestCredentialVending Whether the request is for credential
vending.
+ */
+ public IcebergRequestContext(
+ HttpServletRequest httpRequest, String catalogName, boolean
requestCredentialVending) {
this.httpServletRequest = httpRequest;
this.remoteHostName = httpRequest.getRemoteHost();
this.httpHeaders = IcebergRestUtils.getHttpHeaders(httpRequest);
this.catalogName = catalogName;
this.userName = PrincipalUtils.getCurrentUserName();
+ this.requestCredentialVending = requestCredentialVending;
}
/**
@@ -84,6 +98,15 @@ public class IcebergRequestContext {
return httpHeaders;
}
+ /**
+ * Checks if the request is for credential vending.
+ *
+ * @return true if the request is for credential vending, false otherwise.
+ */
+ public boolean requestCredentialVending() {
+ return requestCredentialVending;
+ }
+
/**
* Retrieves the HttpServletRequest object. This method is deprecated and
should be used
* cautiously.
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
index b663251e0e..52ccb876df 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
@@ -92,7 +92,7 @@ public class IcebergRESTADLSTokenIT extends
IcebergRESTJdbcCatalogIT {
Map configMap = new HashMap<String, String>();
configMap.put(
- IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDERS,
ADLSTokenCredential.ADLS_TOKEN_CREDENTIAL_TYPE);
configMap.put(
IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
index 695b72ed4b..f999f84f58 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
@@ -82,7 +82,7 @@ public class IcebergRESTAzureAccountKeyIT extends
IcebergRESTJdbcCatalogIT {
Map configMap = new HashMap<String, String>();
configMap.put(
- IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDERS,
AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE);
configMap.put(
IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
index 11ee27bf44..523d877374 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
@@ -73,7 +73,7 @@ public class IcebergRESTGCSIT extends
IcebergRESTJdbcCatalogIT {
Map configMap = new HashMap<String, String>();
configMap.put(
- IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDERS,
GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE);
configMap.put(
IcebergConfig.ICEBERG_CONFIG_PREFIX
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
index af70253d84..4c4b4a953b 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
@@ -86,7 +86,7 @@ public class IcebergRESTOSSIT extends
IcebergRESTJdbcCatalogIT {
Map configMap = new HashMap<String, String>();
configMap.put(
- IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDERS,
OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE);
configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX +
OSSProperties.GRAVITINO_OSS_REGION, region);
configMap.put(
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
index cd5c99c46d..0be69cbe3d 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
@@ -79,7 +79,7 @@ public class IcebergRESTOSSSecretIT extends
IcebergRESTJdbcCatalogIT {
Map configMap = new HashMap<String, String>();
configMap.put(
- IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDERS,
OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE);
configMap.put(
IcebergConfig.ICEBERG_CONFIG_PREFIX +
OSSProperties.GRAVITINO_OSS_ENDPOINT, endpoint);
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
index 7e16273245..e906018f52 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
@@ -87,7 +87,7 @@ public class IcebergRESTS3IT extends IcebergRESTJdbcCatalogIT
{
Map configMap = new HashMap<String, String>();
configMap.put(
- IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
+ IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDERS,
S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE);
configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX +
S3Properties.GRAVITINO_S3_REGION, region);
configMap.put(
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManager.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManagerForREST.java
similarity index 98%
rename from
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManager.java
rename to
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManagerForREST.java
index 85a7fdc04e..fad31e816d 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManager.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManagerForREST.java
@@ -28,7 +28,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
-public class TestIcebergCatalogWrapperManager {
+public class TestIcebergCatalogWrapperManagerForREST {
private static final String DEFAULT_CATALOG = "memory";
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/CatalogWrapperForTest.java
similarity index 89%
rename from
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java
rename to
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/CatalogWrapperForTest.java
index f6326dd229..423b52d577 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/CatalogWrapperForTest.java
@@ -19,7 +19,7 @@
package org.apache.gravitino.iceberg.service.rest;
import org.apache.gravitino.iceberg.common.IcebergConfig;
-import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
+import org.apache.gravitino.iceberg.service.CatalogWrapperForREST;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.TableMetadata;
@@ -32,9 +32,9 @@ import org.apache.iceberg.types.Types.StringType;
import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
// Used to override registerTable
-public class IcebergCatalogWrapperForTest extends IcebergCatalogWrapper {
- public IcebergCatalogWrapperForTest(IcebergConfig icebergConfig) {
- super(icebergConfig);
+public class CatalogWrapperForTest extends CatalogWrapperForREST {
+ public CatalogWrapperForTest(String catalogName, IcebergConfig
icebergConfig) {
+ super(catalogName, icebergConfig);
}
@Override
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java
index 361b086d98..445b9f7451 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java
@@ -21,7 +21,7 @@ package org.apache.gravitino.iceberg.service.rest;
import java.util.Map;
import org.apache.gravitino.iceberg.common.IcebergConfig;
-import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
+import org.apache.gravitino.iceberg.service.CatalogWrapperForREST;
import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager;
import org.apache.gravitino.iceberg.service.provider.IcebergConfigProvider;
@@ -33,7 +33,8 @@ public class IcebergCatalogWrapperManagerForTest extends
IcebergCatalogWrapperMa
}
@Override
- public IcebergCatalogWrapper createIcebergCatalogWrapper(IcebergConfig
icebergConfig) {
- return new IcebergCatalogWrapperForTest(icebergConfig);
+ public CatalogWrapperForREST createCatalogWrapper(
+ String catalogName, IcebergConfig icebergConfig) {
+ return new CatalogWrapperForTest(catalogName, icebergConfig);
}
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java
index 19309dc05a..01c063f49c 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java
@@ -104,8 +104,7 @@ public class IcebergRestTestUtil {
StaticIcebergConfigProvider.class.getName());
catalogConf.put(String.format("%s.catalog-backend-name",
catalogConfigPrefix), PREFIX);
catalogConf.put(
- CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
- DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE);
+ CredentialConstants.CREDENTIAL_PROVIDERS,
DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE);
IcebergConfigProvider configProvider =
IcebergConfigProviderFactory.create(catalogConf);
configProvider.initialize(catalogConf);
// used to override register table interface
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/MockIcebergTableOperations.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/MockIcebergTableOperations.java
index 9b9bd93063..a6d9539c95 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/MockIcebergTableOperations.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/MockIcebergTableOperations.java
@@ -21,7 +21,6 @@ package org.apache.gravitino.iceberg.service.rest;
import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
-import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager;
import
org.apache.gravitino.iceberg.service.dispatcher.IcebergTableOperationDispatcher;
import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager;
@@ -29,10 +28,9 @@ public class MockIcebergTableOperations extends
IcebergTableOperations {
@Inject
public MockIcebergTableOperations(
- IcebergCatalogWrapperManager icebergCatalogWrapperManager,
IcebergMetricsManager icebergMetricsManager,
IcebergTableOperationDispatcher tableOperationDispatcher) {
- super(icebergCatalogWrapperManager, icebergMetricsManager,
tableOperationDispatcher);
+ super(icebergMetricsManager, tableOperationDispatcher);
}
// HTTP request is null in Jersey test, create a mock request
