This is an automated email from the ASF dual-hosted git repository.
diqiu50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 2bbb9f2a90 [#13598] feat(credential): Add Aws/Dlf secret-key
credentials (#13599)
2bbb9f2a90 is described below
commit 2bbb9f2a90935ab7fa73883d28289120f389df1b
Author: MaSai <[email protected]>
AuthorDate: Wed Sep 30 14:57:23 2026 +0800
[#13598] feat(credential): Add Aws/Dlf secret-key credentials (#13599)
### What changes were proposed in this pull request?
- Add `AwsSecretKeyCredential` + provider; wire `GlueCatalog`
- Add `DlfSecretKeyCredential` + provider; wire `PaimonCatalog` and
Spark/Flink Paimon injection
- Spark/Trino Glue connectors consume `AwsSecretKeyCredential` (Glue
API) and `S3SecretKeyCredential` (storage)
- Python client factory supports `aws-secret-key` / `dlf-secret-key`
- Hide `dlf-access-key-id`; omit `dlf-security-token` from `getSecrets`
without `INCLUDE_CREDENTIAL_SECRETS`
- When `credential-providers` is set explicitly, Glue still ensures
`aws-secret-key` but does **not** force-append `s3-secret-key`
(preserves `s3-token`)
- Explicit Paimon `credential-providers` still appends JDBC/DLF when
those keys are present (aligned with Jdbc/Iceberg catalogs)
### Why are the changes needed?
Completes Glue AWS and Paimon DLF coverage under credential vending
after #13539.
Fix: #13598
### Does this PR introduce _any_ user-facing change?
Yes:
- New credential types `aws-secret-key` and `dlf-secret-key` via
`getCredentials`
- `dlf-access-key-id` masked on catalog load
- `dlf-security-token` gated like other cloud credential secrets on
`getSecrets`
- Explicit Glue `credential-providers` no longer auto-appends
`s3-secret-key`
- Explicit Paimon `credential-providers` now also gets JDBC/DLF when
configured
### How was this patch tested?
Unit tests for providers, Glue/Paimon wiring, Spark/Trino Glue
credential injection, Python CredentialFactory, Paimon connector
property mapping, and cloud-key filtering.
---------
Co-authored-by: Cursor <[email protected]>
---
.../credential/AwsSecretKeyCredential.java | 112 +++++++++++++++
.../credential/DlfSecretKeyCredential.java | 150 ++++++++++++++++++++
.../org.apache.gravitino.credential.Credential | 2 +
.../lakehouse/paimon/PaimonPropertiesUtils.java | 12 ++
.../CloudStorageCredentialPropertyKeys.java | 3 +-
.../TestCloudStorageCredentialPropertyKeys.java | 25 ++++
.../apache/gravitino/catalog/glue/GlueCatalog.java | 12 +-
.../catalog/glue/TestGlueCatalogCredentials.java | 152 +++++++++++++++++++++
.../catalog/lakehouse/paimon/PaimonCatalog.java | 36 ++++-
.../paimon/PaimonCatalogPropertiesMetadata.java | 2 +-
.../lakehouse/paimon/TestPaimonCatalog.java | 145 +++++++++++++++++++-
.../TestPaimonCatalogPropertiesMetadata.java | 5 +-
.../api/credential/aws_secret_key_credential.py | 62 +++++++++
.../api/credential/dlf_secret_key_credential.py | 70 ++++++++++
.../gravitino/utils/credential_factory.py | 6 +
.../tests/unittests/test_credential_factory.py | 44 ++++++
.../credential/CredentialPropertyUtils.java | 13 +-
.../credential/TestCredentialFactory.java | 67 +++++++++
.../credential/TestCredentialPropertiesUtils.java | 39 ++++++
.../credential/AwsSecretKeyCredentialProvider.java | 70 ++++++++++
.../credential/DlfSecretKeyCredentialProvider.java | 72 ++++++++++
....apache.gravitino.credential.CredentialProvider | 2 +
.../TestAwsSecretKeyCredentialProvider.java | 72 ++++++++++
.../TestDlfSecretKeyCredentialProvider.java | 83 +++++++++++
....apache.gravitino.credential.CredentialProvider | 2 +
design-docs/gravitino-glue-catalog.md | 4 +-
docs/aws-glue-catalog.md | 7 +-
docs/lakehouse-paimon-catalog.md | 8 +-
docs/security/credential-vending.md | 15 +-
.../paimon/TestGravitinoPaimonCatalog.java | 20 +++
.../paimon/TestPaimonPropertiesConverter.java | 27 ++++
.../spark/connector/glue/GravitinoGlueCatalog.java | 29 ++--
.../connector/glue/TestGravitinoGlueCatalog.java | 19 +++
.../catalog/glue/GlueConnectorAdapter.java | 26 +++-
.../catalog/glue/TestGlueConnectorAdapter.java | 15 ++
35 files changed, 1382 insertions(+), 46 deletions(-)
diff --git
a/api/src/main/java/org/apache/gravitino/credential/AwsSecretKeyCredential.java
b/api/src/main/java/org/apache/gravitino/credential/AwsSecretKeyCredential.java
new file mode 100644
index 0000000000..25c7a53bef
--- /dev/null
+++
b/api/src/main/java/org/apache/gravitino/credential/AwsSecretKeyCredential.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Static AWS access-key credential for Glue (and similar) API authentication.
+ *
+ * <p>Distinct from {@link S3SecretKeyCredential}: credential-info keys are
{@code
+ * aws-access-key-id} / {@code aws-secret-access-key}, matching Glue catalog
properties.
+ */
+public class AwsSecretKeyCredential implements Credential {
+
+ /** AWS secret-key credential type. */
+ public static final String AWS_SECRET_KEY_CREDENTIAL_TYPE = "aws-secret-key";
+ /** AWS access key ID. */
+ public static final String GRAVITINO_AWS_ACCESS_KEY_ID = "aws-access-key-id";
+ /** AWS secret access key. */
+ public static final String GRAVITINO_AWS_SECRET_ACCESS_KEY =
"aws-secret-access-key";
+
+ private String accessKeyId;
+ private String secretAccessKey;
+
+ /**
+ * Constructs an {@link AwsSecretKeyCredential}.
+ *
+ * @param accessKeyId the AWS access key ID
+ * @param secretAccessKey the AWS secret access key
+ */
+ public AwsSecretKeyCredential(String accessKeyId, String secretAccessKey) {
+ validate(accessKeyId, secretAccessKey, 0);
+ this.accessKeyId = accessKeyId;
+ this.secretAccessKey = secretAccessKey;
+ }
+
+ /** Used by the credential factory. */
+ public AwsSecretKeyCredential() {}
+
+ @Override
+ public String credentialType() {
+ return AWS_SECRET_KEY_CREDENTIAL_TYPE;
+ }
+
+ @Override
+ public long expireTimeInMs() {
+ return 0;
+ }
+
+ @Override
+ public Map<String, String> credentialInfo() {
+ return new ImmutableMap.Builder<String, String>()
+ .put(GRAVITINO_AWS_ACCESS_KEY_ID, accessKeyId)
+ .put(GRAVITINO_AWS_SECRET_ACCESS_KEY, secretAccessKey)
+ .build();
+ }
+
+ @Override
+ public void initialize(Map<String, String> credentialInfo, long
expireTimeInMs) {
+ String accessKeyId = credentialInfo.get(GRAVITINO_AWS_ACCESS_KEY_ID);
+ String secretAccessKey =
credentialInfo.get(GRAVITINO_AWS_SECRET_ACCESS_KEY);
+ validate(accessKeyId, secretAccessKey, expireTimeInMs);
+ this.accessKeyId = accessKeyId;
+ this.secretAccessKey = secretAccessKey;
+ }
+
+ /**
+ * Returns the AWS access key ID.
+ *
+ * @return access key ID
+ */
+ public String accessKeyId() {
+ return accessKeyId;
+ }
+
+ /**
+ * Returns the AWS secret access key.
+ *
+ * @return secret access key
+ */
+ public String secretAccessKey() {
+ return secretAccessKey;
+ }
+
+ private void validate(String accessKeyId, String secretAccessKey, long
expireTimeInMs) {
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(accessKeyId), "AWS access key ID should not be
empty");
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(secretAccessKey), "AWS secret access key should
not be empty");
+ Preconditions.checkArgument(
+ expireTimeInMs == 0, "The expiration time of AwsSecretKeyCredential
should be 0");
+ }
+}
diff --git
a/api/src/main/java/org/apache/gravitino/credential/DlfSecretKeyCredential.java
b/api/src/main/java/org/apache/gravitino/credential/DlfSecretKeyCredential.java
new file mode 100644
index 0000000000..a10921512a
--- /dev/null
+++
b/api/src/main/java/org/apache/gravitino/credential/DlfSecretKeyCredential.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Static Alibaba Cloud DLF (Data Lake Formation) access-key credential for
Paimon DLF catalogs.
+ *
+ * <p>Credential-info keys match Gravitino Paimon catalog properties: {@code
dlf-access-key-id},
+ * {@code dlf-access-key-secret}, and optionally {@code dlf-security-token}.
Connectors map these to
+ * Paimon REST keys {@code dlf.access-key-id}, {@code dlf.access-key-secret},
and {@code
+ * dlf.security-token}.
+ *
+ * <p>{@code dlf-security-token} is passed through as-is. {@link
#expireTimeInMs()} is always {@code
+ * 0}; Gravitino does not refresh this STS token. For rotating DLF tokens,
configure {@code
+ * dlf-token-loader} / {@code dlf-token-path} on the Paimon catalog instead.
+ */
+public class DlfSecretKeyCredential implements Credential {
+
+ /** DLF secret-key credential type. */
+ public static final String DLF_SECRET_KEY_CREDENTIAL_TYPE = "dlf-secret-key";
+ /** DLF access key ID. */
+ public static final String GRAVITINO_DLF_ACCESS_KEY_ID = "dlf-access-key-id";
+ /** DLF access key secret. */
+ public static final String GRAVITINO_DLF_ACCESS_KEY_SECRET =
"dlf-access-key-secret";
+ /** Optional DLF security token. */
+ public static final String GRAVITINO_DLF_SECURITY_TOKEN =
"dlf-security-token";
+
+ private String accessKeyId;
+ private String accessKeySecret;
+ @Nullable private String securityToken;
+
+ /**
+ * Constructs a {@link DlfSecretKeyCredential} without a security token.
+ *
+ * @param accessKeyId the DLF access key ID
+ * @param accessKeySecret the DLF access key secret
+ */
+ public DlfSecretKeyCredential(String accessKeyId, String accessKeySecret) {
+ this(accessKeyId, accessKeySecret, null);
+ }
+
+ /**
+ * Constructs a {@link DlfSecretKeyCredential}.
+ *
+ * @param accessKeyId the DLF access key ID
+ * @param accessKeySecret the DLF access key secret
+ * @param securityToken optional security token
+ */
+ public DlfSecretKeyCredential(
+ String accessKeyId, String accessKeySecret, @Nullable String
securityToken) {
+ validate(accessKeyId, accessKeySecret, 0);
+ this.accessKeyId = accessKeyId;
+ this.accessKeySecret = accessKeySecret;
+ this.securityToken = securityToken;
+ }
+
+ /** Used by the credential factory. */
+ public DlfSecretKeyCredential() {}
+
+ @Override
+ public String credentialType() {
+ return DLF_SECRET_KEY_CREDENTIAL_TYPE;
+ }
+
+ @Override
+ public long expireTimeInMs() {
+ return 0;
+ }
+
+ @Override
+ public Map<String, String> credentialInfo() {
+ ImmutableMap.Builder<String, String> builder =
+ new ImmutableMap.Builder<String, String>()
+ .put(GRAVITINO_DLF_ACCESS_KEY_ID, accessKeyId)
+ .put(GRAVITINO_DLF_ACCESS_KEY_SECRET, accessKeySecret);
+ if (StringUtils.isNotBlank(securityToken)) {
+ builder.put(GRAVITINO_DLF_SECURITY_TOKEN, securityToken);
+ }
+ return builder.build();
+ }
+
+ @Override
+ public void initialize(Map<String, String> credentialInfo, long
expireTimeInMs) {
+ String accessKeyId = credentialInfo.get(GRAVITINO_DLF_ACCESS_KEY_ID);
+ String accessKeySecret =
credentialInfo.get(GRAVITINO_DLF_ACCESS_KEY_SECRET);
+ validate(accessKeyId, accessKeySecret, expireTimeInMs);
+ this.accessKeyId = accessKeyId;
+ this.accessKeySecret = accessKeySecret;
+ this.securityToken = credentialInfo.get(GRAVITINO_DLF_SECURITY_TOKEN);
+ }
+
+ /**
+ * Returns the DLF access key ID.
+ *
+ * @return access key ID
+ */
+ public String accessKeyId() {
+ return accessKeyId;
+ }
+
+ /**
+ * Returns the DLF access key secret.
+ *
+ * @return access key secret
+ */
+ public String accessKeySecret() {
+ return accessKeySecret;
+ }
+
+ /**
+ * Returns the optional DLF security token.
+ *
+ * @return security token, or null
+ */
+ @Nullable
+ public String securityToken() {
+ return securityToken;
+ }
+
+ private void validate(String accessKeyId, String accessKeySecret, long
expireTimeInMs) {
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(accessKeyId), "DLF access key ID should not be
empty");
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(accessKeySecret), "DLF access key secret should
not be empty");
+ Preconditions.checkArgument(
+ expireTimeInMs == 0, "The expiration time of DlfSecretKeyCredential
should be 0");
+ }
+}
diff --git
a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
index 5e4f705c5b..bfed13388b 100644
---
a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
+++
b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
@@ -28,3 +28,5 @@ org.apache.gravitino.credential.ADLSTokenCredential
org.apache.gravitino.credential.AzureAccountKeyCredential
org.apache.gravitino.credential.AwsIrsaCredential
org.apache.gravitino.credential.JdbcCredential
+org.apache.gravitino.credential.AwsSecretKeyCredential
+org.apache.gravitino.credential.DlfSecretKeyCredential
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java
index 7b1832fe56..415e47db0b 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java
@@ -60,6 +60,18 @@ public class PaimonPropertiesUtils {
OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID,
PaimonConstants.OSS_ACCESS_KEY);
gravitinoConfigToPaimon.put(
OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET,
PaimonConstants.OSS_SECRET_KEY);
+ // DLF (Data Lake Formation)
+ gravitinoConfigToPaimon.put(
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID,
PaimonConstants.PAIMON_DLF_ACCESS_KEY_ID);
+ gravitinoConfigToPaimon.put(
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ PaimonConstants.PAIMON_DLF_ACCESS_KEY_SECRET);
+ gravitinoConfigToPaimon.put(
+ PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN,
PaimonConstants.PAIMON_DLF_SECURITY_TOKEN);
+ gravitinoConfigToPaimon.put(
+ PaimonConstants.GRAVITINO_DLF_TOKEN_PATH,
PaimonConstants.PAIMON_DLF_TOKEN_PATH);
+ gravitinoConfigToPaimon.put(
+ PaimonConstants.GRAVITINO_DLF_TOKEN_LOADER,
PaimonConstants.PAIMON_DLF_TOKEN_LOADER);
GRAVITINO_CONFIG_TO_PAIMON =
Collections.unmodifiableMap(gravitinoConfigToPaimon);
gravitinoConfigToPaimon.forEach(
(key, value) -> {
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/CloudStorageCredentialPropertyKeys.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/CloudStorageCredentialPropertyKeys.java
index 400c46fb14..fd97207ed8 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/CloudStorageCredentialPropertyKeys.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/CloudStorageCredentialPropertyKeys.java
@@ -80,7 +80,8 @@ public final class CloudStorageCredentialPropertyKeys {
AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
AzureProperties.GRAVITINO_AZURE_CLIENT_SECRET,
PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID,
- PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET);
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN);
private CloudStorageCredentialPropertyKeys() {}
diff --git
a/catalogs/catalog-common/src/test/java/org/apache/gravitino/storage/TestCloudStorageCredentialPropertyKeys.java
b/catalogs/catalog-common/src/test/java/org/apache/gravitino/storage/TestCloudStorageCredentialPropertyKeys.java
index 7319467126..09e25f1b64 100644
---
a/catalogs/catalog-common/src/test/java/org/apache/gravitino/storage/TestCloudStorageCredentialPropertyKeys.java
+++
b/catalogs/catalog-common/src/test/java/org/apache/gravitino/storage/TestCloudStorageCredentialPropertyKeys.java
@@ -156,6 +156,31 @@ public class TestCloudStorageCredentialPropertyKeys {
assertTrue(
CloudStorageCredentialPropertyKeys.isCloudAccessKeyPairKey(
PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET));
+ assertTrue(
+ CloudStorageCredentialPropertyKeys.isCloudAccessKeyPairKey(
+ PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN));
assertFalse(CloudStorageCredentialPropertyKeys.isCloudAccessKeyPairKey("jdbc-password"));
}
+
+ @Test
+ void testOmitCloudAccessKeyPairOmitsDlfSecurityToken() {
+ Map<String, String> secrets =
+ Map.of(
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID,
+ "dlf-ak",
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ "dlf-sk",
+ PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN,
+ "dlf-token",
+ "jdbc-password",
+ "db-pass");
+
+ Map<String, String> filtered =
+
CloudStorageCredentialPropertyKeys.omitCloudAccessKeyPairProperties(secrets);
+
+ assertEquals("db-pass", filtered.get("jdbc-password"));
+
assertFalse(filtered.containsKey(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID));
+
assertFalse(filtered.containsKey(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET));
+
assertFalse(filtered.containsKey(PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN));
+ }
}
diff --git
a/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GlueCatalog.java
b/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GlueCatalog.java
index 8222c0d71b..ae460201e9 100644
---
a/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GlueCatalog.java
+++
b/catalogs/catalog-glue/src/main/java/org/apache/gravitino/catalog/glue/GlueCatalog.java
@@ -25,6 +25,7 @@ import org.apache.gravitino.connector.BaseCatalog;
import org.apache.gravitino.connector.CatalogOperations;
import org.apache.gravitino.connector.PropertiesMetadata;
import org.apache.gravitino.connector.capability.Capability;
+import org.apache.gravitino.credential.AwsSecretKeyCredential;
import org.apache.gravitino.storage.S3Properties;
/**
@@ -89,14 +90,17 @@ public class GlueCatalog extends BaseCatalog<GlueCatalog> {
public Map<String, String> propertiesWithCredentialProviders() {
Map<String, String> props = super.propertiesWithCredentialProviders();
// super() skips addCatalogSpecificCredentialProviders() when
credential-providers is already
- // set, so the aws-* → s3-* key mapping never runs. Apply it
unconditionally here so that
- // S3SecretKeyProvider.initialize() can read s3-access-key-id regardless
of how the catalog
- // was configured.
+ // set, so the aws-* → s3-* key mapping never runs. Remap keys and ensure
aws-secret-key so Glue
+ // API credentials stay available via getCredentials. Do not force-append
s3-secret-key when the
+ // user set credential-providers explicitly (e.g. s3-token); that would
inject static S3 keys
+ // beside STS and break path-based credential selection. Auto-detect still
registers
+ // s3-secret-key via addStorageCredentialProviders when providers are
omitted.
String accessKeyId = props.get(GlueConstants.AWS_ACCESS_KEY_ID);
String secretAccessKey = props.get(GlueConstants.AWS_SECRET_ACCESS_KEY);
if (StringUtils.isNotBlank(accessKeyId) &&
StringUtils.isNotBlank(secretAccessKey)) {
props.putIfAbsent(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKeyId);
props.putIfAbsent(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY,
secretAccessKey);
+ ensureCredentialProviderListed(props,
AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE);
}
return props;
}
@@ -107,6 +111,8 @@ public class GlueCatalog extends BaseCatalog<GlueCatalog> {
String accessKeyId = properties.get(GlueConstants.AWS_ACCESS_KEY_ID);
String secretAccessKey =
properties.get(GlueConstants.AWS_SECRET_ACCESS_KEY);
if (StringUtils.isNotBlank(accessKeyId) &&
StringUtils.isNotBlank(secretAccessKey)) {
+ // Glue API auth via aws-* keys; also remap to s3-* for storage
credential vending.
+
credentialProviders.add(AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE);
properties.putIfAbsent(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID,
accessKeyId);
properties.putIfAbsent(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY,
secretAccessKey);
}
diff --git
a/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/TestGlueCatalogCredentials.java
b/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/TestGlueCatalogCredentials.java
new file mode 100644
index 0000000000..90ee4804c5
--- /dev/null
+++
b/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/TestGlueCatalogCredentials.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.glue;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.gravitino.Catalog;
+import org.apache.gravitino.Namespace;
+import org.apache.gravitino.credential.AwsSecretKeyCredential;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.credential.S3SecretKeyCredential;
+import org.apache.gravitino.credential.S3TokenCredential;
+import org.apache.gravitino.meta.AuditInfo;
+import org.apache.gravitino.meta.CatalogEntity;
+import org.apache.gravitino.storage.S3Properties;
+import org.junit.jupiter.api.Test;
+
+public class TestGlueCatalogCredentials {
+
+ @Test
+ void testAwsKeysAutoDetectAwsAndS3Providers() {
+ Map<String, String> props =
+ Map.of(
+ GlueConstants.AWS_ACCESS_KEY_ID,
+ "AKIA",
+ GlueConstants.AWS_SECRET_ACCESS_KEY,
+ "secret",
+ GlueConstants.AWS_REGION,
+ "us-east-1");
+
+ CatalogEntity entity =
+ CatalogEntity.builder()
+ .withId(1L)
+ .withName("glue")
+ .withNamespace(Namespace.of("metalake"))
+ .withType(Catalog.Type.RELATIONAL)
+ .withProvider("glue")
+ .withProperties(props)
+ .withAuditInfo(
+
AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build())
+ .build();
+
+ GlueCatalog catalog = new
GlueCatalog().withCatalogEntity(entity).withCatalogConf(props);
+ Map<String, String> withProviders =
catalog.propertiesWithCredentialProviders();
+
+ List<String> providers = splitProviders(withProviders);
+
assertTrue(providers.contains(AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE));
+
assertTrue(providers.contains(S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE));
+ assertEquals("AKIA",
withProviders.get(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID));
+ assertEquals("secret",
withProviders.get(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY));
+ assertEquals("AKIA", withProviders.get(GlueConstants.AWS_ACCESS_KEY_ID));
+ }
+
+ @Test
+ void testExplicitCredentialProvidersStillGetsAwsButNotS3SecretKey() {
+ Map<String, String> props =
+ Map.of(
+ GlueConstants.AWS_ACCESS_KEY_ID,
+ "AKIA",
+ GlueConstants.AWS_SECRET_ACCESS_KEY,
+ "secret",
+ GlueConstants.AWS_REGION,
+ "us-east-1",
+ CredentialConstants.CREDENTIAL_PROVIDERS,
+ "custom-provider");
+
+ CatalogEntity entity =
+ CatalogEntity.builder()
+ .withId(2L)
+ .withName("glue-explicit")
+ .withNamespace(Namespace.of("metalake"))
+ .withType(Catalog.Type.RELATIONAL)
+ .withProvider("glue")
+ .withProperties(props)
+ .withAuditInfo(
+
AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build())
+ .build();
+
+ GlueCatalog catalog = new
GlueCatalog().withCatalogEntity(entity).withCatalogConf(props);
+ Map<String, String> withProviders =
catalog.propertiesWithCredentialProviders();
+
+ List<String> providers = splitProviders(withProviders);
+ assertEquals(
+ List.of("custom-provider",
AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE),
+ providers);
+
assertFalse(providers.contains(S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE));
+ assertEquals("AKIA",
withProviders.get(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID));
+ }
+
+ @Test
+ void testExplicitS3TokenDoesNotAppendS3SecretKey() {
+ Map<String, String> props =
+ Map.of(
+ GlueConstants.AWS_ACCESS_KEY_ID,
+ "AKIA",
+ GlueConstants.AWS_SECRET_ACCESS_KEY,
+ "secret",
+ GlueConstants.AWS_REGION,
+ "us-east-1",
+ CredentialConstants.CREDENTIAL_PROVIDERS,
+ S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE);
+
+ CatalogEntity entity =
+ CatalogEntity.builder()
+ .withId(3L)
+ .withName("glue-s3-token")
+ .withNamespace(Namespace.of("metalake"))
+ .withType(Catalog.Type.RELATIONAL)
+ .withProvider("glue")
+ .withProperties(props)
+ .withAuditInfo(
+
AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build())
+ .build();
+
+ GlueCatalog catalog = new
GlueCatalog().withCatalogEntity(entity).withCatalogConf(props);
+ Map<String, String> withProviders =
catalog.propertiesWithCredentialProviders();
+
+ assertEquals(
+ List.of(
+ S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE,
+ AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE),
+ splitProviders(withProviders));
+ }
+
+ private static List<String> splitProviders(Map<String, String>
withProviders) {
+ String providers =
withProviders.get(CredentialConstants.CREDENTIAL_PROVIDERS);
+ return
Arrays.stream(providers.split(",")).map(String::trim).collect(Collectors.toList());
+ }
+}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java
index 5c7afa7a63..1898072b67 100644
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java
+++
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java
@@ -20,12 +20,14 @@ package org.apache.gravitino.catalog.lakehouse.paimon;
import java.util.List;
import java.util.Map;
+import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.connector.BaseCatalog;
import org.apache.gravitino.connector.CatalogOperations;
import org.apache.gravitino.connector.PropertiesMetadata;
import org.apache.gravitino.connector.capability.Capability;
+import org.apache.gravitino.credential.DlfSecretKeyCredential;
import org.apache.gravitino.credential.JdbcCredential;
import org.apache.gravitino.rel.ViewCatalog;
@@ -88,8 +90,24 @@ public class PaimonCatalog extends
BaseCatalog<PaimonCatalog> {
}
/**
- * Adds a JDBC credential provider when the backend is JDBC and credentials
are configured, then
- * delegates to the parent for storage (S3/OSS/Azure/GCS) credential
provider detection.
+ * Ensures JDBC / DLF providers stay listed even when {@code
credential-providers} was set
+ * explicitly. {@code super} skips {@link
#addCatalogSpecificCredentialProviders} in that case, so
+ * without this override {@code getCredentials} would not vend JDBC/DLF
(callers with {@code
+ * USE_SECRETS} can still recover those keys via {@code getSecrets}; DLF
cloud pairs also need
+ * {@code INCLUDE_CREDENTIAL_SECRETS}).
+ *
+ * @return catalog properties with credential providers
+ */
+ @Override
+ public Map<String, String> propertiesWithCredentialProviders() {
+ Map<String, String> props = super.propertiesWithCredentialProviders();
+ detectJdbcAndDlfProviders(props, type ->
ensureCredentialProviderListed(props, type));
+ return props;
+ }
+
+ /**
+ * Adds JDBC and/or DLF credential providers when those credentials are
configured, then delegates
+ * to the parent for storage (S3/OSS/Azure/GCS) credential provider
detection.
*
* @param properties the raw catalog properties
* @param credentialProviders the list to append detected provider names to
@@ -97,15 +115,25 @@ public class PaimonCatalog extends
BaseCatalog<PaimonCatalog> {
@Override
protected void addCatalogSpecificCredentialProviders(
Map<String, String> properties, List<String> credentialProviders) {
+ detectJdbcAndDlfProviders(properties, credentialProviders::add);
+ addStorageCredentialProviders(properties, credentialProviders);
+ }
+
+ private static void detectJdbcAndDlfProviders(
+ Map<String, String> properties, Consumer<String> onProvider) {
String catalogBackend = properties.get(PaimonConstants.CATALOG_BACKEND);
if (catalogBackend != null
&& PaimonCatalogBackend.JDBC.name().equalsIgnoreCase(catalogBackend)) {
String jdbcUser = properties.get(PaimonConstants.GRAVITINO_JDBC_USER);
String jdbcPassword =
properties.get(PaimonConstants.GRAVITINO_JDBC_PASSWORD);
if (StringUtils.isNotBlank(jdbcUser) && jdbcPassword != null) {
- credentialProviders.add(JdbcCredential.JDBC_CREDENTIAL_TYPE);
+ onProvider.accept(JdbcCredential.JDBC_CREDENTIAL_TYPE);
}
}
- addStorageCredentialProviders(properties, credentialProviders);
+ String dlfAccessKeyId =
properties.get(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID);
+ String dlfAccessKeySecret =
properties.get(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET);
+ if (StringUtils.isNotBlank(dlfAccessKeyId) &&
StringUtils.isNotBlank(dlfAccessKeySecret)) {
+ onProvider.accept(DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE);
+ }
}
}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
index 2092d53ced..270a1602bf 100644
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
@@ -132,7 +132,7 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
"The access key ID for Aliyun DLF",
false /* immutable */,
null /* defaultValue */,
- false /* hidden */))
+ true /* hidden */))
.put(
PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET,
stringOptionalPropertyEntry(
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
index 998d1d9386..a6c267de84 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
@@ -39,6 +39,7 @@ import org.apache.gravitino.connector.HasPropertyMetadata;
import org.apache.gravitino.connector.PropertiesMetadata;
import org.apache.gravitino.credential.AzureAccountKeyCredential;
import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.credential.DlfSecretKeyCredential;
import org.apache.gravitino.credential.JdbcCredential;
import org.apache.gravitino.credential.OSSSecretKeyCredential;
import org.apache.gravitino.credential.S3SecretKeyCredential;
@@ -313,6 +314,38 @@ public class TestPaimonCatalog {
Assertions.assertNull(credentialProviders);
}
+ @Test
+ void testDlfCredentialProviderAutoDetected() {
+ AuditInfo auditInfo =
+
AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build();
+
+ Map<String, String> dlfProps = Maps.newHashMap();
+ dlfProps.put(PaimonConstants.CATALOG_BACKEND, "rest");
+ dlfProps.put(PaimonConstants.WAREHOUSE, tempDir);
+ dlfProps.put(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID, "dlf-ak");
+ dlfProps.put(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET, "dlf-sk");
+
+ CatalogEntity dlfEntity =
+ CatalogEntity.builder()
+ .withId(4L)
+ .withName("dlf-catalog")
+ .withNamespace(Namespace.of("metalake"))
+ .withType(PaimonCatalog.Type.RELATIONAL)
+ .withProvider("lakehouse-paimon")
+ .withAuditInfo(auditInfo)
+ .withProperties(dlfProps)
+ .build();
+
+ PaimonCatalog dlfCatalog =
+ new
PaimonCatalog().withCatalogConf(dlfProps).withCatalogEntity(dlfEntity);
+ Map<String, String> properties =
dlfCatalog.propertiesWithCredentialProviders();
+
+ String credentialProviders =
properties.get(CredentialConstants.CREDENTIAL_PROVIDERS);
+ Assertions.assertNotNull(credentialProviders);
+ Assertions.assertTrue(
+
credentialProviders.contains(DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE));
+ }
+
@Test
void testJdbcBackendWithOSSCredentialProviders() {
AuditInfo auditInfo =
@@ -390,11 +423,40 @@ public class TestPaimonCatalog {
}
@Test
- void testExplicitCredentialProvidersNotOverridden() {
+ void testExplicitCredentialProvidersWithoutJdbcOrDlfUnchanged() {
AuditInfo auditInfo =
AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build();
- // Test that explicit credential-providers setting is not overridden
+ Map<String, String> explicitProps = Maps.newHashMap();
+ explicitProps.put(PaimonConstants.CATALOG_BACKEND, "filesystem");
+ explicitProps.put(PaimonConstants.WAREHOUSE, tempDir);
+ explicitProps.put(CredentialConstants.CREDENTIAL_PROVIDERS,
"custom-provider");
+
+ CatalogEntity explicitEntity =
+ CatalogEntity.builder()
+ .withId(4L)
+ .withName("explicit-catalog")
+ .withNamespace(Namespace.of("metalake"))
+ .withType(PaimonCatalog.Type.RELATIONAL)
+ .withProvider("lakehouse-paimon")
+ .withAuditInfo(auditInfo)
+ .withProperties(explicitProps)
+ .build();
+
+ PaimonCatalog explicitCatalog =
+ new
PaimonCatalog().withCatalogConf(explicitProps).withCatalogEntity(explicitEntity);
+ Map<String, String> properties =
explicitCatalog.propertiesWithCredentialProviders();
+
+ Assertions.assertEquals(
+ "custom-provider",
properties.get(CredentialConstants.CREDENTIAL_PROVIDERS));
+ }
+
+ @Test
+ void testExplicitCredentialProvidersStillGetsJdbc() {
+ AuditInfo auditInfo =
+
AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build();
+
+ // Explicit credential-providers is kept, and JDBC is still appended when
JDBC keys exist.
Map<String, String> explicitProps = Maps.newHashMap();
explicitProps.put(PaimonConstants.CATALOG_BACKEND, "jdbc");
explicitProps.put(PaimonConstants.URI, "jdbc:sqlite::memory:");
@@ -405,8 +467,8 @@ public class TestPaimonCatalog {
CatalogEntity explicitEntity =
CatalogEntity.builder()
- .withId(4L)
- .withName("explicit-catalog")
+ .withId(5L)
+ .withName("explicit-jdbc-catalog")
.withNamespace(Namespace.of("metalake"))
.withType(PaimonCatalog.Type.RELATIONAL)
.withProvider("lakehouse-paimon")
@@ -418,9 +480,78 @@ public class TestPaimonCatalog {
new
PaimonCatalog().withCatalogConf(explicitProps).withCatalogEntity(explicitEntity);
Map<String, String> properties =
explicitCatalog.propertiesWithCredentialProviders();
- // Should keep explicit credential providers, not override
- String credentialProviders =
properties.get(CredentialConstants.CREDENTIAL_PROVIDERS);
- Assertions.assertEquals("custom-provider", credentialProviders);
+ Assertions.assertEquals(
+ "custom-provider," + JdbcCredential.JDBC_CREDENTIAL_TYPE,
+ properties.get(CredentialConstants.CREDENTIAL_PROVIDERS));
+ }
+
+ @Test
+ void testExplicitCredentialProvidersStillGetsDlf() {
+ AuditInfo auditInfo =
+
AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build();
+
+ Map<String, String> dlfProps = Maps.newHashMap();
+ dlfProps.put(PaimonConstants.CATALOG_BACKEND, "rest");
+ dlfProps.put(PaimonConstants.WAREHOUSE, tempDir);
+ dlfProps.put(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID, "dlf-ak");
+ dlfProps.put(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET, "dlf-sk");
+ dlfProps.put(CredentialConstants.CREDENTIAL_PROVIDERS, "custom-provider");
+
+ CatalogEntity dlfEntity =
+ CatalogEntity.builder()
+ .withId(8L)
+ .withName("dlf-explicit-catalog")
+ .withNamespace(Namespace.of("metalake"))
+ .withType(PaimonCatalog.Type.RELATIONAL)
+ .withProvider("lakehouse-paimon")
+ .withAuditInfo(auditInfo)
+ .withProperties(dlfProps)
+ .build();
+
+ PaimonCatalog dlfCatalog =
+ new
PaimonCatalog().withCatalogConf(dlfProps).withCatalogEntity(dlfEntity);
+ Map<String, String> properties =
dlfCatalog.propertiesWithCredentialProviders();
+
+ Assertions.assertEquals(
+ "custom-provider," +
DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
+ properties.get(CredentialConstants.CREDENTIAL_PROVIDERS));
+ }
+
+ @Test
+ void testExplicitCredentialProvidersStillGetsJdbcAndDlf() {
+ AuditInfo auditInfo =
+
AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build();
+
+ Map<String, String> props = Maps.newHashMap();
+ props.put(PaimonConstants.CATALOG_BACKEND, "jdbc");
+ props.put(PaimonConstants.URI, "jdbc:sqlite::memory:");
+ props.put(PaimonConstants.WAREHOUSE, tempDir);
+ props.put(PaimonConstants.GRAVITINO_JDBC_USER, "test-user");
+ props.put(PaimonConstants.GRAVITINO_JDBC_PASSWORD, "test-password");
+ props.put(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID, "dlf-ak");
+ props.put(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET, "dlf-sk");
+ props.put(CredentialConstants.CREDENTIAL_PROVIDERS, "custom-provider");
+
+ CatalogEntity entity =
+ CatalogEntity.builder()
+ .withId(9L)
+ .withName("jdbc-dlf-explicit-catalog")
+ .withNamespace(Namespace.of("metalake"))
+ .withType(PaimonCatalog.Type.RELATIONAL)
+ .withProvider("lakehouse-paimon")
+ .withAuditInfo(auditInfo)
+ .withProperties(props)
+ .build();
+
+ PaimonCatalog catalog = new
PaimonCatalog().withCatalogConf(props).withCatalogEntity(entity);
+ Map<String, String> properties =
catalog.propertiesWithCredentialProviders();
+
+ Assertions.assertEquals(
+ "custom-provider,"
+ + JdbcCredential.JDBC_CREDENTIAL_TYPE
+ + ","
+ + DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
+ properties.get(CredentialConstants.CREDENTIAL_PROVIDERS));
}
private PaimonCatalog newPaimonCatalog(String catalogName) {
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalogPropertiesMetadata.java
index bdfe09903a..d19b24061d 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalogPropertiesMetadata.java
@@ -42,10 +42,9 @@ public class TestPaimonCatalogPropertiesMetadata {
assertTrue(metadata.isHiddenProperty(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY));
assertTrue(metadata.isHiddenProperty(COSProperties.GRAVITINO_COS_ACCESS_KEY_ID));
assertTrue(metadata.isHiddenProperty(COSProperties.GRAVITINO_COS_ACCESS_KEY_SECRET));
- // REST bearer token and DLF secret halves are hidden; DLF access-key-id
stays visible until
- // a dedicated credential type lands.
+ // REST bearer token and DLF credential keys (including access-key-id) are
hidden.
assertTrue(metadata.isHiddenProperty(PaimonConstants.TOKEN));
-
assertFalse(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID));
+
assertTrue(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID));
assertTrue(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET));
assertTrue(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN));
}
diff --git
a/clients/client-python/gravitino/api/credential/aws_secret_key_credential.py
b/clients/client-python/gravitino/api/credential/aws_secret_key_credential.py
new file mode 100644
index 0000000000..d0f4cb7ef1
--- /dev/null
+++
b/clients/client-python/gravitino/api/credential/aws_secret_key_credential.py
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC
+from typing import Dict
+
+from gravitino.api.credential.credential import Credential
+from gravitino.utils.precondition import Precondition
+
+
+class AwsSecretKeyCredential(Credential, ABC):
+ """Represents AWS secret key credential for Glue API authentication."""
+
+ AWS_SECRET_KEY_CREDENTIAL_TYPE: str = "aws-secret-key"
+ _ACCESS_KEY_ID: str = "aws-access-key-id"
+ _SECRET_ACCESS_KEY: str = "aws-secret-access-key"
+
+ def __init__(self, credential_info: Dict[str, str], expire_time: int):
+ self._access_key_id = credential_info.get(self._ACCESS_KEY_ID, None)
+ self._secret_access_key = credential_info.get(self._SECRET_ACCESS_KEY,
None)
+ Precondition.check_string_not_empty(
+ self._access_key_id, "AWS access key id should not be empty"
+ )
+ Precondition.check_string_not_empty(
+ self._secret_access_key, "AWS secret access key should not be
empty"
+ )
+ Precondition.check_argument(
+ expire_time == 0,
+ "The expiration time of AWS secret key credential should be 0",
+ )
+
+ def credential_type(self) -> str:
+ return self.AWS_SECRET_KEY_CREDENTIAL_TYPE
+
+ def expire_time_in_ms(self) -> int:
+ return 0
+
+ def credential_info(self) -> Dict[str, str]:
+ return {
+ self._ACCESS_KEY_ID: self._access_key_id,
+ self._SECRET_ACCESS_KEY: self._secret_access_key,
+ }
+
+ def access_key_id(self) -> str:
+ return self._access_key_id
+
+ def secret_access_key(self) -> str:
+ return self._secret_access_key
diff --git
a/clients/client-python/gravitino/api/credential/dlf_secret_key_credential.py
b/clients/client-python/gravitino/api/credential/dlf_secret_key_credential.py
new file mode 100644
index 0000000000..7b3d048e4d
--- /dev/null
+++
b/clients/client-python/gravitino/api/credential/dlf_secret_key_credential.py
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC
+from typing import Dict, Optional
+
+from gravitino.api.credential.credential import Credential
+from gravitino.utils.precondition import Precondition
+
+
+class DlfSecretKeyCredential(Credential, ABC):
+ """Represents DLF secret key credential for Paimon DLF catalogs."""
+
+ DLF_SECRET_KEY_CREDENTIAL_TYPE: str = "dlf-secret-key"
+ _ACCESS_KEY_ID: str = "dlf-access-key-id"
+ _ACCESS_KEY_SECRET: str = "dlf-access-key-secret"
+ _SECURITY_TOKEN: str = "dlf-security-token"
+
+ def __init__(self, credential_info: Dict[str, str], expire_time: int):
+ self._access_key_id = credential_info.get(self._ACCESS_KEY_ID, None)
+ self._access_key_secret = credential_info.get(self._ACCESS_KEY_SECRET,
None)
+ self._security_token = credential_info.get(self._SECURITY_TOKEN, None)
+ Precondition.check_string_not_empty(
+ self._access_key_id, "DLF access key id should not be empty"
+ )
+ Precondition.check_string_not_empty(
+ self._access_key_secret, "DLF access key secret should not be
empty"
+ )
+ Precondition.check_argument(
+ expire_time == 0,
+ "The expiration time of DLF secret key credential should be 0",
+ )
+
+ def credential_type(self) -> str:
+ return self.DLF_SECRET_KEY_CREDENTIAL_TYPE
+
+ def expire_time_in_ms(self) -> int:
+ return 0
+
+ def credential_info(self) -> Dict[str, str]:
+ info = {
+ self._ACCESS_KEY_ID: self._access_key_id,
+ self._ACCESS_KEY_SECRET: self._access_key_secret,
+ }
+ if self._security_token:
+ info[self._SECURITY_TOKEN] = self._security_token
+ return info
+
+ def access_key_id(self) -> str:
+ return self._access_key_id
+
+ def access_key_secret(self) -> str:
+ return self._access_key_secret
+
+ def security_token(self) -> Optional[str]:
+ return self._security_token
diff --git a/clients/client-python/gravitino/utils/credential_factory.py
b/clients/client-python/gravitino/utils/credential_factory.py
index 5d566e509b..1ad32887f1 100644
--- a/clients/client-python/gravitino/utils/credential_factory.py
+++ b/clients/client-python/gravitino/utils/credential_factory.py
@@ -17,7 +17,9 @@
from typing import Dict
+from gravitino.api.credential.aws_secret_key_credential import
AwsSecretKeyCredential
from gravitino.api.credential.credential import Credential
+from gravitino.api.credential.dlf_secret_key_credential import
DlfSecretKeyCredential
from gravitino.api.credential.gcs_token_credential import GCSTokenCredential
from gravitino.api.credential.oss_token_credential import OSSTokenCredential
from gravitino.api.credential.s3_secret_key_credential import
S3SecretKeyCredential
@@ -53,6 +55,10 @@ class CredentialFactory:
== AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE
):
credential = AzureAccountKeyCredential(credential_info,
expire_time_in_ms)
+ elif credential_type ==
AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE:
+ credential = AwsSecretKeyCredential(credential_info,
expire_time_in_ms)
+ elif credential_type ==
DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE:
+ credential = DlfSecretKeyCredential(credential_info,
expire_time_in_ms)
else:
raise NotImplementedError(
f"Credential type {credential_type} is not supported"
diff --git a/clients/client-python/tests/unittests/test_credential_factory.py
b/clients/client-python/tests/unittests/test_credential_factory.py
index 0e67e81fea..a2385b72de 100644
--- a/clients/client-python/tests/unittests/test_credential_factory.py
+++ b/clients/client-python/tests/unittests/test_credential_factory.py
@@ -19,6 +19,8 @@
import unittest
+from gravitino.api.credential.aws_secret_key_credential import
AwsSecretKeyCredential
+from gravitino.api.credential.dlf_secret_key_credential import
DlfSecretKeyCredential
from gravitino.api.credential.gcs_token_credential import GCSTokenCredential
from gravitino.api.credential.oss_token_credential import OSSTokenCredential
from gravitino.api.credential.s3_secret_key_credential import
S3SecretKeyCredential
@@ -185,3 +187,45 @@ class TestCredentialFactory(unittest.TestCase):
self.assertEqual("account_name", check_credential.account_name())
self.assertEqual("account_key", check_credential.account_key())
self.assertEqual(0, check_credential.expire_time_in_ms())
+
+ def test_aws_secret_key_credential(self):
+ aws_credential_info = {
+ AwsSecretKeyCredential._ACCESS_KEY_ID: "access_key",
+ AwsSecretKeyCredential._SECRET_ACCESS_KEY: "secret_key",
+ }
+ aws_credential = AwsSecretKeyCredential(aws_credential_info, 0)
+ check_credential = CredentialFactory.create(
+ aws_credential.credential_type(),
+ aws_credential.credential_info(),
+ aws_credential.expire_time_in_ms(),
+ )
+ self.assertEqual(
+ AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE,
+ check_credential.credential_type(),
+ )
+ self.assertIsInstance(check_credential, AwsSecretKeyCredential)
+ self.assertEqual("access_key", check_credential.access_key_id())
+ self.assertEqual("secret_key", check_credential.secret_access_key())
+ self.assertEqual(0, check_credential.expire_time_in_ms())
+
+ def test_dlf_secret_key_credential(self):
+ dlf_credential_info = {
+ DlfSecretKeyCredential._ACCESS_KEY_ID: "dlf_ak",
+ DlfSecretKeyCredential._ACCESS_KEY_SECRET: "dlf_sk",
+ DlfSecretKeyCredential._SECURITY_TOKEN: "dlf_token",
+ }
+ dlf_credential = DlfSecretKeyCredential(dlf_credential_info, 0)
+ check_credential = CredentialFactory.create(
+ dlf_credential.credential_type(),
+ dlf_credential.credential_info(),
+ dlf_credential.expire_time_in_ms(),
+ )
+ self.assertEqual(
+ DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
+ check_credential.credential_type(),
+ )
+ self.assertIsInstance(check_credential, DlfSecretKeyCredential)
+ self.assertEqual("dlf_ak", check_credential.access_key_id())
+ self.assertEqual("dlf_sk", check_credential.access_key_secret())
+ self.assertEqual("dlf_token", check_credential.security_token())
+ self.assertEqual(0, check_credential.expire_time_in_ms())
diff --git
a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
index 734c991d28..12e17abb78 100644
---
a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
+++
b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
@@ -27,6 +27,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.Catalog;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -88,6 +89,9 @@ public class CredentialPropertyUtils {
private static final String PAIMON_S3_SECRET_KEY = "s3.secret-key";
private static final String PAIMON_OSS_ACCESS_KEY_ID = "fs.oss.accessKeyId";
private static final String PAIMON_OSS_ACCESS_KEY_SECRET =
"fs.oss.accessKeySecret";
+ @VisibleForTesting static final String PAIMON_DLF_ACCESS_KEY_ID =
"dlf.access-key-id";
+ @VisibleForTesting static final String PAIMON_DLF_ACCESS_KEY_SECRET =
"dlf.access-key-secret";
+ @VisibleForTesting static final String PAIMON_DLF_SECURITY_TOKEN =
"dlf.security-token";
private static Map<String, String> icebergCredentialPropertyMap =
ImmutableMap.<String, String>builder()
@@ -157,7 +161,7 @@ public class CredentialPropertyUtils {
}
/**
- * Injects vended credentials into Paimon catalog properties. Supports JDBC,
S3, and OSS
+ * Injects vended credentials into Paimon catalog properties. Supports JDBC,
S3, OSS, and DLF
* credential types.
*
* @param credentials the credentials to apply
@@ -177,6 +181,13 @@ public class CredentialPropertyUtils {
OSSSecretKeyCredential oss = (OSSSecretKeyCredential) credential;
props.put(PAIMON_OSS_ACCESS_KEY_ID, oss.accessKeyId());
props.put(PAIMON_OSS_ACCESS_KEY_SECRET, oss.secretAccessKey());
+ } else if (credential instanceof DlfSecretKeyCredential) {
+ DlfSecretKeyCredential dlf = (DlfSecretKeyCredential) credential;
+ props.put(PAIMON_DLF_ACCESS_KEY_ID, dlf.accessKeyId());
+ props.put(PAIMON_DLF_ACCESS_KEY_SECRET, dlf.accessKeySecret());
+ if (StringUtils.isNotBlank(dlf.securityToken())) {
+ props.put(PAIMON_DLF_SECURITY_TOKEN, dlf.securityToken());
+ }
} else {
LOG.warn(
"Received unrecognized credential type '{}' for Paimon catalog,
skipping",
diff --git
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
index 4feb4b4e49..c0f58a225b 100644
---
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
+++
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
@@ -285,6 +285,73 @@ public class TestCredentialFactory {
Assertions.assertEquals(expireTime, jdbcCredential.expireTimeInMs());
}
+ @Test
+ void testAwsSecretKeyCredential() {
+ Map<String, String> awsSecretKeyCredentialInfo =
+ ImmutableMap.of(
+ AwsSecretKeyCredential.GRAVITINO_AWS_ACCESS_KEY_ID,
+ "accessKeyId",
+ AwsSecretKeyCredential.GRAVITINO_AWS_SECRET_ACCESS_KEY,
+ "secretAccessKey");
+ long expireTime = 0;
+ Credential credential =
+ CredentialFactory.create(
+ AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE,
+ awsSecretKeyCredentialInfo,
+ expireTime);
+ Assertions.assertEquals(
+ AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE,
credential.credentialType());
+ Assertions.assertInstanceOf(AwsSecretKeyCredential.class, credential);
+ AwsSecretKeyCredential aws = (AwsSecretKeyCredential) credential;
+ Assertions.assertEquals("accessKeyId", aws.accessKeyId());
+ Assertions.assertEquals("secretAccessKey", aws.secretAccessKey());
+ Assertions.assertEquals(expireTime, aws.expireTimeInMs());
+ }
+
+ @Test
+ void testDlfSecretKeyCredential() {
+ Map<String, String> dlfSecretKeyCredentialInfo =
+ ImmutableMap.of(
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_ID,
+ "dlf-ak",
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ "dlf-sk",
+ DlfSecretKeyCredential.GRAVITINO_DLF_SECURITY_TOKEN,
+ "dlf-token");
+ long expireTime = 0;
+ Credential credential =
+ CredentialFactory.create(
+ DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
+ dlfSecretKeyCredentialInfo,
+ expireTime);
+ Assertions.assertEquals(
+ DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
credential.credentialType());
+ Assertions.assertInstanceOf(DlfSecretKeyCredential.class, credential);
+ DlfSecretKeyCredential dlf = (DlfSecretKeyCredential) credential;
+ Assertions.assertEquals("dlf-ak", dlf.accessKeyId());
+ Assertions.assertEquals("dlf-sk", dlf.accessKeySecret());
+ Assertions.assertEquals("dlf-token", dlf.securityToken());
+ Assertions.assertEquals(expireTime, dlf.expireTimeInMs());
+ }
+
+ @Test
+ void testDlfSecretKeyCredentialWithoutToken() {
+ Map<String, String> dlfSecretKeyCredentialInfo =
+ ImmutableMap.of(
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_ID,
+ "dlf-ak",
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ "dlf-sk");
+ Credential credential =
+ CredentialFactory.create(
+ DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
dlfSecretKeyCredentialInfo, 0);
+ Assertions.assertInstanceOf(DlfSecretKeyCredential.class, credential);
+ DlfSecretKeyCredential dlf = (DlfSecretKeyCredential) credential;
+ Assertions.assertEquals("dlf-ak", dlf.accessKeyId());
+ Assertions.assertEquals("dlf-sk", dlf.accessKeySecret());
+ Assertions.assertNull(dlf.securityToken());
+ }
+
@Test
void testUnknownCredentialType() {
RuntimeException e =
diff --git
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
index 0ee13adac5..cf53c36b39 100644
---
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
+++
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java
@@ -20,6 +20,7 @@
package org.apache.gravitino.credential;
import com.google.common.collect.ImmutableMap;
+import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -147,4 +148,42 @@ public class TestCredentialPropertiesUtils {
refreshEndpointProperties.get(
CredentialPropertyUtils.ICEBERG_ADLS_REFRESH_CREDENTIALS_ENDPOINT));
}
+
+ @Test
+ void testApplyPaimonCredentials() {
+ Map<String, String> props = new HashMap<>();
+ CredentialPropertyUtils.applyPaimonCredentials(
+ new Credential[] {
+ new JdbcCredential("jdbc-user", "jdbc-password"),
+ new S3SecretKeyCredential("s3-key", "s3-secret"),
+ new OSSSecretKeyCredential("oss-key", "oss-secret"),
+ new DlfSecretKeyCredential("dlf-ak", "dlf-sk", "dlf-token")
+ },
+ props);
+
+ Assertions.assertEquals("jdbc-user",
props.get(CredentialPropertyUtils.ICEBERG_JDBC_USER));
+ Assertions.assertEquals(
+ "jdbc-password",
props.get(CredentialPropertyUtils.ICEBERG_JDBC_PASSWORD));
+ Assertions.assertEquals("s3-key", props.get("s3.access-key"));
+ Assertions.assertEquals("s3-secret", props.get("s3.secret-key"));
+ Assertions.assertEquals("oss-key", props.get("fs.oss.accessKeyId"));
+ Assertions.assertEquals("oss-secret", props.get("fs.oss.accessKeySecret"));
+ Assertions.assertEquals("dlf-ak",
props.get(CredentialPropertyUtils.PAIMON_DLF_ACCESS_KEY_ID));
+ Assertions.assertEquals(
+ "dlf-sk",
props.get(CredentialPropertyUtils.PAIMON_DLF_ACCESS_KEY_SECRET));
+ Assertions.assertEquals(
+ "dlf-token",
props.get(CredentialPropertyUtils.PAIMON_DLF_SECURITY_TOKEN));
+ }
+
+ @Test
+ void testApplyPaimonDlfCredentialsWithoutToken() {
+ Map<String, String> props = new HashMap<>();
+ CredentialPropertyUtils.applyPaimonCredentials(
+ new Credential[] {new DlfSecretKeyCredential("dlf-ak", "dlf-sk")},
props);
+
+ Assertions.assertEquals("dlf-ak",
props.get(CredentialPropertyUtils.PAIMON_DLF_ACCESS_KEY_ID));
+ Assertions.assertEquals(
+ "dlf-sk",
props.get(CredentialPropertyUtils.PAIMON_DLF_ACCESS_KEY_SECRET));
+
Assertions.assertFalse(props.containsKey(CredentialPropertyUtils.PAIMON_DLF_SECURITY_TOKEN));
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/AwsSecretKeyCredentialProvider.java
b/core/src/main/java/org/apache/gravitino/credential/AwsSecretKeyCredentialProvider.java
new file mode 100644
index 0000000000..f6a3ecd9f9
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/credential/AwsSecretKeyCredentialProvider.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+
+/** Generates static AWS access-key credentials for Glue API authentication. */
+public class AwsSecretKeyCredentialProvider implements CredentialProvider {
+
+ private String accessKeyId;
+ private String secretAccessKey;
+
+ @Override
+ public void initialize(Map<String, String> properties) {
+ if (properties == null) {
+ return;
+ }
+ this.accessKeyId =
properties.get(AwsSecretKeyCredential.GRAVITINO_AWS_ACCESS_KEY_ID);
+ this.secretAccessKey =
properties.get(AwsSecretKeyCredential.GRAVITINO_AWS_SECRET_ACCESS_KEY);
+ boolean hasAccessKeyId = StringUtils.isNotBlank(accessKeyId);
+ boolean hasSecretAccessKey = StringUtils.isNotBlank(secretAccessKey);
+ Preconditions.checkArgument(
+ hasAccessKeyId == hasSecretAccessKey,
+ "Incomplete AWS static credential pair for %s: both %s and %s are
required",
+ AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE,
+ AwsSecretKeyCredential.GRAVITINO_AWS_ACCESS_KEY_ID,
+ AwsSecretKeyCredential.GRAVITINO_AWS_SECRET_ACCESS_KEY);
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public String credentialType() {
+ return AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE;
+ }
+
+ @Override
+ public boolean supportsScheme(String scheme) {
+ return false;
+ }
+
+ @Nullable
+ @Override
+ public Credential getCredential(CredentialContext context) {
+ if (StringUtils.isBlank(accessKeyId) ||
StringUtils.isBlank(secretAccessKey)) {
+ return null;
+ }
+ return new AwsSecretKeyCredential(accessKeyId, secretAccessKey);
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/DlfSecretKeyCredentialProvider.java
b/core/src/main/java/org/apache/gravitino/credential/DlfSecretKeyCredentialProvider.java
new file mode 100644
index 0000000000..ce695c4f0f
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/credential/DlfSecretKeyCredentialProvider.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+
+/** Generates static DLF access-key credentials for Paimon DLF catalogs. */
+public class DlfSecretKeyCredentialProvider implements CredentialProvider {
+
+ private String accessKeyId;
+ private String accessKeySecret;
+ @Nullable private String securityToken;
+
+ @Override
+ public void initialize(Map<String, String> properties) {
+ if (properties == null) {
+ return;
+ }
+ this.accessKeyId =
properties.get(DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_ID);
+ this.accessKeySecret =
properties.get(DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_SECRET);
+ this.securityToken =
properties.get(DlfSecretKeyCredential.GRAVITINO_DLF_SECURITY_TOKEN);
+ boolean hasAccessKeyId = StringUtils.isNotBlank(accessKeyId);
+ boolean hasAccessKeySecret = StringUtils.isNotBlank(accessKeySecret);
+ Preconditions.checkArgument(
+ hasAccessKeyId == hasAccessKeySecret,
+ "Incomplete DLF static credential pair for %s: both %s and %s are
required",
+ DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_ID,
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_SECRET);
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public String credentialType() {
+ return DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE;
+ }
+
+ @Override
+ public boolean supportsScheme(String scheme) {
+ return false;
+ }
+
+ @Nullable
+ @Override
+ public Credential getCredential(CredentialContext context) {
+ if (StringUtils.isBlank(accessKeyId) ||
StringUtils.isBlank(accessKeySecret)) {
+ return null;
+ }
+ return new DlfSecretKeyCredential(accessKeyId, accessKeySecret,
securityToken);
+ }
+}
diff --git
a/core/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
b/core/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
index 6b992cebfb..1fab871e48 100644
---
a/core/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
+++
b/core/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
@@ -17,3 +17,5 @@
# under the License.
#
org.apache.gravitino.credential.JdbcCredentialProvider
+org.apache.gravitino.credential.AwsSecretKeyCredentialProvider
+org.apache.gravitino.credential.DlfSecretKeyCredentialProvider
diff --git
a/core/src/test/java/org/apache/gravitino/credential/TestAwsSecretKeyCredentialProvider.java
b/core/src/test/java/org/apache/gravitino/credential/TestAwsSecretKeyCredentialProvider.java
new file mode 100644
index 0000000000..ae5736f6e1
--- /dev/null
+++
b/core/src/test/java/org/apache/gravitino/credential/TestAwsSecretKeyCredentialProvider.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.credential;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import java.util.Optional;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestAwsSecretKeyCredentialProvider {
+
+ @Test
+ void testAwsSecretKeyCredentialProvider() {
+ Map<String, String> catalogProperties =
+ ImmutableMap.of(
+ AwsSecretKeyCredential.GRAVITINO_AWS_ACCESS_KEY_ID,
+ "AKIA",
+ AwsSecretKeyCredential.GRAVITINO_AWS_SECRET_ACCESS_KEY,
+ "secret");
+ CredentialProvider provider =
+ CredentialProviderFactory.create(
+ AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE,
catalogProperties);
+ Assertions.assertEquals(
+ AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE,
provider.credentialType());
+ Optional<Credential> credential =
+ provider.getCredentialOptional(new CatalogCredentialContext("u"));
+ Assertions.assertTrue(credential.isPresent());
+ AwsSecretKeyCredential aws = (AwsSecretKeyCredential) credential.get();
+ Assertions.assertEquals("AKIA", aws.accessKeyId());
+ Assertions.assertEquals("secret", aws.secretAccessKey());
+ Assertions.assertEquals("AKIA",
aws.credentialInfo().get("aws-access-key-id"));
+ Assertions.assertEquals("secret",
aws.credentialInfo().get("aws-secret-access-key"));
+ }
+
+ @Test
+ void testMissingPropertiesReturnsNull() {
+ CredentialProvider provider =
+ CredentialProviderFactory.create(
+ AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE,
ImmutableMap.of());
+ Assertions.assertTrue(
+ provider.getCredentialOptional(new
CatalogCredentialContext("u")).isEmpty());
+ }
+
+ @Test
+ void testIncompletePairThrows() {
+ RuntimeException e =
+ Assertions.assertThrows(
+ RuntimeException.class,
+ () ->
+ CredentialProviderFactory.create(
+ AwsSecretKeyCredential.AWS_SECRET_KEY_CREDENTIAL_TYPE,
+
ImmutableMap.of(AwsSecretKeyCredential.GRAVITINO_AWS_ACCESS_KEY_ID, "AKIA")));
+ Assertions.assertInstanceOf(IllegalArgumentException.class, e.getCause());
+ }
+}
diff --git
a/core/src/test/java/org/apache/gravitino/credential/TestDlfSecretKeyCredentialProvider.java
b/core/src/test/java/org/apache/gravitino/credential/TestDlfSecretKeyCredentialProvider.java
new file mode 100644
index 0000000000..2508670ef5
--- /dev/null
+++
b/core/src/test/java/org/apache/gravitino/credential/TestDlfSecretKeyCredentialProvider.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.credential;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import java.util.Optional;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestDlfSecretKeyCredentialProvider {
+
+ @Test
+ void testDlfSecretKeyCredentialProvider() {
+ Map<String, String> catalogProperties =
+ ImmutableMap.of(
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_ID,
+ "dlf-ak",
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ "dlf-sk",
+ DlfSecretKeyCredential.GRAVITINO_DLF_SECURITY_TOKEN,
+ "tok");
+ CredentialProvider provider =
+ CredentialProviderFactory.create(
+ DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
catalogProperties);
+ Optional<Credential> credential =
+ provider.getCredentialOptional(new CatalogCredentialContext("u"));
+ Assertions.assertTrue(credential.isPresent());
+ DlfSecretKeyCredential dlf = (DlfSecretKeyCredential) credential.get();
+ Assertions.assertEquals("dlf-ak", dlf.accessKeyId());
+ Assertions.assertEquals("dlf-sk", dlf.accessKeySecret());
+ Assertions.assertEquals("tok", dlf.securityToken());
+ Assertions.assertEquals("dlf-ak",
dlf.credentialInfo().get("dlf-access-key-id"));
+ Assertions.assertEquals("dlf-sk",
dlf.credentialInfo().get("dlf-access-key-secret"));
+ Assertions.assertEquals("tok",
dlf.credentialInfo().get("dlf-security-token"));
+ }
+
+ @Test
+ void testWithoutSecurityToken() {
+ Map<String, String> catalogProperties =
+ ImmutableMap.of(
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_ID,
+ "dlf-ak",
+ DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ "dlf-sk");
+ CredentialProvider provider =
+ CredentialProviderFactory.create(
+ DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
catalogProperties);
+ DlfSecretKeyCredential dlf =
+ (DlfSecretKeyCredential)
+ provider.getCredentialOptional(new
CatalogCredentialContext("u")).get();
+ Assertions.assertNull(dlf.securityToken());
+
Assertions.assertFalse(dlf.credentialInfo().containsKey("dlf-security-token"));
+ }
+
+ @Test
+ void testIncompletePairThrows() {
+ RuntimeException e =
+ Assertions.assertThrows(
+ RuntimeException.class,
+ () ->
+ CredentialProviderFactory.create(
+ DlfSecretKeyCredential.DLF_SECRET_KEY_CREDENTIAL_TYPE,
+
ImmutableMap.of(DlfSecretKeyCredential.GRAVITINO_DLF_ACCESS_KEY_ID, "dlf-ak")));
+ Assertions.assertInstanceOf(IllegalArgumentException.class, e.getCause());
+ }
+}
diff --git
a/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
b/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
index e672bd50eb..d2afa97d36 100644
---
a/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
+++
b/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
@@ -19,3 +19,5 @@
org.apache.gravitino.credential.DummyCredentialProvider
org.apache.gravitino.credential.Dummy2CredentialProvider
org.apache.gravitino.credential.JdbcCredentialProvider
+org.apache.gravitino.credential.AwsSecretKeyCredentialProvider
+org.apache.gravitino.credential.DlfSecretKeyCredentialProvider
diff --git a/design-docs/gravitino-glue-catalog.md
b/design-docs/gravitino-glue-catalog.md
index c64f449325..cca5624a96 100644
--- a/design-docs/gravitino-glue-catalog.md
+++ b/design-docs/gravitino-glue-catalog.md
@@ -127,8 +127,8 @@ Glue is a separate AWS service from S3. The Glue region and
credentials may diff
| Property | Required | Default |
Description
|
| ----------------------- | -------- | ----------------------------- |
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
| `aws-region` | Yes | — | AWS
region for the Glue Data Catalog
|
-| `aws-access-key-id` | No | Default credential chain | AWS
access key for Glue API authentication. Hidden from catalog readers; returned
as `******`. Plaintext via `getSecrets` with `USE_SECRETS` and
`INCLUDE_CREDENTIAL_SECRETS`. Also remapped to `s3-*` for storage credential
vending.
|
-| `aws-secret-access-key` | No | Default credential chain | AWS
secret key for Glue API authentication. Hidden from catalog readers; returned
as `******`. Plaintext via `getSecrets` with `USE_SECRETS` and
`INCLUDE_CREDENTIAL_SECRETS`.
|
+| `aws-access-key-id` | No | Default credential chain | AWS
access key for Glue API authentication. Hidden from catalog readers; returned
as `******`. Plaintext via `getCredentials` (`AwsSecretKeyCredential`), or via
`getSecrets` with `USE_SECRETS` and `INCLUDE_CREDENTIAL_SECRETS`. Also remapped
to `s3-*` for storage credential vending.
[...]
+| `aws-secret-access-key` | No | Default credential chain | AWS
secret key for Glue API authentication. Hidden from catalog readers; returned
as `******`. Plaintext via `getCredentials` (`AwsSecretKeyCredential`), or via
`getSecrets` with `USE_SECRETS` and `INCLUDE_CREDENTIAL_SECRETS`.
|
| `aws-glue-catalog-id` | Yes | — | Glue
catalog ID. Required because an AWS account can have multiple Glue catalogs
(e.g., default catalog and federated S3 Tables catalog).
|
| `aws-glue-endpoint` | No | AWS default regional endpoint | Custom
Glue endpoint URL (for VPC endpoints or LocalStack testing).
|
| `default-table-format` | No | `iceberg` | Default
format for tables created via Gravitino's `createTable()` API. Accepted values:
`iceberg`, `hive`.
|
diff --git a/docs/aws-glue-catalog.md b/docs/aws-glue-catalog.md
index 9ee2cdcf12..a0ed1ff042 100644
--- a/docs/aws-glue-catalog.md
+++ b/docs/aws-glue-catalog.md
@@ -55,7 +55,12 @@ Besides the [common catalog
properties](./gravitino-server-config.md#catalog-pro
Refer to [Manage Catalogs and
Schemas](./manage-catalogs-and-schemas.md#catalog-operations) for more details.
:::note
-Sensitive catalog properties such as credential-vending keys are hidden from
the default load catalog response. Retrieve secret-manager-backed properties
(including keys that overlap with credential vending) via `getSecrets` / `GET
.../objects/{type}/{fullName}/secrets`. The [credential vending
API](security/credential-vending.md) remains available for typed credential
delivery.
+Sensitive catalog properties such as `aws-access-key-id` /
`aws-secret-access-key` are hidden from the default load catalog response.
Recover them via:
+
+- `getCredentials` / `AwsSecretKeyCredential` (and remapped
`S3SecretKeyCredential` when auto-detected) — no dedicated privilege beyond
loading the catalog
+- `getSecrets` / `GET .../objects/{type}/{fullName}/secrets` with
`USE_SECRETS` and `INCLUDE_CREDENTIAL_SECRETS`
+
+See [credential vending](security/credential-vending.md) for details.
:::
## Schema
diff --git a/docs/lakehouse-paimon-catalog.md b/docs/lakehouse-paimon-catalog.md
index 2ebd10fe3e..2cc78c4f07 100644
--- a/docs/lakehouse-paimon-catalog.md
+++ b/docs/lakehouse-paimon-catalog.md
@@ -136,7 +136,13 @@ Download the corresponding JDBC driver and place it to the
`catalogs/lakehouse-p
Refer to [Manage Catalogs and
Schemas](./manage-catalogs-and-schemas.md#catalog-operations) for more details.
:::note
-Sensitive catalog properties such as `jdbc-password` are hidden from the
default load catalog response (`jdbc-user` is returned in plaintext). Retrieve
secret-manager-backed properties (including `jdbc-password` when stored as a
secret URN) via `getSecrets` / `GET .../objects/{type}/{fullName}/secrets`. The
[credential vending API](security/credential-vending.md) (`getCredentials` /
`JdbcCredential`) remains available for typed credential delivery.
+Sensitive catalog properties such as `jdbc-password` and DLF credential keys
are hidden from the default load catalog response (`jdbc-user` is returned in
plaintext). Recover JDBC / DLF credential fields via either path:
+
+- `getCredentials` / `JdbcCredential` / `DlfSecretKeyCredential` (no dedicated
privilege beyond loading the catalog)
+- `getSecrets` / `GET .../objects/{type}/{fullName}/secrets` with
`USE_SECRETS` (DLF cloud access-key pairs also need
`INCLUDE_CREDENTIAL_SECRETS`)
+
+Other non-credential secrets (secret-manager URNs, declared `hidden`
properties, undeclared sensitive-named keys) use `getSecrets` only.
+
:::
## Schema
diff --git a/docs/security/credential-vending.md
b/docs/security/credential-vending.md
index 839f8ab953..0db656e06d 100755
--- a/docs/security/credential-vending.md
+++ b/docs/security/credential-vending.md
@@ -116,6 +116,8 @@ Catalogs defined in `gravitino.conf` are not registered in
a metalake, so Gravit
| `cos-token` | COS | A temporary STS token
|
| `cos-secret-key` | COS | The configured static access key and secret
|
| `jdbc-user-password` | JDBC | The configured JDBC username and password
|
+| `aws-secret-key` | Glue | The configured AWS access key and secret
for Glue API auth |
+| `dlf-secret-key` | Paimon | The configured DLF access key and secret
(optional token) |
Each value has its own properties, listed in the sections below. To vend for
more than one storage type on a catalog, separate values with a comma. Custom
providers can be added by implementing `CredentialProvider`, described under
[Custom Credentials](#custom-credentials).
@@ -129,6 +131,8 @@ If a catalog does not set `credential-providers`, Gravitino
infers providers fro
| `oss-access-key-id` and `oss-secret-access-key` |
`oss-secret-key` |
| `azure-storage-account-name` and `azure-storage-account-key` |
`azure-account-key` |
| `gcs-service-account-file` | `gcs-token`
|
+| `aws-access-key-id` and `aws-secret-access-key` |
`aws-secret-key` |
+| `dlf-access-key-id` and `dlf-access-key-secret` |
`dlf-secret-key` |
JDBC catalogs additionally infer `jdbc-user-password` from `jdbc-user` and
`jdbc-password`.
@@ -137,10 +141,10 @@ Four providers have no inference rule and must always be
set explicitly: `s3-tok
### Static providers and `getCredentials` privilege risk
Static providers such as `s3-secret-key`, `oss-secret-key`, `cos-secret-key`,
`azure-account-key`,
-and `jdbc-user-password` return the configured long-lived plaintext keys from
-`getCredentials` / `GET .../credentials`. That endpoint does **not** require a
dedicated privilege
-beyond being able to load the metadata object (unlike `getSecrets`, which
requires `USE_SECRETS`,
-with cloud access-key pairs gated by `INCLUDE_CREDENTIAL_SECRETS`).
+`jdbc-user-password`, `aws-secret-key`, and `dlf-secret-key` return the
configured long-lived
+plaintext keys from `getCredentials` / `GET .../credentials`. That endpoint
does **not** require a
+dedicated privilege beyond being able to load the metadata object (unlike
`getSecrets`, which
+requires `USE_SECRETS`, with cloud access-key pairs gated by
`INCLUDE_CREDENTIAL_SECRETS`).
**Risk:** any principal that can load a catalog (or fileset) configured with
these static providers
can retrieve the same static AK/SK or JDBC password that Gravitino uses
server-side.
@@ -505,7 +509,8 @@ Bundle jars on Maven Central:
## Upgrading From a Release Earlier Than 1.3.0
-Sensitive catalog properties such as `s3-access-key-id`,
`s3-secret-access-key`, `jdbc-password`, `aws-access-key-id` /
`aws-secret-access-key` (Glue), and `dlf-access-key-id` /
`dlf-access-key-secret` (Paimon DLF) are masked or excluded from the default
`GET /api/metalakes/{metalake}/catalogs/{catalog}` response (`jdbc-user` and
`azure-storage-account-name` are returned in plaintext when not hidden).
Plaintext secrets are available via `getSecrets` when the caller holds
`USE_SECRETS` (o [...]
+Sensitive catalog properties such as `s3-access-key-id`,
`s3-secret-access-key`, `jdbc-password`, `aws-access-key-id` /
`aws-secret-access-key` (Glue), and `dlf-access-key-id` /
`dlf-access-key-secret` / `dlf-security-token` (Paimon DLF) are masked or
excluded from the default `GET /api/metalakes/{metalake}/catalogs/{catalog}`
response (`jdbc-user` and `azure-storage-account-name` are returned in
plaintext when not hidden). Plaintext secrets are available via `getSecrets`
when the caller [...]
+
For a zero-downtime migration, set the following in `gravitino.conf`:
diff --git
a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/paimon/TestGravitinoPaimonCatalog.java
b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/paimon/TestGravitinoPaimonCatalog.java
index f2586ec68f..519b2c33a0 100644
---
a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/paimon/TestGravitinoPaimonCatalog.java
+++
b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/paimon/TestGravitinoPaimonCatalog.java
@@ -40,6 +40,7 @@ import org.apache.flink.table.factories.CatalogFactory;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.catalog.lakehouse.paimon.PaimonConstants;
import org.apache.gravitino.credential.Credential;
+import org.apache.gravitino.credential.DlfSecretKeyCredential;
import org.apache.gravitino.credential.JdbcCredential;
import org.apache.gravitino.credential.OSSSecretKeyCredential;
import org.apache.gravitino.credential.S3SecretKeyCredential;
@@ -628,6 +629,25 @@ public abstract class TestGravitinoPaimonCatalog {
"jdbc-password",
catalog.capturedOptions.get(PaimonConstants.PAIMON_JDBC_PASSWORD));
}
+ /** Verifies that DLF backend credentials remain Paimon catalog options. */
+ @Test
+ public void testOpenKeepsDlfCredentialsInPaimonOptions() {
+ Catalog mockCatalog =
+ catalogWithCredentials(new DlfSecretKeyCredential("dlf-ak", "dlf-sk",
"dlf-token"));
+ Map<String, String> options = new HashMap<>();
+ options.put("warehouse", "file:/tmp/test-paimon-warehouse");
+
+ CapturingPaimonCatalog catalog = new CapturingPaimonCatalog(options,
mockCatalog);
+ catalog.open();
+
+ Assertions.assertEquals(
+ "dlf-ak",
catalog.capturedOptions.get(PaimonConstants.PAIMON_DLF_ACCESS_KEY_ID));
+ Assertions.assertEquals(
+ "dlf-sk",
catalog.capturedOptions.get(PaimonConstants.PAIMON_DLF_ACCESS_KEY_SECRET));
+ Assertions.assertEquals(
+ "dlf-token",
catalog.capturedOptions.get(PaimonConstants.PAIMON_DLF_SECURITY_TOKEN));
+ }
+
private static Catalog catalogWithCredentials(Credential... credentials) {
Catalog catalog = mock(Catalog.class);
SupportsCredentials supportsCredentials = mock(SupportsCredentials.class);
diff --git
a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/paimon/TestPaimonPropertiesConverter.java
b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/paimon/TestPaimonPropertiesConverter.java
index c59135b64b..6dd4b3085d 100644
---
a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/paimon/TestPaimonPropertiesConverter.java
+++
b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/paimon/TestPaimonPropertiesConverter.java
@@ -77,6 +77,33 @@ public class TestPaimonPropertiesConverter {
Assertions.assertEquals(testUri,
flinkCatalogProperties.get(PaimonConstants.URI));
}
+ @Test
+ public void testToPaimonDlfCatalog() {
+ Map<String, String> catalogProperties =
+ ImmutableMap.<String, String>builder()
+ .put(PaimonConstants.WAREHOUSE, localWarehouse)
+ .put(PaimonConstants.CATALOG_BACKEND, "rest")
+ .put(PaimonConstants.URI, "https://dlf.aliyuncs.com";)
+ .put(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID, "dlf-ak")
+ .put(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET, "dlf-sk")
+ .put(PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN, "dlf-token")
+ .build();
+ Map<String, String> flinkCatalogProperties =
+ CONVERTER.toFlinkCatalogProperties(catalogProperties);
+ Assertions.assertEquals(
+ GravitinoPaimonCatalogFactoryOptions.IDENTIFIER,
flinkCatalogProperties.get("type"));
+ Assertions.assertEquals(localWarehouse,
flinkCatalogProperties.get(PaimonConstants.WAREHOUSE));
+ Assertions.assertEquals("rest",
flinkCatalogProperties.get(PaimonConstants.METASTORE));
+ Assertions.assertEquals(
+ "https://dlf.aliyuncs.com";,
flinkCatalogProperties.get(PaimonConstants.URI));
+ Assertions.assertEquals(
+ "dlf-ak",
flinkCatalogProperties.get(PaimonConstants.PAIMON_DLF_ACCESS_KEY_ID));
+ Assertions.assertEquals(
+ "dlf-sk",
flinkCatalogProperties.get(PaimonConstants.PAIMON_DLF_ACCESS_KEY_SECRET));
+ Assertions.assertEquals(
+ "dlf-token",
flinkCatalogProperties.get(PaimonConstants.PAIMON_DLF_SECURITY_TOKEN));
+ }
+
@Test
public void testToGravitinoCatalogProperties() {
String testUser = "testUser";
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GravitinoGlueCatalog.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GravitinoGlueCatalog.java
index 6ac600140a..99c692dc9c 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GravitinoGlueCatalog.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/glue/GravitinoGlueCatalog.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import java.util.HashMap;
import java.util.Map;
import org.apache.gravitino.catalog.glue.GlueConstants;
+import org.apache.gravitino.credential.AwsSecretKeyCredential;
import org.apache.gravitino.credential.Credential;
import org.apache.gravitino.credential.CredentialPropertyUtils;
import org.apache.gravitino.credential.S3SecretKeyCredential;
@@ -112,10 +113,10 @@ public class GravitinoGlueCatalog extends BaseCatalog {
}
/**
- * Obtains S3 credentials via Gravitino credential vending and injects them
into {@code props} as
- * {@code hadoop.fs.s3a.*} for the non-Iceberg (Hive) path's S3 data access.
Returns the vended
- * credentials keyed by Gravitino catalog property names for reuse in the
Iceberg path. Returns an
- * empty map if credential vending is unavailable.
+ * Obtains credentials via Gravitino credential vending. {@link
AwsSecretKeyCredential} supplies
+ * Glue API keys (returned for the Iceberg path). {@link
S3SecretKeyCredential} supplies {@code
+ * hadoop.fs.s3a.*} for Hive-path S3 data access. When only {@link
S3SecretKeyCredential} is
+ * present (legacy remap), it also fills Glue API keys for backward
compatibility.
*
* @return map of vended AWS credentials (Gravitino key names), or empty map
if none vended
*/
@@ -130,19 +131,27 @@ public class GravitinoGlueCatalog extends BaseCatalog {
"Failed to obtain credentials from Glue catalog, S3 credential
injection skipped", e);
return vended;
}
+ S3SecretKeyCredential s3SecretKey = null;
for (Credential credential : credentials) {
- if (credential instanceof S3SecretKeyCredential) {
- S3SecretKeyCredential s3 = (S3SecretKeyCredential) credential;
- props.put("hadoop.fs.s3a.access.key", s3.accessKeyId());
- props.put("hadoop.fs.s3a.secret.key", s3.secretAccessKey());
- vended.put(GluePropertiesConverter.AWS_ACCESS_KEY_ID,
s3.accessKeyId());
- vended.put(GluePropertiesConverter.AWS_SECRET_ACCESS_KEY,
s3.secretAccessKey());
+ if (credential instanceof AwsSecretKeyCredential) {
+ AwsSecretKeyCredential aws = (AwsSecretKeyCredential) credential;
+ vended.put(GluePropertiesConverter.AWS_ACCESS_KEY_ID,
aws.accessKeyId());
+ vended.put(GluePropertiesConverter.AWS_SECRET_ACCESS_KEY,
aws.secretAccessKey());
+ } else if (credential instanceof S3SecretKeyCredential) {
+ s3SecretKey = (S3SecretKeyCredential) credential;
+ props.put("hadoop.fs.s3a.access.key", s3SecretKey.accessKeyId());
+ props.put("hadoop.fs.s3a.secret.key", s3SecretKey.secretAccessKey());
} else {
LOG.warn(
"Received unrecognized credential type '{}' for Glue catalog,
skipping",
credential.getClass().getName());
}
}
+ // Legacy catalogs that only vend remapped S3SecretKeyCredential still
need Glue API keys.
+ if (vended.isEmpty() && s3SecretKey != null) {
+ vended.put(GluePropertiesConverter.AWS_ACCESS_KEY_ID,
s3SecretKey.accessKeyId());
+ vended.put(GluePropertiesConverter.AWS_SECRET_ACCESS_KEY,
s3SecretKey.secretAccessKey());
+ }
return vended;
}
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGravitinoGlueCatalog.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGravitinoGlueCatalog.java
index 364dcc74e2..ab2ed9a193 100644
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGravitinoGlueCatalog.java
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/glue/TestGravitinoGlueCatalog.java
@@ -32,6 +32,7 @@ import java.util.Map;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.catalog.glue.GlueConstants;
import org.apache.gravitino.client.GravitinoClient;
+import org.apache.gravitino.credential.AwsSecretKeyCredential;
import org.apache.gravitino.credential.Credential;
import org.apache.gravitino.credential.S3SecretKeyCredential;
import org.apache.gravitino.credential.SupportsCredentials;
@@ -212,6 +213,24 @@ public class TestGravitinoGlueCatalog {
"test-secret-key",
vended.get(GluePropertiesConverter.AWS_SECRET_ACCESS_KEY));
}
+ @Test
+ void testApplyS3CredentialUsesAwsForGlueAndS3ForStorage() {
+ Catalog mockCatalog = mock(Catalog.class);
+ AwsSecretKeyCredential awsCred = new AwsSecretKeyCredential("glue-ak",
"glue-sk");
+ S3SecretKeyCredential s3Cred = new S3SecretKeyCredential("s3-ak", "s3-sk");
+ SupportsCredentials supportsCredentials = mock(SupportsCredentials.class);
+ when(mockCatalog.supportsCredentials()).thenReturn(supportsCredentials);
+ when(supportsCredentials.getCredentials()).thenReturn(new Credential[]
{awsCred, s3Cred});
+
+ Map<String, String> props = new HashMap<>();
+ Map<String, String> vended =
GravitinoGlueCatalog.applyS3Credential(mockCatalog, props);
+
+ Assertions.assertEquals("s3-ak", props.get("hadoop.fs.s3a.access.key"));
+ Assertions.assertEquals("s3-sk", props.get("hadoop.fs.s3a.secret.key"));
+ Assertions.assertEquals("glue-ak",
vended.get(GluePropertiesConverter.AWS_ACCESS_KEY_ID));
+ Assertions.assertEquals("glue-sk",
vended.get(GluePropertiesConverter.AWS_SECRET_ACCESS_KEY));
+ }
+
@Test
void testApplyS3CredentialReturnsEmptyWhenNoCredentials() {
Catalog mockCatalog = mock(Catalog.class);
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/glue/GlueConnectorAdapter.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/glue/GlueConnectorAdapter.java
index ac6a56ad01..78b55fb77d 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/glue/GlueConnectorAdapter.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/glue/GlueConnectorAdapter.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.credential.AwsSecretKeyCredential;
import org.apache.gravitino.credential.Credential;
import org.apache.gravitino.credential.S3SecretKeyCredential;
import org.apache.gravitino.trino.connector.catalog.CatalogConnectorAdapter;
@@ -91,17 +92,28 @@ public class GlueConnectorAdapter implements
CatalogConnectorAdapter {
return config;
}
+ /**
+ * Applies Glue API keys from {@link AwsSecretKeyCredential} and S3
data-access keys from {@link
+ * S3SecretKeyCredential}. When only {@link S3SecretKeyCredential} is
present (legacy remap), it
+ * fills both Glue and S3 keys for backward compatibility.
+ */
static void applyS3Credential(Credential[] credentials, Map<String, String>
config) {
+ S3SecretKeyCredential s3SecretKey = null;
for (Credential credential : credentials) {
- if (credential instanceof S3SecretKeyCredential) {
- S3SecretKeyCredential s3 = (S3SecretKeyCredential) credential;
- config.put(HIVE_METASTORE_GLUE_ACCESS_KEY, s3.accessKeyId());
- config.put(HIVE_METASTORE_GLUE_SECRET_KEY, s3.secretAccessKey());
- config.put(HIVE_S3_ACCESS_KEY, s3.accessKeyId());
- config.put(HIVE_S3_SECRET_KEY, s3.secretAccessKey());
- return;
+ if (credential instanceof AwsSecretKeyCredential) {
+ AwsSecretKeyCredential aws = (AwsSecretKeyCredential) credential;
+ config.put(HIVE_METASTORE_GLUE_ACCESS_KEY, aws.accessKeyId());
+ config.put(HIVE_METASTORE_GLUE_SECRET_KEY, aws.secretAccessKey());
+ } else if (credential instanceof S3SecretKeyCredential) {
+ s3SecretKey = (S3SecretKeyCredential) credential;
+ config.put(HIVE_S3_ACCESS_KEY, s3SecretKey.accessKeyId());
+ config.put(HIVE_S3_SECRET_KEY, s3SecretKey.secretAccessKey());
}
}
+ if (!config.containsKey(HIVE_METASTORE_GLUE_ACCESS_KEY) && s3SecretKey !=
null) {
+ config.put(HIVE_METASTORE_GLUE_ACCESS_KEY, s3SecretKey.accessKeyId());
+ config.put(HIVE_METASTORE_GLUE_SECRET_KEY,
s3SecretKey.secretAccessKey());
+ }
}
@Override
diff --git
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/glue/TestGlueConnectorAdapter.java
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/glue/TestGlueConnectorAdapter.java
index 81313f305d..58200a0e19 100644
---
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/glue/TestGlueConnectorAdapter.java
+++
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/glue/TestGlueConnectorAdapter.java
@@ -19,8 +19,10 @@
package org.apache.gravitino.trino.connector.catalog.glue;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
import java.util.Map;
import org.apache.gravitino.Catalog;
+import org.apache.gravitino.credential.AwsSecretKeyCredential;
import org.apache.gravitino.credential.Credential;
import org.apache.gravitino.credential.S3SecretKeyCredential;
import org.apache.gravitino.trino.connector.metadata.GravitinoCatalog;
@@ -76,4 +78,17 @@ class TestGlueConnectorAdapter {
Assertions.assertEquals(
"https://glue.custom.endpoint";,
config.get("hive.metastore.glue.endpoint-url"));
}
+
+ @Test
+ void testApplyS3CredentialUsesAwsForGlueAndS3ForStorage() {
+ Map<String, String> config = Maps.newHashMap();
+ Credential[] credentials = {
+ new AwsSecretKeyCredential("glue-ak", "glue-sk"), new
S3SecretKeyCredential("s3-ak", "s3-sk")
+ };
+ GlueConnectorAdapter.applyS3Credential(credentials, config);
+ Assertions.assertEquals("glue-ak",
config.get("hive.metastore.glue.aws-access-key"));
+ Assertions.assertEquals("glue-sk",
config.get("hive.metastore.glue.aws-secret-key"));
+ Assertions.assertEquals("s3-ak", config.get("hive.s3.aws-access-key"));
+ Assertions.assertEquals("s3-sk", config.get("hive.s3.aws-secret-key"));
+ }
}
