This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new e155af80ec [Cherry-pick to branch-1.3] [#11644] fix(paimon-catalog):
Paimon catalog supports credential vending and hides sensitive properties
(#11669) (#11692)
e155af80ec is described below
commit e155af80ecc07da700515e46d558f0ff2a558b6e
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Jun 17 09:45:19 2026 +0800
[Cherry-pick to branch-1.3] [#11644] fix(paimon-catalog): Paimon catalog
supports credential vending and hides sensitive properties (#11669) (#11692)
**Cherry-pick Information:**
- Original commit: dc6938f8ce4af61ad2d6991333d3ab99f843112b
- Target branch: `branch-1.3`
- Status: ✅ Clean cherry-pick (no conflicts)
Co-authored-by: Yuhui <[email protected]>
---
.../catalog/lakehouse/paimon/PaimonCatalog.java | 7 ++
.../paimon/PaimonCatalogPropertiesMetadata.java | 46 ++++-----
.../paimon/storage/PaimonOSSFileSystemConfig.java | 105 ---------------------
.../paimon/storage/PaimonS3FileSystemConfig.java | 105 ---------------------
.../lakehouse/paimon/storage/S3StorageConfig.java | 105 ---------------------
.../TestPaimonCatalogPropertiesMetadata.java | 56 +++++++++++
docs/apache-hive-catalog.md | 4 +
docs/aws-glue-catalog.md | 4 +
docs/jdbc-doris-catalog.md | 4 +
docs/jdbc-hologres-catalog.md | 4 +
docs/jdbc-mysql-catalog.md | 4 +
docs/jdbc-oceanbase-catalog.md | 4 +
docs/jdbc-postgresql-catalog.md | 4 +
docs/jdbc-starrocks-catalog.md | 4 +
docs/lakehouse-iceberg-catalog.md | 4 +
docs/lakehouse-paimon-catalog.md | 9 +-
docs/security/credential-vending.md | 63 ++++++++++++-
17 files changed, 191 insertions(+), 341 deletions(-)
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java
index 3bf8d7e244..5c7afa7a63 100644
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java
+++
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java
@@ -87,6 +87,13 @@ public class PaimonCatalog extends
BaseCatalog<PaimonCatalog> {
return SCHEMA_PROPERTIES_META;
}
+ /**
+ * Adds a JDBC credential provider when the backend is JDBC and credentials
are configured, then
+ * delegates to the parent for storage (S3/OSS/Azure/GCS) credential
provider detection.
+ *
+ * @param properties the raw catalog properties
+ * @param credentialProviders the list to append detected provider names to
+ */
@Override
protected void addCatalogSpecificCredentialProviders(
Map<String, String> properties, List<String> credentialProviders) {
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
index 7f64c711f7..66234fa76a 100644
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
@@ -31,9 +31,9 @@ import java.util.List;
import java.util.Map;
import
org.apache.gravitino.catalog.lakehouse.paimon.authentication.AuthenticationConfig;
import
org.apache.gravitino.catalog.lakehouse.paimon.authentication.kerberos.KerberosConfig;
-import
org.apache.gravitino.catalog.lakehouse.paimon.storage.PaimonOSSFileSystemConfig;
-import
org.apache.gravitino.catalog.lakehouse.paimon.storage.PaimonS3FileSystemConfig;
import org.apache.gravitino.catalog.lakehouse.paimon.utils.CatalogUtils;
+import org.apache.gravitino.cloud.storage.OSSPropertiesMetadata;
+import org.apache.gravitino.cloud.storage.S3PropertiesMetadata;
import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata;
import org.apache.gravitino.connector.PropertiesMetadata;
import org.apache.gravitino.connector.PropertyEntry;
@@ -62,6 +62,11 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
public static final String S3_ACCESS_KEY = PaimonConstants.S3_ACCESS_KEY;
public static final String S3_SECRET_KEY = PaimonConstants.S3_SECRET_KEY;
+ // OSS properties needed by Paimon
+ public static final String OSS_ENDPOINT = PaimonConstants.OSS_ENDPOINT;
+ public static final String OSS_ACCESS_KEY = PaimonConstants.OSS_ACCESS_KEY;
+ public static final String OSS_SECRET_KEY = PaimonConstants.OSS_SECRET_KEY;
+
public static final Map<String, String> GRAVITINO_CONFIG_TO_PAIMON =
ImmutableMap.<String, String>builder()
.put(GRAVITINO_CATALOG_BACKEND, PAIMON_METASTORE)
@@ -85,13 +90,12 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
.build();
private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
public static final Map<String, String> KERBEROS_CONFIGURATION =
- ImmutableMap.<String, String>builder()
- .put(KerberosConfig.PRINCIPAL_KEY, KerberosConfig.PRINCIPAL_KEY)
- .put(KerberosConfig.KEY_TAB_URI_KEY, KerberosConfig.KEY_TAB_URI_KEY)
- .put(KerberosConfig.CHECK_INTERVAL_SEC_KEY,
KerberosConfig.CHECK_INTERVAL_SEC_KEY)
- .put(KerberosConfig.FETCH_TIMEOUT_SEC_KEY,
KerberosConfig.FETCH_TIMEOUT_SEC_KEY)
- .put(AuthenticationConfig.AUTH_TYPE_KEY,
AuthenticationConfig.AUTH_TYPE_KEY)
- .build();
+ ImmutableMap.of(
+ KerberosConfig.PRINCIPAL_KEY, KerberosConfig.PRINCIPAL_KEY,
+ KerberosConfig.KEY_TAB_URI_KEY, KerberosConfig.KEY_TAB_URI_KEY,
+ KerberosConfig.CHECK_INTERVAL_SEC_KEY,
KerberosConfig.CHECK_INTERVAL_SEC_KEY,
+ KerberosConfig.FETCH_TIMEOUT_SEC_KEY,
KerberosConfig.FETCH_TIMEOUT_SEC_KEY,
+ AuthenticationConfig.AUTH_TYPE_KEY,
AuthenticationConfig.AUTH_TYPE_KEY);
public static final Map<String, String> S3_CONFIGURATION =
ImmutableMap.of(
@@ -101,9 +105,9 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
public static final Map<String, String> OSS_CONFIGURATION =
ImmutableMap.of(
- OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID,
PaimonOSSFileSystemConfig.OSS_ACCESS_KEY,
- OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET,
PaimonOSSFileSystemConfig.OSS_SECRET_KEY,
- OSSProperties.GRAVITINO_OSS_ENDPOINT,
PaimonOSSFileSystemConfig.OSS_ENDPOINT);
+ OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY,
+ OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, OSS_SECRET_KEY,
+ OSSProperties.GRAVITINO_OSS_ENDPOINT, OSS_ENDPOINT);
public static final Map<String, PropertyEntry<?>> REST_PROPERTY_ENTRIES =
new ImmutableMap.Builder<String, PropertyEntry<?>>()
@@ -119,10 +123,10 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
PaimonConstants.TOKEN,
stringOptionalPropertyEntry(
PaimonConstants.TOKEN,
- "The bear token for REST catalog authentication",
+ "The bearer token for REST catalog authentication",
false /* immutable */,
null /* defaultValue */,
- false /* hidden */))
+ true /* hidden */))
.put(
PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID,
stringOptionalPropertyEntry(
@@ -130,7 +134,7 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
"The access key ID for Aliyun DLF",
false /* immutable */,
null /* defaultValue */,
- false /* hidden */))
+ true /* hidden */))
.put(
PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET,
stringOptionalPropertyEntry(
@@ -138,7 +142,7 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
"The access key secret for Aliyun DLF",
false /* immutable */,
null /* defaultValue */,
- false /* hidden */))
+ true /* hidden */))
.put(
PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN,
stringOptionalPropertyEntry(
@@ -146,7 +150,7 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
"The security token for Aliyun DLF",
false /* immutable */,
null /* defaultValue */,
- false /* hidden */))
+ true /* hidden */))
.put(
PaimonConstants.GRAVITINO_DLF_TOKEN_PATH,
stringOptionalPropertyEntry(
@@ -193,13 +197,13 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
"Gravitino Paimon catalog jdbc user",
false /* immutable */,
null /* defaultValue */,
- false /* hidden */),
+ true /* hidden */),
stringOptionalPropertyEntry(
GRAVITINO_JDBC_PASSWORD,
"Gravitino Paimon catalog jdbc password",
false /* immutable */,
null /* defaultValue */,
- false /* hidden */),
+ true /* hidden */),
stringOptionalPropertyEntry(
GRAVITINO_JDBC_DRIVER,
"The driver of the Jdbc connection",
@@ -210,8 +214,8 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
result.putAll(Maps.uniqueIndex(propertyEntries, PropertyEntry::getName));
result.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES);
result.putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES);
- result.putAll(PaimonS3FileSystemConfig.S3_FILESYSTEM_PROPERTY_ENTRIES);
- result.putAll(PaimonOSSFileSystemConfig.OSS_FILESYSTEM_PROPERTY_ENTRIES);
+ result.putAll(S3PropertiesMetadata.PROPERTY_ENTRIES);
+ result.putAll(OSSPropertiesMetadata.PROPERTY_ENTRIES);
result.putAll(REST_PROPERTY_ENTRIES);
PROPERTIES_METADATA = ImmutableMap.copyOf(result);
}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/storage/PaimonOSSFileSystemConfig.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/storage/PaimonOSSFileSystemConfig.java
deleted file mode 100644
index 7b703b5b74..0000000000
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/storage/PaimonOSSFileSystemConfig.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.gravitino.catalog.lakehouse.paimon.storage;
-
-import com.google.common.collect.ImmutableMap;
-import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.gravitino.Config;
-import org.apache.gravitino.catalog.lakehouse.paimon.PaimonConstants;
-import org.apache.gravitino.config.ConfigBuilder;
-import org.apache.gravitino.config.ConfigConstants;
-import org.apache.gravitino.config.ConfigEntry;
-import org.apache.gravitino.connector.PropertyEntry;
-
-public class PaimonOSSFileSystemConfig extends Config {
- // OSS related properties
- public static final String OSS_ENDPOINT = PaimonConstants.OSS_ENDPOINT;
- public static final String OSS_ACCESS_KEY = PaimonConstants.OSS_ACCESS_KEY;
- public static final String OSS_SECRET_KEY = PaimonConstants.OSS_SECRET_KEY;
-
- public PaimonOSSFileSystemConfig(Map<String, String> properties) {
- super(false);
- loadFromMap(properties, k -> true);
- }
-
- public static final ConfigEntry<String> PAIMON_OSS_ENDPOINT_ENTRY =
- new ConfigBuilder(OSS_ENDPOINT)
- .doc("The endpoint of the Aliyun oss")
- .version(ConfigConstants.VERSION_0_7_0)
- .stringConf()
- .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
- .create();
-
- public static final ConfigEntry<String> PAIMON_OSS_ACCESS_KEY_ENTRY =
- new ConfigBuilder(OSS_ACCESS_KEY)
- .doc("The access key of the Aliyun oss")
- .version(ConfigConstants.VERSION_0_7_0)
- .stringConf()
- .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
- .create();
-
- public static final ConfigEntry<String> PAIMON_OSS_SECRET_KEY_ENTRY =
- new ConfigBuilder(OSS_SECRET_KEY)
- .doc("The secret key of the Aliyun oss")
- .version(ConfigConstants.VERSION_0_7_0)
- .stringConf()
- .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
- .create();
-
- public String getOSSEndpoint() {
- return get(PAIMON_OSS_ENDPOINT_ENTRY);
- }
-
- public String getOSSAccessKey() {
- return get(PAIMON_OSS_ACCESS_KEY_ENTRY);
- }
-
- public String getOSSSecretKey() {
- return get(PAIMON_OSS_SECRET_KEY_ENTRY);
- }
-
- public static final Map<String, PropertyEntry<?>>
OSS_FILESYSTEM_PROPERTY_ENTRIES =
- new ImmutableMap.Builder<String, PropertyEntry<?>>()
- .put(
- OSS_ENDPOINT,
- PropertyEntry.stringOptionalPropertyEntry(
- OSS_ENDPOINT,
- "The endpoint of the Aliyun oss",
- false /* immutable */,
- null /* defaultValue */,
- false /* hidden */))
- .put(
- OSS_ACCESS_KEY,
- PropertyEntry.stringOptionalPropertyEntry(
- OSS_ACCESS_KEY,
- "The access key of the Aliyun oss",
- false /* immutable */,
- null /* defaultValue */,
- false /* hidden */))
- .put(
- OSS_SECRET_KEY,
- PropertyEntry.stringOptionalPropertyEntry(
- OSS_SECRET_KEY,
- "The secret key of the Aliyun oss",
- false /* immutable */,
- null /* defaultValue */,
- false /* hidden */))
- .build();
-}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/storage/PaimonS3FileSystemConfig.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/storage/PaimonS3FileSystemConfig.java
deleted file mode 100644
index 6588e4a526..0000000000
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/storage/PaimonS3FileSystemConfig.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.gravitino.catalog.lakehouse.paimon.storage;
-
-import com.google.common.collect.ImmutableMap;
-import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.gravitino.Config;
-import org.apache.gravitino.catalog.lakehouse.paimon.PaimonConstants;
-import org.apache.gravitino.config.ConfigBuilder;
-import org.apache.gravitino.config.ConfigConstants;
-import org.apache.gravitino.config.ConfigEntry;
-import org.apache.gravitino.connector.PropertyEntry;
-
-public class PaimonS3FileSystemConfig extends Config {
- // S3 related properties
- public static final String S3_ENDPOINT = PaimonConstants.S3_ENDPOINT;
- public static final String S3_ACCESS_KEY = PaimonConstants.S3_ACCESS_KEY;
- public static final String S3_SECRET_KEY = PaimonConstants.S3_SECRET_KEY;
-
- public PaimonS3FileSystemConfig(Map<String, String> properties) {
- super(false);
- loadFromMap(properties, k -> true);
- }
-
- public static final ConfigEntry<String> PAIMON_S3_ENDPOINT_ENTRY =
- new ConfigBuilder(S3_ENDPOINT)
- .doc("The endpoint of the AWS s3")
- .version(ConfigConstants.VERSION_0_7_0)
- .stringConf()
- .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
- .create();
-
- public static final ConfigEntry<String> PAIMON_S3_ACCESS_KEY_ENTRY =
- new ConfigBuilder(S3_ACCESS_KEY)
- .doc("The access key of the AWS s3")
- .version(ConfigConstants.VERSION_0_7_0)
- .stringConf()
- .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
- .create();
-
- public static final ConfigEntry<String> PAIMON_S3_SECRET_KEY_ENTRY =
- new ConfigBuilder(S3_SECRET_KEY)
- .doc("The secret key of the AWS s3")
- .version(ConfigConstants.VERSION_0_7_0)
- .stringConf()
- .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
- .create();
-
- public String getS3Endpoint() {
- return get(PAIMON_S3_ENDPOINT_ENTRY);
- }
-
- public String getS3AccessKey() {
- return get(PAIMON_S3_ACCESS_KEY_ENTRY);
- }
-
- public String getS3SecretKey() {
- return get(PAIMON_S3_SECRET_KEY_ENTRY);
- }
-
- public static final Map<String, PropertyEntry<?>>
S3_FILESYSTEM_PROPERTY_ENTRIES =
- new ImmutableMap.Builder<String, PropertyEntry<?>>()
- .put(
- S3_ENDPOINT,
- PropertyEntry.stringOptionalPropertyEntry(
- S3_ENDPOINT,
- "The endpoint of the AWS s3",
- false /* immutable */,
- null /* defaultValue */,
- false /* hidden */))
- .put(
- S3_ACCESS_KEY,
- PropertyEntry.stringOptionalPropertyEntry(
- S3_ACCESS_KEY,
- "The access key of the AWS s3",
- false /* immutable */,
- null /* defaultValue */,
- false /* hidden */))
- .put(
- S3_SECRET_KEY,
- PropertyEntry.stringOptionalPropertyEntry(
- S3_SECRET_KEY,
- "The secret key of the AWS s3",
- false /* immutable */,
- null /* defaultValue */,
- false /* hidden */))
- .build();
-}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/storage/S3StorageConfig.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/storage/S3StorageConfig.java
deleted file mode 100644
index 213589d788..0000000000
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/storage/S3StorageConfig.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.gravitino.catalog.lakehouse.paimon.storage;
-
-import static
org.apache.gravitino.storage.S3Properties.GRAVITINO_S3_ACCESS_KEY_ID;
-import static org.apache.gravitino.storage.S3Properties.GRAVITINO_S3_ENDPOINT;
-import static
org.apache.gravitino.storage.S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY;
-
-import com.google.common.collect.ImmutableMap;
-import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.gravitino.Config;
-import org.apache.gravitino.config.ConfigBuilder;
-import org.apache.gravitino.config.ConfigConstants;
-import org.apache.gravitino.config.ConfigEntry;
-import org.apache.gravitino.connector.PropertyEntry;
-
-public class S3StorageConfig extends Config {
-
- public S3StorageConfig(Map<String, String> properties) {
- super(false);
- loadFromMap(properties, k -> true);
- }
-
- // Unified S3
- public static final ConfigEntry<String> PAIMON_S3_ENDPOINT_ENTRY =
- new ConfigBuilder(GRAVITINO_S3_ENDPOINT)
- .doc("The endpoint of the AWS s3")
- .version(ConfigConstants.VERSION_0_7_0)
- .stringConf()
- .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
- .create();
-
- public static final ConfigEntry<String> PAIMON_S3_ACCESS_KEY_ENTRY =
- new ConfigBuilder(GRAVITINO_S3_ACCESS_KEY_ID)
- .doc("The access key of the AWS s3")
- .version(ConfigConstants.VERSION_0_7_0)
- .stringConf()
- .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
- .create();
-
- public static final ConfigEntry<String> PAIMON_S3_SECRET_KEY_ENTRY =
- new ConfigBuilder(GRAVITINO_S3_SECRET_ACCESS_KEY)
- .doc("The secret key of the AWS s3")
- .version(ConfigConstants.VERSION_0_7_0)
- .stringConf()
- .checkValue(StringUtils::isNotBlank,
ConfigConstants.NOT_BLANK_ERROR_MSG)
- .create();
-
- public String getS3Endpoint() {
- return get(PAIMON_S3_ENDPOINT_ENTRY);
- }
-
- public String getS3AccessKey() {
- return get(PAIMON_S3_ACCESS_KEY_ENTRY);
- }
-
- public String getS3SecretKey() {
- return get(PAIMON_S3_SECRET_KEY_ENTRY);
- }
-
- public static final Map<String, PropertyEntry<?>>
S3_FILESYSTEM_PROPERTY_ENTRIES =
- new ImmutableMap.Builder<String, PropertyEntry<?>>()
- .put(
- GRAVITINO_S3_ENDPOINT,
- PropertyEntry.stringOptionalPropertyEntry(
- GRAVITINO_S3_ENDPOINT,
- "The endpoint of the AWS s3",
- false /* immutable */,
- null /* defaultValue */,
- false /* hidden */))
- .put(
- GRAVITINO_S3_ACCESS_KEY_ID,
- PropertyEntry.stringOptionalPropertyEntry(
- GRAVITINO_S3_ACCESS_KEY_ID,
- "The access key of the AWS s3",
- false /* immutable */,
- null /* defaultValue */,
- false /* hidden */))
- .put(
- GRAVITINO_S3_SECRET_ACCESS_KEY,
- PropertyEntry.stringOptionalPropertyEntry(
- GRAVITINO_S3_SECRET_ACCESS_KEY,
- "The secret key of the AWS s3",
- false /* immutable */,
- null /* defaultValue */,
- false /* hidden */))
- .build();
-}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalogPropertiesMetadata.java
new file mode 100644
index 0000000000..c4025daa1f
--- /dev/null
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalogPropertiesMetadata.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.paimon;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.gravitino.storage.OSSProperties;
+import org.apache.gravitino.storage.S3Properties;
+import org.junit.jupiter.api.Test;
+
+public class TestPaimonCatalogPropertiesMetadata {
+
+ private final PaimonCatalogPropertiesMetadata metadata = new
PaimonCatalogPropertiesMetadata();
+
+ @Test
+ void testSensitivePropertiesAreHidden() {
+
assertTrue(metadata.isHiddenProperty(PaimonCatalogPropertiesMetadata.GRAVITINO_JDBC_USER));
+
assertTrue(metadata.isHiddenProperty(PaimonCatalogPropertiesMetadata.GRAVITINO_JDBC_PASSWORD));
+
assertTrue(metadata.isHiddenProperty(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID));
+
assertTrue(metadata.isHiddenProperty(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY));
+
assertTrue(metadata.isHiddenProperty(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID));
+
assertTrue(metadata.isHiddenProperty(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET));
+ // REST bearer token and DLF credentials share the same hidden requirement
+ assertTrue(metadata.isHiddenProperty(PaimonConstants.TOKEN));
+
assertTrue(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID));
+
assertTrue(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET));
+
assertTrue(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN));
+ }
+
+ @Test
+ void testNonSensitivePropertiesAreNotHidden() {
+
assertFalse(metadata.isHiddenProperty(PaimonCatalogPropertiesMetadata.WAREHOUSE));
+
assertFalse(metadata.isHiddenProperty(PaimonCatalogPropertiesMetadata.URI));
+ // DLF token metadata (provider type, path, loader) is not a credential —
stays visible.
+
assertFalse(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_TOKEN_PROVIDER));
+
assertFalse(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_DLF_TOKEN_PATH));
+
assertFalse(metadata.isHiddenProperty(PaimonConstants.GRAVITINO_DLF_TOKEN_LOADER));
+ }
+}
diff --git a/docs/apache-hive-catalog.md b/docs/apache-hive-catalog.md
index 8b509ab445..2391a99690 100644
--- a/docs/apache-hive-catalog.md
+++ b/docs/apache-hive-catalog.md
@@ -82,6 +82,10 @@ When using Gravitino authorization for Hive with Apache
Ranger, see the [Authori
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `s3-access-key-id`,
`s3-secret-access-key`, `oss-access-key-id`, and `oss-secret-access-key` are
hidden from the load catalog response since Gravitino 1.3.0. Use the
[credential vending API](security/credential-vending.md) to retrieve them at
runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/aws-glue-catalog.md b/docs/aws-glue-catalog.md
index ccdff0fb99..63174a1180 100644
--- a/docs/aws-glue-catalog.md
+++ b/docs/aws-glue-catalog.md
@@ -54,6 +54,10 @@ Besides the [common catalog
properties](./gravitino-server-config.md#catalog-pro
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `aws-access-key-id` and
`aws-secret-access-key` are hidden from the load catalog response since
Gravitino 1.3.0. Use the [credential vending
API](security/credential-vending.md) to retrieve them at runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/jdbc-doris-catalog.md b/docs/jdbc-doris-catalog.md
index 1a0093a22d..8f45476d5f 100644
--- a/docs/jdbc-doris-catalog.md
+++ b/docs/jdbc-doris-catalog.md
@@ -84,6 +84,10 @@ Returning null for DATETIME type precision. Driver version:
mysql-connector-java
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `jdbc-user` and `jdbc-password` are
hidden from the load catalog response since Gravitino 1.3.0. Use the
[credential vending API](security/credential-vending.md) to retrieve them at
runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/jdbc-hologres-catalog.md b/docs/jdbc-hologres-catalog.md
index 8c1aee3165..0fc76cd0d6 100644
--- a/docs/jdbc-hologres-catalog.md
+++ b/docs/jdbc-hologres-catalog.md
@@ -61,6 +61,10 @@ Hologres uses the PostgreSQL JDBC Driver (version 42.3.2 or
later recommended).
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `jdbc-user` and `jdbc-password` are
hidden from the load catalog response since Gravitino 1.3.0. Use the
[credential vending API](security/credential-vending.md) to retrieve them at
runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/jdbc-mysql-catalog.md b/docs/jdbc-mysql-catalog.md
index f447ae76c1..dfee4ac223 100644
--- a/docs/jdbc-mysql-catalog.md
+++ b/docs/jdbc-mysql-catalog.md
@@ -86,6 +86,10 @@ Returning null for TIMESTAMP type precision. Driver version:
mysql-connector-jav
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `jdbc-user` and `jdbc-password` are
hidden from the load catalog response since Gravitino 1.3.0. Use the
[credential vending API](security/credential-vending.md) to retrieve them at
runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/jdbc-oceanbase-catalog.md b/docs/jdbc-oceanbase-catalog.md
index 00face9557..222b0d3643 100644
--- a/docs/jdbc-oceanbase-catalog.md
+++ b/docs/jdbc-oceanbase-catalog.md
@@ -86,6 +86,10 @@ Returning null for TIMESTAMP type precision. Driver version:
mysql-connector-jav
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `jdbc-user` and `jdbc-password` are
hidden from the load catalog response since Gravitino 1.3.0. Use the
[credential vending API](security/credential-vending.md) to retrieve them at
runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/jdbc-postgresql-catalog.md b/docs/jdbc-postgresql-catalog.md
index 235abee1ff..8235f1b069 100644
--- a/docs/jdbc-postgresql-catalog.md
+++ b/docs/jdbc-postgresql-catalog.md
@@ -62,6 +62,10 @@ In PostgreSQL, the database corresponds to the Gravitino
catalog, and the schema
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `jdbc-user` and `jdbc-password` are
hidden from the load catalog response since Gravitino 1.3.0. Use the
[credential vending API](security/credential-vending.md) to retrieve them at
runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/jdbc-starrocks-catalog.md b/docs/jdbc-starrocks-catalog.md
index aeb6dd9c86..a69d808b11 100644
--- a/docs/jdbc-starrocks-catalog.md
+++ b/docs/jdbc-starrocks-catalog.md
@@ -82,6 +82,10 @@ Returning null for DATETIME type precision. Driver version:
mysql-connector-java
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `jdbc-user` and `jdbc-password` are
hidden from the load catalog response since Gravitino 1.3.0. Use the
[credential vending API](security/credential-vending.md) to retrieve them at
runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/lakehouse-iceberg-catalog.md
b/docs/lakehouse-iceberg-catalog.md
index 68931a3672..b9c7a912e4 100644
--- a/docs/lakehouse-iceberg-catalog.md
+++ b/docs/lakehouse-iceberg-catalog.md
@@ -254,6 +254,10 @@ Gravitino provides the build-in
`org.apache.gravitino.iceberg.common.cache.Local
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `s3-access-key-id`,
`s3-secret-access-key`, `oss-access-key-id`, and `oss-secret-access-key` are
hidden from the load catalog response since Gravitino 1.3.0. Use the
[credential vending API](security/credential-vending.md) to retrieve them at
runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/lakehouse-paimon-catalog.md b/docs/lakehouse-paimon-catalog.md
index fed8d2a2e1..3548efdcfa 100644
--- a/docs/lakehouse-paimon-catalog.md
+++ b/docs/lakehouse-paimon-catalog.md
@@ -47,8 +47,8 @@ Builds with Apache Paimon `1.2`.
| `s3-endpoint` | The endpoint of the AWS
S3.
| (none)
| required if the value of `warehouse` is a S3 path
[...]
| `s3-access-key-id` | The access key of the
AWS S3.
| (none)
| required if the value of `warehouse` is a S3 path
[...]
| `s3-secret-access-key` | The secret key of the
AWS S3.
| (none)
| required if the value of `warehouse` is a S3 path
[...]
-| `token-provider` | The token provider type
for Paimon catalog backend.
| Token provider could be `bear` or `dlf`.
| required if the value of `catalog-backend` is `rest`.
[...]
-| `token` | The bear token for
Paimon REST catalog authentication.
| (none)
| required if the value of `token-provider` is
`bear`.
[...]
+| `token-provider` | The token provider type
for Paimon catalog backend.
| Token provider could be `bearer` or `dlf`.
| required if the value of `catalog-backend` is `rest`.
[...]
+| `token` | The bearer token for
Paimon REST catalog authentication.
| (none)
| required if the value of `token-provider` is
`bearer`.
[...]
| `dlf-access-key-id` | The access key ID for
Aliyun DLF (Data Lake Formation).
| (none)
| required if the value of `catalog-backend` is `rest`
and accessing Aliyun DLF Paimon REST server.
[...]
| `dlf-access-key-secret` | The access key secret
for Aliyun DLF.
| (none)
| required if the value of `catalog-backend` is `rest`
and accessing Aliyun DLF Paimon REST server.
[...]
| `dlf-security-token` | The security token for
Aliyun DLF.
| (none)
| No
[...]
@@ -57,6 +57,7 @@ Builds with Apache Paimon `1.2`.
:::note
- If you want to use the `oss` or `s3` warehouse, you need to place related
jars in the `catalogs/lakehouse-paimon/lib` directory, more information can be
found in the [Paimon
S3](https://paimon.apache.org/docs/1.2/maintenance/filesystems/#s3).
+- If you use an S3 warehouse, also download
[`gravitino-aws-<version>.jar`](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws)
and place it in the `catalogs/lakehouse-paimon/libs` directory to enable
credential vending. For OSS, use
[`gravitino-aliyun-<version>.jar`](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun)
instead.
- If you want to use REST backend, Gravitino Paimon catalog supports Aliyun
DLF (Data Lake Formation) as the REST catalog service. You need to configure
the DLF-related properties eg:
```
{
@@ -97,6 +98,10 @@ Download the corresponding JDBC driver and place it to the
`catalogs/lakehouse-p
Refer to [Manage Relational Metadata Using
Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations)
for more details.
+:::note
+Sensitive catalog properties such as `s3-access-key-id`,
`s3-secret-access-key`, `jdbc-user`, and `jdbc-password` are hidden from the
load catalog response since Gravitino 1.3.0. Use the [credential vending
API](security/credential-vending.md) to retrieve them at runtime.
+:::
+
## Schema
### Schema Capabilities
diff --git a/docs/security/credential-vending.md
b/docs/security/credential-vending.md
index 4c7769c226..1df065d12e 100644
--- a/docs/security/credential-vending.md
+++ b/docs/security/credential-vending.md
@@ -12,13 +12,22 @@ Gravitino credential vending is used to generate temporary
or static credentials
### Capabilities
- Supports Gravitino Iceberg REST server.
-- Supports Gravitino server, only support Hadoop catalog.
+- Supports Gravitino server with the following catalog types:
+
+ | Catalog type | Supported credential types
| Since version |
+
|-----------------|-----------------------------------------------------------------|------------------|
+ | Hadoop (Fileset)| S3, OSS, GCS, ADLS
| 0.7.0-incubating |
+ | Hive | S3, OSS, GCS, ADLS
| 1.3.0 |
+ | Iceberg | S3, OSS, GCS, ADLS
| 1.3.0 |
+ | Glue | S3
| 1.3.0 |
+ | JDBC | JDBC user/password (`jdbc-user-password`)
| 1.3.0 |
+ | Paimon | S3, OSS, JDBC user/password (`jdbc-user-password`)
| 1.3.0 |
- Supports pluggable credentials with build-in credentials:
- S3: `S3TokenCredential`, `S3SecretKeyCredential`, `AwsIrsaCredential`
- GCS: `GCSTokenCredential`
- ADLS: `ADLSTokenCredential`, `AzureAccountKeyCredential`
- OSS: `OSSTokenCredential`, `OSSSecretKeyCredential`
-- No support for Spark/Trino/Flink connector yet.
+- Spark, Flink, and Trino connectors automatically consume vended credentials
for Hive, Iceberg, Glue, JDBC, and Paimon catalogs since 1.3.0.
## General Configurations
@@ -184,6 +193,54 @@ The classpath of the server:
- Iceberg REST server: the classpath differs by deployment mode; see the
[Deployment](../iceberg-rest-service.md#deployment) section.
- Fileset catalog: `catalogs/fileset/libs/`
+## Credential Vending for Catalogs
+
+Hive, Iceberg, Glue, JDBC, and Paimon catalogs support server-side credential
vending since Gravitino 1.4.0. This section explains how credential vending
works for these catalog types and how it differs from the Hadoop Fileset and
Iceberg REST server cases.
+
+### Auto-Detection of Credential Providers
+
+Relational catalogs **do not** require an explicit `credential-providers`
property in the catalog configuration. Instead, Gravitino automatically detects
which credential providers to enable based on the catalog properties you supply:
+
+- If `s3-access-key-id` / `s3-secret-access-key` are present, the S3
secret-key credential provider is enabled.
+- If `s3-role-arn` is also present, the S3 token (STS) credential provider is
enabled instead.
+- If `oss-access-key-id` / `oss-secret-access-key` are present, the OSS
secret-key or token credential provider is enabled.
+- For JDBC catalogs (and Paimon with `catalog-backend=jdbc`), if `jdbc-user` /
`jdbc-password` are set, the JDBC credential provider is enabled.
+- Paimon additionally supports OSS and S3 credential vending when the
corresponding cloud properties are set.
+
+No extra configuration is required beyond the catalog's normal properties.
+
+### Security: Sensitive Properties Hidden from REST API
+
+To protect credentials, all sensitive catalog properties (such as
`s3-access-key-id`, `s3-secret-access-key`, `jdbc-user`, `jdbc-password`, etc.)
are excluded from the `GET /api/metalakes/{metalake}/catalogs/{catalog}`
response. Clients retrieve credentials through the dedicated credential vending
endpoint instead.
+
+### Credential Vending REST API
+
+Clients retrieve vended credentials from:
+
+```
+GET /api/metalakes/{metalake}/objects/catalog/{catalog}/credentials
+```
+
+The server returns short-lived or static credentials that the client can use
to access the underlying storage directly.
+
+### Spark and Flink Connector Integration
+
+The Gravitino Spark and Flink connectors automatically call the credential
vending API and inject the returned credentials into the connector's
configuration, so no connector-side credential configuration is needed. For
example, `GravitinoHiveCatalog`, `GravitinoGlueCatalog`,
`GravitinoJdbcCatalog`, and `GravitinoIcebergCatalog` all consume the vended
credentials transparently.
+
+### Backward Compatibility: `gravitino.catalog.credential.backfillToProperties`
+
+During a rolling upgrade from Gravitino < 1.4.0 to 1.4.0, older clients that
read catalog properties directly (rather than calling `/credentials`) would
lose access to credentials because the properties are now hidden. To allow a
zero-downtime migration, set the following property in `gravitino.conf`:
+
+```properties
+gravitino.catalog.credential.backfillToProperties = true
+```
+
+When enabled, the server re-includes hidden credential properties in `GET
/catalogs/{catalog}` responses for backward compatibility.
+
+:::caution
+`gravitino.catalog.credential.backfillToProperties = true` exposes credentials
in plaintext in catalog GET responses. Disable it once all clients have been
upgraded to use the credential vending API.
+:::
+
## Example
### Credential Vending for Iceberg REST Server
@@ -204,7 +261,7 @@ gravitino.iceberg-rest.s3-region = {region_name}
gravitino.iceberg-rest.s3-role-arn = {role_arn}
```
-3. Exploring the Iceberg table with a Spark client with credential vending
enabled.
+3. Explore the Iceberg table with a Spark client with credential vending
enabled.
```shell
./bin/spark-sql -v \
