This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 3f277bee15 [#12457] feat(secret): Wire connectors and aux services to
getSecrets (#12572)
3f277bee15 is described below
commit 3f277bee159543785810b05f015ce96096277a8a
Author: MaSai <[email protected]>
AuthorDate: Tue Aug 25 15:58:17 2026 +0800
[#12457] feat(secret): Wire connectors and aux services to getSecrets
(#12572)
### What changes were proposed in this pull request?
Wire engines and aux services to the `getSecrets` API from #12458 so
runtime
conf merges secret-manager plaintext without changing default `load*`
omit
behavior:
- Spark / Flink / Trino: `properties() + getSecrets()`
- IRC (`DynamicIcebergConfigProvider`): merge `getSecrets()`, then
overlay
`JdbcCredential` so **credentials win** (aligned with JDBC connectors);
auxiliary mode uses `SecretManager.toPlaintextProperties`
- Lance: catalog/schema properties with secrets (in-process dispatcher
when
available, otherwise client `supportsSecrets()`)
- GVFS (Java + Python): merge catalog/schema/fileset secrets into FS
conf
(fileset > schema > catalog precedence)
### Why are the changes needed?
Fix: #12457
Connectors need secret plaintext at runtime. #12458 added the platform
`getSecrets` API; this PR wires the call sites.
### Does this PR introduce _any_ user-facing change?
- Yes: connector/aux runtime behavior now merges secret-manager
plaintext via
`getSecrets`
- No new public REST/API surface beyond #12458
### How was this patch tested?
```bash
./gradlew spotlessApply
./gradlew \
:spark-connector:spark-common:test \
--tests
'org.apache.gravitino.spark.connector.catalog.TestBaseCatalogSecrets' \
:flink-connector:flink-common:test \
--tests
'org.apache.gravitino.flink.connector.store.TestGravitinoCatalogStore' \
:trino-connector:trino-connector:test \
--tests
'org.apache.gravitino.trino.connector.catalog.TestCatalogConnectorManager' \
:iceberg:iceberg-rest-server:test \
--tests
'org.apache.gravitino.iceberg.service.provider.TestDynamicIcebergConfigProvider'
\
:lance:lance-common:test \
--tests
'org.apache.gravitino.lance.common.ops.gravitino.TestGravitinoLanceNamespaceWrapper'
\
:clients:filesystem-hadoop3:test \
--tests
'org.apache.gravitino.filesystem.hadoop.TestBaseGVFSOperationsSecrets' \
-PskipITs
# Python
python -m unittest \
clients/client-python/tests/unittests/test_gvfs_merge_secrets.py
```
---------
Co-authored-by: Cursor <[email protected]>
---
.../gravitino/filesystem/gvfs_base_operations.py | 36 +--
clients/client-python/tests/unittests/mock_base.py | 14 +-
.../tests/unittests/test_gvfs_merge_secrets.py | 125 ++++++++++
.../filesystem/hadoop/BaseGVFSOperations.java | 41 ++--
.../filesystem/hadoop/GravitinoMockServerBase.java | 19 ++
.../hadoop/TestBaseGVFSOperationsSecrets.java | 260 +++++++++++++++++++++
.../connector/store/GravitinoCatalogStore.java | 22 +-
.../connector/store/TestGravitinoCatalogStore.java | 150 ++++++++++++
.../provider/DynamicIcebergConfigProvider.java | 65 +++---
.../provider/TestDynamicIcebergConfigProvider.java | 254 ++++++++++++++++++--
.../GravitinoLanceNameSpaceOperations.java | 7 +-
.../gravitino/GravitinoLanceNamespaceWrapper.java | 44 ++++
.../gravitino/GravitinoLanceTableOperations.java | 6 +-
.../TestGravitinoLanceNamespaceWrapper.java | 233 +++++++++++++++++-
.../gravitino/TestGravitinoLanceModeParsing.java | 3 +
.../spark/connector/catalog/BaseCatalog.java | 12 +-
.../connector/catalog/TestBaseCatalogSecrets.java | 180 ++++++++++++++
.../connector/catalog/CatalogConnectorManager.java | 13 +-
.../trino/connector/metadata/GravitinoCatalog.java | 13 +-
.../trino/connector/GravitinoMockServer.java | 40 ++--
.../catalog/TestCatalogConnectorManager.java | 33 +++
.../connector/metadata/TestGravitinoCatalog.java | 17 ++
22 files changed, 1472 insertions(+), 115 deletions(-)
diff --git a/clients/client-python/gravitino/filesystem/gvfs_base_operations.py
b/clients/client-python/gravitino/filesystem/gvfs_base_operations.py
index 37c091caf3..71cac7801d 100644
--- a/clients/client-python/gravitino/filesystem/gvfs_base_operations.py
+++ b/clients/client-python/gravitino/filesystem/gvfs_base_operations.py
@@ -495,21 +495,32 @@ class BaseGVFSOperations(ABC):
def _merge_fileset_properties(
self,
- catalog: FilesetCatalog,
- schema: Schema,
- fileset: Fileset,
+ fileset_ident: NameIdentifier,
actual_location: str,
) -> Dict[str, str]:
- """Merge properties from catalog, schema, fileset, options, and
user-defined configs.
- :param catalog: The fileset catalog
- :param schema: The schema
- :param fileset: The fileset
+ """Merge properties from catalog, schema, fileset, options, and
configs.
+
+ Combines default load*.properties() with get_secrets() so every secret
URN
+ (including keys that may also appear in credential vending) becomes
plaintext
+ for FS access. Typed credentials remain available via get_credentials.
+
+ :param fileset_ident: The fileset identifier
:param actual_location: The actual storage location
:return: Merged properties dictionary
"""
+ catalog_name = fileset_ident.namespace().level(1)
+ schema_name = fileset_ident.namespace().level(2)
+ catalog = self._get_gravitino_client().load_catalog(catalog_name)
+ schema = catalog.as_schemas().load_schema(schema_name)
+ fileset = catalog.as_fileset_catalog().load_fileset(
+ NameIdentifier.of(schema_name, fileset_ident.name())
+ )
fileset_props = dict(catalog.properties() or {})
+ fileset_props.update(catalog.get_secrets())
fileset_props.update(schema.properties() or {})
+ fileset_props.update(schema.get_secrets())
fileset_props.update(fileset.properties() or {})
+ fileset_props.update(fileset.get_secrets())
if self._options:
fileset_props.update(self._options)
# Get user-defined configurations for the actual location
@@ -530,13 +541,6 @@ class BaseGVFSOperations(ABC):
:param location_name: The location name, None means the default
location
:return: The actual filesystem
"""
- catalog_ident: NameIdentifier = NameIdentifier.of(
- self._metalake, fileset_ident.namespace().level(1)
- )
- catalog = self._get_fileset_catalog(catalog_ident)
- schema = self._get_fileset_schema(
- NameIdentifier.parse(str(fileset_ident.namespace()))
- )
fileset = self._get_fileset(fileset_ident)
# Determine target location name
@@ -553,9 +557,7 @@ class BaseGVFSOperations(ABC):
f"Cannot find the location: {target_location_name} in fileset:
{fileset_ident}"
)
- fileset_props = self._merge_fileset_properties(
- catalog, schema, fileset, actual_location
- )
+ fileset_props = self._merge_fileset_properties(fileset_ident,
actual_location)
# Set caller context for credential vending
if location_name:
diff --git a/clients/client-python/tests/unittests/mock_base.py
b/clients/client-python/tests/unittests/mock_base.py
index 684dc77992..ca247d3170 100644
--- a/clients/client-python/tests/unittests/mock_base.py
+++ b/clients/client-python/tests/unittests/mock_base.py
@@ -224,15 +224,27 @@ def mock_load_schema(name: str):
_last_modifier="test",
_last_modified_time="2024-04-05T10:10:35.218Z",
)
- return SchemaDTO(
+ schema = SchemaDTO(
_name=name,
_comment="this is schema test",
_properties={"schema-prop": "schema-val"},
_audit=audit_dto,
)
+ mock_schema = MagicMock()
+ mock_schema.properties.return_value = schema.properties()
+ mock_schema.get_secrets.return_value = {}
+ return mock_schema
def mock_data(cls):
+ @patch(
+ "gravitino.client.generic_fileset.GenericFileset.get_secrets",
+ return_value={},
+ )
+ @patch(
+ "gravitino.client.fileset_catalog.FilesetCatalog.get_secrets",
+ return_value={},
+ )
@patch(
"gravitino.client.gravitino_client_base.GravitinoClientBase.load_metalake",
return_value=mock_load_metalake(),
diff --git a/clients/client-python/tests/unittests/test_gvfs_merge_secrets.py
b/clients/client-python/tests/unittests/test_gvfs_merge_secrets.py
new file mode 100644
index 0000000000..e13319dd9d
--- /dev/null
+++ b/clients/client-python/tests/unittests/test_gvfs_merge_secrets.py
@@ -0,0 +1,125 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+from unittest.mock import MagicMock, patch
+
+from gravitino.filesystem.gvfs_default_operations import DefaultGVFSOperations
+from gravitino.name_identifier import NameIdentifier
+
+
+# pylint: disable=protected-access
+class TestGVFSMergeSecrets(unittest.TestCase):
+ """Unit tests for _merge_fileset_properties including get_secrets()."""
+
+ def test_merge_secrets(self):
+ operations = DefaultGVFSOperations(
+ server_uri="http://localhost:8090";, metalake_name="ml", options={}
+ )
+
+ catalog = MagicMock()
+ schema = MagicMock()
+ fileset = MagicMock()
+ catalog.properties.return_value = {"c-vis": "1"}
+ catalog.get_secrets.return_value = {"c-secret": "cs"}
+ catalog.as_schemas.return_value.load_schema.return_value = schema
+ schema.properties.return_value = {"s-vis": "2"}
+ schema.get_secrets.return_value = {"s-secret": "ss"}
+ catalog.as_fileset_catalog.return_value.load_fileset.return_value =
fileset
+ fileset.properties.return_value = {"f-vis": "3"}
+ fileset.get_secrets.return_value = {"f-secret": "fs"}
+
+ client = MagicMock()
+ client.load_catalog.return_value = catalog
+
+ with patch.object(operations, "_get_gravitino_client",
return_value=client):
+ with patch.object(operations, "_get_user_defined_configs",
return_value={}):
+ merged = operations._merge_fileset_properties(
+ NameIdentifier.of("ml", "catalog", "schema", "fs"),
+ "file:///tmp/data",
+ )
+
+ self.assertEqual(merged["c-vis"], "1")
+ self.assertEqual(merged["c-secret"], "cs")
+ self.assertEqual(merged["s-vis"], "2")
+ self.assertEqual(merged["s-secret"], "ss")
+ self.assertEqual(merged["f-vis"], "3")
+ self.assertEqual(merged["f-secret"], "fs")
+
+ def test_secret_override(self):
+ operations = DefaultGVFSOperations(
+ server_uri="http://localhost:8090";, metalake_name="ml", options={}
+ )
+
+ catalog = MagicMock()
+ schema = MagicMock()
+ fileset = MagicMock()
+ catalog.properties.return_value = {"shared": "from-catalog-props"}
+ catalog.get_secrets.return_value = {"shared": "from-catalog-secret"}
+ catalog.as_schemas.return_value.load_schema.return_value = schema
+ schema.properties.return_value = {"shared": "from-schema-props"}
+ schema.get_secrets.return_value = {"shared": "from-schema-secret"}
+ catalog.as_fileset_catalog.return_value.load_fileset.return_value =
fileset
+ fileset.properties.return_value = {"shared": "from-fileset-props"}
+ fileset.get_secrets.return_value = {"shared": "from-fileset-secret"}
+
+ client = MagicMock()
+ client.load_catalog.return_value = catalog
+
+ with patch.object(operations, "_get_gravitino_client",
return_value=client):
+ with patch.object(operations, "_get_user_defined_configs",
return_value={}):
+ merged = operations._merge_fileset_properties(
+ NameIdentifier.of("ml", "catalog", "schema", "fs"),
+ "file:///tmp/data",
+ )
+
+ self.assertEqual(merged["shared"], "from-fileset-secret")
+
+ def test_null_props(self):
+ operations = DefaultGVFSOperations(
+ server_uri="http://localhost:8090";, metalake_name="ml", options={}
+ )
+
+ catalog = MagicMock()
+ schema = MagicMock()
+ fileset = MagicMock()
+ catalog.properties.return_value = None
+ catalog.get_secrets.return_value = {"c-secret": "cs"}
+ catalog.as_schemas.return_value.load_schema.return_value = schema
+ schema.properties.return_value = None
+ schema.get_secrets.return_value = {"s-secret": "ss"}
+ catalog.as_fileset_catalog.return_value.load_fileset.return_value =
fileset
+ fileset.properties.return_value = None
+ fileset.get_secrets.return_value = {"f-secret": "fs"}
+
+ client = MagicMock()
+ client.load_catalog.return_value = catalog
+
+ with patch.object(operations, "_get_gravitino_client",
return_value=client):
+ with patch.object(operations, "_get_user_defined_configs",
return_value={}):
+ merged = operations._merge_fileset_properties(
+ NameIdentifier.of("ml", "catalog", "schema", "fs"),
+ "file:///tmp/data",
+ )
+
+ self.assertEqual(merged["c-secret"], "cs")
+ self.assertEqual(merged["s-secret"], "ss")
+ self.assertEqual(merged["f-secret"], "fs")
+ self.assertEqual(len(merged), 3)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/BaseGVFSOperations.java
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/BaseGVFSOperations.java
index bece7047f5..19fb0f72ee 100644
---
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/BaseGVFSOperations.java
+++
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/BaseGVFSOperations.java
@@ -80,6 +80,7 @@ import org.apache.gravitino.exceptions.NoSuchFilesetException;
import org.apache.gravitino.exceptions.NoSuchLocationNameException;
import org.apache.gravitino.file.Fileset;
import org.apache.gravitino.file.FilesetCatalog;
+import org.apache.gravitino.secret.SupportsSecrets;
import org.apache.gravitino.storage.AzureProperties;
import org.apache.gravitino.storage.OSSProperties;
import org.apache.gravitino.storage.S3Properties;
@@ -726,7 +727,7 @@ public abstract class BaseGVFSOperations implements
Closeable {
filesetIdent);
Path targetLocation = new
Path(fileset.storageLocations().get(targetLocationName));
- Map<String, String> allProperties = getAllProperties(filesetIdent,
fileset.properties());
+ Map<String, String> allProperties = getAllProperties(filesetIdent);
allProperties.putAll(
FilesetUtil.getUserDefinedFileSystemConfigs(
targetLocation.toUri(), allProperties,
FS_GRAVITINO_PATH_CONFIG_PREFIX));
@@ -961,21 +962,29 @@ public abstract class BaseGVFSOperations implements
Closeable {
return cacheBuilder.build();
}
- private Map<String, String> getAllProperties(
- NameIdentifier filesetIdent, Map<String, String> filesetProperties) {
- Map<String, String> allProperties = new HashMap<>();
- Catalog catalog =
- (Catalog)
- getFilesetCatalog(
- NameIdentifier.of(
- filesetIdent.namespace().level(0),
filesetIdent.namespace().level(1)));
- allProperties.putAll(catalog.properties());
-
- Schema schema =
getSchema(NameIdentifier.parse(filesetIdent.namespace().toString()));
- allProperties.putAll(schema.properties());
- allProperties.putAll(filesetProperties);
- allProperties.putAll(extractNonDefaultConfig(conf));
- return allProperties;
+ @VisibleForTesting
+ Map<String, String> getAllProperties(NameIdentifier filesetIdent) {
+ String catalogName = filesetIdent.namespace().level(1);
+ String schemaName = filesetIdent.namespace().level(2);
+ Catalog catalog = getGravitinoClient().loadCatalog(catalogName);
+ Schema schema = catalog.asSchemas().loadSchema(schemaName);
+ Fileset fileset =
+ catalog.asFilesetCatalog().loadFileset(NameIdentifier.of(schemaName,
filesetIdent.name()));
+
+ Map<String, String> all = new HashMap<>();
+ putPropsAndSecrets(all, catalog.properties(), catalog.supportsSecrets());
+ putPropsAndSecrets(all, schema.properties(), schema.supportsSecrets());
+ putPropsAndSecrets(all, fileset.properties(), fileset.supportsSecrets());
+ all.putAll(extractNonDefaultConfig(conf));
+ return all;
+ }
+
+ private static void putPropsAndSecrets(
+ Map<String, String> target, Map<String, String> props, SupportsSecrets
secrets) {
+ if (props != null) {
+ target.putAll(props);
+ }
+ target.putAll(secrets.getSecrets());
}
private Map<String, String> getNecessaryProperties(Map<String, String>
properties) {
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/GravitinoMockServerBase.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/GravitinoMockServerBase.java
index b950536d12..aa8b1b6784 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/GravitinoMockServerBase.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/GravitinoMockServerBase.java
@@ -37,6 +37,7 @@ import org.apache.gravitino.dto.file.FilesetDTO;
import org.apache.gravitino.dto.responses.CatalogResponse;
import org.apache.gravitino.dto.responses.FilesetResponse;
import org.apache.gravitino.dto.responses.MetalakeResponse;
+import org.apache.gravitino.dto.responses.SecretsResponse;
import org.apache.gravitino.dto.responses.VersionResponse;
import org.apache.gravitino.file.Fileset;
import org.apache.gravitino.json.JsonUtils;
@@ -67,12 +68,14 @@ public abstract class GravitinoMockServerBase {
mockServer = ClientAndServer.startClientAndServer(0);
port = mockServer.getLocalPort();
mockAPIVersion();
+ mockEmptySecretsAPI();
}
@AfterEach
public void reset() {
mockServer.reset();
mockAPIVersion();
+ mockEmptySecretsAPI();
}
@AfterAll
@@ -132,6 +135,22 @@ public abstract class GravitinoMockServerBase {
}
}
+ /** Mocks metadata-object secrets endpoints used by {@code
getAllProperties}. */
+ protected static void mockEmptySecretsAPI() {
+ try {
+ String emptySecretsJson = MAPPER.writeValueAsString(new
SecretsResponse(ImmutableMap.of()));
+ mockServer
+ .when(
+ HttpRequest.request()
+ .withMethod("GET")
+ .withPath("/api/metalakes/.*/objects/.*/.*/secrets"),
+ Times.unlimited())
+
.respond(HttpResponse.response().withStatusCode(SC_OK).withBody(emptySecretsJson));
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
protected static void mockMetalakeDTO(String name, String comment) {
MetalakeDTO mockMetalake =
MetalakeDTO.builder()
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestBaseGVFSOperationsSecrets.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestBaseGVFSOperationsSecrets.java
new file mode 100644
index 0000000000..2aa4c129c6
--- /dev/null
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestBaseGVFSOperationsSecrets.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.filesystem.hadoop;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.gravitino.Catalog;
+import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.Schema;
+import org.apache.gravitino.SupportsSchemas;
+import org.apache.gravitino.client.GravitinoClient;
+import org.apache.gravitino.file.Fileset;
+import org.apache.gravitino.file.FilesetCatalog;
+import org.apache.gravitino.secret.SupportsSecrets;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.junit.jupiter.api.Test;
+
+public class TestBaseGVFSOperationsSecrets {
+
+ @Test
+ public void testMergeSecrets() throws Exception {
+ Configuration conf = new Configuration();
+
conf.set(GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_METALAKE_KEY,
"ml");
+ conf.set(
+ GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_SERVER_URI_KEY,
+ "http://localhost:8090";);
+
+ Catalog catalog = mock(Catalog.class);
+ Schema schema = mock(Schema.class);
+ Fileset fileset = mock(Fileset.class);
+ SupportsSchemas schemas = mock(SupportsSchemas.class);
+ FilesetCatalog filesetCatalog = mock(FilesetCatalog.class);
+ SupportsSecrets catalogSecrets = mock(SupportsSecrets.class);
+ SupportsSecrets schemaSecrets = mock(SupportsSecrets.class);
+ SupportsSecrets filesetSecrets = mock(SupportsSecrets.class);
+
+ when(catalog.properties()).thenReturn(Map.of("c-vis", "1"));
+ when(catalog.supportsSecrets()).thenReturn(catalogSecrets);
+ when(catalogSecrets.getSecrets()).thenReturn(Map.of("c-secret", "cs"));
+ when(catalog.asSchemas()).thenReturn(schemas);
+ when(schemas.loadSchema("schema")).thenReturn(schema);
+ when(schema.properties()).thenReturn(Map.of("s-vis", "2"));
+ when(schema.supportsSecrets()).thenReturn(schemaSecrets);
+ when(schemaSecrets.getSecrets()).thenReturn(Map.of("s-secret", "ss"));
+ when(catalog.asFilesetCatalog()).thenReturn(filesetCatalog);
+ when(filesetCatalog.loadFileset(NameIdentifier.of("schema",
"fs"))).thenReturn(fileset);
+ when(fileset.properties()).thenReturn(Map.of("f-vis", "3"));
+ when(fileset.supportsSecrets()).thenReturn(filesetSecrets);
+ when(filesetSecrets.getSecrets()).thenReturn(Map.of("f-secret", "fs"));
+
+ GravitinoClient client = mock(GravitinoClient.class);
+ when(client.loadCatalog("catalog")).thenReturn(catalog);
+
+ TestOps ops = new TestOps(conf, client);
+ Map<String, String> all =
+ ops.getAllProperties(NameIdentifier.of("ml", "catalog", "schema",
"fs"));
+
+ assertEquals("1", all.get("c-vis"));
+ assertEquals("cs", all.get("c-secret"));
+ assertEquals("2", all.get("s-vis"));
+ assertEquals("ss", all.get("s-secret"));
+ assertEquals("3", all.get("f-vis"));
+ assertEquals("fs", all.get("f-secret"));
+ }
+
+ @Test
+ public void testSecretOverride() throws Exception {
+ Configuration conf = new Configuration();
+
conf.set(GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_METALAKE_KEY,
"ml");
+ conf.set(
+ GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_SERVER_URI_KEY,
+ "http://localhost:8090";);
+
+ Catalog catalog = mock(Catalog.class);
+ Schema schema = mock(Schema.class);
+ Fileset fileset = mock(Fileset.class);
+ SupportsSchemas schemas = mock(SupportsSchemas.class);
+ FilesetCatalog filesetCatalog = mock(FilesetCatalog.class);
+ SupportsSecrets catalogSecrets = mock(SupportsSecrets.class);
+ SupportsSecrets schemaSecrets = mock(SupportsSecrets.class);
+ SupportsSecrets filesetSecrets = mock(SupportsSecrets.class);
+
+ when(catalog.properties()).thenReturn(Map.of("shared",
"from-catalog-props"));
+ when(catalog.supportsSecrets()).thenReturn(catalogSecrets);
+ when(catalogSecrets.getSecrets()).thenReturn(Map.of("shared",
"from-catalog-secret"));
+ when(catalog.asSchemas()).thenReturn(schemas);
+ when(schemas.loadSchema("schema")).thenReturn(schema);
+ when(schema.properties()).thenReturn(Map.of("shared",
"from-schema-props"));
+ when(schema.supportsSecrets()).thenReturn(schemaSecrets);
+ when(schemaSecrets.getSecrets()).thenReturn(Map.of("shared",
"from-schema-secret"));
+ when(catalog.asFilesetCatalog()).thenReturn(filesetCatalog);
+ when(filesetCatalog.loadFileset(NameIdentifier.of("schema",
"fs"))).thenReturn(fileset);
+ when(fileset.properties()).thenReturn(Map.of("shared",
"from-fileset-props"));
+ when(fileset.supportsSecrets()).thenReturn(filesetSecrets);
+ when(filesetSecrets.getSecrets()).thenReturn(Map.of("shared",
"from-fileset-secret"));
+
+ GravitinoClient client = mock(GravitinoClient.class);
+ when(client.loadCatalog("catalog")).thenReturn(catalog);
+
+ TestOps ops = new TestOps(conf, client);
+ Map<String, String> all =
+ ops.getAllProperties(NameIdentifier.of("ml", "catalog", "schema",
"fs"));
+
+ assertEquals("from-fileset-secret", all.get("shared"));
+ }
+
+ @Test
+ public void testNullProps() throws Exception {
+ Configuration conf = new Configuration();
+
conf.set(GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_METALAKE_KEY,
"ml");
+ conf.set(
+ GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_SERVER_URI_KEY,
+ "http://localhost:8090";);
+
+ Catalog catalog = mock(Catalog.class);
+ Schema schema = mock(Schema.class);
+ Fileset fileset = mock(Fileset.class);
+ SupportsSchemas schemas = mock(SupportsSchemas.class);
+ FilesetCatalog filesetCatalog = mock(FilesetCatalog.class);
+ SupportsSecrets catalogSecrets = mock(SupportsSecrets.class);
+ SupportsSecrets schemaSecrets = mock(SupportsSecrets.class);
+ SupportsSecrets filesetSecrets = mock(SupportsSecrets.class);
+
+ when(catalog.properties()).thenReturn(null);
+ when(catalog.supportsSecrets()).thenReturn(catalogSecrets);
+ when(catalogSecrets.getSecrets()).thenReturn(Map.of("c-secret", "cs"));
+ when(catalog.asSchemas()).thenReturn(schemas);
+ when(schemas.loadSchema("schema")).thenReturn(schema);
+ when(schema.properties()).thenReturn(null);
+ when(schema.supportsSecrets()).thenReturn(schemaSecrets);
+ when(schemaSecrets.getSecrets()).thenReturn(Map.of("s-secret", "ss"));
+ when(catalog.asFilesetCatalog()).thenReturn(filesetCatalog);
+ when(filesetCatalog.loadFileset(NameIdentifier.of("schema",
"fs"))).thenReturn(fileset);
+ when(fileset.properties()).thenReturn(null);
+ when(fileset.supportsSecrets()).thenReturn(filesetSecrets);
+ when(filesetSecrets.getSecrets()).thenReturn(Map.of("f-secret", "fs"));
+
+ GravitinoClient client = mock(GravitinoClient.class);
+ when(client.loadCatalog("catalog")).thenReturn(catalog);
+
+ TestOps ops = new TestOps(conf, client);
+ Map<String, String> all =
+ ops.getAllProperties(NameIdentifier.of("ml", "catalog", "schema",
"fs"));
+
+ assertEquals("cs", all.get("c-secret"));
+ assertEquals("ss", all.get("s-secret"));
+ assertEquals("fs", all.get("f-secret"));
+ }
+
+ private static final class TestOps extends BaseGVFSOperations {
+ private final GravitinoClient client;
+
+ private TestOps(Configuration configuration, GravitinoClient client) {
+ super(configuration);
+ this.client = client;
+ }
+
+ @Override
+ GravitinoClient getGravitinoClient() {
+ return client;
+ }
+
+ @Override
+ public FSDataInputStream open(Path gvfsPath, int bufferSize) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setWorkingDirectory(Path gvfsDir) {}
+
+ @Override
+ public FSDataOutputStream create(
+ Path gvfsPath,
+ FsPermission permission,
+ boolean overwrite,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FSDataOutputStream append(Path gvfsPath, int bufferSize,
Progressable progress) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean rename(Path srcGvfsPath, Path dstGvfsPath) {
+ return false;
+ }
+
+ @Override
+ public boolean delete(Path gvfsPath, boolean recursive) {
+ return false;
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path gvfsPath) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path gvfsPath) {
+ return new FileStatus[0];
+ }
+
+ @Override
+ public boolean mkdirs(Path gvfsPath, FsPermission permission) {
+ return false;
+ }
+
+ @Override
+ public short getDefaultReplication(Path gvfsPath) {
+ return 1;
+ }
+
+ @Override
+ public long getDefaultBlockSize(Path gvfsPath) {
+ return 1L;
+ }
+
+ @Override
+ public Token<?>[] addDelegationTokens(String renewer, Credentials
credentials) {
+ return new Token<?>[0];
+ }
+
+ @Override
+ public void close() {}
+ }
+}
diff --git
a/flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java
b/flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java
index fb7c15d29f..ac45fa74e0 100644
---
a/flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java
+++
b/flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java
@@ -20,6 +20,8 @@
package org.apache.gravitino.flink.connector.store;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -97,11 +99,12 @@ public class GravitinoCatalogStore extends
AbstractCatalogStore {
public Optional<CatalogDescriptor> getCatalog(String catalogName) throws
CatalogException {
try {
Catalog catalog =
gravitinoCatalogManager.getGravitinoCatalogInfo(catalogName);
- BaseCatalogFactory catalogFactory =
getCatalogFactory(catalog.provider());
+ BaseCatalogFactory catalogFactory =
catalogFactoryForProvider(catalog.provider());
CatalogPropertiesConverter catalogPropertiesConverter =
catalogFactory.catalogPropertiesConverter();
+ Map<String, String> catalogProperties = propsWithSecrets(catalog);
Map<String, String> flinkCatalogProperties =
-
catalogPropertiesConverter.toFlinkCatalogProperties(catalog.properties());
+
catalogPropertiesConverter.toFlinkCatalogProperties(catalogProperties);
CatalogDescriptor descriptor =
newCatalogDescriptor(catalogName,
Configuration.fromMap(flinkCatalogProperties));
return Optional.of(descriptor);
@@ -150,7 +153,13 @@ public class GravitinoCatalogStore extends
AbstractCatalogStore {
catalogType));
}
- private BaseCatalogFactory getCatalogFactory(String provider) {
+ /**
+ * Resolve the Flink catalog factory for a Gravitino provider.
Package-visible for unit tests.
+ *
+ * @param provider Gravitino catalog provider name
+ * @return matching {@link BaseCatalogFactory}
+ */
+ BaseCatalogFactory catalogFactoryForProvider(String provider) {
return discoverFactories(
catalogFactory ->
((BaseCatalogFactory) catalogFactory)
@@ -210,4 +219,11 @@ public class GravitinoCatalogStore extends
AbstractCatalogStore {
}
return (BaseCatalogFactory) factories.get(0);
}
+
+ private static Map<String, String> propsWithSecrets(Catalog catalog) {
+ Map<String, String> props =
+ new HashMap<>(catalog.properties() == null ? Collections.emptyMap() :
catalog.properties());
+ props.putAll(catalog.supportsSecrets().getSecrets());
+ return props;
+ }
}
diff --git
a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/store/TestGravitinoCatalogStore.java
b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/store/TestGravitinoCatalogStore.java
index ac3a74a42e..fc83d072cd 100644
---
a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/store/TestGravitinoCatalogStore.java
+++
b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/store/TestGravitinoCatalogStore.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.flink.connector.store;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -25,13 +26,28 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
import java.util.ServiceConfigurationError;
import java.util.function.Predicate;
+import org.apache.flink.table.catalog.CatalogDescriptor;
import org.apache.flink.table.catalog.exceptions.CatalogException;
import org.apache.flink.table.factories.Factory;
+import org.apache.gravitino.Catalog;
+import org.apache.gravitino.Config;
+import org.apache.gravitino.flink.connector.CatalogPropertiesConverter;
import org.apache.gravitino.flink.connector.catalog.BaseCatalogFactory;
import org.apache.gravitino.flink.connector.catalog.GravitinoCatalogManager;
+import org.apache.gravitino.secret.SecretBinding;
+import org.apache.gravitino.secret.SecretManager;
+import org.apache.gravitino.secret.SecretMaterial;
+import org.apache.gravitino.secret.SecretPropertyUtils;
+import org.apache.gravitino.secret.SecretProviderRegistry;
+import org.apache.gravitino.secret.SupportsSecrets;
+import org.apache.gravitino.secret.memory.InMemorySecretsProvider;
import org.junit.Before;
import org.junit.Test;
@@ -131,4 +147,138 @@ public class TestGravitinoCatalogStore {
assertSame(expectedFactory, actualFactory);
}
+
+ @Test
+ public void testMergeSecrets() {
+ Catalog catalog = mock(Catalog.class);
+ SupportsSecrets supportsSecrets = mock(SupportsSecrets.class);
+
when(gravitinoCatalogMockManager.getGravitinoCatalogInfo("sec")).thenReturn(catalog);
+ when(catalog.provider()).thenReturn("test-provider");
+ when(catalog.properties()).thenReturn(Map.of("visible", "v1"));
+ when(catalog.supportsSecrets()).thenReturn(supportsSecrets);
+ when(supportsSecrets.getSecrets()).thenReturn(Map.of("jdbc-password",
"secret"));
+
+ BaseCatalogFactory factory = mock(BaseCatalogFactory.class);
+ CatalogPropertiesConverter converter =
mock(CatalogPropertiesConverter.class);
+ when(factory.catalogPropertiesConverter()).thenReturn(converter);
+
when(converter.toFlinkCatalogProperties(org.mockito.ArgumentMatchers.anyMap()))
+ .thenAnswer(
+ invocation -> {
+ Map<String, String> in = invocation.getArgument(0);
+ Map<String, String> out = new HashMap<>(in);
+ out.put("type", "generic_in_memory");
+ return out;
+ });
+
+ GravitinoCatalogStore store =
+ new GravitinoCatalogStore(gravitinoCatalogMockManager) {
+ @Override
+ BaseCatalogFactory catalogFactoryForProvider(String provider) {
+ return factory;
+ }
+ };
+
+ Optional<CatalogDescriptor> descriptor = store.getCatalog("sec");
+ assertTrue(descriptor.isPresent());
+ assertEquals("secret",
descriptor.get().getConfiguration().toMap().get("jdbc-password"));
+ assertEquals("v1",
descriptor.get().getConfiguration().toMap().get("visible"));
+ }
+
+ @Test
+ public void testMergeSecretsNullProps() {
+ Catalog catalog = mock(Catalog.class);
+ SupportsSecrets supportsSecrets = mock(SupportsSecrets.class);
+
when(gravitinoCatalogMockManager.getGravitinoCatalogInfo("sec-null")).thenReturn(catalog);
+ when(catalog.provider()).thenReturn("test-provider");
+ when(catalog.properties()).thenReturn(null);
+ when(catalog.supportsSecrets()).thenReturn(supportsSecrets);
+ when(supportsSecrets.getSecrets()).thenReturn(Map.of("jdbc-password",
"secret"));
+
+ BaseCatalogFactory factory = mock(BaseCatalogFactory.class);
+ CatalogPropertiesConverter converter =
mock(CatalogPropertiesConverter.class);
+ when(factory.catalogPropertiesConverter()).thenReturn(converter);
+
when(converter.toFlinkCatalogProperties(org.mockito.ArgumentMatchers.anyMap()))
+ .thenAnswer(
+ invocation -> {
+ Map<String, String> in = invocation.getArgument(0);
+ Map<String, String> out = new HashMap<>(in);
+ out.put("type", "generic_in_memory");
+ return out;
+ });
+
+ GravitinoCatalogStore store =
+ new GravitinoCatalogStore(gravitinoCatalogMockManager) {
+ @Override
+ BaseCatalogFactory catalogFactoryForProvider(String provider) {
+ return factory;
+ }
+ };
+
+ Optional<CatalogDescriptor> descriptor = store.getCatalog("sec-null");
+ assertTrue(descriptor.isPresent());
+ assertEquals("secret",
descriptor.get().getConfiguration().toMap().get("jdbc-password"));
+ }
+
+ @Test
+ public void testMergeMemorySecrets() {
+ try (SecretManager sm = memorySecretManager()) {
+ Map<String, String> entityProps = new HashMap<>();
+ entityProps.put("jdbc-user", "root");
+ java.util.List<SecretMaterial> writes =
+ sm.assembleSecretMaterials(
+ Map.of("jdbc-user", "root"),
+ entityProps,
+ "catalog",
+ 2L,
+ Map.of("jdbc-password", new SecretBinding("memory", "mem-pwd")),
+ Map.of());
+ sm.writeSecrets(writes);
+ Map<String, String> secrets = SecretPropertyUtils.buildSecrets(sm,
entityProps);
+
+ Catalog catalog = mock(Catalog.class);
+ SupportsSecrets supportsSecrets = mock(SupportsSecrets.class);
+
when(gravitinoCatalogMockManager.getGravitinoCatalogInfo("mem")).thenReturn(catalog);
+ when(catalog.provider()).thenReturn("test-provider");
+ when(catalog.properties()).thenReturn(Map.of("jdbc-url",
"jdbc:mysql://localhost/db"));
+ when(catalog.supportsSecrets()).thenReturn(supportsSecrets);
+ when(supportsSecrets.getSecrets()).thenReturn(secrets);
+
+ BaseCatalogFactory factory = mock(BaseCatalogFactory.class);
+ CatalogPropertiesConverter converter =
mock(CatalogPropertiesConverter.class);
+ when(factory.catalogPropertiesConverter()).thenReturn(converter);
+
when(converter.toFlinkCatalogProperties(org.mockito.ArgumentMatchers.anyMap()))
+ .thenAnswer(
+ invocation -> {
+ Map<String, String> in = invocation.getArgument(0);
+ Map<String, String> out = new HashMap<>(in);
+ out.put("type", "generic_in_memory");
+ return out;
+ });
+
+ GravitinoCatalogStore store =
+ new GravitinoCatalogStore(gravitinoCatalogMockManager) {
+ @Override
+ BaseCatalogFactory catalogFactoryForProvider(String provider) {
+ return factory;
+ }
+ };
+
+ Optional<CatalogDescriptor> descriptor = store.getCatalog("mem");
+ assertTrue(descriptor.isPresent());
+ assertEquals("mem-pwd",
descriptor.get().getConfiguration().toMap().get("jdbc-password"));
+ }
+ }
+
+ private static SecretManager memorySecretManager() {
+ Config config = new Config(false) {};
+ Properties properties = new Properties();
+ properties.setProperty(SecretProviderRegistry.GRAVITINO_SECRET_PROVIDERS,
"memory");
+ properties.setProperty(
+ SecretProviderRegistry.GRAVITINO_SECRET_PROVIDER_PREFIX
+ + "memory."
+ + SecretProviderRegistry.CLASS_NAME,
+ InMemorySecretsProvider.class.getName());
+ config.loadFromProperties(properties);
+ return new SecretManager(config);
+ }
}
diff --git
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/provider/DynamicIcebergConfigProvider.java
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/provider/DynamicIcebergConfigProvider.java
index 43ec66812e..b4f724626f 100644
---
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/provider/DynamicIcebergConfigProvider.java
+++
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/provider/DynamicIcebergConfigProvider.java
@@ -43,6 +43,7 @@ import org.apache.gravitino.credential.SupportsCredentials;
import org.apache.gravitino.exceptions.NoSuchCatalogException;
import org.apache.gravitino.iceberg.common.IcebergConfig;
import
org.apache.gravitino.iceberg.service.authorization.IcebergRESTServerContext;
+import org.apache.gravitino.secret.SupportsSecrets;
import org.apache.gravitino.server.web.JettyServerConfig;
import org.apache.gravitino.utils.MapUtils;
import org.apache.gravitino.utils.NameIdentifierUtil;
@@ -104,36 +105,44 @@ public class DynamicIcebergConfigProvider implements
IcebergConfigProvider {
"lakehouse-iceberg".equals(catalog.provider()),
String.format("%s.%s is not iceberg catalog", gravitinoMetalake,
catalogName));
- // Sensitive credentials (e.g. jdbc-password) are marked hidden in
PropertiesMetadata and
- // filtered out of catalog.properties(). We need two different strategies
to recover them:
- //
- // Auxiliary mode: the catalog is a BaseCatalog running in the same JVM as
the Gravitino
- // server. Call propertiesWithCredentialProviders() which returns the raw
entity properties
- // including all hidden fields.
- //
- // Standalone mode: the catalog is a client-side object obtained via the
Gravitino REST API.
- // Call getCredentials() to retrieve vended credentials, then inject any
JdbcCredential
- // fields into the properties map so the JDBC backend can connect.
- Map<String, String> catalogProperties;
+ // Auxiliary: BaseCatalog + SecretManager plaintext. Standalone:
properties + getSecrets,
+ // then JdbcCredential overlays so credentials win.
+ return
Optional.of(getIcebergConfigFromCatalogProperties(resolveProps(catalog)));
+ }
+
+ private static Map<String, String> resolveProps(Catalog catalog) {
if (catalog instanceof BaseCatalog) {
- catalogProperties = ((BaseCatalog<?>)
catalog).propertiesWithCredentialProviders();
- } else {
- catalogProperties = new HashMap<>(catalog.properties());
- if (catalog instanceof SupportsCredentials) {
- Arrays.stream(((SupportsCredentials) catalog).getCredentials())
- .filter(c -> c instanceof JdbcCredential)
- .map(c -> (JdbcCredential) c)
- .findFirst()
- .ifPresent(
- jdbc -> {
- catalogProperties.putIfAbsent(
- IcebergConstants.GRAVITINO_JDBC_USER, jdbc.jdbcUser());
- catalogProperties.putIfAbsent(
- IcebergConstants.GRAVITINO_JDBC_PASSWORD,
jdbc.jdbcPassword());
- });
+ return new HashMap<>(
+ GravitinoEnv.getInstance()
+ .secretManager()
+ .toPlaintextProperties(
+ ((BaseCatalog<?>)
catalog).propertiesWithCredentialProviders()));
+ }
+ Map<String, String> props =
+ new HashMap<>(catalog.properties() == null ? Map.of() :
catalog.properties());
+ try {
+ SupportsSecrets supportsSecrets = catalog.supportsSecrets();
+ if (supportsSecrets != null) {
+ Map<String, String> secrets = supportsSecrets.getSecrets();
+ if (secrets != null) {
+ props.putAll(secrets);
+ }
}
+ } catch (UnsupportedOperationException ignored) {
+ // Catalog does not support secret property operations.
+ }
+ if (catalog instanceof SupportsCredentials) {
+ Arrays.stream(((SupportsCredentials) catalog).getCredentials())
+ .filter(c -> c instanceof JdbcCredential)
+ .map(c -> (JdbcCredential) c)
+ .findFirst()
+ .ifPresent(
+ jdbc -> {
+ props.put(IcebergConstants.GRAVITINO_JDBC_USER,
jdbc.jdbcUser());
+ props.put(IcebergConstants.GRAVITINO_JDBC_PASSWORD,
jdbc.jdbcPassword());
+ });
}
- return
Optional.of(getIcebergConfigFromCatalogProperties(catalogProperties));
+ return props;
}
/**
@@ -317,7 +326,7 @@ public class DynamicIcebergConfigProvider implements
IcebergConfigProvider {
@Override
public Catalog loadCatalog(String catalogName) throws
NoSuchCatalogException {
- return
getGravitinoClient().loadMetalake(metalake).loadCatalog(catalogName);
+ return getGravitinoClient().loadCatalog(catalogName);
}
@Override
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/provider/TestDynamicIcebergConfigProvider.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/provider/TestDynamicIcebergConfigProvider.java
index d5b2024152..42807c1816 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/provider/TestDynamicIcebergConfigProvider.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/provider/TestDynamicIcebergConfigProvider.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -31,14 +32,26 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.gravitino.Catalog;
+import org.apache.gravitino.Config;
import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.catalog.CatalogDispatcher;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.connector.BaseCatalog;
+import org.apache.gravitino.credential.Credential;
+import org.apache.gravitino.credential.JdbcCredential;
+import org.apache.gravitino.credential.SupportsCredentials;
import org.apache.gravitino.exceptions.NoSuchCatalogException;
import org.apache.gravitino.iceberg.common.IcebergConfig;
import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
import
org.apache.gravitino.iceberg.service.authorization.IcebergRESTServerContext;
+import org.apache.gravitino.secret.SecretBinding;
+import org.apache.gravitino.secret.SecretManager;
+import org.apache.gravitino.secret.SecretMaterial;
+import org.apache.gravitino.secret.SecretPropertyUtils;
+import org.apache.gravitino.secret.SecretProviderRegistry;
+import org.apache.gravitino.secret.SupportsSecrets;
+import org.apache.gravitino.secret.memory.InMemorySecretsProvider;
import org.apache.gravitino.utils.NameIdentifierUtil;
import org.apache.iceberg.hive.HiveCatalog;
import org.apache.iceberg.jdbc.JdbcCatalog;
@@ -63,6 +76,7 @@ public class TestDynamicIcebergConfigProvider {
public void tearDown() throws IllegalAccessException {
// Clean up GravitinoEnv and IcebergRESTServerContext state after each test
FieldUtils.writeField(GravitinoEnv.getInstance(),
"internalCatalogDispatcher", null, true);
+ FieldUtils.writeField(GravitinoEnv.getInstance(), "secretManager", null,
true);
resetServerContext();
}
@@ -94,12 +108,15 @@ public class TestDynamicIcebergConfigProvider {
private void setMockCatalogFetcher(
DynamicIcebergConfigProvider provider, Map<String, Catalog> catalogMap) {
DynamicIcebergConfigProvider.CatalogFetcher mockFetcher =
- catalogName -> {
- Catalog catalog = catalogMap.get(catalogName);
- if (catalog == null) {
- throw new NoSuchCatalogException("Catalog not found: %s",
catalogName);
+ new DynamicIcebergConfigProvider.CatalogFetcher() {
+ @Override
+ public Catalog loadCatalog(String catalogName) throws
NoSuchCatalogException {
+ Catalog catalog = catalogMap.get(catalogName);
+ if (catalog == null) {
+ throw new NoSuchCatalogException("Catalog not found: %s",
catalogName);
+ }
+ return catalog;
}
- return catalog;
};
provider.setCatalogFetcher(mockFetcher);
}
@@ -289,15 +306,15 @@ public class TestDynamicIcebergConfigProvider {
NameIdentifier catalogIdent = NameIdentifierUtil.ofCatalog(metalakeName,
catalogName);
Mockito.when(mockInternalCatalogDispatcher.loadCatalog(catalogIdent)).thenReturn(mockCatalog);
+ Map<String, String> catalogProperties =
+ new HashMap<String, String>() {
+ {
+ put(IcebergConstants.CATALOG_BACKEND, "custom");
+ put(IcebergConstants.CATALOG_BACKEND_NAME, catalogName);
+ }
+ };
Mockito.when(mockCatalog.provider()).thenReturn("lakehouse-iceberg");
- Mockito.when(mockCatalog.properties())
- .thenReturn(
- new HashMap<String, String>() {
- {
- put(IcebergConstants.CATALOG_BACKEND, "custom");
- put(IcebergConstants.CATALOG_BACKEND_NAME, catalogName);
- }
- });
+ Mockito.when(mockCatalog.properties()).thenReturn(catalogProperties);
// Set the mock CatalogDispatchers to GravitinoEnv
FieldUtils.writeField(
@@ -516,15 +533,15 @@ public class TestDynamicIcebergConfigProvider {
NameIdentifier catalogIdent = NameIdentifierUtil.ofCatalog(metalakeName,
catalogName);
Mockito.when(mockInternalCatalogDispatcher.loadCatalog(catalogIdent)).thenReturn(mockCatalog);
+ Map<String, String> catalogProperties =
+ new HashMap<String, String>() {
+ {
+ put(IcebergConstants.CATALOG_BACKEND, "custom");
+ put(IcebergConstants.CATALOG_BACKEND_NAME, catalogName);
+ }
+ };
Mockito.when(mockCatalog.provider()).thenReturn("lakehouse-iceberg");
- Mockito.when(mockCatalog.properties())
- .thenReturn(
- new HashMap<String, String>() {
- {
- put(IcebergConstants.CATALOG_BACKEND, "custom");
- put(IcebergConstants.CATALOG_BACKEND_NAME, catalogName);
- }
- });
+ Mockito.when(mockCatalog.properties()).thenReturn(catalogProperties);
// Set the mock CatalogDispatchers to GravitinoEnv
FieldUtils.writeField(
@@ -584,4 +601,199 @@ public class TestDynamicIcebergConfigProvider {
executor.shutdown();
executor.awaitTermination(5, TimeUnit.SECONDS);
}
+
+ @Test
+ public void testMergeSecrets() {
+ String metalakeName = "test_metalake";
+ String catalogName = "jdbc_catalog";
+
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+ SupportsSecrets supportsSecrets = Mockito.mock(SupportsSecrets.class);
+ Mockito.when(mockCatalog.provider()).thenReturn("lakehouse-iceberg");
+ Mockito.when(mockCatalog.properties())
+ .thenReturn(
+ new HashMap<String, String>() {
+ {
+ put(IcebergConstants.CATALOG_BACKEND, "jdbc");
+ put(IcebergConstants.CATALOG_BACKEND_NAME, catalogName);
+ put(IcebergConstants.WAREHOUSE, "s3://bucket/wh");
+ }
+ });
+ Mockito.when(mockCatalog.supportsSecrets()).thenReturn(supportsSecrets);
+ Mockito.when(supportsSecrets.getSecrets())
+ .thenReturn(Map.of(IcebergConstants.GRAVITINO_JDBC_PASSWORD,
"secret-pwd"));
+
+ Map<String, String> properties = new HashMap<>();
+ properties.put(IcebergConstants.GRAVITINO_URI, "http://localhost:8090";);
+ properties.put(IcebergConstants.GRAVITINO_METALAKE, metalakeName);
+
+ DynamicIcebergConfigProvider provider = new DynamicIcebergConfigProvider();
+ provider.initialize(properties);
+ setMockCatalogFetcher(provider, Map.of(catalogName, mockCatalog));
+
+ Optional<IcebergConfig> config =
provider.getIcebergCatalogConfig(catalogName);
+ Assertions.assertTrue(config.isPresent());
+ Assertions.assertEquals(
+ "secret-pwd",
+
config.get().getIcebergCatalogProperties().get(IcebergConstants.GRAVITINO_JDBC_PASSWORD));
+ }
+
+ @Test
+ public void testCredsOverrideSecrets() {
+ String metalakeName = "test_metalake";
+ String catalogName = "jdbc_catalog";
+
+ Catalog mockCatalog =
+ Mockito.mock(
+ Catalog.class,
+ Mockito.withSettings()
+ .extraInterfaces(SupportsCredentials.class,
SupportsSecrets.class));
+ SupportsSecrets supportsSecrets = (SupportsSecrets) mockCatalog;
+ SupportsCredentials supportsCredentials = (SupportsCredentials)
mockCatalog;
+
+ Mockito.when(mockCatalog.provider()).thenReturn("lakehouse-iceberg");
+ Mockito.when(mockCatalog.properties())
+ .thenReturn(
+ new HashMap<String, String>() {
+ {
+ put(IcebergConstants.CATALOG_BACKEND, "jdbc");
+ put(IcebergConstants.CATALOG_BACKEND_NAME, catalogName);
+ }
+ });
+ Mockito.when(mockCatalog.supportsSecrets()).thenReturn(supportsSecrets);
+ Mockito.when(supportsSecrets.getSecrets())
+ .thenReturn(
+ Map.of(
+ IcebergConstants.GRAVITINO_JDBC_USER,
+ "from-secret",
+ IcebergConstants.GRAVITINO_JDBC_PASSWORD,
+ "secret-pwd"));
+ Mockito.when(supportsCredentials.getCredentials())
+ .thenReturn(new Credential[] {new JdbcCredential("cred-user",
"cred-pwd")});
+
+ Map<String, String> properties = new HashMap<>();
+ properties.put(IcebergConstants.GRAVITINO_URI, "http://localhost:8090";);
+ properties.put(IcebergConstants.GRAVITINO_METALAKE, metalakeName);
+
+ DynamicIcebergConfigProvider provider = new DynamicIcebergConfigProvider();
+ provider.initialize(properties);
+ setMockCatalogFetcher(provider, Map.of(catalogName, mockCatalog));
+
+ Optional<IcebergConfig> config =
provider.getIcebergCatalogConfig(catalogName);
+ Assertions.assertTrue(config.isPresent());
+ Map<String, String> icebergProps =
config.get().getIcebergCatalogProperties();
+ Assertions.assertEquals("cred-user",
icebergProps.get(IcebergConstants.GRAVITINO_JDBC_USER));
+ Assertions.assertEquals("cred-pwd",
icebergProps.get(IcebergConstants.GRAVITINO_JDBC_PASSWORD));
+ }
+
+ @Test
+ public void testMergeMemorySecrets() {
+ try (SecretManager sm = memorySecretManager()) {
+ Map<String, String> entityProps = new HashMap<>();
+ entityProps.put(IcebergConstants.GRAVITINO_JDBC_USER, "root");
+ List<SecretMaterial> writes =
+ sm.assembleSecretMaterials(
+ Map.of(IcebergConstants.GRAVITINO_JDBC_USER, "root"),
+ entityProps,
+ "catalog",
+ 3L,
+ Map.of(
+ IcebergConstants.GRAVITINO_JDBC_PASSWORD,
+ new SecretBinding("memory", "mem-jdbc-pwd")),
+ Map.of());
+ sm.writeSecrets(writes);
+ Map<String, String> secrets = SecretPropertyUtils.buildSecrets(sm,
entityProps);
+
+ String metalakeName = "test_metalake";
+ String catalogName = "jdbc_catalog";
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+ SupportsSecrets supportsSecrets = Mockito.mock(SupportsSecrets.class);
+ Mockito.when(mockCatalog.provider()).thenReturn("lakehouse-iceberg");
+ Mockito.when(mockCatalog.properties())
+ .thenReturn(
+ new HashMap<String, String>() {
+ {
+ put(IcebergConstants.CATALOG_BACKEND, "jdbc");
+ put(IcebergConstants.CATALOG_BACKEND_NAME, catalogName);
+ }
+ });
+ Mockito.when(mockCatalog.supportsSecrets()).thenReturn(supportsSecrets);
+ Mockito.when(supportsSecrets.getSecrets()).thenReturn(secrets);
+
+ Map<String, String> properties = new HashMap<>();
+ properties.put(IcebergConstants.GRAVITINO_URI, "http://localhost:8090";);
+ properties.put(IcebergConstants.GRAVITINO_METALAKE, metalakeName);
+
+ DynamicIcebergConfigProvider provider = new
DynamicIcebergConfigProvider();
+ provider.initialize(properties);
+ setMockCatalogFetcher(provider, Map.of(catalogName, mockCatalog));
+
+ Optional<IcebergConfig> config =
provider.getIcebergCatalogConfig(catalogName);
+ Assertions.assertTrue(config.isPresent());
+ Assertions.assertEquals(
+ "mem-jdbc-pwd",
+
config.get().getIcebergCatalogProperties().get(IcebergConstants.GRAVITINO_JDBC_PASSWORD));
+ }
+ }
+
+ @Test
+ public void testAuxPlaintext() throws Exception {
+ try (SecretManager sm = memorySecretManager()) {
+ String metalakeName = "test_metalake";
+ String catalogName = "jdbc_catalog";
+
+ Map<String, String> entityProps = new HashMap<>();
+ entityProps.put(IcebergConstants.CATALOG_BACKEND, "jdbc");
+ entityProps.put(IcebergConstants.CATALOG_BACKEND_NAME, catalogName);
+ entityProps.put(IcebergConstants.GRAVITINO_JDBC_USER, "root");
+ List<SecretMaterial> writes =
+ sm.assembleSecretMaterials(
+ Map.of(IcebergConstants.GRAVITINO_JDBC_USER, "root"),
+ entityProps,
+ "catalog",
+ 9L,
+ Map.of(
+ IcebergConstants.GRAVITINO_JDBC_PASSWORD,
+ new SecretBinding("memory", "aux-mem-pwd")),
+ Map.of());
+ sm.writeSecrets(writes);
+
+ @SuppressWarnings("unchecked")
+ BaseCatalog<?> baseCatalog = Mockito.mock(BaseCatalog.class);
+ Mockito.when(baseCatalog.provider()).thenReturn("lakehouse-iceberg");
+
Mockito.when(baseCatalog.propertiesWithCredentialProviders()).thenReturn(entityProps);
+
+ FieldUtils.writeField(GravitinoEnv.getInstance(), "secretManager", sm,
true);
+
+ Map<String, String> properties = new HashMap<>();
+ properties.put(IcebergConstants.GRAVITINO_URI, "http://localhost:8090";);
+ properties.put(IcebergConstants.GRAVITINO_METALAKE, metalakeName);
+
+ DynamicIcebergConfigProvider provider = new
DynamicIcebergConfigProvider();
+ provider.initialize(properties);
+ setMockCatalogFetcher(provider, Map.of(catalogName, baseCatalog));
+
+ Optional<IcebergConfig> config =
provider.getIcebergCatalogConfig(catalogName);
+ Assertions.assertTrue(config.isPresent());
+ Assertions.assertEquals(
+ "aux-mem-pwd",
+
config.get().getIcebergCatalogProperties().get(IcebergConstants.GRAVITINO_JDBC_PASSWORD));
+ Assertions.assertEquals(
+ "root",
+
config.get().getIcebergCatalogProperties().get(IcebergConstants.GRAVITINO_JDBC_USER));
+ }
+ }
+
+ private static SecretManager memorySecretManager() {
+ Config config = new Config(false) {};
+ Properties properties = new Properties();
+ properties.setProperty(SecretProviderRegistry.GRAVITINO_SECRET_PROVIDERS,
"memory");
+ properties.setProperty(
+ SecretProviderRegistry.GRAVITINO_SECRET_PROVIDER_PREFIX
+ + "memory."
+ + SecretProviderRegistry.CLASS_NAME,
+ InMemorySecretsProvider.class.getName());
+ config.loadFromProperties(properties);
+ return new SecretManager(config);
+ }
}
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceNameSpaceOperations.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceNameSpaceOperations.java
index 025cac747e..73afbd411a 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceNameSpaceOperations.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceNameSpaceOperations.java
@@ -142,12 +142,13 @@ public class GravitinoLanceNameSpaceOperations implements
LanceNamespaceOperatio
switch (nsId.levels()) {
case 1:
-
Optional.ofNullable(catalog.properties()).ifPresent(properties::putAll);
+ Optional.ofNullable(namespaceWrapper.propsWithSecrets(catalog))
+ .ifPresent(properties::putAll);
break;
case 2:
String schemaName = nsId.levelAtListPos(1);
- Schema schema = namespaceWrapper.loadSchema(catalog, schemaName);
- Optional.ofNullable(schema.properties()).ifPresent(properties::putAll);
+ Optional.ofNullable(namespaceWrapper.schemaPropsWithSecrets(catalog,
schemaName))
+ .ifPresent(properties::putAll);
break;
default:
throw new IllegalArgumentException(
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceNamespaceWrapper.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceNamespaceWrapper.java
index 001f993f7c..d6880ce320 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceNamespaceWrapper.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceNamespaceWrapper.java
@@ -31,6 +31,7 @@ import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.CatalogChange;
+import org.apache.gravitino.Entity;
import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.Namespace;
@@ -64,6 +65,7 @@ import
org.apache.gravitino.rel.expressions.distributions.Distribution;
import org.apache.gravitino.rel.expressions.sorts.SortOrder;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.indexes.Index;
+import org.apache.gravitino.secret.SecretPropertyOperationDispatcher;
import org.apache.gravitino.utils.NameIdentifierUtil;
import org.lance.namespace.errors.NamespaceNotFoundException;
import org.slf4j.Logger;
@@ -129,6 +131,48 @@ public class GravitinoLanceNamespaceWrapper extends
NamespaceWrapper {
return catalogOperator.loadCatalog(catalogName);
}
+ Map<String, String> propsWithSecrets(Catalog catalog) {
+ Map<String, String> props = copyProps(catalog.properties());
+ props.putAll(catalogSecrets(catalog.name()));
+ return props;
+ }
+
+ Map<String, String> schemaPropsWithSecrets(Catalog catalog, String
schemaName) {
+ Schema schema = loadSchema(catalog, schemaName);
+ Map<String, String> props = copyProps(schema.properties());
+ props.putAll(schemaSecrets(catalog.name(), schemaName));
+ return props;
+ }
+
+ private Map<String, String> catalogSecrets(String catalogName) {
+ SecretPropertyOperationDispatcher dispatcher = secretDispatcher();
+ if (dispatcher != null) {
+ return dispatcher.getSecrets(
+ NameIdentifierUtil.ofCatalog(metalakeName, catalogName),
Entity.EntityType.CATALOG);
+ }
+ return loadCatalog(catalogName).supportsSecrets().getSecrets();
+ }
+
+ private Map<String, String> schemaSecrets(String catalogName, String
schemaName) {
+ SecretPropertyOperationDispatcher dispatcher = secretDispatcher();
+ if (dispatcher != null) {
+ return dispatcher.getSecrets(schemaIdent(catalogName, schemaName),
Entity.EntityType.SCHEMA);
+ }
+ return loadSchema(loadCatalog(catalogName),
schemaName).supportsSecrets().getSecrets();
+ }
+
+ private SecretPropertyOperationDispatcher secretDispatcher() {
+ try {
+ return GravitinoEnv.getInstance().secretPropertyOperationDispatcher();
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ private static Map<String, String> copyProps(Map<String, String> props) {
+ return new HashMap<>(props == null ? Map.of() : props);
+ }
+
Catalog createCatalog(
String catalogName,
Catalog.Type type,
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
index 6a1fcd6e11..08bd9bae3a 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
@@ -146,7 +146,8 @@ public class GravitinoLanceTableOperations implements
LanceTableOperations {
.map(Long::valueOf)
.orElse(null));
response.setStorageOptions(
- LancePropertiesUtils.resolveLanceStorageOptions(catalog.properties(),
table.properties()));
+ LancePropertiesUtils.resolveLanceStorageOptions(
+ namespaceWrapper.propsWithSecrets(catalog), table.properties()));
response.setManagedVersioning(false);
if (checkDeclared) {
response.setIsOnlyDeclared(
@@ -200,7 +201,8 @@ public class GravitinoLanceTableOperations implements
LanceTableOperations {
tableIdentifier, columns.toArray(new Column[0]), null,
createTableProperties);
Map<String, String> properties = t.properties();
Map<String, String> effectiveStorageOptions =
- LancePropertiesUtils.resolveLanceStorageOptions(catalog.properties(),
properties);
+ LancePropertiesUtils.resolveLanceStorageOptions(
+ namespaceWrapper.propsWithSecrets(catalog), properties);
CreateTableResponse response = new CreateTableResponse();
response.setStorageOptions(effectiveStorageOptions);
diff --git
a/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestGravitinoLanceNamespaceWrapper.java
b/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestGravitinoLanceNamespaceWrapper.java
index ee8fd98c93..0b1eaec04c 100644
---
a/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestGravitinoLanceNamespaceWrapper.java
+++
b/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestGravitinoLanceNamespaceWrapper.java
@@ -26,16 +26,20 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.CatalogChange;
+import org.apache.gravitino.Entity;
import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.Namespace;
import org.apache.gravitino.Schema;
+import org.apache.gravitino.SupportsSchemas;
import org.apache.gravitino.catalog.CatalogDispatcher;
import org.apache.gravitino.catalog.SchemaDispatcher;
import org.apache.gravitino.catalog.TableDispatcher;
import org.apache.gravitino.lance.common.config.LanceConfig;
import org.apache.gravitino.rel.Table;
import org.apache.gravitino.rel.TableCatalog;
+import org.apache.gravitino.secret.SecretPropertyOperationDispatcher;
+import org.apache.gravitino.secret.SupportsSecrets;
import org.apache.gravitino.utils.NameIdentifierUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@@ -49,6 +53,8 @@ public class TestGravitinoLanceNamespaceWrapper {
FieldUtils.writeField(GravitinoEnv.getInstance(), "catalogManager", null,
true);
FieldUtils.writeField(GravitinoEnv.getInstance(), "schemaDispatcher",
null, true);
FieldUtils.writeField(GravitinoEnv.getInstance(), "tableDispatcher", null,
true);
+ FieldUtils.writeField(
+ GravitinoEnv.getInstance(), "secretPropertyOperationDispatcher", null,
true);
}
@Test
@@ -310,6 +316,140 @@ public class TestGravitinoLanceNamespaceWrapper {
Assertions.assertSame(expectedCatalog,
wrapper.loadAndValidateLakehouseCatalog("test_catalog"));
}
+ @Test
+ public void testPropsWithSecrets() {
+ GravitinoLanceNamespaceWrapper wrapper = new
GravitinoLanceNamespaceWrapper();
+ Catalog catalog =
+ createCatalogProxy(
+ Catalog.Type.RELATIONAL,
+ "lakehouse-generic",
+ Map.of("visible", "v1"),
+ Map.of("jdbc-password", "secret"));
+ wrapper.setCatalogOperator(
+ new GravitinoLanceNamespaceWrapper.CatalogOperator() {
+ @Override
+ public Catalog[] listCatalogsInfo() {
+ return new Catalog[0];
+ }
+
+ @Override
+ public Catalog loadCatalog(String catalogName) {
+ return catalog;
+ }
+
+ @Override
+ public Catalog createCatalog(
+ String catalogName,
+ Catalog.Type type,
+ String provider,
+ String comment,
+ Map<String, String> properties) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Catalog alterCatalog(String catalogName, CatalogChange...
changes) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean dropCatalog(String catalogName, boolean force) {
+ throw new UnsupportedOperationException();
+ }
+ });
+
+ Map<String, String> merged = wrapper.propsWithSecrets(catalog);
+ Assertions.assertEquals("v1", merged.get("visible"));
+ Assertions.assertEquals("secret", merged.get("jdbc-password"));
+ }
+
+ @Test
+ public void testSchemaPropsWithSecrets() {
+ Schema schema = createSchemaProxy(Map.of("visible", "s1"),
Map.of("schema-pwd", "s-secret"));
+ Catalog catalog =
+ createCatalogProxy(
+ Catalog.Type.RELATIONAL, "lakehouse-generic", Map.of(), Map.of(),
schema);
+ GravitinoLanceNamespaceWrapper wrapper =
+ new GravitinoLanceNamespaceWrapper(
+ new
LanceConfig(ImmutableMap.of(LanceConfig.METALAKE_NAME.getKey(),
"test_metalake")),
+ false);
+ wrapper.setCatalogOperator(catalogOperatorReturning(catalog));
+
+ Map<String, String> merged = wrapper.schemaPropsWithSecrets(catalog,
"test_schema");
+ Assertions.assertEquals("s1", merged.get("visible"));
+ Assertions.assertEquals("s-secret", merged.get("schema-pwd"));
+ }
+
+ @Test
+ public void testPropsViaDispatcher() throws Exception {
+ Catalog catalog =
+ createCatalogProxy(
+ Catalog.Type.RELATIONAL, "lakehouse-generic", Map.of("visible",
"c1"), Map.of());
+ Schema schema = createSchemaProxy(Map.of("visible", "s1"), Map.of());
+
+ NameIdentifier catalogIdent =
NameIdentifierUtil.ofCatalog("test_metalake", "test_catalog");
+ NameIdentifier schemaIdent =
+ NameIdentifierUtil.ofSchema("test_metalake", "test_catalog",
"test_schema");
+ SecretPropertyOperationDispatcher dispatcher =
+ new SecretPropertyOperationDispatcher(null, null, null, null) {
+ @Override
+ public Map<String, String> getSecrets(
+ NameIdentifier identifier, Entity.EntityType entityType) {
+ if (catalogIdent.equals(identifier) && entityType ==
Entity.EntityType.CATALOG) {
+ return Map.of("jdbc-password", "from-dispatcher");
+ }
+ if (schemaIdent.equals(identifier) && entityType ==
Entity.EntityType.SCHEMA) {
+ return Map.of("schema-pwd", "from-dispatcher-schema");
+ }
+ return Map.of();
+ }
+ };
+ FieldUtils.writeField(
+ GravitinoEnv.getInstance(), "secretPropertyOperationDispatcher",
dispatcher, true);
+
+ SchemaDispatcher schemaDispatcher =
+ (SchemaDispatcher)
+ Proxy.newProxyInstance(
+ SchemaDispatcher.class.getClassLoader(),
+ new Class<?>[] {SchemaDispatcher.class},
+ (proxy, method, args) -> {
+ if ("loadSchema".equals(method.getName())) {
+ return schema;
+ }
+ Class<?> returnType = method.getReturnType();
+ if (returnType.equals(boolean.class)) {
+ return false;
+ }
+ if (returnType.equals(int.class)) {
+ return 0;
+ }
+ return null;
+ });
+ FieldUtils.writeField(
+ GravitinoEnv.getInstance(),
+ "catalogDispatcher",
+ Proxy.newProxyInstance(
+ CatalogDispatcher.class.getClassLoader(),
+ new Class<?>[] {CatalogDispatcher.class},
+ (proxy, method, args) -> null),
+ true);
+ FieldUtils.writeField(GravitinoEnv.getInstance(), "schemaDispatcher",
schemaDispatcher, true);
+
+ LanceConfig lanceConfig =
+ new LanceConfig(ImmutableMap.of(LanceConfig.METALAKE_NAME.getKey(),
"test_metalake"));
+ GravitinoLanceNamespaceWrapper wrapper = new
GravitinoLanceNamespaceWrapper(lanceConfig, true);
+ wrapper.asNamespaceOps();
+ wrapper.setCatalogOperator(catalogOperatorReturning(catalog));
+
+ Map<String, String> catalogMerged = wrapper.propsWithSecrets(catalog);
+ Assertions.assertEquals("c1", catalogMerged.get("visible"));
+ Assertions.assertEquals("from-dispatcher",
catalogMerged.get("jdbc-password"));
+
+ Map<String, String> schemaMerged = wrapper.schemaPropsWithSecrets(catalog,
"test_schema");
+ Assertions.assertEquals("s1", schemaMerged.get("visible"));
+ Assertions.assertEquals("from-dispatcher-schema",
schemaMerged.get("schema-pwd"));
+ }
+
@Test
public void testLoadSchemaUsesSchemaDispatcherInAuxMode() throws Exception {
Schema expectedSchema = createSchemaProxy();
@@ -419,11 +559,35 @@ public class TestGravitinoLanceNamespaceWrapper {
}
private Schema createSchemaProxy() {
+ return createSchemaProxy(null, Map.of());
+ }
+
+ private Schema createSchemaProxy(Map<String, String> properties, Map<String,
String> secrets) {
return (Schema)
Proxy.newProxyInstance(
Schema.class.getClassLoader(),
- new Class<?>[] {Schema.class},
- (proxy, method, args) -> null);
+ new Class<?>[] {Schema.class, SupportsSecrets.class},
+ (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "name":
+ return "test_schema";
+ case "properties":
+ return properties;
+ case "supportsSecrets":
+ return proxy;
+ case "getSecrets":
+ return secrets;
+ default:
+ Class<?> returnType = method.getReturnType();
+ if (returnType.equals(boolean.class)) {
+ return false;
+ }
+ if (returnType.equals(int.class)) {
+ return 0;
+ }
+ return null;
+ }
+ });
}
private Table createTableProxy() {
@@ -435,10 +599,27 @@ public class TestGravitinoLanceNamespaceWrapper {
}
private Catalog createCatalogProxy(Catalog.Type type, String provider) {
+ return createCatalogProxy(type, provider, null, Map.of());
+ }
+
+ private Catalog createCatalogProxy(
+ Catalog.Type type,
+ String provider,
+ Map<String, String> properties,
+ Map<String, String> secrets) {
+ return createCatalogProxy(type, provider, properties, secrets, null);
+ }
+
+ private Catalog createCatalogProxy(
+ Catalog.Type type,
+ String provider,
+ Map<String, String> properties,
+ Map<String, String> secrets,
+ Schema schema) {
return (Catalog)
Proxy.newProxyInstance(
Catalog.class.getClassLoader(),
- new Class<?>[] {Catalog.class},
+ new Class<?>[] {Catalog.class, SupportsSecrets.class,
SupportsSchemas.class},
(proxy, method, args) -> {
switch (method.getName()) {
case "type":
@@ -448,7 +629,17 @@ public class TestGravitinoLanceNamespaceWrapper {
case "name":
return "test_catalog";
case "properties":
- return null;
+ return properties;
+ case "supportsSecrets":
+ return proxy;
+ case "getSecrets":
+ return secrets;
+ case "asSchemas":
+ return proxy;
+ case "loadSchema":
+ return schema;
+ case "schemaExists":
+ return schema != null;
default:
Class<?> returnType = method.getReturnType();
if (returnType.equals(boolean.class)) {
@@ -464,4 +655,38 @@ public class TestGravitinoLanceNamespaceWrapper {
}
});
}
+
+ private GravitinoLanceNamespaceWrapper.CatalogOperator
catalogOperatorReturning(Catalog catalog) {
+ return new GravitinoLanceNamespaceWrapper.CatalogOperator() {
+ @Override
+ public Catalog[] listCatalogsInfo() {
+ return new Catalog[0];
+ }
+
+ @Override
+ public Catalog loadCatalog(String catalogName) {
+ return catalog;
+ }
+
+ @Override
+ public Catalog createCatalog(
+ String catalogName,
+ Catalog.Type type,
+ String provider,
+ String comment,
+ Map<String, String> properties) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Catalog alterCatalog(String catalogName, CatalogChange...
changes) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean dropCatalog(String catalogName, boolean force) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
}
diff --git
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestGravitinoLanceModeParsing.java
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestGravitinoLanceModeParsing.java
index 2c142e55b3..e18820bb9e 100644
---
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestGravitinoLanceModeParsing.java
+++
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestGravitinoLanceModeParsing.java
@@ -227,7 +227,10 @@ class TestGravitinoLanceModeParsing {
GravitinoLanceNamespaceWrapper namespaceWrapper =
Mockito.mock(GravitinoLanceNamespaceWrapper.class);
Catalog catalog = Mockito.mock(Catalog.class);
+ when(catalog.name()).thenReturn("catalog");
+ when(catalog.properties()).thenReturn(Map.of());
when(namespaceWrapper.loadAndValidateLakehouseCatalog("catalog")).thenReturn(catalog);
+ when(namespaceWrapper.propsWithSecrets(catalog)).thenReturn(Map.of());
when(namespaceWrapper.asTableCatalog(catalog)).thenReturn(tableCatalog);
return new GravitinoLanceTableOperations(namespaceWrapper);
}
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/BaseCatalog.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/BaseCatalog.java
index 7ebdf160a3..b59fea1c7a 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/BaseCatalog.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/BaseCatalog.java
@@ -22,6 +22,7 @@ package org.apache.gravitino.spark.connector.catalog;
import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
@@ -165,8 +166,8 @@ public abstract class BaseCatalog implements TableCatalog,
SupportsNamespaces, F
String provider = gravitinoCatalogClient.provider();
Preconditions.checkArgument(
StringUtils.isNotBlank(provider), name + " catalog provider is empty");
- this.sparkCatalog =
- createAndInitSparkCatalog(name, options,
gravitinoCatalogClient.properties());
+ Map<String, String> catalogProperties =
propsWithSecrets(gravitinoCatalogClient);
+ this.sparkCatalog = createAndInitSparkCatalog(name, options,
catalogProperties);
this.propertiesConverter = getPropertiesConverter();
this.sparkTransformConverter = getSparkTransformConverter();
this.sparkTypeConverter = getSparkTypeConverter();
@@ -709,6 +710,13 @@ public abstract class BaseCatalog implements TableCatalog,
SupportsNamespaces, F
}
}
+ private static Map<String, String> propsWithSecrets(Catalog catalog) {
+ Map<String, String> props =
+ new HashMap<>(catalog.properties() == null ? Collections.emptyMap() :
catalog.properties());
+ props.putAll(catalog.supportsSecrets().getSecrets());
+ return props;
+ }
+
@Override
public void invalidateTable(Identifier ident) {
sparkCatalog.invalidateTable(ident);
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/catalog/TestBaseCatalogSecrets.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/catalog/TestBaseCatalogSecrets.java
new file mode 100644
index 0000000000..593ba6c529
--- /dev/null
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/catalog/TestBaseCatalogSecrets.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.spark.connector.catalog;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.gravitino.Catalog;
+import org.apache.gravitino.Config;
+import org.apache.gravitino.client.GravitinoClient;
+import org.apache.gravitino.secret.SecretBinding;
+import org.apache.gravitino.secret.SecretManager;
+import org.apache.gravitino.secret.SecretMaterial;
+import org.apache.gravitino.secret.SecretPropertyUtils;
+import org.apache.gravitino.secret.SecretProviderRegistry;
+import org.apache.gravitino.secret.SupportsSecrets;
+import org.apache.gravitino.secret.memory.InMemorySecretsProvider;
+import org.apache.gravitino.spark.connector.PropertiesConverter;
+import org.apache.gravitino.spark.connector.SparkTransformConverter;
+import org.apache.gravitino.spark.connector.SparkTypeConverter;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.connector.catalog.Identifier;
+import org.apache.spark.sql.connector.catalog.Table;
+import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.apache.spark.sql.util.CaseInsensitiveStringMap;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestBaseCatalogSecrets {
+
+ private GravitinoClient gravitinoClient;
+ private CapturingCatalog catalog;
+
+ @BeforeAll
+ void initCatalogManager() {
+ gravitinoClient = mock(GravitinoClient.class);
+ GravitinoCatalogManager.create(new SparkConf(false), "user", identity ->
gravitinoClient);
+ }
+
+ @AfterAll
+ void cleanupCatalogManager() {
+ GravitinoCatalogManager.get().close();
+ }
+
+ @Test
+ void testMergeSecrets() {
+ setUpCatalog(Map.of("metastore.uris", "thrift://localhost:9083"),
Map.of("s3-sk", "secret-sk"));
+
+ catalog.initialize("hive", new CaseInsensitiveStringMap(Map.of()));
+
+ assertEquals("thrift://localhost:9083",
catalog.lastProperties.get("metastore.uris"));
+ assertEquals("secret-sk", catalog.lastProperties.get("s3-sk"));
+ }
+
+ @Test
+ void testMergeSecretsNullProps() {
+ setUpCatalog(null, Map.of("s3-sk", "secret-sk"));
+
+ catalog.initialize("hive", new CaseInsensitiveStringMap(Map.of()));
+
+ assertEquals("secret-sk", catalog.lastProperties.get("s3-sk"));
+ assertEquals(1, catalog.lastProperties.size());
+ }
+
+ @Test
+ void testMergeMemorySecrets() {
+ try (SecretManager sm = memorySecretManager()) {
+ Map<String, String> entityProps = new HashMap<>();
+ entityProps.put("jdbc-user", "root");
+ List<SecretMaterial> writes =
+ sm.assembleSecretMaterials(
+ Map.of("jdbc-user", "root"),
+ entityProps,
+ "catalog",
+ 1L,
+ Map.of("jdbc-password", new SecretBinding("memory",
"from-memory")),
+ Map.of());
+ sm.writeSecrets(writes);
+ Map<String, String> secrets = SecretPropertyUtils.buildSecrets(sm,
entityProps);
+
+ setUpCatalog(Map.of("jdbc-url", "jdbc:mysql://localhost/db"), secrets);
+ catalog.initialize("jdbc", new CaseInsensitiveStringMap(Map.of()));
+
+ assertEquals("jdbc:mysql://localhost/db",
catalog.lastProperties.get("jdbc-url"));
+ assertEquals("from-memory", catalog.lastProperties.get("jdbc-password"));
+ }
+ }
+
+ private void setUpCatalog(Map<String, String> properties, Map<String,
String> secrets) {
+ Catalog gravitinoCatalog = mock(Catalog.class);
+ SupportsSecrets supportsSecrets = mock(SupportsSecrets.class);
+ TableCatalog sparkCatalog = mock(TableCatalog.class);
+ when(gravitinoCatalog.type()).thenReturn(Catalog.Type.RELATIONAL);
+ when(gravitinoCatalog.provider()).thenReturn("hive");
+ when(gravitinoCatalog.properties()).thenReturn(properties);
+ when(gravitinoCatalog.supportsSecrets()).thenReturn(supportsSecrets);
+ when(supportsSecrets.getSecrets()).thenReturn(secrets);
+ when(gravitinoClient.loadCatalog(any())).thenReturn(gravitinoCatalog);
+ // Catalog info is cached; recreate the manager so each test loads the new
mock.
+ GravitinoCatalogManager.get().close();
+ GravitinoCatalogManager.create(new SparkConf(false), "user", identity ->
gravitinoClient);
+ catalog = new CapturingCatalog(sparkCatalog);
+ }
+
+ private static SecretManager memorySecretManager() {
+ Config config = new Config(false) {};
+ Properties properties = new Properties();
+ properties.setProperty(SecretProviderRegistry.GRAVITINO_SECRET_PROVIDERS,
"memory");
+ properties.setProperty(
+ SecretProviderRegistry.GRAVITINO_SECRET_PROVIDER_PREFIX
+ + "memory."
+ + SecretProviderRegistry.CLASS_NAME,
+ InMemorySecretsProvider.class.getName());
+ config.loadFromProperties(properties);
+ return new SecretManager(config);
+ }
+
+ private static class CapturingCatalog extends BaseCatalog {
+ private final TableCatalog backing;
+ private Map<String, String> lastProperties = Map.of();
+
+ private CapturingCatalog(TableCatalog backing) {
+ this.backing = backing;
+ }
+
+ @Override
+ protected TableCatalog createAndInitSparkCatalog(
+ String name, CaseInsensitiveStringMap options, Map<String, String>
properties) {
+ this.lastProperties = Map.copyOf(properties);
+ return backing;
+ }
+
+ @Override
+ protected Table createSparkTable(
+ Identifier identifier,
+ org.apache.gravitino.rel.Table gravitinoTable,
+ Table sparkTable,
+ TableCatalog sparkCatalog,
+ PropertiesConverter propertiesConverter,
+ SparkTransformConverter sparkTransformConverter,
+ SparkTypeConverter sparkTypeConverter) {
+ return sparkTable;
+ }
+
+ @Override
+ protected PropertiesConverter getPropertiesConverter() {
+ return mock(PropertiesConverter.class);
+ }
+
+ @Override
+ protected SparkTransformConverter getSparkTransformConverter() {
+ return mock(SparkTransformConverter.class);
+ }
+ }
+}
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorManager.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorManager.java
index 34872d2e2f..9d477676af 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorManager.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorManager.java
@@ -23,6 +23,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
import io.trino.spi.TrinoException;
import io.trino.spi.connector.ConnectorContext;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -263,7 +264,9 @@ public class CatalogConnectorManager {
(String catalogName) -> {
try {
Catalog catalog = metalake.loadCatalog(catalogName);
- GravitinoCatalog gravitinoCatalog = new
GravitinoCatalog(metalake.name(), catalog);
+ Map<String, String> properties = propsWithSecrets(catalog);
+ GravitinoCatalog gravitinoCatalog =
+ new GravitinoCatalog(metalake.name(), catalog, properties);
if
(catalogConnectors.containsKey(getTrinoCatalogName(gravitinoCatalog))) {
// Reload catalogs that have been updated in Gravitino
server.
reloadCatalog(gravitinoCatalog);
@@ -484,6 +487,14 @@ public class CatalogConnectorManager {
return false;
}
+ /** Visible catalog properties overlaid with {@code getSecrets()}. */
+ static Map<String, String> propsWithSecrets(Catalog catalog) {
+ Map<String, String> props =
+ new HashMap<>(catalog.properties() == null ? Map.of() :
catalog.properties());
+ props.putAll(catalog.supportsSecrets().getSecrets());
+ return props;
+ }
+
public interface TrinoCatalogNameHandler {
String getCatalogName(String metalake, String catalog);
}
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java
index e595d763f8..08b6831c8a 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java
@@ -62,10 +62,21 @@ public class GravitinoCatalog {
* @param catalog the catalog
*/
public GravitinoCatalog(String metalake, Catalog catalog) {
+ this(metalake, catalog, catalog.properties());
+ }
+
+ /**
+ * Constructs a new GravitinoCatalog with resolved plaintext properties.
+ *
+ * @param metalake the name of the metalake
+ * @param catalog the catalog
+ * @param properties resolved catalog properties (secret URNs replaced with
plaintext)
+ */
+ public GravitinoCatalog(String metalake, Catalog catalog, Map<String,
String> properties) {
this.metalake = metalake;
this.provider = catalog.provider();
this.name = catalog.name();
- this.properties = catalog.properties();
+ this.properties = properties;
Instant time =
catalog.auditInfo().lastModifiedTime() == null
? catalog.auditInfo().createTime()
diff --git
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java
index e578f7e23d..648de60c07 100644
---
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java
+++
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java
@@ -59,6 +59,7 @@ import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.Table;
import org.apache.gravitino.rel.TableCatalog;
import org.apache.gravitino.rel.TableChange;
+import org.apache.gravitino.secret.SupportsSecrets;
import org.apache.gravitino.trino.connector.catalog.CatalogConnectorManager;
import
org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter;
import
org.apache.gravitino.trino.connector.catalog.hive.HiveDataTypeTransformer;
@@ -208,19 +209,20 @@ public class GravitinoMockServer implements AutoCloseable
{
}
});
- when(metaLake.loadCatalog(anyString()))
- .thenAnswer(
- new Answer<Catalog>() {
- @Override
- public Catalog answer(InvocationOnMock invocation) throws
Throwable {
- String catalogName = invocation.getArgument(0);
- if
(!metalakes.get(metalakeName).catalogs.containsKey(catalogName)) {
- throw new NoSuchCatalogException("catalog does not be
found");
- }
-
- return metalakes.get(metalakeName).catalogs.get(catalogName);
- }
- });
+ Answer<Catalog> loadCatalogAnswer =
+ new Answer<Catalog>() {
+ @Override
+ public Catalog answer(InvocationOnMock invocation) throws Throwable {
+ String catalogName = invocation.getArgument(0);
+ if
(!metalakes.get(metalakeName).catalogs.containsKey(catalogName)) {
+ throw new NoSuchCatalogException("catalog does not be found");
+ }
+
+ return metalakes.get(metalakeName).catalogs.get(catalogName);
+ }
+ };
+ when(metaLake.loadCatalog(anyString())).thenAnswer(loadCatalogAnswer);
+ // Connectors load catalogs with secrets resolved for backend
configuration.
when(metaLake.listCatalogsInfo())
.thenAnswer(
new Answer<Catalog[]>() {
@@ -240,15 +242,21 @@ public class GravitinoMockServer implements AutoCloseable
{
when(catalog.name()).thenReturn(catalogName);
when(catalog.provider()).thenReturn(testCatalogProvider);
when(catalog.type()).thenReturn(Catalog.Type.RELATIONAL);
- when(catalog.properties())
- .thenReturn(properties.isEmpty() ? Map.of("max_ttl", "10") :
properties);
+ Map<String, String> baseProperties =
+ properties.isEmpty() ? Map.of("max_ttl", "10") : properties;
+ when(catalog.properties()).thenReturn(baseProperties);
+ SupportsSecrets supportsSecrets = mock(SupportsSecrets.class);
+ when(supportsSecrets.getSecrets()).thenReturn(Map.of());
+ when(catalog.supportsSecrets()).thenReturn(supportsSecrets);
Audit mockAudit = mock(Audit.class);
when(mockAudit.creator()).thenReturn("gravitino");
when(mockAudit.createTime()).thenReturn(Instant.now());
when(catalog.auditInfo()).thenReturn(mockAudit);
- GravitinoCatalog gravitinoCatalog = new GravitinoCatalog(testMetalake,
catalog);
+ Map<String, String> resolved = new HashMap<>(baseProperties);
+ resolved.putAll(supportsSecrets.getSecrets());
+ GravitinoCatalog gravitinoCatalog = new GravitinoCatalog(testMetalake,
catalog, resolved);
when(catalog.asTableCatalog()).thenAnswer(answer ->
createTableCatalog(gravitinoCatalog));
when(catalog.asSchemas()).thenAnswer(answer ->
createSchemas(gravitinoCatalog));
diff --git
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/TestCatalogConnectorManager.java
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/TestCatalogConnectorManager.java
index f03cf0a474..781cf6755c 100644
---
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/TestCatalogConnectorManager.java
+++
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/TestCatalogConnectorManager.java
@@ -30,7 +30,10 @@ import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableMap;
import io.trino.spi.TrinoException;
import io.trino.spi.connector.ConnectorContext;
+import java.util.Map;
+import org.apache.gravitino.Catalog;
import org.apache.gravitino.client.GravitinoAdminClient;
+import org.apache.gravitino.secret.SupportsSecrets;
import org.apache.gravitino.trino.connector.GravitinoConfig;
import org.apache.gravitino.trino.connector.GravitinoErrorCode;
import org.apache.gravitino.trino.connector.metadata.GravitinoCatalog;
@@ -140,6 +143,36 @@ public class TestCatalogConnectorManager {
assertFalse(manager.skipCatalog("b2"));
}
+ @Test
+ public void testPropsWithSecrets() {
+ Catalog catalog = mock(Catalog.class);
+ SupportsSecrets supportsSecrets = mock(SupportsSecrets.class);
+ when(catalog.properties()).thenReturn(Map.of("visible", "v1", "shared",
"from-props"));
+ when(catalog.supportsSecrets()).thenReturn(supportsSecrets);
+ when(supportsSecrets.getSecrets())
+ .thenReturn(Map.of("jdbc-password", "secret", "shared",
"from-secret"));
+
+ Map<String, String> merged =
CatalogConnectorManager.propsWithSecrets(catalog);
+
+ assertEquals("v1", merged.get("visible"));
+ assertEquals("secret", merged.get("jdbc-password"));
+ assertEquals("from-secret", merged.get("shared"));
+ }
+
+ @Test
+ public void testPropsWithSecretsNullProps() {
+ Catalog catalog = mock(Catalog.class);
+ SupportsSecrets supportsSecrets = mock(SupportsSecrets.class);
+ when(catalog.properties()).thenReturn(null);
+ when(catalog.supportsSecrets()).thenReturn(supportsSecrets);
+ when(supportsSecrets.getSecrets()).thenReturn(Map.of("jdbc-password",
"secret"));
+
+ Map<String, String> merged =
CatalogConnectorManager.propsWithSecrets(catalog);
+
+ assertEquals("secret", merged.get("jdbc-password"));
+ assertEquals(1, merged.size());
+ }
+
private CatalogConnectorManager createManager(ImmutableMap<String, String>
configMap)
throws Exception {
return createManager(createCatalogConnectorFactory(), configMap);
diff --git
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/metadata/TestGravitinoCatalog.java
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/metadata/TestGravitinoCatalog.java
index d5a42e3f8b..3fa867c5c0 100644
---
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/metadata/TestGravitinoCatalog.java
+++
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/metadata/TestGravitinoCatalog.java
@@ -106,6 +106,23 @@ public class TestGravitinoCatalog {
assertTrue(catalog.isSameRegion("c2"));
}
+ @Test
+ public void testCatalogWithResolvedSecretsProperties() {
+ String catalogName = "mock";
+ String provider = "hive";
+ HashMap<String, String> properties = new HashMap<>();
+ properties.put("metastore.uris", "thrift://localhost:9083");
+ Catalog mockCatalog =
+ mockCatalog(catalogName, provider, "test catalog",
Catalog.Type.RELATIONAL, properties);
+
+ HashMap<String, String> resolved = new HashMap<>(properties);
+ resolved.put("jdbc-password", "from-secrets");
+ GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog,
resolved);
+
+ assertEquals("from-secrets", catalog.getProperty("jdbc-password", ""));
+ assertEquals("thrift://localhost:9083",
catalog.getProperty("metastore.uris", ""));
+ }
+
public static Catalog mockCatalog(
String name,
String provider,
