This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 7c89a6bae8 [#7816] feat(filesystem-hadoop3): Support custom client
config (#7819)
7c89a6bae8 is described below
commit 7c89a6bae89263dad5b223f242caf74df2df3e98
Author: qbhan <[email protected]>
AuthorDate: Tue Aug 12 18:19:38 2025 +0800
[#7816] feat(filesystem-hadoop3): Support custom client config (#7819)
### What changes were proposed in this pull request?
Support custom client config when use `GVFS`
### Why are the changes needed?
Fix: #7816
### Does this PR introduce _any_ user-facing change?
- add config `fs.gravitino.client.` which is the configuration key
prefix for the Gravitino client config.
### How was this patch tested?
local tests
---
.../GravitinoVirtualFileSystemConfiguration.java | 14 ++++++
.../hadoop/GravitinoVirtualFileSystemUtils.java | 32 +++++++++++++
.../gravitino/filesystem/hadoop/TestGvfsBase.java | 56 ++++++++++++++++++++++
docs/how-to-use-gvfs.md | 7 +++
4 files changed, 109 insertions(+)
diff --git
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java
index ef84b39014..4ffb8d3731 100644
---
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java
+++
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java
@@ -18,6 +18,9 @@
*/
package org.apache.gravitino.filesystem.hadoop;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+
/** Configuration class for Gravitino Virtual File System. */
public class GravitinoVirtualFileSystemConfiguration {
@@ -50,6 +53,8 @@ public class GravitinoVirtualFileSystemConfiguration {
/** The authentication type for kerberos authentication. */
public static final String KERBEROS_AUTH_TYPE = "kerberos";
// oauth2
+ /** The configuration key prefix for oauth2 */
+ public static final String FS_GRAVITINO_CLIENT_OAUTH2_PREFIX =
"fs.gravitino.client.oauth2.";
/** The configuration key for the URI of the default OAuth server. */
public static final String FS_GRAVITINO_CLIENT_OAUTH2_SERVER_URI_KEY =
"fs.gravitino.client.oauth2.serverUri";
@@ -66,6 +71,8 @@ public class GravitinoVirtualFileSystemConfiguration {
public static final String FS_GRAVITINO_CLIENT_OAUTH2_SCOPE_KEY =
"fs.gravitino.client.oauth2.scope";
+ /** The configuration key prefix for kerberos */
+ public static final String FS_GRAVITINO_CLIENT_KERBEROS_PREFIX =
"fs.gravitino.client.kerberos.";
/** The configuration key for the principal. */
public static final String FS_GRAVITINO_CLIENT_KERBEROS_PRINCIPAL_KEY =
"fs.gravitino.client.kerberos.principal";
@@ -143,8 +150,15 @@ public class GravitinoVirtualFileSystemConfiguration {
public static final String FS_GRAVITINO_ENABLE_CREDENTIAL_VENDING =
"fs.gravitino.enableCredentialVending";
+ /** The configuration key prefix for the Gravitino client config. */
+ public static final String FS_GRAVITINO_CLIENT_CONFIG_PREFIX =
"fs.gravitino.client.";
+
/** The default value for whether to enable credential vending. */
public static final boolean FS_GRAVITINO_ENABLE_CREDENTIAL_VENDING_DEFAULT =
false;
+ /** The configuration key list which not a Gravitino client config */
+ public static final List<String> NOT_GRAVITINO_CLIENT_CONFIG_LIST =
+ ImmutableList.of(FS_GRAVITINO_CLIENT_METALAKE_KEY,
FS_GRAVITINO_CLIENT_AUTH_TYPE_KEY);
+
private GravitinoVirtualFileSystemConfiguration() {}
}
diff --git
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemUtils.java
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemUtils.java
index 926637072b..3a28fd6ac0 100644
---
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemUtils.java
+++
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemUtils.java
@@ -19,8 +19,14 @@
package org.apache.gravitino.filesystem.hadoop;
+import static
org.apache.gravitino.client.GravitinoClientConfiguration.GRAVITINO_CLIENT_CONFIG_PREFIX;
+import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_CONFIG_PREFIX;
+import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_KERBEROS_PREFIX;
+import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_OAUTH2_PREFIX;
import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_REQUEST_HEADER_PREFIX;
+import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.NOT_GRAVITINO_CLIENT_CONFIG_LIST;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import java.io.File;
@@ -94,6 +100,8 @@ public class GravitinoVirtualFileSystemUtils {
e ->
e.getKey().substring(FS_GRAVITINO_CLIENT_REQUEST_HEADER_PREFIX.length()),
Map.Entry::getValue));
+ Map<String, String> clientConfig = extractClientConfig(configuration);
+
String authType =
configuration.getOrDefault(
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_AUTH_TYPE_KEY,
@@ -103,6 +111,7 @@ public class GravitinoVirtualFileSystemUtils {
.withMetalake(metalakeValue)
.withSimpleAuth()
.withHeaders(requestHeaders)
+ .withClientConfig(clientConfig)
.build();
} else if (authType.equalsIgnoreCase(
GravitinoVirtualFileSystemConfiguration.OAUTH2_AUTH_TYPE)) {
@@ -150,6 +159,7 @@ public class GravitinoVirtualFileSystemUtils {
.withMetalake(metalakeValue)
.withOAuth(authDataProvider)
.withHeaders(requestHeaders)
+ .withClientConfig(clientConfig)
.build();
} else if (authType.equalsIgnoreCase(
GravitinoVirtualFileSystemConfiguration.KERBEROS_AUTH_TYPE)) {
@@ -181,6 +191,7 @@ public class GravitinoVirtualFileSystemUtils {
.withMetalake(metalakeValue)
.withKerberosAuth(authDataProvider)
.withHeaders(requestHeaders)
+ .withClientConfig(clientConfig)
.build();
} else {
throw new IllegalArgumentException(
@@ -190,6 +201,27 @@ public class GravitinoVirtualFileSystemUtils {
}
}
+ @VisibleForTesting
+ static Map<String, String> extractClientConfig(Map<String, String>
configuration) {
+ return configuration.entrySet().stream()
+ .filter(e -> isClientConfigKey(e.getKey()))
+ .collect(
+ Collectors.toMap(
+ e ->
+ e.getKey()
+ .replace(FS_GRAVITINO_CLIENT_CONFIG_PREFIX,
GRAVITINO_CLIENT_CONFIG_PREFIX),
+ Map.Entry::getValue,
+ (oldVal, newVal) -> newVal));
+ }
+
+ private static boolean isClientConfigKey(String key) {
+ return key.startsWith(FS_GRAVITINO_CLIENT_CONFIG_PREFIX)
+ && !key.startsWith(FS_GRAVITINO_CLIENT_OAUTH2_PREFIX)
+ && !key.startsWith(FS_GRAVITINO_CLIENT_KERBEROS_PREFIX)
+ && !key.startsWith(FS_GRAVITINO_CLIENT_REQUEST_HEADER_PREFIX)
+ && !NOT_GRAVITINO_CLIENT_CONFIG_LIST.contains(key);
+ }
+
/**
* Extract the full identifier of the fileset from the virtual path.
*
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
index d9084f90f3..7d266341bc 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
@@ -18,11 +18,15 @@
*/
package org.apache.gravitino.filesystem.hadoop;
+import static
org.apache.gravitino.client.GravitinoClientConfiguration.CLIENT_CONNECTION_TIMEOUT_MS;
+import static
org.apache.gravitino.client.GravitinoClientConfiguration.CLIENT_SOCKET_TIMEOUT_MS;
import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
import static org.apache.gravitino.file.Fileset.PROPERTY_DEFAULT_LOCATION_NAME;
import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_BLOCK_SIZE_DEFAULT;
+import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_CONFIG_PREFIX;
import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_REQUEST_HEADER_PREFIX;
import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemUtils.extractIdentifier;
+import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemUtils.getConfigMap;
import static org.apache.hc.core5.http.HttpStatus.SC_NOT_FOUND;
import static org.apache.hc.core5.http.HttpStatus.SC_OK;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -43,6 +47,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.net.SocketTimeoutException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@@ -67,6 +72,7 @@ import org.apache.gravitino.dto.responses.VersionResponse;
import org.apache.gravitino.exceptions.NoSuchCatalogException;
import org.apache.gravitino.exceptions.NoSuchFilesetException;
import org.apache.gravitino.exceptions.NoSuchLocationNameException;
+import org.apache.gravitino.exceptions.RESTException;
import org.apache.gravitino.file.Fileset;
import org.apache.gravitino.rest.RESTUtils;
import org.apache.hadoop.conf.Configuration;
@@ -76,6 +82,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hc.core5.http.Method;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -985,6 +992,55 @@ public class TestGvfsBase extends GravitinoMockServerBase {
}
}
+ @Test
+ public void testGravitinoClientConfig() {
+ Configuration configuration = new Configuration(conf);
+ // test valid client property
+ configuration.set(FS_GRAVITINO_CLIENT_CONFIG_PREFIX +
"connectionTimeoutMs", "8000");
+ configuration.set(FS_GRAVITINO_CLIENT_CONFIG_PREFIX + "socketTimeoutMs",
"4000");
+ Map<String, String> clientConfig =
+
GravitinoVirtualFileSystemUtils.extractClientConfig(getConfigMap(configuration));
+ Assertions.assertEquals(clientConfig.get(CLIENT_CONNECTION_TIMEOUT_MS),
"8000");
+ Assertions.assertEquals(clientConfig.get(CLIENT_SOCKET_TIMEOUT_MS),
"4000");
+
+ // test invalid client property
+ configuration.set(FS_GRAVITINO_CLIENT_CONFIG_PREFIX + "xxxx", "2000");
+ Throwable throwable =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> {
+ try (FileSystem fs = new
Path("gvfs://fileset/").getFileSystem(configuration)) {}
+ });
+ Assertions.assertEquals(
+ "Invalid property for client: gravitino.client.xxxx",
throwable.getMessage());
+ }
+
+ @Test
+ public void testSocketTimeout() throws IOException {
+
+ Configuration configuration = new Configuration(conf);
+ configuration.set(FS_GRAVITINO_CLIENT_CONFIG_PREFIX + "socketTimeoutMs",
"2000");
+
+ mockServer().clear(request().withPath("/api/version"));
+ HttpRequest req = request().withPath("/api/version");
+ mockServer()
+ .when(req, Times.once())
+ .respond(
+ response()
+ .withStatusCode(SC_OK)
+ .withBody(getJsonString(new
VersionResponse(Version.getCurrentVersionDTO())))
+ .withDelay(TimeUnit.MILLISECONDS, 5000));
+
+ Throwable throwable =
+ Assertions.assertThrows(
+ RESTException.class,
+ () -> {
+ try (FileSystem fs = new
Path("gvfs://fileset/").getFileSystem(configuration)) {}
+ });
+ Assertions.assertInstanceOf(SocketTimeoutException.class,
throwable.getCause());
+ Assertions.assertEquals("Read timed out",
throwable.getCause().getMessage());
+ }
+
private void buildMockResourceForCredential(String filesetName, String
filesetLocation)
throws JsonProcessingException {
String filesetPath =
diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md
index 4a197862ef..468930778c 100644
--- a/docs/how-to-use-gvfs.md
+++ b/docs/how-to-use-gvfs.md
@@ -71,6 +71,13 @@ the path mapping and convert automatically.
| `fs.gravitino.hook.class` | The hook class to
inject into the <br/>Gravitino Virtual File System. Users can implement their
own `GravitinoVirtualFileSystemHook` and configure the class name in this conf
to inject custom code.
|
`org.apache.gravitino.filesystem.hadoop.NoOpHook` | No
| 0.9.0-incubating |
| `fs.gravitino.client.request.header.` | The configuration
key prefix for the Gravitino client request header. You can set the request
header for the Gravitino client.
| (none)
| No |
0.9.0-incubating |
| `fs.gravitino.enableCredentialVending` | Whether to enable
credential vending for the Gravitino Virtual File System.
| `false`
| No |
0.9.0-incubating |
+| `fs.gravitino.client.` | The configuration
key prefix for the Gravitino client config.
| (none)
| No |
1.0.0 |
+
+To configure the Gravitino client, use properties prefixed with
`fs.gravitino.client.`. These properties undergo automatic transformation: the
prefix is replaced with `gravitino.client.` and passed to the `GravitinoClient`.
+
+**Example:** Setting `fs.gravitino.client.socketTimeoutMs` is equivalent to
setting `gravitino.client.socketTimeoutMs` for the `GravitinoClient`.
+
+**Note:** Invalid configuration properties will result in exceptions. Please
see [Gravitino Java client
configurations](./how-to-use-gravitino-client.md#gravitino-java-client-configuration)
for more support client configuration.
Apart from the above properties, to access fileset like S3, GCS, OSS and
custom fileset, extra properties are needed, please see
[S3 GVFS Java client
configurations](./fileset-catalog-with-s3.md#using-the-gvfs-java-client-to-access-the-fileset),
