This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 0e0a19e026 [#8948]Improve(gvfs-java) Lazy load gravitino client in
gvfs (#8950)
0e0a19e026 is described below
commit 0e0a19e0266cb0a87a8137bee012602206299756
Author: Junda Yang <[email protected]>
AuthorDate: Tue Oct 28 23:10:49 2025 -0700
[#8948]Improve(gvfs-java) Lazy load gravitino client in gvfs (#8950)
### What changes were proposed in this pull request?
- Lazy load GravitinoClient and FilesetMetadataCache in
BaseGVFSOperations using double-checked locking pattern
- Client and cache are now created on first filesystem operation instead
of during construction
- Fixed resource ownership: FilesetMetadataCache no longer closes the
shared GravitinoClient
### Why are the changes needed?
- Improves startup performance by deferring expensive client
initialization
- Reduces resource usage when filesystem is created but never used
- Better resource management with clear ownership (only
BaseGVFSOperations closes the client)
Fix: #8948
### Does this PR introduce _any_ user-facing change?
Yes, minor behavioral change:
Configuration validation and connection errors are now deferred until
the first filesystem operation (e.g., exists(), listStatus()) instead of
being thrown during filesystem initialization.
### How was this patch tested?
unit tests added and updated
---
.../filesystem/hadoop/BaseGVFSOperations.java | 82 +++++--
.../filesystem/hadoop/FilesetMetadataCache.java | 9 +-
.../TestClientAndCacheLazyLoadingBehavior.java | 253 +++++++++++++++++++++
.../hadoop/TestFilesetMetadataCache.java | 96 ++++++++
.../gravitino/filesystem/hadoop/TestGvfsBase.java | 15 +-
.../filesystem/hadoop/TestKerberosClient.java | 39 +++-
.../filesystem/hadoop/TestOauth2Client.java | 59 ++++-
.../filesystem/hadoop/TestSimpleClient.java | 8 +-
8 files changed, 524 insertions(+), 37 deletions(-)
diff --git
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/BaseGVFSOperations.java
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/BaseGVFSOperations.java
index 1fc0bc6b80..920ed90128 100644
---
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/BaseGVFSOperations.java
+++
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/BaseGVFSOperations.java
@@ -114,8 +114,14 @@ public abstract class BaseGVFSOperations implements
Closeable {
private final String metalakeName;
- private final Optional<FilesetMetadataCache> filesetMetadataCache;
- private final GravitinoClient gravitinoClient;
+ private final boolean enableFilesetMetadataCache;
+
+ // Lazy initialization of FilesetCatalogCache, see getFilesetMetadataCache()
for details.
+ private volatile Optional<FilesetMetadataCache> filesetMetadataCache;
+ private final Object filesetMetadataCacheLock = new Object();
+
+ // Lazy initialization of GravitinoClient, see getGravitinoClient() for
details.
+ private volatile GravitinoClient gravitinoClient;
private final Configuration conf;
@@ -148,15 +154,10 @@ public abstract class BaseGVFSOperations implements
Closeable {
"'%s' is not set in the configuration",
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_METALAKE_KEY);
- this.gravitinoClient =
GravitinoVirtualFileSystemUtils.createClient(configuration);
- boolean enableFilesetCatalogCache =
+ this.enableFilesetMetadataCache =
configuration.getBoolean(
FS_GRAVITINO_FILESET_METADATA_CACHE_ENABLE,
FS_GRAVITINO_FILESET_METADATA_CACHE_ENABLE_DEFAULT);
- this.filesetMetadataCache =
- enableFilesetCatalogCache
- ? Optional.of(new FilesetMetadataCache(gravitinoClient))
- : Optional.empty();
this.internalFileSystemCache = newFileSystemCache(configuration);
@@ -187,6 +188,26 @@ public abstract class BaseGVFSOperations implements
Closeable {
this.conf = configuration;
}
+ /**
+ * Lazy initialization of FilesetMetadataCache, see
getFilesetMetadataCache() for details.
+ *
+ * @return the FilesetMetadataCache.
+ */
+ @VisibleForTesting
+ protected Optional<FilesetMetadataCache> getFilesetMetadataCache() {
+ if (filesetMetadataCache == null) {
+ synchronized (filesetMetadataCacheLock) {
+ if (filesetMetadataCache == null) {
+ this.filesetMetadataCache =
+ enableFilesetMetadataCache
+ ? Optional.of(new FilesetMetadataCache(getGravitinoClient()))
+ : Optional.empty();
+ }
+ }
+ }
+ return filesetMetadataCache;
+ }
+
@Override
public void close() throws IOException {
// close all actual FileSystems
@@ -200,12 +221,21 @@ public abstract class BaseGVFSOperations implements
Closeable {
internalFileSystemCache.invalidateAll();
try {
- if (filesetMetadataCache.isPresent()) {
+ if (filesetMetadataCache != null && filesetMetadataCache.isPresent()) {
filesetMetadataCache.get().close();
}
} catch (IOException e) {
// ignore
}
+
+ // Close the GravitinoClient if it was initialized
+ try {
+ if (gravitinoClient != null) {
+ gravitinoClient.close();
+ }
+ } catch (Exception e) {
+ // ignore
+ }
}
/**
@@ -548,9 +578,9 @@ public abstract class BaseGVFSOperations implements
Closeable {
* @return the fileset catalog.
*/
protected FilesetCatalog getFilesetCatalog(NameIdentifier catalogIdent) {
- return filesetMetadataCache
+ return getFilesetMetadataCache()
.map(cache -> cache.getFilesetCatalog(catalogIdent))
- .orElseGet(() ->
gravitinoClient.loadCatalog(catalogIdent.name()).asFilesetCatalog());
+ .orElseGet(() ->
getGravitinoClient().loadCatalog(catalogIdent.name()).asFilesetCatalog());
}
/**
@@ -561,7 +591,7 @@ public abstract class BaseGVFSOperations implements
Closeable {
* @return the fileset.
*/
protected Fileset getFileset(NameIdentifier filesetIdent) {
- return filesetMetadataCache
+ return getFilesetMetadataCache()
.map(cache -> cache.getFileset(filesetIdent))
.orElseGet(
() ->
@@ -577,6 +607,24 @@ public abstract class BaseGVFSOperations implements
Closeable {
return internalFileSystemCache;
}
+ /**
+ * Lazy initialization of GravitinoClient using double-checked locking
pattern. This ensures the
+ * expensive client creation only happens when actually needed.
+ *
+ * @return the GravitinoClient
+ */
+ @VisibleForTesting
+ GravitinoClient getGravitinoClient() {
+ if (gravitinoClient == null) {
+ synchronized (this) {
+ if (gravitinoClient == null) {
+ this.gravitinoClient =
GravitinoVirtualFileSystemUtils.createClient(conf);
+ }
+ }
+ }
+ return gravitinoClient;
+ }
+
private void setCallerContextForGetFileLocation(FilesetDataOperation
operation) {
Map<String, String> contextMap = Maps.newHashMap();
contextMap.put(
@@ -594,7 +642,15 @@ public abstract class BaseGVFSOperations implements
Closeable {
CallerContext.CallerContextHolder.set(callerContext);
}
- private FileSystem getActualFileSystemByLocationName(
+ /**
+ * Get the actual file system corresponding to the given fileset identifier
and location name.
+ *
+ * @param filesetIdent the fileset identifier.
+ * @param locationName the location name. null means the default location.
+ * @return the actual file system.
+ * @throws FileNotFoundException if the target location name is not found in
the fileset.
+ */
+ protected FileSystem getActualFileSystemByLocationName(
NameIdentifier filesetIdent, String locationName) throws
FileNotFoundException {
NameIdentifier catalogIdent =
NameIdentifier.of(filesetIdent.namespace().level(0),
filesetIdent.namespace().level(1));
diff --git
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/FilesetMetadataCache.java
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/FilesetMetadataCache.java
index 2972e31142..239e1a144b 100644
---
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/FilesetMetadataCache.java
+++
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/FilesetMetadataCache.java
@@ -118,13 +118,6 @@ public class FilesetMetadataCache implements Closeable {
public void close() throws IOException {
catalogCache.invalidateAll();
filesetCache.invalidateAll();
- // close the client
- try {
- if (client != null) {
- client.close();
- }
- } catch (Exception e) {
- // ignore
- }
+ // Note: We don't close the client here since it's owned and managed by
BaseGVFSOperations
}
}
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestClientAndCacheLazyLoadingBehavior.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestClientAndCacheLazyLoadingBehavior.java
new file mode 100644
index 0000000000..66319c689f
--- /dev/null
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestClientAndCacheLazyLoadingBehavior.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.filesystem.hadoop;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Optional;
+import org.apache.gravitino.client.GravitinoClient;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for lazy loading functionality in {@link BaseGVFSOperations},
verifying that
+ * GravitinoClient and FilesetMetadataCache are NOT created during
construction and remain null
+ * until first access. These tests use reflection to verify the lazy
initialization pattern without
+ * requiring an actual Gravitino server.
+ */
+public class TestClientAndCacheLazyLoadingBehavior {
+
+ /** A minimal concrete implementation of BaseGVFSOperations for testing lazy
loading behavior. */
+ private static class TestableGVFSOperations extends BaseGVFSOperations {
+ protected TestableGVFSOperations(Configuration configuration) {
+ super(configuration);
+ }
+
+ // Minimal implementations - not used in lazy loading tests
+ @Override
+ public FSDataInputStream open(Path gvfsPath, int bufferSize) throws
IOException {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public void setWorkingDirectory(Path gvfsDir) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public FSDataOutputStream create(
+ Path gvfsPath,
+ FsPermission permission,
+ boolean overwrite,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress)
+ throws IOException {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public FSDataOutputStream append(Path gvfsPath, int bufferSize,
Progressable progress)
+ throws IOException {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public boolean rename(Path srcGvfsPath, Path dstGvfsPath) throws
IOException {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public boolean delete(Path gvfsPath, boolean recursive) throws IOException
{
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path gvfsPath) throws IOException {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path gvfsPath) throws IOException {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public boolean mkdirs(Path gvfsPath, FsPermission permission) throws
IOException {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public short getDefaultReplication(Path gvfsPath) {
+ return 1;
+ }
+
+ @Override
+ public long getDefaultBlockSize(Path gvfsPath) {
+ return 134217728L;
+ }
+
+ @Override
+ public Token<?>[] addDelegationTokens(String renewer, Credentials
credentials) {
+ return new Token<?>[0];
+ }
+ }
+
+ private Configuration createTestConfiguration() {
+ Configuration conf = new Configuration();
+ conf.set(
+
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_METALAKE_KEY,
"test_metalake");
+ conf.set(
+ GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_SERVER_URI_KEY,
+ "http://localhost:8090";);
+ return conf;
+ }
+
+ /**
+ * Test that GravitinoClient is NOT created during BaseGVFSOperations
construction.
+ *
+ * <p>This verifies the lazy initialization behavior - the expensive client
creation should be
+ * deferred until first use.
+ */
+ @Test
+ public void testClientNotCreatedDuringConstruction() throws Exception {
+ Configuration conf = createTestConfiguration();
+ TestableGVFSOperations operations = new TestableGVFSOperations(conf);
+
+ // Access the private gravitinoClient field via reflection
+ Field clientField =
BaseGVFSOperations.class.getDeclaredField("gravitinoClient");
+ clientField.setAccessible(true);
+ GravitinoClient client = (GravitinoClient) clientField.get(operations);
+
+ // Client should be null - not yet initialized
+ assertNull(client, "GravitinoClient should not be created during
construction");
+
+ operations.close();
+ }
+
+ /**
+ * Test that FilesetMetadataCache is NOT created during construction.
+ *
+ * <p>This verifies lazy initialization of the cache.
+ */
+ @Test
+ public void testCacheNotCreatedDuringConstruction() throws Exception {
+ Configuration conf = createTestConfiguration();
+ conf.setBoolean(
+
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_FILESET_METADATA_CACHE_ENABLE,
true);
+
+ TestableGVFSOperations operations = new TestableGVFSOperations(conf);
+
+ // Access the private filesetMetadataCache field
+ Field cacheField =
BaseGVFSOperations.class.getDeclaredField("filesetMetadataCache");
+ cacheField.setAccessible(true);
+ Optional<?> cache = (Optional<?>) cacheField.get(operations);
+
+ // Cache should be null - not yet initialized
+ assertNull(cache, "FilesetMetadataCache should not be created during
construction");
+
+ operations.close();
+ }
+
+ /**
+ * Test that FilesetMetadataCache returns empty Optional when disabled.
+ *
+ * <p>This verifies that when disabled, the cache initialization creates an
empty Optional rather
+ * than attempting to create a cache with a client.
+ */
+ @Test
+ public void testCacheNotCreatedWhenDisabled() throws Exception {
+ Configuration conf = createTestConfiguration();
+ conf.setBoolean(
+
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_FILESET_METADATA_CACHE_ENABLE,
false);
+
+ TestableGVFSOperations operations = new TestableGVFSOperations(conf);
+
+ // Access the cache - should return empty without trying to create a client
+ Optional<FilesetMetadataCache> cache =
operations.getFilesetMetadataCache();
+
+ assertFalse(cache.isPresent(), "FilesetMetadataCache should not be created
when disabled");
+
+ // Verify the field is now set (to Optional.empty())
+ Field cacheField =
BaseGVFSOperations.class.getDeclaredField("filesetMetadataCache");
+ cacheField.setAccessible(true);
+ Optional<?> fieldCache = (Optional<?>) cacheField.get(operations);
+ assertNotNull(fieldCache, "Cache field should be set to Optional after
first access");
+ assertFalse(fieldCache.isPresent(), "Cache Optional should be empty when
disabled");
+
+ operations.close();
+ }
+
+ /**
+ * Test that closing BaseGVFSOperations without ever accessing the client
completes safely.
+ *
+ * <p>This verifies that the lazy initialization doesn't break cleanup when
resources are never
+ * used.
+ */
+ @Test
+ public void testCloseWithoutClientAccess() throws Exception {
+ Configuration conf = createTestConfiguration();
+ TestableGVFSOperations operations = new TestableGVFSOperations(conf);
+
+ // Close without ever accessing the client
+ operations.close();
+
+ // Verify client is still null
+ Field clientField =
BaseGVFSOperations.class.getDeclaredField("gravitinoClient");
+ clientField.setAccessible(true);
+ GravitinoClient client = (GravitinoClient) clientField.get(operations);
+ assertNull(client, "Client should remain null if never accessed");
+ }
+
+ /**
+ * Test that closing BaseGVFSOperations without ever accessing the cache
completes safely.
+ *
+ * <p>This verifies lazy initialization doesn't break cleanup for the cache.
+ */
+ @Test
+ public void testCloseWithoutCacheAccess() throws Exception {
+ Configuration conf = createTestConfiguration();
+ conf.setBoolean(
+
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_FILESET_METADATA_CACHE_ENABLE,
true);
+
+ TestableGVFSOperations operations = new TestableGVFSOperations(conf);
+
+ // Close without ever accessing the cache
+ operations.close();
+
+ // Verify cache is still null
+ Field cacheField =
BaseGVFSOperations.class.getDeclaredField("filesetMetadataCache");
+ cacheField.setAccessible(true);
+ Optional<?> cache = (Optional<?>) cacheField.get(operations);
+ assertNull(cache, "Cache should remain null if never accessed");
+ }
+}
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestFilesetMetadataCache.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestFilesetMetadataCache.java
new file mode 100644
index 0000000000..3c04482fef
--- /dev/null
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestFilesetMetadataCache.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.filesystem.hadoop;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import org.apache.gravitino.client.GravitinoClient;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link FilesetMetadataCache} to verify proper resource management
and client ownership
+ * semantics.
+ */
+public class TestFilesetMetadataCache {
+
+ /**
+ * Test that FilesetMetadataCache does NOT close the GravitinoClient when
closed.
+ *
+ * <p>This verifies the resource ownership principle: FilesetMetadataCache
receives a shared
+ * reference to the client but doesn't own it, so it should not close it.
The owner
+ * (BaseGVFSOperations) is responsible for closing the client.
+ */
+ @Test
+ public void testCacheDoesNotCloseClient() throws IOException {
+ // Create a mock GravitinoClient
+ GravitinoClient mockClient = mock(GravitinoClient.class);
+
+ // Create the cache with the mock client
+ FilesetMetadataCache cache = new FilesetMetadataCache(mockClient);
+
+ // Close the cache
+ cache.close();
+
+ // Verify that the client's close() method was NEVER called
+ verify(mockClient, never()).close();
+ }
+
+ /**
+ * Test that FilesetMetadataCache can be closed multiple times safely.
+ *
+ * <p>Since the cache doesn't close the client, multiple close() calls
should be idempotent and
+ * not cause any issues.
+ */
+ @Test
+ public void testCacheCanBeClosedMultipleTimes() throws IOException {
+ GravitinoClient mockClient = mock(GravitinoClient.class);
+ FilesetMetadataCache cache = new FilesetMetadataCache(mockClient);
+
+ // Close multiple times - should not throw any exceptions
+ cache.close();
+ cache.close();
+ cache.close();
+
+ // Client should still never be closed
+ verify(mockClient, never()).close();
+ }
+
+ /**
+ * Test that FilesetMetadataCache properly invalidates its internal caches
when closed.
+ *
+ * <p>Even though the client isn't closed, the cache should properly clean
up its internal
+ * Caffeine caches.
+ */
+ @Test
+ public void testCacheInvalidatesInternalCaches() throws IOException {
+ GravitinoClient mockClient = mock(GravitinoClient.class);
+ FilesetMetadataCache cache = new FilesetMetadataCache(mockClient);
+
+ // Note: We can't easily verify cache invalidation without accessing
private fields,
+ // but we can at least verify that close() completes without errors
+ cache.close();
+
+ // If we got here without exceptions, the test passes
+ assertTrue(true, "Cache close completed successfully");
+ }
+}
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
index 0956ae2b8f..b3ae42dba4 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
@@ -290,6 +290,11 @@ public class TestGvfsBase extends GravitinoMockServerBase {
.withBody(getJsonString(new
VersionResponse(Version.getCurrentVersionDTO()))));
try (FileSystem fs = new
Path("gvfs://fileset/").getFileSystem(configuration)) {
+ // Trigger lazy initialization by accessing a path (throws RESTException
for mock server 404)
+ assertThrows(
+ RESTException.class,
+ () -> fs.exists(new
Path("gvfs://fileset/catalog/schema/fileset/file.txt")));
+ // Verify the request was made with correct headers during client
initialization
mockServer().verify(req, VerificationTimes.once());
}
}
@@ -1011,7 +1016,10 @@ public class TestGvfsBase extends
GravitinoMockServerBase {
Assertions.assertThrows(
IllegalArgumentException.class,
() -> {
- try (FileSystem fs = new
Path("gvfs://fileset/").getFileSystem(configuration)) {}
+ try (FileSystem fs = new
Path("gvfs://fileset/").getFileSystem(configuration)) {
+ // Trigger lazy initialization by accessing a path
+ fs.exists(new
Path("gvfs://fileset/catalog/schema/fileset/file.txt"));
+ }
});
Assertions.assertEquals(
"Invalid property for client: gravitino.client.xxxx",
throwable.getMessage());
@@ -1037,7 +1045,10 @@ public class TestGvfsBase extends
GravitinoMockServerBase {
Assertions.assertThrows(
RESTException.class,
() -> {
- try (FileSystem fs = new
Path("gvfs://fileset/").getFileSystem(configuration)) {}
+ try (FileSystem fs = new
Path("gvfs://fileset/").getFileSystem(configuration)) {
+ // Trigger lazy initialization by accessing a path
+ fs.exists(new
Path("gvfs://fileset/catalog/schema/fileset/file.txt"));
+ }
});
Assertions.assertInstanceOf(SocketTimeoutException.class,
throwable.getCause());
Assertions.assertEquals("Read timed out",
throwable.getCause().getMessage());
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestKerberosClient.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestKerberosClient.java
index 564b05cee7..b7e4ea3a38 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestKerberosClient.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestKerberosClient.java
@@ -31,6 +31,7 @@ import java.util.UUID;
import org.apache.gravitino.Config;
import org.apache.gravitino.server.authentication.KerberosAuthenticator;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.Method;
@@ -90,14 +91,24 @@ public class TestKerberosClient extends TestGvfsBase {
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_AUTH_TYPE_KEY,
GravitinoVirtualFileSystemConfiguration.KERBEROS_AUTH_TYPE);
assertThrows(
- IllegalArgumentException.class, () ->
managedFilesetPath.getFileSystem(configuration));
+ IllegalArgumentException.class,
+ () -> {
+ FileSystem fs = managedFilesetPath.getFileSystem(configuration);
+ // Trigger lazy initialization
+ fs.exists(managedFilesetPath);
+ });
// set not exist keytab path
configuration.set(
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_KERBEROS_KEYTAB_FILE_PATH_KEY,
"file://tmp/test.keytab");
assertThrows(
- IllegalArgumentException.class, () ->
managedFilesetPath.getFileSystem(configuration));
+ IllegalArgumentException.class,
+ () -> {
+ FileSystem fs = managedFilesetPath.getFileSystem(configuration);
+ // Trigger lazy initialization
+ fs.exists(managedFilesetPath);
+ });
}
@Test
@@ -184,7 +195,13 @@ public class TestKerberosClient extends TestGvfsBase {
return response().withStatusCode(HttpStatus.SC_OK);
});
Path newPath = new
Path(managedFilesetPath.toString().replace(metalakeName, testMetalake));
- Assertions.assertThrows(IllegalStateException.class, () ->
newPath.getFileSystem(conf1));
+ Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> {
+ FileSystem fs = newPath.getFileSystem(conf1);
+ // Trigger lazy initialization
+ fs.exists(newPath);
+ });
// test with principal and invalid keytab
File invalidKeytabFile =
@@ -201,7 +218,13 @@ public class TestKerberosClient extends TestGvfsBase {
conf2.set(
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_KERBEROS_KEYTAB_FILE_PATH_KEY,
invalidKeytabFile.getAbsolutePath());
- Assertions.assertThrows(IllegalStateException.class, () ->
newPath.getFileSystem(conf2));
+ Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> {
+ FileSystem fs = newPath.getFileSystem(conf2);
+ // Trigger lazy initialization
+ fs.exists(newPath);
+ });
invalidKeytabFile.delete();
// test with principal and no keytab
@@ -212,6 +235,12 @@ public class TestKerberosClient extends TestGvfsBase {
// remove keytab configuration
conf3.unset(
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_KERBEROS_KEYTAB_FILE_PATH_KEY);
- Assertions.assertThrows(IllegalStateException.class, () ->
newPath.getFileSystem(conf3));
+ Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> {
+ FileSystem fs = newPath.getFileSystem(conf3);
+ // Trigger lazy initialization
+ fs.exists(newPath);
+ });
}
}
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestOauth2Client.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestOauth2Client.java
index 2186f53067..e673e1e1b0 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestOauth2Client.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestOauth2Client.java
@@ -56,6 +56,7 @@ import org.apache.gravitino.rest.RESTUtils;
import org.apache.gravitino.server.authentication.OAuthConfig;
import org.apache.gravitino.server.authentication.ServerAuthenticator;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.Method;
@@ -180,27 +181,47 @@ public class TestOauth2Client extends TestGvfsBase {
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_AUTH_TYPE_KEY,
GravitinoVirtualFileSystemConfiguration.OAUTH2_AUTH_TYPE);
assertThrows(
- IllegalArgumentException.class, () ->
managedFilesetPath.getFileSystem(configuration));
+ IllegalArgumentException.class,
+ () -> {
+ FileSystem fs = managedFilesetPath.getFileSystem(configuration);
+ // Trigger lazy initialization
+ fs.exists(managedFilesetPath);
+ });
// set oauth server uri, but do not set other configs
configuration.set(
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_OAUTH2_SERVER_URI_KEY,
Oauth2MockServerBase.serverUri());
assertThrows(
- IllegalArgumentException.class, () ->
managedFilesetPath.getFileSystem(configuration));
+ IllegalArgumentException.class,
+ () -> {
+ FileSystem fs = managedFilesetPath.getFileSystem(configuration);
+ // Trigger lazy initialization
+ fs.exists(managedFilesetPath);
+ });
// set oauth server path, but do not set other configs
configuration.set(
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_OAUTH2_PATH_KEY,
normal_path);
assertThrows(
- IllegalArgumentException.class, () ->
managedFilesetPath.getFileSystem(configuration));
+ IllegalArgumentException.class,
+ () -> {
+ FileSystem fs = managedFilesetPath.getFileSystem(configuration);
+ // Trigger lazy initialization
+ fs.exists(managedFilesetPath);
+ });
// set oauth credential, but do not set other configs
configuration.set(
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_OAUTH2_CREDENTIAL_KEY,
credential);
assertThrows(
- IllegalArgumentException.class, () ->
managedFilesetPath.getFileSystem(configuration));
+ IllegalArgumentException.class,
+ () -> {
+ FileSystem fs = managedFilesetPath.getFileSystem(configuration);
+ // Trigger lazy initialization
+ fs.exists(managedFilesetPath);
+ });
// set oauth scope, all configs are set
configuration.set(
@@ -231,7 +252,12 @@ public class TestOauth2Client extends TestGvfsBase {
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_OAUTH2_PATH_KEY,
invalid_path);
// should throw UnauthorizedException
assertThrows(
- UnauthorizedException.class, () ->
managedFilesetPath.getFileSystem(configuration));
+ UnauthorizedException.class,
+ () -> {
+ FileSystem fs = managedFilesetPath.getFileSystem(configuration);
+ // Trigger lazy initialization
+ fs.exists(managedFilesetPath);
+ });
// 2. test wrong client secret
mockResponse =
response().withStatusCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);
@@ -255,7 +281,12 @@ public class TestOauth2Client extends TestGvfsBase {
Times.exactly(1))
.respond(mockResponse);
assertThrows(
- UnauthorizedException.class, () ->
managedFilesetPath.getFileSystem(configuration));
+ UnauthorizedException.class,
+ () -> {
+ FileSystem fs = managedFilesetPath.getFileSystem(configuration);
+ // Trigger lazy initialization
+ fs.exists(managedFilesetPath);
+ });
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
@@ -333,7 +364,13 @@ public class TestOauth2Client extends TestGvfsBase {
config1.set(
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_METALAKE_KEY,
testMetalake);
// UnauthorizedException will be caught by the client, and the
RESTException will be thrown
- assertThrows(RESTException.class, () -> newPath.getFileSystem(config1));
+ assertThrows(
+ RESTException.class,
+ () -> {
+ FileSystem fs = newPath.getFileSystem(config1);
+ // Trigger lazy initialization
+ fs.exists(newPath);
+ });
}
@Test
@@ -358,6 +395,12 @@ public class TestOauth2Client extends TestGvfsBase {
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_OAUTH2_PATH_KEY,
invalid_path + "/bad");
// should throw BadRequestException
- assertThrows(BadRequestException.class, () ->
managedFilesetPath.getFileSystem(configuration));
+ assertThrows(
+ BadRequestException.class,
+ () -> {
+ FileSystem fs = managedFilesetPath.getFileSystem(configuration);
+ // Trigger lazy initialization
+ fs.exists(managedFilesetPath);
+ });
}
}
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestSimpleClient.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestSimpleClient.java
index b88fbba16b..f80ef2bcb7 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestSimpleClient.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestSimpleClient.java
@@ -19,6 +19,7 @@
package org.apache.gravitino.filesystem.hadoop;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -31,8 +32,10 @@ import org.apache.gravitino.auth.AuthConstants;
import org.apache.gravitino.dto.AuditDTO;
import org.apache.gravitino.dto.MetalakeDTO;
import org.apache.gravitino.dto.responses.MetalakeResponse;
+import org.apache.gravitino.exceptions.RESTException;
import org.apache.gravitino.json.JsonUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.Method;
@@ -98,7 +101,10 @@ public class TestSimpleClient extends TestGvfsBase {
Configuration config1 = new Configuration(conf);
config1.set(
GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_CLIENT_METALAKE_KEY,
testMetalake);
- newPath.getFileSystem(config1);
+ FileSystem fs = newPath.getFileSystem(config1);
+ // Trigger lazy initialization to set auth token (throws RESTException for
non-existent
+ // metalake)
+ assertThrows(RESTException.class, () -> fs.exists(newPath));
String userInformation = user + ":dummy";
assertEquals(
