(hudi) branch master updated: [HUDI-9355] Avoid calling FileSystem#close in HoodieHadoopStorage (#13243)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 6bcee85fe29 [HUDI-9355] Avoid calling FileSystem#close in 
HoodieHadoopStorage (#13243)
6bcee85fe29 is described below

commit 6bcee85fe2965cab9ce32f2e9a51d7c53dc79d43
Author: Y Ethan Guo <ethan.guoyi...@gmail.com>
AuthorDate: Wed Apr 30 17:39:59 2025 -0700

    [HUDI-9355] Avoid calling FileSystem#close in HoodieHadoopStorage (#13243)
---
 .../hudi/storage/hadoop/HoodieHadoopStorage.java    |  4 +++-
 .../storage/hadoop/TestHoodieHadoopStorage.java     | 21 +++++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git 
a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
 
b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
index c125d8e2490..6d5549c7ef9 100644
--- 
a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
+++ 
b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
@@ -299,6 +299,8 @@ public class HoodieHadoopStorage extends HoodieStorage {
 
   @Override
   public void close() throws IOException {
-    fs.close();
+    // Don't close the wrapped `FileSystem` object.
+    // This will end up closing it for every thread since it
+    // could be cached across JVM. We don't own that object anyway.
   }
 }
diff --git 
a/hudi-hadoop-common/src/test/java/org/apache/hudi/storage/hadoop/TestHoodieHadoopStorage.java
 
b/hudi-hadoop-common/src/test/java/org/apache/hudi/storage/hadoop/TestHoodieHadoopStorage.java
index e34f858b859..8d0f64d0c6b 100644
--- 
a/hudi-hadoop-common/src/test/java/org/apache/hudi/storage/hadoop/TestHoodieHadoopStorage.java
+++ 
b/hudi-hadoop-common/src/test/java/org/apache/hudi/storage/hadoop/TestHoodieHadoopStorage.java
@@ -25,6 +25,11 @@ import org.apache.hudi.storage.HoodieStorage;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertSame;
 
 /**
  * Tests {@link HoodieHadoopStorage}.
@@ -49,4 +54,20 @@ public class TestHoodieHadoopStorage extends 
TestHoodieStorageBase {
     conf.set(CONF_KEY, CONF_VALUE);
     return conf;
   }
+
+  @Test
+  void testClose() throws IOException {
+    Configuration conf = new Configuration();
+    String path = getTempDir();
+    FileSystem fileSystem = HadoopFSUtils.getFs(path, conf, true);
+    HoodieStorage storage = new HoodieHadoopStorage(fileSystem);
+    storage.close();
+    // This validates that HoodieHadoopStorage#close does not close the 
underlying FileSystem
+    // object. If the underlying FileSystem object is closed, the cache of the 
object based on
+    // the path is closed and removed, which causes problems if it is reused 
elsewhere. Fetching
+    // the FileSystem object on the same path again in this case returns a 
different object,
+    // which can be caught here.
+    assertSame(fileSystem, storage.getFileSystem());
+    assertSame(fileSystem, HadoopFSUtils.getFs(getTempDir(), conf, true));
+  }
 }