(hudi) branch master updated: [HUDI-7676] Fix serialization in Spark DAG in HoodieBackedTableMetadataWriter (#11103)

[jonvex]

This is an automated email from the ASF dual-hosted git repository.

jonvex pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 93f753d159d [HUDI-7676] Fix serialization in Spark DAG in 
HoodieBackedTableMetadataWriter (#11103)
93f753d159d is described below

commit 93f753d159dd7aef7ee87a8b8b25e5e2f3fc01c3
Author: Y Ethan Guo <ethan.guoyi...@gmail.com>
AuthorDate: Fri Apr 26 09:33:01 2024 -0700

    [HUDI-7676] Fix serialization in Spark DAG in 
HoodieBackedTableMetadataWriter (#11103)
---
 .../org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java    | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
 
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 3064f4073f8..acb81cd8029 100644
--- 
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ 
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -71,6 +71,7 @@ import org.apache.hudi.exception.HoodieMetadataException;
 import org.apache.hudi.exception.TableNotFoundException;
 import org.apache.hudi.index.HoodieIndexUtils;
 import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
 import org.apache.hudi.storage.StoragePath;
 import org.apache.hudi.storage.StoragePathInfo;
 import org.apache.hudi.table.BulkInsertPartitioner;
@@ -640,6 +641,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> 
implements HoodieTableM
 
     List<DirectoryInfo> partitionsToBootstrap = new LinkedList<>();
     final int fileListingParallelism = 
metadataWriteConfig.getFileListingParallelism();
+    SerializableConfiguration conf = new 
SerializableConfiguration(dataMetaClient.getHadoopConf());
     final String dirFilterRegex = 
dataWriteConfig.getMetadataConfig().getDirectoryFilterRegex();
     final String datasetBasePath = dataMetaClient.getBasePathV2().toString();
     StoragePath storageBasePath = new StoragePath(datasetBasePath);
@@ -650,8 +652,9 @@ public abstract class HoodieBackedTableMetadataWriter<I> 
implements HoodieTableM
       // List all directories in parallel
       engineContext.setJobStatus(this.getClass().getSimpleName(), "Listing " + 
numDirsToList + " partitions from filesystem");
       List<DirectoryInfo> processedDirectories = 
engineContext.map(pathsToList.subList(0, numDirsToList), path -> {
+        HoodieStorage storage = HoodieStorageUtils.getStorage(path, 
conf.get());
         String relativeDirPath = 
FSUtils.getRelativePartitionPath(storageBasePath, path);
-        return new DirectoryInfo(relativeDirPath, 
metadataMetaClient.getStorage().listDirectEntries(path), initializationTime);
+        return new DirectoryInfo(relativeDirPath, 
storage.listDirectEntries(path), initializationTime);
       }, numDirsToList);
 
       pathsToList = new LinkedList<>(pathsToList.subList(numDirsToList, 
pathsToList.size()));