(hudi) branch master updated: [HUDI-5128] Use HadoopFSUtils.getFs to get filesystem (#11937)

[yihua]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new a1e0123f4a1 [HUDI-5128] Use HadoopFSUtils.getFs to get filesystem 
(#11937)
a1e0123f4a1 is described below

commit a1e0123f4a1f624f891a7ee1964349a8f66d47b7
Author: Shawn Chang <42792772+c...@users.noreply.github.com>
AuthorDate: Thu Sep 12 15:58:12 2024 -0700

    [HUDI-5128] Use HadoopFSUtils.getFs to get filesystem (#11937)
    
    Co-authored-by: Shawn Chang <yxch...@amazon.com>
---
 .../hudi/utilities/sources/helpers/DatePartitionPathSelector.java    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java
 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java
index 70acd7ca527..16ee630f6fa 100644
--- 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java
+++ 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.ValidationUtils;
 import org.apache.hudi.common.util.collection.ImmutablePair;
 import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.hadoop.fs.HadoopFSUtils;
 import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
 import org.apache.hudi.utilities.config.DatePartitionPathSelectorConfig;
 
@@ -137,7 +138,7 @@ public class DatePartitionPathSelector extends 
DFSPathSelector {
 
     List<FileStatus> eligibleFiles = context.flatMap(prunedPartitionPaths,
         path -> {
-          FileSystem fs = new Path(path).getFileSystem(storageConf.unwrap());
+          FileSystem fs = HadoopFSUtils.getFs(path, storageConf);
           return listEligibleFiles(fs, new Path(path), 
lastCheckpointTime).stream();
         }, partitionsListParallelism);
     // sort them by modification time ascending.
@@ -187,7 +188,7 @@ public class DatePartitionPathSelector extends 
DFSPathSelector {
     for (int i = 0; i < datePartitionDepth; i++) {
       partitionPaths = context.flatMap(partitionPaths, path -> {
         Path subDir = new Path(path);
-        FileSystem fileSystem = subDir.getFileSystem(storageConf.unwrap());
+        FileSystem fileSystem = HadoopFSUtils.getFs(subDir, storageConf);
         // skip files/dirs whose names start with (_, ., etc)
         FileStatus[] statuses = fileSystem.listStatus(subDir,
             file -> IGNORE_FILEPREFIX_LIST.stream().noneMatch(pfx -> 
file.getName().startsWith(pfx)));