(dolphinscheduler) branch dev updated: [Fix-16331] Fix the resource file cannot be displayed hierarchically (#16799)

[zihaoxiang]

This is an automated email from the ASF dual-hosted git repository.

zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git


The following commit(s) were added to refs/heads/dev by this push:
     new 854098bb44 [Fix-16331] Fix the resource file cannot be displayed 
hierarchically (#16799)
854098bb44 is described below

commit 854098bb44bebf0cb5acfa50c249358f42fa3297
Author: xiangzihao <460888...@qq.com>
AuthorDate: Fri Nov 15 12:13:33 2024 +0800

    [Fix-16331] Fix the resource file cannot be displayed hierarchically 
(#16799)
---
 .../plugin/storage/hdfs/HdfsStorageOperator.java             | 12 ++++++------
 .../plugin/storage/hdfs/LocalStorageOperatorTest.java        | 12 ++++++++----
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git 
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
 
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
index d3a2cad7ee..0d21b769a9 100644
--- 
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
+++ 
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
@@ -32,9 +32,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.security.UserGroupInformation;
 
@@ -235,10 +233,12 @@ public class HdfsStorageOperator extends 
AbstractStorageOperator implements Clos
             if (!fs.exists(path)) {
                 continue;
             }
-            RemoteIterator<LocatedFileStatus> remoteIterator = 
fs.listFiles(path, true);
-            while (remoteIterator.hasNext()) {
-                LocatedFileStatus locatedFileStatus = remoteIterator.next();
-                
result.add(transformFileStatusToResourceMetadata(locatedFileStatus));
+            FileStatus[] fileStatuses = fs.listStatus(path);
+            for (FileStatus fileStatus : fileStatuses) {
+                if (fileStatus.isDirectory()) {
+                    foldersToFetch.addLast(fileStatus.getPath().toString());
+                }
+                result.add(transformFileStatusToResourceMetadata(fileStatus));
             }
         }
         return result;
diff --git 
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
 
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
index 0a9b943d5f..5730f46dbd 100644
--- 
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
+++ 
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
@@ -296,10 +296,14 @@ class LocalStorageOperatorTest {
         String resourceFileAbsolutePath = "file:" + baseDir;
         List<StorageEntity> storageEntities =
                 
storageOperator.listFileStorageEntityRecursively(resourceFileAbsolutePath);
-        assertThat(storageEntities.size()).isEqualTo(1);
-
-        StorageEntity storageEntity2 = storageEntities.get(0);
-        assertThat(storageEntity2.getFullName()).isEqualTo("file:" + 
Paths.get(baseDir, "sqlDirectory", "demo.sql"));
+        assertThat(storageEntities.size()).isEqualTo(3);
+
+        StorageEntity storageEntity2 = storageEntities.stream()
+                .filter(storageEntity -> 
storageEntity.getFileName().equals("demo.sql"))
+                .findFirst()
+                .get();
+        assertThat(storageEntity2.getFullName())
+                .isEqualTo("file:" + Paths.get(baseDir, "sqlDirectory", 
"demo.sql"));
         assertThat(storageEntity2.getFileName()).isEqualTo("demo.sql");
         assertThat(storageEntity2.getPfullName()).isEqualTo("file:" + 
Paths.get(baseDir, "sqlDirectory"));
         assertThat(storageEntity2.isDirectory()).isFalse();