(dolphinscheduler) branch dev updated: [Fix-18632][Storage-OSS] Paginate listStorageEntity instead of returning only the first page (#18633)

[zihaoxiang]

This is an automated email from the ASF dual-hosted git repository.

SbloodyS pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git


The following commit(s) were added to refs/heads/dev by this push:
     new 040e5cc4db [Fix-18632][Storage-OSS] Paginate listStorageEntity instead 
of returning only the first page (#18633)
040e5cc4db is described below

commit 040e5cc4db00fa12e4bb6452e02cb998af471ad7
Author: huangrunxing <huangrunxingx...@163.com>
AuthorDate: Fri Sep 11 13:44:59 2026 +0800

    [Fix-18632][Storage-OSS] Paginate listStorageEntity instead of returning 
only the first page (#18633)
---
 .../plugin/storage/oss/OssStorageOperator.java     | 56 ++++++++++++++--------
 1 file changed, 36 insertions(+), 20 deletions(-)

diff --git 
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
 
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
index de8c12e470..c04aa218f2 100644
--- 
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
+++ 
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
@@ -62,6 +62,8 @@ import com.aliyun.oss.model.PutObjectRequest;
 @Slf4j
 public class OssStorageOperator extends AbstractStorageOperator implements 
Closeable, StorageOperator {
 
+    private static final int MAX_KEYS = 1000;
+
     private String region;
 
     private String bucketName;
@@ -224,29 +226,43 @@ public class OssStorageOperator extends 
AbstractStorageOperator implements Close
     public List<StorageEntity> listStorageEntity(String resourceAbsolutePath) {
         final String ossResourceAbsolutePath = 
transformAbsolutePathToOssKey(resourceAbsolutePath);
 
-        ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request()
-                .withBucketName(bucketName)
-                .withDelimiter("/")
-                .withPrefix(ossResourceAbsolutePath);
+        List<StorageEntity> storageEntities = new ArrayList<>();
+        Set<String> commonPrefixSet = new HashSet<>();
+        String continuationToken = null;
+        boolean truncated;
+        do {
+            ListObjectsV2Request listObjectsV2Request = new 
ListObjectsV2Request()
+                    .withBucketName(bucketName)
+                    .withDelimiter("/")
+                    .withPrefix(ossResourceAbsolutePath)
+                    .withMaxKeys(MAX_KEYS);
+            if (continuationToken != null) {
+                listObjectsV2Request.setContinuationToken(continuationToken);
+            }
 
-        ListObjectsV2Result listObjectsV2Result = 
ossClient.listObjectsV2(listObjectsV2Request);
+            ListObjectsV2Result listObjectsV2Result = 
ossClient.listObjectsV2(listObjectsV2Request);
+
+            for (String commonPrefix : 
listObjectsV2Result.getCommonPrefixes()) {
+                if (commonPrefixSet.add(commonPrefix)) {
+                    
storageEntities.add(transformCommonPrefixToStorageEntity(commonPrefix));
+                }
+            }
 
-        // Collect common prefixes (directories)
-        Set<String> commonPrefixSet = new 
HashSet<>(listObjectsV2Result.getCommonPrefixes());
+            for (OSSObjectSummary ossObjectSummary : 
listObjectsV2Result.getObjectSummaries()) {
+                // Filter out the current directory itself
+                if (ossObjectSummary.getKey().equals(ossResourceAbsolutePath)) 
{
+                    continue;
+                }
+                // Filter out directory marker objects that are already in 
commonPrefixes
+                if (commonPrefixSet.contains(ossObjectSummary.getKey())) {
+                    continue;
+                }
+                
storageEntities.add(transformOSSObjectToStorageEntity(ossObjectSummary));
+            }
 
-        List<StorageEntity> storageEntities = new ArrayList<>();
-        storageEntities.addAll(listObjectsV2Result.getCommonPrefixes()
-                .stream()
-                .map(this::transformCommonPrefixToStorageEntity)
-                .collect(Collectors.toList()));
-        storageEntities.addAll(
-                listObjectsV2Result.getObjectSummaries().stream()
-                        // Filter out the current directory itself
-                        .filter(ossObjectSummary -> 
!ossObjectSummary.getKey().equals(ossResourceAbsolutePath))
-                        // Filter out directory marker objects that are 
already in commonPrefixes
-                        .filter(ossObjectSummary -> 
!commonPrefixSet.contains(ossObjectSummary.getKey()))
-                        .map(this::transformOSSObjectToStorageEntity)
-                        .collect(Collectors.toList()));
+            truncated = listObjectsV2Result.isTruncated();
+            continuationToken = listObjectsV2Result.getNextContinuationToken();
+        } while (truncated && continuationToken != null);
 
         return storageEntities;