[asterixdb] branch master updated: [ASTERIXDB-3257][OTH} Fix S3 list in Cloud I/O Manager

[wyk]

This is an automated email from the ASF dual-hosted git repository.

wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git


The following commit(s) were added to refs/heads/master by this push:
     new d3a31a6bf9 [ASTERIXDB-3257][OTH} Fix S3 list in Cloud I/O Manager
d3a31a6bf9 is described below

commit d3a31a6bf9e1f1ad2dde81b877079132b85a38a5
Author: Wail Alkowaileet <wael....@gmail.com>
AuthorDate: Tue Sep 5 15:01:33 2023 -0700

    [ASTERIXDB-3257][OTH} Fix S3 list in Cloud I/O Manager
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    Fix list to return the entire objects stored in the cloud.
    
    Change-Id: I500aab9a51376b63128c78feece334116b5215ce
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17762
    Reviewed-by: Hussain Towaileb <hussai...@gmail.com>
    Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
    Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
---
 .../main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java  | 6 +++++-
 .../src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java  | 5 ++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git 
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
 
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
index 2faba79972..8901dec73a 100644
--- 
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
+++ 
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
@@ -23,6 +23,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLDecoder;
 import java.nio.charset.Charset;
+import java.util.ArrayList;
 import java.util.List;
 
 import software.amazon.awssdk.services.s3.S3Client;
@@ -41,6 +42,7 @@ public class S3Utils {
         ListObjectsV2Response listObjectsResponse;
         ListObjectsV2Request.Builder listObjectsBuilder = 
ListObjectsV2Request.builder().bucket(bucket);
         listObjectsBuilder.prefix(toCloudPrefix(path));
+        List<S3Object> files = new ArrayList<>();
         while (true) {
             // List the objects from the start, or from the last marker in 
case of truncated result
             if (newMarker == null) {
@@ -49,6 +51,8 @@ public class S3Utils {
                 listObjectsResponse = 
s3Client.listObjectsV2(listObjectsBuilder.continuationToken(newMarker).build());
             }
 
+            files.addAll(listObjectsResponse.contents());
+
             // Mark the flag as done if done, otherwise, get the marker of the 
previous response for the next request
             if (Boolean.FALSE.equals(listObjectsResponse.isTruncated())) {
                 break;
@@ -56,7 +60,7 @@ public class S3Utils {
                 newMarker = listObjectsResponse.nextContinuationToken();
             }
         }
-        return listObjectsResponse.contents();
+        return files;
     }
 
     public static String encodeURI(String path) {
diff --git 
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
 
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index 4394361bc5..aee916c4d3 100644
--- 
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++ 
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -66,6 +66,8 @@ public class CloudFileUtil {
         // First get the set of local files
         Set<FileReference> localFiles = ioManager.list(partitionPath);
         Iterator<FileReference> localFilesIter = localFiles.iterator();
+        LOGGER.info("Cleaning partition {}. Total number of unchecked cloud 
files {}", partitionPath.getRelativePath(),
+                cloudFiles.size());
 
         // Reconcile local files and cloud files
         while (localFilesIter.hasNext()) {
@@ -131,7 +133,8 @@ public class CloudFileUtil {
 
     private static void logDeleteFile(FileReference fileReference) {
         if (LOGGER.isDebugEnabled()) {
-            LOGGER.debug("Deleting {} from the local cache as it doesn't 
exists in the cloud", fileReference);
+            LOGGER.debug("Deleting {} from the local cache as {} doesn't exist 
in the cloud", fileReference,
+                    fileReference.getRelativePath());
         }
     }
 }