>From Wail Alkowaileet <[email protected]>: Wail Alkowaileet has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17762 )
Change subject: [ASTERIXDB-3257][OTH} Fix S3 list in Cloud I/O Manager ...................................................................... [ASTERIXDB-3257][OTH} Fix S3 list in Cloud I/O Manager - user model changes: no - storage format changes: no - interface changes: no Details: Fix list to return the entire objects stored in the cloud. Change-Id: I500aab9a51376b63128c78feece334116b5215ce Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17762 Reviewed-by: Hussain Towaileb <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> --- M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java 2 files changed, 29 insertions(+), 2 deletions(-) Approvals: Hussain Towaileb: Looks good to me, approved Jenkins: Verified; Verified Objections: Anon. E. Moose #1000171: Violations found diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java index 2faba79..8901dec 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java @@ -23,6 +23,7 @@ import java.net.URISyntaxException; import java.net.URLDecoder; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.List; import software.amazon.awssdk.services.s3.S3Client; @@ -41,6 +42,7 @@ ListObjectsV2Response listObjectsResponse; ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(bucket); listObjectsBuilder.prefix(toCloudPrefix(path)); + List<S3Object> files = new ArrayList<>(); while (true) { // List the objects from the start, or from the last marker in case of truncated result if (newMarker == null) { @@ -49,6 +51,8 @@ listObjectsResponse = s3Client.listObjectsV2(listObjectsBuilder.continuationToken(newMarker).build()); } + files.addAll(listObjectsResponse.contents()); + // Mark the flag as done if done, otherwise, get the marker of the previous response for the next request if (Boolean.FALSE.equals(listObjectsResponse.isTruncated())) { break; @@ -56,7 +60,7 @@ newMarker = listObjectsResponse.nextContinuationToken(); } } - return listObjectsResponse.contents(); + return files; } public static String encodeURI(String path) { diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java index 4394361..aee916c 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java @@ -66,6 +66,8 @@ // First get the set of local files Set<FileReference> localFiles = ioManager.list(partitionPath); Iterator<FileReference> localFilesIter = localFiles.iterator(); + LOGGER.info("Cleaning partition {}. Total number of unchecked cloud files {}", partitionPath.getRelativePath(), + cloudFiles.size()); // Reconcile local files and cloud files while (localFilesIter.hasNext()) { @@ -131,7 +133,8 @@ private static void logDeleteFile(FileReference fileReference) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Deleting {} from the local cache as it doesn't exists in the cloud", fileReference); + LOGGER.debug("Deleting {} from the local cache as {} doesn't exist in the cloud", fileReference, + fileReference.getRelativePath()); } } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17762 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I500aab9a51376b63128c78feece334116b5215ce Gerrit-Change-Number: 17762 Gerrit-PatchSet: 3 Gerrit-Owner: Wail Alkowaileet <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
