>From Wail Alkowaileet <[email protected]>:
Wail Alkowaileet has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17762 )
Change subject: [ASTERIXDB-3257][OTH} Fix S3 list in Cloud I/O Manager
......................................................................
[ASTERIXDB-3257][OTH} Fix S3 list in Cloud I/O Manager
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Fix list to return the entire objects stored in the cloud.
Change-Id: I500aab9a51376b63128c78feece334116b5215ce
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
2 files changed, 25 insertions(+), 2 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/62/17762/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
index 2faba79..8901dec 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
@@ -23,6 +23,7 @@
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.nio.charset.Charset;
+import java.util.ArrayList;
import java.util.List;
import software.amazon.awssdk.services.s3.S3Client;
@@ -41,6 +42,7 @@
ListObjectsV2Response listObjectsResponse;
ListObjectsV2Request.Builder listObjectsBuilder =
ListObjectsV2Request.builder().bucket(bucket);
listObjectsBuilder.prefix(toCloudPrefix(path));
+ List<S3Object> files = new ArrayList<>();
while (true) {
// List the objects from the start, or from the last marker in
case of truncated result
if (newMarker == null) {
@@ -49,6 +51,8 @@
listObjectsResponse =
s3Client.listObjectsV2(listObjectsBuilder.continuationToken(newMarker).build());
}
+ files.addAll(listObjectsResponse.contents());
+
// Mark the flag as done if done, otherwise, get the marker of the
previous response for the next request
if (Boolean.FALSE.equals(listObjectsResponse.isTruncated())) {
break;
@@ -56,7 +60,7 @@
newMarker = listObjectsResponse.nextContinuationToken();
}
}
- return listObjectsResponse.contents();
+ return files;
}
public static String encodeURI(String path) {
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index 4394361..a0c1318 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -66,6 +66,8 @@
// First get the set of local files
Set<FileReference> localFiles = ioManager.list(partitionPath);
Iterator<FileReference> localFilesIter = localFiles.iterator();
+ LOGGER.info("Cleaning partition {}. Total number of unchecked cloud
files {}", partitionPath.getRelativePath(),
+ cloudFiles.size());
// Reconcile local files and cloud files
while (localFilesIter.hasNext()) {
@@ -131,7 +133,8 @@
private static void logDeleteFile(FileReference fileReference) {
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Deleting {} from the local cache as it doesn't
exists in the cloud", fileReference);
+ LOGGER.debug("Deleting {} from the local cache as {} doesn't exist
the cloud", fileReference,
+ fileReference.getRelativePath());
}
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17762
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I500aab9a51376b63128c78feece334116b5215ce
Gerrit-Change-Number: 17762
Gerrit-PatchSet: 1
Gerrit-Owner: Wail Alkowaileet <[email protected]>
Gerrit-MessageType: newchange
