>From Wail Alkowaileet <[email protected]>: Wail Alkowaileet has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17773 )
Change subject: [ASTERIXDB-3260][STO] Expose stored cloud objects for diagnostics ...................................................................... [ASTERIXDB-3260][STO] Expose stored cloud objects for diagnostics - user model changes: no - storage format changes: no - interface changes: yes Details: Expose the list of objects (as a JSON array) stored in the cloud object store (S3) Change-Id: I37900dc4d5401530a25ef66b916e7b6827ca93dd Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17773 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java 3 files changed, 104 insertions(+), 38 deletions(-) Approvals: Murtadha Hubail: Looks good to me, approved Wail Alkowaileet: Looks good to me, but someone else must approve Jenkins: Verified; Verified diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java index 6973b7b..d9c248e 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java @@ -49,6 +49,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + public abstract class AbstractCloudIOManager extends IOManager implements IPartitionBootstrapper { private static final Logger LOGGER = LogManager.getLogger(); private static final String DATAVERSE_PATH = @@ -260,4 +263,14 @@ super.close(); localIoManager.close(); } + + /** + * Returns a list of all stored objects (sorted ASC by path) in the cloud and their sizes + * + * @param objectMapper to create the result {@link JsonNode} + * @return {@link JsonNode} with stored objects' information + */ + public final JsonNode listAsJson(ObjectMapper objectMapper) { + return cloudClient.listAsJson(objectMapper, bucket); + } } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java index ff26915..f2b7ff4 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java @@ -28,6 +28,9 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + /** * Interface containing methods to perform IO operation on the Cloud Storage */ @@ -137,6 +140,15 @@ void syncFiles(String bucket, Map<String, String> cloudToLocalStoragePaths) throws HyracksDataException; /** + * Produces a {@link JsonNode} that contains information about the stored objects in the cloud + * + * @param objectMapper to create the result {@link JsonNode} + * @param bucket bucket name + * @return {@link JsonNode} with stored objects' information + */ + JsonNode listAsJson(ObjectMapper objectMapper, String bucket); + + /** * Performs any necessary closing and cleaning up */ void close(); diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java index 5619fc8..5faf663 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java @@ -53,6 +53,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.regions.Region; @@ -100,22 +105,6 @@ } - private S3Client buildClient() { - S3ClientBuilder builder = S3Client.builder(); - builder.credentialsProvider(config.createCredentialsProvider()); - builder.region(Region.of(config.getRegion())); - if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) { - URI uri; - try { - uri = new URI(config.getEndpoint()); - } catch (URISyntaxException ex) { - throw new IllegalArgumentException(ex); - } - builder.endpointOverride(uri); - } - return builder.build(); - } - @Override public ICloudBufferedWriter createBufferedWriter(String bucket, String path) { return new S3BufferedWriter(s3Client, profiler, bucket, path); @@ -253,17 +242,6 @@ } } - private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) { - Set<String> files = new HashSet<>(); - for (S3Object s3Object : contents) { - String path = config.isEncodeKeys() ? S3Utils.decodeURI(s3Object.key()) : s3Object.key(); - if (filter.accept(null, IoUtil.getFileNameFromPath(path))) { - files.add(path); - } - } - return files; - } - @Override public void syncFiles(String bucket, Map<String, String> cloudToLocalStoragePaths) throws HyracksDataException { LOGGER.info("Syncing cloud storage to local storage started"); @@ -311,6 +289,58 @@ LOGGER.info("Syncing cloud storage to local storage successful"); } + @Override + public JsonNode listAsJson(ObjectMapper objectMapper, String bucket) { + List<S3Object> objects = listS3Objects(s3Client, bucket, "/"); + ArrayNode objectsInfo = objectMapper.createArrayNode(); + + objects.sort((x, y) -> String.CASE_INSENSITIVE_ORDER.compare(x.key(), y.key())); + for (S3Object object : objects) { + ObjectNode objectInfo = objectsInfo.addObject(); + objectInfo.put("path", object.key()); + objectInfo.put("size", object.size()); + } + return objectsInfo; + } + + @Override + public void close() { + if (s3Client != null) { + s3Client.close(); + } + + if (s3TransferManager != null) { + s3TransferManager.close(); + } + } + + private S3Client buildClient() { + S3ClientBuilder builder = S3Client.builder(); + builder.credentialsProvider(config.createCredentialsProvider()); + builder.region(Region.of(config.getRegion())); + if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) { + URI uri; + try { + uri = new URI(config.getEndpoint()); + } catch (URISyntaxException ex) { + throw new IllegalArgumentException(ex); + } + builder.endpointOverride(uri); + } + return builder.build(); + } + + private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) { + Set<String> files = new HashSet<>(); + for (S3Object s3Object : contents) { + String path = config.isEncodeKeys() ? S3Utils.decodeURI(s3Object.key()) : s3Object.key(); + if (filter.accept(null, IoUtil.getFileNameFromPath(path))) { + files.add(path); + } + } + return files; + } + private void downloadFiles(String bucket, Map<String, String> cloudToLocalStoragePaths) throws HyracksDataException { byte[] buffer = new byte[8 * 1024]; @@ -364,15 +394,4 @@ s3TransferManager = S3TransferManager.builder().s3Client(client).build(); return s3TransferManager; } - - @Override - public void close() { - if (s3Client != null) { - s3Client.close(); - } - - if (s3TransferManager != null) { - s3TransferManager.close(); - } - } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17773 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I37900dc4d5401530a25ef66b916e7b6827ca93dd Gerrit-Change-Number: 17773 Gerrit-PatchSet: 2 Gerrit-Owner: Wail Alkowaileet <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
