>From Wail Alkowaileet <[email protected]>:
Wail Alkowaileet has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17773 )
Change subject: [ASTERIXDB-3260][STO] Expose stored cloud objects for
diagnostics
......................................................................
[ASTERIXDB-3260][STO] Expose stored cloud objects for diagnostics
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Expose the list of objects (as a JSON array) stored in the
cloud object store (S3)
Change-Id: I37900dc4d5401530a25ef66b916e7b6827ca93dd
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
3 files changed, 99 insertions(+), 38 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/73/17773/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 6973b7b..d9c248e 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -49,6 +49,9 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
public abstract class AbstractCloudIOManager extends IOManager implements
IPartitionBootstrapper {
private static final Logger LOGGER = LogManager.getLogger();
private static final String DATAVERSE_PATH =
@@ -260,4 +263,14 @@
super.close();
localIoManager.close();
}
+
+ /**
+ * Returns a list of all stored objects (sorted ASC by path) in the cloud
and their sizes
+ *
+ * @param objectMapper to create the result {@link JsonNode}
+ * @return {@link JsonNode} with stored objects' information
+ */
+ public final JsonNode listAsJson(ObjectMapper objectMapper) {
+ return cloudClient.listAsJson(objectMapper, bucket);
+ }
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index ff26915..f2b7ff4 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -28,6 +28,9 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
/**
* Interface containing methods to perform IO operation on the Cloud Storage
*/
@@ -137,6 +140,15 @@
void syncFiles(String bucket, Map<String, String>
cloudToLocalStoragePaths) throws HyracksDataException;
/**
+ * Produces a {@link JsonNode} that contains information about the stored
objects in the cloud
+ *
+ * @param objectMapper to create the result {@link JsonNode}
+ * @param bucket bucket name
+ * @return {@link JsonNode} with stored objects' information
+ */
+ JsonNode listAsJson(ObjectMapper objectMapper, String bucket);
+
+ /**
* Performs any necessary closing and cleaning up
*/
void close();
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 5619fc8..5faf663 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -53,6 +53,11 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
@@ -100,22 +105,6 @@
}
- private S3Client buildClient() {
- S3ClientBuilder builder = S3Client.builder();
- builder.credentialsProvider(config.createCredentialsProvider());
- builder.region(Region.of(config.getRegion()));
- if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
- URI uri;
- try {
- uri = new URI(config.getEndpoint());
- } catch (URISyntaxException ex) {
- throw new IllegalArgumentException(ex);
- }
- builder.endpointOverride(uri);
- }
- return builder.build();
- }
-
@Override
public ICloudBufferedWriter createBufferedWriter(String bucket, String
path) {
return new S3BufferedWriter(s3Client, profiler, bucket, path);
@@ -253,17 +242,6 @@
}
}
- private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter
filter) {
- Set<String> files = new HashSet<>();
- for (S3Object s3Object : contents) {
- String path = config.isEncodeKeys() ?
S3Utils.decodeURI(s3Object.key()) : s3Object.key();
- if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
- files.add(path);
- }
- }
- return files;
- }
-
@Override
public void syncFiles(String bucket, Map<String, String>
cloudToLocalStoragePaths) throws HyracksDataException {
LOGGER.info("Syncing cloud storage to local storage started");
@@ -311,6 +289,58 @@
LOGGER.info("Syncing cloud storage to local storage successful");
}
+ @Override
+ public JsonNode listAsJson(ObjectMapper objectMapper, String bucket) {
+ List<S3Object> objects = listS3Objects(s3Client, bucket, "/");
+ ArrayNode objectsInfo = objectMapper.createArrayNode();
+
+ objects.sort((x, y) -> String.CASE_INSENSITIVE_ORDER.compare(x.key(),
y.key()));
+ for (S3Object object : objects) {
+ ObjectNode objectInfo = objectsInfo.addObject();
+ objectInfo.put("path", object.key());
+ objectInfo.put("size", object.size());
+ }
+ return objectsInfo;
+ }
+
+ @Override
+ public void close() {
+ if (s3Client != null) {
+ s3Client.close();
+ }
+
+ if (s3TransferManager != null) {
+ s3TransferManager.close();
+ }
+ }
+
+ private S3Client buildClient() {
+ S3ClientBuilder builder = S3Client.builder();
+ builder.credentialsProvider(config.createCredentialsProvider());
+ builder.region(Region.of(config.getRegion()));
+ if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
+ URI uri;
+ try {
+ uri = new URI(config.getEndpoint());
+ } catch (URISyntaxException ex) {
+ throw new IllegalArgumentException(ex);
+ }
+ builder.endpointOverride(uri);
+ }
+ return builder.build();
+ }
+
+ private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter
filter) {
+ Set<String> files = new HashSet<>();
+ for (S3Object s3Object : contents) {
+ String path = config.isEncodeKeys() ?
S3Utils.decodeURI(s3Object.key()) : s3Object.key();
+ if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
+ files.add(path);
+ }
+ }
+ return files;
+ }
+
private void downloadFiles(String bucket, Map<String, String>
cloudToLocalStoragePaths)
throws HyracksDataException {
byte[] buffer = new byte[8 * 1024];
@@ -364,15 +394,4 @@
s3TransferManager =
S3TransferManager.builder().s3Client(client).build();
return s3TransferManager;
}
-
- @Override
- public void close() {
- if (s3Client != null) {
- s3Client.close();
- }
-
- if (s3TransferManager != null) {
- s3TransferManager.close();
- }
- }
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17773
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I37900dc4d5401530a25ef66b916e7b6827ca93dd
Gerrit-Change-Number: 17773
Gerrit-PatchSet: 1
Gerrit-Owner: Wail Alkowaileet <[email protected]>
Gerrit-MessageType: newchange
