This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 53119940cf [ASTERIXDB-3260][STO] Expose stored cloud objects for
diagnostics
53119940cf is described below
commit 53119940cf1f3839de7c1276058588dd633a2635
Author: Wail Alkowaileet <[email protected]>
AuthorDate: Mon Sep 11 15:32:57 2023 -0700
[ASTERIXDB-3260][STO] Expose stored cloud objects for diagnostics
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Expose the list of objects (as a JSON array) stored in the
cloud object store (S3)
Change-Id: I37900dc4d5401530a25ef66b916e7b6827ca93dd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17773
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Wail Alkowaileet <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
---
.../asterix/cloud/AbstractCloudIOManager.java | 13 +++
.../apache/asterix/cloud/clients/ICloudClient.java | 12 +++
.../cloud/clients/aws/s3/S3CloudClient.java | 95 +++++++++++++---------
3 files changed, 82 insertions(+), 38 deletions(-)
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 6973b7bc1b..d9c248ec61 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -49,6 +49,9 @@ import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
public abstract class AbstractCloudIOManager extends IOManager implements
IPartitionBootstrapper {
private static final Logger LOGGER = LogManager.getLogger();
private static final String DATAVERSE_PATH =
@@ -260,4 +263,14 @@ public abstract class AbstractCloudIOManager extends
IOManager implements IParti
super.close();
localIoManager.close();
}
+
+ /**
+ * Returns a list of all stored objects (sorted ASC by path) in the cloud
and their sizes
+ *
+ * @param objectMapper to create the result {@link JsonNode}
+ * @return {@link JsonNode} with stored objects' information
+ */
+ public final JsonNode listAsJson(ObjectMapper objectMapper) {
+ return cloudClient.listAsJson(objectMapper, bucket);
+ }
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index ff26915bc3..f2b7ff45df 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -28,6 +28,9 @@ import java.util.Set;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
/**
* Interface containing methods to perform IO operation on the Cloud Storage
*/
@@ -136,6 +139,15 @@ public interface ICloudClient {
*/
void syncFiles(String bucket, Map<String, String>
cloudToLocalStoragePaths) throws HyracksDataException;
+ /**
+ * Produces a {@link JsonNode} that contains information about the stored
objects in the cloud
+ *
+ * @param objectMapper to create the result {@link JsonNode}
+ * @param bucket bucket name
+ * @return {@link JsonNode} with stored objects' information
+ */
+ JsonNode listAsJson(ObjectMapper objectMapper, String bucket);
+
/**
* Performs any necessary closing and cleaning up
*/
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 5619fc8ca8..5faf663d50 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -53,6 +53,11 @@ import org.apache.hyracks.api.util.IoUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
@@ -100,22 +105,6 @@ public class S3CloudClient implements ICloudClient {
}
- private S3Client buildClient() {
- S3ClientBuilder builder = S3Client.builder();
- builder.credentialsProvider(config.createCredentialsProvider());
- builder.region(Region.of(config.getRegion()));
- if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
- URI uri;
- try {
- uri = new URI(config.getEndpoint());
- } catch (URISyntaxException ex) {
- throw new IllegalArgumentException(ex);
- }
- builder.endpointOverride(uri);
- }
- return builder.build();
- }
-
@Override
public ICloudBufferedWriter createBufferedWriter(String bucket, String
path) {
return new S3BufferedWriter(s3Client, profiler, bucket, path);
@@ -253,17 +242,6 @@ public class S3CloudClient implements ICloudClient {
}
}
- private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter
filter) {
- Set<String> files = new HashSet<>();
- for (S3Object s3Object : contents) {
- String path = config.isEncodeKeys() ?
S3Utils.decodeURI(s3Object.key()) : s3Object.key();
- if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
- files.add(path);
- }
- }
- return files;
- }
-
@Override
public void syncFiles(String bucket, Map<String, String>
cloudToLocalStoragePaths) throws HyracksDataException {
LOGGER.info("Syncing cloud storage to local storage started");
@@ -311,6 +289,58 @@ public class S3CloudClient implements ICloudClient {
LOGGER.info("Syncing cloud storage to local storage successful");
}
+ @Override
+ public JsonNode listAsJson(ObjectMapper objectMapper, String bucket) {
+ List<S3Object> objects = listS3Objects(s3Client, bucket, "/");
+ ArrayNode objectsInfo = objectMapper.createArrayNode();
+
+ objects.sort((x, y) -> String.CASE_INSENSITIVE_ORDER.compare(x.key(),
y.key()));
+ for (S3Object object : objects) {
+ ObjectNode objectInfo = objectsInfo.addObject();
+ objectInfo.put("path", object.key());
+ objectInfo.put("size", object.size());
+ }
+ return objectsInfo;
+ }
+
+ @Override
+ public void close() {
+ if (s3Client != null) {
+ s3Client.close();
+ }
+
+ if (s3TransferManager != null) {
+ s3TransferManager.close();
+ }
+ }
+
+ private S3Client buildClient() {
+ S3ClientBuilder builder = S3Client.builder();
+ builder.credentialsProvider(config.createCredentialsProvider());
+ builder.region(Region.of(config.getRegion()));
+ if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
+ URI uri;
+ try {
+ uri = new URI(config.getEndpoint());
+ } catch (URISyntaxException ex) {
+ throw new IllegalArgumentException(ex);
+ }
+ builder.endpointOverride(uri);
+ }
+ return builder.build();
+ }
+
+ private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter
filter) {
+ Set<String> files = new HashSet<>();
+ for (S3Object s3Object : contents) {
+ String path = config.isEncodeKeys() ?
S3Utils.decodeURI(s3Object.key()) : s3Object.key();
+ if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
+ files.add(path);
+ }
+ }
+ return files;
+ }
+
private void downloadFiles(String bucket, Map<String, String>
cloudToLocalStoragePaths)
throws HyracksDataException {
byte[] buffer = new byte[8 * 1024];
@@ -364,15 +394,4 @@ public class S3CloudClient implements ICloudClient {
s3TransferManager =
S3TransferManager.builder().s3Client(client).build();
return s3TransferManager;
}
-
- @Override
- public void close() {
- if (s3Client != null) {
- s3Client.close();
- }
-
- if (s3TransferManager != null) {
- s3TransferManager.close();
- }
- }
}
