>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17544 )
Change subject: [ASTERIXDB-3190][*DB] Use HeadObject when only metadata info is
needed
......................................................................
[ASTERIXDB-3190][*DB] Use HeadObject when only metadata info is needed
Change-Id: I9c201cbce0bd92ca2c7a2ae87aebf399c88b55af
---
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
M
hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
5 files changed, 36 insertions(+), 22 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/44/17544/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
index 97d8d80..f87adba 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
@@ -217,7 +217,7 @@
}
@Override
- public long getSize(IFileHandle fileHandle) {
+ public long getSize(IFileHandle fileHandle) throws HyracksDataException {
if (!fileHandle.getFileReference().getFile().exists()) {
return cloudClient.getObjectSize(bucket,
fileHandle.getFileReference().getRelativePath());
}
@@ -225,7 +225,7 @@
}
@Override
- public long getSize(FileReference fileReference) {
+ public long getSize(FileReference fileReference) throws
HyracksDataException {
if (!fileReference.getFile().exists()) {
return cloudClient.getObjectSize(bucket,
fileReference.getRelativePath());
}
@@ -265,7 +265,7 @@
}
@Override
- public boolean exists(FileReference fileRef) {
+ public boolean exists(FileReference fileRef) throws HyracksDataException {
// Check if the file exists locally first as newly created files
(i.e., they are empty) are not stored in cloud storage
return fileRef.getFile().exists() || cloudClient.exists(bucket,
fileRef.getRelativePath());
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index fe2a2d6..4f02df2 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -115,7 +115,7 @@
* @param path path
* @return size
*/
- long getObjectSize(String bucket, String path);
+ long getObjectSize(String bucket, String path) throws HyracksDataException;
/**
* Checks if an object exists at the specified path
@@ -124,7 +124,7 @@
* @param path path
* @return {@code true} if the object exists, {@code false} otherwise
*/
- boolean exists(String bucket, String path);
+ boolean exists(String bucket, String path) throws HyracksDataException;
/**
* Syncs files by downloading them from cloud storage to local storage
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index b800b96..6db784d 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -62,6 +62,7 @@
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
@@ -201,23 +202,27 @@
s3Client.deleteObjects(deleteReq);
}
- // TODO(htowaileb): Use the following cheaper request to check if an
object exists
- //
https://stackoverflow.com/questions/3910071/check-file-size-on-s3-without-downloading
@Override
- public long getObjectSize(String bucket, String path) {
- List<S3Object> objects = listS3Objects(s3Client, bucket, path);
- if (objects.isEmpty()) {
+ public long getObjectSize(String bucket, String path) throws
HyracksDataException {
+ try {
+ return
s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build()).contentLength();
+ } catch (NoSuchKeyException ex) {
return 0;
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
}
- return objects.get(0).size();
}
- // TODO(htowaileb): Use the following cheaper request to check if an
object exists
- //
https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_HeadObject_section.html
@Override
- public boolean exists(String bucket, String path) {
- List<S3Object> objects = listS3Objects(s3Client, bucket, path);
- return !objects.isEmpty();
+ public boolean exists(String bucket, String path) throws
HyracksDataException {
+ try {
+
s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build());
+ return true;
+ } catch (NoSuchKeyException ex) {
+ return false;
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
+ }
}
private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter
filter) {
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index 07e1c78..63226ac 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -65,9 +65,9 @@
void truncate(IFileHandle fileHandle, long size) throws
HyracksDataException;
- long getSize(IFileHandle fileHandle);
+ long getSize(IFileHandle fileHandle) throws HyracksDataException;
- long getSize(FileReference fileReference);
+ long getSize(FileReference fileReference) throws HyracksDataException;
WritableByteChannel newWritableChannel(IFileHandle fileHandle);
@@ -139,7 +139,7 @@
// TODO: Remove and use list
Collection<FileReference> getMatchingFiles(FileReference root,
FilenameFilter filter) throws HyracksDataException;
- boolean exists(FileReference fileRef);
+ boolean exists(FileReference fileRef) throws HyracksDataException;
void create(FileReference fileRef) throws HyracksDataException;
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 24d1061..19fbff3 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -399,12 +399,12 @@
}
@Override
- public long getSize(IFileHandle fileHandle) {
+ public long getSize(IFileHandle fileHandle) throws HyracksDataException {
return getSize(fileHandle.getFileReference());
}
@Override
- public long getSize(FileReference fileReference) {
+ public long getSize(FileReference fileReference) throws
HyracksDataException {
return fileReference.getFile().length();
}
@@ -596,7 +596,7 @@
}
@Override
- public boolean exists(FileReference fileRef) {
+ public boolean exists(FileReference fileRef) throws HyracksDataException {
return fileRef.getFile().exists();
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17544
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I9c201cbce0bd92ca2c7a2ae87aebf399c88b55af
Gerrit-Change-Number: 17544
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
