>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17544 )
Change subject: [ASTERIXDB-3190][*DB] Use HeadObject when only metadata info is needed ...................................................................... [ASTERIXDB-3190][*DB] Use HeadObject when only metadata info is needed Change-Id: I9c201cbce0bd92ca2c7a2ae87aebf399c88b55af Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17544 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java 7 files changed, 43 insertions(+), 24 deletions(-) Approvals: Wail Alkowaileet: Looks good to me, approved Hussain Towaileb: Looks good to me, but someone else must approve Jenkins: Verified; Verified Anon. E. Moose #1000171: diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java index 97d8d80..f87adba 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java @@ -217,7 +217,7 @@ } @Override - public long getSize(IFileHandle fileHandle) { + public long getSize(IFileHandle fileHandle) throws HyracksDataException { if (!fileHandle.getFileReference().getFile().exists()) { return cloudClient.getObjectSize(bucket, fileHandle.getFileReference().getRelativePath()); } @@ -225,7 +225,7 @@ } @Override - public long getSize(FileReference fileReference) { + public long getSize(FileReference fileReference) throws HyracksDataException { if (!fileReference.getFile().exists()) { return cloudClient.getObjectSize(bucket, fileReference.getRelativePath()); } @@ -265,7 +265,7 @@ } @Override - public boolean exists(FileReference fileRef) { + public boolean exists(FileReference fileRef) throws HyracksDataException { // Check if the file exists locally first as newly created files (i.e., they are empty) are not stored in cloud storage return fileRef.getFile().exists() || cloudClient.exists(bucket, fileRef.getRelativePath()); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java index fe2a2d6..4f02df2 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java @@ -115,7 +115,7 @@ * @param path path * @return size */ - long getObjectSize(String bucket, String path); + long getObjectSize(String bucket, String path) throws HyracksDataException; /** * Checks if an object exists at the specified path @@ -124,7 +124,7 @@ * @param path path * @return {@code true} if the object exists, {@code false} otherwise */ - boolean exists(String bucket, String path); + boolean exists(String bucket, String path) throws HyracksDataException; /** * Syncs files by downloading them from cloud storage to local storage diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java index b800b96..6db784d 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java @@ -62,6 +62,7 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -201,23 +202,27 @@ s3Client.deleteObjects(deleteReq); } - // TODO(htowaileb): Use the following cheaper request to check if an object exists - // https://stackoverflow.com/questions/3910071/check-file-size-on-s3-without-downloading @Override - public long getObjectSize(String bucket, String path) { - List<S3Object> objects = listS3Objects(s3Client, bucket, path); - if (objects.isEmpty()) { + public long getObjectSize(String bucket, String path) throws HyracksDataException { + try { + return s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build()).contentLength(); + } catch (NoSuchKeyException ex) { return 0; + } catch (Exception ex) { + throw HyracksDataException.create(ex); } - return objects.get(0).size(); } - // TODO(htowaileb): Use the following cheaper request to check if an object exists - // https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_HeadObject_section.html @Override - public boolean exists(String bucket, String path) { - List<S3Object> objects = listS3Objects(s3Client, bucket, path); - return !objects.isEmpty(); + public boolean exists(String bucket, String path) throws HyracksDataException { + try { + s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build()); + return true; + } catch (NoSuchKeyException ex) { + return false; + } catch (Exception ex) { + throw HyracksDataException.create(ex); + } } private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) { diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java index 07e1c78..63226ac 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java @@ -65,9 +65,9 @@ void truncate(IFileHandle fileHandle, long size) throws HyracksDataException; - long getSize(IFileHandle fileHandle); + long getSize(IFileHandle fileHandle) throws HyracksDataException; - long getSize(FileReference fileReference); + long getSize(FileReference fileReference) throws HyracksDataException; WritableByteChannel newWritableChannel(IFileHandle fileHandle); @@ -139,7 +139,7 @@ // TODO: Remove and use list Collection<FileReference> getMatchingFiles(FileReference root, FilenameFilter filter) throws HyracksDataException; - boolean exists(FileReference fileRef); + boolean exists(FileReference fileRef) throws HyracksDataException; void create(FileReference fileRef) throws HyracksDataException; diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java index 24d1061..19fbff3 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java @@ -399,12 +399,12 @@ } @Override - public long getSize(IFileHandle fileHandle) { + public long getSize(IFileHandle fileHandle) throws HyracksDataException { return getSize(fileHandle.getFileReference()); } @Override - public long getSize(FileReference fileReference) { + public long getSize(FileReference fileReference) throws HyracksDataException { return fileReference.getFile().length(); } @@ -596,7 +596,7 @@ } @Override - public boolean exists(FileReference fileRef) { + public boolean exists(FileReference fileRef) throws HyracksDataException { return fileRef.getFile().exists(); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java index 44a2031..6a4f10d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java @@ -284,7 +284,7 @@ return ioManager.doSyncWrite(fileHandle, offset, buf); } - protected final long getFileSize() { + protected final long getFileSize() throws HyracksDataException { return ioManager.getSize(fileHandle); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java index 11862dc..3d4d80b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java @@ -121,7 +121,7 @@ } @Override - public int getNumberOfPages() { + public int getNumberOfPages() throws HyracksDataException { if (DEBUG) { assert getFileSize() % bufferCache.getPageSizeWithHeader() == 0; } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17544 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I9c201cbce0bd92ca2c7a2ae87aebf399c88b55af Gerrit-Change-Number: 17544 Gerrit-PatchSet: 3 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Wail Alkowaileet Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
