This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2098b99a50 [ASTERIXDB-3190][*DB] Use HeadObject when only metadata
info is needed
2098b99a50 is described below
commit 2098b99a503c2404585b2f2cee98dc2edc269056
Author: Hussain Towaileb <[email protected]>
AuthorDate: Thu May 18 22:53:26 2023 +0300
[ASTERIXDB-3190][*DB] Use HeadObject when only metadata info is needed
Change-Id: I9c201cbce0bd92ca2c7a2ae87aebf399c88b55af
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17544
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Hussain Towaileb <[email protected]>
Reviewed-by: Wail Alkowaileet <[email protected]>
---
.../org/apache/asterix/cloud/CloudIOManager.java | 6 ++---
.../apache/asterix/cloud/clients/ICloudClient.java | 4 ++--
.../cloud/clients/aws/s3/S3CloudClient.java | 27 +++++++++++++---------
.../java/org/apache/hyracks/api/io/IIOManager.java | 6 ++---
.../apache/hyracks/control/nc/io/IOManager.java | 6 ++---
.../buffercache/AbstractBufferedFileIOManager.java | 2 +-
.../storage/common/file/BufferedFileHandle.java | 2 +-
7 files changed, 29 insertions(+), 24 deletions(-)
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
index 97d8d80889..f87adbaefd 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
@@ -217,7 +217,7 @@ public class CloudIOManager extends IOManager {
}
@Override
- public long getSize(IFileHandle fileHandle) {
+ public long getSize(IFileHandle fileHandle) throws HyracksDataException {
if (!fileHandle.getFileReference().getFile().exists()) {
return cloudClient.getObjectSize(bucket,
fileHandle.getFileReference().getRelativePath());
}
@@ -225,7 +225,7 @@ public class CloudIOManager extends IOManager {
}
@Override
- public long getSize(FileReference fileReference) {
+ public long getSize(FileReference fileReference) throws
HyracksDataException {
if (!fileReference.getFile().exists()) {
return cloudClient.getObjectSize(bucket,
fileReference.getRelativePath());
}
@@ -265,7 +265,7 @@ public class CloudIOManager extends IOManager {
}
@Override
- public boolean exists(FileReference fileRef) {
+ public boolean exists(FileReference fileRef) throws HyracksDataException {
// Check if the file exists locally first as newly created files
(i.e., they are empty) are not stored in cloud storage
return fileRef.getFile().exists() || cloudClient.exists(bucket,
fileRef.getRelativePath());
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index fe2a2d65ef..4f02df208d 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -115,7 +115,7 @@ public interface ICloudClient {
* @param path path
* @return size
*/
- long getObjectSize(String bucket, String path);
+ long getObjectSize(String bucket, String path) throws HyracksDataException;
/**
* Checks if an object exists at the specified path
@@ -124,7 +124,7 @@ public interface ICloudClient {
* @param path path
* @return {@code true} if the object exists, {@code false} otherwise
*/
- boolean exists(String bucket, String path);
+ boolean exists(String bucket, String path) throws HyracksDataException;
/**
* Syncs files by downloading them from cloud storage to local storage
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index b800b96728..6db784d3d2 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -62,6 +62,7 @@ import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
@@ -201,23 +202,27 @@ public class S3CloudClient implements ICloudClient {
s3Client.deleteObjects(deleteReq);
}
- // TODO(htowaileb): Use the following cheaper request to check if an
object exists
- //
https://stackoverflow.com/questions/3910071/check-file-size-on-s3-without-downloading
@Override
- public long getObjectSize(String bucket, String path) {
- List<S3Object> objects = listS3Objects(s3Client, bucket, path);
- if (objects.isEmpty()) {
+ public long getObjectSize(String bucket, String path) throws
HyracksDataException {
+ try {
+ return
s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build()).contentLength();
+ } catch (NoSuchKeyException ex) {
return 0;
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
}
- return objects.get(0).size();
}
- // TODO(htowaileb): Use the following cheaper request to check if an
object exists
- //
https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_HeadObject_section.html
@Override
- public boolean exists(String bucket, String path) {
- List<S3Object> objects = listS3Objects(s3Client, bucket, path);
- return !objects.isEmpty();
+ public boolean exists(String bucket, String path) throws
HyracksDataException {
+ try {
+
s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build());
+ return true;
+ } catch (NoSuchKeyException ex) {
+ return false;
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
+ }
}
private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter
filter) {
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index 07e1c78a5f..63226ac7d6 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -65,9 +65,9 @@ public interface IIOManager extends Closeable {
void truncate(IFileHandle fileHandle, long size) throws
HyracksDataException;
- long getSize(IFileHandle fileHandle);
+ long getSize(IFileHandle fileHandle) throws HyracksDataException;
- long getSize(FileReference fileReference);
+ long getSize(FileReference fileReference) throws HyracksDataException;
WritableByteChannel newWritableChannel(IFileHandle fileHandle);
@@ -139,7 +139,7 @@ public interface IIOManager extends Closeable {
// TODO: Remove and use list
Collection<FileReference> getMatchingFiles(FileReference root,
FilenameFilter filter) throws HyracksDataException;
- boolean exists(FileReference fileRef);
+ boolean exists(FileReference fileRef) throws HyracksDataException;
void create(FileReference fileRef) throws HyracksDataException;
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 24d1061ae3..19fbff37ec 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -399,12 +399,12 @@ public class IOManager implements IIOManager {
}
@Override
- public long getSize(IFileHandle fileHandle) {
+ public long getSize(IFileHandle fileHandle) throws HyracksDataException {
return getSize(fileHandle.getFileReference());
}
@Override
- public long getSize(FileReference fileReference) {
+ public long getSize(FileReference fileReference) throws
HyracksDataException {
return fileReference.getFile().length();
}
@@ -596,7 +596,7 @@ public class IOManager implements IIOManager {
}
@Override
- public boolean exists(FileReference fileRef) {
+ public boolean exists(FileReference fileRef) throws HyracksDataException {
return fileRef.getFile().exists();
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
index 44a203106f..6a4f10dff0 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
@@ -284,7 +284,7 @@ public abstract class AbstractBufferedFileIOManager {
return ioManager.doSyncWrite(fileHandle, offset, buf);
}
- protected final long getFileSize() {
+ protected final long getFileSize() throws HyracksDataException {
return ioManager.getSize(fileHandle);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
index 11862dc3fb..3d4d80bc3a 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
@@ -121,7 +121,7 @@ public class BufferedFileHandle extends
AbstractBufferedFileIOManager {
}
@Override
- public int getNumberOfPages() {
+ public int getNumberOfPages() throws HyracksDataException {
if (DEBUG) {
assert getFileSize() % bufferCache.getPageSizeWithHeader() == 0;
}
