This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 04c8cbad38 [ASTERIXDB-3390][STO]: Update storage scheme for GCS
04c8cbad38 is described below
commit 04c8cbad386e563213c1fddf781ab81a4b886110
Author: Savyasach Reddy <[email protected]>
AuthorDate: Wed May 8 15:06:45 2024 +0530
[ASTERIXDB-3390][STO]: Update storage scheme for GCS
Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Savyasach Reddy <[email protected]>
Reviewed-by: Wail Alkowaileet <[email protected]>
---
.../asterix-app/src/test/resources/cc-cloud-storage-gcs.conf | 2 +-
.../java/org/apache/asterix/cloud/AbstractCloudIOManager.java | 9 ++++++---
.../org/apache/asterix/cloud/clients/CloudClientProvider.java | 2 +-
.../apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java | 2 +-
.../org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java | 5 +----
5 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
index d0ebd24bca..ccb7f9da0a 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
@@ -64,7 +64,7 @@ messaging.frame.count=512
cloud.deployment=true
storage.buffercache.pagesize=32KB
storage.partitioning=static
-cloud.storage.scheme=gcs
+cloud.storage.scheme=gs
cloud.storage.bucket=cloud-storage-container
cloud.storage.region=us-west2
cloud.storage.endpoint=http://127.0.0.1:4443
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 8cf5ac5e31..9df26f2f5f 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -239,14 +239,17 @@ public abstract class AbstractCloudIOManager extends
IOManager implements IParti
public final long doSyncWrite(IFileHandle fHandle, long offset,
ByteBuffer[] dataArray)
throws HyracksDataException {
long writtenBytes = localIoManager.doSyncWrite(fHandle, offset,
dataArray);
+ dataArray[0].flip();
+ dataArray[1].flip();
cloudWrite(fHandle, dataArray);
return writtenBytes;
}
@Override
- public final int doSyncWrite(IFileHandle fHandle, long offset, ByteBuffer
dataArray) throws HyracksDataException {
- int writtenBytes = localIoManager.doSyncWrite(fHandle, offset,
dataArray);
- cloudWrite(fHandle, dataArray);
+ public final int doSyncWrite(IFileHandle fHandle, long offset, ByteBuffer
data) throws HyracksDataException {
+ int writtenBytes = localIoManager.doSyncWrite(fHandle, offset, data);
+ data.flip();
+ cloudWrite(fHandle, data);
return writtenBytes;
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index 35ab467107..d5f508d8b8 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -27,7 +27,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
public class CloudClientProvider {
private static final String S3 = "s3";
- private static final String GCS = "gcs";
+ private static final String GCS = "gs";
private CloudClientProvider() {
throw new AssertionError("do not instantiate");
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
index 9eb3fe0ab2..9bcd386757 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
@@ -204,7 +204,7 @@ public class GCSCloudClient implements ICloudClient {
public boolean exists(String bucket, String path) {
profiler.objectGet();
Blob blob = gcsClient.get(bucket, path,
Storage.BlobGetOption.fields(Storage.BlobField.values()));
- return blob != null;
+ return blob != null && blob.exists();
}
@Override
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
index cccd9ec05b..a6dade5507 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
@@ -40,7 +40,6 @@ public class GCSWriter implements ICloudWriter {
private final String path;
private final IRequestProfiler profiler;
private final Storage gcsClient;
- private boolean uploadStarted = false;
private WriteChannel writer = null;
public GCSWriter(String bucket, String path, Storage gcsClient,
IRequestProfiler profiler) {
@@ -88,7 +87,6 @@ public class GCSWriter implements ICloudWriter {
try {
writer.close();
writer = null;
- uploadStarted = false;
} catch (IOException e) {
throw HyracksDataException.create(e);
}
@@ -104,8 +102,7 @@ public class GCSWriter implements ICloudWriter {
}
private void setUploadId() {
- if (!uploadStarted) {
- uploadStarted = true;
+ if (writer == null) {
writer = gcsClient.writer(BlobInfo.newBuilder(BlobId.of(bucket,
path)).build());
writer.setChunkSize(WRITE_BUFFER_SIZE);
log("STARTED");
