This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1ac5c5b898 [ASTERIXDB-3196][*DB] Skip upload on empty writes
1ac5c5b898 is described below
commit 1ac5c5b898a97fdd648a06c339324d15f2b949cd
Author: Murtadha Hubail <[email protected]>
AuthorDate: Fri Jun 23 18:12:39 2023 -0700
[ASTERIXDB-3196][*DB] Skip upload on empty writes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When no data was uploaded, skip completing the multipart
upload.
- Workaround S3Mock encoding issues by encoding/decoding keys
on list objects.
Change-Id: I31f9955e8a9c4897eba4f819746b2dd47fa7c50f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17614
Reviewed-by: Ali Alsuliman <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
---
.../apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java | 6 ++++++
.../org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java | 9 +++++++++
.../org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java | 5 ++++-
.../java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java | 2 +-
4 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
index 9104d9afc5..6c02bbc260 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
@@ -69,6 +69,9 @@ public class S3BufferedWriter implements ICloudBufferedWriter
{
@Override
public void finish() throws HyracksDataException {
+ if (uploadId == null) {
+ return;
+ }
CompletedMultipartUpload completedMultipartUpload =
CompletedMultipartUpload.builder().parts(partQueue).build();
CompleteMultipartUploadRequest completeMultipartUploadRequest =
CompleteMultipartUploadRequest.builder()
.bucket(bucket).key(path).uploadId(uploadId).multipartUpload(completedMultipartUpload).build();
@@ -97,6 +100,9 @@ public class S3BufferedWriter implements
ICloudBufferedWriter {
@Override
public void abort() throws HyracksDataException {
+ if (uploadId == null) {
+ return;
+ }
s3Client.abortMultipartUpload(
AbortMultipartUploadRequest.builder().bucket(bucket).key(path).uploadId(uploadId).build());
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index 281fe7707b..bbe8586540 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -55,7 +55,16 @@ public class S3ClientConfig {
return prefix;
}
+ public boolean isEncodeKeys() {
+ // to workaround https://github.com/findify/s3mock/issues/187 in our
S3Mock, we encode/decode keys
+ return isS3Mock();
+ }
+
public AwsCredentialsProvider createCredentialsProvider() {
return anonymousAuth ? AnonymousCredentialsProvider.create() :
DefaultCredentialsProvider.create();
}
+
+ private boolean isS3Mock() {
+ return endpoint != null && !endpoint.isEmpty();
+ }
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 6d65f69039..0fb9c082cc 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.cloud.clients.aws.s3;
+import static org.apache.asterix.cloud.clients.aws.s3.S3Utils.encodeURI;
import static org.apache.asterix.cloud.clients.aws.s3.S3Utils.listS3Objects;
import java.io.File;
@@ -108,6 +109,7 @@ public class S3CloudClient implements ICloudClient {
@Override
public Set<String> listObjects(String bucket, String path, FilenameFilter
filter) {
+ path = config.isEncodeKeys() ? encodeURI(path) : path;
return filterAndGet(listS3Objects(s3Client, bucket, path), filter);
}
@@ -171,6 +173,7 @@ public class S3CloudClient implements ICloudClient {
@Override
public void copy(String bucket, String srcPath, FileReference destPath) {
+ srcPath = config.isEncodeKeys() ? encodeURI(srcPath) : srcPath;
List<S3Object> objects = listS3Objects(s3Client, bucket, srcPath);
for (S3Object object : objects) {
String srcKey = object.key();
@@ -223,7 +226,7 @@ public class S3CloudClient implements ICloudClient {
private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter
filter) {
Set<String> files = new HashSet<>();
for (S3Object s3Object : contents) {
- String path = S3Utils.decodeURI(s3Object.key());
+ String path = config.isEncodeKeys() ?
S3Utils.decodeURI(s3Object.key()) : s3Object.key();
if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
files.add(path);
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
index 4ea11b2ec2..2faba79972 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
@@ -40,7 +40,7 @@ public class S3Utils {
String newMarker = null;
ListObjectsV2Response listObjectsResponse;
ListObjectsV2Request.Builder listObjectsBuilder =
ListObjectsV2Request.builder().bucket(bucket);
- listObjectsBuilder.prefix(encodeURI(toCloudPrefix(path)));
+ listObjectsBuilder.prefix(toCloudPrefix(path));
while (true) {
// List the objects from the start, or from the last marker in
case of truncated result
if (newMarker == null) {
