This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new 1ac5c5b898 [ASTERIXDB-3196][*DB] Skip upload on empty writes 1ac5c5b898 is described below commit 1ac5c5b898a97fdd648a06c339324d15f2b949cd Author: Murtadha Hubail <mhub...@apache.org> AuthorDate: Fri Jun 23 18:12:39 2023 -0700 [ASTERIXDB-3196][*DB] Skip upload on empty writes - user model changes: no - storage format changes: no - interface changes: no Details: - When no data was uploaded, skip completing the multipart upload. - Workaround S3Mock encoding issues by encoding/decoding keys on list objects. Change-Id: I31f9955e8a9c4897eba4f819746b2dd47fa7c50f Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17614 Reviewed-by: Ali Alsuliman <ali.al.solai...@gmail.com> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> --- .../apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java | 6 ++++++ .../org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java | 9 +++++++++ .../org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java | 5 ++++- .../java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java | 2 +- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java index 9104d9afc5..6c02bbc260 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java @@ -69,6 +69,9 @@ public class S3BufferedWriter implements ICloudBufferedWriter { @Override public void finish() throws HyracksDataException { + if (uploadId == null) { + return; + } CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(partQueue).build(); CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder() .bucket(bucket).key(path).uploadId(uploadId).multipartUpload(completedMultipartUpload).build(); @@ -97,6 +100,9 @@ public class S3BufferedWriter implements ICloudBufferedWriter { @Override public void abort() throws HyracksDataException { + if (uploadId == null) { + return; + } s3Client.abortMultipartUpload( AbortMultipartUploadRequest.builder().bucket(bucket).key(path).uploadId(uploadId).build()); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java index 281fe7707b..bbe8586540 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java @@ -55,7 +55,16 @@ public class S3ClientConfig { return prefix; } + public boolean isEncodeKeys() { + // to workaround https://github.com/findify/s3mock/issues/187 in our S3Mock, we encode/decode keys + return isS3Mock(); + } + public AwsCredentialsProvider createCredentialsProvider() { return anonymousAuth ? AnonymousCredentialsProvider.create() : DefaultCredentialsProvider.create(); } + + private boolean isS3Mock() { + return endpoint != null && !endpoint.isEmpty(); + } } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java index 6d65f69039..0fb9c082cc 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.cloud.clients.aws.s3; +import static org.apache.asterix.cloud.clients.aws.s3.S3Utils.encodeURI; import static org.apache.asterix.cloud.clients.aws.s3.S3Utils.listS3Objects; import java.io.File; @@ -108,6 +109,7 @@ public class S3CloudClient implements ICloudClient { @Override public Set<String> listObjects(String bucket, String path, FilenameFilter filter) { + path = config.isEncodeKeys() ? encodeURI(path) : path; return filterAndGet(listS3Objects(s3Client, bucket, path), filter); } @@ -171,6 +173,7 @@ public class S3CloudClient implements ICloudClient { @Override public void copy(String bucket, String srcPath, FileReference destPath) { + srcPath = config.isEncodeKeys() ? encodeURI(srcPath) : srcPath; List<S3Object> objects = listS3Objects(s3Client, bucket, srcPath); for (S3Object object : objects) { String srcKey = object.key(); @@ -223,7 +226,7 @@ public class S3CloudClient implements ICloudClient { private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) { Set<String> files = new HashSet<>(); for (S3Object s3Object : contents) { - String path = S3Utils.decodeURI(s3Object.key()); + String path = config.isEncodeKeys() ? S3Utils.decodeURI(s3Object.key()) : s3Object.key(); if (filter.accept(null, IoUtil.getFileNameFromPath(path))) { files.add(path); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java index 4ea11b2ec2..2faba79972 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java @@ -40,7 +40,7 @@ public class S3Utils { String newMarker = null; ListObjectsV2Response listObjectsResponse; ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(bucket); - listObjectsBuilder.prefix(encodeURI(toCloudPrefix(path))); + listObjectsBuilder.prefix(toCloudPrefix(path)); while (true) { // List the objects from the start, or from the last marker in case of truncated result if (newMarker == null) {