This is an automated email from the ASF dual-hosted git repository. amoghj pushed a commit to branch wip-s3sign-body in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit e0f5229e6a2ddd20959986d7124fde917d4cd375 Author: amogh-jahagirdar <[email protected]> AuthorDate: Mon Aug 21 16:35:44 2023 -0700 AWS: Update S3V4RestSignerClient to send body for DeleteObjects requests --- .../aws/s3/signer/S3V4RestSignerClient.java | 27 ++++++++++++++++++++++ .../iceberg/aws/s3/signer/TestS3RestSigner.java | 20 ++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java index a334221a25..7433462eec 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java @@ -21,6 +21,9 @@ package org.apache.iceberg.aws.s3.signer; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.RemovalListener; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; import java.net.URI; import java.time.Duration; import java.util.Collections; @@ -55,6 +58,8 @@ import software.amazon.awssdk.auth.signer.params.AwsS3V4SignerParams; import software.amazon.awssdk.core.checksums.SdkChecksum; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.utils.IoUtils; @Value.Immutable public abstract class S3V4RestSignerClient @@ -286,6 +291,7 @@ public abstract class S3V4RestSignerClient .uri(request.getUri()) .headers(request.headers()) .properties(requestPropertiesSupplier().get()) + .body(bodyIfApplicable(request)) .build(); Key cacheKey = Key.from(remoteSigningRequest); @@ -328,6 +334,27 @@ public abstract class S3V4RestSignerClient return mutableRequest.build(); } + private String bodyIfApplicable(SdkHttpFullRequest request) { + if (shouldAddBodyToRequest(request) && request.contentStreamProvider().isPresent()) { + try (InputStream is = request.contentStreamProvider().get().newStream()) { + return IoUtils.toUtf8String(is); + } catch (IOException e) { + throw new UncheckedIOException("Failed to set body for S3 sign request", e); + } + } + + return null; + } + + /** + * Only add body for DeleteObjectsRequest. Refer to + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html#API_DeleteObjects_RequestSyntax + */ + private boolean shouldAddBodyToRequest(SdkHttpFullRequest request) { + return request.method() == SdkHttpMethod.POST + && request.rawQueryParameters().containsKey("delete"); + } + private void reconstructHeaders( Map<String, List<String>> signedAndUnsignedHeaders, SdkHttpFullRequest.Builder mutableRequest) { diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java index 1e44e53318..b572b482dc 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java @@ -20,6 +20,7 @@ package org.apache.iceberg.aws.s3.signer; import static org.assertj.core.api.Assertions.assertThat; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import java.util.Map; @@ -56,8 +57,11 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.UploadPartRequest; @@ -177,6 +181,22 @@ public class TestS3RestSigner { PutObjectRequest.builder().bucket(BUCKET).key("some/key").build(), Paths.get("/etc/hosts")); } + @Test + public void validateDeleteObjects() { + Path sourcePath = Paths.get("/etc/hosts"); + s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key("some/key1").build(), sourcePath); + s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key("some/key2").build(), sourcePath); + + Delete objectsToDelete = + Delete.builder() + .objects( + ObjectIdentifier.builder().key("some/key1").build(), + ObjectIdentifier.builder().key("some/key2").build()) + .build(); + + s3.deleteObjects(DeleteObjectsRequest.builder().bucket(BUCKET).delete(objectsToDelete).build()); + } + @Test public void validateListPrefix() { s3.listObjectsV2(ListObjectsV2Request.builder().bucket(BUCKET).prefix("some/prefix/").build());
