This is an automated email from the ASF dual-hosted git repository. amoghj pushed a commit to branch wip-s3sign-body in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit b67a83297317301e1afc5db2230b3640aeab7305 Author: amogh-jahagirdar <[email protected]> AuthorDate: Mon Aug 21 16:35:44 2023 -0700 AWS: Update S3V4RestSignerClient to send body for DeleteObjects requests --- .../aws/s3/signer/S3V4RestSignerClient.java | 34 ++++++++++++++++++++-- .../iceberg/aws/s3/signer/TestS3RestSigner.java | 22 ++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java index a334221a25..703bf9a829 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java @@ -21,6 +21,9 @@ package org.apache.iceberg.aws.s3.signer; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.RemovalListener; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; import java.net.URI; import java.time.Duration; import java.util.Collections; @@ -55,6 +58,8 @@ import software.amazon.awssdk.auth.signer.params.AwsS3V4SignerParams; import software.amazon.awssdk.core.checksums.SdkChecksum; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.utils.IoUtils; @Value.Immutable public abstract class S3V4RestSignerClient @@ -279,15 +284,16 @@ public abstract class S3V4RestSignerClient AwsS3V4SignerParams signerParams = extractSignerParams(AwsS3V4SignerParams.builder(), executionAttributes).build(); - S3SignRequest remoteSigningRequest = + ImmutableS3SignRequest.Builder remoteSigningRequestBuilder = ImmutableS3SignRequest.builder() .method(request.method().name()) .region(signerParams.signingRegion().id()) .uri(request.getUri()) .headers(request.headers()) - .properties(requestPropertiesSupplier().get()) - .build(); + .properties(requestPropertiesSupplier().get()); + addBodyToRequestIfNeeded(request, remoteSigningRequestBuilder); + ImmutableS3SignRequest remoteSigningRequest = remoteSigningRequestBuilder.build(); Key cacheKey = Key.from(remoteSigningRequest); SignedComponent cachedSignedComponent = SIGNED_COMPONENT_CACHE.getIfPresent(cacheKey); SignedComponent signedComponent; @@ -328,6 +334,28 @@ public abstract class S3V4RestSignerClient return mutableRequest.build(); } + private void addBodyToRequestIfNeeded( + SdkHttpFullRequest request, ImmutableS3SignRequest.Builder requestBuilder) { + if (request.contentStreamProvider().isPresent()) { + if (shouldAddBodyToRequest(request)) { + try (InputStream is = request.contentStreamProvider().get().newStream()) { + requestBuilder.body(IoUtils.toUtf8String(is)); + } catch (IOException e) { + throw new UncheckedIOException("Failed to set body for S3 sign request", e); + } + } + } + } + + /** + * Only add body for DeleteObjectsRequest. Refer to + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html#API_DeleteObjects_RequestSyntax + */ + private boolean shouldAddBodyToRequest(SdkHttpFullRequest request) { + return request.method() == SdkHttpMethod.POST + && request.rawQueryParameters().containsKey("delete"); + } + private void reconstructHeaders( Map<String, List<String>> signedAndUnsignedHeaders, SdkHttpFullRequest.Builder mutableRequest) { diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java index 1e44e53318..7972826bea 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java @@ -56,8 +56,11 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.UploadPartRequest; @@ -177,6 +180,25 @@ public class TestS3RestSigner { PutObjectRequest.builder().bucket(BUCKET).key("some/key").build(), Paths.get("/etc/hosts")); } + @Test + public void validateDeleteObjects() { + s3.putObject( + PutObjectRequest.builder().bucket(BUCKET).key("some/key1").build(), + Paths.get("/etc/hosts")); + s3.putObject( + PutObjectRequest.builder().bucket(BUCKET).key("some/key2").build(), + Paths.get("/etc/hosts")); + + Delete objectsToDelete = + Delete.builder() + .objects( + ObjectIdentifier.builder().key("some/key1").build(), + ObjectIdentifier.builder().key("some/key2").build()) + .build(); + + s3.deleteObjects(DeleteObjectsRequest.builder().bucket(BUCKET).delete(objectsToDelete).build()); + } + @Test public void validateListPrefix() { s3.listObjectsV2(ListObjectsV2Request.builder().bucket(BUCKET).prefix("some/prefix/").build());
