This is an automated email from the ASF dual-hosted git repository.
amoghj pushed a commit to branch wip-s3sign-body
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/wip-s3sign-body by this push:
new b48b0fb13a AWS: Update S3V4RestSignerClient to send body for
DeleteObjects requests
b48b0fb13a is described below
commit b48b0fb13abcbdab318ea91515a89338505ea549
Author: amogh-jahagirdar <[email protected]>
AuthorDate: Mon Aug 21 16:35:44 2023 -0700
AWS: Update S3V4RestSignerClient to send body for DeleteObjects requests
---
.../aws/s3/signer/S3V4RestSignerClient.java | 34 ++++++++++++++++++++--
.../iceberg/aws/s3/signer/TestS3RestSigner.java | 22 ++++++++++++++
2 files changed, 53 insertions(+), 3 deletions(-)
diff --git
a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
index a334221a25..703bf9a829 100644
---
a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
+++
b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
@@ -21,6 +21,9 @@ package org.apache.iceberg.aws.s3.signer;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalListener;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
import java.net.URI;
import java.time.Duration;
import java.util.Collections;
@@ -55,6 +58,8 @@ import
software.amazon.awssdk.auth.signer.params.AwsS3V4SignerParams;
import software.amazon.awssdk.core.checksums.SdkChecksum;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.http.SdkHttpFullRequest;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.utils.IoUtils;
@Value.Immutable
public abstract class S3V4RestSignerClient
@@ -279,15 +284,16 @@ public abstract class S3V4RestSignerClient
AwsS3V4SignerParams signerParams =
extractSignerParams(AwsS3V4SignerParams.builder(),
executionAttributes).build();
- S3SignRequest remoteSigningRequest =
+ ImmutableS3SignRequest.Builder remoteSigningRequestBuilder =
ImmutableS3SignRequest.builder()
.method(request.method().name())
.region(signerParams.signingRegion().id())
.uri(request.getUri())
.headers(request.headers())
- .properties(requestPropertiesSupplier().get())
- .build();
+ .properties(requestPropertiesSupplier().get());
+ addBodyToRequestIfNeeded(request, remoteSigningRequestBuilder);
+ ImmutableS3SignRequest remoteSigningRequest =
remoteSigningRequestBuilder.build();
Key cacheKey = Key.from(remoteSigningRequest);
SignedComponent cachedSignedComponent =
SIGNED_COMPONENT_CACHE.getIfPresent(cacheKey);
SignedComponent signedComponent;
@@ -328,6 +334,28 @@ public abstract class S3V4RestSignerClient
return mutableRequest.build();
}
+ private void addBodyToRequestIfNeeded(
+ SdkHttpFullRequest request, ImmutableS3SignRequest.Builder
requestBuilder) {
+ if (request.contentStreamProvider().isPresent()) {
+ if (shouldAddBodyToRequest(request)) {
+ try (InputStream is =
request.contentStreamProvider().get().newStream()) {
+ requestBuilder.body(IoUtils.toUtf8String(is));
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to set body for S3 sign
request", e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Only add body for DeleteObjectsRequest. Refer to
+ *
https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html#API_DeleteObjects_RequestSyntax
+ */
+ private boolean shouldAddBodyToRequest(SdkHttpFullRequest request) {
+ return request.method() == SdkHttpMethod.POST
+ && request.rawQueryParameters().containsKey("delete");
+ }
+
private void reconstructHeaders(
Map<String, List<String>> signedAndUnsignedHeaders,
SdkHttpFullRequest.Builder mutableRequest) {
diff --git
a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
index 1e44e53318..7972826bea 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
@@ -56,8 +56,11 @@ import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
@@ -177,6 +180,25 @@ public class TestS3RestSigner {
PutObjectRequest.builder().bucket(BUCKET).key("some/key").build(),
Paths.get("/etc/hosts"));
}
+ @Test
+ public void validateDeleteObjects() {
+ s3.putObject(
+ PutObjectRequest.builder().bucket(BUCKET).key("some/key1").build(),
+ Paths.get("/etc/hosts"));
+ s3.putObject(
+ PutObjectRequest.builder().bucket(BUCKET).key("some/key2").build(),
+ Paths.get("/etc/hosts"));
+
+ Delete objectsToDelete =
+ Delete.builder()
+ .objects(
+ ObjectIdentifier.builder().key("some/key1").build(),
+ ObjectIdentifier.builder().key("some/key2").build())
+ .build();
+
+
s3.deleteObjects(DeleteObjectsRequest.builder().bucket(BUCKET).delete(objectsToDelete).build());
+ }
+
@Test
public void validateListPrefix() {
s3.listObjectsV2(ListObjectsV2Request.builder().bucket(BUCKET).prefix("some/prefix/").build());
