This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e9e8b303f9 HDDS-12935. Support unsigned chunked upload with
STREAMING-UNSIGNED-PAYLOAD-TRAILER (#8366)
e9e8b303f9 is described below
commit e9e8b303f9dde94b182b43f564e032f389b7b4e8
Author: Ivan Andika <[email protected]>
AuthorDate: Wed May 21 04:36:00 2025 +0800
HDDS-12935. Support unsigned chunked upload with
STREAMING-UNSIGNED-PAYLOAD-TRAILER (#8366)
---
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 43 ++++
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 88 ++++++++
.../hadoop/ozone/s3/SignedChunksInputStream.java | 86 ++++++--
...tStream.java => UnsignedChunksInputStream.java} | 115 ++++++----
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 84 ++++++--
.../ozone/s3/signature/StringToSignProducer.java | 55 +++--
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 7 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 92 ++++++--
.../ozone/s3/TestSignedChunksInputStream.java | 231 ++++++++++++++++-----
.../ozone/s3/TestUnsignedChunkInputStream.java | 223 ++++++++++++++++++++
.../hadoop/ozone/s3/endpoint/TestListParts.java | 3 +
.../s3/endpoint/TestMultipartUploadComplete.java | 3 +
.../s3/endpoint/TestMultipartUploadWithCopy.java | 5 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 2 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 11 +
.../ozone/s3/endpoint/TestObjectTaggingDelete.java | 3 +
.../ozone/s3/endpoint/TestObjectTaggingGet.java | 3 +
.../ozone/s3/endpoint/TestObjectTaggingPut.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 7 +
.../s3/endpoint/TestPartUploadWithStream.java | 3 +
.../ozone/s3/endpoint/TestPermissionCheck.java | 3 +
.../ozone/s3/endpoint/TestUploadWithStream.java | 2 +
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 3 +
23 files changed, 909 insertions(+), 167 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index ccb922eb31..1b79c5722c 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -29,6 +29,7 @@
import com.amazonaws.AmazonServiceException;
import com.amazonaws.AmazonServiceException.ErrorType;
+import com.amazonaws.HttpMethod;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.AccessControlList;
@@ -37,6 +38,7 @@
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.CreateBucketRequest;
+import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.Grantee;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
@@ -69,17 +71,22 @@
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.services.s3.transfer.model.UploadResult;
+import com.amazonaws.util.IOUtils;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -989,6 +996,42 @@ public void testQuotaExceeded() throws IOException {
assertEquals("QuotaExceeded", ase.getErrorCode());
}
+ @Test
+ public void testPresignedUrlGet() throws IOException {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(bucketName);
+
+ InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+
+ s3Client.putObject(bucketName, keyName, is, new ObjectMetadata());
+
+ // Set the presigned URL to expire after one hour.
+ Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60));
+
+ // Generate the presigned URL
+ GeneratePresignedUrlRequest generatePresignedUrlRequest =
+ new GeneratePresignedUrlRequest(bucketName, keyName)
+ .withMethod(HttpMethod.GET)
+ .withExpiration(expiration);
+ generatePresignedUrlRequest.addRequestParameter("x-custom-parameter",
"custom-value");
+ URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
+
+ // Download the object using HttpUrlConnection (since v1.1)
+ // Capture the response body to a byte array.
+ URL presignedUrl = new URL(url.toExternalForm());
+ HttpURLConnection connection = (HttpURLConnection)
presignedUrl.openConnection();
+ connection.setRequestMethod("GET");
+ // Download the result of executing the request.
+ try (InputStream s3is = connection.getInputStream();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(
+ content.getBytes(StandardCharsets.UTF_8).length)) {
+ IOUtils.copy(s3is, bos);
+ assertEquals(content, bos.toString("UTF-8"));
+ }
+ }
+
private boolean isBucketEmpty(Bucket bucket) {
ObjectListing objectListing = s3Client.listObjects(bucket.getName());
return objectListing.getObjectSummaries().isEmpty();
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 1d1867ae60..3d4ac36203 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -25,14 +25,18 @@
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -64,14 +68,22 @@
import org.junit.jupiter.api.io.TempDir;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.HttpExecuteRequest;
+import software.amazon.awssdk.http.HttpExecuteResponse;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
import
software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
@@ -85,10 +97,14 @@
import software.amazon.awssdk.services.s3.model.Tagging;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.model.UploadPartResponse;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import
software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
+import
software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest;
import software.amazon.awssdk.transfer.s3.model.FileDownload;
import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload;
+import software.amazon.awssdk.utils.IoUtils;
/**
* This is an abstract class to test the AWS Java S3 SDK operations.
@@ -390,6 +406,78 @@ public void testResumableDownloadWithEtagMismatch() throws
Exception {
}
}
+ @Test
+ public void testPresignedUrlGet() throws Exception {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ s3Client.putObject(b -> b
+ .bucket(bucketName)
+ .key(keyName),
+ RequestBody.fromString(content));
+
+ try (S3Presigner presigner = S3Presigner.builder()
+ // TODO: Find a way to retrieve the path style configuration from
S3Client instead
+
.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
+
.endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get())
+ .region(s3Client.serviceClientConfiguration().region())
+
.credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build())
{
+ GetObjectRequest objectRequest = GetObjectRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .build();
+
+ GetObjectPresignRequest presignRequest =
GetObjectPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(10)) // The URL will expire
in 10 minutes.
+ .getObjectRequest(objectRequest)
+ .build();
+
+ PresignedGetObjectRequest presignedRequest =
presigner.presignGetObject(presignRequest);
+
+ // Download the object using HttpUrlConnection (since v1.1)
+ // Capture the response body to a byte array.
+ URL presignedUrl = presignedRequest.url();
+ HttpURLConnection connection = (HttpURLConnection)
presignedUrl.openConnection();
+ connection.setRequestMethod("GET");
+ // Download the result of executing the request.
+ try (InputStream s3is = connection.getInputStream();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(
+ content.getBytes(StandardCharsets.UTF_8).length)) {
+ IoUtils.copy(s3is, bos);
+ assertEquals(content, bos.toString("UTF-8"));
+ }
+
+ // Use the AWS SDK for Java SdkHttpClient class to do the download
+ SdkHttpRequest request = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.GET)
+ .uri(presignedUrl.toURI())
+ .build();
+
+ HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
+ .request(request)
+ .build();
+
+ try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(
+ content.getBytes(StandardCharsets.UTF_8).length)) {
+ HttpExecuteResponse response =
sdkHttpClient.prepareRequest(executeRequest).call();
+ assertTrue(response.responseBody().isPresent(), () -> "The presigned
url download request " +
+ "should have a response body");
+ response.responseBody().ifPresent(
+ abortableInputStream -> {
+ try {
+ IoUtils.copy(abortableInputStream, bos);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ assertEquals(content, bos.toString("UTF-8"));
+ }
+ }
+ }
+
private String getBucketName() {
return getBucketName("");
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
index 725b755913..f3c825db4e 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
@@ -17,21 +17,57 @@
package org.apache.hadoop.ozone.s3;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.eol;
+
import java.io.IOException;
import java.io.InputStream;
+import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
- * Input stream implementation to read body with chunked signatures. This
should also work
+ * Input stream implementation to read body of a signed chunked upload. This
should also work
* with the chunked payloads with trailer.
*
+ * <p>
+ * Example chunk data:
+ * <pre>
+ *
10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n
+ * <65536-bytes>\r\n
+ *
400;chunk-signature=1c1344b170168f8e65b41376b44b20fe354e373826ccbbe2c1d40a8cae51e5c7\r\n
+ * <1024-bytes>\r\n
+ *
0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n
+ * x-amz-checksum-crc32c:sOO8/Q==\r\n
+ *
x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n
+ * </pre>
+ * </p>
+ * For the first chunk 10000 will be read and decoded from base-16
representation to 65536, which is the size of
+ * the first chunk payload. Each chunk upload ends with a zero-byte final
additional chunk.
+ * At the end, there might be a trailer checksum payload and signature,
depending on whether the x-amz-content-sha256
+ * header value contains "-TRAILER" suffix (e.g.
STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER
+ * and STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER) and "x-amz-trailer"
is specified (e.g. x-amz-checksum-crc32c).
+ * <p>
+ *
+ * <p>
+ * The logic is similar to {@link UnsignedChunksInputStream}, but there is a
"chunk-signature" to parse.
+ * </p>
+ *
+ * <p>
* Note that there are no actual chunk signature verification taking place.
The InputStream only
* returns the actual chunk payload from chunked signatures format.
+ * </p>
*
- * See
- * - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
- * -
https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
+ * Reference:
+ * <ul>
+ * <li>
+ * <a
href="https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html";>
+ * Signature Calculation: Transfer Payload in Multiple Chunks</a>
+ * </li>
+ * <li>
+ * <a
href="https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html";>
+ * Signature Calculation: Including Trailing Headers</a>
+ * </li>
+ * </ul>
*/
public class SignedChunksInputStream extends InputStream {
@@ -46,12 +82,21 @@ public class SignedChunksInputStream extends InputStream {
*/
private int remainingData = 0;
+ /**
+ * Every chunked uploads (multiple chunks) contains an additional final
zero-byte
+ * chunk. This can be used as the end-of-file marker.
+ */
+ private boolean isFinalChunkEncountered = false;
+
public SignedChunksInputStream(InputStream inputStream) {
originalStream = inputStream;
}
@Override
public int read() throws IOException {
+ if (isFinalChunkEncountered) {
+ return -1;
+ }
if (remainingData > 0) {
int curr = originalStream.read();
remainingData--;
@@ -63,7 +108,10 @@ public int read() throws IOException {
return curr;
} else {
remainingData = readContentLengthFromHeader();
- if (remainingData == -1) {
+ if (remainingData <= 0) {
+ // there is always a final zero byte chunk so we can stop reading
+ // if we encounter this chunk
+ isFinalChunkEncountered = true;
return -1;
}
return read();
@@ -72,12 +120,14 @@ public int read() throws IOException {
@Override
public int read(byte[] b, int off, int len) throws IOException {
- if (b == null) {
- throw new NullPointerException();
- } else if (off < 0 || len < 0 || len > b.length - off) {
- throw new IndexOutOfBoundsException();
+ Objects.requireNonNull(b, "b == null");
+ if (off < 0 || len < 0 || len > b.length - off) {
+ throw new IndexOutOfBoundsException("Offset=" + off + " and len="
+ + len + " don't match the array length of " + b.length);
} else if (len == 0) {
return 0;
+ } else if (isFinalChunkEncountered) {
+ return -1;
}
int currentOff = off;
int currentLen = len;
@@ -103,7 +153,12 @@ public int read(byte[] b, int off, int len) throws
IOException {
}
} else {
remainingData = readContentLengthFromHeader();
- if (remainingData == -1) {
+ if (remainingData == 0) {
+ // there is always a final zero byte chunk so we can stop reading
+ // if we encounter this chunk
+ isFinalChunkEncountered = true;
+ }
+ if (isFinalChunkEncountered || remainingData == -1) {
break;
}
}
@@ -125,10 +180,9 @@ private int readContentLengthFromHeader() throws
IOException {
prev = curr;
curr = next;
}
- // Example
- // The chunk data sent:
- //
10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2
- // <65536-bytes>
+ // Example of a single chunk data:
+ //
10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n
+ // <65536-bytes>\r\n
//
// 10000 will be read and decoded from base-16 representation to 65536,
which is the size of
// the subsequent chunk payload.
@@ -145,8 +199,4 @@ private int readContentLengthFromHeader() throws
IOException {
throw new IOException("Invalid signature line: " + signatureLine);
}
}
-
- private boolean eol(int prev, int curr) {
- return prev == 13 && curr == 10;
- }
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java
similarity index 51%
copy from
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
copy to
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java
index 725b755913..93565b0d82 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java
@@ -17,41 +17,82 @@
package org.apache.hadoop.ozone.s3;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.eol;
+
import java.io.IOException;
import java.io.InputStream;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import java.util.Objects;
/**
- * Input stream implementation to read body with chunked signatures. This
should also work
- * with the chunked payloads with trailer.
+ * Input stream implementation to read body of an unsigned chunked upload.
+ * <p>
+ * Currently, the only valid value of x-amz-content-sha256 header to indicate
+ * transfer unsigned payload in multiple chunks is
STREAMING-UNSIGNED-PAYLOAD-TRAILER.
+ * Therefore, the input stream should work with chunked payloads with checksum
trailer.
+ * Nevertheless, this input stream also supports chunked upload without
trailer.
+ * </p>
+ * <p>
+ * Example chunk data:
+ * <pre>
+ * 10000\r\n
+ * <65536-bytes>\r\n
+ * 0\r\n
+ * x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n
+ * </pre>
+ * </p>
+ * <p>
+ * The 10000 will be read and decoded from base-16 representation to 65536,
which is the size of
+ * the subsequent chunk payload. Each chunk upload ends with a zero-byte final
additional chunk.
+ * At the end, there will be a trailer checksum payload
+ * </p>
+ *
+ * <p>
+ * The logic is similar to {@link SignedChunksInputStream}, but since it is an
unsigned chunked upload
+ * there is no "chunk-signature" to parse.
+ * </p>
*
- * Note that there are no actual chunk signature verification taking place.
The InputStream only
+ * <p>
+ * Note that there is not actual trailer checksum verification taking place.
The InputStream only
* returns the actual chunk payload from chunked signatures format.
+ * </p>
*
- * See
- * - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
- * -
https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
+ * Reference:
+ * <ul>
+ * <li>
+ * <a
href="https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html";>
+ * Signature Calculation: Transfer Payload in Multiple Chunks</a>
+ * </li>
+ * <li>
+ * <a
href="https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html";>
+ * Signature Calculation: Including Trailing Headers</a>
+ * </li>
+ * </ul>
*/
-public class SignedChunksInputStream extends InputStream {
-
- private final Pattern signatureLinePattern =
- Pattern.compile("([0-9A-Fa-f]+);chunk-signature=.*");
+public class UnsignedChunksInputStream extends InputStream {
private final InputStream originalStream;
/**
- * Size of the chunk payload. If zero, the signature line should be parsed to
+ * Size of the chunk payload. If zero, the content length should be parsed to
* retrieve the subsequent chunk payload size.
*/
private int remainingData = 0;
- public SignedChunksInputStream(InputStream inputStream) {
+ /**
+ * Every chunked uploads (multiple chunks) contains an additional final
zero-byte
+ * chunk. This can be used as the end-of-file marker.
+ */
+ private boolean isFinalChunkEncountered = false;
+
+ public UnsignedChunksInputStream(InputStream inputStream) {
originalStream = inputStream;
}
@Override
public int read() throws IOException {
+ if (isFinalChunkEncountered) {
+ return -1;
+ }
if (remainingData > 0) {
int curr = originalStream.read();
remainingData--;
@@ -63,7 +104,10 @@ public int read() throws IOException {
return curr;
} else {
remainingData = readContentLengthFromHeader();
- if (remainingData == -1) {
+ if (remainingData <= 0) {
+ // since currently trailer checksum verification is not supported, we
can
+ // stop reading after encountering the final zero-byte chunk.
+ isFinalChunkEncountered = true;
return -1;
}
return read();
@@ -72,12 +116,14 @@ public int read() throws IOException {
@Override
public int read(byte[] b, int off, int len) throws IOException {
- if (b == null) {
- throw new NullPointerException();
- } else if (off < 0 || len < 0 || len > b.length - off) {
- throw new IndexOutOfBoundsException();
+ Objects.requireNonNull(b, "b == null");
+ if (off < 0 || len < 0 || len > b.length - off) {
+ throw new IndexOutOfBoundsException("Offset=" + off + " and len="
+ + len + " don't match the array length of " + b.length);
} else if (len == 0) {
return 0;
+ } else if (isFinalChunkEncountered) {
+ return -1;
}
int currentOff = off;
int currentLen = len;
@@ -103,7 +149,12 @@ public int read(byte[] b, int off, int len) throws
IOException {
}
} else {
remainingData = readContentLengthFromHeader();
- if (remainingData == -1) {
+ if (remainingData == 0) {
+ // there is always a final zero byte chunk so we can stop reading
+ // if we encounter this chunk
+ isFinalChunkEncountered = true;
+ }
+ if (isFinalChunkEncountered || remainingData == -1) {
break;
}
}
@@ -125,28 +176,16 @@ private int readContentLengthFromHeader() throws
IOException {
prev = curr;
curr = next;
}
- // Example
- // The chunk data sent:
- //
10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2
- // <65536-bytes>
+ // Example of a single chunk data:
+ // 10000\r\n
+ // <65536-bytes>\r\n
//
// 10000 will be read and decoded from base-16 representation to 65536,
which is the size of
// the subsequent chunk payload.
- String signatureLine = buf.toString().trim();
- if (signatureLine.isEmpty()) {
+ String readString = buf.toString().trim();
+ if (readString.isEmpty()) {
return -1;
}
-
- //parse the data length.
- Matcher matcher = signatureLinePattern.matcher(signatureLine);
- if (matcher.matches()) {
- return Integer.parseInt(matcher.group(1), 16);
- } else {
- throw new IOException("Invalid signature line: " + signatureLine);
- }
- }
-
- private boolean eol(int prev, int curr) {
- return prev == 13 && curr == 10;
+ return Integer.parseInt(readString, 16);
}
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index bf3d0c5992..45e1f9d34e 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -57,7 +57,11 @@
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_DIRECTIVE_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.hasMultiChunksPayload;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.hasUnsignedPayload;
import static org.apache.hadoop.ozone.s3.util.S3Utils.urlDecode;
+import static
org.apache.hadoop.ozone.s3.util.S3Utils.validateMultiChunksUpload;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.validateSignatureHeader;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
@@ -100,6 +104,7 @@
import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.StreamingOutput;
import javax.xml.bind.DatatypeConverter;
+import net.jcip.annotations.Immutable;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -126,6 +131,7 @@
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.s3.HeaderPreprocessor;
import org.apache.hadoop.ozone.s3.SignedChunksInputStream;
+import org.apache.hadoop.ozone.s3.UnsignedChunksInputStream;
import org.apache.hadoop.ozone.s3.endpoint.S3Tagging.Tag;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
@@ -303,17 +309,13 @@ public Response put(
}
// Normal put object
+ S3ChunkInputStreamInfo chunkInputStreamInfo =
getS3ChunkInputStreamInfo(body,
+ length, amzDecodedLength, keyPath);
+ digestInputStream = chunkInputStreamInfo.getDigestInputStream();
+ length = chunkInputStreamInfo.getEffectiveLength();
+
Map<String, String> customMetadata =
getCustomMetadataFromHeaders(headers.getRequestHeaders());
-
- if (S3Utils.hasSignedPayloadHeader(headers)) {
- digestInputStream = new DigestInputStream(new
SignedChunksInputStream(body),
- getMessageDigestInstance());
- length = Long.parseLong(amzDecodedLength);
- } else {
- digestInputStream = new DigestInputStream(body,
getMessageDigestInstance());
- }
-
Map<String, String> tags = getTaggingFromHeaders(headers);
long putLength;
@@ -962,15 +964,11 @@ private Response createMultipartKey(OzoneVolume volume,
String bucket,
String copyHeader = null;
DigestInputStream digestInputStream = null;
try {
-
- if (S3Utils.hasSignedPayloadHeader(headers)) {
- digestInputStream = new DigestInputStream(new
SignedChunksInputStream(body),
- getMessageDigestInstance());
- length = Long.parseLong(
- headers.getHeaderString(DECODED_CONTENT_LENGTH_HEADER));
- } else {
- digestInputStream = new DigestInputStream(body,
getMessageDigestInstance());
- }
+ String amzDecodedLength =
headers.getHeaderString(DECODED_CONTENT_LENGTH_HEADER);
+ S3ChunkInputStreamInfo chunkInputStreamInfo = getS3ChunkInputStreamInfo(
+ body, length, amzDecodedLength, key);
+ digestInputStream = chunkInputStreamInfo.getDigestInputStream();
+ length = chunkInputStreamInfo.getEffectiveLength();
copyHeader = headers.getHeaderString(COPY_SOURCE_HEADER);
String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER);
@@ -1536,4 +1534,54 @@ private int getIOBufferSize(long fileLength) {
return fileLength < bufferSize ? (int) fileLength : bufferSize;
}
}
+
+ /**
+ * Create a {@link S3ChunkInputStreamInfo} that contains the necessary
information to handle
+ * the S3 chunk upload.
+ */
+ private S3ChunkInputStreamInfo getS3ChunkInputStreamInfo(
+ InputStream body, long contentLength, String amzDecodedLength, String
keyPath) throws OS3Exception {
+ final String amzContentSha256Header = validateSignatureHeader(headers,
keyPath);
+ final InputStream chunkInputStream;
+ final long effectiveLength;
+ if (hasMultiChunksPayload(amzContentSha256Header)) {
+ validateMultiChunksUpload(headers, amzDecodedLength, keyPath);
+ if (hasUnsignedPayload(amzContentSha256Header)) {
+ chunkInputStream = new UnsignedChunksInputStream(body);
+ } else {
+ chunkInputStream = new SignedChunksInputStream(body);
+ }
+ effectiveLength = Long.parseLong(amzDecodedLength);
+ } else {
+ // Single chunk upload:
https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+ // Possible x-amz-content-sha256 header values
+ // - Actual payload checksum value: For signed payload
+ // - UNSIGNED-PAYLOAD: For unsigned payload
+ chunkInputStream = body;
+ effectiveLength = contentLength;
+ }
+
+ // DigestInputStream is used for ETag calculation
+ DigestInputStream digestInputStream = new
DigestInputStream(chunkInputStream, getMessageDigestInstance());
+ return new S3ChunkInputStreamInfo(digestInputStream, effectiveLength);
+ }
+
+ @Immutable
+ static final class S3ChunkInputStreamInfo {
+ private final DigestInputStream digestInputStream;
+ private final long effectiveLength;
+
+ S3ChunkInputStreamInfo(DigestInputStream digestInputStream, long
effectiveLength) {
+ this.digestInputStream = digestInputStream;
+ this.effectiveLength = effectiveLength;
+ }
+
+ public DigestInputStream getDigestInputStream() {
+ return digestInputStream;
+ }
+
+ public long getEffectiveLength() {
+ return effectiveLength;
+ }
+ }
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
index 5a474a024e..ea460f62d4 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
@@ -19,7 +19,6 @@
import static java.time.temporal.ChronoUnit.SECONDS;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR;
-import static
org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_UNSIGNED_PAYLOAD_TRAILER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.UNSIGNED_PAYLOAD;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
@@ -201,31 +200,45 @@ public static String buildCanonicalRequest(
validateCanonicalHeaders(canonicalHeaders.toString(), headers,
unsignedPayload);
- String payloadHash;
- if (UNSIGNED_PAYLOAD.equals(headers.get(X_AMZ_CONTENT_SHA256)) ||
-
STREAMING_UNSIGNED_PAYLOAD_TRAILER.equals(headers.get(X_AMZ_CONTENT_SHA256)) ||
- unsignedPayload) {
- payloadHash = UNSIGNED_PAYLOAD;
- } else {
- // According to AWS Sig V4 documentation
- // https://docs.aws.amazon.com/AmazonS3/latest/API/
- // sig-v4-header-based-auth.html
- // Note: The x-amz-content-sha256 header is required
- // for all AWS Signature Version 4 requests.(using Authorization header)
- if (!headers.containsKey(X_AMZ_CONTENT_SHA256)) {
- LOG.error("The request must include " + X_AMZ_CONTENT_SHA256
- + " header for signed payload");
- throw S3_AUTHINFO_CREATION_ERROR;
- }
- payloadHash = headers.get(X_AMZ_CONTENT_SHA256);
- }
- String canonicalRequest = method + NEWLINE
+ String payloadHash = getPayloadHash(headers, unsignedPayload);
+
+ return method + NEWLINE
+ canonicalUri + NEWLINE
+ canonicalQueryStr + NEWLINE
+ canonicalHeaders + NEWLINE
+ signedHeaders + NEWLINE
+ payloadHash;
- return canonicalRequest;
+ }
+
+ private static String getPayloadHash(Map<String, String> headers, boolean
isUsingQueryParameter)
+ throws OS3Exception {
+ if (isUsingQueryParameter) {
+ // According to AWS Signature V4 documentation using Query Parameters
+ //
https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+ return UNSIGNED_PAYLOAD;
+ }
+ String contentSignatureHeaderValue = headers.get(X_AMZ_CONTENT_SHA256);
+ // According to AWS Signature V4 documentation using Authorization Header
+ //
https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+ // The x-amz-content-sha256 header is required
+ // for all AWS Signature Version 4 requests using Authorization header.
+ if (contentSignatureHeaderValue == null) {
+ LOG.error("The request must include " + X_AMZ_CONTENT_SHA256
+ + " header for signed payload");
+ throw S3_AUTHINFO_CREATION_ERROR;
+ }
+ // Simply return the header value of x-amz-content-sha256 as the payload
hash
+ // These are the possible cases:
+ // 1. Actual payload checksum for single chunk upload
+ // 2. Unsigned payloads for multiple chunks upload
+ // - UNSIGNED-PAYLOAD
+ // - STREAMING-UNSIGNED-PAYLOAD-TRAILER
+ // 3. Signed payloads for multiple chunks upload
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
+ return contentSignatureHeaderValue;
}
/**
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 1060f2568c..e5f49383fc 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -32,8 +32,8 @@ public final class S3Consts {
public static final String STORAGE_CLASS_HEADER = "x-amz-storage-class";
public static final String ENCODING_TYPE = "url";
- // Constants related to Signature calculation
- //
https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
+ // Constants related to AWS Signature Version V4 calculation
+ //
https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
public static final String X_AMZ_CONTENT_SHA256 = "x-amz-content-sha256";
public static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
@@ -45,6 +45,9 @@ public final class S3Consts {
public static final String STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD_TRAILER =
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER";
+ public static final String AWS_CHUNKED = "aws-chunked";
+ public static final String MULTI_CHUNKS_UPLOAD_PREFIX = "STREAMING";
+
// Constants related to Range Header
public static final String COPY_SOURCE_IF_PREFIX = "x-amz-copy-source-if-";
public static final String COPY_SOURCE_IF_MODIFIED_SINCE =
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
index 6355f8cd24..54ffc23c32 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
@@ -20,15 +20,19 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
-import static
org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD;
-import static
org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD_TRAILER;
-import static
org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_AWS4_HMAC_SHA256_PAYLOAD;
-import static
org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_AWS4_HMAC_SHA256_PAYLOAD_TRAILER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_CHUNKED;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.MULTI_CHUNKS_UPLOAD_PREFIX;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_UNSIGNED_PAYLOAD_TRAILER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.UNSIGNED_PAYLOAD;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
+import jakarta.annotation.Nonnull;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
+import java.util.Arrays;
+import java.util.Objects;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
@@ -38,6 +42,7 @@
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
/**
* Utilities.
@@ -132,17 +137,78 @@ public static WebApplicationException
wrapOS3Exception(OS3Exception ex) {
.build());
}
- public static boolean hasSignedPayloadHeader(HttpHeaders headers) {
- final String signingAlgorithm =
headers.getHeaderString(X_AMZ_CONTENT_SHA256);
- if (signingAlgorithm == null) {
- return false;
+ public static boolean hasUnsignedPayload(@Nonnull String
amzContentSha256Header) {
+ Objects.requireNonNull(amzContentSha256Header);
+ return amzContentSha256Header.equals(UNSIGNED_PAYLOAD) ||
+ amzContentSha256Header.equals(STREAMING_UNSIGNED_PAYLOAD_TRAILER);
+ }
+
+ public static boolean hasMultiChunksPayload(@Nonnull String
amzContentSha256Header) {
+ Objects.requireNonNull(amzContentSha256Header);
+ // Multiple chunk uploads
+ // - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+ // -
https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
+ // Possible values
+ // - STREAMING-UNSIGNED-PAYLOAD-TRAILER
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
+ // Currently since all the multi chunks values have x-amz-content-sha256
header value that starts
+ // with STREAMING, we can use this prefix to differentiates between multi
chunks and single chunks upload.
+ // In the future if there are more multi chunks signature algorithms that
has the same prefix,
+ // this function will be able to handle detect it.
+ return amzContentSha256Header.startsWith(MULTI_CHUNKS_UPLOAD_PREFIX);
+ }
+
+ public static void validateMultiChunksUpload(HttpHeaders headers, String
amzDecodedContentLength,
+ String resource) throws
OS3Exception {
+ final String contentEncoding =
headers.getHeaderString(HttpHeaders.CONTENT_ENCODING);
+ // "Content-Encoding : aws-chunked" seems to only be sent for SDK V2, so
ignore if there is no
+ // Content-Encoding header
+ if (contentEncoding != null) {
+ // Amazon S3 supports multiple content encoding values for example
"Content-Encoding : aws-chunked,gzip"
+ // We are only interested on "aws-chunked"
+ boolean containsAwsChunked = Arrays.stream(contentEncoding.split(","))
+ .map(String::trim)
+ .anyMatch(AWS_CHUNKED::equals);
+ if (!containsAwsChunked) {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
resource);
+ ex.setErrorMessage("An error occurred (InvalidArgument) for multi
chunks upload: " +
+ "The " + HttpHeaders.CONTENT_ENCODING + " header does not contain
" + AWS_CHUNKED);
+ throw ex;
+ }
+ }
+
+ if (amzDecodedContentLength == null) {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
resource);
+ ex.setErrorMessage("An error occurred (InvalidArgument) for multi chunks
upload: " +
+ "The " + DECODED_CONTENT_LENGTH_HEADER + " header is not specified");
+ throw ex;
}
+ }
- // Handles both AWS Signature Version 4 (HMAC-256) and AWS Signature
Version 4A (ECDSA-P256-SHA256)
- return signingAlgorithm.equals(STREAMING_AWS4_HMAC_SHA256_PAYLOAD) ||
- signingAlgorithm.equals(STREAMING_AWS4_HMAC_SHA256_PAYLOAD_TRAILER) ||
- signingAlgorithm.equals(STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD) ||
-
signingAlgorithm.equals(STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD_TRAILER);
+ public static String validateSignatureHeader(HttpHeaders headers, String
resource) throws OS3Exception {
+ String xAmzContentSha256Header =
headers.getHeaderString(X_AMZ_CONTENT_SHA256);
+ if (xAmzContentSha256Header == null) {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
resource);
+ ex.setErrorMessage("An error occurred (InvalidArgument): " +
+ "The " + X_AMZ_CONTENT_SHA256 + " header is not specified");
+ throw ex;
+ }
+
+ return xAmzContentSha256Header;
+ }
+
+ /**
+ * Checks if the given pair of bytes represent the end-of-line sequence
(\r\n).
+ *
+ * @param prev the previous byte value (should be 13 for '\r')
+ * @param curr the current byte value (should be 10 for '\n')
+ * @return true if the pair forms a CRLF sequence, false otherwise
+ */
+ public static boolean eol(int prev, int curr) {
+ return prev == 13 && curr == 10;
}
/**
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
index cf4334ab4d..8706840988 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
@@ -27,90 +27,209 @@
import org.junit.jupiter.api.Test;
/**
- * Test input stream parsing with signatures.
+ * Test {@link SignedChunksInputStream}.
*/
public class TestSignedChunksInputStream {
@Test
- public void emptyfile() throws IOException {
- InputStream is = fileContent("0;chunk-signature"
- +
- "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40");
- String result = IOUtils.toString(is, UTF_8);
- assertEquals("", result);
-
- is = fileContent("0;chunk-signature"
- +
- "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
- + "\n");
- result = IOUtils.toString(is, UTF_8);
- assertEquals("", result);
+ void testEmptyFile() throws IOException {
+ try (InputStream is = wrapContent("0;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n")) {
+ assertEquals("", IOUtils.toString(is, UTF_8));
+ }
}
@Test
- public void singlechunk() throws IOException {
+ void testEmptyFileWithTrailer() throws IOException {
+ try (InputStream is = wrapContent("0;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "x-amz-checksum-crc32c:sOO8/Q==\r\n"
+ +
"x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n"))
{
+ assertEquals("", IOUtils.toString(is, UTF_8));
+ }
+ }
+
+ @Test
+ void testEmptyFileWithoutEnd() throws IOException {
+ try (InputStream is = wrapContent("0;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40")) {
+ assertEquals("", IOUtils.toString(is, UTF_8));
+ }
+ }
+
+ @Test
+ void testSingleChunk() throws IOException {
+ //test simple read()
+ try (InputStream is = wrapContent("0A;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "1234567890\r\n")) {
+ assertEquals("1234567890", IOUtils.toString(is, UTF_8));
+ }
+
+ //test read(byte[],int,int)
+ try (InputStream is = wrapContent("0A;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "1234567890\r\n")) {
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ assertEquals("1234567890", new String(bytes, UTF_8));
+ }
+
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0A;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "1234567890\r\n")) {
+ byte[] bytes = new byte[10];
+ int readLength = IOUtils.read(is, bytes, 0, 10);
+ assertEquals(10, readLength);
+ assertEquals("1234567890", new String(bytes, UTF_8));
+ }
+ }
+
+ @Test
+ void testSingleChunkWithTrailer() throws IOException {
//test simple read()
- InputStream is = fileContent("0A;chunk-signature"
- +
- "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
- + "\n1234567890\r\n");
- String result = IOUtils.toString(is, UTF_8);
- assertEquals("1234567890", result);
+ try (InputStream is = wrapContent("0A;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "1234567890\r\n"
+ + "0;chunk-signature=signature\r\n"
+ + "x-amz-checksum-crc32c:sOO8/Q==\r\n"
+ +
"x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n"))
{
+ assertEquals("1234567890", IOUtils.toString(is, UTF_8));
+ }
+
+ //test read(byte[],int,int)
+ try (InputStream is = wrapContent("0A;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "1234567890\r\n"
+ + "0;chunk-signature=signature\r\n"
+ + "x-amz-checksum-crc32c:sOO8/Q==\r\n"
+ +
"x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n"))
{
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ assertEquals("1234567890", new String(bytes, UTF_8));
+ }
+
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0A;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "1234567890\r\n"
+ + "0;chunk-signature=signature\r\n"
+ + "x-amz-checksum-crc32c:sOO8/Q==\r\n"
+ +
"x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n"))
{
+ byte[] bytes = new byte[10];
+ int readLength = IOUtils.read(is, bytes, 0, 10);
+ assertEquals(10, readLength);
+ assertEquals("1234567890", new String(bytes, UTF_8));
+ }
+ }
+ @Test
+ void testSingleChunkWithoutEnd() throws IOException {
+ //test simple read()
+ try (InputStream is = wrapContent("0A;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "1234567890")) {
+ assertEquals("1234567890", IOUtils.toString(is, UTF_8));
+ }
//test read(byte[],int,int)
- is = fileContent("0A;chunk-signature"
- +
- "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
- + "\n1234567890\r\n");
- byte[] bytes = new byte[10];
- IOUtils.read(is, bytes, 0, 10);
- assertEquals("1234567890",
- new String(bytes, UTF_8));
+ try (InputStream is = wrapContent("0A;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "1234567890")) {
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ assertEquals("1234567890", new String(bytes, UTF_8));
+ }
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0A;chunk-signature"
+ +
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n"
+ + "1234567890")) {
+ byte[] bytes = new byte[15];
+ int readLength = IOUtils.read(is, bytes, 0, 15);
+ assertEquals(10, readLength);
+ assertEquals("1234567890", new String(bytes, UTF_8).substring(0, 10));
+ }
}
@Test
- public void singlechunkwithoutend() throws IOException {
+ void testMultiChunks() throws IOException {
//test simple read()
- InputStream is = fileContent("0A;chunk-signature"
- +
- "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
- + "\n1234567890");
- String result = IOUtils.toString(is, UTF_8);
- assertEquals("1234567890", result);
+ try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n"
+ + "1234567890\r\n"
+ + "05;chunk-signature=signature\r\n"
+ + "abcde\r\n"
+ + "0;chunk-signature=signature\r\n")) {
+ String result = IOUtils.toString(is, UTF_8);
+ assertEquals("1234567890abcde", result);
+ }
//test read(byte[],int,int)
- is = fileContent("0A;chunk-signature"
- +
- "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
- + "\n1234567890");
- byte[] bytes = new byte[10];
- IOUtils.read(is, bytes, 0, 10);
- assertEquals("1234567890",
- new String(bytes, UTF_8));
+ try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n"
+ + "1234567890\r\n"
+ + "05;chunk-signature=signature\r\n"
+ + "abcde\r\n"
+ + "0;chunk-signature=signature\r\n")) {
+ byte[] bytes = new byte[15];
+ IOUtils.read(is, bytes, 0, 15);
+ assertEquals("1234567890abcde", new String(bytes, UTF_8));
+ }
+
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n"
+ + "1234567890\r\n"
+ + "05;chunk-signature=signature\r\n"
+ + "abcde\r\n"
+ + "0;chunk-signature=signature\r\n")) {
+ byte[] bytes = new byte[20];
+ int readLength = IOUtils.read(is, bytes, 0, 20);
+ assertEquals(15, readLength);
+ assertEquals("1234567890abcde", new String(bytes, UTF_8).substring(0,
15));
+ }
}
@Test
- public void multichunks() throws IOException {
+ void testMultiChunksWithTrailer() throws Exception {
//test simple read()
- InputStream is = fileContent("0a;chunk-signature=signature\r\n"
+ try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n"
+ "1234567890\r\n"
+ "05;chunk-signature=signature\r\n"
- + "abcde\r\n");
- String result = IOUtils.toString(is, UTF_8);
- assertEquals("1234567890abcde", result);
+ + "abcde\r\n"
+ + "0;chunk-signature=signature\r\n"
+ + "x-amz-checksum-crc32c:sOO8/Q==\r\n"
+ +
"x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n"))
{
+ String result = IOUtils.toString(is, UTF_8);
+ assertEquals("1234567890abcde", result);
+ }
//test read(byte[],int,int)
- is = fileContent("0a;chunk-signature=signature\r\n"
+ try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n"
+ + "1234567890\r\n"
+ + "05;chunk-signature=signature\r\n"
+ + "abcde\r\n"
+ + "0;chunk-signature=signature\r\n"
+ + "x-amz-checksum-crc32c:sOO8/Q==\r\n"
+ +
"x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n"))
{
+ byte[] bytes = new byte[15];
+ IOUtils.read(is, bytes, 0, 15);
+ assertEquals("1234567890abcde", new String(bytes, UTF_8));
+ }
+
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n"
+ "1234567890\r\n"
+ "05;chunk-signature=signature\r\n"
- + "abcde\r\n");
- byte[] bytes = new byte[15];
- IOUtils.read(is, bytes, 0, 15);
- assertEquals("1234567890abcde",
- new String(bytes, UTF_8));
+ + "abcde\r\n"
+ + "0;chunk-signature=signature\r\n"
+ + "x-amz-checksum-crc32c:sOO8/Q==\r\n"
+ +
"x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n"))
{
+ byte[] bytes = new byte[20];
+ int readLength = IOUtils.read(is, bytes, 0, 20);
+ assertEquals(15, readLength);
+ assertEquals("1234567890abcde", new String(bytes, UTF_8).substring(0,
15));
+ }
}
- private InputStream fileContent(String content) {
+ private InputStream wrapContent(String content) {
return new SignedChunksInputStream(
new ByteArrayInputStream(content.getBytes(UTF_8)));
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestUnsignedChunkInputStream.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestUnsignedChunkInputStream.java
new file mode 100644
index 0000000000..0f468923cf
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestUnsignedChunkInputStream.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test {@link UnsignedChunksInputStream}.
+ */
+public class TestUnsignedChunkInputStream {
+
+ @Test
+ void testEmptyFile() throws IOException {
+ try (InputStream is = wrapContent("0\r\n")) {
+ assertEquals("", IOUtils.toString(is, UTF_8));
+ }
+ }
+
+ @Test
+ void testEmptyFileWithTrailer() throws IOException {
+ try (InputStream is = wrapContent("0\r\n"
+ + "x-amz-checksum-crc64nvme:AAAAAAAAAAA=\r\n")) {
+ assertEquals("", IOUtils.toString(is, UTF_8));
+ }
+ }
+
+ @Test
+ public void testEmptyFileWithoutEnd() throws IOException {
+ try (InputStream is = wrapContent("0\r\n"
+ + "x-amz-checksum-crc64nvme:AAAAAAAAAAA=")) {
+ assertEquals("", IOUtils.toString(is, UTF_8));
+ }
+ }
+
+ @Test
+ void testSingleChunk() throws IOException {
+ //test simple read()
+ try (InputStream is = wrapContent("0A\r\n"
+ + "1234567890\r\n"
+ + "0\r\n")) {
+ assertEquals("1234567890", IOUtils.toString(is, UTF_8));
+ }
+
+ //test read(byte[],int,int)
+ try (InputStream is = wrapContent("0A\r\n"
+ + "1234567890\r\n"
+ + "0\r\n")) {
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ assertEquals("1234567890", new String(bytes, UTF_8));
+ }
+
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0A\r\n"
+ + "1234567890\r\n"
+ + "0\r\n")) {
+ byte[] bytes = new byte[15];
+ int readLength = IOUtils.read(is, bytes, 0, 15);
+ assertEquals(10, readLength);
+ assertEquals("1234567890", new String(bytes, UTF_8).substring(0, 10));
+ }
+ }
+
+ @Test
+ void testSingleChunkWithTrailer() throws IOException {
+ try (InputStream is = wrapContent("0A\r\n"
+ + "1234567890\r\n"
+ + "0\r\n"
+ + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n")) {
+ assertEquals("1234567890", IOUtils.toString(is, UTF_8));
+ }
+
+ //test read(byte[],int,int)
+ try (InputStream is = wrapContent("0A\r\n"
+ + "1234567890\r\n"
+ + "0\r\n"
+ + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n")) {
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ assertEquals("1234567890", new String(bytes, UTF_8));
+ }
+
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0A\r\n"
+ + "1234567890\r\n"
+ + "0\r\n"
+ + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n")) {
+ byte[] bytes = new byte[15];
+ int readLength = IOUtils.read(is, bytes, 0, 15);
+ assertEquals(10, readLength);
+ assertEquals("1234567890", new String(bytes, UTF_8).substring(0, 10));
+ }
+ }
+
+ @Test
+ void testSingleChunkWithoutEnd() throws IOException {
+ try (InputStream is = wrapContent("0A\r\n"
+ + "1234567890\r\n"
+ + "0")) {
+ assertEquals("1234567890", IOUtils.toString(is, UTF_8));
+ }
+ //test read(byte[],int,int)
+ try (InputStream is = wrapContent("0A\r\n"
+ + "1234567890\r\n"
+ + "0")) {
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ assertEquals("1234567890", new String(bytes, UTF_8));
+ }
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0A\r\n"
+ + "1234567890\r\n"
+ + "0")) {
+ byte[] bytes = new byte[15];
+ int readLength = IOUtils.read(is, bytes, 0, 10);
+ assertEquals(10, readLength);
+ assertEquals("1234567890", new String(bytes, UTF_8).substring(0, 10));
+ }
+ }
+
+ @Test
+ void testMultiChunks() throws IOException {
+ //test simple read()
+ try (InputStream is = wrapContent("0a\r\n"
+ + "1234567890\r\n"
+ + "05\r\n"
+ + "abcde\r\n"
+ + "0\r\n")) {
+ String result = IOUtils.toString(is, UTF_8);
+ assertEquals("1234567890abcde", result);
+ }
+
+ //test read(byte[],int,int)
+ try (InputStream is = wrapContent("0a\r\n"
+ + "1234567890\r\n"
+ + "05\r\n"
+ + "abcde\r\n"
+ + "0\r\n")) {
+ byte[] bytes = new byte[15];
+ IOUtils.read(is, bytes, 0, 15);
+ assertEquals("1234567890abcde", new String(bytes, UTF_8));
+ }
+
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0a\r\n"
+ + "1234567890\r\n"
+ + "05\r\n"
+ + "abcde\r\n"
+ + "0\r\n")) {
+ byte[] bytes = new byte[20];
+ int readLength = IOUtils.read(is, bytes, 0, 20);
+ assertEquals(15, readLength);
+ assertEquals("1234567890abcde", new String(bytes, UTF_8).substring(0,
15));
+ }
+ }
+
+ @Test
+ void testMultiChunksWithTrailer() throws IOException {
+ //test simple read()
+ try (InputStream is = wrapContent("0a\r\n"
+ + "1234567890\r\n"
+ + "05\r\n"
+ + "abcde\r\n"
+ + "0\r\n"
+ + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n")) {
+ String result = IOUtils.toString(is, UTF_8);
+ assertEquals("1234567890abcde", result);
+ }
+
+ //test read(byte[],int,int)
+ try (InputStream is = wrapContent("0a\r\n"
+ + "1234567890\r\n"
+ + "05\r\n"
+ + "abcde\r\n"
+ + "0\r\n"
+ + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\n")) {
+ byte[] bytes = new byte[15];
+ IOUtils.read(is, bytes, 0, 15);
+ assertEquals("1234567890abcde", new String(bytes, UTF_8));
+ }
+
+ //test read(byte[],int,int) with length parameter larger than the payload
+ try (InputStream is = wrapContent("0a\r\n"
+ + "1234567890\r\n"
+ + "05\r\n"
+ + "abcde\r\n"
+ + "0\r\n"
+ + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\n")) {
+ byte[] bytes = new byte[20];
+ int readLength = IOUtils.read(is, bytes, 0, 20);
+ assertEquals(15, readLength);
+ assertEquals("1234567890abcde", new String(bytes, UTF_8).substring(0,
15));
+ }
+ }
+
+ private InputStream wrapContent(String content) {
+ return new UnsignedChunksInputStream(
+ new ByteArrayInputStream(content.getBytes(UTF_8)));
+ }
+
+}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
index 0800b81bb7..30be715b53 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
@@ -19,6 +19,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -52,6 +53,8 @@ public void setUp() throws Exception {
client.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET);
HttpHeaders headers = mock(HttpHeaders.class);
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
"STANDARD");
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
index 46a141df74..fde336f480 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
@@ -20,6 +20,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -64,6 +65,8 @@ public void setUp() throws Exception {
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
"STANDARD");
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
rest = EndpointBuilder.newObjectEndpointBuilder()
.setHeaders(headers)
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
index f1321820f1..5189ddf39a 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
@@ -23,6 +23,7 @@
import static
org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_MODIFIED_SINCE;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_UNMODIFIED_SINCE;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;
@@ -121,6 +122,8 @@ public static void setUp() throws Exception {
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
"STANDARD");
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
REST.setHeaders(headers);
REST.setClient(CLIENT);
@@ -434,6 +437,8 @@ private void setHeaders(Map<String, String>
additionalHeaders) {
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
"STANDARD");
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
additionalHeaders
.forEach((k, v) -> when(headers.getHeaderString(k)).thenReturn(v));
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
index 9bf5f27ddd..3e772f8b8b 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
@@ -23,6 +23,7 @@
import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -83,6 +84,7 @@ public void init() throws OS3Exception, IOException {
client.getObjectStore().createS3Bucket(BUCKET_NAME);
headers = mock(HttpHeaders.class);
+
when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
rest = EndpointBuilder.newObjectEndpointBuilder()
.setClient(client)
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 4f22bd4184..673fb6a756 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -30,6 +30,7 @@
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_KEY_LENGTH_LIMIT;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_NUM_LIMIT;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_VALUE_LENGTH_LIMIT;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.apache.hadoop.ozone.s3.util.S3Utils.urlEncode;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -127,6 +128,7 @@ void setup() throws IOException {
clientStub.getObjectStore().createS3Bucket(DEST_BUCKET_NAME);
headers = mock(HttpHeaders.class);
+
when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
// Create PutObject and setClient to OzoneClientStub
objectEndpoint = EndpointBuilder.newObjectEndpointBuilder()
@@ -208,6 +210,7 @@ void testPutObjectContentLengthForStreaming()
@Test
public void testPutObjectWithTags() throws IOException, OS3Exception {
HttpHeaders headersWithTags = Mockito.mock(HttpHeaders.class);
+
when(headersWithTags.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
when(headersWithTags.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2");
ByteArrayInputStream body =
@@ -232,6 +235,7 @@ public void testPutObjectWithOnlyTagKey() throws Exception {
ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
HttpHeaders headerWithOnlyTagKey = Mockito.mock(HttpHeaders.class);
+
when(headerWithOnlyTagKey.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
// Try to send with only the key (no value)
when(headerWithOnlyTagKey.getHeaderString(TAG_HEADER)).thenReturn("tag1");
objectEndpoint.setHeaders(headerWithOnlyTagKey);
@@ -252,6 +256,7 @@ public void testPutObjectWithDuplicateTagKey() throws
Exception {
ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
HttpHeaders headersWithDuplicateTagKey = Mockito.mock(HttpHeaders.class);
+
when(headersWithDuplicateTagKey.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
when(headersWithDuplicateTagKey.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag1=value2");
objectEndpoint.setHeaders(headersWithDuplicateTagKey);
try {
@@ -270,6 +275,7 @@ public void testPutObjectWithLongTagKey() throws Exception {
ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
HttpHeaders headersWithLongTagKey = Mockito.mock(HttpHeaders.class);
+
when(headersWithLongTagKey.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
String longTagKey = StringUtils.repeat('k', TAG_KEY_LENGTH_LIMIT + 1);
when(headersWithLongTagKey.getHeaderString(TAG_HEADER)).thenReturn(longTagKey +
"=value1");
objectEndpoint.setHeaders(headersWithLongTagKey);
@@ -289,6 +295,7 @@ public void testPutObjectWithLongTagValue() throws
Exception {
ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
HttpHeaders headersWithLongTagValue = Mockito.mock(HttpHeaders.class);
+
when(headersWithLongTagValue.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
objectEndpoint.setHeaders(headersWithLongTagValue);
String longTagValue = StringUtils.repeat('v', TAG_VALUE_LENGTH_LIMIT + 1);
when(headersWithLongTagValue.getHeaderString(TAG_HEADER)).thenReturn("tag1=" +
longTagValue);
@@ -308,6 +315,7 @@ public void testPutObjectWithTooManyTags() throws Exception
{
ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
HttpHeaders headersWithTooManyTags = Mockito.mock(HttpHeaders.class);
+
when(headersWithTooManyTags.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
StringBuilder sb = new StringBuilder();
for (int i = 0; i < TAG_NUM_LIMIT + 1; i++) {
sb.append(String.format("tag%d=value%d", i, i));
@@ -579,6 +587,7 @@ public void
testCopyObjectMessageDigestResetDuringException() throws IOException
public void testCopyObjectWithTags() throws IOException, OS3Exception {
// Put object in to source bucket
HttpHeaders headersForPut = Mockito.mock(HttpHeaders.class);
+
when(headersForPut.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
when(headersForPut.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2");
ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
@@ -600,6 +609,7 @@ public void testCopyObjectWithTags() throws IOException,
OS3Exception {
// Copy object without x-amz-tagging-directive (default to COPY)
String destKey = "key=value/2";
HttpHeaders headersForCopy = Mockito.mock(HttpHeaders.class);
+
when(headersForCopy.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
when(headersForCopy.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
BUCKET_NAME + "/" + urlEncode(sourceKeyName));
@@ -738,6 +748,7 @@ void testDirectoryCreationOverFile() throws IOException,
OS3Exception {
@Test
public void testPutEmptyObject() throws IOException, OS3Exception {
HttpHeaders headersWithTags = Mockito.mock(HttpHeaders.class);
+
when(headersWithTags.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
String emptyString = "";
ByteArrayInputStream body = new
ByteArrayInputStream(emptyString.getBytes(UTF_8));
objectEndpoint.setHeaders(headersWithTags);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java
index 1d33634879..04c4bfd65c 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java
@@ -25,6 +25,7 @@
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -83,6 +84,8 @@ public void init() throws OS3Exception, IOException {
body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
// Create a key with object tags
Mockito.when(headers.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2");
+ Mockito.when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
rest.put(BUCKET_NAME, KEY_WITH_TAG, CONTENT.length(),
1, null, null, null, body);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java
index 8b5ffcb3b2..c4eb4c25ff 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java
@@ -23,6 +23,7 @@
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;
@@ -61,6 +62,8 @@ public void init() throws OS3Exception, IOException {
client.getObjectStore().createS3Bucket(BUCKET_NAME);
HttpHeaders headers = Mockito.mock(HttpHeaders.class);
+ Mockito.when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
rest = EndpointBuilder.newObjectEndpointBuilder()
.setClient(client)
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java
index de698116f5..02b71e8772 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java
@@ -26,6 +26,7 @@
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.doThrow;
@@ -75,6 +76,7 @@ void setup() throws IOException, OS3Exception {
clientStub.getObjectStore().createS3Bucket(BUCKET_NAME);
HttpHeaders headers = mock(HttpHeaders.class);
+
when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
// Create PutObject and setClient to OzoneClientStub
objectEndpoint = EndpointBuilder.newObjectEndpointBuilder()
@@ -83,7 +85,7 @@ void setup() throws IOException, OS3Exception {
.setHeaders(headers)
.build();
-
+
ByteArrayInputStream body =
new ByteArrayInputStream("".getBytes(UTF_8));
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
index 7a3faf6bd5..4981069528 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
@@ -21,6 +21,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -71,6 +72,8 @@ public void setUp() throws Exception {
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
"STANDARD");
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
rest = EndpointBuilder.newObjectEndpointBuilder()
.setHeaders(headers)
@@ -148,6 +151,8 @@ public void testPartUploadWithIncorrectUploadID() throws
Exception {
public void testPartUploadStreamContentLength()
throws IOException, OS3Exception {
HttpHeaders headers = mock(HttpHeaders.class);
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
ObjectEndpoint objectEndpoint = EndpointBuilder.newObjectEndpointBuilder()
.setHeaders(headers)
.setClient(client)
@@ -206,6 +211,8 @@ public void
testPartUploadMessageDigestResetDuringException() throws IOException
HttpHeaders headers = mock(HttpHeaders.class);
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
"STANDARD");
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java
index 2e3025e0e9..4b2d8a49ef 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java
@@ -20,6 +20,7 @@
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -58,6 +59,8 @@ public void setUp() throws Exception {
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD");
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
OzoneConfiguration conf = new OzoneConfiguration();
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
index 0e4319d695..9b34b6ec86 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
@@ -19,6 +19,7 @@
import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -87,6 +88,8 @@ public void setup() {
when(client.getObjectStore()).thenReturn(objectStore);
when(client.getConfiguration()).thenReturn(conf);
headers = mock(HttpHeaders.class);
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
clientProtocol = mock(ClientProtocol.class);
S3GatewayMetrics.create(conf);
when(client.getProxy()).thenReturn(clientProtocol);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
index 1fcb5d8d40..02026c2ef9 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
@@ -21,6 +21,7 @@
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
@@ -69,6 +70,7 @@ public void setUp() throws Exception {
client.getObjectStore().createS3Bucket(S3BUCKET);
HttpHeaders headers = mock(HttpHeaders.class);
+
when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature");
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD");
OzoneConfiguration conf = new OzoneConfiguration();
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
index 53a8736bdd..63465ef755 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
@@ -23,6 +23,7 @@
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_EXISTS;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.apache.hadoop.ozone.s3.util.S3Utils.urlEncode;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
@@ -97,6 +98,8 @@ public void setup() throws Exception {
headers = mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
"STANDARD");
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("mockSignature");
keyEndpoint.setHeaders(headers);
metrics = bucketEndpoint.getMetrics();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
