This is an automated email from the ASF dual-hosted git repository.
peterxcli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b63347345a2 HDDS-14346. Reduce duplication in multipart upload unit
tests (#9588)
b63347345a2 is described below
commit b63347345a25f009fc5d453fc758ad2a3dcfced6
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Jan 6 08:37:26 2026 +0100
HDDS-14346. Reduce duplication in multipart upload unit tests (#9588)
---
.../ozone/s3/endpoint/EndpointTestUtils.java | 64 +++++++
.../s3/endpoint/TestAbortMultipartUpload.java | 13 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 27 +--
.../hadoop/ozone/s3/endpoint/TestListParts.java | 34 +---
.../s3/endpoint/TestMultipartUploadComplete.java | 136 ++------------
.../s3/endpoint/TestMultipartUploadWithCopy.java | 101 +++-------
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 41 ++--
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 207 +++++++--------------
.../s3/endpoint/TestPartUploadWithStream.java | 145 ---------------
.../ozone/s3/endpoint/TestUploadWithStream.java | 15 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 20 +-
11 files changed, 206 insertions(+), 597 deletions(-)
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
index a67bd28317a..d2e0a362f1b 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
@@ -19,11 +19,14 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.util.List;
import javax.ws.rs.core.Response;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.http.HttpStatus;
import org.apache.ratis.util.function.CheckedSupplier;
@@ -104,6 +107,67 @@ public static Response deleteTagging(
return subject.delete(bucket, key, null, "");
}
+ /** Initiate multipart upload.
+ * @return upload ID */
+ public static String initiateMultipartUpload(ObjectEndpoint subject, String
bucket, String key)
+ throws IOException, OS3Exception {
+ try (Response response = subject.initializeMultipartUpload(bucket, key)) {
+ assertEquals(HttpStatus.SC_OK, response.getStatus());
+ MultipartUploadInitiateResponse entity =
(MultipartUploadInitiateResponse) response.getEntity();
+ String uploadID = entity.getUploadID();
+ assertNotNull(uploadID, "uploadID == null");
+ return uploadID;
+ }
+ }
+
+ /** Upload part of multipart key.
+ * @return Part to be used for completion request */
+ public static CompleteMultipartUploadRequest.Part uploadPart(
+ ObjectEndpoint subject,
+ String bucket,
+ String key,
+ int partNumber,
+ String uploadID,
+ String content
+ ) throws IOException, OS3Exception {
+ CompleteMultipartUploadRequest.Part part = new
CompleteMultipartUploadRequest.Part();
+
+ try (Response response = put(subject, bucket, key, partNumber, uploadID,
content)) {
+ assertEquals(HttpStatus.SC_OK, response.getStatus());
+ String eTag = response.getHeaderString(OzoneConsts.ETAG);
+ assertNotNull(eTag);
+ part.setETag(eTag);
+ }
+
+ part.setPartNumber(partNumber);
+
+ return part;
+ }
+
+ /** Complete multipart upload. */
+ public static void completeMultipartUpload(
+ ObjectEndpoint subject,
+ String bucket,
+ String key,
+ String uploadID,
+ List<CompleteMultipartUploadRequest.Part> parts
+ ) throws IOException, OS3Exception {
+ CompleteMultipartUploadRequest completeMultipartUploadRequest = new
CompleteMultipartUploadRequest();
+ completeMultipartUploadRequest.setPartList(parts);
+
+ try (Response response = subject.completeMultipartUpload(bucket, key,
uploadID, completeMultipartUploadRequest)) {
+ assertEquals(HttpStatus.SC_OK, response.getStatus());
+
+ CompleteMultipartUploadResponse completeMultipartUploadResponse =
+ (CompleteMultipartUploadResponse) response.getEntity();
+
+ assertEquals(bucket, completeMultipartUploadResponse.getBucket());
+ assertEquals(key, completeMultipartUploadResponse.getKey());
+ assertEquals(bucket, completeMultipartUploadResponse.getLocation());
+ assertNotNull(completeMultipartUploadResponse.getETag());
+ }
+ }
+
/** Verify response is success for {@code request}. */
public static <E extends Exception> void
assertSucceeds(CheckedSupplier<Response, E> request) throws E {
try (Response response = request.get()) {
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
index 9c46a718508..518275c7026 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
@@ -17,9 +17,9 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -55,17 +55,10 @@ public void testAbortMultipartUpload() throws Exception {
.build();
- Response response = rest.initializeMultipartUpload(bucket, key);
-
- assertEquals(200, response.getStatus());
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
-
+ String uploadID = initiateMultipartUpload(rest, bucket, key);
// Abort multipart upload
- response = rest.delete(bucket, key, uploadID, null);
+ Response response = rest.delete(bucket, key, uploadID, null);
assertEquals(204, response.getStatus());
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
index 71113cec45a..a425c166af4 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
@@ -17,16 +17,14 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
-import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import jakarta.annotation.Nonnull;
import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Response;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneClient;
@@ -52,21 +50,11 @@ public void testInitiateMultipartUpload() throws Exception {
ObjectEndpoint rest = getObjectEndpoint(client, headers);
- Response response = rest.initializeMultipartUpload(bucket, key);
-
- assertEquals(200, response.getStatus());
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
+ String uploadID = initiateMultipartUpload(rest, bucket, key);
// Calling again should return different uploadID.
- response = rest.initializeMultipartUpload(bucket, key);
- assertEquals(200, response.getStatus());
- multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- assertNotEquals(multipartUploadInitiateResponse.getUploadID(), uploadID);
+ String nextID = initiateMultipartUpload(rest, bucket, key);
+ assertNotEquals(uploadID, nextID);
}
@Test
@@ -79,12 +67,7 @@ public void testInitiateMultipartUploadWithECKey() throws
Exception {
ObjectEndpoint rest = getObjectEndpoint(client, headers);
client.getObjectStore().getS3Bucket(bucket)
.setReplicationConfig(new ECReplicationConfig("rs-3-2-1024K"));
- Response response = rest.initializeMultipartUpload(bucket, key);
-
- assertEquals(200, response.getStatus());
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
+ initiateMultipartUpload(rest, bucket, key);
}
@Nonnull
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
index 30be715b530..f42c5849aa2 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
@@ -17,17 +17,16 @@
package org.apache.hadoop.ozone.s3.endpoint;
-import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload;
+import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.uploadPart;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.io.ByteArrayInputStream;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -63,32 +62,11 @@ public void setUp() throws Exception {
.setClient(client)
.build();
- Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET,
- OzoneConsts.KEY);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- uploadID = multipartUploadInitiateResponse.getUploadID();
+ uploadID = initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET,
OzoneConsts.KEY);
- assertEquals(200, response.getStatus());
-
- String content = "Multipart Upload";
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
- content.length(), 1, uploadID, null, null, body);
-
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
-
- response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
- content.length(), 2, uploadID, null, null, body);
-
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
-
- response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
- content.length(), 3, uploadID, null, null, body);
-
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
+ for (int i = 1; i <= 3; i++) {
+ uploadPart(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, i, uploadID,
"Multipart Upload");
+ }
}
@Test
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
index fde336f4807..53500bbd2e5 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
@@ -17,20 +17,19 @@
package org.apache.hadoop.ozone.s3.endpoint;
-import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.singletonList;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.completeMultipartUpload;
+import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.uploadPart;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -76,7 +75,7 @@ public void setUp() throws Exception {
private String initiateMultipartUpload(String key) throws IOException,
OS3Exception {
- return initiateMultipartUpload(key, Collections.emptyMap());
+ return EndpointTestUtils.initiateMultipartUpload(rest,
OzoneConsts.S3_BUCKET, key);
}
private String initiateMultipartUpload(String key, Map<String, String>
metadata) throws IOException,
@@ -90,82 +89,19 @@ private String initiateMultipartUpload(String key,
Map<String, String> metadata)
when(headers.getRequestHeaders()).thenReturn(metadataHeaders);
- Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET,
- key);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
-
- assertEquals(200, response.getStatus());
-
- return uploadID;
- }
-
- private Part uploadPart(String key, String uploadID, int partNumber, String
- content) throws IOException, OS3Exception {
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- Response response = rest.put(OzoneConsts.S3_BUCKET, key, content.length(),
- partNumber, uploadID, null, null, body);
- assertEquals(200, response.getStatus());
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
- Part part = new Part();
- part.setETag(response.getHeaderString(OzoneConsts.ETAG));
- part.setPartNumber(partNumber);
-
- return part;
- }
-
- private void completeMultipartUpload(String key,
- CompleteMultipartUploadRequest completeMultipartUploadRequest,
- String uploadID) throws IOException, OS3Exception {
- Response response = rest.completeMultipartUpload(OzoneConsts.S3_BUCKET,
key,
- uploadID, completeMultipartUploadRequest);
-
- assertEquals(200, response.getStatus());
-
- CompleteMultipartUploadResponse completeMultipartUploadResponse =
- (CompleteMultipartUploadResponse) response.getEntity();
-
- assertEquals(OzoneConsts.S3_BUCKET,
- completeMultipartUploadResponse.getBucket());
- assertEquals(key, completeMultipartUploadResponse.getKey());
- assertEquals(OzoneConsts.S3_BUCKET,
- completeMultipartUploadResponse.getLocation());
- assertNotNull(completeMultipartUploadResponse.getETag());
+ return EndpointTestUtils.initiateMultipartUpload(rest,
OzoneConsts.S3_BUCKET, key);
}
@Test
public void testMultipart() throws Exception {
-
- // Initiate multipart upload
String uploadID = initiateMultipartUpload(OzoneConsts.KEY);
- List<Part> partsList = new ArrayList<>();
-
-
// Upload parts
- String content = "Multipart Upload 1";
- int partNumber = 1;
-
- Part part1 = uploadPart(OzoneConsts.KEY, uploadID, partNumber, content);
- partsList.add(part1);
-
- content = "Multipart Upload 2";
- partNumber = 2;
- Part part2 = uploadPart(OzoneConsts.KEY, uploadID, partNumber, content);
- partsList.add(part2);
-
- // complete multipart upload
- CompleteMultipartUploadRequest completeMultipartUploadRequest = new
- CompleteMultipartUploadRequest();
- completeMultipartUploadRequest.setPartList(partsList);
-
-
- completeMultipartUpload(OzoneConsts.KEY, completeMultipartUploadRequest,
- uploadID);
+ List<Part> partsList = new ArrayList<>();
+ partsList.add(uploadPart(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 1,
uploadID, "Multipart Upload 1"));
+ partsList.add(uploadPart(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 2,
uploadID, "Multipart Upload 2"));
+ completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
uploadID, partsList);
}
@Test
@@ -177,21 +113,8 @@ public void testMultipartWithCustomMetadata() throws
Exception {
customMetadata.put("custom-key2", "custom-value2");
String uploadID = initiateMultipartUpload(key, customMetadata);
-
- List<Part> partsList = new ArrayList<>();
-
- // Upload parts
- String content = "Multipart Upload 1";
- int partNumber = 1;
-
- Part part1 = uploadPart(key, uploadID, partNumber, content);
- partsList.add(part1);
-
- CompleteMultipartUploadRequest completeMultipartUploadRequest = new
- CompleteMultipartUploadRequest();
- completeMultipartUploadRequest.setPartList(partsList);
-
- completeMultipartUpload(key, completeMultipartUploadRequest, uploadID);
+ Part part1 = uploadPart(rest, OzoneConsts.S3_BUCKET, key, 1, uploadID,
"Multipart Upload 1");
+ completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, key, uploadID,
singletonList(part1));
Response headResponse = rest.head(OzoneConsts.S3_BUCKET, key);
@@ -209,27 +132,17 @@ public void testMultipartInvalidPartOrderError() throws
Exception {
List<Part> partsList = new ArrayList<>();
// Upload parts
- String content = "Multipart Upload 1";
- int partNumber = 1;
- Part part1 = uploadPart(key, uploadID, partNumber, content);
+ Part part1 = uploadPart(rest, OzoneConsts.S3_BUCKET, key, 1, uploadID,
"Multipart Upload 1");
// Change part number
part1.setPartNumber(3);
partsList.add(part1);
- content = "Multipart Upload 2";
- partNumber = 2;
-
- Part part2 = uploadPart(key, uploadID, partNumber, content);
- partsList.add(part2);
+ partsList.add(uploadPart(rest, OzoneConsts.S3_BUCKET, key, 2, uploadID,
"Multipart Upload 2"));
// complete multipart upload
- CompleteMultipartUploadRequest completeMultipartUploadRequest = new
- CompleteMultipartUploadRequest();
- completeMultipartUploadRequest.setPartList(partsList);
- OS3Exception ex =
- assertThrows(OS3Exception.class,
- () -> completeMultipartUpload(key, completeMultipartUploadRequest,
uploadID));
+ OS3Exception ex = assertThrows(OS3Exception.class,
+ () -> completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, key,
uploadID, partsList));
assertEquals(S3ErrorTable.INVALID_PART_ORDER.getCode(), ex.getCode());
}
@@ -243,27 +156,16 @@ public void testMultipartInvalidPartError() throws
Exception {
List<Part> partsList = new ArrayList<>();
// Upload parts
- String content = "Multipart Upload 1";
- int partNumber = 1;
-
- Part part1 = uploadPart(key, uploadID, partNumber, content);
+ Part part1 = uploadPart(rest, OzoneConsts.S3_BUCKET, key, 1, uploadID,
"Multipart Upload 1");
// Change part name.
part1.setETag("random");
partsList.add(part1);
- content = "Multipart Upload 2";
- partNumber = 2;
-
- Part part2 = uploadPart(key, uploadID, partNumber, content);
- partsList.add(part2);
+ partsList.add(uploadPart(rest, OzoneConsts.S3_BUCKET, key, 2, uploadID,
"Multipart Upload 2"));
// complete multipart upload
- CompleteMultipartUploadRequest completeMultipartUploadRequest = new
- CompleteMultipartUploadRequest();
- completeMultipartUploadRequest.setPartList(partsList);
- OS3Exception ex =
- assertThrows(OS3Exception.class,
- () -> completeMultipartUpload(key, completeMultipartUploadRequest,
uploadID));
+ OS3Exception ex = assertThrows(OS3Exception.class,
+ () -> completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, key,
uploadID, partsList));
assertEquals(ex.getCode(), S3ErrorTable.INVALID_PART.getCode());
}
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
index 702c32d1aba..8bb2cf3fa62 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
@@ -18,6 +18,11 @@
package org.apache.hadoop.ozone.s3.endpoint;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.completeMultipartUpload;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload;
+import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put;
+import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.uploadPart;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER_RANGE;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_MODIFIED_SINCE;
@@ -30,7 +35,6 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -132,14 +136,15 @@ public static void setUp() throws Exception {
@Test
public void testMultipart() throws Exception {
// Initiate multipart upload
- String uploadID = initiateMultipartUpload(KEY);
+ String uploadID = initiateMultipartUpload(endpoint, OzoneConsts.S3_BUCKET,
KEY);
List<Part> partsList = new ArrayList<>();
// Upload parts
String content = "Multipart Upload 1";
- Part part1 = uploadPart(KEY, uploadID, 1, content);
+ setHeaders();
+ Part part1 = uploadPart(endpoint, OzoneConsts.S3_BUCKET, KEY, 1, uploadID,
content);
partsList.add(part1);
Part part2 =
@@ -160,13 +165,8 @@ public void testMultipart() throws Exception {
);
partsList.add(part4);
- // complete multipart upload
- CompleteMultipartUploadRequest completeMultipartUploadRequest = new
- CompleteMultipartUploadRequest();
- completeMultipartUploadRequest.setPartList(partsList);
-
- completeMultipartUpload(KEY, completeMultipartUploadRequest,
- uploadID);
+ setHeaders();
+ completeMultipartUpload(endpoint, OzoneConsts.S3_BUCKET,
TestMultipartUploadWithCopy.KEY, uploadID, partsList);
OzoneBucket bucket =
client.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET);
@@ -309,38 +309,6 @@ public void testMultipartTSHeaders() throws Exception {
}
}
- private String initiateMultipartUpload(String key) throws IOException,
- OS3Exception {
- setHeaders();
- Response response =
endpoint.initializeMultipartUpload(OzoneConsts.S3_BUCKET,
- key);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
-
- assertEquals(200, response.getStatus());
-
- return uploadID;
-
- }
-
- private Part uploadPart(String key, String uploadID, int partNumber, String
- content) throws IOException, OS3Exception {
- setHeaders();
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- Response response = endpoint.put(OzoneConsts.S3_BUCKET, key,
content.length(),
- partNumber, uploadID, null, null, body);
- assertEquals(200, response.getStatus());
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
- Part part = new Part();
- part.setETag(response.getHeaderString(OzoneConsts.ETAG));
- part.setPartNumber(partNumber);
-
- return part;
- }
-
private Part uploadPartWithCopy(String key, String uploadID, int partNumber,
String keyOrigin, String range) throws IOException, OS3Exception {
return uploadPartWithCopy(key, uploadID, partNumber, keyOrigin,
@@ -350,7 +318,8 @@ private Part uploadPartWithCopy(String key, String
uploadID, int partNumber,
private Part uploadPartWithCopy(String ifModifiedSinceStr,
String ifUnmodifiedSinceStr) throws IOException, OS3Exception {
// Initiate multipart upload
- String uploadID = initiateMultipartUpload(KEY);
+ setHeaders();
+ String uploadID = initiateMultipartUpload(endpoint, OzoneConsts.S3_BUCKET,
KEY);
return uploadPartWithCopy(KEY, uploadID, 1,
OzoneConsts.S3_BUCKET + "/" + EXISTING_KEY, "bytes=0-3",
@@ -374,19 +343,18 @@ private Part uploadPartWithCopy(String key, String
uploadID, int partNumber,
}
setHeaders(additionalHeaders);
- ByteArrayInputStream body = new ByteArrayInputStream("".getBytes(UTF_8));
- Response response = endpoint.put(OzoneConsts.S3_BUCKET, key, 0, partNumber,
- uploadID, null, null, body);
- assertEquals(200, response.getStatus());
+ try (Response response = put(endpoint, OzoneConsts.S3_BUCKET, key,
partNumber, uploadID, "")) {
+ assertEquals(200, response.getStatus());
- CopyPartResult result = (CopyPartResult) response.getEntity();
- assertNotNull(result.getETag());
- assertNotNull(result.getLastModified());
- Part part = new Part();
- part.setETag(result.getETag());
- part.setPartNumber(partNumber);
+ CopyPartResult result = (CopyPartResult) response.getEntity();
+ assertNotNull(result.getETag());
+ assertNotNull(result.getLastModified());
+ Part part = new Part();
+ part.setETag(result.getETag());
+ part.setPartNumber(partNumber);
- return part;
+ return part;
+ }
}
@Test
@@ -396,14 +364,13 @@ public void testUploadWithRangeCopyContentLength()
// the Content-Length, the key Commit will compare the Content-Length with
// the actual length of the data written.
- String uploadID = initiateMultipartUpload(KEY);
- ByteArrayInputStream body = new ByteArrayInputStream("".getBytes(UTF_8));
+ String uploadID = initiateMultipartUpload(endpoint, OzoneConsts.S3_BUCKET,
KEY);
Map<String, String> additionalHeaders = new HashMap<>();
additionalHeaders.put(COPY_SOURCE_HEADER,
OzoneConsts.S3_BUCKET + "/" + EXISTING_KEY);
additionalHeaders.put(COPY_SOURCE_HEADER_RANGE, "bytes=0-3");
setHeaders(additionalHeaders);
- endpoint.put(OzoneConsts.S3_BUCKET, KEY, 0, 1, uploadID, null, null, body);
+ assertSucceeds(() -> put(endpoint, OzoneConsts.S3_BUCKET, KEY, 1,
uploadID, ""));
OzoneMultipartUploadPartListParts parts =
client.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET)
.listParts(KEY, uploadID, 0, 100);
@@ -411,26 +378,6 @@ public void testUploadWithRangeCopyContentLength()
assertEquals(4, parts.getPartInfoList().get(0).getSize());
}
- private void completeMultipartUpload(String key,
- CompleteMultipartUploadRequest completeMultipartUploadRequest,
- String uploadID) throws IOException, OS3Exception {
- setHeaders();
- Response response =
endpoint.completeMultipartUpload(OzoneConsts.S3_BUCKET, key,
- uploadID, completeMultipartUploadRequest);
-
- assertEquals(200, response.getStatus());
-
- CompleteMultipartUploadResponse completeMultipartUploadResponse =
- (CompleteMultipartUploadResponse) response.getEntity();
-
- assertEquals(OzoneConsts.S3_BUCKET,
- completeMultipartUploadResponse.getBucket());
- assertEquals(KEY, completeMultipartUploadResponse.getKey());
- assertEquals(OzoneConsts.S3_BUCKET,
- completeMultipartUploadResponse.getLocation());
- assertNotNull(completeMultipartUploadResponse.getETag());
- }
-
private void setHeaders(Map<String, String> additionalHeaders) {
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
index 7c1352e59dd..ee2158d7334 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
@@ -17,8 +17,9 @@
package org.apache.hadoop.ozone.s3.endpoint;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
+import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY;
import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER;
@@ -31,18 +32,16 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.time.format.DateTimeFormatter;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
-import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.OzoneClientTestUtils;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -72,13 +71,14 @@ public class TestObjectGet {
private HttpHeaders headers;
private ObjectEndpoint rest;
- private OzoneClient client;
+ private OzoneBucket bucket;
@BeforeEach
public void init() throws OS3Exception, IOException {
//GIVEN
- client = new OzoneClientStub();
+ OzoneClient client = new OzoneClientStub();
client.getObjectStore().createS3Bucket(BUCKET_NAME);
+ bucket = client.getObjectStore().getS3Bucket(BUCKET_NAME);
headers = mock(HttpHeaders.class);
when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("UNSIGNED-PAYLOAD");
@@ -88,13 +88,11 @@ public void init() throws OS3Exception, IOException {
.setHeaders(headers)
.build();
- ByteArrayInputStream body = new
ByteArrayInputStream(CONTENT.getBytes(UTF_8));
- rest.put(BUCKET_NAME, KEY_NAME, CONTENT.length(),
- 1, null, null, null, body);
+ assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT));
+
// Create a key with object tags
when(headers.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2");
- rest.put(BUCKET_NAME, KEY_WITH_TAG, CONTENT.length(),
- 1, null, null, null, body);
+ assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_WITH_TAG, CONTENT));
}
@Test
@@ -103,14 +101,8 @@ public void get() throws IOException, OS3Exception {
Response response = rest.get(BUCKET_NAME, KEY_NAME, 0, null, 0, null,
null);
//THEN
- OzoneInputStream ozoneInputStream =
- client.getObjectStore().getS3Bucket(BUCKET_NAME)
- .readKey(KEY_NAME);
- String keyContent =
- IOUtils.toString(ozoneInputStream, UTF_8);
-
- assertEquals(CONTENT, keyContent);
- assertEquals(String.valueOf(keyContent.length()),
+ OzoneClientTestUtils.assertKeyContent(bucket, KEY_NAME, CONTENT);
+ assertEquals(String.valueOf(CONTENT.length()),
response.getHeaderString("Content-Length"));
DateTimeFormatter.RFC_1123_DATE_TIME
@@ -125,14 +117,8 @@ public void getKeyWithTag() throws IOException,
OS3Exception {
Response response = rest.get(BUCKET_NAME, KEY_WITH_TAG, 0, null, 0, null,
null);
//THEN
- OzoneInputStream ozoneInputStream =
- client.getObjectStore().getS3Bucket(BUCKET_NAME)
- .readKey(KEY_NAME);
- String keyContent =
- IOUtils.toString(ozoneInputStream, UTF_8);
-
- assertEquals(CONTENT, keyContent);
- assertEquals(String.valueOf(keyContent.length()),
+ OzoneClientTestUtils.assertKeyContent(bucket, KEY_WITH_TAG, CONTENT);
+ assertEquals(String.valueOf(CONTENT.length()),
response.getHeaderString("Content-Length"));
DateTimeFormatter.RFC_1123_DATE_TIME
@@ -251,7 +237,6 @@ public void
testGetWhenKeyIsDirectoryAndDoesNotEndWithASlash()
OzoneConfiguration config = new OzoneConfiguration();
config.set(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, "true");
rest.setOzoneConfiguration(config);
- OzoneBucket bucket = client.getObjectStore().getS3Bucket(BUCKET_NAME);
bucket.createDirectory(keyPath);
// WHEN
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
index 2da2a42a05b..0ac2508565c 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
@@ -17,17 +17,19 @@
package org.apache.hadoop.ozone.s3.endpoint;
-import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
-import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload;
+import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
@@ -36,7 +38,6 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -45,211 +46,131 @@
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.Parameter;
+import org.junit.jupiter.params.ParameterizedClass;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.MockedStatic;
/**
* This class tests Upload part request.
*/
+@ParameterizedClass
+@ValueSource(booleans = {false, true})
public class TestPartUpload {
private ObjectEndpoint rest;
private OzoneClient client;
+ private HttpHeaders headers;
+
+ @Parameter
+ private boolean enableDataStream;
+
@BeforeEach
public void setUp() throws Exception {
-
client = new OzoneClientStub();
client.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET);
-
- HttpHeaders headers = mock(HttpHeaders.class);
+ headers = mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
"STANDARD");
when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
.thenReturn("mockSignature");
- rest = EndpointBuilder.newObjectEndpointBuilder()
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED,
enableDataStream);
+
+ rest = spy(EndpointBuilder.newObjectEndpointBuilder()
.setHeaders(headers)
.setClient(client)
- .build();
+ .setConfig(conf)
+ .build());
+ assertEquals(enableDataStream, rest.isDatastreamEnabled());
}
@Test
public void testPartUpload() throws Exception {
-
- Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET,
- OzoneConsts.KEY);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
-
- assertEquals(200, response.getStatus());
+ String uploadID = initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET,
OzoneConsts.KEY);
String content = "Multipart Upload";
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
- content.length(), 1, uploadID, null, null, body);
-
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
-
- }
-
- @Test
- public void testPartUploadWithOverride() throws Exception {
-
- Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET,
- OzoneConsts.KEY);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
-
- assertEquals(200, response.getStatus());
-
- String content = "Multipart Upload";
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
- content.length(), 1, uploadID, null, null, body);
-
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
-
- String eTag = response.getHeaderString(OzoneConsts.ETAG);
+ String eTag;
+ try (Response response = put(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
1, uploadID, content)) {
+ eTag = response.getHeaderString(OzoneConsts.ETAG);
+ assertNotNull(eTag);
+ }
+ assertContentLength(uploadID, OzoneConsts.KEY, content.length());
// Upload part again with same part Number, the ETag should be changed.
- content = "Multipart Upload Changed";
- response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
- content.length(), 1, uploadID, null, null, body);
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
- assertNotEquals(eTag, response.getHeaderString(OzoneConsts.ETAG));
-
+ String newContent = "Multipart Upload Changed";
+ try (Response response = put(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
1, uploadID, newContent)) {
+ String newETag = response.getHeaderString(OzoneConsts.ETAG);
+ assertNotNull(newETag);
+ assertNotEquals(eTag, newETag);
+ }
}
@Test
- public void testPartUploadWithIncorrectUploadID() throws Exception {
- OS3Exception ex = assertThrows(OS3Exception.class, () -> {
- String content = "Multipart Upload With Incorrect uploadID";
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, content.length(), 1,
- "random", null, null, body);
- });
- assertEquals("NoSuchUpload", ex.getCode());
- assertEquals(HTTP_NOT_FOUND, ex.getHttpCode());
+ public void testPartUploadWithIncorrectUploadID() {
+ assertErrorResponse(S3ErrorTable.NO_SUCH_UPLOAD,
+ () -> put(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 1, "random",
"any"));
}
@Test
public void testPartUploadStreamContentLength()
throws IOException, OS3Exception {
- HttpHeaders headers = mock(HttpHeaders.class);
- when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
- .thenReturn("mockSignature");
- ObjectEndpoint objectEndpoint = EndpointBuilder.newObjectEndpointBuilder()
- .setHeaders(headers)
- .setClient(client)
- .build();
String keyName = UUID.randomUUID().toString();
+ int contentLength = 15;
String chunkedContent = "0a;chunk-signature=signature\r\n"
+ "1234567890\r\n"
+ "05;chunk-signature=signature\r\n"
+ "abcde\r\n";
- when(headers.getHeaderString("x-amz-content-sha256"))
+ when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
.thenReturn("STREAMING-AWS4-HMAC-SHA256-PAYLOAD");
when(headers.getHeaderString(DECODED_CONTENT_LENGTH_HEADER))
- .thenReturn("15");
+ .thenReturn(String.valueOf(contentLength));
- Response response = objectEndpoint.initializeMultipartUpload(
- OzoneConsts.S3_BUCKET, keyName);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
- long contentLength = chunkedContent.length();
+ String uploadID = initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET,
keyName);
- objectEndpoint.put(OzoneConsts.S3_BUCKET, keyName, contentLength, 1,
- uploadID, null, null, new
ByteArrayInputStream(chunkedContent.getBytes(UTF_8)));
- assertContentLength(uploadID, keyName, 15);
- }
-
- @Test
- public void testPartUploadContentLength() throws IOException, OS3Exception {
- // The contentLength specified when creating the Key should be the same as
- // the Content-Length, the key Commit will compare the Content-Length with
- // the actual length of the data written.
-
- String keyName = UUID.randomUUID().toString();
- Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET,
- keyName);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
- String content = "Multipart Upload";
- long contentLength = content.length();
-
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- rest.put(OzoneConsts.S3_BUCKET, keyName,
- contentLength, 1, uploadID, null, null, body);
- assertContentLength(uploadID, keyName, content.length());
+ assertSucceeds(() -> put(rest, OzoneConsts.S3_BUCKET, keyName, 1,
uploadID, chunkedContent));
+ assertContentLength(uploadID, keyName, contentLength);
}
@Test
public void testPartUploadMessageDigestResetDuringException() throws
IOException, OS3Exception {
- OzoneClient clientStub = new OzoneClientStub();
- clientStub.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET);
-
-
- HttpHeaders headers = mock(HttpHeaders.class);
- when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
- .thenReturn("mockSignature");
- when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
- "STANDARD");
-
- ObjectEndpoint objectEndpoint = EndpointBuilder.newObjectEndpointBuilder()
- .setHeaders(headers)
- .setClient(clientStub)
- .build();
-
- objectEndpoint = spy(objectEndpoint);
-
- Response response =
objectEndpoint.initializeMultipartUpload(OzoneConsts.S3_BUCKET,
- OzoneConsts.KEY);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
-
- assertEquals(200, response.getStatus());
+ String uploadID = initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET,
OzoneConsts.KEY);
MessageDigest messageDigest = mock(MessageDigest.class);
when(messageDigest.getAlgorithm()).thenReturn("MD5");
MessageDigest sha256Digest = mock(MessageDigest.class);
when(sha256Digest.getAlgorithm()).thenReturn("SHA-256");
- try (MockedStatic<IOUtils> mocked = mockStatic(IOUtils.class)) {
- // Add the mocked methods only during the copy request
-
when(objectEndpoint.getMessageDigestInstance()).thenReturn(messageDigest);
- when(objectEndpoint.getSha256DigestInstance()).thenReturn(sha256Digest);
- mocked.when(() -> IOUtils.copyLarge(any(InputStream.class),
any(OutputStream.class), anyLong(),
- anyLong(), any(byte[].class)))
- .thenThrow(IOException.class);
+ try (MockedStatic<IOUtils> ioutils = mockStatic(IOUtils.class);
+ MockedStatic<ObjectEndpointStreaming> streaming =
mockStatic(ObjectEndpointStreaming.class)) {
+ // Add the mocked methods only during part upload
+ when(rest.getMessageDigestInstance()).thenReturn(messageDigest);
+ when(rest.getSha256DigestInstance()).thenReturn(sha256Digest);
+ if (enableDataStream) {
+ streaming.when(() -> ObjectEndpointStreaming.createMultipartKey(any(),
any(), anyLong(), anyInt(), any(),
+ anyInt(), any(), any()))
+ .thenThrow(IOException.class);
+ } else {
+ ioutils.when(() -> IOUtils.copyLarge(any(InputStream.class),
any(OutputStream.class), anyLong(),
+ anyLong(), any(byte[].class)))
+ .thenThrow(IOException.class);
+ }
String content = "Multipart Upload";
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- try {
- objectEndpoint.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
- content.length(), 1, uploadID, null, null, body);
+ try (Response ignored = put(rest, OzoneConsts.S3_BUCKET,
OzoneConsts.KEY, 1, uploadID, content)) {
fail("Should throw IOException");
} catch (IOException ignored) {
// Verify that the message digest is reset so that the instance can be
reused for the
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java
deleted file mode 100644
index 736660073d5..00000000000
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.s3.endpoint;
-
-import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
-import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.ByteArrayInputStream;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Response;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientStub;
-import org.apache.hadoop.ozone.s3.exception.OS3Exception;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-/**
- * This class tests Upload part request.
- */
-public class TestPartUploadWithStream {
-
- private ObjectEndpoint rest;
-
- private static final String S3BUCKET = "streampartb1";
- private static final String S3KEY = "testkey";
-
- @BeforeEach
- public void setUp() throws Exception {
- OzoneClient client = new OzoneClientStub();
- client.getObjectStore().createS3Bucket(S3BUCKET);
-
-
- HttpHeaders headers = mock(HttpHeaders.class);
- when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD");
- when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
- .thenReturn("mockSignature");
-
-
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED,
- true);
-
- rest = EndpointBuilder.newObjectEndpointBuilder()
- .setHeaders(headers)
- .setClient(client)
- .setConfig(conf)
- .build();
- }
-
- @Test
- public void testEnableStream() {
- assertTrue(rest.isDatastreamEnabled());
- }
-
- @Test
- public void testPartUpload() throws Exception {
-
- Response response = rest.initializeMultipartUpload(S3BUCKET, S3KEY);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
-
- assertEquals(200, response.getStatus());
-
- String content = "Multipart Upload";
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- response = rest.put(S3BUCKET, S3KEY,
- content.length(), 1, uploadID, null, null, body);
-
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
-
- }
-
- @Test
- public void testPartUploadWithOverride() throws Exception {
-
- Response response = rest.initializeMultipartUpload(S3BUCKET, S3KEY);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- assertNotNull(multipartUploadInitiateResponse.getUploadID());
- String uploadID = multipartUploadInitiateResponse.getUploadID();
-
- assertEquals(200, response.getStatus());
-
- String content = "Multipart Upload";
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- response = rest.put(S3BUCKET, S3KEY,
- content.length(), 1, uploadID, null, null, body);
-
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
-
- String eTag = response.getHeaderString(OzoneConsts.ETAG);
-
- // Upload part again with same part Number, the ETag should be changed.
- content = "Multipart Upload Changed";
- response = rest.put(S3BUCKET, S3KEY,
- content.length(), 1, uploadID, null, null, body);
- assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
- assertNotEquals(eTag, response.getHeaderString(OzoneConsts.ETAG));
-
- }
-
- @Test
- public void testPartUploadWithIncorrectUploadID() throws Exception {
- OS3Exception ex = assertThrows(OS3Exception.class, () -> {
- String content = "Multipart Upload With Incorrect uploadID";
- ByteArrayInputStream body =
- new ByteArrayInputStream(content.getBytes(UTF_8));
- rest.put(S3BUCKET, S3KEY, content.length(), 1,
- "random", null, null, body);
- });
- assertEquals("NoSuchUpload", ex.getCode());
- assertEquals(HTTP_NOT_FOUND, ex.getHttpCode());
- }
-}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
index e9d70f67982..66d4a4cbef6 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
@@ -19,6 +19,8 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
+import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
@@ -27,12 +29,10 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.io.ByteArrayInputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Response;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
@@ -89,12 +89,7 @@ public void testEnableStream() {
@Test
public void testUpload() throws Exception {
- byte[] keyContent = S3_COPY_EXISTING_KEY_CONTENT.getBytes(UTF_8);
- ByteArrayInputStream body =
- new ByteArrayInputStream(keyContent);
- Response response = rest.put(S3BUCKET, S3KEY, 0, 0, null, null, null,
body);
-
- assertEquals(200, response.getStatus());
+ assertSucceeds(() -> put(rest, S3BUCKET, S3KEY,
S3_COPY_EXISTING_KEY_CONTENT));
}
@Test
@@ -126,9 +121,7 @@ public void testUploadWithCopy() throws Exception {
.forEach((k, v) -> when(headers.getHeaderString(k)).thenReturn(v));
rest.setHeaders(headers);
- Response response = rest.put(S3BUCKET, S3KEY, 0, 0, null, null, null,
null);
-
- assertEquals(200, response.getStatus());
+ assertSucceeds(() -> put(rest, S3BUCKET, S3KEY, null));
final long newDataSize = bucket.getKey(S3KEY).getDataSize();
assertEquals(dataSize, newDataSize);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
index 8aa4ba707ef..d01acbe708b 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
@@ -51,7 +51,7 @@
import org.apache.hadoop.ozone.s3.endpoint.BucketEndpoint;
import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest;
import org.apache.hadoop.ozone.s3.endpoint.EndpointBuilder;
-import org.apache.hadoop.ozone.s3.endpoint.MultipartUploadInitiateResponse;
+import org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils;
import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint;
import org.apache.hadoop.ozone.s3.endpoint.RootEndpoint;
import org.apache.hadoop.ozone.s3.endpoint.TestBucketAcl;
@@ -357,7 +357,7 @@ public void testGetKeyFailure() {
public void testInitMultiPartUploadSuccess() throws Exception {
long oriMetric = metrics.getInitMultiPartUploadSuccess();
- keyEndpoint.initializeMultipartUpload(bucketName, keyName);
+ EndpointTestUtils.initiateMultipartUpload(keyEndpoint, bucketName,
keyName);
long curMetric = metrics.getInitMultiPartUploadSuccess();
assertEquals(1L, curMetric - oriMetric);
}
@@ -560,20 +560,8 @@ public void testDeleteObjectTaggingFailure() {
assertEquals(1L, curMetric - oriMetric);
}
- private String initiateMultipartUpload(String bktName, String key)
- throws IOException,
- OS3Exception {
- // Initiate the Upload
- Response response =
- keyEndpoint.initializeMultipartUpload(bktName, key);
- MultipartUploadInitiateResponse multipartUploadInitiateResponse =
- (MultipartUploadInitiateResponse) response.getEntity();
- if (response.getStatus() == 200) {
- // Fetch the Upload-Id
- String uploadID = multipartUploadInitiateResponse.getUploadID();
- return uploadID;
- }
- return "Invalid-Id";
+ private String initiateMultipartUpload(String bktName, String key) throws
IOException, OS3Exception {
+ return EndpointTestUtils.initiateMultipartUpload(keyEndpoint, bktName,
key);
}
private static String getPutTaggingBody() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
