This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b6921533d52 HDDS-13477. Add OM S3 multipart commit and complete
response tests (#10605)
b6921533d52 is described below
commit b6921533d524e5c58feec4c0d6f524006d275a92
Author: Chi-Hsuan Huang <[email protected]>
AuthorDate: Fri Jun 26 22:31:20 2026 +0800
HDDS-13477. Add OM S3 multipart commit and complete response tests (#10605)
Co-authored-by: Michael Chu <[email protected]>
---
.../s3/multipart/TestS3MultipartResponse.java | 103 +++++
...> TestS3MultipartUploadCommitPartResponse.java} | 167 ++++----
...S3MultipartUploadCommitPartResponseWithFSO.java | 246 +++--------
.../TestS3MultipartUploadCompleteResponse.java | 388 ++++++++++++++++++
...stS3MultipartUploadCompleteResponseWithFSO.java | 450 ++++-----------------
5 files changed, 695 insertions(+), 659 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
index ac56273d628..cd407d39eb8 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadAbortResponse;
@@ -295,6 +296,95 @@ public S3MultipartUploadCommitPartResponse
createS3CommitMPUResponseFSO(
getBucketLayout());
}
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public S3MultipartUploadCommitPartResponse createS3CommitMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID,
+ OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo,
+ OmMultipartKeyInfo multipartKeyInfo,
+ OzoneManagerProtocolProtos.Status status, String openKey)
+ throws IOException {
+ if (multipartKeyInfo == null) {
+ multipartKeyInfo = new OmMultipartKeyInfo.Builder()
+ .setUploadID(multipartUploadID)
+ .setCreationTime(Time.now())
+ .setReplicationConfig(RatisReplicationConfig.getInstance(
+ HddsProtos.ReplicationFactor.ONE))
+ .build();
+ }
+
+ String multipartKey = omMetadataManager
+ .getMultipartKey(volumeName, bucketName, keyName, multipartUploadID);
+
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo omBucketInfo =
+ omMetadataManager.getBucketTable().get(bucketKey);
+
+ OmKeyInfo openPartKeyInfoToBeDeleted = new OmKeyInfo.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setCreationTime(Time.now())
+ .setModificationTime(Time.now())
+ .setReplicationConfig(RatisReplicationConfig.getInstance(
+ HddsProtos.ReplicationFactor.ONE))
+ .setOmKeyLocationInfos(Collections.singletonList(
+ new OmKeyLocationInfoGroup(0, new ArrayList<>(), true)))
+ .build();
+
+ OMResponse omResponse = OMResponse.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.CommitMultiPartUpload)
+ .setStatus(status).setSuccess(true)
+ .setCommitMultiPartUploadResponse(
+
OzoneManagerProtocolProtos.MultipartCommitUploadPartResponse
+
.newBuilder().setETag(volumeName).setPartName(volumeName)).build();
+
+ Map<String, RepeatedOmKeyInfo> keyToDeleteMap = new HashMap<>();
+ if (oldPartKeyInfo != null) {
+ OmKeyInfo partKeyToBeDeleted =
+ OmKeyInfo.getFromProtobuf(oldPartKeyInfo.getPartKeyInfo());
+ String delKeyName = omMetadataManager.getOzoneDeletePathKey(
+ partKeyToBeDeleted.getObjectID(), multipartKey);
+
+ keyToDeleteMap.put(delKeyName, new RepeatedOmKeyInfo(partKeyToBeDeleted,
omBucketInfo.getObjectID()));
+ }
+
+ return new S3MultipartUploadCommitPartResponse(omResponse,
+ multipartKey, openKey, multipartKeyInfo, keyToDeleteMap,
+ openPartKeyInfoToBeDeleted, omBucketInfo, omBucketInfo.getObjectID(),
+ getBucketLayout());
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public S3MultipartUploadCompleteResponse createS3CompleteMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID, OmKeyInfo omKeyInfo,
+ OzoneManagerProtocolProtos.Status status,
+ List<OmKeyInfo> allKeyInfoToRemove,
+ OmBucketInfo omBucketInfo) throws IOException {
+
+ String multipartKey = omMetadataManager
+ .getMultipartKey(volumeName, bucketName, keyName, multipartUploadID);
+ // In legacy/OBS buckets, the MPU open key and the multipart key share the
+ // same format, so the complete response deletes them using the same key.
+ String multipartOpenKey = multipartKey;
+
+ long bucketId = omBucketInfo != null ? omBucketInfo.getObjectID()
+ : omMetadataManager.getBucketId(volumeName, bucketName);
+
+ OMResponse omResponse = OMResponse.newBuilder()
+
.setCmdType(OzoneManagerProtocolProtos.Type.CompleteMultiPartUpload)
+ .setStatus(status).setSuccess(true)
+ .setCompleteMultiPartUploadResponse(
+ OzoneManagerProtocolProtos.MultipartUploadCompleteResponse
+ .newBuilder().setBucket(bucketName)
+ .setVolume(volumeName).setKey(keyName)).build();
+
+ return new S3MultipartUploadCompleteResponse(omResponse, multipartKey,
+ multipartOpenKey, omKeyInfo, allKeyInfoToRemove, getBucketLayout(),
+ omBucketInfo, bucketId);
+ }
+
@SuppressWarnings("checkstyle:ParameterNumber")
public S3MultipartUploadCompleteResponse createS3CompleteMPUResponseFSO(
String volumeName, String bucketName, long parentID, String keyName,
@@ -346,6 +436,19 @@ protected S3MultipartUploadAbortResponse
getS3MultipartUploadAbortResp(
getBucketLayout());
}
+ /**
+ * Seed the part's open key into the open key table, simulating the open
+ * entry created during part upload that the commit response later removes.
+ */
+ protected void addPartToOpenKeyTable(String volumeName, String bucketName,
+ String keyName, String openKey) throws IOException {
+ OmKeyInfo partKeyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName,
+ bucketName, keyName, RatisReplicationConfig.getInstance(
+ HddsProtos.ReplicationFactor.ONE)).build();
+ omMetadataManager.getOpenKeyTable(getBucketLayout())
+ .put(openKey, partKeyInfo);
+ }
+
public BucketLayout getBucketLayout() {
return BucketLayout.DEFAULT;
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponse.java
similarity index 61%
copy from
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponseWithFSO.java
copy to
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponse.java
index 9414c62a543..b3263cfb5f6 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponse.java
@@ -22,16 +22,15 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
-import java.util.ArrayList;
+import java.io.IOException;
import java.util.List;
import java.util.UUID;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
import org.apache.hadoop.util.Time;
@@ -40,13 +39,9 @@
/**
* Test multipart upload commit part response.
*/
-public class TestS3MultipartUploadCommitPartResponseWithFSO
+public class TestS3MultipartUploadCommitPartResponse
extends TestS3MultipartResponse {
- private String dirName = "a/b/c/";
-
- private long parentID;
-
@Test
public void testAddDBToBatch() throws Exception {
String volumeName = UUID.randomUUID().toString();
@@ -56,22 +51,22 @@ public void testAddDBToBatch() throws Exception {
OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
omMetadataManager);
-
createParentPath(volumeName, bucketName);
- String fileName = OzoneFSUtils.getFileName(keyName);
+
String multipartKey = omMetadataManager
.getMultipartKey(volumeName, bucketName, keyName, multipartUploadID);
- final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
long clientId = Time.now();
- String openKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
- parentID, fileName, clientId);
+ String openKey = getPartOpenKey(volumeName, bucketName, keyName, clientId);
+
+ // Seed the part's open key so the commit can be verified to remove it.
+ addPartToOpenKeyTable(volumeName, bucketName, keyName, openKey);
+ assertNotNull(
+ omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey));
S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
- createS3CommitMPUResponseFSO(volumeName, bucketName, parentID, keyName,
+ createCommitMPUResponse(volumeName, bucketName, keyName,
multipartUploadID, null, null,
- OzoneManagerProtocolProtos.Status.OK, openKey);
+ OzoneManagerProtocolProtos.Status.OK, openKey);
s3MultipartUploadCommitPartResponse.addToDBBatch(omMetadataManager,
batchOperation);
@@ -92,61 +87,64 @@ public void testAddDBToBatchWithParts() throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
String keyName = getKeyName();
+ String multipartUploadID = UUID.randomUUID().toString();
OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
omMetadataManager);
createParentPath(volumeName, bucketName);
- String multipartUploadID = UUID.randomUUID().toString();
-
- String fileName = OzoneFSUtils.getFileName(keyName);
String multipartKey = omMetadataManager
.getMultipartKey(volumeName, bucketName, keyName, multipartUploadID);
- final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
+ String multipartOpenKey = OMMultipartUploadUtils.getMultipartOpenKey(
+ volumeName, bucketName, keyName, multipartUploadID, omMetadataManager,
+ getBucketLayout());
- S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponseFSO =
- createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, new ArrayList<>(), volumeId, bucketId);
+ S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponse =
+ createInitiateMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID);
- s3InitiateMultipartUploadResponseFSO.addToDBBatch(omMetadataManager,
+ s3InitiateMultipartUploadResponse.addToDBBatch(omMetadataManager,
batchOperation);
// Add some dummy parts for testing.
// Not added any key locations, as this just test is to see entries are
// adding to delete table or not.
OmMultipartKeyInfo omMultipartKeyInfo =
- s3InitiateMultipartUploadResponseFSO.getOmMultipartKeyInfo();
+ s3InitiateMultipartUploadResponse.getOmMultipartKeyInfo();
- PartKeyInfo part1 = createPartKeyInfoFSO(volumeName, bucketName, parentID,
- fileName, 1);
+ PartKeyInfo part1 = createPartKeyInfo(volumeName, bucketName, keyName, 1);
- addPart(1, part1, omMultipartKeyInfo);
+ omMultipartKeyInfo.addPartKeyInfo(part1);
long clientId = Time.now();
+ String openKey = getPartOpenKey(volumeName, bucketName, keyName, clientId);
- String openKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
- parentID, fileName, clientId);
+ // Seed the part's open key so the commit can be verified to remove it.
+ addPartToOpenKeyTable(volumeName, bucketName, keyName, openKey);
S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
- createS3CommitMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID,
- omMultipartKeyInfo.getPartKeyInfo(1),
+ createCommitMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID, omMultipartKeyInfo.getPartKeyInfo(1),
omMultipartKeyInfo,
- OzoneManagerProtocolProtos.Status.OK, openKey);
+ OzoneManagerProtocolProtos.Status.OK, openKey);
s3MultipartUploadCommitPartResponse.checkAndUpdateDB(omMetadataManager,
batchOperation);
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ // The part's open key is removed from the open key table, while the
+ // committed part is persisted to the multipart info table. The open key
+ // created by initiate MPU uses a different key format and is not removed.
assertNull(
omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey));
- assertNull(
+ assertNotNull(
+ omMetadataManager.getOpenKeyTable(getBucketLayout())
+ .get(multipartOpenKey));
+ assertNotNull(
omMetadataManager.getMultipartInfoTable().get(multipartKey));
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
- // As 1 parts are created, so 1 entry should be there in delete table.
+ // As 1 part is overwritten, so 1 entry should be there in delete table.
assertEquals(1, omMetadataManager.countRowsInTable(
omMetadataManager.getDeletedTable()));
@@ -170,86 +168,95 @@ public void testWithMultipartUploadError() throws
Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
String keyName = getKeyName();
+ String multipartUploadID = UUID.randomUUID().toString();
OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
omMetadataManager);
createParentPath(volumeName, bucketName);
- String multipartUploadID = UUID.randomUUID().toString();
-
- final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
-
- String fileName = OzoneFSUtils.getFileName(keyName);
- String multipartKey = omMetadataManager.getMultipartKey(volumeId, bucketId,
- parentID, fileName, multipartUploadID);
-
- S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponseFSO =
- createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, new ArrayList<>(), volumeId, bucketId);
+ S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponse =
+ createInitiateMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID);
- s3InitiateMultipartUploadResponseFSO.addToDBBatch(omMetadataManager,
+ s3InitiateMultipartUploadResponse.addToDBBatch(omMetadataManager,
batchOperation);
// Add some dummy parts for testing.
// Not added any key locations, as this just test is to see entries are
// adding to delete table or not.
OmMultipartKeyInfo omMultipartKeyInfo =
- s3InitiateMultipartUploadResponseFSO.getOmMultipartKeyInfo();
+ s3InitiateMultipartUploadResponse.getOmMultipartKeyInfo();
- PartKeyInfo part1 = createPartKeyInfoFSO(volumeName, bucketName, parentID,
- fileName, 1);
+ PartKeyInfo part1 = createPartKeyInfo(volumeName, bucketName, keyName, 1);
- addPart(1, part1, omMultipartKeyInfo);
+ omMultipartKeyInfo.addPartKeyInfo(part1);
long clientId = Time.now();
- String openKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
- parentID, fileName, clientId);
+ String openKey = getPartOpenKey(volumeName, bucketName, keyName, clientId);
String keyNameInvalid = keyName + "invalid";
S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
- createS3CommitMPUResponseFSO(volumeName, bucketName, parentID,
- keyNameInvalid, multipartUploadID,
- omMultipartKeyInfo.getPartKeyInfo(1),
+ createCommitMPUResponse(volumeName, bucketName, keyNameInvalid,
+ multipartUploadID, omMultipartKeyInfo.getPartKeyInfo(1),
omMultipartKeyInfo, OzoneManagerProtocolProtos.Status
.NO_SUCH_MULTIPART_UPLOAD_ERROR, openKey);
s3MultipartUploadCommitPartResponse.checkAndUpdateDB(omMetadataManager,
batchOperation);
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ // The aborted upload neither persists the invalid multipart key nor adds
+ // the open key back to the open key table.
+ String multipartKeyInvalid = omMetadataManager.getMultipartKey(volumeName,
+ bucketName, keyNameInvalid, multipartUploadID);
assertNull(
omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey));
assertNull(
- omMetadataManager.getMultipartInfoTable().get(multipartKey));
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
omMetadataManager.getMultipartInfoTable().get(multipartKeyInvalid));
// openkey entry should be there in delete table.
assertEquals(1, omMetadataManager.countRowsInTable(
omMetadataManager.getDeletedTable()));
- String deletedKey = omMetadataManager
- .getMultipartKey(volumeName, bucketName, keyNameInvalid,
- multipartUploadID);
List<? extends Table.KeyValue<String, RepeatedOmKeyInfo>> rangeKVs
= omMetadataManager.getDeletedTable().getRangeKVs(
- null, 100, deletedKey);
+ null, 100, multipartKeyInvalid);
assertThat(rangeKVs.size()).isGreaterThan(0);
}
- private String getKeyName() {
- return dirName + UUID.randomUUID().toString();
+ protected String getKeyName() {
+ return UUID.randomUUID().toString();
}
- private void createParentPath(String volumeName, String bucketName)
+ /**
+ * Set up the parent path. No-op for legacy/OBS buckets; FSO buckets
+ * override this to create the parent directories.
+ */
+ protected void createParentPath(String volumeName, String bucketName)
throws Exception {
- // Create parent dirs for the path
- parentID = OMRequestTestUtils.addParentsToDirTable(volumeName, bucketName,
- dirName, omMetadataManager);
}
- @Override
- public BucketLayout getBucketLayout() {
- return BucketLayout.FILE_SYSTEM_OPTIMIZED;
+ protected String getPartOpenKey(String volumeName, String bucketName,
+ String keyName, long clientId) throws IOException {
+ return omMetadataManager.getOpenKey(volumeName, bucketName, keyName,
+ String.valueOf(clientId));
+ }
+
+ protected S3InitiateMultipartUploadResponse createInitiateMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID) throws IOException {
+ return createS3InitiateMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID);
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected S3MultipartUploadCommitPartResponse createCommitMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID, PartKeyInfo oldPartKeyInfo,
+ OmMultipartKeyInfo multipartKeyInfo,
+ OzoneManagerProtocolProtos.Status status, String openKey)
+ throws IOException {
+ return createS3CommitMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID, oldPartKeyInfo, multipartKeyInfo, status, openKey);
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponseWithFSO.java
index 9414c62a543..3660f9ba832 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCommitPartResponseWithFSO.java
@@ -17,235 +17,79 @@
package org.apache.hadoop.ozone.om.response.s3.multipart;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-
+import java.io.IOException;
import java.util.ArrayList;
-import java.util.List;
import java.util.UUID;
-import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
-import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
-import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
-import org.apache.hadoop.util.Time;
-import org.junit.jupiter.api.Test;
/**
- * Test multipart upload commit part response.
+ * Test multipart upload commit part response for FSO bucket.
*/
public class TestS3MultipartUploadCommitPartResponseWithFSO
- extends TestS3MultipartResponse {
+ extends TestS3MultipartUploadCommitPartResponse {
private String dirName = "a/b/c/";
private long parentID;
- @Test
- public void testAddDBToBatch() throws Exception {
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- String keyName = getKeyName();
- String multipartUploadID = UUID.randomUUID().toString();
-
- OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
- omMetadataManager);
-
- createParentPath(volumeName, bucketName);
- String fileName = OzoneFSUtils.getFileName(keyName);
- String multipartKey = omMetadataManager
- .getMultipartKey(volumeName, bucketName, keyName, multipartUploadID);
- final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
- long clientId = Time.now();
- String openKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
- parentID, fileName, clientId);
-
- S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
- createS3CommitMPUResponseFSO(volumeName, bucketName, parentID, keyName,
- multipartUploadID, null, null,
- OzoneManagerProtocolProtos.Status.OK, openKey);
-
- s3MultipartUploadCommitPartResponse.addToDBBatch(omMetadataManager,
- batchOperation);
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
-
assertNull(omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey));
- assertNotNull(omMetadataManager.getMultipartInfoTable().get(multipartKey));
-
- // As no parts are created, so no entries should be there in delete table.
- assertEquals(0, omMetadataManager.countRowsInTable(
- omMetadataManager.getDeletedTable()));
+ @Override
+ protected String getKeyName() {
+ return dirName + UUID.randomUUID().toString();
}
- @Test
- public void testAddDBToBatchWithParts() throws Exception {
-
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- String keyName = getKeyName();
-
- OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
- omMetadataManager);
- createParentPath(volumeName, bucketName);
-
- String multipartUploadID = UUID.randomUUID().toString();
-
- String fileName = OzoneFSUtils.getFileName(keyName);
- String multipartKey = omMetadataManager
- .getMultipartKey(volumeName, bucketName, keyName, multipartUploadID);
- final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
-
- S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponseFSO =
- createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, new ArrayList<>(), volumeId, bucketId);
-
- s3InitiateMultipartUploadResponseFSO.addToDBBatch(omMetadataManager,
- batchOperation);
-
- // Add some dummy parts for testing.
- // Not added any key locations, as this just test is to see entries are
- // adding to delete table or not.
- OmMultipartKeyInfo omMultipartKeyInfo =
- s3InitiateMultipartUploadResponseFSO.getOmMultipartKeyInfo();
-
- PartKeyInfo part1 = createPartKeyInfoFSO(volumeName, bucketName, parentID,
- fileName, 1);
-
- addPart(1, part1, omMultipartKeyInfo);
-
- long clientId = Time.now();
-
- String openKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
- parentID, fileName, clientId);
-
- S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
- createS3CommitMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID,
- omMultipartKeyInfo.getPartKeyInfo(1),
- omMultipartKeyInfo,
- OzoneManagerProtocolProtos.Status.OK, openKey);
-
- s3MultipartUploadCommitPartResponse.checkAndUpdateDB(omMetadataManager,
- batchOperation);
-
- assertNull(
- omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey));
- assertNull(
- omMetadataManager.getMultipartInfoTable().get(multipartKey));
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
- // As 1 parts are created, so 1 entry should be there in delete table.
- assertEquals(1, omMetadataManager.countRowsInTable(
- omMetadataManager.getDeletedTable()));
-
- String part1DeletedKeyName =
- omMetadataManager.getOzoneDeletePathKey(
- omMultipartKeyInfo.getPartKeyInfo(1).getPartKeyInfo()
- .getObjectID(), multipartKey);
-
- assertNotNull(omMetadataManager.getDeletedTable().get(
- part1DeletedKeyName));
-
- RepeatedOmKeyInfo ro =
- omMetadataManager.getDeletedTable().get(part1DeletedKeyName);
- assertEquals(OmKeyInfo.getFromProtobuf(part1.getPartKeyInfo()),
- ro.getOmKeyInfoList().get(0));
+ @Override
+ protected void createParentPath(String volumeName, String bucketName)
+ throws Exception {
+ // Create parent dirs for the path
+ parentID = OMRequestTestUtils.addParentsToDirTable(volumeName, bucketName,
+ dirName, omMetadataManager);
}
- @Test
- public void testWithMultipartUploadError() throws Exception {
-
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- String keyName = getKeyName();
-
- OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
- omMetadataManager);
- createParentPath(volumeName, bucketName);
-
- String multipartUploadID = UUID.randomUUID().toString();
-
+ @Override
+ protected String getPartOpenKey(String volumeName, String bucketName,
+ String keyName, long clientId) throws IOException {
final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
-
+ final long bucketId = omMetadataManager.getBucketId(volumeName,
bucketName);
String fileName = OzoneFSUtils.getFileName(keyName);
- String multipartKey = omMetadataManager.getMultipartKey(volumeId, bucketId,
- parentID, fileName, multipartUploadID);
-
- S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponseFSO =
- createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, new ArrayList<>(), volumeId, bucketId);
-
- s3InitiateMultipartUploadResponseFSO.addToDBBatch(omMetadataManager,
- batchOperation);
-
- // Add some dummy parts for testing.
- // Not added any key locations, as this just test is to see entries are
- // adding to delete table or not.
- OmMultipartKeyInfo omMultipartKeyInfo =
- s3InitiateMultipartUploadResponseFSO.getOmMultipartKeyInfo();
-
- PartKeyInfo part1 = createPartKeyInfoFSO(volumeName, bucketName, parentID,
- fileName, 1);
-
- addPart(1, part1, omMultipartKeyInfo);
-
- long clientId = Time.now();
- String openKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
- parentID, fileName, clientId);
-
- String keyNameInvalid = keyName + "invalid";
- S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
- createS3CommitMPUResponseFSO(volumeName, bucketName, parentID,
- keyNameInvalid, multipartUploadID,
- omMultipartKeyInfo.getPartKeyInfo(1),
- omMultipartKeyInfo, OzoneManagerProtocolProtos.Status
- .NO_SUCH_MULTIPART_UPLOAD_ERROR, openKey);
-
- s3MultipartUploadCommitPartResponse.checkAndUpdateDB(omMetadataManager,
- batchOperation);
-
- assertNull(
- omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey));
- assertNull(
- omMetadataManager.getMultipartInfoTable().get(multipartKey));
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
+ return omMetadataManager.getOpenFileName(volumeId, bucketId, parentID,
+ fileName, clientId);
+ }
- // openkey entry should be there in delete table.
- assertEquals(1, omMetadataManager.countRowsInTable(
- omMetadataManager.getDeletedTable()));
- String deletedKey = omMetadataManager
- .getMultipartKey(volumeName, bucketName, keyNameInvalid,
- multipartUploadID);
- List<? extends Table.KeyValue<String, RepeatedOmKeyInfo>> rangeKVs
- = omMetadataManager.getDeletedTable().getRangeKVs(
- null, 100, deletedKey);
- assertThat(rangeKVs.size()).isGreaterThan(0);
+ @Override
+ protected S3InitiateMultipartUploadResponse createInitiateMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID) throws IOException {
+ final long volumeId = omMetadataManager.getVolumeId(volumeName);
+ final long bucketId = omMetadataManager.getBucketId(volumeName,
bucketName);
+ return createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
+ keyName, multipartUploadID, new ArrayList<>(), volumeId, bucketId);
}
- private String getKeyName() {
- return dirName + UUID.randomUUID().toString();
+ @Override
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected S3MultipartUploadCommitPartResponse createCommitMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID, PartKeyInfo oldPartKeyInfo,
+ OmMultipartKeyInfo multipartKeyInfo,
+ OzoneManagerProtocolProtos.Status status, String openKey)
+ throws IOException {
+ return createS3CommitMPUResponseFSO(volumeName, bucketName, parentID,
+ keyName, multipartUploadID, oldPartKeyInfo, multipartKeyInfo, status,
+ openKey);
}
- private void createParentPath(String volumeName, String bucketName)
- throws Exception {
- // Create parent dirs for the path
- parentID = OMRequestTestUtils.addParentsToDirTable(volumeName, bucketName,
- dirName, omMetadataManager);
+ @Override
+ public PartKeyInfo createPartKeyInfo(
+ String volumeName, String bucketName, String keyName, int partNumber)
+ throws IOException {
+ String fileName = OzoneFSUtils.getFileName(keyName);
+ return createPartKeyInfoFSO(volumeName, bucketName, parentID, fileName,
+ partNumber);
}
@Override
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponse.java
new file mode 100644
index 00000000000..9aeaf89cf5f
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponse.java
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.s3.multipart;
+
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
+import org.apache.hadoop.util.Time;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test multipart upload complete response.
+ */
+public class TestS3MultipartUploadCompleteResponse
+ extends TestS3MultipartResponse {
+
+ @Test
+ public void testAddDBToBatch() throws Exception {
+ runAddDBToBatch(true);
+ }
+
+ @Test
+ // similar to testAddDBToBatch(), but omBucketInfo is null
+ public void testAddDBToBatchWithNullBucketInfo() throws Exception {
+ runAddDBToBatch(false);
+ }
+
+ private void runAddDBToBatch(boolean withBucketInfo) throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = getKeyName();
+ String multipartUploadID = UUID.randomUUID().toString();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager);
+ createParentPath(volumeName, bucketName);
+
+ String dbMultipartKey = omMetadataManager.getMultipartKey(volumeName,
+ bucketName, keyName, multipartUploadID);
+ String dbMultipartOpenKey = getMultipartOpenKey(volumeName, bucketName,
+ keyName, multipartUploadID);
+
+ // add MPU entry to open table and multipart info table
+ S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponse =
+ createInitiateMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID);
+ s3InitiateMultipartUploadResponse.addToDBBatch(omMetadataManager,
+ batchOperation);
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ // commit a part without any overwritten part
+ OmMultipartKeyInfo omMultipartKeyInfo =
+ s3InitiateMultipartUploadResponse.getOmMultipartKeyInfo();
+ addCommittedPart(volumeName, bucketName, keyName, multipartUploadID,
+ omMultipartKeyInfo);
+
+ OmKeyInfo omKeyInfo = createCompletedKeyInfo(volumeName, bucketName,
+ keyName, 1000, 50);
+
+ OmBucketInfo omBucketInfo = withBucketInfo ? omMetadataManager
+ .getBucketTable().get(omMetadataManager
+ .getBucketKey(volumeName, bucketName)) : null;
+
+
assertNotNull(omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
+ assertNotNull(omMetadataManager.getOpenKeyTable(
+ getBucketLayout()).get(dbMultipartOpenKey));
+
+ List<OmKeyInfo> unUsedParts = new ArrayList<>();
+ S3MultipartUploadCompleteResponse s3MultipartUploadCompleteResponse =
+ createCompleteMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID, omKeyInfo,
+ OzoneManagerProtocolProtos.Status.OK, unUsedParts,
+ omBucketInfo);
+
+ s3MultipartUploadCompleteResponse.addToDBBatch(omMetadataManager,
+ batchOperation);
+
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ assertNotNull(omMetadataManager.getKeyTable(getBucketLayout())
+ .get(getFinalDbKey(omKeyInfo)));
+ assertNull(omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
+ assertNull(omMetadataManager.getOpenKeyTable(getBucketLayout())
+ .get(dbMultipartOpenKey));
+
+ // As no parts are unused, so no entries should be there in delete table.
+ assertEquals(0, omMetadataManager.countRowsInTable(
+ omMetadataManager.getDeletedTable()));
+ }
+
+ @Test
+ public void testAddDBToBatchWithParts() throws Exception {
+
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = getKeyName();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager);
+ createParentPath(volumeName, bucketName);
+ runAddDBToBatchWithParts(volumeName, bucketName, keyName, 0);
+
+ // As 1 unused part exists, so 1 unused entry should be there in delete
+ // table, in addition to the 1 overwritten part committed earlier.
+ assertEquals(2, omMetadataManager.countRowsInTable(
+ omMetadataManager.getDeletedTable()));
+ }
+
+ @Test
+ public void testAddDBToBatchWithPartsWithKeyInDeleteTable() throws Exception
{
+
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = getKeyName();
+
+ OmBucketInfo bucketInfo = OMRequestTestUtils.addVolumeAndBucketToDB(
+ volumeName, bucketName, omMetadataManager);
+ createParentPath(volumeName, bucketName);
+
+ // Put an entry to delete table with the same key prior to multipart commit
+ OmKeyInfo prevKey = OMRequestTestUtils.createOmKeyInfo(volumeName,
+ bucketName, keyName, RatisReplicationConfig.getInstance(ONE),
+ new OmKeyLocationInfoGroup(0L, new ArrayList<>(), true))
+ .setObjectID(8)
+ .setUpdateID(8)
+ .build();
+ RepeatedOmKeyInfo prevKeys = new RepeatedOmKeyInfo(prevKey,
+ bucketInfo.getObjectID());
+ String ozoneKey = omMetadataManager.getOzoneDeletePathKey(
+ prevKey.getObjectID(),
+ omMetadataManager.getOzoneKey(prevKey.getVolumeName(),
+ prevKey.getBucketName(), prevKey.getFileName()));
+ omMetadataManager.getDeletedTable().put(ozoneKey, prevKeys);
+
+ long oId = runAddDBToBatchWithParts(volumeName, bucketName, keyName, 1);
+
+ // Make sure new object isn't in delete table
+ RepeatedOmKeyInfo ds = omMetadataManager.getDeletedTable().get(ozoneKey);
+ for (OmKeyInfo omKeyInfo : ds.getOmKeyInfoList()) {
+ assertNotEquals(oId, omKeyInfo.getObjectID());
+ }
+
+ // As 1 unused part, 1 overwritten part and 1 previously put-and-deleted
+ // object exist, so 3 entries should be there in delete table.
+ assertEquals(3, omMetadataManager.countRowsInTable(
+ omMetadataManager.getDeletedTable()));
+ }
+
+ private long runAddDBToBatchWithParts(String volumeName,
+ String bucketName, String keyName, int expectedDeleteEntryCount)
+ throws Exception {
+
+ String multipartUploadID = UUID.randomUUID().toString();
+
+ String dbMultipartKey = omMetadataManager.getMultipartKey(volumeName,
+ bucketName, keyName, multipartUploadID);
+ String dbMultipartOpenKey = getMultipartOpenKey(volumeName, bucketName,
+ keyName, multipartUploadID);
+
+ S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponse =
+ createInitiateMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID);
+ s3InitiateMultipartUploadResponse.addToDBBatch(omMetadataManager,
+ batchOperation);
+
+ OmMultipartKeyInfo omMultipartKeyInfo =
+ s3InitiateMultipartUploadResponse.getOmMultipartKeyInfo();
+
+ // Committing the overwritten part adds one entry to the deleted table,
+ // which commitOnePart also asserts internally.
+ OmKeyInfo committedPartKeyInfo = commitOnePart(volumeName, bucketName,
+ keyName, multipartUploadID, dbMultipartKey, omMultipartKeyInfo,
+ expectedDeleteEntryCount + 1);
+
+ OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName, bucketName));
+
+ // 1 unused part that should be moved to the deleted table on completion.
+ OmKeyInfo unUsedPartKeyInfo =
+ OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
+ RatisReplicationConfig.getInstance(ONE),
+ new OmKeyLocationInfoGroup(0L, new ArrayList<>(), true))
+ .setObjectID(9)
+ .setUpdateID(100)
+ .build();
+ List<OmKeyInfo> unUsedParts = new ArrayList<>();
+ unUsedParts.add(unUsedPartKeyInfo);
+ S3MultipartUploadCompleteResponse s3MultipartUploadCompleteResponse =
+ createCompleteMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID, committedPartKeyInfo,
+ OzoneManagerProtocolProtos.Status.OK, unUsedParts,
+ omBucketInfo);
+
+ s3MultipartUploadCompleteResponse.addToDBBatch(omMetadataManager,
+ batchOperation);
+
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ assertNotNull(omMetadataManager.getKeyTable(getBucketLayout())
+ .get(getFinalDbKey(committedPartKeyInfo)));
+ assertNull(omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
+ assertNull(omMetadataManager.getOpenKeyTable(getBucketLayout())
+ .get(dbMultipartOpenKey));
+
+ return committedPartKeyInfo.getObjectID();
+ }
+
+ /**
+ * Commit a single part with an overwritten part and assert that the
+ * overwritten part is moved to the deleted table.
+ *
+ * @return the committed part key info, used as the completed key.
+ */
+ private OmKeyInfo commitOnePart(String volumeName, String bucketName,
+ String keyName, String multipartUploadID, String dbMultipartKey,
+ OmMultipartKeyInfo omMultipartKeyInfo, int expectedDeleteEntryCount)
+ throws Exception {
+
+ PartKeyInfo part1 = createPartKeyInfo(volumeName, bucketName, keyName, 1);
+
+ omMultipartKeyInfo.addPartKeyInfo(part1);
+
+ long clientId = Time.now();
+ String openKey = getPartOpenKey(volumeName, bucketName, keyName, clientId);
+
+ // Seed the part's open key so the commit can be verified to remove it.
+ addPartToOpenKeyTable(volumeName, bucketName, keyName, openKey);
+
+ S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
+ createCommitMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID,
+ omMultipartKeyInfo.getPartKeyInfo(1),
+ omMultipartKeyInfo,
+ OzoneManagerProtocolProtos.Status.OK, openKey);
+
+ s3MultipartUploadCommitPartResponse.checkAndUpdateDB(omMetadataManager,
+ batchOperation);
+
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ // The part's open key is removed and the part is persisted to the
+ // multipart info table.
+ assertNull(
+ omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey));
+ assertNotNull(
+ omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
+
+ // The overwritten part is added to the deleted table.
+ assertEquals(expectedDeleteEntryCount,
+ omMetadataManager.countRowsInTable(
+ omMetadataManager.getDeletedTable()));
+
+ String part1DeletedKeyName = omMetadataManager.getOzoneDeletePathKey(
+ omMultipartKeyInfo.getPartKeyInfo(1).getPartKeyInfo().getObjectID(),
+ dbMultipartKey);
+
+ assertNotNull(omMetadataManager.getDeletedTable().get(
+ part1DeletedKeyName));
+
+ RepeatedOmKeyInfo ro =
+ omMetadataManager.getDeletedTable().get(part1DeletedKeyName);
+ OmKeyInfo omPartKeyInfo =
OmKeyInfo.getFromProtobuf(part1.getPartKeyInfo());
+ assertEquals(omPartKeyInfo, ro.getOmKeyInfoList().get(0));
+
+ return omPartKeyInfo;
+ }
+
+ /**
+ * Commit a single part without an overwritten part. Used by the
+ * {@link #runAddDBToBatch(boolean)} flow.
+ */
+ private void addCommittedPart(String volumeName, String bucketName,
+ String keyName, String multipartUploadID,
+ OmMultipartKeyInfo omMultipartKeyInfo) throws Exception {
+ long clientId = Time.now();
+ String openKey = getPartOpenKey(volumeName, bucketName, keyName, clientId);
+
+ S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
+ createCommitMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID, null, omMultipartKeyInfo,
+ OzoneManagerProtocolProtos.Status.OK, openKey);
+
+ s3MultipartUploadCommitPartResponse.addToDBBatch(omMetadataManager,
+ batchOperation);
+
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+ }
+
+ protected String getKeyName() {
+ return UUID.randomUUID().toString();
+ }
+
+ /**
+ * Set up the parent path. No-op for legacy/OBS buckets; FSO buckets
+ * override this to create the parent directories.
+ */
+ protected void createParentPath(String volumeName, String bucketName)
+ throws Exception {
+ }
+
+ protected String getMultipartOpenKey(String volumeName, String bucketName,
+ String keyName, String multipartUploadID) throws IOException {
+ return OMMultipartUploadUtils.getMultipartOpenKey(volumeName, bucketName,
+ keyName, multipartUploadID, omMetadataManager, getBucketLayout());
+ }
+
+ protected String getPartOpenKey(String volumeName, String bucketName,
+ String keyName, long clientId) throws IOException {
+ return omMetadataManager.getOpenKey(volumeName, bucketName, keyName,
+ String.valueOf(clientId));
+ }
+
+ protected String getFinalDbKey(OmKeyInfo omKeyInfo) throws IOException {
+ return omMetadataManager.getOzoneKey(omKeyInfo.getVolumeName(),
+ omKeyInfo.getBucketName(), omKeyInfo.getKeyName());
+ }
+
+ protected OmKeyInfo createCompletedKeyInfo(String volumeName,
+ String bucketName, String keyName, long objectId, long txnId) {
+ return OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
+ RatisReplicationConfig.getInstance(ONE),
+ new OmKeyLocationInfoGroup(0L, new ArrayList<>(), true))
+ .setObjectID(objectId)
+ .setUpdateID(txnId)
+ .build();
+ }
+
+ protected S3InitiateMultipartUploadResponse createInitiateMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID) throws IOException {
+ return createS3InitiateMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID);
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected S3MultipartUploadCommitPartResponse createCommitMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID, PartKeyInfo oldPartKeyInfo,
+ OmMultipartKeyInfo multipartKeyInfo,
+ OzoneManagerProtocolProtos.Status status, String openKey)
+ throws IOException {
+ return createS3CommitMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID, oldPartKeyInfo, multipartKeyInfo, status, openKey);
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected S3MultipartUploadCompleteResponse createCompleteMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID, OmKeyInfo omKeyInfo,
+ OzoneManagerProtocolProtos.Status status,
+ List<OmKeyInfo> allKeyInfoToRemove, OmBucketInfo omBucketInfo)
+ throws IOException {
+ return createS3CompleteMPUResponse(volumeName, bucketName, keyName,
+ multipartUploadID, omKeyInfo, status, allKeyInfoToRemove,
omBucketInfo);
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponseWithFSO.java
index acc6cfbd530..e9be745dff3 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponseWithFSO.java
@@ -18,10 +18,6 @@
package org.apache.hadoop.ozone.om.response.s3.multipart;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
import java.io.IOException;
import java.util.ArrayList;
@@ -30,417 +26,115 @@
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
-import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
-import org.apache.hadoop.util.Time;
-import org.junit.jupiter.api.Test;
/**
- * Test multipart upload complete response.
+ * Test multipart upload complete response for FSO bucket.
*/
public class TestS3MultipartUploadCompleteResponseWithFSO
- extends TestS3MultipartResponse {
+ extends TestS3MultipartUploadCompleteResponse {
private String dirName = "a/b/c/";
private long parentID;
- @Test
- public void testAddDBToBatch() throws Exception {
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- String keyName = getKeyName();
- String multipartUploadID = UUID.randomUUID().toString();
-
- OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
- omMetadataManager);
-
- long txnId = 50;
- long objectId = parentID + 1;
- String fileName = OzoneFSUtils.getFileName(keyName);
- final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
- String dbMultipartKey = omMetadataManager.getMultipartKey(volumeName,
- bucketName, keyName, multipartUploadID);
- String dbMultipartOpenKey = omMetadataManager.getMultipartKey(volumeId,
- bucketId, parentID, fileName, multipartUploadID);
- long clientId = Time.now();
-
- // add MPU entry to OpenFileTable
- List<OmDirectoryInfo> parentDirInfos = new ArrayList<>();
- S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponseFSO =
- createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, parentDirInfos, volumeId, bucketId);
-
- s3InitiateMultipartUploadResponseFSO.addToDBBatch(omMetadataManager,
- batchOperation);
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
- String dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
- parentID, fileName, clientId);
- String dbKey = omMetadataManager.getOzonePathKey(volumeId, bucketId,
- parentID, fileName);
- OmKeyInfo omKeyInfoFSO =
- OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
- RatisReplicationConfig.getInstance(ONE), new
OmKeyLocationInfoGroup(0L, new ArrayList<>(), true))
- .setObjectID(objectId)
- .setParentObjectID(parentID)
- .setUpdateID(txnId)
- .build();
-
- // add key to openFileTable
- omKeyInfoFSO.setKeyName(fileName);
- OMRequestTestUtils.addFileToKeyTable(true, false,
- fileName, omKeyInfoFSO, clientId, omKeyInfoFSO.getObjectID(),
- omMetadataManager);
-
- addS3MultipartUploadCommitPartResponseFSO(volumeName, bucketName, keyName,
- multipartUploadID, dbOpenKey);
-
- String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
- OmBucketInfo omBucketInfo =
- omMetadataManager.getBucketTable().get(bucketKey);
-
-
assertNotNull(omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
- assertNotNull(omMetadataManager.getOpenKeyTable(
- getBucketLayout()).get(dbMultipartOpenKey));
-
- List<OmKeyInfo> unUsedParts = new ArrayList<>();
- S3MultipartUploadCompleteResponse s3MultipartUploadCompleteResponse =
- createS3CompleteMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, omKeyInfoFSO,
- OzoneManagerProtocolProtos.Status.OK, unUsedParts,
- omBucketInfo);
-
- s3MultipartUploadCompleteResponse.addToDBBatch(omMetadataManager,
- batchOperation);
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
- assertNotNull(omMetadataManager.getKeyTable(getBucketLayout()).get(dbKey));
- assertNull(omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
- assertNull(omMetadataManager.getOpenKeyTable(getBucketLayout())
- .get(dbMultipartOpenKey));
-
- // As no parts are created, so no entries should be there in delete table.
- assertEquals(0, omMetadataManager.countRowsInTable(
- omMetadataManager.getDeletedTable()));
- }
-
- @Test
- // similar to testAddDBToBatch(), but omBucketInfo is null
- public void testAddDBToBatchWithNullBucketInfo() throws Exception {
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- String keyName = getKeyName();
- String multipartUploadID = UUID.randomUUID().toString();
-
- OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
- omMetadataManager);
-
- long txnId = 150;
- long objectId = parentID + 1;
- String fileName = OzoneFSUtils.getFileName(keyName);
- final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
- String dbMultipartKey = omMetadataManager.getMultipartKey(volumeName,
- bucketName, keyName, multipartUploadID);
- String dbMultipartOpenKey = omMetadataManager.getMultipartKey(volumeId,
- bucketId, parentID, fileName, multipartUploadID);
- long clientId = Time.now();
-
- // add MPU entry to OpenFileTable
- List<OmDirectoryInfo> parentDirInfos = new ArrayList<>();
- S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponseFSO =
- createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, parentDirInfos, volumeId, bucketId);
-
- s3InitiateMultipartUploadResponseFSO.addToDBBatch(omMetadataManager,
- batchOperation);
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
- String dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
- parentID, fileName, clientId);
- String dbKey = omMetadataManager.getOzonePathKey(volumeId, bucketId,
- parentID, fileName);
- OmKeyInfo omKeyInfoFSO =
- OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
- RatisReplicationConfig.getInstance(ONE), new
OmKeyLocationInfoGroup(0L, new ArrayList<>(), true))
- .setObjectID(objectId)
- .setParentObjectID(parentID)
- .setUpdateID(txnId)
- .build();
-
- // add key to openFileTable
- omKeyInfoFSO.setKeyName(fileName);
- OMRequestTestUtils.addFileToKeyTable(true, false,
- fileName, omKeyInfoFSO, clientId, omKeyInfoFSO.getObjectID(),
- omMetadataManager);
-
- addS3MultipartUploadCommitPartResponseFSO(volumeName, bucketName, keyName,
- multipartUploadID, dbOpenKey);
-
- assertNotNull(
- omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
- assertNotNull(omMetadataManager.getOpenKeyTable(
- getBucketLayout()).get(dbMultipartOpenKey));
-
- // S3MultipartUploadCompleteResponseWithFSO should accept null bucketInfo
- List<OmKeyInfo> unUsedParts = new ArrayList<>();
- S3MultipartUploadCompleteResponse s3MultipartUploadCompleteResponse =
- createS3CompleteMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, omKeyInfoFSO,
- OzoneManagerProtocolProtos.Status.OK, unUsedParts,
- null);
-
- s3MultipartUploadCompleteResponse.addToDBBatch(omMetadataManager,
- batchOperation);
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
- assertNotNull(
- omMetadataManager.getKeyTable(getBucketLayout()).get(dbKey));
- assertNull(
- omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
- assertNull(omMetadataManager.getOpenKeyTable(getBucketLayout())
- .get(dbMultipartOpenKey));
-
- // As no parts are created, so no entries should be there in delete table.
- assertEquals(0, omMetadataManager.countRowsInTable(
- omMetadataManager.getDeletedTable()));
- }
-
- @Test
- public void testAddDBToBatchWithParts() throws Exception {
-
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- String keyName = getKeyName();
-
- OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
- omMetadataManager);
- createParentPath(volumeName, bucketName);
- runAddDBToBatchWithParts(volumeName, bucketName, keyName, 0);
-
- // As 1 unused parts exists, so 1 unused entry should be there in delete
- // table.
- assertEquals(2, omMetadataManager.countRowsInTable(
- omMetadataManager.getDeletedTable()));
+ @Override
+ protected String getKeyName() {
+ return dirName + UUID.randomUUID().toString();
}
- @Test
- public void testAddDBToBatchWithPartsWithKeyInDeleteTable() throws Exception
{
-
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- String keyName = getKeyName();
-
- OmBucketInfo bucketInfo =
OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
- omMetadataManager);
- createParentPath(volumeName, bucketName);
-
- // Put an entry to delete table with the same key prior to multipart commit
- OmKeyInfo prevKey = OMRequestTestUtils.createOmKeyInfo(volumeName,
bucketName, keyName,
- RatisReplicationConfig.getInstance(ONE), new
OmKeyLocationInfoGroup(0L, new ArrayList<>(), true))
- .setObjectID(parentID + 8)
- .setParentObjectID(parentID)
- .setUpdateID(8)
- .build();
- RepeatedOmKeyInfo prevKeys = new RepeatedOmKeyInfo(prevKey,
bucketInfo.getObjectID());
- String ozoneKey = omMetadataManager
- .getOzoneKey(prevKey.getVolumeName(),
- prevKey.getBucketName(), prevKey.getFileName());
- omMetadataManager.getDeletedTable().put(ozoneKey, prevKeys);
-
- long oId = runAddDBToBatchWithParts(volumeName, bucketName, keyName, 1);
-
- // Make sure new object isn't in delete table
- RepeatedOmKeyInfo ds = omMetadataManager.getDeletedTable().get(ozoneKey);
- for (OmKeyInfo omKeyInfo : ds.getOmKeyInfoList()) {
- assertNotEquals(oId, omKeyInfo.getObjectID());
- }
-
- // As 1 unused parts and 1 previously put-and-deleted object exist,
- // so 2 entries should be there in delete table.
- assertEquals(3, omMetadataManager.countRowsInTable(
- omMetadataManager.getDeletedTable()));
+ @Override
+ protected void createParentPath(String volumeName, String bucketName)
+ throws Exception {
+ // Create parent dirs for the path
+ parentID = OMRequestTestUtils.addParentsToDirTable(volumeName, bucketName,
+ dirName, omMetadataManager);
}
- private long runAddDBToBatchWithParts(String volumeName,
- String bucketName, String keyName, int deleteEntryCount)
- throws Exception {
-
- String multipartUploadID = UUID.randomUUID().toString();
+ @Override
+ protected String getPartOpenKey(String volumeName, String bucketName,
+ String keyName, long clientId) throws IOException {
final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
-
+ final long bucketId = omMetadataManager.getBucketId(volumeName,
bucketName);
String fileName = OzoneFSUtils.getFileName(keyName);
- String dbMultipartKey = omMetadataManager.getMultipartKey(volumeName,
- bucketName, keyName, multipartUploadID);
- String dbMultipartOpenKey = omMetadataManager.getMultipartKey(volumeId,
- bucketId, parentID, fileName, multipartUploadID);
-
- S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponseFSO =
- addS3InitiateMultipartUpload(volumeName, bucketName, keyName,
- multipartUploadID, volumeId, bucketId);
-
- // Add some dummy parts for testing.
- // Not added any key locations, as this just test is to see entries are
- // adding to delete table or not.
- OmMultipartKeyInfo omMultipartKeyInfo =
- s3InitiateMultipartUploadResponseFSO.getOmMultipartKeyInfo();
-
- // After commits, it adds an entry to the deleted table. Incrementing the
- // variable before the method call, because this method also has entry
- // count check inside.
- deleteEntryCount++;
- OmKeyInfo omKeyInfoFSO = commitS3MultipartUpload(volumeName, bucketName,
- keyName, multipartUploadID, fileName, dbMultipartKey,
- omMultipartKeyInfo, deleteEntryCount);
+ return omMetadataManager.getOpenFileName(volumeId, bucketId, parentID,
+ fileName, clientId);
+ }
- String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
- OmBucketInfo omBucketInfo =
- omMetadataManager.getBucketTable().get(bucketKey);
+ @Override
+ protected String getFinalDbKey(OmKeyInfo omKeyInfo) throws IOException {
+ final long volumeId = omMetadataManager.getVolumeId(
+ omKeyInfo.getVolumeName());
+ final long bucketId = omMetadataManager.getBucketId(
+ omKeyInfo.getVolumeName(), omKeyInfo.getBucketName());
+ return omMetadataManager.getOzonePathKey(volumeId, bucketId,
+ omKeyInfo.getParentObjectID(), omKeyInfo.getKeyName());
+ }
+ @Override
+ protected OmKeyInfo createCompletedKeyInfo(String volumeName,
+ String bucketName, String keyName, long objectId, long txnId) {
OmKeyInfo omKeyInfo =
OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
- RatisReplicationConfig.getInstance(ONE), new
OmKeyLocationInfoGroup(0L, new ArrayList<>(), true))
- .setObjectID(parentID + 9)
+ RatisReplicationConfig.getInstance(ONE),
+ new OmKeyLocationInfoGroup(0L, new ArrayList<>(), true))
+ .setObjectID(objectId)
.setParentObjectID(parentID)
- .setUpdateID(100)
+ .setUpdateID(txnId)
.build();
- List<OmKeyInfo> unUsedParts = new ArrayList<>();
- unUsedParts.add(omKeyInfo);
- S3MultipartUploadCompleteResponse s3MultipartUploadCompleteResponse =
- createS3CompleteMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, omKeyInfoFSO,
- OzoneManagerProtocolProtos.Status.OK, unUsedParts,
- omBucketInfo);
-
- s3MultipartUploadCompleteResponse.addToDBBatch(omMetadataManager,
- batchOperation);
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
- String dbKey = omMetadataManager.getOzonePathKey(volumeId, bucketId,
- parentID, omKeyInfoFSO.getFileName());
- assertNotNull(
- omMetadataManager.getKeyTable(getBucketLayout()).get(dbKey));
- assertNull(
- omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
- assertNull(omMetadataManager.getOpenKeyTable(getBucketLayout())
- .get(dbMultipartOpenKey));
-
- return omKeyInfoFSO.getObjectID();
+ omKeyInfo.setKeyName(OzoneFSUtils.getFileName(keyName));
+ return omKeyInfo;
}
- @SuppressWarnings("parameterNumber")
- private OmKeyInfo commitS3MultipartUpload(String volumeName,
- String bucketName, String keyName, String multipartUploadID,
- String fileName, String multipartKey,
- OmMultipartKeyInfo omMultipartKeyInfo,
- int deleteEntryCount) throws IOException {
-
+ @Override
+ protected S3InitiateMultipartUploadResponse createInitiateMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID) throws IOException {
final long volumeId = omMetadataManager.getVolumeId(volumeName);
- final long bucketId = omMetadataManager.getBucketId(volumeName,
- bucketName);
-
- PartKeyInfo part1 = createPartKeyInfoFSO(volumeName, bucketName, parentID,
- fileName, 1);
-
- addPart(1, part1, omMultipartKeyInfo);
-
- long clientId = Time.now();
- String openKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
- parentID, fileName, clientId);
-
- S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
- createS3CommitMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID,
- omMultipartKeyInfo.getPartKeyInfo(1),
- omMultipartKeyInfo,
- OzoneManagerProtocolProtos.Status.OK, openKey);
-
- s3MultipartUploadCommitPartResponse.checkAndUpdateDB(omMetadataManager,
- batchOperation);
-
- assertNull(
-
omMetadataManager.getOpenKeyTable(getBucketLayout()).get(multipartKey));
- assertNull(
- omMetadataManager.getMultipartInfoTable().get(multipartKey));
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
- // As 1 parts are created, so 1 entry should be there in delete table.
- assertEquals(deleteEntryCount,
- omMetadataManager.countRowsInTable(
- omMetadataManager.getDeletedTable()));
-
- String part1DeletedKeyName = omMetadataManager.getOzoneDeletePathKey(
- omMultipartKeyInfo.getPartKeyInfo(1).getPartKeyInfo().getObjectID(),
- multipartKey);
-
- assertNotNull(omMetadataManager.getDeletedTable().get(
- part1DeletedKeyName));
-
- RepeatedOmKeyInfo ro =
- omMetadataManager.getDeletedTable().get(part1DeletedKeyName);
- OmKeyInfo omPartKeyInfo =
OmKeyInfo.getFromProtobuf(part1.getPartKeyInfo());
- assertEquals(omPartKeyInfo, ro.getOmKeyInfoList().get(0));
-
- return omPartKeyInfo;
+ final long bucketId = omMetadataManager.getBucketId(volumeName,
bucketName);
+ return createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
+ keyName, multipartUploadID, new ArrayList<>(), volumeId, bucketId);
}
- private S3InitiateMultipartUploadResponse addS3InitiateMultipartUpload(
- String volumeName, String bucketName, String keyName,
- String multipartUploadID, long volumeId,
- long bucketId) throws IOException {
-
- S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponseFSO =
- createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, new ArrayList<>(), volumeId,
- bucketId);
-
- s3InitiateMultipartUploadResponseFSO.addToDBBatch(omMetadataManager,
- batchOperation);
-
- return s3InitiateMultipartUploadResponseFSO;
- }
-
- private String getKeyName() {
- return dirName + UUID.randomUUID().toString();
+ @Override
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected S3MultipartUploadCommitPartResponse createCommitMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID, PartKeyInfo oldPartKeyInfo,
+ OmMultipartKeyInfo multipartKeyInfo,
+ OzoneManagerProtocolProtos.Status status, String openKey)
+ throws IOException {
+ return createS3CommitMPUResponseFSO(volumeName, bucketName, parentID,
+ keyName, multipartUploadID, oldPartKeyInfo, multipartKeyInfo, status,
+ openKey);
}
- private void createParentPath(String volumeName, String bucketName)
- throws Exception {
- // Create parent dirs for the path
- parentID = OMRequestTestUtils.addParentsToDirTable(volumeName, bucketName,
- dirName, omMetadataManager);
+ @Override
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected S3MultipartUploadCompleteResponse createCompleteMPUResponse(
+ String volumeName, String bucketName, String keyName,
+ String multipartUploadID, OmKeyInfo omKeyInfo,
+ OzoneManagerProtocolProtos.Status status,
+ List<OmKeyInfo> allKeyInfoToRemove, OmBucketInfo omBucketInfo)
+ throws IOException {
+ return createS3CompleteMPUResponseFSO(volumeName, bucketName, parentID,
+ keyName, multipartUploadID, omKeyInfo, status, allKeyInfoToRemove,
+ omBucketInfo);
}
- private void addS3MultipartUploadCommitPartResponseFSO(String volumeName,
- String bucketName, String keyName, String multipartUploadID,
- String openKey) throws IOException {
- S3MultipartUploadCommitPartResponse s3MultipartUploadCommitPartResponse =
- createS3CommitMPUResponseFSO(volumeName, bucketName, parentID,
- keyName, multipartUploadID, null, null,
- OzoneManagerProtocolProtos.Status.OK, openKey);
-
- s3MultipartUploadCommitPartResponse.addToDBBatch(omMetadataManager,
- batchOperation);
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
+ @Override
+ public PartKeyInfo createPartKeyInfo(
+ String volumeName, String bucketName, String keyName, int partNumber)
+ throws IOException {
+ String fileName = OzoneFSUtils.getFileName(keyName);
+ return createPartKeyInfoFSO(volumeName, bucketName, parentID, fileName,
+ partNumber);
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
