This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a commit to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c4236b67f7ff009b7dff969a40ee721ff618bc32 Author: Rakesh Radhakrishnan <[email protected]> AuthorDate: Tue Mar 9 17:11:31 2021 +0530 HDDS-4835. [FSO]S3Multipart: Implement UploadAbortRequest (#1997) --- .../rpc/TestOzoneClientMultipartUploadV1.java | 179 +++++++++++++++++++++ .../om/ratis/utils/OzoneManagerRatisUtils.java | 4 + .../multipart/S3MultipartUploadAbortRequest.java | 43 +++-- .../multipart/S3MultipartUploadAbortRequestV1.java | 90 +++++++++++ .../S3MultipartUploadCommitPartRequestV1.java | 23 +-- .../S3MultipartUploadCompleteRequest.java | 17 +- .../S3MultipartUploadCompleteRequestV1.java | 16 ++ .../S3MultipartUploadAbortResponseV1.java | 55 +++++++ .../TestS3InitiateMultipartUploadRequest.java | 24 +-- .../TestS3InitiateMultipartUploadRequestV1.java | 57 +------ .../s3/multipart/TestS3MultipartRequest.java | 7 +- .../TestS3MultipartUploadAbortRequest.java | 29 +++- .../TestS3MultipartUploadAbortRequestV1.java | 69 ++++++++ .../s3/multipart/TestS3MultipartResponse.java | 25 ++- .../TestS3MultipartUploadAbortResponse.java | 22 ++- .../TestS3MultipartUploadAbortResponseV1.java | 75 +++++++++ 16 files changed, 622 insertions(+), 113 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java index 1ab2cc3..76feec8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java @@ -31,7 +31,10 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; @@ -39,8 +42,12 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.StringUtils.string2Bytes; import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -49,7 +56,10 @@ import org.junit.Test; import org.junit.rules.Timeout; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.TreeMap; @@ -58,6 +68,9 @@ import java.util.UUID; import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * This test verifies all the S3 multipart client apis - layout version V1. @@ -450,6 +463,172 @@ public class TestOzoneClientMultipartUploadV1 { } } + @Test + public void testAbortUploadFail() throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + OzoneTestUtils.expectOmException(NO_SUCH_MULTIPART_UPLOAD_ERROR, + () -> bucket.abortMultipartUpload(keyName, "random")); + } + + @Test + public void testAbortUploadFailWithInProgressPartUpload() throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String parentDir = "a/b/c/d/"; + String keyName = parentDir + UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + OmMultipartInfo omMultipartInfo = bucket.initiateMultipartUpload(keyName, + STAND_ALONE, ONE); + + Assert.assertNotNull(omMultipartInfo.getUploadID()); + + // Do not close output stream. + byte[] data = "data".getBytes(UTF_8); + OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName, + data.length, 1, omMultipartInfo.getUploadID()); + ozoneOutputStream.write(data, 0, data.length); + + // Abort before completing part upload. + bucket.abortMultipartUpload(keyName, omMultipartInfo.getUploadID()); + + try { + ozoneOutputStream.close(); + fail("testAbortUploadFailWithInProgressPartUpload failed"); + } catch (IOException ex) { + assertTrue(ex instanceof OMException); + assertEquals(NO_SUCH_MULTIPART_UPLOAD_ERROR, + ((OMException) ex).getResult()); + } + } + + @Test + public void testAbortUploadSuccessWithOutAnyParts() throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String parentDir = "a/b/c/d/"; + String keyName = parentDir + UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE, + ONE); + bucket.abortMultipartUpload(keyName, uploadID); + } + + @Test + public void testAbortUploadSuccessWithParts() throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String parentDir = "a/b/c/d/"; + String keyName = parentDir + UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE, + ONE); + String partName = uploadPart(bucket, keyName, uploadID, 1, + "data".getBytes(UTF_8)); + + OMMetadataManager metadataMgr = + cluster.getOzoneManager().getMetadataManager(); + String multipartKey = verifyUploadedPart(volumeName, bucketName, keyName, + uploadID, partName, metadataMgr); + + bucket.abortMultipartUpload(keyName, uploadID); + + OmKeyInfo omKeyInfo = metadataMgr.getOpenKeyTable().get(multipartKey); + OmMultipartKeyInfo omMultipartKeyInfo = + metadataMgr.getMultipartInfoTable().get(multipartKey); + Assert.assertNull(omKeyInfo); + Assert.assertNull(omMultipartKeyInfo); + + // Since deleteTable operation is performed via + // batchOp - Table.putWithBatch(), which is an async operation and + // not making any assertion for the same. + } + + private String verifyUploadedPart(String volumeName, String bucketName, + String keyName, String uploadID, String partName, + OMMetadataManager metadataMgr) throws IOException { + String multipartKey = getMultipartKey(uploadID, volumeName, bucketName, + keyName, metadataMgr); + OmKeyInfo omKeyInfo = metadataMgr.getOpenKeyTable().get(multipartKey); + OmMultipartKeyInfo omMultipartKeyInfo = + metadataMgr.getMultipartInfoTable().get(multipartKey); + + Assert.assertNotNull(omKeyInfo); + Assert.assertNotNull(omMultipartKeyInfo); + Assert.assertEquals(OzoneFSUtils.getFileName(keyName), + omKeyInfo.getKeyName()); + Assert.assertEquals(uploadID, omMultipartKeyInfo.getUploadID()); + + long parentID = getParentID(volumeName, bucketName, keyName, + metadataMgr); + + TreeMap<Integer, OzoneManagerProtocolProtos.PartKeyInfo> partKeyInfoMap = + omMultipartKeyInfo.getPartKeyInfoMap(); + for (Map.Entry<Integer, OzoneManagerProtocolProtos.PartKeyInfo> entry : + partKeyInfoMap.entrySet()) { + OzoneManagerProtocolProtos.PartKeyInfo partKeyInfo = entry.getValue(); + OmKeyInfo currentKeyPartInfo = + OmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo()); + + Assert.assertEquals(OzoneFSUtils.getFileName(keyName), + currentKeyPartInfo.getKeyName()); + + // prepare dbPartName <parentID>/partFileName + String partFileName = OzoneFSUtils.getFileName(partName); + String dbPartName = metadataMgr.getOzonePathKey(parentID, partFileName); + + Assert.assertEquals(dbPartName, partKeyInfo.getPartName()); + } + return multipartKey; + } + + private String getMultipartKey(String multipartUploadID, String volumeName, + String bucketName, String keyName, OMMetadataManager omMetadataManager) + throws IOException { + + String fileName = OzoneFSUtils.getFileName(keyName); + long parentID = getParentID(volumeName, bucketName, keyName, + omMetadataManager); + + String multipartKey = omMetadataManager.getMultipartKey(parentID, + fileName, multipartUploadID); + + return multipartKey; + } + + private long getParentID(String volumeName, String bucketName, + String keyName, OMMetadataManager omMetadataManager) throws IOException { + Iterator<Path> pathComponents = Paths.get(keyName).iterator(); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = + omMetadataManager.getBucketTable().get(bucketKey); + long bucketId = omBucketInfo.getObjectID(); + return OMFileRequest.getParentID(bucketId, pathComponents, + keyName, omMetadataManager); + } + private String initiateMultipartUpload(OzoneBucket bucket, String keyName, ReplicationType replicationType, ReplicationFactor replicationFactor) throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 2b39b8d..1498a73 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -59,6 +59,7 @@ import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixSetAclRequest; import org.apache.hadoop.ozone.om.request.s3.multipart.S3InitiateMultipartUploadRequest; import org.apache.hadoop.ozone.om.request.s3.multipart.S3InitiateMultipartUploadRequestV1; import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadAbortRequest; +import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadAbortRequestV1; import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCommitPartRequest; import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCommitPartRequestV1; import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCompleteRequest; @@ -195,6 +196,9 @@ public final class OzoneManagerRatisUtils { } return new S3MultipartUploadCommitPartRequest(omRequest); case AbortMultiPartUpload: + if (isBucketFSOptimized()) { + return new S3MultipartUploadAbortRequestV1(omRequest); + } return new S3MultipartUploadAbortRequest(omRequest); case CompleteMultiPartUpload: if (isBucketFSOptimized()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index e7773cf..abf90a8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -118,8 +118,8 @@ public class S3MultipartUploadAbortRequest extends OMKeyRequest { validateBucketAndVolume(omMetadataManager, volumeName, bucketName); - multipartKey = omMetadataManager.getMultipartKey( - volumeName, bucketName, keyName, keyArgs.getMultipartUploadID()); + multipartKey = getMultipartKey(keyArgs.getMultipartUploadID(), + volumeName, bucketName, keyName, omMetadataManager); OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable().get(multipartKey); @@ -161,19 +161,14 @@ public class S3MultipartUploadAbortRequest extends OMKeyRequest { new CacheKey<>(multipartKey), new CacheValue<>(Optional.absent(), trxnLogIndex)); - omClientResponse = new S3MultipartUploadAbortResponse( - omResponse.setAbortMultiPartUploadResponse( - MultipartUploadAbortResponse.newBuilder()).build(), - multipartKey, multipartKeyInfo, ozoneManager.isRatisEnabled(), - omBucketInfo.copyObject()); + omClientResponse = getOmClientResponse(ozoneManager, multipartKeyInfo, + multipartKey, omResponse, omBucketInfo); result = Result.SUCCESS; } catch (IOException ex) { result = Result.FAILURE; exception = ex; - omClientResponse = - new S3MultipartUploadAbortResponse(createErrorOMResponse(omResponse, - exception)); + omClientResponse = getOmClientResponse(exception, omResponse); } finally { addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, omDoubleBufferHelper); @@ -207,4 +202,32 @@ public class S3MultipartUploadAbortRequest extends OMKeyRequest { return omClientResponse; } + + protected OMClientResponse getOmClientResponse(IOException exception, + OMResponse.Builder omResponse) { + + return new S3MultipartUploadAbortResponse(createErrorOMResponse(omResponse, + exception)); + } + + protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, + OmMultipartKeyInfo multipartKeyInfo, String multipartKey, + OMResponse.Builder omResponse, OmBucketInfo omBucketInfo) { + + OMClientResponse omClientResponse = new S3MultipartUploadAbortResponse( + omResponse.setAbortMultiPartUploadResponse( + MultipartUploadAbortResponse.newBuilder()).build(), + multipartKey, multipartKeyInfo, ozoneManager.isRatisEnabled(), + omBucketInfo.copyObject()); + return omClientResponse; + } + + protected String getMultipartKey(String multipartUploadID, String volumeName, + String bucketName, String keyName, OMMetadataManager omMetadataManager) + throws IOException { + + String multipartKey = omMetadataManager.getMultipartKey( + volumeName, bucketName, keyName, multipartUploadID); + return multipartKey; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequestV1.java new file mode 100644 index 0000000..ddeb10c --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequestV1.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.s3.multipart; + +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadAbortResponseV1; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadAbortResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; + +/** + * Handles Abort of multipart upload request. + */ +public class S3MultipartUploadAbortRequestV1 + extends S3MultipartUploadAbortRequest { + + private static final Logger LOG = + LoggerFactory.getLogger(S3MultipartUploadAbortRequestV1.class); + + public S3MultipartUploadAbortRequestV1(OMRequest omRequest) { + super(omRequest); + } + + protected OMClientResponse getOmClientResponse(IOException exception, + OMResponse.Builder omResponse) { + + return new S3MultipartUploadAbortResponseV1(createErrorOMResponse( + omResponse, exception)); + } + + protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, + OmMultipartKeyInfo multipartKeyInfo, String multipartKey, + OMResponse.Builder omResponse, OmBucketInfo omBucketInfo) { + + OMClientResponse omClientResponse = new S3MultipartUploadAbortResponseV1( + omResponse.setAbortMultiPartUploadResponse( + MultipartUploadAbortResponse.newBuilder()).build(), + multipartKey, multipartKeyInfo, ozoneManager.isRatisEnabled(), + omBucketInfo.copyObject()); + return omClientResponse; + } + + protected String getMultipartKey(String multipartUploadID, String volumeName, + String bucketName, String keyName, OMMetadataManager omMetadataManager) + throws IOException { + + String fileName = OzoneFSUtils.getFileName(keyName); + Iterator<Path> pathComponents = Paths.get(keyName).iterator(); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = + omMetadataManager.getBucketTable().get(bucketKey); + long bucketId = omBucketInfo.getObjectID(); + long parentID = OMFileRequest.getParentID(bucketId, pathComponents, + keyName, omMetadataManager); + + String multipartKey = omMetadataManager.getMultipartKey(parentID, + fileName, multipartUploadID); + + return multipartKey; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestV1.java index 7aa21cf..0f23745 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestV1.java @@ -92,7 +92,7 @@ public class S3MultipartUploadCommitPartRequestV1 getOmRequest()); OMClientResponse omClientResponse = null; OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo = null; - String openKey = null; + String openFileKey = null; OmKeyInfo omKeyInfo = null; String multipartKey = null; OmMultipartKeyInfo multipartKeyInfo = null; @@ -127,14 +127,15 @@ public class S3MultipartUploadCommitPartRequestV1 long clientID = multipartCommitUploadPartRequest.getClientID(); - openKey = omMetadataManager.getOpenFileName(parentID, fileName, clientID); + openFileKey = omMetadataManager.getOpenFileName(parentID, fileName, + clientID); omKeyInfo = OMFileRequest.getOmKeyInfoFromFileTable(true, - omMetadataManager, openKey, keyName); + omMetadataManager, openFileKey, keyName); if (omKeyInfo == null) { throw new OMException("Failed to commit Multipart Upload key, as " + - openKey + " entry is not found in the openKey table", + openFileKey + " entry is not found in the openFileTable", KEY_NOT_FOUND); } @@ -150,7 +151,7 @@ public class S3MultipartUploadCommitPartRequestV1 /** * Format of PartName stored into MultipartInfoTable is, - * "fileName + ClientID". + * "<parentID>/fileName + ClientID". * * Contract is that all part names present in a multipart info will * have same key prefix path. @@ -159,7 +160,9 @@ public class S3MultipartUploadCommitPartRequestV1 * /vol1/buck1/a/b/c/part-1, /vol1/buck1/a/b/c/part-2, * /vol1/buck1/a/b/c/part-n */ - dbPartName = fileName + clientID; + String ozoneFileKey = omMetadataManager.getOzonePathKey(parentID, + fileName); + dbPartName = ozoneFileKey + clientID; if (multipartKeyInfo == null) { // This can occur when user started uploading part by the time commit @@ -204,7 +207,7 @@ public class S3MultipartUploadCommitPartRequestV1 trxnLogIndex)); omMetadataManager.getOpenKeyTable().addCacheEntry( - new CacheKey<>(openKey), + new CacheKey<>(openFileKey), new CacheValue<>(Optional.absent(), trxnLogIndex)); long scmBlockSize = ozoneManager.getScmBlockSize(); @@ -228,7 +231,7 @@ public class S3MultipartUploadCommitPartRequestV1 .setPartName(fullKeyPartName)); omClientResponse = new S3MultipartUploadCommitPartResponseV1( - omResponse.build(), multipartKey, openKey, + omResponse.build(), multipartKey, openFileKey, multipartKeyInfo, oldPartKeyInfo, omKeyInfo, ozoneManager.isRatisEnabled(), omBucketInfo.copyObject()); @@ -238,8 +241,8 @@ public class S3MultipartUploadCommitPartRequestV1 result = Result.FAILURE; exception = ex; omClientResponse = new S3MultipartUploadCommitPartResponseV1( - createErrorOMResponse(omResponse, exception), multipartKey, openKey, - multipartKeyInfo, oldPartKeyInfo, omKeyInfo, + createErrorOMResponse(omResponse, exception), multipartKey, + openFileKey, multipartKeyInfo, oldPartKeyInfo, omKeyInfo, ozoneManager.isRatisEnabled(), copyBucketInfo); } finally { addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 1112608..3cde78d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -34,9 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; -import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; -import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -429,22 +427,11 @@ public class S3MultipartUploadCompleteRequest extends OMKeyRequest { return dataSize; } - private String preparePartName(String requestedVolume, + protected String preparePartName(String requestedVolume, String requestedBucket, String keyName, PartKeyInfo partKeyInfo, OMMetadataManager omMetadataManager) { - String partName; - if (OzoneManagerRatisUtils.isBucketFSOptimized()) { - String parentPath = OzoneFSUtils.getParent(keyName); - StringBuffer keyPath = new StringBuffer(parentPath); - keyPath.append(partKeyInfo.getPartName()); - - partName = omMetadataManager.getOzoneKey(requestedVolume, - requestedBucket, keyPath.toString()); - } else { - partName = partKeyInfo.getPartName(); - } - return partName; + return partKeyInfo.getPartName(); } private static String failureMessage(String volume, String bucket, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestV1.java index 4ab9ee7..37f2dad 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestV1.java @@ -252,6 +252,7 @@ public class S3MultipartUploadCompleteRequestV1 new CacheValue<>(Optional.absent(), transactionLogIndex)); } + @Override protected void updatePrefixFSOInfo(OmKeyInfo dbOpenKeyInfo, OmKeyInfo.Builder builder) { // updates parentID and fileName @@ -259,6 +260,21 @@ public class S3MultipartUploadCompleteRequestV1 builder.setFileName(dbOpenKeyInfo.getFileName()); } + @Override + protected String preparePartName(String requestedVolume, + String requestedBucket, String keyName, PartKeyInfo partKeyInfo, + OMMetadataManager omMetadataManager) { + + String parentPath = OzoneFSUtils.getParent(keyName); + StringBuffer keyPath = new StringBuffer(parentPath); + String partFileName = OzoneFSUtils.getFileName(partKeyInfo.getPartName()); + keyPath.append(partFileName); + + return omMetadataManager.getOzoneKey(requestedVolume, + requestedBucket, keyPath.toString()); + } + + private static String failureMessage(String volume, String bucket, String keyName) { return "Complete Multipart Upload Failed: volume: " + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponseV1.java new file mode 100644 index 0000000..3ed05ef --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponseV1.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.s3.multipart; + +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + +import javax.annotation.Nonnull; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.MULTIPARTFILEINFO_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE; + +/** + * Response for Multipart Abort Request layout version V1. + */ +@CleanupTableInfo(cleanupTables = {OPEN_FILE_TABLE, DELETED_TABLE, + MULTIPARTFILEINFO_TABLE}) +public class S3MultipartUploadAbortResponseV1 + extends S3MultipartUploadAbortResponse { + + public S3MultipartUploadAbortResponseV1(@Nonnull OMResponse omResponse, + String multipartKey, @Nonnull OmMultipartKeyInfo omMultipartKeyInfo, + boolean isRatisEnabled, @Nonnull OmBucketInfo omBucketInfo) { + + super(omResponse, multipartKey, omMultipartKeyInfo, isRatisEnabled, + omBucketInfo); + } + + /** + * For when the request is not successful. + * For a successful request, the other constructor should be used. + */ + public S3MultipartUploadAbortResponseV1(@Nonnull OMResponse omResponse) { + super(omResponse); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java index 088b232..01561e9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java @@ -56,7 +56,7 @@ public class TestS3InitiateMultipartUploadRequest bucketName, keyName); S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = - new S3InitiateMultipartUploadRequest(modifiedRequest); + getS3InitiateMultipartUploadReq(modifiedRequest); OMClientResponse omClientResponse = s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, @@ -65,8 +65,8 @@ public class TestS3InitiateMultipartUploadRequest Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK, omClientResponse.getOMResponse().getStatus()); - String multipartKey = omMetadataManager.getMultipartKey(volumeName, - bucketName, keyName, modifiedRequest.getInitiateMultiPartUploadRequest() + String multipartKey = getMultipartKey(volumeName, bucketName, keyName, + modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getMultipartUploadID()); Assert.assertNotNull(omMetadataManager.getOpenKeyTable().get(multipartKey)); @@ -102,7 +102,7 @@ public class TestS3InitiateMultipartUploadRequest volumeName, bucketName, keyName); S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = - new S3InitiateMultipartUploadRequest(modifiedRequest); + getS3InitiateMultipartUploadReq(modifiedRequest); OMClientResponse omClientResponse = s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, @@ -111,8 +111,8 @@ public class TestS3InitiateMultipartUploadRequest Assert.assertEquals(OzoneManagerProtocolProtos.Status.BUCKET_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); - String multipartKey = omMetadataManager.getMultipartKey(volumeName, - bucketName, keyName, modifiedRequest.getInitiateMultiPartUploadRequest() + String multipartKey = getMultipartKey(volumeName, bucketName, keyName, + modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getMultipartUploadID()); Assert.assertNull(omMetadataManager.getOpenKeyTable().get(multipartKey)); @@ -130,7 +130,7 @@ public class TestS3InitiateMultipartUploadRequest keyName); S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = - new S3InitiateMultipartUploadRequest(modifiedRequest); + getS3InitiateMultipartUploadReq(modifiedRequest); OMClientResponse omClientResponse = s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, @@ -139,12 +139,18 @@ public class TestS3InitiateMultipartUploadRequest Assert.assertEquals(OzoneManagerProtocolProtos.Status.VOLUME_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); - String multipartKey = omMetadataManager.getMultipartKey(volumeName, - bucketName, keyName, modifiedRequest.getInitiateMultiPartUploadRequest() + String multipartKey = getMultipartKey(volumeName, bucketName, keyName, + modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getMultipartUploadID()); Assert.assertNull(omMetadataManager.getOpenKeyTable().get(multipartKey)); Assert.assertNull(omMetadataManager.getMultipartInfoTable() .get(multipartKey)); } + + protected String getMultipartKey(String volumeName, String bucketName, + String keyName, String multipartUploadID) { + return omMetadataManager.getMultipartKey(volumeName, + bucketName, keyName, multipartUploadID); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java index 5fa75ba..4c9dbb5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java @@ -65,8 +65,8 @@ public class TestS3InitiateMultipartUploadRequestV1 OMRequest modifiedRequest = doPreExecuteInitiateMPUV1(volumeName, bucketName, keyName); - S3InitiateMultipartUploadRequestV1 s3InitiateMultipartUploadRequestV1 = - new S3InitiateMultipartUploadRequestV1(modifiedRequest); + S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequestV1 = + getS3InitiateMultipartUploadReq(modifiedRequest); OMClientResponse omClientResponse = s3InitiateMultipartUploadRequestV1.validateAndUpdateCache( @@ -112,54 +112,6 @@ public class TestS3InitiateMultipartUploadRequestV1 .getCreationTime()); } - @Test - public void testValidateAndUpdateCacheWithBucketNotFound() throws Exception { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - String keyName = UUID.randomUUID().toString(); - - TestOMRequestUtils.addVolumeToDB(volumeName, omMetadataManager); - - OMRequest modifiedRequest = doPreExecuteInitiateMPU( - volumeName, bucketName, keyName); - - S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = - new S3InitiateMultipartUploadRequest(modifiedRequest); - - OMClientResponse omClientResponse = - s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, - 100L, ozoneManagerDoubleBufferHelper); - - Assert.assertEquals(OzoneManagerProtocolProtos.Status.BUCKET_NOT_FOUND, - omClientResponse.getOMResponse().getStatus()); - - Assert.assertTrue(omMetadataManager.getOpenKeyTable().isEmpty()); - Assert.assertTrue(omMetadataManager.getMultipartInfoTable().isEmpty()); - } - - @Test - public void testValidateAndUpdateCacheWithVolumeNotFound() throws Exception { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - String keyName = UUID.randomUUID().toString(); - - OMRequest modifiedRequest = doPreExecuteInitiateMPU(volumeName, bucketName, - keyName); - - S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = - new S3InitiateMultipartUploadRequest(modifiedRequest); - - OMClientResponse omClientResponse = - s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, - 100L, ozoneManagerDoubleBufferHelper); - - Assert.assertEquals(OzoneManagerProtocolProtos.Status.VOLUME_NOT_FOUND, - omClientResponse.getOMResponse().getStatus()); - - Assert.assertTrue(omMetadataManager.getOpenKeyTable().isEmpty()); - Assert.assertTrue(omMetadataManager.getMultipartInfoTable().isEmpty()); - } - private long verifyDirectoriesInDB(List<String> dirs, long bucketID) throws IOException { // bucketID is the parent @@ -179,4 +131,9 @@ public class TestS3InitiateMultipartUploadRequestV1 } return parentID; } + + protected S3InitiateMultipartUploadRequest getS3InitiateMultipartUploadReq( + OMRequest initiateMPURequest) { + return new S3InitiateMultipartUploadRequestV1(initiateMPURequest); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java index 16cb4ae..d08b4ae 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java @@ -183,7 +183,7 @@ public class TestS3MultipartRequest { S3MultipartUploadAbortRequest s3MultipartUploadAbortRequest = - new S3MultipartUploadAbortRequest(omRequest); + getS3MultipartUploadAbortReq(omRequest); OMRequest modifiedRequest = s3MultipartUploadAbortRequest.preExecute(ozoneManager); @@ -262,4 +262,9 @@ public class TestS3MultipartRequest { return new S3InitiateMultipartUploadRequest(initiateMPURequest); } + protected S3MultipartUploadAbortRequest getS3MultipartUploadAbortReq( + OMRequest omRequest) { + return new S3MultipartUploadAbortRequest(omRequest); + } + } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java index d0b61c7..9bff636 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java @@ -50,16 +50,18 @@ public class TestS3MultipartUploadAbortRequest extends TestS3MultipartRequest { public void testValidateAndUpdateCache() throws Exception { String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); - String keyName = UUID.randomUUID().toString(); + String keyName = getKeyName(); TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager); + createParentPath(volumeName, bucketName); + OMRequest initiateMPURequest = doPreExecuteInitiateMPU(volumeName, bucketName, keyName); S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = - new S3InitiateMultipartUploadRequest(initiateMPURequest); + getS3InitiateMultipartUploadReq(initiateMPURequest); OMClientResponse omClientResponse = s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, @@ -73,15 +75,15 @@ public class TestS3MultipartUploadAbortRequest extends TestS3MultipartRequest { multipartUploadID); S3MultipartUploadAbortRequest s3MultipartUploadAbortRequest = - new S3MultipartUploadAbortRequest(abortMPURequest); + getS3MultipartUploadAbortReq(abortMPURequest); omClientResponse = s3MultipartUploadAbortRequest.validateAndUpdateCache(ozoneManager, 2L, ozoneManagerDoubleBufferHelper); - String multipartKey = omMetadataManager.getMultipartKey(volumeName, - bucketName, keyName, multipartUploadID); + String multipartKey = getMultipartKey(volumeName, bucketName, keyName, + multipartUploadID); // Check table and response. Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK, @@ -108,7 +110,7 @@ public class TestS3MultipartUploadAbortRequest extends TestS3MultipartRequest { multipartUploadID); S3MultipartUploadAbortRequest s3MultipartUploadAbortRequest = - new S3MultipartUploadAbortRequest(abortMPURequest); + getS3MultipartUploadAbortReq(abortMPURequest); OMClientResponse omClientResponse = s3MultipartUploadAbortRequest.validateAndUpdateCache(ozoneManager, 2L, @@ -177,4 +179,19 @@ public class TestS3MultipartUploadAbortRequest extends TestS3MultipartRequest { omClientResponse.getOMResponse().getStatus()); } + + protected String getKeyName() { + return UUID.randomUUID().toString(); + } + + protected void createParentPath(String volumeName, String bucketName) + throws Exception { + // no parent hierarchy + } + + protected String getMultipartKey(String volumeName, String bucketName, + String keyName, String multipartUploadID) { + return omMetadataManager.getMultipartKey(volumeName, + bucketName, keyName, multipartUploadID); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequestV1.java new file mode 100644 index 0000000..fd8a158 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequestV1.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.s3.multipart; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; + +import java.util.UUID; + +/** + * Test Multipart upload abort request. + */ +public class TestS3MultipartUploadAbortRequestV1 + extends TestS3MultipartUploadAbortRequest { + + private String dirName = "a/b/c/"; + + private long parentID; + + @Override + protected S3MultipartUploadAbortRequest getS3MultipartUploadAbortReq( + OMRequest omRequest) { + return new S3MultipartUploadAbortRequestV1(omRequest); + } + + @Override + protected S3InitiateMultipartUploadRequest getS3InitiateMultipartUploadReq( + OMRequest initiateMPURequest) { + return new S3InitiateMultipartUploadRequestV1(initiateMPURequest); + } + + @Override + protected String getKeyName() { + return dirName + UUID.randomUUID().toString(); + } + + @Override + protected void createParentPath(String volumeName, String bucketName) + throws Exception { + // Create parent dirs for the path + parentID = TestOMRequestUtils.addParentsToDirTable(volumeName, bucketName, + dirName, omMetadataManager); + } + + @Override + protected String getMultipartKey(String volumeName, String bucketName, + String keyName, String multipartUploadID) { + String fileName = StringUtils.substringAfter(keyName, dirName); + return omMetadataManager.getMultipartKey(parentID, fileName, + multipartUploadID); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java index 6f4d6fa..c10ff77 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java @@ -116,8 +116,8 @@ public class TestS3MultipartResponse { .setKeyName(keyName) .setMultipartUploadID(multipartUploadID)).build(); - return new S3InitiateMultipartUploadResponse(omResponse, multipartKeyInfo, - omKeyInfo); + return getS3InitiateMultipartUploadResp(multipartKeyInfo, omKeyInfo, + omResponse); } public S3MultipartUploadAbortResponse createS3AbortMPUResponse( @@ -130,11 +130,10 @@ public class TestS3MultipartResponse { .setAbortMultiPartUploadResponse( MultipartUploadAbortResponse.newBuilder().build()).build(); - return new S3MultipartUploadAbortResponse(omResponse, multipartKey, - omMultipartKeyInfo, true, omBucketInfo); + return getS3MultipartUploadAbortResp(multipartKey, + omMultipartKeyInfo, omBucketInfo, omResponse); } - public void addPart(int partNumber, PartKeyInfo partKeyInfo, OmMultipartKeyInfo omMultipartKeyInfo) { omMultipartKeyInfo.addPartKeyInfo(partNumber, partKeyInfo); @@ -162,7 +161,7 @@ public class TestS3MultipartResponse { int partNumber) { return PartKeyInfo.newBuilder() .setPartNumber(partNumber) - .setPartName(omMetadataManager.getMultipartKey(parentID, fileName, + .setPartName(omMetadataManager.getOzonePathKey(parentID, fileName + UUID.randomUUID().toString())) .setPartKeyInfo(KeyInfo.newBuilder() .setVolumeName(volumeName) @@ -295,4 +294,18 @@ public class TestS3MultipartResponse { return omMetadataManager.getMultipartKey(parentID, fileName, multipartUploadID); } + + protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp( + OmMultipartKeyInfo multipartKeyInfo, OmKeyInfo omKeyInfo, + OMResponse omResponse) { + return new S3InitiateMultipartUploadResponse(omResponse, multipartKeyInfo, + omKeyInfo); + } + + protected S3MultipartUploadAbortResponse getS3MultipartUploadAbortResp( + String multipartKey, OmMultipartKeyInfo omMultipartKeyInfo, + OmBucketInfo omBucketInfo, OMResponse omResponse) { + return new S3MultipartUploadAbortResponse(omResponse, multipartKey, + omMultipartKeyInfo, true, omBucketInfo); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponse.java index a11c4db..ae8650e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponse.java @@ -43,10 +43,10 @@ public class TestS3MultipartUploadAbortResponse String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); - String keyName = UUID.randomUUID().toString(); + String keyName = getKeyName(); String multipartUploadID = UUID.randomUUID().toString(); - String multipartKey = omMetadataManager.getMultipartKey(volumeName, - bucketName, keyName, multipartUploadID); + String multipartKey = getMultipartKey(volumeName, bucketName, keyName, + multipartUploadID); OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder() .setVolumeName(volumeName).setBucketName(bucketName) @@ -82,10 +82,10 @@ public class TestS3MultipartUploadAbortResponse String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); - String keyName = UUID.randomUUID().toString(); + String keyName = getKeyName(); String multipartUploadID = UUID.randomUUID().toString(); - String multipartKey = omMetadataManager.getMultipartKey(volumeName, - bucketName, keyName, multipartUploadID); + String multipartKey = getMultipartKey(volumeName, bucketName, keyName, + multipartUploadID); OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder() .setVolumeName(volumeName).setBucketName(bucketName) @@ -154,4 +154,14 @@ public class TestS3MultipartUploadAbortResponse ro.getOmKeyInfoList().get(0)); } + protected String getKeyName() { + return UUID.randomUUID().toString(); + } + + protected String getMultipartKey(String volumeName, String bucketName, + String keyName, String multipartUploadID) { + return omMetadataManager.getMultipartKey(volumeName, + bucketName, keyName, multipartUploadID); + } + } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseV1.java new file mode 100644 index 0000000..115b277 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseV1.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.s3.multipart; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; + +import java.util.ArrayList; +import java.util.UUID; + +/** + * Test multipart upload abort response. + */ +public class TestS3MultipartUploadAbortResponseV1 + extends TestS3MultipartUploadAbortResponse { + + private String dirName = "abort/b/c/"; + + private long parentID = 1027; + + protected String getKeyName() { + return dirName + UUID.randomUUID().toString(); + } + + protected String getMultipartKey(String volumeName, String bucketName, + String keyName, String multipartUploadID) { + String fileName = StringUtils.substringAfter(keyName, dirName); + return omMetadataManager.getMultipartKey(parentID, fileName, + multipartUploadID); + } + + protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp( + OmMultipartKeyInfo multipartKeyInfo, OmKeyInfo omKeyInfo, + OzoneManagerProtocolProtos.OMResponse omResponse) { + return new S3InitiateMultipartUploadResponseV1(omResponse, multipartKeyInfo, + omKeyInfo, new ArrayList<>()); + } + + protected S3MultipartUploadAbortResponse getS3MultipartUploadAbortResp( + String multipartKey, OmMultipartKeyInfo omMultipartKeyInfo, + OmBucketInfo omBucketInfo, + OzoneManagerProtocolProtos.OMResponse omResponse) { + return new S3MultipartUploadAbortResponseV1(omResponse, multipartKey, + omMultipartKeyInfo, true, omBucketInfo); + } + + @Override + public OzoneManagerProtocolProtos.PartKeyInfo createPartKeyInfo( + String volumeName, String bucketName, String keyName, int partNumber) { + + String fileName = OzoneFSUtils.getFileName(keyName); + return createPartKeyInfoV1(volumeName, bucketName, parentID, fileName, + partNumber); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
