This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a commit to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit cdc78552fd6d37ed85a3a86ef8193948eca6fb2f Author: Rakesh Radhakrishnan <[email protected]> AuthorDate: Wed Mar 10 22:04:39 2021 +0530 HDDS-4924. [FSO]S3Multipart: Implement OzoneBucket#listParts (#2016) --- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 2 +- .../rpc/TestOzoneClientMultipartUploadV1.java | 265 +++++++++++++++++++++ .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 58 ++++- .../S3MultipartUploadCompleteRequest.java | 1 - 4 files changed, 321 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java index c63c21f..f1f6454 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java @@ -178,7 +178,7 @@ public final class OzoneFSUtils { if (fileName != null) { return fileName.toString(); } - // failed to find a parent directory. + // no parent directory. return ""; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java index 76feec8..0e981d6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java @@ -18,15 +18,18 @@ package org.apache.hadoop.ozone.client.rpc; import org.apache.commons.lang3.RandomUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneTestUtils; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; @@ -48,6 +51,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -58,9 +62,11 @@ import org.junit.rules.Timeout; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -566,6 +572,265 @@ public class TestOzoneClientMultipartUploadV1 { // not making any assertion for the same. } + @Test + public void testListMultipartUploadParts() throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String parentDir = "a/b/c/d/e/f/"; + String keyName = parentDir + "file-ABC"; + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + Map<Integer, String> partsMap = new TreeMap<>(); + String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE, + ONE); + String partName1 = uploadPart(bucket, keyName, uploadID, 1, + generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97)); + partsMap.put(1, partName1); + + String partName2 =uploadPart(bucket, keyName, uploadID, 2, + generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97)); + partsMap.put(2, partName2); + + String partName3 =uploadPart(bucket, keyName, uploadID, 3, + generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97)); + partsMap.put(3, partName3); + + OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts = + bucket.listParts(keyName, uploadID, 0, 3); + + Assert.assertEquals(STAND_ALONE, + ozoneMultipartUploadPartListParts.getReplicationType()); + Assert.assertEquals(3, + ozoneMultipartUploadPartListParts.getPartInfoList().size()); + + verifyPartNamesInDB(volumeName, bucketName, parentDir, keyName, partsMap, + ozoneMultipartUploadPartListParts, uploadID); + + Assert.assertFalse(ozoneMultipartUploadPartListParts.isTruncated()); + } + + private void verifyPartNamesInDB(String volumeName, String bucketName, + String parentDir, String keyName, Map<Integer, String> partsMap, + OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts, + String uploadID) throws IOException { + + List<String> listPartNames = new ArrayList<>(); + String keyPartName = verifyPartNames(partsMap, 0, + ozoneMultipartUploadPartListParts); + listPartNames.add(keyPartName); + + keyPartName = verifyPartNames(partsMap, 1, + ozoneMultipartUploadPartListParts); + listPartNames.add(keyPartName); + + keyPartName = verifyPartNames(partsMap, 2, + ozoneMultipartUploadPartListParts); + listPartNames.add(keyPartName); + + OMMetadataManager metadataMgr = + cluster.getOzoneManager().getMetadataManager(); + String multipartKey = getMultipartKey(uploadID, volumeName, bucketName, + keyName, metadataMgr); + OmMultipartKeyInfo omMultipartKeyInfo = + metadataMgr.getMultipartInfoTable().get(multipartKey); + Assert.assertNotNull(omMultipartKeyInfo); + + long parentID = getParentID(volumeName, bucketName, keyName, metadataMgr); + TreeMap<Integer, OzoneManagerProtocolProtos.PartKeyInfo> partKeyInfoMap = + omMultipartKeyInfo.getPartKeyInfoMap(); + for (Map.Entry<Integer, OzoneManagerProtocolProtos.PartKeyInfo> entry : + partKeyInfoMap.entrySet()) { + OzoneManagerProtocolProtos.PartKeyInfo partKeyInfo = entry.getValue(); + String partKeyName = partKeyInfo.getPartName(); + + // partKeyName format in DB - <parentID>/partFileName + ClientID + Assert.assertTrue("Invalid partKeyName format in DB", + partKeyName.startsWith(parentID + OzoneConsts.OM_KEY_PREFIX)); + partKeyName = StringUtils.remove(partKeyName, + parentID + OzoneConsts.OM_KEY_PREFIX); + + // reconstruct full part name with volume, bucket, partKeyName + String fullKeyPartName = metadataMgr.getOzoneKey(volumeName, bucketName, + parentDir + partKeyName); + + listPartNames.remove(fullKeyPartName); + } + + Assert.assertTrue("Wrong partKeyName format in DB!", + listPartNames.isEmpty()); + } + + private String verifyPartNames(Map<Integer, String> partsMap, int index, + OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts) { + + Assert.assertEquals(partsMap.get(ozoneMultipartUploadPartListParts + .getPartInfoList().get(index).getPartNumber()), + ozoneMultipartUploadPartListParts.getPartInfoList().get(index) + .getPartName()); + + return ozoneMultipartUploadPartListParts.getPartInfoList().get(index) + .getPartName(); + } + + @Test + public void testListMultipartUploadPartsWithContinuation() + throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + Map<Integer, String> partsMap = new TreeMap<>(); + String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE, + ONE); + String partName1 = uploadPart(bucket, keyName, uploadID, 1, + generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97)); + partsMap.put(1, partName1); + + String partName2 =uploadPart(bucket, keyName, uploadID, 2, + generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97)); + partsMap.put(2, partName2); + + String partName3 =uploadPart(bucket, keyName, uploadID, 3, + generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97)); + partsMap.put(3, partName3); + + OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts = + bucket.listParts(keyName, uploadID, 0, 2); + + Assert.assertEquals(STAND_ALONE, + ozoneMultipartUploadPartListParts.getReplicationType()); + + Assert.assertEquals(2, + ozoneMultipartUploadPartListParts.getPartInfoList().size()); + + Assert.assertEquals(partsMap.get(ozoneMultipartUploadPartListParts + .getPartInfoList().get(0).getPartNumber()), + ozoneMultipartUploadPartListParts.getPartInfoList().get(0) + .getPartName()); + Assert.assertEquals(partsMap.get(ozoneMultipartUploadPartListParts + .getPartInfoList().get(1).getPartNumber()), + ozoneMultipartUploadPartListParts.getPartInfoList().get(1) + .getPartName()); + + // Get remaining + Assert.assertTrue(ozoneMultipartUploadPartListParts.isTruncated()); + ozoneMultipartUploadPartListParts = bucket.listParts(keyName, uploadID, + ozoneMultipartUploadPartListParts.getNextPartNumberMarker(), 2); + + Assert.assertEquals(1, + ozoneMultipartUploadPartListParts.getPartInfoList().size()); + Assert.assertEquals(partsMap.get(ozoneMultipartUploadPartListParts + .getPartInfoList().get(0).getPartNumber()), + ozoneMultipartUploadPartListParts.getPartInfoList().get(0) + .getPartName()); + + + // As we don't have any parts for this, we should get false here + Assert.assertFalse(ozoneMultipartUploadPartListParts.isTruncated()); + + } + + @Test + public void testListPartsInvalidPartMarker() throws Exception { + try { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + bucket.listParts(keyName, "random", -1, 2); + Assert.fail("Should throw exception as partNumber is an invalid number!"); + } catch (IllegalArgumentException ex) { + GenericTestUtils.assertExceptionContains("Should be greater than or " + + "equal to zero", ex); + } + } + + @Test + public void testListPartsInvalidMaxParts() throws Exception { + try { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + bucket.listParts(keyName, "random", 1, -1); + Assert.fail("Should throw exception as max parts is an invalid number!"); + } catch (IllegalArgumentException ex) { + GenericTestUtils.assertExceptionContains("Max Parts Should be greater " + + "than zero", ex); + } + } + + @Test + public void testListPartsWithPartMarkerGreaterThanPartCount() + throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + + String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE, + ONE); + uploadPart(bucket, keyName, uploadID, 1, + generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97)); + + + OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts = + bucket.listParts(keyName, uploadID, 100, 2); + + // Should return empty + + Assert.assertEquals(0, + ozoneMultipartUploadPartListParts.getPartInfoList().size()); + Assert.assertEquals(STAND_ALONE, + ozoneMultipartUploadPartListParts.getReplicationType()); + + // As we don't have any parts with greater than partNumberMarker and list + // is not truncated, so it should return false here. + Assert.assertFalse(ozoneMultipartUploadPartListParts.isTruncated()); + + } + + @Test + public void testListPartsWithInvalidUploadID() throws Exception { + OzoneTestUtils + .expectOmException(NO_SUCH_MULTIPART_UPLOAD_ERROR, () -> { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts = + bucket.listParts(keyName, "random", 100, 2); + }); + } + private String verifyUploadedPart(String volumeName, String bucketName, String keyName, String uploadID, String partName, OMMetadataManager metadataMgr) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 24afc5f..f45a96b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1416,8 +1416,8 @@ public class KeyManagerImpl implements KeyManager { metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName); try { - String multipartKey = metadataManager.getMultipartKey(volumeName, - bucketName, keyName, uploadID); + String multipartKey = getMultipartKey(volumeName, bucketName, + keyName, uploadID); OmMultipartKeyInfo multipartKeyInfo = metadataManager.getMultipartInfoTable().get(multipartKey); @@ -1445,8 +1445,10 @@ public class KeyManagerImpl implements KeyManager { // than part number marker if (partKeyInfoEntry.getKey() > partNumberMarker) { PartKeyInfo partKeyInfo = partKeyInfoEntry.getValue(); + String partName = getPartName(partKeyInfo, volumeName, bucketName, + keyName); OmPartInfo omPartInfo = new OmPartInfo(partKeyInfo.getPartNumber(), - partKeyInfo.getPartName(), + partName, partKeyInfo.getPartKeyInfo().getModificationTime(), partKeyInfo.getPartKeyInfo().getDataSize()); omPartInfoList.add(omPartInfo); @@ -1506,6 +1508,56 @@ public class KeyManagerImpl implements KeyManager { } } + private String getPartName(PartKeyInfo partKeyInfo, String volName, + String buckName, String keyName) { + + String partName = partKeyInfo.getPartName(); + + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { + String parentDir = OzoneFSUtils.getParentDir(keyName); + String partFileName = OzoneFSUtils.getFileName(partKeyInfo.getPartName()); + + StringBuilder fullKeyPartName = new StringBuilder(); + fullKeyPartName.append(OZONE_URI_DELIMITER); + fullKeyPartName.append(volName); + fullKeyPartName.append(OZONE_URI_DELIMITER); + fullKeyPartName.append(buckName); + if (StringUtils.isNotEmpty(parentDir)) { + fullKeyPartName.append(OZONE_URI_DELIMITER); + fullKeyPartName.append(parentDir); + } + fullKeyPartName.append(OZONE_URI_DELIMITER); + fullKeyPartName.append(partFileName); + + return fullKeyPartName.toString(); + } + return partName; + } + + private String getMultipartKey(String volumeName, String bucketName, + String keyName, String uploadID) throws IOException { + + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { + OMMetadataManager metaMgr = ozoneManager.getMetadataManager(); + String fileName = OzoneFSUtils.getFileName(keyName); + Iterator<Path> pathComponents = Paths.get(keyName).iterator(); + String bucketKey = metaMgr.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = + metaMgr.getBucketTable().get(bucketKey); + long bucketId = omBucketInfo.getObjectID(); + long parentID = OMFileRequest.getParentID(bucketId, pathComponents, + keyName, metaMgr); + + String multipartKey = metaMgr.getMultipartKey(parentID, fileName, + uploadID); + + return multipartKey; + } else { + return metadataManager.getMultipartKey(volumeName, + bucketName, keyName, uploadID); + } + } + /** * Add acl for Ozone object. Return true if acl is added successfully else * false. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 162cf2f..8fcf992 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -33,7 +33,6 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
