This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ea75a1275d HDDS-6399. Update bucket usage for discarded parts on
MultipartUploadComplete. (#5231)
ea75a1275d is described below
commit ea75a1275d3b9dbcc0a941808e1a47b5bf77de2b
Author: Arafat2198 <[email protected]>
AuthorDate: Tue Sep 26 00:10:45 2023 +0530
HDDS-6399. Update bucket usage for discarded parts on
MultipartUploadComplete. (#5231)
---
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 30 ++++++++++++++++++++++
.../S3MultipartUploadCompleteRequest.java | 8 +++---
2 files changed, 35 insertions(+), 3 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
index f89aa85d15..ae1a222f5c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
@@ -360,6 +360,36 @@ public class TestOzoneClientMultipartUploadWithFSO {
() -> completeMultipartUpload(bucket, keyName, uploadID, partsMap));
}
+ @Test
+ public void testMultipartUploadWithDiscardedUnusedPartSize()
+ throws Exception {
+ // Initiate multipart upload
+ String uploadID = initiateMultipartUpload(bucket, keyName, RATIS, ONE);
+ byte[] data = generateData(10000000, (byte) 97);
+
+ // Upload Parts
+ Map<Integer, String> partsMap = new TreeMap<>();
+
+ // Upload part 1 and add it to the partsMap for completing the upload.
+ String partName1 = uploadPart(bucket, keyName, uploadID, 1, data);
+ partsMap.put(1, partName1);
+
+ // Upload part 2 and add it to the partsMap for completing the upload.
+ String partName2 = uploadPart(bucket, keyName, uploadID, 2, data);
+ partsMap.put(2, partName2);
+
+ // Upload part 3 but do not add it to the partsMap.
+ uploadPart(bucket, keyName, uploadID, 3, data);
+
+ completeMultipartUpload(bucket, keyName, uploadID, partsMap);
+
+ // Check the bucket size. Since part number 3 was not added to the
partsMap,
+ // the unused part size should be discarded from the bucket size,
+ // 30000000 - 10000000 = 20000000
+ long bucketSize = volume.getBucket(bucketName).getUsedBytes();
+ Assert.assertEquals(bucketSize, data.length * 2);
+ }
+
@Test
public void testMultipartUploadWithPartsMisMatchWithListSizeDifferent()
throws Exception {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index 52bc3feba4..ab6d6f8b95 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -203,12 +203,15 @@ public class S3MultipartUploadCompleteRequest extends
OMKeyRequest {
bucketName, keyName, dbMultipartOpenKey, omMetadataManager,
dbOzoneKey, partKeyInfoMap, partLocationInfos, dataSize);
+ long usedBytesDiff = 0;
//Find all unused parts.
List<OmKeyInfo> allKeyInfoToRemove = new ArrayList<>();
for (PartKeyInfo partKeyInfo : partKeyInfoMap) {
if (!partNumbers.contains(partKeyInfo.getPartNumber())) {
- allKeyInfoToRemove.add(OmKeyInfo
- .getFromProtobuf(partKeyInfo.getPartKeyInfo()));
+ OmKeyInfo delPartKeyInfo =
+ OmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo());
+ allKeyInfoToRemove.add(delPartKeyInfo);
+ usedBytesDiff -= delPartKeyInfo.getReplicatedSize();
}
}
@@ -218,7 +221,6 @@ public class S3MultipartUploadCompleteRequest extends
OMKeyRequest {
// creation after the knob turned on.
OmKeyInfo keyToDelete =
omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey);
- long usedBytesDiff = 0;
boolean isNamespaceUpdate = false;
if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()) {
RepeatedOmKeyInfo oldKeyVersionsToDelete = getOldVersionsToCleanUp(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
