This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a4edeb8cd9 HDDS-9164. [Hsync] moves blocks to deleted table on final
commit (#5183)
a4edeb8cd9 is described below
commit a4edeb8cd9d4a17b059c012160004e8dc14dc873
Author: Sumit Agrawal <[email protected]>
AuthorDate: Fri Sep 8 08:13:52 2023 +0530
HDDS-9164. [Hsync] moves blocks to deleted table on final commit (#5183)
---
.../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 13 ++++++++++++-
.../ozone/om/request/key/OMKeyCommitRequestWithFSO.java | 13 ++++++++++++-
.../hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java | 5 ++---
3 files changed, 26 insertions(+), 5 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 33bf839167..69e24bf41e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.WithMetadata;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
@@ -217,9 +218,19 @@ public class OMKeyCommitRequest extends OMKeyRequest {
// creation and key commit, old versions will be just overwritten and
// not kept. Bucket versioning will be effective from the first key
// creation after the knob turned on.
+ boolean isPreviousCommitHsync = false;
Map<String, RepeatedOmKeyInfo> oldKeyVersionsToDeleteMap = null;
OmKeyInfo keyToDelete =
omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey);
+ if (null != keyToDelete) {
+ final String clientIdString
+ = String.valueOf(commitKeyRequest.getClientID());
+ isPreviousCommitHsync = java.util.Optional.ofNullable(keyToDelete)
+ .map(WithMetadata::getMetadata)
+ .map(meta -> meta.get(OzoneConsts.HSYNC_CLIENT_ID))
+ .filter(id -> id.equals(clientIdString))
+ .isPresent();
+ }
omKeyInfo =
omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKey);
@@ -248,7 +259,7 @@ public class OMKeyCommitRequest extends OMKeyRequest {
long correctedSpace = omKeyInfo.getReplicatedSize();
// if keyToDelete isn't null, usedNamespace needn't check and
// increase.
- if (keyToDelete != null && isHSync) {
+ if (keyToDelete != null && (isHSync || isPreviousCommitHsync)) {
correctedSpace -= keyToDelete.getReplicatedSize();
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
correctedSpace);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
index 9b4094a381..89ed0573e9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.WithMetadata;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
@@ -180,15 +181,25 @@ public class OMKeyCommitRequestWithFSO extends
OMKeyCommitRequest {
// creation and key commit, old versions will be just overwritten and
// not kept. Bucket versioning will be effective from the first key
// creation after the knob turned on.
+ boolean isPreviousCommitHsync = false;
Map<String, RepeatedOmKeyInfo> oldKeyVersionsToDeleteMap = null;
OmKeyInfo keyToDelete =
omMetadataManager.getKeyTable(getBucketLayout()).get(dbFileKey);
+ if (null != keyToDelete) {
+ final String clientIdString
+ = String.valueOf(commitKeyRequest.getClientID());
+ isPreviousCommitHsync = java.util.Optional.ofNullable(keyToDelete)
+ .map(WithMetadata::getMetadata)
+ .map(meta -> meta.get(OzoneConsts.HSYNC_CLIENT_ID))
+ .filter(id -> id.equals(clientIdString))
+ .isPresent();
+ }
long correctedSpace = omKeyInfo.getReplicatedSize();
// if keyToDelete isn't null, usedNamespace shouldn't check and
// increase.
- if (keyToDelete != null && isHSync) {
+ if (keyToDelete != null && (isHSync || isPreviousCommitHsync)) {
correctedSpace -= keyToDelete.getReplicatedSize();
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
correctedSpace);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
index 4c610141b6..4a12fb28d4 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
@@ -352,10 +352,9 @@ public class TestOMKeyCommitRequest extends
TestOMKeyRequest {
// 3rd and final commit of all 10 blocks, HSync = false
keyToDeleteMap = doKeyCommit(false, allocatedKeyLocationList);
- // keyToDeleteMap should be empty because none of the previous blocks
+ // keyToDeleteMap should be null / empty because none of the previous
blocks
// should be deleted.
- Assert.assertNotNull(keyToDeleteMap);
- Assert.assertTrue(keyToDeleteMap.isEmpty());
+ Assert.assertTrue(keyToDeleteMap == null || keyToDeleteMap.isEmpty());
bucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
long thirdCommitUsedBytes = bucketInfo.getUsedBytes();
Assert.assertEquals(1000, thirdCommitUsedBytes - usedBytes);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
