This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4fe6e1f24c0 HDDS-16175. Do not change usedNamespace on hsync re-commit
(#11092)
4fe6e1f24c0 is described below
commit 4fe6e1f24c08d03692bc1e96f76048bf05dc8fd9
Author: Chi-Hsuan Huang <[email protected]>
AuthorDate: Tue Aug 25 17:54:23 2026 +0800
HDDS-16175. Do not change usedNamespace on hsync re-commit (#11092)
---
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 44 ++++++++++++++++++++++
.../ozone/om/request/key/OMKeyCommitRequest.java | 6 +--
.../om/request/key/OMKeyCommitRequestWithFSO.java | 5 ++-
.../om/request/key/TestOMKeyCommitRequest.java | 26 +++++++++++++
4 files changed, 76 insertions(+), 5 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
index b3314579c13..02fa289c023 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
@@ -1288,6 +1288,50 @@ public void writeWithBigBuffer(boolean
incrementalChunkList, int bufferSize)
bucket.deleteKey(keyName);
}
+ @Test
+ public void testUsedNamespaceWithRepeatedHsync() throws Exception {
+ final String rootPath = String.format("%s://%s/",
+ OZONE_OFS_URI_SCHEME, CONF.get(OZONE_OM_ADDRESS_KEY));
+ CONF.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+
+ final String dir = OZONE_ROOT + bucket.getVolumeName()
+ + OZONE_URI_DELIMITER + bucket.getName();
+ final Path file = new Path(dir, "file-hsync-used-namespace");
+
+ OzoneManager ozoneManager = cluster.getOzoneManager();
+ OMMetadataManager metadataManager = ozoneManager.getMetadataManager();
+ String bucketKey = metadataManager.getBucketKey(bucket.getVolumeName(),
bucket.getName());
+ final long usedNamespace =
metadataManager.getBucketTable().get(bucketKey).getUsedNamespace();
+
+ try (FileSystem fs = FileSystem.get(CONF)) {
+ try {
+ try (FSDataOutputStream out = fs.create(file, true)) {
+ // Each block triggers an hsync commit; only the first adds the key.
+ byte[] blockData =
RandomStringUtils.secure().nextAlphabetic(BLOCK_SIZE)
+ .getBytes(UTF_8);
+ for (int i = 0; i < 4; i++) {
+ out.write(blockData);
+ out.hsync();
+ }
+ }
+ assertEquals(usedNamespace + 1,
+
metadataManager.getBucketTable().get(bucketKey).getUsedNamespace());
+
+ assertTrue(fs.delete(file, false));
+ GenericTestUtils.waitFor((CheckedSupplier<Boolean, IOException>) () ->
+ metadataManager.getBucketTable().get(bucketKey).getUsedNamespace()
== usedNamespace,
+ 100, 30000);
+ } finally {
+ try {
+ fs.delete(file, false);
+ waitForEmptyDeletedTable();
+ } finally {
+ cleanupOpenKeyTable(ozoneManager, BUCKET_LAYOUT);
+ }
+ }
+ }
+ }
+
@Test
public void testNormalKeyOverwriteHSyncKey() throws Exception {
// Set the fs.defaultFS
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 6c34443058b..0789e128850 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -319,8 +319,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
Map<String, RepeatedOmKeyInfo> oldKeyVersionsToDeleteMap = null;
long correctedSpace = omKeyInfo.getReplicatedSize();
- // if keyToDelete isn't null, usedNamespace needn't check and
- // increase.
+ // Same-client hsync re-commit does not consume namespace.
if (keyToDelete != null && (isSameHsyncKey)) {
correctedSpace -= keyToDelete.getReplicatedSize();
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
@@ -367,12 +366,13 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
// Subtract the used namespace of empty overwritten keys.
omBucketInfo.decrUsedNamespace(filteredUsedBlockCnt.getRight(), false);
omBucketInfo.decrUsedBytes(totalSize, true);
+ omBucketInfo.incrUsedNamespace(1L);
} else {
checkBucketQuotaInNamespace(omBucketInfo, 1L);
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
correctedSpace);
+ omBucketInfo.incrUsedNamespace(1L);
}
- omBucketInfo.incrUsedNamespace(1L);
// let the uncommitted blocks pretend as key's old version blocks
// which will be deleted as RepeatedOmKeyInfo
final OmKeyInfo pseudoKeyInfo = isHSync ? null
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
index f9aa275ef75..9bc6f7ec0d9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
@@ -250,7 +250,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
.build();
long correctedSpace = omKeyInfo.getReplicatedSize();
- // if keyToDelete isn't null, usedNamespace shouldn't check and increase.
+ // Same-client hsync re-commit does not consume namespace.
if (keyToDelete != null && isSameHsyncKey) {
correctedSpace -= keyToDelete.getReplicatedSize();
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
@@ -299,12 +299,13 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
omBucketInfo.decrUsedNamespace(totalNamespace, true);
omBucketInfo.decrUsedNamespace(filteredUsedBlockCnt.getRight(), false);
omBucketInfo.decrUsedBytes(totalSize, true);
+ omBucketInfo.incrUsedNamespace(1L);
} else {
checkBucketQuotaInNamespace(omBucketInfo, 1L);
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
correctedSpace);
+ omBucketInfo.incrUsedNamespace(1L);
}
- omBucketInfo.incrUsedNamespace(1L);
// let the uncommitted blocks pretend as key's old version blocks
// which will be deleted as RepeatedOmKeyInfo
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
index e602d4faa04..2a87c8e00a8 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
@@ -511,6 +511,32 @@ public void testCommitWithHsyncIncrementalUsages() throws
Exception {
assertEquals(1000, thirdCommitUsedBytes - usedBytes);
}
+ @Test
+ public void testCommitWithHsyncUsedNamespace() throws Exception {
+ BucketLayout bucketLayout = getBucketLayout();
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, bucketLayout);
+ List<KeyLocation> allocatedKeyLocationList = getKeyLocation(10);
+
+ doKeyCommit(true, allocatedKeyLocationList.subList(0, 3));
+ doKeyCommit(true, allocatedKeyLocationList.subList(0, 6));
+ doKeyCommit(false, allocatedKeyLocationList);
+
+ OmBucketInfo bucketInfo =
omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(1, bucketInfo.getUsedNamespace());
+
+ clientID = Time.now();
+ version += 1;
+ Map<String, RepeatedOmKeyInfo> keyToDeleteMap =
+ doKeyCommit(false, getKeyLocation(20).subList(10, 20));
+ assertThat(keyToDeleteMap).isNotEmpty();
+
+ bucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(1, bucketInfo.getUsedNamespace());
+ }
+
private Map<String, RepeatedOmKeyInfo> doKeyCommit(boolean isHSync,
List<KeyLocation> keyLocations) throws Exception {
// allocated block list
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
