This is an automated email from the ASF dual-hosted git repository. jojochuang pushed a commit to branch ozone-2.1 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit a22823ebff5fa643b298b0016cd753cc8d475079 Author: Chi-Hsuan Huang <[email protected]> AuthorDate: Tue Aug 25 17:54:23 2026 +0800 HDDS-16175. Do not change usedNamespace on hsync re-commit (#11092) (cherry picked from commit 4fe6e1f24c08d03692bc1e96f76048bf05dc8fd9) --- .../java/org/apache/hadoop/fs/ozone/TestHSync.java | 44 ++++++++++++++++++++++ .../ozone/om/request/key/OMKeyCommitRequest.java | 6 +-- .../om/request/key/OMKeyCommitRequestWithFSO.java | 5 ++- .../om/request/key/TestOMKeyCommitRequest.java | 26 +++++++++++++ 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index 9643598e7e9..07ae0195513 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java @@ -1288,6 +1288,50 @@ public void writeWithBigBuffer(boolean incrementalChunkList, int bufferSize) bucket.deleteKey(keyName); } + @Test + public void testUsedNamespaceWithRepeatedHsync() throws Exception { + final String rootPath = String.format("%s://%s/", + OZONE_OFS_URI_SCHEME, CONF.get(OZONE_OM_ADDRESS_KEY)); + CONF.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); + + final String dir = OZONE_ROOT + bucket.getVolumeName() + + OZONE_URI_DELIMITER + bucket.getName(); + final Path file = new Path(dir, "file-hsync-used-namespace"); + + OzoneManager ozoneManager = cluster.getOzoneManager(); + OMMetadataManager metadataManager = ozoneManager.getMetadataManager(); + String bucketKey = metadataManager.getBucketKey(bucket.getVolumeName(), bucket.getName()); + final long usedNamespace = metadataManager.getBucketTable().get(bucketKey).getUsedNamespace(); + + try (FileSystem fs = FileSystem.get(CONF)) { + try { + try (FSDataOutputStream out = fs.create(file, true)) { + // Each block triggers an hsync commit; only the first adds the key. + byte[] blockData = RandomStringUtils.secure().nextAlphabetic(BLOCK_SIZE) + .getBytes(UTF_8); + for (int i = 0; i < 4; i++) { + out.write(blockData); + out.hsync(); + } + } + assertEquals(usedNamespace + 1, + metadataManager.getBucketTable().get(bucketKey).getUsedNamespace()); + + assertTrue(fs.delete(file, false)); + GenericTestUtils.waitFor((CheckedSupplier<Boolean, IOException>) () -> + metadataManager.getBucketTable().get(bucketKey).getUsedNamespace() == usedNamespace, + 100, 30000); + } finally { + try { + fs.delete(file, false); + waitForEmptyDeletedTable(); + } finally { + cleanupOpenKeyTable(ozoneManager, BUCKET_LAYOUT); + } + } + } + } + @Test public void testNormalKeyOverwriteHSyncKey() throws Exception { // Set the fs.defaultFS diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 31f1d9d7180..7d0b1e95d40 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -312,8 +312,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut Map<String, RepeatedOmKeyInfo> oldKeyVersionsToDeleteMap = null; long correctedSpace = omKeyInfo.getReplicatedSize(); - // if keyToDelete isn't null, usedNamespace needn't check and - // increase. + // Same-client hsync re-commit does not consume namespace. if (keyToDelete != null && (isSameHsyncKey)) { correctedSpace -= keyToDelete.getReplicatedSize(); checkBucketQuotaInBytes(omMetadataManager, omBucketInfo, @@ -357,12 +356,13 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Subtract the used namespace of empty overwritten keys. omBucketInfo.decrUsedNamespace(filteredUsedBlockCnt.getRight(), false); omBucketInfo.decrUsedBytes(totalSize, true); + omBucketInfo.incrUsedNamespace(1L); } else { checkBucketQuotaInNamespace(omBucketInfo, 1L); checkBucketQuotaInBytes(omMetadataManager, omBucketInfo, correctedSpace); + omBucketInfo.incrUsedNamespace(1L); } - omBucketInfo.incrUsedNamespace(1L); // let the uncommitted blocks pretend as key's old version blocks // which will be deleted as RepeatedOmKeyInfo final OmKeyInfo pseudoKeyInfo = isHSync ? null diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index a23716d40d1..377471f3d22 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -243,7 +243,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omKeyInfo.setExpectedDataGeneration(null); long correctedSpace = omKeyInfo.getReplicatedSize(); - // if keyToDelete isn't null, usedNamespace shouldn't check and increase. + // Same-client hsync re-commit does not consume namespace. if (keyToDelete != null && isSameHsyncKey) { correctedSpace -= keyToDelete.getReplicatedSize(); checkBucketQuotaInBytes(omMetadataManager, omBucketInfo, @@ -289,12 +289,13 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omBucketInfo.decrUsedNamespace(totalNamespace, true); omBucketInfo.decrUsedNamespace(filteredUsedBlockCnt.getRight(), false); omBucketInfo.decrUsedBytes(totalSize, true); + omBucketInfo.incrUsedNamespace(1L); } else { checkBucketQuotaInNamespace(omBucketInfo, 1L); checkBucketQuotaInBytes(omMetadataManager, omBucketInfo, correctedSpace); + omBucketInfo.incrUsedNamespace(1L); } - omBucketInfo.incrUsedNamespace(1L); // let the uncommitted blocks pretend as key's old version blocks // which will be deleted as RepeatedOmKeyInfo diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index e8bd2b07941..18ec9595181 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -444,6 +444,32 @@ public void testCommitWithHsyncIncrementalUsages() throws Exception { assertEquals(1000, thirdCommitUsedBytes - usedBytes); } + @Test + public void testCommitWithHsyncUsedNamespace() throws Exception { + BucketLayout bucketLayout = getBucketLayout(); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, bucketLayout); + List<KeyLocation> allocatedKeyLocationList = getKeyLocation(10); + + doKeyCommit(true, allocatedKeyLocationList.subList(0, 3)); + doKeyCommit(true, allocatedKeyLocationList.subList(0, 6)); + doKeyCommit(false, allocatedKeyLocationList); + + OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(bucketKey); + assertEquals(1, bucketInfo.getUsedNamespace()); + + clientID = Time.now(); + version += 1; + Map<String, RepeatedOmKeyInfo> keyToDeleteMap = + doKeyCommit(false, getKeyLocation(20).subList(10, 20)); + assertThat(keyToDeleteMap).isNotEmpty(); + + bucketInfo = omMetadataManager.getBucketTable().get(bucketKey); + assertEquals(1, bucketInfo.getUsedNamespace()); + } + private Map<String, RepeatedOmKeyInfo> doKeyCommit(boolean isHSync, List<KeyLocation> keyLocations) throws Exception { // allocated block list --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
