This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 930aee22f5 HDDS-8259. [hsync] OMKeyRequest: Detect allocated but
uncommitted blocks (#4761)
930aee22f5 is described below
commit 930aee22f558ad5e613f54ec5e49a4d50eea9bd8
Author: Wei-Chiu Chuang <[email protected]>
AuthorDate: Wed May 31 15:32:31 2023 -0700
HDDS-8259. [hsync] OMKeyRequest: Detect allocated but uncommitted blocks
(#4761)
---
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 24 ++++++++++++++++++++++
.../ozone/om/request/key/OMKeyCommitRequest.java | 8 ++++----
.../om/request/key/OMKeyCommitRequestWithFSO.java | 8 ++++----
3 files changed, 32 insertions(+), 8 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
index 8cd2809c3b..d3fe1cda0b 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
@@ -160,6 +160,30 @@ public class TestHSync {
}
}
+ @Test
+ public void testUncommittedBlocks() throws Exception {
+ // Set the fs.defaultFS
+ final String rootPath = String.format("%s://%s/",
+ OZONE_OFS_URI_SCHEME, CONF.get(OZONE_OM_ADDRESS_KEY));
+ CONF.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+
+ final String dir = OZONE_ROOT + bucket.getVolumeName()
+ + OZONE_URI_DELIMITER + bucket.getName();
+ final byte[] data = new byte[1];
+ ThreadLocalRandom.current().nextBytes(data);
+
+ try (FileSystem fs = FileSystem.get(CONF)) {
+ final Path file = new Path(dir, "file");
+ try (FSDataOutputStream outputStream = fs.create(file, true)) {
+ outputStream.hsync();
+ outputStream.write(data);
+ outputStream.hsync();
+ assertTrue(cluster.getOzoneManager().getMetadataManager()
+ .getDeletedTable().isEmpty());
+ }
+ }
+ }
+
static void runTestHSync(FileSystem fs, Path file, int initialDataSize)
throws Exception {
try (StreamWithLength out = new StreamWithLength(
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 39879e0333..80e3c33301 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -217,8 +217,8 @@ public class OMKeyCommitRequest extends OMKeyRequest {
omKeyInfo.setModificationTime(commitKeyArgs.getModificationTime());
// Update the block length for each block, return the allocated but
// uncommitted blocks
- List<OmKeyLocationInfo> uncommitted = omKeyInfo.updateLocationInfoList(
- locationInfoList, false);
+ List<OmKeyLocationInfo> uncommitted =
+ omKeyInfo.updateLocationInfoList(locationInfoList, false);
// Set the UpdateID to current transactionLogIndex
omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
@@ -252,8 +252,8 @@ public class OMKeyCommitRequest extends OMKeyRequest {
// let the uncommitted blocks pretend as key's old version blocks
// which will be deleted as RepeatedOmKeyInfo
- OmKeyInfo pseudoKeyInfo = wrapUncommittedBlocksAsPseudoKey(uncommitted,
- omKeyInfo);
+ final OmKeyInfo pseudoKeyInfo = isHSync ? null
+ : wrapUncommittedBlocksAsPseudoKey(uncommitted, omKeyInfo);
if (pseudoKeyInfo != null) {
long pseudoObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex);
String delKeyName = omMetadataManager.getOzoneDeletePathKey(
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
index 0de87732f5..fa7f92e9a8 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
@@ -163,8 +163,8 @@ public class OMKeyCommitRequestWithFSO extends
OMKeyCommitRequest {
omKeyInfo.setModificationTime(commitKeyArgs.getModificationTime());
- List<OmKeyLocationInfo> uncommitted = omKeyInfo.updateLocationInfoList(
- locationInfoList, false);
+ List<OmKeyLocationInfo> uncommitted =
+ omKeyInfo.updateLocationInfoList(locationInfoList, false);
// Set the UpdateID to current transactionLogIndex
omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
@@ -209,8 +209,8 @@ public class OMKeyCommitRequestWithFSO extends
OMKeyCommitRequest {
// let the uncommitted blocks pretend as key's old version blocks
// which will be deleted as RepeatedOmKeyInfo
- OmKeyInfo pseudoKeyInfo = wrapUncommittedBlocksAsPseudoKey(uncommitted,
- omKeyInfo);
+ final OmKeyInfo pseudoKeyInfo = isHSync ? null
+ : wrapUncommittedBlocksAsPseudoKey(uncommitted, omKeyInfo);
if (pseudoKeyInfo != null) {
String delKeyName = omMetadataManager
.getOzoneKey(volumeName, bucketName, fileName);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
