This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-7593 by this push:
new f5f737dc5c HDDS-10190. [hsync] Handle lease recovery for file without
blocks. (#6094)
f5f737dc5c is described below
commit f5f737dc5cdb8e8658952e4361d70bf0eb6a781e
Author: ashishkumar50 <[email protected]>
AuthorDate: Sat Jan 27 00:13:17 2024 +0530
HDDS-10190. [hsync] Handle lease recovery for file without blocks. (#6094)
Co-authored-by: ashishk <[email protected]>
---
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 21 ++++++++++++++
.../om/request/file/OMRecoverLeaseRequest.java | 32 ++++++++++++----------
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 18 ++++++------
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 18 ++++++------
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 18 ++++++------
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 18 ++++++------
6 files changed, 78 insertions(+), 47 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
index 68c2d43471..a40c7f5275 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
@@ -433,6 +433,27 @@ public class TestLeaseRecovery {
}
}
+ @Test
+ public void testRecoveryWithoutBlocks() throws Exception {
+ RootedOzoneFileSystem fs = (RootedOzoneFileSystem)FileSystem.get(conf);
+
+ final FSDataOutputStream stream = fs.create(file, true);
+ try {
+ stream.hsync();
+ assertFalse(fs.isFileClosed(file));
+
+ int count = 0;
+ while (count++ < 15 && !fs.recoverLease(file)) {
+ Thread.sleep(1000);
+ }
+ // The lease should have been recovered.
+ assertTrue(fs.isFileClosed(file), "File should be closed");
+
+ } finally {
+ closeIgnoringKeyNotFound(stream);
+ }
+ }
+
private void verifyData(byte[] data, int dataSize, Path filePath,
RootedOzoneFileSystem fs) throws IOException {
try (FSDataInputStream fdis = fs.open(filePath)) {
int bufferSize = dataSize > data.length ? dataSize / 2 : dataSize;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
index addcc54977..6d997ed272 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
@@ -248,29 +248,31 @@ public class OMRecoverLeaseRequest extends OMKeyRequest {
List<OmKeyLocationInfo> keyLocationInfoList =
keyLatestVersionLocations.getLocationList();
OmKeyLocationInfoGroup openKeyLatestVersionLocations =
openKeyInfo.getLatestVersionLocations();
List<OmKeyLocationInfo> openKeyLocationInfoList =
openKeyLatestVersionLocations.getLocationList();
- OmKeyLocationInfo finalBlock;
+
+ OmKeyLocationInfo finalBlock = null;
boolean returnKeyInfo = true;
if (openKeyLocationInfoList.size() > keyLocationInfoList.size() &&
- openKeyModificationTime > keyInfo.getModificationTime()) {
+ openKeyModificationTime > keyInfo.getModificationTime() &&
+ openKeyLocationInfoList.size() > 0) {
finalBlock = openKeyLocationInfoList.get(openKeyLocationInfoList.size()
- 1);
returnKeyInfo = false;
- } else {
+ } else if (keyLocationInfoList.size() > 0) {
finalBlock = keyLocationInfoList.get(keyLocationInfoList.size() - 1);
}
-
- // set token to last block if enabled
- if (ozoneManager.isGrpcBlockTokenEnabled()) {
- String remoteUser = getRemoteUser().getShortUserName();
- OzoneBlockTokenSecretManager secretManager =
ozoneManager.getBlockTokenSecretManager();
- finalBlock.setToken(secretManager.generateToken(remoteUser,
finalBlock.getBlockID(),
- EnumSet.of(READ, WRITE), finalBlock.getLength()));
+ if (finalBlock != null) {
+ // set token to last block if enabled
+ if (ozoneManager.isGrpcBlockTokenEnabled()) {
+ String remoteUser = getRemoteUser().getShortUserName();
+ OzoneBlockTokenSecretManager secretManager =
ozoneManager.getBlockTokenSecretManager();
+ finalBlock.setToken(secretManager.generateToken(remoteUser,
finalBlock.getBlockID(),
+ EnumSet.of(READ, WRITE), finalBlock.getLength()));
+ }
+ // refresh last block pipeline
+ ContainerWithPipeline containerWithPipeline =
+
ozoneManager.getScmClient().getContainerClient().getContainerWithPipeline(finalBlock.getContainerID());
+ finalBlock.setPipeline(containerWithPipeline.getPipeline());
}
- // refresh last block pipeline
- ContainerWithPipeline containerWithPipeline =
-
ozoneManager.getScmClient().getContainerClient().getContainerWithPipeline(finalBlock.getContainerID());
- finalBlock.setPipeline(containerWithPipeline.getPipeline());
-
RecoverLeaseResponse.Builder rb = RecoverLeaseResponse.newBuilder();
rb.setKeyInfo(returnKeyInfo ?
keyInfo.getNetworkProtobuf(getOmRequest().getVersion(), true) :
openKeyInfo.getNetworkProtobuf(getOmRequest().getVersion(), true));
diff --git
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 415801a789..b6cc22bbad 100644
---
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -155,15 +155,17 @@ public class OzoneFileSystem extends BasicOzoneFileSystem
// finalize the final block and get block length
List<OmKeyLocationInfo> locationInfoList =
keyInfo.getLatestVersionLocations().getLocationList();
- OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable e) {
- if (!forceRecovery) {
- throw e;
+ if (!locationInfoList.isEmpty()) {
+ OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
+ try {
+ block.setLength(getAdapter().finalizeBlock(block));
+ } catch (Throwable e) {
+ if (!forceRecovery) {
+ throw e;
+ }
+ LOG.warn("Failed to finalize block. Continue to recover the file since
{} is enabled.",
+ FORCE_LEASE_RECOVERY_ENV, e);
}
- LOG.warn("Failed to finalize block. Continue to recover the file since
{} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, e);
}
// recover and commit file
diff --git
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
index f5216cb451..e6eba955e4 100644
---
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
@@ -159,15 +159,17 @@ public class RootedOzoneFileSystem extends
BasicRootedOzoneFileSystem
// finalize the final block and get block length
List<OmKeyLocationInfo> locationInfoList =
keyInfo.getLatestVersionLocations().getLocationList();
- OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable e) {
- if (!forceRecovery) {
- throw e;
+ if (!locationInfoList.isEmpty()) {
+ OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
+ try {
+ block.setLength(getAdapter().finalizeBlock(block));
+ } catch (Throwable e) {
+ if (!forceRecovery) {
+ throw e;
+ }
+ LOG.warn("Failed to finalize block. Continue to recover the file since
{} is enabled.",
+ FORCE_LEASE_RECOVERY_ENV, e);
}
- LOG.warn("Failed to finalize block. Continue to recover the file since
{} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, e);
}
// recover and commit file
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 415801a789..b6cc22bbad 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -155,15 +155,17 @@ public class OzoneFileSystem extends BasicOzoneFileSystem
// finalize the final block and get block length
List<OmKeyLocationInfo> locationInfoList =
keyInfo.getLatestVersionLocations().getLocationList();
- OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable e) {
- if (!forceRecovery) {
- throw e;
+ if (!locationInfoList.isEmpty()) {
+ OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
+ try {
+ block.setLength(getAdapter().finalizeBlock(block));
+ } catch (Throwable e) {
+ if (!forceRecovery) {
+ throw e;
+ }
+ LOG.warn("Failed to finalize block. Continue to recover the file since
{} is enabled.",
+ FORCE_LEASE_RECOVERY_ENV, e);
}
- LOG.warn("Failed to finalize block. Continue to recover the file since
{} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, e);
}
// recover and commit file
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
index 2784626287..36aa0e5f27 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
@@ -152,15 +152,17 @@ public class RootedOzoneFileSystem extends
BasicRootedOzoneFileSystem
// finalize the final block and get block length
List<OmKeyLocationInfo> keyLocationInfoList =
keyInfo.getLatestVersionLocations().getLocationList();
- OmKeyLocationInfo block =
keyLocationInfoList.get(keyLocationInfoList.size() - 1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable e) {
- if (!forceRecovery) {
- throw e;
+ if (!keyLocationInfoList.isEmpty()) {
+ OmKeyLocationInfo block =
keyLocationInfoList.get(keyLocationInfoList.size() - 1);
+ try {
+ block.setLength(getAdapter().finalizeBlock(block));
+ } catch (Throwable e) {
+ if (!forceRecovery) {
+ throw e;
+ }
+ LOG.warn("Failed to finalize block. Continue to recover the file since
{} is enabled.",
+ FORCE_LEASE_RECOVERY_ENV, e);
}
- LOG.warn("Failed to finalize block. Continue to recover the file since
{} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, e);
}
// recover and commit file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
