This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-7593 by this push:
new 16c3f094d6 HDDS-10632. Handle inconsistent read issue for hsync keys
after lease recovery. (#6810)
16c3f094d6 is described below
commit 16c3f094d65ead1e9a855a06f197fbc0fd21e5ac
Author: Ashish Kumar <[email protected]>
AuthorDate: Sat Jun 15 01:18:49 2024 +0530
HDDS-10632. Handle inconsistent read issue for hsync keys after lease
recovery. (#6810)
Co-authored-by: ashishk <[email protected]>
---
.../hadoop/ozone/om/helpers/LeaseKeyInfo.java | 16 +--
...OzoneManagerProtocolClientSideTranslatorPB.java | 2 +-
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 90 ++++++++++++++-
.../src/main/proto/OmClientProtocol.proto | 2 +-
.../om/request/file/OMRecoverLeaseRequest.java | 30 ++---
.../fs/ozone/LeaseRecoveryClientDNHandler.java | 123 +++++++++++++++++++++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 41 +------
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 41 +------
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 41 +------
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 40 +------
10 files changed, 250 insertions(+), 176 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/LeaseKeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/LeaseKeyInfo.java
index a97ca68168..aab091672b 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/LeaseKeyInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/LeaseKeyInfo.java
@@ -22,19 +22,15 @@ package org.apache.hadoop.ozone.om.helpers;
*/
public class LeaseKeyInfo {
private final OmKeyInfo keyInfo;
- /**
- * isKeyInfo = true indicates keyInfo is from keyTable.
- * isKeyInfo = false indicates keyInfo is from openKeyTable.
- */
- private boolean isKeyInfo;
+ private final OmKeyInfo openKeyInfo;
- public LeaseKeyInfo(OmKeyInfo info, boolean isKeyInfo) {
- this.keyInfo = info;
- this.isKeyInfo = isKeyInfo;
+ public LeaseKeyInfo(OmKeyInfo keyInfo, OmKeyInfo openKeyInfo) {
+ this.keyInfo = keyInfo;
+ this.openKeyInfo = openKeyInfo;
}
- public boolean getIsKeyInfo() {
- return this.isKeyInfo;
+ public OmKeyInfo getOpenKeyInfo() {
+ return openKeyInfo;
}
public OmKeyInfo getKeyInfo() {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index b061091f4c..3e021ffde7 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -2562,7 +2562,7 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
handleError(submitRequest(omRequest)).getRecoverLeaseResponse();
return new
LeaseKeyInfo(OmKeyInfo.getFromProtobuf(recoverLeaseResponse.getKeyInfo()),
- recoverLeaseResponse.getIsKeyInfo());
+ OmKeyInfo.getFromProtobuf(recoverLeaseResponse.getOpenKeyInfo()));
}
@Override
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
index 494f3d5ca2..6ec233fc35 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
@@ -162,7 +162,7 @@ public class TestLeaseRecovery {
}
@ParameterizedTest
- @ValueSource(ints = {1 << 20, (1 << 20) + 1, (1 << 20) - 1})
+ @ValueSource(ints = {1 << 17, (1 << 17) + 1, (1 << 17) - 1})
public void testRecovery(int dataSize) throws Exception {
RootedOzoneFileSystem fs = (RootedOzoneFileSystem)FileSystem.get(conf);
@@ -508,6 +508,94 @@ public class TestLeaseRecovery {
}
}
+ @Test
+ public void testRecoveryWithPartialFilledHsyncBlock() throws Exception {
+ RootedOzoneFileSystem fs = (RootedOzoneFileSystem)FileSystem.get(conf);
+ int blockSize = (int)
cluster.getOzoneManager().getConfiguration().getStorageSize(
+ OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+ final byte[] data = getData(blockSize - 1);
+
+ final FSDataOutputStream stream = fs.create(file, true);
+ try {
+ // Write data into 1st block with total length = blockSize - 1
+ stream.write(data);
+ stream.hsync();
+
+ StorageContainerManager scm = cluster.getStorageContainerManager();
+ ContainerInfo container =
scm.getContainerManager().getContainers().get(0);
+ // Close container so that new data won't be written into the same block
+ // block1 is partially filled
+ OzoneTestUtils.closeContainer(scm, container);
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return
scm.getPipelineManager().getPipeline(container.getPipelineID()).isClosed();
+ } catch (PipelineNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }, 200, 30000);
+
+ assertFalse(fs.isFileClosed(file));
+
+ // write data, this data will completely go into block2 even though
block1 had 1 space left
+ stream.write(data);
+ stream.flush();
+ // At this point both block1 and block2 has (blockSize - 1) length data
+
+ int count = 0;
+ while (count++ < 15 && !fs.recoverLease(file)) {
+ Thread.sleep(1000);
+ }
+ // The lease should have been recovered.
+ assertTrue(fs.isFileClosed(file), "File should be closed");
+
+ // A second call to recoverLease should succeed too.
+ assertTrue(fs.recoverLease(file));
+ } finally {
+ closeIgnoringKeyNotFound(stream);
+ }
+
+ // open it again, make sure the data is correct
+ verifyData(data, (blockSize - 1) * 2, file, fs);
+ }
+
+ @Test
+ public void testRecoveryWithSameBlockCountInOpenFileAndFileTable() throws
Exception {
+ RootedOzoneFileSystem fs = (RootedOzoneFileSystem)FileSystem.get(conf);
+ int blockSize = (int)
cluster.getOzoneManager().getConfiguration().getStorageSize(
+ OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+ final byte[] data = getData(blockSize / 2 - 1);
+
+ final FSDataOutputStream stream = fs.create(file, true);
+ try {
+ stream.write(data);
+ // block 1 exist in fileTable after hsync with length (blockSize / 2 - 1)
+ stream.hsync();
+ assertFalse(fs.isFileClosed(file));
+
+ // Write more data without hsync on same block
+ // File table block length will be still (blockSize / 2 - 1)
+ stream.write(data);
+ stream.flush();
+
+ int count = 0;
+ // fileTable and openFileTable will have same block count.
+ // Both table contains block1
+ while (count++ < 15 && !fs.recoverLease(file)) {
+ Thread.sleep(1000);
+ }
+ // The lease should have been recovered.
+ assertTrue(fs.isFileClosed(file), "File should be closed");
+
+ // A second call to recoverLease should succeed too.
+ assertTrue(fs.recoverLease(file));
+ } finally {
+ closeIgnoringKeyNotFound(stream);
+ }
+
+ // open it again, make sure the data is correct
+ verifyData(data, (blockSize / 2 - 1) * 2, file, fs);
+ }
+
private void verifyData(byte[] data, int dataSize, Path filePath,
RootedOzoneFileSystem fs) throws IOException {
try (FSDataInputStream fdis = fs.open(filePath)) {
int bufferSize = dataSize > data.length ? dataSize / 2 : dataSize;
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index be6a1e1797..4f20bf889e 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -2144,7 +2144,7 @@ message RecoverLeaseRequest {
message RecoverLeaseResponse {
optional bool response = 1 [deprecated=true];
optional KeyInfo keyInfo = 2;
- optional bool isKeyInfo = 3 [default = true];
+ optional KeyInfo openKeyInfo = 3;
}
message SetTimesRequest {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
index be12886a68..2d13140cac 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
@@ -225,7 +225,6 @@ public class OMRecoverLeaseRequest extends OMKeyRequest {
throw new OMException("Open Key " + keyName + " is already deleted",
KEY_NOT_FOUND);
}
- long openKeyModificationTime = openKeyInfo.getModificationTime();
if (openKeyInfo.getMetadata().containsKey(OzoneConsts.LEASE_RECOVERY)) {
LOG.debug("Key: " + keyName + " is already under recovery");
} else {
@@ -251,28 +250,19 @@ public class OMRecoverLeaseRequest extends OMKeyRequest {
OmKeyLocationInfoGroup openKeyLatestVersionLocations =
openKeyInfo.getLatestVersionLocations();
List<OmKeyLocationInfo> openKeyLocationInfoList =
openKeyLatestVersionLocations.getLocationList();
- OmKeyLocationInfo finalBlock = null;
- OmKeyLocationInfo penultimateBlock = null;
- boolean returnKeyInfo = true;
- if (openKeyLocationInfoList.size() > keyLocationInfoList.size() &&
- openKeyModificationTime > keyInfo.getModificationTime() &&
- openKeyLocationInfoList.size() > 0) {
- finalBlock = openKeyLocationInfoList.get(openKeyLocationInfoList.size()
- 1);
- if (openKeyLocationInfoList.size() > 1) {
- penultimateBlock =
openKeyLocationInfoList.get(openKeyLocationInfoList.size() - 2);
- }
- returnKeyInfo = false;
- } else if (keyLocationInfoList.size() > 0) {
- finalBlock = keyLocationInfoList.get(keyLocationInfoList.size() - 1);
+ if (keyLocationInfoList.size() > 0) {
+ updateBlockInfo(ozoneManager,
keyLocationInfoList.get(keyLocationInfoList.size() - 1));
+ }
+ if (openKeyLocationInfoList.size() > 1) {
+ updateBlockInfo(ozoneManager,
openKeyLocationInfoList.get(openKeyLocationInfoList.size() - 1));
+ updateBlockInfo(ozoneManager,
openKeyLocationInfoList.get(openKeyLocationInfoList.size() - 2));
+ } else if (openKeyLocationInfoList.size() > 0) {
+ updateBlockInfo(ozoneManager, openKeyLocationInfoList.get(0));
}
- updateBlockInfo(ozoneManager, finalBlock);
- updateBlockInfo(ozoneManager, penultimateBlock);
RecoverLeaseResponse.Builder rb = RecoverLeaseResponse.newBuilder();
- rb.setKeyInfo(returnKeyInfo ?
keyInfo.getNetworkProtobuf(getOmRequest().getVersion(), true) :
- openKeyInfo.getNetworkProtobuf(getOmRequest().getVersion(), true));
- rb.setIsKeyInfo(returnKeyInfo);
-
+ rb.setKeyInfo(keyInfo.getNetworkProtobuf(getOmRequest().getVersion(),
true));
+
rb.setOpenKeyInfo(openKeyInfo.getNetworkProtobuf(getOmRequest().getVersion(),
true));
return rb.build();
}
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/LeaseRecoveryClientDNHandler.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/LeaseRecoveryClientDNHandler.java
new file mode 100644
index 0000000000..1eea2ea4c9
--- /dev/null
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/LeaseRecoveryClientDNHandler.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import jakarta.annotation.Nonnull;
+import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_NOT_FOUND;
+import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK;
+import static org.apache.hadoop.ozone.OzoneConsts.FORCE_LEASE_RECOVERY_ENV;
+
+/**
+ * Handles lease recovery call between client and DN.
+ */
+public final class LeaseRecoveryClientDNHandler {
+ static final Logger LOG =
LoggerFactory.getLogger(LeaseRecoveryClientDNHandler.class);
+
+ private LeaseRecoveryClientDNHandler() {
+ // Not required.
+ }
+
+ /**
+ * Get actual block length from DN for the last/penultimate block and update
in KeyLocationInfo.
+ * @param leaseKeyInfo keyInfo received from OM
+ * @param adapter client adapter
+ * @param forceRecovery whether to do force recovery
+ * @return List<OmKeyLocationInfo>
+ */
+ @Nonnull
+ public static List<OmKeyLocationInfo> getOmKeyLocationInfos(LeaseKeyInfo
leaseKeyInfo,
+ OzoneClientAdapter adapter, boolean forceRecovery) throws IOException {
+ OmKeyLocationInfoGroup keyLatestVersionLocations =
leaseKeyInfo.getKeyInfo().getLatestVersionLocations();
+ List<OmKeyLocationInfo> keyLocationInfoList =
keyLatestVersionLocations.getLocationList();
+ OmKeyLocationInfoGroup openKeyLatestVersionLocations =
leaseKeyInfo.getOpenKeyInfo().getLatestVersionLocations();
+ List<OmKeyLocationInfo> openKeyLocationInfoList =
openKeyLatestVersionLocations.getLocationList();
+
+ int openKeyLocationSize = openKeyLocationInfoList.size();
+ int keyLocationSize = keyLocationInfoList.size();
+ OmKeyLocationInfo openKeyFinalBlock = null;
+ OmKeyLocationInfo openKeyPenultimateBlock = null;
+ OmKeyLocationInfo keyFinalBlock;
+
+ if (keyLocationSize > 0) {
+ // Block info from fileTable
+ keyFinalBlock = keyLocationInfoList.get(keyLocationSize - 1);
+ // Block info from openFileTable
+ if (openKeyLocationSize > 1) {
+ openKeyFinalBlock = openKeyLocationInfoList.get(openKeyLocationSize -
1);
+ openKeyPenultimateBlock =
openKeyLocationInfoList.get(openKeyLocationSize - 2);
+ } else if (openKeyLocationSize > 0) {
+ openKeyFinalBlock = openKeyLocationInfoList.get(0);
+ }
+ // Finalize the final block and get block length
+ try {
+ // CASE 1: When openFileTable has more block than fileTable
+ // Try to finalize last block of openFileTable
+ // Add that block into fileTable locationInfo
+ if (openKeyLocationSize > keyLocationSize) {
+
openKeyFinalBlock.setLength(adapter.finalizeBlock(openKeyFinalBlock));
+ keyLocationInfoList.add(openKeyFinalBlock);
+ }
+ // CASE 2: When openFileTable penultimate block length is not equal to
fileTable block length of last block
+ // Finalize and get the actual block length and update in fileTable
last block
+ if ((openKeyPenultimateBlock != null && keyFinalBlock != null) &&
+ openKeyPenultimateBlock.getLength() != keyFinalBlock.getLength() &&
+ openKeyPenultimateBlock.getBlockID().getLocalID() ==
keyFinalBlock.getBlockID().getLocalID()) {
+ keyFinalBlock.setLength(adapter.finalizeBlock(keyFinalBlock));
+ }
+ // CASE 3: When openFileTable has same number of blocks as fileTable
+ // Finalize and get actual length of fileTable final block
+ if (keyLocationInfoList.size() == openKeyLocationInfoList.size() &&
keyFinalBlock != null) {
+ keyFinalBlock.setLength(adapter.finalizeBlock(keyFinalBlock));
+ }
+ } catch (Throwable e) {
+ if (e instanceof StorageContainerException &&
+ (((StorageContainerException) e).getResult().equals(NO_SUCH_BLOCK)
+ || ((StorageContainerException)
e).getResult().equals(CONTAINER_NOT_FOUND))
+ && openKeyPenultimateBlock != null && keyFinalBlock != null &&
+ openKeyPenultimateBlock.getBlockID().getLocalID() ==
keyFinalBlock.getBlockID().getLocalID()) {
+ try {
+ keyFinalBlock.setLength(adapter.finalizeBlock(keyFinalBlock));
+ } catch (Throwable exp) {
+ if (!forceRecovery) {
+ throw exp;
+ }
+ LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
+ FORCE_LEASE_RECOVERY_ENV, exp);
+ }
+ } else if (!forceRecovery) {
+ throw e;
+ } else {
+ LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
+ FORCE_LEASE_RECOVERY_ENV, e);
+ }
+ }
+ }
+ return keyLocationInfoList;
+ }
+}
diff --git
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 4de4b22908..65e3145e7d 100644
---
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -35,15 +35,12 @@ import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.security.token.DelegationTokenIssuer;
-import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_NOT_FOUND;
-import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK;
import static org.apache.hadoop.ozone.OzoneConsts.FORCE_LEASE_RECOVERY_ENV;
/**
@@ -156,43 +153,15 @@ public class OzoneFileSystem extends BasicOzoneFileSystem
throw e;
}
- // finalize the final block and get block length
- List<OmKeyLocationInfo> locationInfoList =
leaseKeyInfo.getKeyInfo().getLatestVersionLocations().getLocationList();
- if (!locationInfoList.isEmpty()) {
- OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable e) {
- if (e instanceof StorageContainerException &&
(((StorageContainerException) e).getResult().equals(NO_SUCH_BLOCK)
- || ((StorageContainerException)
e).getResult().equals(CONTAINER_NOT_FOUND))
- && !leaseKeyInfo.getIsKeyInfo() && locationInfoList.size() > 1) {
- locationInfoList =
leaseKeyInfo.getKeyInfo().getLatestVersionLocations().getLocationList().subList(0,
- locationInfoList.size() - 1);
- block = locationInfoList.get(locationInfoList.size() - 1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable exp) {
- if (!forceRecovery) {
- throw exp;
- }
- LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, exp);
- }
- } else if (!forceRecovery) {
- throw e;
- } else {
- LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, e);
- }
- }
- }
-
+ // Get keyLocationInfo
+ List<OmKeyLocationInfo> keyLocationInfoList =
LeaseRecoveryClientDNHandler.getOmKeyLocationInfos(
+ leaseKeyInfo, getAdapter(), forceRecovery);
// recover and commit file
- long keyLength =
locationInfoList.stream().mapToLong(OmKeyLocationInfo::getLength).sum();
+ long keyLength =
keyLocationInfoList.stream().mapToLong(OmKeyLocationInfo::getLength).sum();
OmKeyArgs keyArgs = new
OmKeyArgs.Builder().setVolumeName(leaseKeyInfo.getKeyInfo().getVolumeName())
.setBucketName(leaseKeyInfo.getKeyInfo().getBucketName()).setKeyName(leaseKeyInfo.getKeyInfo().getKeyName())
.setReplicationConfig(leaseKeyInfo.getKeyInfo().getReplicationConfig()).setDataSize(keyLength)
- .setLocationInfoList(locationInfoList)
+ .setLocationInfoList(keyLocationInfoList)
.build();
getAdapter().recoverFile(keyArgs);
return true;
diff --git
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
index 3025b1af03..bae01eafde 100644
---
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.StorageStatistics;
-import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -42,8 +41,6 @@ import java.io.InputStream;
import java.net.URI;
import java.util.List;
-import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_NOT_FOUND;
-import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK;
import static org.apache.hadoop.ozone.OzoneConsts.FORCE_LEASE_RECOVERY_ENV;
/**
@@ -160,43 +157,15 @@ public class RootedOzoneFileSystem extends
BasicRootedOzoneFileSystem
throw e;
}
- // finalize the final block and get block length
- List<OmKeyLocationInfo> locationInfoList =
leaseKeyInfo.getKeyInfo().getLatestVersionLocations().getLocationList();
- if (!locationInfoList.isEmpty()) {
- OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable e) {
- if (e instanceof StorageContainerException &&
(((StorageContainerException) e).getResult().equals(NO_SUCH_BLOCK)
- || ((StorageContainerException)
e).getResult().equals(CONTAINER_NOT_FOUND))
- && !leaseKeyInfo.getIsKeyInfo() && locationInfoList.size() > 1) {
- locationInfoList =
leaseKeyInfo.getKeyInfo().getLatestVersionLocations().getLocationList().subList(0,
- locationInfoList.size() - 1);
- block = locationInfoList.get(locationInfoList.size() - 1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable exp) {
- if (!forceRecovery) {
- throw exp;
- }
- LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, exp);
- }
- } else if (!forceRecovery) {
- throw e;
- } else {
- LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, e);
- }
- }
- }
-
+ // Get keyLocationInfo
+ List<OmKeyLocationInfo> keyLocationInfoList =
LeaseRecoveryClientDNHandler.getOmKeyLocationInfos(
+ leaseKeyInfo, getAdapter(), forceRecovery);
// recover and commit file
- long keyLength =
locationInfoList.stream().mapToLong(OmKeyLocationInfo::getLength).sum();
+ long keyLength =
keyLocationInfoList.stream().mapToLong(OmKeyLocationInfo::getLength).sum();
OmKeyArgs keyArgs = new
OmKeyArgs.Builder().setVolumeName(leaseKeyInfo.getKeyInfo().getVolumeName())
.setBucketName(leaseKeyInfo.getKeyInfo().getBucketName()).setKeyName(leaseKeyInfo.getKeyInfo().getKeyName())
.setReplicationConfig(leaseKeyInfo.getKeyInfo().getReplicationConfig()).setDataSize(keyLength)
- .setLocationInfoList(locationInfoList)
+ .setLocationInfoList(keyLocationInfoList)
.build();
getAdapter().recoverFile(keyArgs);
return true;
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 4de4b22908..65e3145e7d 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -35,15 +35,12 @@ import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.security.token.DelegationTokenIssuer;
-import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_NOT_FOUND;
-import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK;
import static org.apache.hadoop.ozone.OzoneConsts.FORCE_LEASE_RECOVERY_ENV;
/**
@@ -156,43 +153,15 @@ public class OzoneFileSystem extends BasicOzoneFileSystem
throw e;
}
- // finalize the final block and get block length
- List<OmKeyLocationInfo> locationInfoList =
leaseKeyInfo.getKeyInfo().getLatestVersionLocations().getLocationList();
- if (!locationInfoList.isEmpty()) {
- OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable e) {
- if (e instanceof StorageContainerException &&
(((StorageContainerException) e).getResult().equals(NO_SUCH_BLOCK)
- || ((StorageContainerException)
e).getResult().equals(CONTAINER_NOT_FOUND))
- && !leaseKeyInfo.getIsKeyInfo() && locationInfoList.size() > 1) {
- locationInfoList =
leaseKeyInfo.getKeyInfo().getLatestVersionLocations().getLocationList().subList(0,
- locationInfoList.size() - 1);
- block = locationInfoList.get(locationInfoList.size() - 1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable exp) {
- if (!forceRecovery) {
- throw exp;
- }
- LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, exp);
- }
- } else if (!forceRecovery) {
- throw e;
- } else {
- LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, e);
- }
- }
- }
-
+ // Get keyLocationInfo
+ List<OmKeyLocationInfo> keyLocationInfoList =
LeaseRecoveryClientDNHandler.getOmKeyLocationInfos(
+ leaseKeyInfo, getAdapter(), forceRecovery);
// recover and commit file
- long keyLength =
locationInfoList.stream().mapToLong(OmKeyLocationInfo::getLength).sum();
+ long keyLength =
keyLocationInfoList.stream().mapToLong(OmKeyLocationInfo::getLength).sum();
OmKeyArgs keyArgs = new
OmKeyArgs.Builder().setVolumeName(leaseKeyInfo.getKeyInfo().getVolumeName())
.setBucketName(leaseKeyInfo.getKeyInfo().getBucketName()).setKeyName(leaseKeyInfo.getKeyInfo().getKeyName())
.setReplicationConfig(leaseKeyInfo.getKeyInfo().getReplicationConfig()).setDataSize(keyLength)
- .setLocationInfoList(locationInfoList)
+ .setLocationInfoList(keyLocationInfoList)
.build();
getAdapter().recoverFile(keyArgs);
return true;
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
index c06a6b7644..bed3505fe9 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo;
@@ -44,8 +43,6 @@ import java.io.InputStream;
import java.net.URI;
import java.util.List;
-import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_NOT_FOUND;
-import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK;
import static org.apache.hadoop.ozone.OzoneConsts.FORCE_LEASE_RECOVERY_ENV;
/**
@@ -160,43 +157,16 @@ public class RootedOzoneFileSystem extends
BasicRootedOzoneFileSystem
throw e;
}
- // finalize the final block and get block length
- List<OmKeyLocationInfo> locationInfoList =
leaseKeyInfo.getKeyInfo().getLatestVersionLocations().getLocationList();
- if (!locationInfoList.isEmpty()) {
- OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() -
1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable e) {
- if (e instanceof StorageContainerException &&
(((StorageContainerException) e).getResult().equals(NO_SUCH_BLOCK)
- || ((StorageContainerException)
e).getResult().equals(CONTAINER_NOT_FOUND))
- && !leaseKeyInfo.getIsKeyInfo() && locationInfoList.size() > 1) {
- locationInfoList =
leaseKeyInfo.getKeyInfo().getLatestVersionLocations().getLocationList().subList(0,
- locationInfoList.size() - 1);
- block = locationInfoList.get(locationInfoList.size() - 1);
- try {
- block.setLength(getAdapter().finalizeBlock(block));
- } catch (Throwable exp) {
- if (!forceRecovery) {
- throw exp;
- }
- LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, exp);
- }
- } else if (!forceRecovery) {
- throw e;
- } else {
- LOG.warn("Failed to finalize block. Continue to recover the file
since {} is enabled.",
- FORCE_LEASE_RECOVERY_ENV, e);
- }
- }
- }
+ // Get keyLocationInfo
+ List<OmKeyLocationInfo> keyLocationInfoList =
LeaseRecoveryClientDNHandler.getOmKeyLocationInfos(
+ leaseKeyInfo, getAdapter(), forceRecovery);
// recover and commit file
- long keyLength =
locationInfoList.stream().mapToLong(OmKeyLocationInfo::getLength).sum();
+ long keyLength =
keyLocationInfoList.stream().mapToLong(OmKeyLocationInfo::getLength).sum();
OmKeyArgs keyArgs = new
OmKeyArgs.Builder().setVolumeName(leaseKeyInfo.getKeyInfo().getVolumeName())
.setBucketName(leaseKeyInfo.getKeyInfo().getBucketName()).setKeyName(leaseKeyInfo.getKeyInfo().getKeyName())
.setReplicationConfig(leaseKeyInfo.getKeyInfo().getReplicationConfig()).setDataSize(keyLength)
- .setLocationInfoList(locationInfoList)
+ .setLocationInfoList(keyLocationInfoList)
.build();
getAdapter().recoverFile(keyArgs);
return true;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
