This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-7593 by this push:
new 074b6cc4c4 HDDS-10104. [hsync]Introduce soft limit support for lease
recovery. (#5974)
074b6cc4c4 is described below
commit 074b6cc4c4e2bc22e71fb5b90cd70f5cfa1d7a89
Author: ashishkumar50 <[email protected]>
AuthorDate: Tue Jan 16 13:15:52 2024 +0530
HDDS-10104. [hsync]Introduce soft limit support for lease recovery. (#5974)
Co-authored-by: ashishk <[email protected]>
---
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 +++
.../common/src/main/resources/ozone-default.xml | 8 +++
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 1 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 1 +
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 1 +
.../hadoop/ozone/debug/TestLeaseRecoverer.java | 1 +
.../src/main/proto/OmClientProtocol.proto | 2 +
.../om/request/file/OMRecoverLeaseRequest.java | 14 ++++-
.../om/request/file/TestOMRecoverLeaseRequest.java | 67 +++++++++++++++++++++-
10 files changed, 101 insertions(+), 5 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 0b62b887a3..0a024ba0f6 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -128,6 +128,14 @@ public final class OzoneConfigKeys {
public static final boolean OZONE_FS_HSYNC_ENABLED_DEFAULT
= false;
+ /**
+ * hsync lease soft limit.
+ */
+ public static final String OZONE_OM_LEASE_SOFT_LIMIT
+ = "ozone.om.lease.soft.limit";
+ public static final String OZONE_OM_LEASE_SOFT_LIMIT_DEFAULT
+ = "60s";
+
/**
* When set to true, allocate a random free port for ozone container, so that
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index cb37a7b8b3..54da8e2ed9 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -3999,6 +3999,14 @@
Enable hsync/hflush. By default they are disabled.
</description>
</property>
+ <property>
+ <name>ozone.om.lease.soft.limit</name>
+ <value>60s</value>
+ <tag>OZONE, OM</tag>
+ <description>
+ Hsync soft limit lease period.
+ </description>
+ </property>
<property>
<name>ozone.recon.scm.snapshot.task.initial.delay</name>
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index c5cf619e37..8d4a552c82 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@ -269,6 +269,7 @@ public class OMException extends IOException {
INVALID_PATH,
KEY_UNDER_LEASE_RECOVERY,
- KEY_ALREADY_CLOSED
+ KEY_ALREADY_CLOSED,
+ KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
index a161146718..70f8a26b87 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
@@ -95,6 +95,7 @@ public class TestLeaseRecovery {
conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name());
+ conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(5)
.setTotalPipelineNumLimit(10)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index 166084ee2b..faadc20332 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -193,6 +193,7 @@ public class TestOzoneFileSystem {
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
conf.setBoolean(OZONE_ACL_ENABLED, true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
+ conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
if (!bucketLayout.equals(FILE_SYSTEM_OPTIMIZED)) {
conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
enabledFileSystemPaths);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 65b8ebcc93..08ac282109 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -256,6 +256,7 @@ public class TestRootedOzoneFileSystem {
conf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, TRASH_INTERVAL / 2);
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
+ conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
if (isBucketFSOptimized) {
bucketLayout = BucketLayout.FILE_SYSTEM_OPTIMIZED;
conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
index ef4fe1456f..dc0c065802 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
@@ -74,6 +74,7 @@ public class TestLeaseRecoverer {
public static void init() throws Exception {
conf = new OzoneConfiguration();
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
+ conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
String clusterId = UUID.randomUUID().toString();
String scmId = UUID.randomUUID().toString();
String omId = UUID.randomUUID().toString();
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 737bf166c3..6135c4c261 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -526,6 +526,7 @@ enum Status {
INVALID_PATH = 93;
KEY_UNDER_LEASE_RECOVERY = 94;
KEY_ALREADY_CLOSED = 95;
+ KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 96;
}
/**
@@ -2070,6 +2071,7 @@ message RecoverLeaseRequest {
optional string volumeName = 1;
optional string bucketName = 2;
optional string keyName = 3;
+ optional bool force = 4;
}
message RecoverLeaseResponse {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
index 7b920f9949..26ba47900f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
@@ -40,6 +40,9 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRespo
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseResponse;
+
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT_DEFAULT;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.RecoverLease;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
@@ -56,9 +59,11 @@ import java.nio.file.Paths;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_CLOSED;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD;
import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
/**
@@ -75,6 +80,7 @@ public class OMRecoverLeaseRequest extends OMKeyRequest {
private String dbFileKey;
private OmKeyInfo openKeyInfo;
private String dbOpenFileKey;
+ private boolean force;
private OMMetadataManager omMetadataManager;
@@ -87,6 +93,7 @@ public class OMRecoverLeaseRequest extends OMKeyRequest {
volumeName = recoverLeaseRequest.getVolumeName();
bucketName = recoverLeaseRequest.getBucketName();
keyName = recoverLeaseRequest.getKeyName();
+ force = recoverLeaseRequest.getForce();
}
@Override
@@ -144,7 +151,6 @@ public class OMRecoverLeaseRequest extends OMKeyRequest {
RecoverLeaseResponse recoverLeaseResponse = doWork(ozoneManager,
transactionLogIndex);
// Prepare response
- boolean responseCode = true;
omResponse.setRecoverLeaseResponse(recoverLeaseResponse).setCmdType(RecoverLease);
omClientResponse = new OMRecoverLeaseResponse(omResponse.build(),
getBucketLayout(),
dbOpenFileKey, openKeyInfo);
@@ -212,6 +218,12 @@ public class OMRecoverLeaseRequest extends OMKeyRequest {
if (openKeyInfo.getMetadata().containsKey(OzoneConsts.LEASE_RECOVERY)) {
LOG.debug("Key: " + keyName + " is already under recovery");
} else {
+ final long leaseSoftLimit = ozoneManager.getConfiguration()
+ .getTimeDuration(OZONE_OM_LEASE_SOFT_LIMIT,
OZONE_OM_LEASE_SOFT_LIMIT_DEFAULT, TimeUnit.MILLISECONDS);
+ if (!force && Time.now() < openKeyInfo.getModificationTime() +
leaseSoftLimit) {
+ throw new OMException("Open Key " + keyName + " updated recently and
is inside soft limit period",
+ KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD);
+ }
openKeyInfo.getMetadata().put(OzoneConsts.LEASE_RECOVERY, "true");
openKeyInfo.setUpdateID(transactionLogIndex,
ozoneManager.isRatisEnabled());
openKeyInfo.setModificationTime(Time.now());
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java
index 476e2b2f93..c4d2404729 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.om.request.file;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.ClientVersion;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.KeyValueUtil;
@@ -48,6 +49,8 @@ import org.apache.hadoop.util.Time;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.util.ArrayList;
@@ -64,6 +67,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
public class TestOMRecoverLeaseRequest extends TestOMKeyRequest {
private long parentId;
+ private boolean forceRecovery = false;
@Override
public BucketLayout getBucketLayout() {
@@ -297,6 +301,60 @@ public class TestOMRecoverLeaseRequest extends
TestOMKeyRequest {
verifyTables(false, false);
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testLeaseSoftLimitForHsyncRecoverFile(boolean force) throws
Exception {
+ forceRecovery = force;
+ populateNamespace(true, true, true, true);
+
+ // update soft limit to high value
+
ozoneManager.getConfiguration().set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT,
"2s");
+ OMClientResponse omClientResponse = validateAndUpdateCache();
+ OMResponse omResponse = omClientResponse.getOMResponse();
+ RecoverLeaseResponse recoverLeaseResponse;
+ if (force) {
+ // In case of force it should always succeed irrespective of soft limit
value.
+ Assertions.assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ omResponse.getStatus());
+ recoverLeaseResponse = omResponse.getRecoverLeaseResponse();
+ KeyInfo keyInfo = recoverLeaseResponse.getKeyInfo();
+ Assertions.assertNotNull(keyInfo);
+ } else {
+ // Call recovery inside soft limit period it should fail
+
Assertions.assertEquals(OzoneManagerProtocolProtos.Status.KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD,
+ omResponse.getStatus());
+ }
+ omClientResponse = validateAndUpdateCache();
+ omResponse = omClientResponse.getOMResponse();
+ if (force) {
+ Assertions.assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ omResponse.getStatus());
+ recoverLeaseResponse = omResponse.getRecoverLeaseResponse();
+ KeyInfo keyInfo = recoverLeaseResponse.getKeyInfo();
+ Assertions.assertNotNull(keyInfo);
+ } else {
+ // Call second time inside soft limit period also should fail
+
Assertions.assertEquals(OzoneManagerProtocolProtos.Status.KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD,
+ omResponse.getStatus());
+ }
+ Thread.sleep(2000);
+ // Call recovery after soft limit period it should succeed
+ omClientResponse = validateAndUpdateCache();
+ omResponse = omClientResponse.getOMResponse();
+ Assertions.assertEquals(OzoneManagerProtocolProtos.Status.OK,
omResponse.getStatus());
+ recoverLeaseResponse = omResponse.getRecoverLeaseResponse();
+ KeyInfo keyInfo = recoverLeaseResponse.getKeyInfo();
+ Assertions.assertNotNull(keyInfo);
+
+ // Call recovery again it should succeed
+ omClientResponse = validateAndUpdateCache();
+ omResponse = omClientResponse.getOMResponse();
+ Assertions.assertEquals(OzoneManagerProtocolProtos.Status.OK,
omResponse.getStatus());
+ recoverLeaseResponse = omResponse.getRecoverLeaseResponse();
+ keyInfo = recoverLeaseResponse.getKeyInfo();
+ Assertions.assertNotNull(keyInfo);
+ }
+
private KeyArgs getNewKeyArgs(OmKeyInfo omKeyInfo, long deltaLength) throws
IOException {
OmKeyLocationInfoGroup omKeyLocationInfoGroup =
omKeyInfo.getLatestVersionLocations();
List<OmKeyLocationInfo> omKeyLocationInfoList =
omKeyLocationInfoGroup.getBlocksLatestVersionOnly();
@@ -355,6 +413,9 @@ public class TestOMRecoverLeaseRequest extends
TestOMKeyRequest {
.get(openKey);
assertNotNull(omKeyInfo);
}
+
+ // Set lease soft limit to 0
+
ozoneManager.getConfiguration().set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT,
"0s");
}
protected OMRequest createAllocateBlockRequest(String volumeName, String
bucketName, String keyName) {
@@ -375,9 +436,9 @@ public class TestOMRecoverLeaseRequest extends
TestOMKeyRequest {
@NotNull
protected OMRequest createRecoverLeaseRequest(
- String volumeName, String bucketName, String keyName) {
+ String volumeName, String bucketName, String keyName, boolean force) {
RecoverLeaseRequest.Builder rb = RecoverLeaseRequest.newBuilder();
- rb.setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName);
+
rb.setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setForce(force);
return OMRequest.newBuilder()
.setCmdType(OzoneManagerProtocolProtos.Type.RecoverLease)
.setClientId(UUID.randomUUID().toString())
@@ -386,7 +447,7 @@ public class TestOMRecoverLeaseRequest extends
TestOMKeyRequest {
private OMClientResponse validateAndUpdateCache() throws Exception {
OMRequest modifiedOmRequest = doPreExecute(createRecoverLeaseRequest(
- volumeName, bucketName, keyName));
+ volumeName, bucketName, keyName, forceRecovery));
assertNotNull(modifiedOmRequest.getUserInfo());
OMRecoverLeaseRequest omRecoverLeaseRequest = getOmRecoverLeaseRequest(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
