This is an automated email from the ASF dual-hosted git repository. sammichen pushed a commit to branch ozone-0.6.0 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 075423bc5f4fca68ab92acaeedc68481bf185097 Author: Doroszlai, Attila <[email protected]> AuthorDate: Fri Jul 17 08:20:19 2020 +0200 HDDS-3964. Ratis config key mismatch (#1204) (cherry picked from commit f15b011ec6ae80fb9eb03dd0e1bdaf6b730fd033) --- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 24 +++++++--------- .../hadoop/hdds/ratis/conf/RatisClientConfig.java | 6 ++-- .../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java | 7 +++-- .../hdds/conf/DatanodeRatisServerConfig.java | 32 +++++++--------------- .../hadoop/fs/ozone/contract/OzoneContract.java | 17 ++++++------ .../ozone/contract/rooted/RootedOzoneContract.java | 17 ++++++------ .../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +- .../ozone/client/rpc/Test2WayCommitInRatis.java | 16 +++++------ .../rpc/TestBlockOutputStreamWithFailures.java | 28 ++++++++++--------- ...estBlockOutputStreamWithFailuresFlushDelay.java | 28 ++++++++++--------- .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 27 +++++++++--------- .../TestContainerStateMachineFailureOnRead.java | 27 +++++++++--------- .../rpc/TestContainerStateMachineFailures.java | 27 +++++++++--------- .../client/rpc/TestDeleteWithSlowFollower.java | 28 ++++++++++--------- .../client/rpc/TestFailureHandlingByClient.java | 28 ++++++++++--------- .../rpc/TestFailureHandlingByClientFlushDelay.java | 28 ++++++++++--------- .../rpc/TestMultiBlockWritesWithDnFailures.java | 28 ++++++++++--------- .../client/rpc/TestValidateBCSIDOnRestart.java | 28 +++++++++---------- .../ozone/client/rpc/TestWatchForCommit.java | 28 ++++++++++--------- .../hadoop/ozone/freon/TestDataValidate.java | 17 ++++++------ .../ozone/freon/TestFreonWithDatanodeRestart.java | 17 ++++++------ .../ozone/freon/TestFreonWithPipelineDestroy.java | 17 ++++++------ .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 17 ++++++------ 23 files changed, 246 insertions(+), 248 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java index 47523bc..505b6c9 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java @@ -75,13 +75,6 @@ public final class RatisHelper { // Prefix for Ratis Server GRPC and Ratis client conf. public static final String HDDS_DATANODE_RATIS_PREFIX_KEY = "hdds.ratis"; - private static final String RAFT_SERVER_PREFIX_KEY = "raft.server"; - public static final String HDDS_DATANODE_RATIS_SERVER_PREFIX_KEY = - HDDS_DATANODE_RATIS_PREFIX_KEY + "." + RAFT_SERVER_PREFIX_KEY; - public static final String HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY = - HDDS_DATANODE_RATIS_PREFIX_KEY + "." + RaftClientConfigKeys.PREFIX; - public static final String HDDS_DATANODE_RATIS_GRPC_PREFIX_KEY = - HDDS_DATANODE_RATIS_PREFIX_KEY + "." + GrpcConfigKeys.PREFIX; private static final Class[] NO_RETRY_EXCEPTIONS = new Class[] {NotReplicatedException.class, GroupMismatchException.class, @@ -227,7 +220,7 @@ public final class RatisHelper { } /** - * Set all the properties matching with regex + * Set all client properties matching with regex * {@link RatisHelper#HDDS_DATANODE_RATIS_PREFIX_KEY} in * ozone configuration object and configure it to RaftProperties. * @param ozoneConf @@ -237,23 +230,26 @@ public final class RatisHelper { RaftProperties raftProperties) { // As for client we do not require server and grpc server/tls. exclude them. - Map<String, String> ratisClientConf = ozoneConf.getPropsWithPrefix( - StringUtils.appendIfNotPresent(HDDS_DATANODE_RATIS_PREFIX_KEY, '.')); + Map<String, String> ratisClientConf = + getDatanodeRatisPrefixProps(ozoneConf); ratisClientConf.forEach((key, val) -> { - if (key.startsWith(RaftClientConfigKeys.PREFIX) || isGrpcClientConfig( - key)) { + if (isClientConfig(key) || isGrpcClientConfig(key)) { raftProperties.set(key, val); } }); } + private static boolean isClientConfig(String key) { + return key.startsWith(RaftClientConfigKeys.PREFIX); + } + private static boolean isGrpcClientConfig(String key) { return key.startsWith(GrpcConfigKeys.PREFIX) && !key .startsWith(GrpcConfigKeys.TLS.PREFIX) && !key .startsWith(GrpcConfigKeys.Server.PREFIX); } /** - * Set all the properties matching with prefix + * Set all server properties matching with prefix * {@link RatisHelper#HDDS_DATANODE_RATIS_PREFIX_KEY} in * ozone configuration object and configure it to RaftProperties. * @param ozoneConf @@ -266,7 +262,7 @@ public final class RatisHelper { getDatanodeRatisPrefixProps(ozoneConf); ratisServerConf.forEach((key, val) -> { // Exclude ratis client configuration. - if (!key.startsWith(RaftClientConfigKeys.PREFIX)) { + if (!isClientConfig(key)) { raftProperties.set(key, val); } }); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/conf/RatisClientConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/conf/RatisClientConfig.java index 18d174d..b88583d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/conf/RatisClientConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/conf/RatisClientConfig.java @@ -29,7 +29,7 @@ import static org.apache.hadoop.hdds.conf.ConfigTag.PERFORMANCE; /** * Configuration related to Ratis Client. This is the config used in creating - * RaftClient creation. + * RaftClient. */ @ConfigGroup(prefix = RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY) public class RatisClientConfig { @@ -38,7 +38,9 @@ public class RatisClientConfig { * Configurations which will be set in RaftProperties. RaftProperties is a * configuration object for Ratis client. */ - @ConfigGroup(prefix = RaftClientConfigKeys.PREFIX) + @ConfigGroup(prefix = + RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY + "." + + RaftClientConfigKeys.PREFIX) public static class RaftConfig { @Config(key = "async.outstanding-requests.max", defaultValue = "32", diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisGrpcConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisGrpcConfig.java index ed036fd..443b879 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisGrpcConfig.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisGrpcConfig.java @@ -18,15 +18,18 @@ package org.apache.hadoop.hdds.conf; +import org.apache.ratis.grpc.GrpcConfigKeys; + import static org.apache.hadoop.hdds.conf.ConfigTag.CLIENT; import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE; import static org.apache.hadoop.hdds.conf.ConfigTag.PERFORMANCE; -import static org.apache.hadoop.hdds.ratis.RatisHelper.HDDS_DATANODE_RATIS_GRPC_PREFIX_KEY; +import static org.apache.hadoop.hdds.ratis.RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY; /** * Ratis Grpc Config Keys. */ -@ConfigGroup(prefix = HDDS_DATANODE_RATIS_GRPC_PREFIX_KEY) +@ConfigGroup(prefix = HDDS_DATANODE_RATIS_PREFIX_KEY + "." + + GrpcConfigKeys.PREFIX) public class DatanodeRatisGrpcConfig { @Config(key = "message.size.max", defaultValue = "32MB", diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java index 39f58bb..b0034ee 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java @@ -18,36 +18,24 @@ package org.apache.hadoop.hdds.conf; +import org.apache.ratis.server.RaftServerConfigKeys; + import java.time.Duration; import static org.apache.hadoop.hdds.conf.ConfigTag.DATANODE; import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE; import static org.apache.hadoop.hdds.conf.ConfigTag.PERFORMANCE; import static org.apache.hadoop.hdds.conf.ConfigTag.RATIS; -import static org.apache.hadoop.hdds.ratis.RatisHelper.HDDS_DATANODE_RATIS_SERVER_PREFIX_KEY; +import static org.apache.hadoop.hdds.ratis.RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY; /** * Datanode Ratis server Configuration. */ -@ConfigGroup(prefix = HDDS_DATANODE_RATIS_SERVER_PREFIX_KEY) +@ConfigGroup(prefix = HDDS_DATANODE_RATIS_PREFIX_KEY + "." + + RaftServerConfigKeys.PREFIX) public class DatanodeRatisServerConfig { - private static final String RATIS_SERVER_REQUEST_TIMEOUT_KEY = - "rpc.request.timeout"; - - private static final String RATIS_SERVER_WATCH_REQUEST_TIMEOUT_KEY = - "watch.timeout"; - - private static final String RATIS_SERVER_NO_LEADER_TIMEOUT_KEY = - "Notification.no-leader.timeout"; - - private static final String RATIS_FOLLOWER_SLOWNESS_TIMEOUT_KEY = - "rpcslowness.timeout"; - - private static final String RATIS_LEADER_NUM_PENDING_REQUESTS_KEY = - "write.element-limit"; - - @Config(key = RATIS_SERVER_REQUEST_TIMEOUT_KEY, + @Config(key = "rpc.request.timeout", defaultValue = "60s", type = ConfigType.TIME, tags = {OZONE, DATANODE, RATIS}, @@ -64,7 +52,7 @@ public class DatanodeRatisServerConfig { this.requestTimeOut = duration.toMillis(); } - @Config(key = RATIS_SERVER_WATCH_REQUEST_TIMEOUT_KEY, + @Config(key = "watch.timeout", defaultValue = "180s", type = ConfigType.TIME, tags = {OZONE, DATANODE, RATIS}, @@ -82,7 +70,7 @@ public class DatanodeRatisServerConfig { this.watchTimeOut = duration.toMillis(); } - @Config(key = RATIS_SERVER_NO_LEADER_TIMEOUT_KEY, + @Config(key = "notification.no-leader.timeout", defaultValue = "300s", type = ConfigType.TIME, tags = {OZONE, DATANODE, RATIS}, @@ -100,7 +88,7 @@ public class DatanodeRatisServerConfig { this.noLeaderTimeout = duration.toMillis(); } - @Config(key = RATIS_FOLLOWER_SLOWNESS_TIMEOUT_KEY, + @Config(key = "rpcslowness.timeout", defaultValue = "300s", type = ConfigType.TIME, tags = {OZONE, DATANODE, RATIS}, @@ -118,7 +106,7 @@ public class DatanodeRatisServerConfig { this.followerSlownessTimeout = duration.toMillis(); } - @Config(key = RATIS_LEADER_NUM_PENDING_REQUESTS_KEY, + @Config(key = "write.element-limit", defaultValue = "1024", type = ConfigType.INT, tags = {OZONE, DATANODE, RATIS, PERFORMANCE}, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java index 7335a93..e832dd7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -71,14 +71,13 @@ class OzoneContract extends AbstractFSContract { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY + "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY + "." + - "watch.request.timeout", - 10, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(raftClientConfig); + conf.addResource(CONTRACT_XML); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/RootedOzoneContract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/RootedOzoneContract.java index c90a7ba..cd68929 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/RootedOzoneContract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/RootedOzoneContract.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -68,14 +68,13 @@ class RootedOzoneContract extends AbstractFSContract { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY + "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY + "." + - "watch.request.timeout", - 10, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(raftClientConfig); + conf.addResource(CONTRACT_XML); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java index 2bb1fb1..6f58eae 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java @@ -63,7 +63,7 @@ public class TestNodeFailure { DatanodeRatisServerConfig ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class); ratisServerConfig.setFollowerSlownessTimeout(Duration.ofSeconds(10)); - ratisServerConfig.setNoLeaderTimeout(Duration.ofSeconds(10)); + ratisServerConfig.setNoLeaderTimeout(Duration.ofMinutes(5)); conf.setFromObject(ratisServerConfig); conf.set(HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL, "2s"); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java index 087376b..7da9d11 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientRatis; import org.apache.hadoop.hdds.scm.XceiverClientSpi; @@ -97,14 +97,12 @@ public class Test2WayCommitInRatis { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 10, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(raftClientConfig); conf.setQuietMode(false); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java index 40756b0..37094d5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientRatis; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; @@ -108,19 +108,21 @@ public class TestBlockOutputStreamWithFailures { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + conf.setBoolean(OZONE_CLIENT_STREAM_BUFFER_FLUSH_DELAY, false); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 30, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 30, TimeUnit.SECONDS); + + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + conf.setFromObject(ratisClientConfig); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7) .setTotalPipelineNumLimit(10).setBlockSize(blockSize) .setChunkSize(chunkSize).setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailuresFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailuresFlushDelay.java index 6fa54ed..3462e50 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailuresFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailuresFlushDelay.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientRatis; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; @@ -102,18 +102,20 @@ public class TestBlockOutputStreamWithFailuresFlushDelay { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 30, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 30, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + + + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + conf.setFromObject(ratisClientConfig); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7) .setTotalPipelineNumLimit(10).setBlockSize(blockSize) .setChunkSize(chunkSize).setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCommitWatcher.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCommitWatcher.java index b9b7e5e..c3eca6a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCommitWatcher.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCommitWatcher.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientRatis; import org.apache.hadoop.hdds.scm.XceiverClientReply; @@ -112,18 +112,19 @@ public class TestCommitWatcher { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 30, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 30, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + conf.setFromObject(ratisClientConfig); + conf.set(OzoneConfigKeys.OZONE_CLIENT_CHECKSUM_TYPE, "NONE"); conf.setQuietMode(false); conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailureOnRead.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailureOnRead.java index 7af57de..d390ca3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailureOnRead.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailureOnRead.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientRatis; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException; @@ -97,18 +97,19 @@ public class TestContainerStateMachineFailureOnRead { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 30, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 30, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + + + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + conf.setFromObject(ratisClientConfig); conf.setQuietMode(false); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java index fba3ba1..e266ff4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; @@ -121,23 +121,24 @@ public class TestContainerStateMachineFailures { conf.setTimeDuration(OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 1, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 10, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 10, TimeUnit.SECONDS); + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(10)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(ratisClientConfig); + DatanodeRatisServerConfig ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class); ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 10, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(raftClientConfig); + conf.setLong(OzoneConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY, 1); conf.setQuietMode(false); cluster = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java index cac16f1..3dc46b9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; @@ -120,20 +120,22 @@ public class TestDeleteWithSlowFollower { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 10, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(raftClientConfig); + conf.setTimeDuration(OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 30, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 30, TimeUnit.SECONDS); + + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + conf.setFromObject(ratisClientConfig); + conf.setQuietMode(false); int numOfDatanodes = 3; cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java index e35a393..887acbb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -97,10 +97,13 @@ public class TestFailureHandlingByClient { chunkSize = (int) OzoneConsts.MB; blockSize = 4 * chunkSize; conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 100, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 30, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 30, TimeUnit.SECONDS); + + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + conf.setFromObject(ratisClientConfig); + conf.setTimeDuration( OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY, 1, TimeUnit.SECONDS); @@ -112,14 +115,13 @@ public class TestFailureHandlingByClient { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + conf.setBoolean( OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_DELAY, false); conf.setQuietMode(false); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java index 8f145b4..a7bf514 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -95,10 +95,13 @@ public class TestFailureHandlingByClientFlushDelay { maxFlushSize = 2 * flushSize; blockSize = 4 * chunkSize; conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 100, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 30, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 30, TimeUnit.SECONDS); + + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + conf.setFromObject(ratisClientConfig); + conf.setTimeDuration( OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY, 1, TimeUnit.SECONDS); @@ -110,14 +113,13 @@ public class TestFailureHandlingByClientFlushDelay { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + conf.setQuietMode(false); conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, StaticMapping.class, DNSToSwitchMapping.class); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java index 1ca073d..1329cc3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -94,18 +94,20 @@ public class TestMultiBlockWritesWithDnFailures { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 30, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 30, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + + + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(30)); + conf.setFromObject(ratisClientConfig); + conf.setTimeDuration( OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY, 1, TimeUnit.SECONDS); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java index 63e63af..315a1df 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; @@ -106,24 +106,24 @@ public class TestValidateBCSIDOnRestart { conf.setTimeDuration(OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 1, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 10, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 10, TimeUnit.SECONDS); + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(10)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(ratisClientConfig); + DatanodeRatisServerConfig ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class); ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY + "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY + "." + - "watch.request.timeout", - 10, TimeUnit.SECONDS); - conf.setQuietMode(false); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(raftClientConfig); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(2). setHbInterval(200) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java index 131d289..ed9642a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.*; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; @@ -100,23 +100,25 @@ public class TestWatchForCommit { conf.setTimeDuration(OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 10, TimeUnit.SECONDS); conf.setQuietMode(false); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.write.timeout", 10, TimeUnit.SECONDS); - conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY - + ".client.request.watch.timeout", 10, TimeUnit.SECONDS); + + RatisClientConfig ratisClientConfig = + conf.getObject(RatisClientConfig.class); + ratisClientConfig.setWriteRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(10)); + ratisClientConfig.setWatchRequestTimeoutInMs(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(ratisClientConfig); + DatanodeRatisServerConfig ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class); ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 10, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(raftClientConfig); + conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(9) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java index 154bd55..86b004d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.junit.Assert; import org.junit.Test; @@ -44,14 +44,13 @@ public abstract class TestDataValidate { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 10, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(10)); + conf.setFromObject(raftClientConfig); + cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5).setTotalPipelineNumLimit(8).build(); cluster.waitForClusterToBeReady(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java index 8984b66..1207bab 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.junit.AfterClass; @@ -74,14 +74,13 @@ public class TestFreonWithDatanodeRestart { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + cluster = MiniOzoneCluster.newBuilder(conf) .setHbProcessorInterval(1000) .setHbInterval(1000) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java index 61ac6af..615444f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; @@ -67,14 +67,13 @@ public class TestFreonWithPipelineDestroy { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + cluster = MiniOzoneCluster.newBuilder(conf) .setHbProcessorInterval(1000) .setHbInterval(1000) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java index ddb27ce..f05ce07 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.ratis.RatisHelper; +import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.junit.AfterClass; @@ -57,14 +57,13 @@ public class TestRandomKeyGenerator { ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); + + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + raftClientConfig.setRpcWatchRequestTimeout(TimeUnit.SECONDS.toMillis(3)); + conf.setFromObject(raftClientConfig); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); cluster.waitForClusterToBeReady(); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
