This is an automated email from the ASF dual-hosted git repository.
licheng pushed a commit to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit d7728b308cb3bd326ea8370ade40a2feb727e858
Author: Li Cheng <bloodhell2...@gmail.com>
AuthorDate: Wed May 20 10:05:57 2020 +0800
HDDS-3556 Refactor conf in SCMRatisServer to Java-based conf. (#907)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 4 +-
.../java/org/apache/hadoop/hdds/conf/Config.java | 5 +
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 4 +
.../hdds/scm/server/StorageContainerManager.java | 89 ++++---
.../hdds/scm/server/ratis/SCMRatisServer.java | 282 ++++++++++++++-------
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 2 +-
.../hdds/scm/server/ratis/TestSCMRatisServer.java | 6 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 2 +-
10 files changed, 270 insertions(+), 129 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 6c4f9e8..e5de261 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -433,7 +433,7 @@ public final class ScmConfigKeys {
// SCM Ratis Leader Election configurations
public static final String
OZONE_SCM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
- "ozone.scm.leader.election.minimum.timeout.duration";
+ "ozone.scm.ratis.leader.election.minimum.timeout.duration";
public static final TimeDuration
OZONE_SCM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(1, TimeUnit.SECONDS);
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index bd62474..126121b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2043,7 +2043,7 @@
<name>ozone.scm.ratis.server.request.timeout</name>
<value>3s</value>
<tag>OZONE, SCM, HA, RATIS</tag>
- <description>The timeout duration for SCM's ratis server request
.</description>
+ <description>The timeout duration for SCM's ratis server
request.</description>
</property>
<property>
@@ -2062,7 +2062,7 @@
</property>
<property>
- <name>ozone.scm.leader.election.minimum.timeout.duration</name>
+ <name>ozone.scm.ratis.leader.election.minimum.timeout.duration</name>
<value>1s</value>
<tag>OZONE, SCM, HA, RATIS</tag>
<description>The minimum timeout duration for SCM ratis leader election.
diff --git
a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java
b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java
index 316c867..5d4b477 100644
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java
@@ -55,5 +55,10 @@ public @interface Config {
*/
TimeUnit timeUnit() default TimeUnit.MILLISECONDS;
+ /**
+ * If type == SIZE the unit should be defined with this attribute.
+ */
+ StorageUnit sizeUnit() default StorageUnit.BYTES;
+
ConfigTag[] tags();
}
diff --git
a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
index 385840a..39d481e 100644
---
a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
+++
b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
@@ -42,5 +42,6 @@ public enum ConfigTag {
STANDALONE,
S3GATEWAY,
DATANODE,
- RECON
+ RECON,
+ HA
}
diff --git
a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java
b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java
index 48f06ea..1d5e654 100644
---
a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java
+++
b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java
@@ -84,6 +84,10 @@ public final class ConfigurationReflectionUtil {
forcedFieldSet(field, configuration,
from.getTimeDuration(key, "0s", configAnnotation.timeUnit()));
break;
+ case SIZE:
+ forcedFieldSet(field, configuration,
+ from.getStorageSize(key, "0B", configAnnotation.sizeUnit()));
+ break;
default:
throw new ConfigurationException(
"Unsupported ConfigType " + type + " on " + fieldLocation);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 1a9b70c..6bf5da5 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -220,7 +220,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
*
* @param conf configuration
*/
- public StorageContainerManager(OzoneConfiguration conf)
+ private StorageContainerManager(OzoneConfiguration conf)
throws IOException, AuthenticationException {
// default empty configurator means default managers will be used.
this(conf, new SCMConfigurator());
@@ -236,7 +236,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
* @param conf - Configuration
* @param configurator - configurator
*/
- public StorageContainerManager(OzoneConfiguration conf,
+ private StorageContainerManager(OzoneConfiguration conf,
SCMConfigurator configurator)
throws IOException, AuthenticationException {
super(HddsVersionInfo.HDDS_VERSION_INFO);
@@ -269,14 +269,9 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
loginAsSCMUser(conf);
}
- if (SCMHAUtils.isSCMHAEnabled(conf)) {
- this.scmRatisSnapshotInfo = new SCMRatisSnapshotInfo(
- scmStorageConfig.getCurrentDir());
- this.scmRatisSnapshotDir = SCMHAUtils.createSCMRatisDir(conf);
- initializeRatisServer();
- } else {
- scmRatisServer = null;
- }
+ this.scmRatisSnapshotInfo = new SCMRatisSnapshotInfo(
+ scmStorageConfig.getCurrentDir());
+ this.scmRatisSnapshotDir = SCMHAUtils.createSCMRatisDir(conf);
// Creates the SCM DBs or opens them if it exists.
// A valid pointer to the store is required by all the other services
below.
@@ -388,6 +383,38 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
}
/**
+ * Create an SCM instance based on the supplied configuration.
+ *
+ * @param conf HDDS configuration
+ * @param configurator SCM configurator
+ * @return SCM instance
+ * @throws IOException, AuthenticationException
+ */
+ public static StorageContainerManager createSCM(
+ OzoneConfiguration conf, SCMConfigurator configurator)
+ throws IOException, AuthenticationException {
+ StorageContainerManager scm = new StorageContainerManager(
+ conf, configurator);
+ if (SCMHAUtils.isSCMHAEnabled(conf) && scm.getScmRatisServer() == null) {
+ SCMRatisServer scmRatisServer = initializeRatisServer(conf, scm);
+ scm.setScmRatisServer(scmRatisServer);
+ }
+ return scm;
+ }
+
+ /**
+ * Create an SCM instance based on the supplied configuration.
+ *
+ * @param conf HDDS configuration
+ * @return SCM instance
+ * @throws IOException, AuthenticationException
+ */
+ public static StorageContainerManager createSCM(OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
+ return createSCM(conf, new SCMConfigurator());
+ }
+
+ /**
* This function initializes the following managers. If the configurator
* specifies a value, we will use it, else we will use the default value.
*
@@ -634,18 +661,6 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
}
/**
- * Create an SCM instance based on the supplied configuration.
- *
- * @param conf HDDS configuration
- * @return SCM instance
- * @throws IOException, AuthenticationException
- */
- public static StorageContainerManager createSCM(OzoneConfiguration conf)
- throws IOException, AuthenticationException {
- return new StorageContainerManager(conf);
- }
-
- /**
* Routine to set up the Version info for StorageContainerManager.
*
* @param conf OzoneConfiguration
@@ -1137,18 +1152,20 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
return this.clusterMap;
}
- private void initializeRatisServer() throws IOException {
- if (scmRatisServer == null) {
- SCMNodeDetails scmNodeDetails = SCMNodeDetails
- .initStandAlone(configuration);
- //TODO enable Ratis ring
- scmRatisServer = SCMRatisServer.newSCMRatisServer(configuration, this,
- scmNodeDetails, Collections.EMPTY_LIST);
- if (scmRatisServer != null) {
- LOG.info("SCM Ratis server initialized at port {}",
- scmRatisServer.getServerPort());
- }
- }
+ private static SCMRatisServer initializeRatisServer(
+ OzoneConfiguration conf, StorageContainerManager scm) throws IOException
{
+ SCMNodeDetails scmNodeDetails = SCMNodeDetails
+ .initStandAlone(conf);
+ //TODO enable Ratis group
+ SCMRatisServer scmRatisServer = SCMRatisServer.newSCMRatisServer(
+ conf.getObject(SCMRatisServer.SCMRatisServerConfiguration.class),
+ scm, scmNodeDetails, Collections.EMPTY_LIST,
+ SCMRatisServer.getSCMRatisDirectory(conf));
+ if (scmRatisServer != null) {
+ LOG.info("SCM Ratis server initialized at port {}",
+ scmRatisServer.getServerPort());
+ } // TODO error handling for scmRatisServer creation failure
+ return scmRatisServer;
}
@VisibleForTesting
@@ -1156,6 +1173,10 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
return scmRatisServer;
}
+ public void setScmRatisServer(SCMRatisServer scmRatisServer) {
+ this.scmRatisServer = scmRatisServer;
+ }
+
@VisibleForTesting
public SCMRatisSnapshotInfo getSnapshotInfo() {
return scmRatisSnapshotInfo;
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMRatisServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMRatisServer.java
index 89a9d55..9ab8c66 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMRatisServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMRatisServer.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.hdds.scm.server.ratis;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigType;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -66,6 +68,11 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
+import static org.apache.hadoop.hdds.conf.ConfigTag.RATIS;
+import static org.apache.hadoop.hdds.conf.ConfigTag.SCM;
+import static org.apache.hadoop.hdds.conf.ConfigTag.HA;
+
/**
* Class for SCM Ratis Server.
*/
@@ -76,6 +83,7 @@ public final class SCMRatisServer {
private final StorageContainerManager scm;
private final SCMStateMachine scmStateMachine;
+ private final String storageDir;
private final int port;
private final InetSocketAddress scmRatisAddress;
private final RaftServer server;
@@ -100,14 +108,15 @@ public final class SCMRatisServer {
* Creates a SCM Ratis Server.
* @throws IOException
*/
- private SCMRatisServer(ConfigurationSource conf,
- StorageContainerManager scm,
+ private SCMRatisServer(SCMRatisServerConfiguration conf,
+ StorageContainerManager scm, String ratisStorageDir,
String raftGroupIdStr, RaftPeerId localRaftPeerId,
InetSocketAddress addr, List<RaftPeer> raftPeers)
throws IOException {
this.scm = scm;
this.scmRatisAddress = addr;
this.port = addr.getPort();
+ this.storageDir = ratisStorageDir;
RaftProperties serverProperties = newRaftProperties(conf);
this.raftPeerId = localRaftPeerId;
@@ -149,8 +158,9 @@ public final class SCMRatisServer {
* Create a SCM Ratis Server instance.
*/
public static SCMRatisServer newSCMRatisServer(
- ConfigurationSource conf, StorageContainerManager scm,
- SCMNodeDetails scmNodeDetails, List<SCMNodeDetails> peers)
+ SCMRatisServerConfiguration conf, StorageContainerManager scm,
+ SCMNodeDetails scmNodeDetails, List<SCMNodeDetails> peers,
+ String ratisStorageDir)
throws IOException {
String scmServiceId = scmNodeDetails.getSCMServiceId();
@@ -174,8 +184,8 @@ public final class SCMRatisServer {
raftPeers.add(raftPeer);
}
- return new SCMRatisServer(conf, scm, scmServiceId, localRaftPeerId,
- ratisAddr, raftPeers);
+ return new SCMRatisServer(conf, scm, ratisStorageDir, scmServiceId,
+ localRaftPeerId, ratisAddr, raftPeers);
}
private UUID getRaftGroupIdFromOmServiceId(String scmServiceId) {
@@ -187,13 +197,10 @@ public final class SCMRatisServer {
return new SCMStateMachine(this);
}
- private RaftProperties newRaftProperties(ConfigurationSource conf) {
+ private RaftProperties newRaftProperties(SCMRatisServerConfiguration conf) {
final RaftProperties properties = new RaftProperties();
// Set RPC type
- final String rpcType = conf.get(
- ScmConfigKeys.OZONE_SCM_RATIS_RPC_TYPE_KEY,
- ScmConfigKeys.OZONE_SCM_RATIS_RPC_TYPE_DEFAULT);
- final RpcType rpc = SupportedRpcType.valueOfIgnoreCase(rpcType);
+ final RpcType rpc = SupportedRpcType.valueOfIgnoreCase(conf.getRpcType());
RaftConfigKeys.Rpc.setType(properties, rpc);
// Set the ratis port number
if (rpc == SupportedRpcType.GRPC) {
@@ -202,80 +209,41 @@ public final class SCMRatisServer {
NettyConfigKeys.Server.setPort(properties, port);
}
// Set Ratis storage directory
- String storageDir = SCMRatisServer.getSCMRatisDirectory(conf);
RaftServerConfigKeys.setStorageDir(properties,
Collections.singletonList(new File(storageDir)));
// Set RAFT segment size
- final int raftSegmentSize = (int) conf.getStorageSize(
- ScmConfigKeys.OZONE_SCM_RATIS_SEGMENT_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_RATIS_SEGMENT_SIZE_DEFAULT,
- org.apache.hadoop.hdds.conf.StorageUnit.BYTES);
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
- SizeInBytes.valueOf(raftSegmentSize));
+ SizeInBytes.valueOf((long)conf.getSegmentSize()));
// Set RAFT segment pre-allocated size
- final int raftSegmentPreallocatedSize = (int) conf.getStorageSize(
- ScmConfigKeys.OZONE_SCM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
- org.apache.hadoop.hdds.conf.StorageUnit.BYTES);
- int logAppenderQueueNumElements = conf.getInt(
- ScmConfigKeys.OZONE_SCM_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS,
- ScmConfigKeys.OZONE_SCM_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT);
- final int logAppenderQueueByteLimit = (int) conf.getStorageSize(
- ScmConfigKeys.OZONE_SCM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT,
- ScmConfigKeys.OZONE_SCM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT,
- StorageUnit.BYTES);
RaftServerConfigKeys.Log.Appender.setBufferElementLimit(properties,
- logAppenderQueueNumElements);
+ (int)conf.getLogAppenderQueueByteLimit());
RaftServerConfigKeys.Log.Appender.setBufferByteLimit(properties,
- SizeInBytes.valueOf(logAppenderQueueByteLimit));
+ SizeInBytes.valueOf(conf.getLogAppenderQueueNum()));
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
- SizeInBytes.valueOf(raftSegmentPreallocatedSize));
+ SizeInBytes.valueOf((int)conf.getPreallocatedSize()));
RaftServerConfigKeys.Log.Appender.setInstallSnapshotEnabled(properties,
false);
- final int logPurgeGap = conf.getInt(
- ScmConfigKeys.OZONE_SCM_RATIS_LOG_PURGE_GAP,
- ScmConfigKeys.OZONE_SCM_RATIS_LOG_PURGE_GAP_DEFAULT);
- RaftServerConfigKeys.Log.setPurgeGap(properties, logPurgeGap);
+ RaftServerConfigKeys.Log.setPurgeGap(properties, conf.getLogPurgeGap());
// For grpc set the maximum message size
// TODO: calculate the optimal max message size
GrpcConfigKeys.setMessageSizeMax(properties,
- SizeInBytes.valueOf(logAppenderQueueByteLimit));
+ SizeInBytes.valueOf((int)conf.getLogAppenderQueueByteLimit()));
// Set the server request timeout
- TimeUnit serverRequestTimeoutUnit =
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_REQUEST_TIMEOUT_DEFAULT.getUnit();
- long serverRequestTimeoutDuration = conf.getTimeDuration(
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_REQUEST_TIMEOUT_KEY,
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_REQUEST_TIMEOUT_DEFAULT
- .getDuration(), serverRequestTimeoutUnit);
final TimeDuration serverRequestTimeout = TimeDuration.valueOf(
- serverRequestTimeoutDuration, serverRequestTimeoutUnit);
+ conf.getRequestTimeout(), TimeUnit.MILLISECONDS);
RaftServerConfigKeys.Rpc.setRequestTimeout(properties,
serverRequestTimeout);
// Set timeout for server retry cache entry
- TimeUnit retryCacheTimeoutUnit = ScmConfigKeys
- .OZONE_SCM_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DEFAULT.getUnit();
- long retryCacheTimeoutDuration = conf.getTimeDuration(
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_RETRY_CACHE_TIMEOUT_KEY,
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DEFAULT
- .getDuration(), retryCacheTimeoutUnit);
final TimeDuration retryCacheTimeout = TimeDuration.valueOf(
- retryCacheTimeoutDuration, retryCacheTimeoutUnit);
+ conf.getRetryCacheTimeout(), TimeUnit.MILLISECONDS);
RaftServerConfigKeys.RetryCache.setExpiryTime(properties,
retryCacheTimeout);
// Set the server min and max timeout
- TimeUnit serverMinTimeoutUnit =
- ScmConfigKeys.OZONE_SCM_RATIS_MINIMUM_TIMEOUT_DEFAULT.getUnit();
- long serverMinTimeoutDuration = conf.getTimeDuration(
- ScmConfigKeys.OZONE_SCM_RATIS_MINIMUM_TIMEOUT_KEY,
- ScmConfigKeys.OZONE_SCM_RATIS_MINIMUM_TIMEOUT_DEFAULT
- .getDuration(), serverMinTimeoutUnit);
final TimeDuration serverMinTimeout = TimeDuration.valueOf(
- serverMinTimeoutDuration, serverMinTimeoutUnit);
- long serverMaxTimeoutDuration =
- serverMinTimeout.toLong(TimeUnit.MILLISECONDS) + 200;
+ conf.getMinTimeout(), TimeUnit.MILLISECONDS);
final TimeDuration serverMaxTimeout = TimeDuration.valueOf(
- serverMaxTimeoutDuration, serverMinTimeoutUnit);
+ conf.getMinTimeout() + 200L, TimeUnit.MILLISECONDS);
RaftServerConfigKeys.Rpc.setTimeoutMin(properties,
serverMinTimeout);
RaftServerConfigKeys.Rpc.setTimeoutMax(properties,
@@ -284,46 +252,24 @@ public final class SCMRatisServer {
RaftServerConfigKeys.Log.setSegmentCacheNumMax(properties, 2);
// TODO: set max write buffer size
// Set the ratis leader election timeout
- TimeUnit leaderElectionMinTimeoutUnit =
-
ScmConfigKeys.OZONE_SCM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT
- .getUnit();
- long leaderElectionMinTimeoutduration = conf.getTimeDuration(
- ScmConfigKeys.OZONE_SCM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
-
ScmConfigKeys.OZONE_SCM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT
- .getDuration(), leaderElectionMinTimeoutUnit);
final TimeDuration leaderElectionMinTimeout = TimeDuration.valueOf(
- leaderElectionMinTimeoutduration, leaderElectionMinTimeoutUnit);
+ conf.getMinLeaderElectionTimeout(), TimeUnit.MILLISECONDS);
RaftServerConfigKeys.Rpc.setTimeoutMin(properties,
leaderElectionMinTimeout);
long leaderElectionMaxTimeout = leaderElectionMinTimeout.toLong(
TimeUnit.MILLISECONDS) + 200;
RaftServerConfigKeys.Rpc.setTimeoutMax(properties,
TimeDuration.valueOf(leaderElectionMaxTimeout, TimeUnit.MILLISECONDS));
- TimeUnit nodeFailureTimeoutUnit =
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT
- .getUnit();
- long nodeFailureTimeoutDuration = conf.getTimeDuration(
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT
- .getDuration(), nodeFailureTimeoutUnit);
+
final TimeDuration nodeFailureTimeout = TimeDuration.valueOf(
- nodeFailureTimeoutDuration, nodeFailureTimeoutUnit);
+ conf.getFailureTimeout(), TimeUnit.MILLISECONDS);
RaftServerConfigKeys.Notification.setNoLeaderTimeout(properties,
nodeFailureTimeout);
RaftServerConfigKeys.Rpc.setSlownessTimeout(properties,
nodeFailureTimeout);
// Ratis leader role check
- TimeUnit roleCheckIntervalUnit =
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_ROLE_CHECK_INTERVAL_DEFAULT
- .getUnit();
- long roleCheckIntervalDuration = conf.getTimeDuration(
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_ROLE_CHECK_INTERVAL_KEY,
- ScmConfigKeys.OZONE_SCM_RATIS_SERVER_ROLE_CHECK_INTERVAL_DEFAULT
- .getDuration(), nodeFailureTimeoutUnit);
- this.roleCheckIntervalMs = TimeDuration.valueOf(
- roleCheckIntervalDuration, roleCheckIntervalUnit)
- .toLong(TimeUnit.MILLISECONDS);
+ this.roleCheckIntervalMs = conf.getRoleCheckerInterval();
this.roleCheckInitialDelayMs = leaderElectionMinTimeout
.toLong(TimeUnit.MILLISECONDS);
@@ -479,4 +425,166 @@ public final class SCMRatisServer {
this.roleCheckLock.writeLock().unlock();
}
}
+
+ /**
+ * Configuration used by SCM Ratis Server.
+ */
+ @ConfigGroup(prefix = "ozone.scm.ratis")
+ public static class SCMRatisServerConfiguration {
+ @Config(key = "rpc.type",
+ type = ConfigType.STRING,
+ defaultValue = "GRPC",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "Ratis supports different kinds of transports like" +
+ " netty, GRPC， Hadoop RPC etc. This picks one of those for" +
+ " this cluster."
+ )
+ private String rpcType;
+
+ @Config(key = "segment.size",
+ type = ConfigType.SIZE,
+ defaultValue = "16KB",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "The size of the raft segment used by Apache Ratis on" +
+ " SCM. (16 KB by default)"
+ )
+ private double segmentSize = 16 * 1024;
+
+ @Config(key = "segment.preallocated.size",
+ type = ConfigType.SIZE,
+ defaultValue = "16KB",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "The size of the buffer which is preallocated for" +
+ " raft segment used by Apache Ratis on SCM.(16 KB by default)"
+ )
+ private double preallocatedSize = 16 * 1024;
+
+ @Config(key = "log.appender.queue.num-elements",
+ type = ConfigType.INT,
+ defaultValue = "1024",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "Number of operation pending with Raft's Log Worker."
+ )
+ private int logAppenderQueueNum = 1024;
+
+ @Config(key = "log.appender.queue.byte-limit",
+ type = ConfigType.SIZE,
+ defaultValue = "32MB",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "Byte limit for Raft's Log Worker queue."
+ )
+ private double logAppenderQueueByteLimit = 32 * 1024 * 1024;
+
+ @Config(key = "log.purge.gap",
+ type = ConfigType.INT,
+ defaultValue = "1000000",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "The minimum gap between log indices for Raft server to"
+
+ " purge its log segments after taking snapshot."
+ )
+ private int logPurgeGap = 1000000;
+
+ @Config(key = "server.request.timeout",
+ type = ConfigType.TIME,
+ defaultValue = "3s",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "The timeout duration for SCM's ratis server request."
+ )
+ private long requestTimeout = 3 * 1000L;
+
+ @Config(key = "server.retry.cache.timeout",
+ type = ConfigType.TIME,
+ defaultValue = "60s",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "Retry Cache entry timeout for SCM's ratis server."
+ )
+ private long retryCacheTimeout = 60 * 1000L;
+
+ @Config(key = "minimum.timeout",
+ type = ConfigType.TIME,
+ defaultValue = "1s",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "The minimum timeout duration for SCM's Ratis server
rpc."
+ )
+ private long minTimeout = 1 * 1000L;
+
+ @Config(key = "leader.election.minimum.timeout.duration",
+ type = ConfigType.TIME,
+ defaultValue = "1s",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "The minimum timeout duration for SCM ratis leader" +
+ " election. Default is 1s."
+ )
+ private long minLeaderElectionTimeout = 1 * 1000L;
+
+ @Config(key = "server.failure.timeout.duration",
+ type = ConfigType.TIME,
+ defaultValue = "120s",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "The timeout duration for ratis server failure" +
+ " detection, once the threshold has reached, the ratis state" +
+ " machine will be informed about the failure in the ratis ring."
+ )
+ private long failureTimeout = 120 * 1000L;
+
+ @Config(key = "server.role.check.interval",
+ type = ConfigType.TIME,
+ defaultValue = "15s",
+ tags = {SCM, OZONE, HA, RATIS},
+ description = "The interval between SCM leader performing a role" +
+ " check on its ratis server. Ratis server informs SCM if it loses"
+
+ " the leader role. The scheduled check is an secondary check to" +
+ " ensure that the leader role is updated periodically"
+ )
+ private long roleCheckerInterval = 15 * 1000L;
+
+ public String getRpcType() {
+ return rpcType;
+ }
+
+ public double getSegmentSize() {
+ return segmentSize;
+ }
+
+ public double getPreallocatedSize() {
+ return preallocatedSize;
+ }
+
+ public int getLogAppenderQueueNum() {
+ return logAppenderQueueNum;
+ }
+
+ public double getLogAppenderQueueByteLimit() {
+ return logAppenderQueueByteLimit;
+ }
+
+ public int getLogPurgeGap() {
+ return logPurgeGap;
+ }
+
+ public long getRequestTimeout() {
+ return requestTimeout;
+ }
+
+ public long getRetryCacheTimeout() {
+ return retryCacheTimeout;
+ }
+
+ public long getMinTimeout() {
+ return minTimeout;
+ }
+
+ public long getMinLeaderElectionTimeout() {
+ return minLeaderElectionTimeout;
+ }
+
+ public long getFailureTimeout() {
+ return failureTimeout;
+ }
+
+
+ public long getRoleCheckerInterval() {
+ return roleCheckerInterval;
+ }
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
index 64752da..5d1ed46 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
@@ -497,7 +497,7 @@ public final class TestUtils {
// writes the version file properties
scmStore.initialize();
}
- return new StorageContainerManager(conf, configurator);
+ return StorageContainerManager.createSCM(conf, configurator);
}
public static ContainerInfo getContainer(
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java
index d6981d3..4079965 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java
@@ -117,8 +117,10 @@ public class TestSCMRatisServer {
// Starts a single node Ratis server
scmRatisServer.stop();
SCMRatisServer newScmRatisServer = SCMRatisServer
- .newSCMRatisServer(newConf, scm, nodeDetails,
- Collections.emptyList());
+ .newSCMRatisServer(newConf.getObject(SCMRatisServer
+ .SCMRatisServerConfiguration.class), scm, nodeDetails,
+ Collections.emptyList(),
+ SCMRatisServer.getSCMRatisDirectory(newConf));
newScmRatisServer.start();
UUID uuid = UUID.nameUUIDFromBytes(customScmServiceId.getBytes());
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
index 797c805..b1a79d9 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
@@ -133,7 +133,7 @@ public final class GenesisUtil {
// writes the version file properties
scmStore.initialize();
}
- return new StorageContainerManager(conf, configurator);
+ return StorageContainerManager.createSCM(conf, configurator);
}
static void configureSCM(OzoneConfiguration conf, int numHandlers) {
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
