This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 838cc2691b HDDS-10088. Refine the number of handlers for each RPC of
SCM (#5954)
838cc2691b is described below
commit 838cc2691b0652ade8de13c0c6156d4ca1b64751
Author: jianghuazhu <[email protected]>
AuthorDate: Fri Jan 12 23:12:06 2024 +0800
HDDS-10088. Refine the number of handlers for each RPC of SCM (#5954)
---
.../hadoop/hdds/conf/OzoneConfiguration.java | 24 +++++++++++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 ++++
.../common/src/main/resources/ozone-default.xml | 38 +++++++++++++++++++--
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 39 +++++++++++++++++++++-
.../hdds/scm/server/SCMBlockProtocolServer.java | 7 ++--
.../hdds/scm/server/SCMClientProtocolServer.java | 7 ++--
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 7 ++--
7 files changed, 116 insertions(+), 12 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
index 10c2189f56..bb5ff0067f 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
@@ -37,6 +37,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
@@ -335,4 +336,27 @@ public class OzoneConfiguration extends Configuration
OZONE_CONTAINER_COPY_WORKDIR)
});
}
+
+ /**
+ * Gets backwards-compatible configuration property values.
+ * @param name Primary configuration attribute key name.
+ * @param fallbackName The key name of the configuration property that needs
+ * to be backward compatible.
+ * @param defaultValue The default value to be returned.
+ */
+ public int getInt(String name, String fallbackName, int defaultValue,
+ Consumer<String> log) {
+ String value = this.getTrimmed(name);
+ if (value == null) {
+ value = this.getTrimmed(fallbackName);
+ if (log != null) {
+ log.accept(name + " is not set. Fallback to " + fallbackName +
+ ", which is set to " + value);
+ }
+ }
+ if (value == null) {
+ return defaultValue;
+ }
+ return Integer.parseInt(value);
+ }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 7e01afd559..c6760451c6 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -236,6 +236,12 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_HANDLER_COUNT_KEY =
"ozone.scm.handler.count.key";
+ public static final String OZONE_SCM_CLIENT_HANDLER_COUNT_KEY =
+ "ozone.scm.client.handler.count.key";
+ public static final String OZONE_SCM_BLOCK_HANDLER_COUNT_KEY =
+ "ozone.scm.block.handler.count.key";
+ public static final String OZONE_SCM_DATANODE_HANDLER_COUNT_KEY =
+ "ozone.scm.datanode.handler.count.key";
public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 100;
public static final String OZONE_SCM_SECURITY_HANDLER_COUNT_KEY =
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index f7a1e7ef66..6a29dc81ec 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1097,8 +1097,42 @@
Set a value that is appropriate for the cluster size. Generally, HDFS
recommends RPC handler count is set to 20 * log2(Cluster Size) with an
- upper limit of 200. However, SCM will not have the same amount of
- traffic as Namenode, so a value much smaller than that will work well
too.
+ upper limit of 200. However, Ozone SCM will not have the same amount of
+ traffic as HDFS Namenode, so a value much smaller than that will work
well too.
+
+ To specify handlers for individual RPC servers,
+ set the following configuration properties instead:
+ ---- RPC type ---- : ---- Configuration properties ----
+ SCMClientProtocolServer : 'ozone.scm.client.handler.count.key'
+ SCMBlockProtocolServer : 'ozone.scm.block.handler.count.key'
+ SCMDatanodeProtocolServer: 'ozone.scm.datanode.handler.count.key'
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.client.handler.count.key</name>
+ <value>100</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ Used to set the number of RPC handlers used by Client to access SCM.
+ The default value is 100.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.block.handler.count.key</name>
+ <value>100</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ Used to set the number of RPC handlers when accessing blocks.
+ The default value is 100.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.datanode.handler.count.key</name>
+ <value>100</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ Used to set the number of RPC handlers used by DataNode to access SCM.
+ The default value is 100.
</description>
</property>
<property>
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
index c9ee219a67..26da27921e 100644
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
@@ -26,6 +26,7 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -34,8 +35,17 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.EnumSource;
-
+import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_HANDLER_COUNT_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -48,6 +58,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
*/
public class TestOzoneConfiguration {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestOzoneConfiguration.class);
+
private OzoneConfiguration conf;
@BeforeEach
@@ -250,6 +263,30 @@ public class TestOzoneConfiguration {
assertEquals(0, subject.getDouble("test.scm.client.threshold", 20.5));
}
+ private static Stream<Arguments> getIntBackwardCompatibilityScenarios() {
+ return Stream.of(
+ Arguments.of(OZONE_SCM_CLIENT_HANDLER_COUNT_KEY, 10, true,
+ OZONE_SCM_HANDLER_COUNT_KEY, 10, OZONE_SCM_HANDLER_COUNT_DEFAULT),
+ Arguments.of(OZONE_SCM_BLOCK_HANDLER_COUNT_KEY, -1, false,
+ OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT,
+ OZONE_SCM_HANDLER_COUNT_DEFAULT),
+ Arguments.of(OZONE_SCM_DATANODE_HANDLER_COUNT_KEY, 105, true,
+ OZONE_SCM_HANDLER_COUNT_KEY, 105, OZONE_SCM_HANDLER_COUNT_DEFAULT)
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("getIntBackwardCompatibilityScenarios")
+ public void testGetIntBackwardCompatibility(String name, int newVal,
+ boolean isGen, String fallbackName, int targetVal, int defaultVal) {
+ OzoneConfiguration ozoneConfig = new OzoneConfiguration();
+ if (isGen) {
+ ozoneConfig.setInt(name, newVal);
+ }
+ int value = ozoneConfig.getInt(name, fallbackName, defaultVal, LOG::info);
+ assertEquals(value, targetVal);
+ }
+
@Test
public void postConstructValidation() {
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
index 41c96d969f..69f190c7fb 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
@@ -70,6 +70,7 @@ import com.google.protobuf.BlockingService;
import com.google.protobuf.ProtocolMessageEnum;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_HANDLER_COUNT_KEY;
import static
org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.IO_EXCEPTION;
import static org.apache.hadoop.hdds.scm.net.NetConstants.NODE_COST_DEFAULT;
import static
org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
@@ -106,9 +107,9 @@ public class SCMBlockProtocolServer implements
StorageContainerManager scm) throws IOException {
this.scm = scm;
this.conf = conf;
- final int handlerCount =
- conf.getInt(OZONE_SCM_HANDLER_COUNT_KEY,
- OZONE_SCM_HANDLER_COUNT_DEFAULT);
+ final int handlerCount = conf.getInt(OZONE_SCM_BLOCK_HANDLER_COUNT_KEY,
+ OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT,
+ LOG::info);
RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
ProtobufRpcEngine.class);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index f9fe52fb1c..3d38fdbe81 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -113,6 +113,7 @@ import java.util.stream.Stream;
import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StorageContainerLocationProtocolService.newReflectiveBlockingService;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_HANDLER_COUNT_KEY;
import static
org.apache.hadoop.hdds.scm.ScmUtils.checkIfCertSignRequestAllowed;
import static
org.apache.hadoop.hdds.scm.ha.HASecurityUtils.createSCMRatisTLSConfig;
import static
org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
@@ -140,9 +141,9 @@ public class SCMClientProtocolServer implements
ReconfigurationHandler reconfigurationHandler) throws IOException {
this.scm = scm;
this.config = conf;
- final int handlerCount =
- conf.getInt(OZONE_SCM_HANDLER_COUNT_KEY,
- OZONE_SCM_HANDLER_COUNT_DEFAULT);
+ final int handlerCount = conf.getInt(OZONE_SCM_CLIENT_HANDLER_COUNT_KEY,
+ OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT,
+ LOG::info);
RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
ProtobufRpcEngine.class);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index dc6ed2ccb1..3d864d4ea2 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -101,6 +101,7 @@ import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProt
import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.setNodeOperationalStateCommand;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_HANDLER_COUNT_KEY;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_REPORT;
import static
org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
@@ -157,9 +158,9 @@ public class SCMDatanodeProtocolServer implements
conf, scm.getScmNodeDetails());
protocolMessageMetrics = getProtocolMessageMetrics();
-
- final int handlerCount = conf.getInt(OZONE_SCM_HANDLER_COUNT_KEY,
- OZONE_SCM_HANDLER_COUNT_DEFAULT);
+ final int handlerCount = conf.getInt(OZONE_SCM_DATANODE_HANDLER_COUNT_KEY,
+ OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT,
+ LOG::info);
RPC.setProtocolEngine(conf, getProtocolClass(), ProtobufRpcEngine.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
