This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8a7c23d9a4 HDDS-11302. Ozone wrapper configurations to increase
ipc.server.read.threadpool.size for SCM and Datanode (#7058)
8a7c23d9a4 is described below
commit 8a7c23d9a4af3a2dd720c2c42178304c34f20c86
Author: tanvipenumudy <[email protected]>
AuthorDate: Thu Aug 15 02:39:21 2024 +0530
HDDS-11302. Ozone wrapper configurations to increase
ipc.server.read.threadpool.size for SCM and Datanode (#7058)
Co-authored-by: tanvipenumudy <[email protected]>
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 ++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 12 ++++++
.../common/src/main/resources/ozone-default.xml | 50 ++++++++++++++++++++++
.../ozone/HddsDatanodeClientProtocolServer.java | 9 +++-
.../hdds/scm/server/SCMBlockProtocolServer.java | 7 ++-
.../hdds/scm/server/SCMClientProtocolServer.java | 7 ++-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 7 ++-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 5 ++-
.../hdds/scm/server/StorageContainerManager.java | 4 +-
9 files changed, 97 insertions(+), 7 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index abdd25f068..87707f75dc 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -367,6 +367,9 @@ public final class HddsConfigKeys {
public static final String HDDS_DATANODE_HANDLER_COUNT_KEY =
"hdds.datanode.handler.count";
public static final int HDDS_DATANODE_HANDLER_COUNT_DEFAULT = 10;
+ public static final String HDDS_DATANODE_READ_THREADPOOL_KEY =
+ "hdds.datanode.read.threadpool";
+ public static final int HDDS_DATANODE_READ_THREADPOOL_DEFAULT = 10;
public static final String HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT = 9882;
public static final int HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT = 9883;
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 593764ee09..36d4dbd45a 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -228,15 +228,27 @@ public final class ScmConfigKeys {
"ozone.scm.handler.count.key";
public static final String OZONE_SCM_CLIENT_HANDLER_COUNT_KEY =
"ozone.scm.client.handler.count.key";
+ public static final String OZONE_SCM_CLIENT_READ_THREADPOOL_KEY =
+ "ozone.scm.client.read.threadpool";
+ public static final int OZONE_SCM_CLIENT_READ_THREADPOOL_DEFAULT = 10;
public static final String OZONE_SCM_BLOCK_HANDLER_COUNT_KEY =
"ozone.scm.block.handler.count.key";
+ public static final String OZONE_SCM_BLOCK_READ_THREADPOOL_KEY =
+ "ozone.scm.block.read.threadpool";
+ public static final int OZONE_SCM_BLOCK_READ_THREADPOOL_DEFAULT = 10;
public static final String OZONE_SCM_DATANODE_HANDLER_COUNT_KEY =
"ozone.scm.datanode.handler.count.key";
+ public static final String OZONE_SCM_DATANODE_READ_THREADPOOL_KEY =
+ "ozone.scm.datanode.read.threadpool";
+ public static final int OZONE_SCM_DATANODE_READ_THREADPOOL_DEFAULT = 10;
public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 100;
public static final String OZONE_SCM_SECURITY_HANDLER_COUNT_KEY =
"ozone.scm.security.handler.count.key";
public static final int OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT = 2;
+ public static final String OZONE_SCM_SECURITY_READ_THREADPOOL_KEY =
+ "ozone.scm.security.read.threadpool";
+ public static final int OZONE_SCM_SECURITY_READ_THREADPOOL_DEFAULT = 1;
public static final String OZONE_SCM_DEADNODE_INTERVAL =
"ozone.scm.dead.node.interval";
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 186b5aaded..ad06f2f6e3 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1140,6 +1140,36 @@
The default value is 100.
</description>
</property>
+ <property>
+ <name>ozone.scm.client.read.threadpool</name>
+ <value>10</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ The number of threads in RPC server reading from the socket used by
Client to access SCM.
+ This config overrides Hadoop configuration
"ipc.server.read.threadpool.size" for SCMClientProtocolServer.
+ The default value is 10.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.block.read.threadpool</name>
+ <value>10</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ The number of threads in RPC server reading from the socket when
accessing blocks.
+ This config overrides Hadoop configuration
"ipc.server.read.threadpool.size" for SCMBlockProtocolServer.
+ The default value is 10.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.datanode.read.threadpool</name>
+ <value>10</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ The number of threads in RPC server reading from the socket used by
DataNode to access SCM.
+ This config overrides Hadoop configuration
"ipc.server.read.threadpool.size" for SCMDatanodeProtocolServer.
+ The default value is 10.
+ </description>
+ </property>
<property>
<name>hdds.heartbeat.interval</name>
<value>30s</value>
@@ -2493,6 +2523,16 @@
<tag>OZONE, HDDS, SECURITY</tag>
<description>Threads configured for
SCMSecurityProtocolServer.</description>
</property>
+ <property>
+ <name>ozone.scm.security.read.threadpool</name>
+ <value>1</value>
+ <tag>OZONE, HDDS, SECURITY, PERFORMANCE</tag>
+ <description>
+ The number of threads in RPC server reading from the socket when
performing security related operations with SCM.
+ This config overrides Hadoop configuration
"ipc.server.read.threadpool.size" for SCMSecurityProtocolServer.
+ The default value is 1.
+ </description>
+ </property>
<property>
<name>ozone.scm.security.service.address</name>
<value/>
@@ -2935,6 +2975,16 @@
service endpoints.
</description>
</property>
+ <property>
+ <name>hdds.datanode.read.threadpool</name>
+ <value>10</value>
+ <tag>OZONE, HDDS, PERFORMANCE</tag>
+ <description>
+ The number of threads in RPC server reading from the socket for Datanode
client service endpoints.
+ This config overrides Hadoop configuration
"ipc.server.read.threadpool.size" for HddsDatanodeClientProtocolServer.
+ The default value is 10.
+ </description>
+ </property>
<property>
<name>ozone.client.failover.max.attempts</name>
<value>500</value>
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeClientProtocolServer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeClientProtocolServer.java
index 8b0b3a7ca2..b0308f79ed 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeClientProtocolServer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeClientProtocolServer.java
@@ -42,6 +42,8 @@ import java.net.InetSocketAddress;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_CLIENT_ADDRESS_KEY;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_HANDLER_COUNT_DEFAULT;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_READ_THREADPOOL_KEY;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_READ_THREADPOOL_DEFAULT;
import static org.apache.hadoop.hdds.HddsUtils.preserveThreadName;
import static
org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.CLIENT_RPC;
@@ -106,6 +108,8 @@ public class HddsDatanodeClientProtocolServer extends
ServiceRuntimeInfoImpl {
final int handlerCount = conf.getInt(HDDS_DATANODE_HANDLER_COUNT_KEY,
HDDS_DATANODE_HANDLER_COUNT_DEFAULT);
+ final int readThreads = conf.getInt(HDDS_DATANODE_READ_THREADPOOL_KEY,
+ HDDS_DATANODE_READ_THREADPOOL_DEFAULT);
ReconfigureProtocolServerSideTranslatorPB reconfigureServerProtocol
= new
ReconfigureProtocolServerSideTranslatorPB(reconfigurationHandler);
BlockingService reconfigureService = ReconfigureProtocolProtos
@@ -113,7 +117,7 @@ public class HddsDatanodeClientProtocolServer extends
ServiceRuntimeInfoImpl {
reconfigureServerProtocol);
return preserveThreadName(() -> startRpcServer(configuration, rpcAddress,
- ReconfigureProtocolDatanodePB.class, reconfigureService,
handlerCount));
+ ReconfigureProtocolDatanodePB.class, reconfigureService, handlerCount,
readThreads));
}
/**
@@ -130,7 +134,7 @@ public class HddsDatanodeClientProtocolServer extends
ServiceRuntimeInfoImpl {
private RPC.Server startRpcServer(
Configuration configuration, InetSocketAddress addr,
Class<?> protocol, BlockingService instance,
- int handlerCount)
+ int handlerCount, int readThreads)
throws IOException {
return new RPC.Builder(configuration)
.setProtocol(protocol)
@@ -138,6 +142,7 @@ public class HddsDatanodeClientProtocolServer extends
ServiceRuntimeInfoImpl {
.setBindAddress(addr.getHostString())
.setPort(addr.getPort())
.setNumHandlers(handlerCount)
+ .setNumReaders(readThreads)
.setVerbose(false)
.setSecretManager(null)
.build();
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
index a035751796..7ed23b110c 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
@@ -73,6 +73,8 @@ import com.google.protobuf.ProtocolMessageEnum;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_READ_THREADPOOL_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_READ_THREADPOOL_DEFAULT;
import static
org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.IO_EXCEPTION;
import static org.apache.hadoop.hdds.scm.net.NetConstants.NODE_COST_DEFAULT;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
@@ -117,6 +119,8 @@ public class SCMBlockProtocolServer implements
final int handlerCount = conf.getInt(OZONE_SCM_BLOCK_HANDLER_COUNT_KEY,
OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT,
LOG::info);
+ final int readThreads = conf.getInt(OZONE_SCM_BLOCK_READ_THREADPOOL_KEY,
+ OZONE_SCM_BLOCK_READ_THREADPOOL_DEFAULT);
RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
ProtobufRpcEngine.class);
@@ -142,7 +146,8 @@ public class SCMBlockProtocolServer implements
scmBlockAddress,
ScmBlockLocationProtocolPB.class,
blockProtoPbService,
- handlerCount);
+ handlerCount,
+ readThreads);
blockRpcAddress =
updateRPCListenAddress(
conf, scm.getScmNodeDetails().getBlockProtocolServerAddressKey(),
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 828b452d30..40d153a6bb 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -120,6 +120,8 @@ import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProt
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_HANDLER_COUNT_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_READ_THREADPOOL_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_READ_THREADPOOL_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmUtils.checkIfCertSignRequestAllowed;
import static
org.apache.hadoop.hdds.scm.ha.HASecurityUtils.createSCMRatisTLSConfig;
import static
org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
@@ -150,6 +152,8 @@ public class SCMClientProtocolServer implements
final int handlerCount = conf.getInt(OZONE_SCM_CLIENT_HANDLER_COUNT_KEY,
OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT,
LOG::info);
+ final int readThreads = conf.getInt(OZONE_SCM_CLIENT_READ_THREADPOOL_KEY,
+ OZONE_SCM_CLIENT_READ_THREADPOOL_DEFAULT);
RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
ProtobufRpcEngine.class);
@@ -173,7 +177,8 @@ public class SCMClientProtocolServer implements
scmAddress,
StorageContainerLocationProtocolPB.class,
storageProtoPbService,
- handlerCount);
+ handlerCount,
+ readThreads);
// Add reconfigureProtocolService.
ReconfigureProtocolServerSideTranslatorPB reconfigureServerProtocol
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 3d864d4ea2..0cc6ab7ab9 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -102,6 +102,8 @@ import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProt
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_READ_THREADPOOL_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_READ_THREADPOOL_DEFAULT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_REPORT;
import static
org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
@@ -161,6 +163,8 @@ public class SCMDatanodeProtocolServer implements
final int handlerCount = conf.getInt(OZONE_SCM_DATANODE_HANDLER_COUNT_KEY,
OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT,
LOG::info);
+ final int readThreads = conf.getInt(OZONE_SCM_DATANODE_READ_THREADPOOL_KEY,
+ OZONE_SCM_DATANODE_READ_THREADPOOL_DEFAULT);
RPC.setProtocolEngine(conf, getProtocolClass(), ProtobufRpcEngine.class);
@@ -176,7 +180,8 @@ public class SCMDatanodeProtocolServer implements
datanodeRpcAddr,
getProtocolClass(),
dnProtoPbService,
- handlerCount);
+ handlerCount,
+ readThreads);
datanodeRpcAddress = updateRPCListenAddress(
conf, getDatanodeAddressKey(), datanodeRpcAddr,
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index 6fea1f106f..88b3c88774 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -129,6 +129,8 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol,
final int handlerCount =
conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY,
ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT);
+ final int readThreads =
conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_READ_THREADPOOL_KEY,
+ ScmConfigKeys.OZONE_SCM_SECURITY_READ_THREADPOOL_DEFAULT);
rpcAddress = HddsServerUtil
.getScmSecurityInetAddress(conf);
// SCM security service RPC service.
@@ -157,7 +159,8 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol,
rpcAddress,
SCMSecurityProtocolPB.class,
secureProtoPbService,
- handlerCount);
+ handlerCount,
+ readThreads);
HddsServerUtil.addPBProtocol(conf, SecretKeyProtocolDatanodePB.class,
secretKeyService, rpcServer);
HddsServerUtil.addPBProtocol(conf, SecretKeyProtocolOmPB.class,
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 5c0248f162..aaf6bbfc9c 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -1103,7 +1103,8 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
InetSocketAddress addr,
Class<?> protocol,
BlockingService instance,
- int handlerCount)
+ int handlerCount,
+ int readThreads)
throws IOException {
RPC.Server rpcServer = preserveThreadName(() -> new RPC.Builder(conf)
@@ -1112,6 +1113,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
.setBindAddress(addr.getHostString())
.setPort(addr.getPort())
.setNumHandlers(handlerCount)
+ .setNumReaders(readThreads)
.setVerbose(false)
.setSecretManager(null)
.build());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
