This is an automated email from the ASF dual-hosted git repository.
jojochuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 24cf9aaa28e HDDS-16418. Make Inter-SCM gRPC timeouts and keepalive
configurable (#11295)
24cf9aaa28e is described below
commit 24cf9aaa28e5855db3185ae847efbc3331a038d6
Author: hani-fouladgar <[email protected]>
AuthorDate: Tue Oct 6 14:26:28 2026 -0500
HDDS-16418. Make Inter-SCM gRPC timeouts and keepalive configurable (#11295)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 30 +++++++++++++
.../common/src/main/resources/ozone-default.xml | 49 ++++++++++++++++++++++
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 10 +++++
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 23 +++++++++-
.../hadoop/hdds/scm/ha/TestInterSCMGrpcClient.java | 28 +++++++++++++
5 files changed, 139 insertions(+), 1 deletion(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 365a0ab16d3..604b09d807e 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -563,6 +563,36 @@ public final class ScmConfigKeys {
public static final long
OZONE_SCM_HA_GRPC_DEADLINE_INTERVAL_DEFAULT = 30 * 60 * 1000L;
+ public static final String OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIME =
+ "ozone.scm.ha.grpc.client.keepalive.time";
+ public static final long
+ OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIME_DEFAULT = 5 * 60 * 1000L;
+
+ public static final String OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIMEOUT =
+ "ozone.scm.ha.grpc.client.keepalive.timeout";
+ public static final long
+ OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIMEOUT_DEFAULT = 30 * 1000L;
+
+ public static final String OZONE_SCM_HA_GRPC_SERVER_MAX_CONNECTION_IDLE =
+ "ozone.scm.ha.grpc.server.max.connection.idle";
+ public static final long
+ OZONE_SCM_HA_GRPC_SERVER_MAX_CONNECTION_IDLE_DEFAULT = 15 * 60 *
1000L;
+
+ public static final String OZONE_SCM_HA_GRPC_SERVER_KEEPALIVE_TIME =
+ "ozone.scm.ha.grpc.server.keepalive.time";
+ public static final long
+ OZONE_SCM_HA_GRPC_SERVER_KEEPALIVE_TIME_DEFAULT = 5 * 60 * 1000L;
+
+ public static final String OZONE_SCM_HA_GRPC_SERVER_KEEPALIVE_TIMEOUT =
+ "ozone.scm.ha.grpc.server.keepalive.timeout";
+ public static final long
+ OZONE_SCM_HA_GRPC_SERVER_KEEPALIVE_TIMEOUT_DEFAULT = 30 * 1000L;
+
+ public static final String OZONE_SCM_HA_GRPC_SERVER_PERMIT_KEEPALIVE_TIME =
+ "ozone.scm.ha.grpc.server.permit.keepalive.time";
+ public static final long
+ OZONE_SCM_HA_GRPC_SERVER_PERMIT_KEEPALIVE_TIME_DEFAULT = 5 * 60 *
1000L;
+
public static final String OZONE_SCM_HA_RATIS_NODE_FAILURE_TIMEOUT =
"ozone.scm.ha.ratis.server.failure.timeout.duration";
public static final long
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index ec3ec030b76..23e2ac5d950 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -4384,6 +4384,55 @@
<tag>SCM, OZONE, HA, RATIS</tag>
<description>Deadline for SCM DB checkpoint interval.</description>
</property>
+ <property>
+ <name>ozone.scm.ha.grpc.client.keepalive.time</name>
+ <value>5m</value>
+ <tag>SCM, OZONE, HA, RATIS</tag>
+ <description>Interval after which an Inter-SCM gRPC client (a follower SCM
+ or Recon downloading a DB checkpoint) sends an HTTP/2 keepalive ping to
+ the leader SCM when the connection is otherwise quiet, to keep long
+ checkpoint downloads alive across proxies and load balancers.
+ Must be >= ozone.scm.ha.grpc.server.permit.keepalive.time to prevent
+ server disconnections.</description>
+ </property>
+ <property>
+ <name>ozone.scm.ha.grpc.client.keepalive.timeout</name>
+ <value>30s</value>
+ <tag>SCM, OZONE, HA, RATIS</tag>
+ <description>Time an Inter-SCM gRPC client waits for a keepalive ping ack
+ before considering the connection dead and closing it.</description>
+ </property>
+ <property>
+ <name>ozone.scm.ha.grpc.server.max.connection.idle</name>
+ <value>15m</value>
+ <tag>SCM, OZONE, HA, RATIS</tag>
+ <description>Maximum time the Inter-SCM gRPC server keeps a connection with
+ no active RPC before closing it with a GOAWAY. Does not affect an
+ in-progress checkpoint download, which holds an active
stream.</description>
+ </property>
+ <property>
+ <name>ozone.scm.ha.grpc.server.keepalive.time</name>
+ <value>5m</value>
+ <tag>SCM, OZONE, HA, RATIS</tag>
+ <description>Interval after which the Inter-SCM gRPC server sends an HTTP/2
+ keepalive ping to a client when the connection is otherwise
quiet.</description>
+ </property>
+ <property>
+ <name>ozone.scm.ha.grpc.server.keepalive.timeout</name>
+ <value>30s</value>
+ <tag>SCM, OZONE, HA, RATIS</tag>
+ <description>Time the Inter-SCM gRPC server waits for a keepalive ping ack
+ before considering the connection dead and closing it.</description>
+ </property>
+ <property>
+ <name>ozone.scm.ha.grpc.server.permit.keepalive.time</name>
+ <value>5m</value>
+ <tag>SCM, OZONE, HA, RATIS</tag>
+ <description>Minimum interval the Inter-SCM gRPC server permits between
+ keepalive pings from a client. Clients pinging more frequently are
+ disconnected. Should be no larger than
+ ozone.scm.ha.grpc.client.keepalive.time so client pings are
accepted.</description>
+ </property>
<property>
<name>ozone.scm.close.container.wait.duration</name>
<value>150s</value>
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java
index 8b74f0b0f82..aae26dcb981 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java
@@ -63,9 +63,19 @@ public InterSCMGrpcClient(final String host,
ScmConfigKeys.OZONE_SCM_HA_GRPC_DEADLINE_INTERVAL,
ScmConfigKeys.OZONE_SCM_HA_GRPC_DEADLINE_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
+ final long keepAliveTime = conf.getTimeDuration(
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIME,
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIME_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ final long keepAliveTimeout = conf.getTimeDuration(
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIMEOUT,
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
NettyChannelBuilder channelBuilder =
NettyChannelBuilder.forAddress(host, port).usePlaintext()
.maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .keepAliveTime(keepAliveTime, TimeUnit.MILLISECONDS)
+ .keepAliveTimeout(keepAliveTimeout, TimeUnit.MILLISECONDS)
.proxyDetector(uri -> null);
SecurityConfig securityConfig = new SecurityConfig(conf);
if (securityConfig.isSecurityEnabled()
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java
index 8b4086a69d9..7a165e8e55c 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java
@@ -57,9 +57,30 @@ public class InterSCMGrpcProtocolService {
this.port = conf.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
+ final long maxConnectionIdle = conf.getTimeDuration(
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_SERVER_MAX_CONNECTION_IDLE,
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_SERVER_MAX_CONNECTION_IDLE_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ final long keepAliveTime = conf.getTimeDuration(
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_SERVER_KEEPALIVE_TIME,
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_SERVER_KEEPALIVE_TIME_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ final long keepAliveTimeout = conf.getTimeDuration(
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_SERVER_KEEPALIVE_TIMEOUT,
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_SERVER_KEEPALIVE_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ final long permitKeepAliveTime = conf.getTimeDuration(
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_SERVER_PERMIT_KEEPALIVE_TIME,
+ ScmConfigKeys.OZONE_SCM_HA_GRPC_SERVER_PERMIT_KEEPALIVE_TIME_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxConnectionIdle(maxConnectionIdle, TimeUnit.MILLISECONDS)
+ .keepAliveTime(keepAliveTime, TimeUnit.MILLISECONDS)
+ .keepAliveTimeout(keepAliveTimeout, TimeUnit.MILLISECONDS)
+ .permitKeepAliveTime(permitKeepAliveTime, TimeUnit.MILLISECONDS);
InterSCMGrpcService service = new InterSCMGrpcService(scm);
ServerBuilder b = nettyServerBuilder.addService(service);
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestInterSCMGrpcClient.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestInterSCMGrpcClient.java
index ca964aac970..02676d83c18 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestInterSCMGrpcClient.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestInterSCMGrpcClient.java
@@ -75,6 +75,34 @@ void testDownloadIsAbortedAtConfiguredDeadline() throws
Exception {
}
}
+ /**
+ * gRPC rejects a non-positive keepalive time at build time, so a zero value
+ * fails construction only if the configured keepalive time is applied.
+ */
+ @Test
+ void testClientAppliesConfiguredKeepAliveTime() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIME, "0ms");
+
+ assertThrows(IllegalArgumentException.class, () ->
+ new InterSCMGrpcClient(
+ "localhost", PortAllocator.getFreePort(), conf, null));
+ }
+
+ /**
+ * gRPC rejects a non-positive keepalive timeout at build time, so a zero
+ * value fails construction only if the configured timeout is applied.
+ */
+ @Test
+ void testClientAppliesConfiguredKeepAliveTimeout() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_HA_GRPC_CLIENT_KEEPALIVE_TIMEOUT, "0ms");
+
+ assertThrows(IllegalArgumentException.class, () ->
+ new InterSCMGrpcClient(
+ "localhost", PortAllocator.getFreePort(), conf, null));
+ }
+
/**
* Keeps the download stream pending, so that the client side deadline is
* the only way the call can end.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
