This is an automated email from the ASF dual-hosted git repository.
wernerdv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 3269aedfe81 IGNITE-29054 Add socket read/write timeouts count metrics
to TcpDiscoverySpi (#13590)
3269aedfe81 is described below
commit 3269aedfe8186d121adcf6dc336bfbeab7d34e22
Author: Dmitry Werner <[email protected]>
AuthorDate: Wed Oct 14 21:12:07 2026 +0800
IGNITE-29054 Add socket read/write timeouts count metrics to
TcpDiscoverySpi (#13590)
---
docs/_docs/monitoring-metrics/new-metrics.adoc | 4 +-
.../ignite/spi/discovery/tcp/TcpDiscoverySpi.java | 9 +-
.../tcp/internal/TcpDiscoveryStatistics.java | 30 ++++
.../discovery/tcp/TcpDiscoverySpiMetricsTest.java | 165 +++++++++++++++++++++
.../IgniteSpiDiscoverySelfTestSuite2.java | 2 +
5 files changed, 208 insertions(+), 2 deletions(-)
diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc
b/docs/_docs/monitoring-metrics/new-metrics.adoc
index f89008c0b61..200b9708a17 100644
--- a/docs/_docs/monitoring-metrics/new-metrics.adoc
+++ b/docs/_docs/monitoring-metrics/new-metrics.adoc
@@ -523,12 +523,14 @@ Register name: `io.discovery`
|Next | UUID | Next in the ring node ID.
|PendingMessagesRegistered | integer | Pending messages registered count.
|RejectedSslConnectionsCount | integer | TCP discovery connections count that
were rejected due to SSL errors.
+|SocketReadTimeoutsCount | integer | TCP discovery socket read operations
count that timed out.
+|SocketWriteTimeoutsCount | integer | TCP discovery socket write operations
count that timed out.
|SslEnabled | boolean | Whether SSL is enabled.
|TotalProcessedMessages | integer | Total processed messages count.
|TotalReceivedMessages | integer | Total received messages count.
|===
-NOTE: `Coordinator`, `CoordinatorSince`, `LocalPort`, `MaxMsgQueueSize` and
`Next` are registered on server nodes only; `ClientRouterNodeId` is registered
on client nodes only.
+NOTE: `Coordinator`, `CoordinatorSince`, `LocalPort`, `MaxMsgQueueSize` and
`Next` are registered on server nodes only; `ClientRouterNodeId` is registered
on client nodes only. `SocketReadTimeoutsCount` indicates the remote node is
not sending data (not responding to pings or not sending delivery receipts) —
likely a long GC pause on the remote node, a network issue in the receive
direction, or the remote node is down. `SocketWriteTimeoutsCount` indicates the
remote node is not accepting [...]
== Data Region IO
diff --git
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 19903ffb5fb..1ec99c1b038 100644
---
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1790,11 +1790,14 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter
implements IgniteDiscovery
return ses.readMessage(timeout);
}
catch (IOException | IgniteCheckedException e) {
- if (X.hasCause(e, SocketTimeoutException.class))
+ if (X.hasCause(e, SocketTimeoutException.class)) {
+ stats.onSocketReadTimeout();
+
LT.warn(log, "Timed out waiting for message to be read (most
probably, the reason is " +
"long GC pauses on remote node) [curTimeout=" + timeout +
", rmtAddr=" + ses.socket().getRemoteSocketAddress() +
", rmtPort=" + ses.socket().getPort() + ']');
+ }
StreamCorruptedException streamCorruptedCause = X.cause(e,
StreamCorruptedException.class);
@@ -1878,6 +1881,8 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter
implements IgniteDiscovery
return ses.read(timeout);
}
catch (SocketTimeoutException e) {
+ stats.onSocketReadTimeout();
+
LT.warn(log, "Timed out waiting for message delivery receipt (most
probably, the reason is " +
"in long GC pauses on remote node; consider tuning GC and
increasing 'ackTimeout' " +
"configuration property). Will retry to send message with
increased timeout " +
@@ -2499,6 +2504,8 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter
implements IgniteDiscovery
// Close session - timeout occurred.
ses.close();
+ stats.onSocketWriteTimeout();
+
LT.warn(log, "Socket write has timed out (consider increasing
" +
(failureDetectionTimeoutEnabled() ?
"'IgniteConfiguration.failureDetectionTimeout'
configuration property) [" +
diff --git
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
index 82f30d11c68..63d0cdb4167 100644
---
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
+++
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
@@ -76,6 +76,12 @@ public class TcpDiscoveryStatistics {
/** Metric that indicates connections count that were rejected due to SSL
errors. */
private final IntMetricImpl rejectedSslConnectionsCnt;
+ /** Metric that indicates socket read timeouts count. */
+ private final IntMetricImpl sockReadTimeoutsCnt;
+
+ /** Metric that indicates socket write timeouts count. */
+ private final IntMetricImpl sockWriteTimeoutsCnt;
+
/** */
public TcpDiscoveryStatistics() {
joinedNodesCnt = new IntMetricImpl(metricName(DISCO_METRICS,
"JoinedNodes"), "Joined nodes count");
@@ -91,6 +97,16 @@ public class TcpDiscoveryStatistics {
metricName(DISCO_METRICS, "RejectedSslConnectionsCount"),
"TCP discovery connections count that were rejected due to SSL
errors."
);
+
+ sockReadTimeoutsCnt = new IntMetricImpl(
+ metricName(DISCO_METRICS, "SocketReadTimeoutsCount"),
+ "TCP discovery socket read operations count that timed out."
+ );
+
+ sockWriteTimeoutsCnt = new IntMetricImpl(
+ metricName(DISCO_METRICS, "SocketWriteTimeoutsCount"),
+ "TCP discovery socket write operations count that timed out."
+ );
}
/**
@@ -106,6 +122,8 @@ public class TcpDiscoveryStatistics {
discoReg.register(leftNodesCnt);
discoReg.register(pendingMsgsRegistered);
discoReg.register(rejectedSslConnectionsCnt);
+ discoReg.register(sockReadTimeoutsCnt);
+ discoReg.register(sockWriteTimeoutsCnt);
}
/**
@@ -141,6 +159,16 @@ public class TcpDiscoveryStatistics {
rejectedSslConnectionsCnt.increment();
}
+ /** Increments socket read timeouts count. */
+ public void onSocketReadTimeout() {
+ sockReadTimeoutsCnt.increment();
+ }
+
+ /** Increments socket write timeouts count. */
+ public void onSocketWriteTimeout() {
+ sockWriteTimeoutsCnt.increment();
+ }
+
/**
* Collects necessary stats for message received by SPI.
*
@@ -334,6 +362,8 @@ public class TcpDiscoveryStatistics {
rcvdMsgs.clear();
sentMsgs.clear();
rejectedSslConnectionsCnt.reset();
+ sockReadTimeoutsCnt.reset();
+ sockWriteTimeoutsCnt.reset();
}
/** {@inheritDoc} */
diff --git
a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMetricsTest.java
b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMetricsTest.java
new file mode 100644
index 00000000000..320ec32e1c5
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMetricsTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import
org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingRequest;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingResponse;
+import org.apache.ignite.spi.metric.IntMetric;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.DISCO_METRICS;
+
+/** Tests {@link TcpDiscoverySpi} metrics registered in the {@code
io.discovery} metric registry. */
+public class TcpDiscoverySpiMetricsTest extends GridCommonAbstractTest {
+ /** */
+ private volatile CountDownLatch latch;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setFailureDetectionTimeout(1_000);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /** */
+ @Test
+ public void testSocketReadTimeoutsMetric() throws Exception {
+ checkMetric(true);
+ }
+
+ /** */
+ @Test
+ public void testSocketWriteTimeoutsMetric() throws Exception {
+ checkMetric(false);
+ }
+
+ /** */
+ private void checkMetric(boolean read) throws Exception {
+ IgniteEx node0;
+
+ if (read)
+ node0 = startGrid(0);
+ else {
+ node0 = startGrid(0, cfg -> {
+ BlockingWriteDiscoverySpi spi = new
BlockingWriteDiscoverySpi();
+
+
spi.setIpFinder(((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder());
+
+ cfg.setDiscoverySpi(spi);
+ });
+ }
+
+ IgniteEx node1;
+
+ if (read) {
+ node1 = startGrid(1, cfg -> {
+ BlockingReadDiscoverySpi spi = new BlockingReadDiscoverySpi();
+
+
spi.setIpFinder(((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder());
+
+ cfg.setDiscoverySpi(spi);
+ });
+ }
+ else
+ node1 = startGrid(1);
+
+ latch = new CountDownLatch(1);
+
+ IntMetric metric = node0.context().metric().registry(DISCO_METRICS)
+ .findMetric(read ? "SocketReadTimeoutsCount" :
"SocketWriteTimeoutsCount");
+
+ assertEquals(0, metric.value());
+
+ IgniteInternalFuture<?> pingFut = GridTestUtils.runAsync(() ->
node0.cluster().pingNode(node1.cluster().localNode().id()));
+
+ assertTrue(GridTestUtils.waitForCondition(() -> metric.value() == 1,
15_000));
+
+ latch.countDown();
+
+ pingFut.get(5_000);
+ }
+
+ /** */
+ private class BlockingReadDiscoverySpi extends TcpDiscoverySpi {
+ /** {@inheritDoc} */
+ @Override protected void writeMessage(
+ TcpDiscoveryIoSession ses,
+ TcpDiscoveryAbstractMessage msg,
+ long timeout
+ ) throws IOException, IgniteCheckedException {
+ if (msg instanceof TcpDiscoveryPingResponse && latch != null)
+ U.awaitQuiet(latch);
+
+ super.writeMessage(ses, msg, timeout);
+ }
+ }
+
+ /** */
+ private class BlockingWriteDiscoverySpi extends TcpDiscoverySpi {
+ /** {@inheritDoc} */
+ @Override protected TcpDiscoveryIoSession openSession(
+ Socket sock,
+ InetSocketAddress remAddr,
+ IgniteSpiOperationTimeoutHelper timeoutHelper
+ ) throws IgniteSpiOperationTimeoutException, IgniteCheckedException,
IOException {
+ super.openSession(sock, remAddr, timeoutHelper);
+
+ return new BlockingIoSession(ignite.context(), sock);
+ }
+ }
+
+ /** */
+ private class BlockingIoSession extends TcpDiscoveryIoSession {
+ /** */
+ BlockingIoSession(GridKernalContext ctx, Socket sock) {
+ super(ctx, sock);
+ }
+
+ /** {@inheritDoc} */
+ @Override synchronized void writeMessage(TcpDiscoveryAbstractMessage
msg) throws IgniteCheckedException, IOException {
+ if (msg instanceof TcpDiscoveryPingRequest && latch != null)
+ U.awaitQuiet(latch);
+
+ super.writeMessage(msg);
+ }
+ }
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite2.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite2.java
index 206020d5747..64c4377e012 100644
---
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite2.java
+++
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite2.java
@@ -47,6 +47,7 @@ import
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySelfTest;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySnapshotHistoryTest;
import
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiFailureTimeoutSelfTest;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiMBeanTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiMetricsTest;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSslSelfTest;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslTrustedUntrustedTest;
@@ -69,6 +70,7 @@ import org.junit.runners.Suite;
TcpDiscoverySpiSslSelfTest.class,
TcpDiscoverySpiFailureTimeoutSelfTest.class,
TcpDiscoverySpiMBeanTest.class,
+ TcpDiscoverySpiMetricsTest.class,
TcpDiscoverySnapshotHistoryTest.class,
ExponentialBackoffTimeoutStrategyTest.class,
TcpClientDiscoverySpiSelfTest.class,