This is an automated email from the ASF dual-hosted git repository.

wernerdv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git


The following commit(s) were added to refs/heads/master by this push:
     new 3269aedfe81 IGNITE-29054 Add socket read/write timeouts count metrics 
to TcpDiscoverySpi (#13590)
3269aedfe81 is described below

commit 3269aedfe8186d121adcf6dc336bfbeab7d34e22
Author: Dmitry Werner <[email protected]>
AuthorDate: Wed Oct 14 21:12:07 2026 +0800

    IGNITE-29054 Add socket read/write timeouts count metrics to 
TcpDiscoverySpi (#13590)
---
 docs/_docs/monitoring-metrics/new-metrics.adoc     |   4 +-
 .../ignite/spi/discovery/tcp/TcpDiscoverySpi.java  |   9 +-
 .../tcp/internal/TcpDiscoveryStatistics.java       |  30 ++++
 .../discovery/tcp/TcpDiscoverySpiMetricsTest.java  | 165 +++++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite2.java          |   2 +
 5 files changed, 208 insertions(+), 2 deletions(-)

diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc 
b/docs/_docs/monitoring-metrics/new-metrics.adoc
index f89008c0b61..200b9708a17 100644
--- a/docs/_docs/monitoring-metrics/new-metrics.adoc
+++ b/docs/_docs/monitoring-metrics/new-metrics.adoc
@@ -523,12 +523,14 @@ Register name: `io.discovery`
 |Next | UUID | Next in the ring node ID.
 |PendingMessagesRegistered | integer | Pending messages registered count.
 |RejectedSslConnectionsCount | integer | TCP discovery connections count that 
were rejected due to SSL errors.
+|SocketReadTimeoutsCount | integer | TCP discovery socket read operations 
count that timed out.
+|SocketWriteTimeoutsCount | integer | TCP discovery socket write operations 
count that timed out.
 |SslEnabled | boolean | Whether SSL is enabled.
 |TotalProcessedMessages | integer | Total processed messages count.
 |TotalReceivedMessages | integer | Total received messages count.
 |===
 
-NOTE: `Coordinator`, `CoordinatorSince`, `LocalPort`, `MaxMsgQueueSize` and 
`Next` are registered on server nodes only; `ClientRouterNodeId` is registered 
on client nodes only.
+NOTE: `Coordinator`, `CoordinatorSince`, `LocalPort`, `MaxMsgQueueSize` and 
`Next` are registered on server nodes only; `ClientRouterNodeId` is registered 
on client nodes only. `SocketReadTimeoutsCount` indicates the remote node is 
not sending data (not responding to pings or not sending delivery receipts) — 
likely a long GC pause on the remote node, a network issue in the receive 
direction, or the remote node is down. `SocketWriteTimeoutsCount` indicates the 
remote node is not accepting [...]
 
 == Data Region IO
 
diff --git 
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
 
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 19903ffb5fb..1ec99c1b038 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1790,11 +1790,14 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter 
implements IgniteDiscovery
             return ses.readMessage(timeout);
         }
         catch (IOException | IgniteCheckedException e) {
-            if (X.hasCause(e, SocketTimeoutException.class))
+            if (X.hasCause(e, SocketTimeoutException.class)) {
+                stats.onSocketReadTimeout();
+
                 LT.warn(log, "Timed out waiting for message to be read (most 
probably, the reason is " +
                     "long GC pauses on remote node) [curTimeout=" + timeout +
                     ", rmtAddr=" + ses.socket().getRemoteSocketAddress() +
                     ", rmtPort=" + ses.socket().getPort() + ']');
+            }
 
             StreamCorruptedException streamCorruptedCause = X.cause(e, 
StreamCorruptedException.class);
 
@@ -1878,6 +1881,8 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter 
implements IgniteDiscovery
             return ses.read(timeout);
         }
         catch (SocketTimeoutException e) {
+            stats.onSocketReadTimeout();
+
             LT.warn(log, "Timed out waiting for message delivery receipt (most 
probably, the reason is " +
                 "in long GC pauses on remote node; consider tuning GC and 
increasing 'ackTimeout' " +
                 "configuration property). Will retry to send message with 
increased timeout " +
@@ -2499,6 +2504,8 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter 
implements IgniteDiscovery
                 // Close session - timeout occurred.
                 ses.close();
 
+                stats.onSocketWriteTimeout();
+
                 LT.warn(log, "Socket write has timed out (consider increasing 
" +
                     (failureDetectionTimeoutEnabled() ?
                             "'IgniteConfiguration.failureDetectionTimeout' 
configuration property) [" +
diff --git 
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
 
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
index 82f30d11c68..63d0cdb4167 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
@@ -76,6 +76,12 @@ public class TcpDiscoveryStatistics {
     /** Metric that indicates connections count that were rejected due to SSL 
errors. */
     private final IntMetricImpl rejectedSslConnectionsCnt;
 
+    /** Metric that indicates socket read timeouts count. */
+    private final IntMetricImpl sockReadTimeoutsCnt;
+
+    /** Metric that indicates socket write timeouts count. */
+    private final IntMetricImpl sockWriteTimeoutsCnt;
+
     /** */
     public TcpDiscoveryStatistics() {
         joinedNodesCnt = new IntMetricImpl(metricName(DISCO_METRICS, 
"JoinedNodes"), "Joined nodes count");
@@ -91,6 +97,16 @@ public class TcpDiscoveryStatistics {
             metricName(DISCO_METRICS, "RejectedSslConnectionsCount"),
             "TCP discovery connections count that were rejected due to SSL 
errors."
         );
+
+        sockReadTimeoutsCnt = new IntMetricImpl(
+            metricName(DISCO_METRICS, "SocketReadTimeoutsCount"),
+            "TCP discovery socket read operations count that timed out."
+        );
+
+        sockWriteTimeoutsCnt = new IntMetricImpl(
+            metricName(DISCO_METRICS, "SocketWriteTimeoutsCount"),
+            "TCP discovery socket write operations count that timed out."
+        );
     }
 
     /**
@@ -106,6 +122,8 @@ public class TcpDiscoveryStatistics {
         discoReg.register(leftNodesCnt);
         discoReg.register(pendingMsgsRegistered);
         discoReg.register(rejectedSslConnectionsCnt);
+        discoReg.register(sockReadTimeoutsCnt);
+        discoReg.register(sockWriteTimeoutsCnt);
     }
 
     /**
@@ -141,6 +159,16 @@ public class TcpDiscoveryStatistics {
         rejectedSslConnectionsCnt.increment();
     }
 
+    /** Increments socket read timeouts count. */
+    public void onSocketReadTimeout() {
+        sockReadTimeoutsCnt.increment();
+    }
+
+    /** Increments socket write timeouts count. */
+    public void onSocketWriteTimeout() {
+        sockWriteTimeoutsCnt.increment();
+    }
+
     /**
      * Collects necessary stats for message received by SPI.
      *
@@ -334,6 +362,8 @@ public class TcpDiscoveryStatistics {
         rcvdMsgs.clear();
         sentMsgs.clear();
         rejectedSslConnectionsCnt.reset();
+        sockReadTimeoutsCnt.reset();
+        sockWriteTimeoutsCnt.reset();
     }
 
     /** {@inheritDoc} */
diff --git 
a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMetricsTest.java
 
b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMetricsTest.java
new file mode 100644
index 00000000000..320ec32e1c5
--- /dev/null
+++ 
b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMetricsTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import 
org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingRequest;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingResponse;
+import org.apache.ignite.spi.metric.IntMetric;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static 
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.DISCO_METRICS;
+
+/** Tests {@link TcpDiscoverySpi} metrics registered in the {@code 
io.discovery} metric registry. */
+public class TcpDiscoverySpiMetricsTest extends GridCommonAbstractTest {
+    /** */
+    private volatile CountDownLatch latch;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String 
igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setFailureDetectionTimeout(1_000);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /** */
+    @Test
+    public void testSocketReadTimeoutsMetric() throws Exception {
+        checkMetric(true);
+    }
+
+    /** */
+    @Test
+    public void testSocketWriteTimeoutsMetric() throws Exception {
+        checkMetric(false);
+    }
+
+    /** */
+    private void checkMetric(boolean read) throws Exception {
+        IgniteEx node0;
+
+        if (read)
+            node0 = startGrid(0);
+        else {
+            node0 = startGrid(0, cfg -> {
+                BlockingWriteDiscoverySpi spi = new 
BlockingWriteDiscoverySpi();
+
+                
spi.setIpFinder(((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder());
+
+                cfg.setDiscoverySpi(spi);
+            });
+        }
+
+        IgniteEx node1;
+
+        if (read) {
+            node1 = startGrid(1, cfg -> {
+                BlockingReadDiscoverySpi spi = new BlockingReadDiscoverySpi();
+
+                
spi.setIpFinder(((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder());
+
+                cfg.setDiscoverySpi(spi);
+            });
+        }
+        else
+            node1 = startGrid(1);
+
+        latch = new CountDownLatch(1);
+
+        IntMetric metric = node0.context().metric().registry(DISCO_METRICS)
+            .findMetric(read ? "SocketReadTimeoutsCount" : 
"SocketWriteTimeoutsCount");
+
+        assertEquals(0, metric.value());
+
+        IgniteInternalFuture<?> pingFut = GridTestUtils.runAsync(() -> 
node0.cluster().pingNode(node1.cluster().localNode().id()));
+
+        assertTrue(GridTestUtils.waitForCondition(() -> metric.value() == 1, 
15_000));
+
+        latch.countDown();
+
+        pingFut.get(5_000);
+    }
+
+    /** */
+    private class BlockingReadDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void writeMessage(
+            TcpDiscoveryIoSession ses,
+            TcpDiscoveryAbstractMessage msg,
+            long timeout
+        ) throws IOException, IgniteCheckedException {
+            if (msg instanceof TcpDiscoveryPingResponse && latch != null)
+                U.awaitQuiet(latch);
+
+            super.writeMessage(ses, msg, timeout);
+        }
+    }
+
+    /** */
+    private class BlockingWriteDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected TcpDiscoveryIoSession openSession(
+            Socket sock,
+            InetSocketAddress remAddr,
+            IgniteSpiOperationTimeoutHelper timeoutHelper
+        ) throws IgniteSpiOperationTimeoutException, IgniteCheckedException, 
IOException {
+            super.openSession(sock, remAddr, timeoutHelper);
+
+            return new BlockingIoSession(ignite.context(), sock);
+        }
+    }
+
+    /** */
+    private class BlockingIoSession extends TcpDiscoveryIoSession {
+        /** */
+        BlockingIoSession(GridKernalContext ctx, Socket sock) {
+            super(ctx, sock);
+        }
+
+        /** {@inheritDoc} */
+        @Override synchronized void writeMessage(TcpDiscoveryAbstractMessage 
msg) throws IgniteCheckedException, IOException {
+            if (msg instanceof TcpDiscoveryPingRequest && latch != null)
+                U.awaitQuiet(latch);
+
+            super.writeMessage(msg);
+        }
+    }
+}
diff --git 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite2.java
 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite2.java
index 206020d5747..64c4377e012 100644
--- 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite2.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite2.java
@@ -47,6 +47,7 @@ import 
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySnapshotHistoryTest;
 import 
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiMBeanTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiMetricsTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSslSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslTrustedUntrustedTest;
@@ -69,6 +70,7 @@ import org.junit.runners.Suite;
     TcpDiscoverySpiSslSelfTest.class,
     TcpDiscoverySpiFailureTimeoutSelfTest.class,
     TcpDiscoverySpiMBeanTest.class,
+    TcpDiscoverySpiMetricsTest.class,
     TcpDiscoverySnapshotHistoryTest.class,
     ExponentialBackoffTimeoutStrategyTest.class,
     TcpClientDiscoverySpiSelfTest.class,

Reply via email to