This is an automated email from the ASF dual-hosted git repository.

wernerdv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git


The following commit(s) were added to refs/heads/master by this push:
     new d625c06e00f IGNITE-29047 Add ignored failure count metric to 
FailureProcessor (#13574)
d625c06e00f is described below

commit d625c06e00f2e2aa65beda24b95cdbbe571bc181
Author: Dmitry Werner <[email protected]>
AuthorDate: Sat Oct 3 00:43:29 2026 +0500

    IGNITE-29047 Add ignored failure count metric to FailureProcessor (#13574)
---
 docs/_docs/monitoring-metrics/new-metrics.adoc     |  12 +++
 .../processors/failure/FailureProcessor.java       |  33 ++++++
 .../failure/FailureProcessorMetricsTest.java       | 119 +++++++++++++++++++++
 .../ignite/testsuites/IgniteBasicTestSuite2.java   |   2 +
 4 files changed, 166 insertions(+)

diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc 
b/docs/_docs/monitoring-metrics/new-metrics.adoc
index 7482933ed43..bba01c74dd8 100644
--- a/docs/_docs/monitoring-metrics/new-metrics.adoc
+++ b/docs/_docs/monitoring-metrics/new-metrics.adoc
@@ -773,6 +773,18 @@ Register name: `Services.{service_name}`
 |{method_name} | histogram | Duration in nanoseconds of '{method_name}()'.
 |===
 
+== Failure Handling
+
+Counters of the failures suppressed by the configured failure handler. One 
counter is registered per failure type the handler ignores, see 
link:perf-and-troubleshooting/handling-exceptions#critical-workers-health-check[Critical
 Workers Health Check].
+
+Register name: `failure.ignored`
+
+[cols="2,1,3",opts="header"]
+|===
+|Name | Type | Description
+|{failure_type} | long | The number of failures of the given type suppressed 
in accordance with the configured failure handler.
+|===
+
 == Custom Metrics
 
 Registers created by the application through `Ignite.metrics()`. Their names 
start with `custom.`, see link:monitoring-metrics/custom-metrics[Custom 
Metrics].
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
index b2fabbe4109..57a0b65d53d 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -34,12 +34,15 @@ import 
org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import 
org.apache.ignite.internal.processors.cache.persistence.CorruptedDataStructureException;
 import org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor;
+import org.apache.ignite.internal.processors.metric.MetricRegistryImpl;
+import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static 
org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE;
 import static 
org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT;
 import static 
org.apache.ignite.IgniteSystemProperties.IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE;
+import static 
org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName;
 import static org.apache.ignite.internal.util.IgniteUtils.validateRamUsage;
 
 /**
@@ -67,6 +70,16 @@ public class FailureProcessor extends GridProcessorAdapter {
     static final String FAILURE_LOG_MSG = "Critical system error detected. " +
         "Will be handled accordingly to configured handler ";
 
+    /** Ignored failures metrics registry name. */
+    static final String IGNORED_FAILURES_REG = metricName("failure", 
"ignored");
+
+    /** Description template for per-type ignored failures count metrics. */
+    private static final String IGNORED_FAILURES_DESC =
+        "The number of %s failures suppressed in accordance with a configured 
failure handler.";
+
+    /** Per-type ignored failures count metrics. */
+    private final Map<FailureType, AtomicLongMetric> ignoredFailuresCntMetrics 
= new EnumMap<>(FailureType.class);
+
     /** Thread dump per failure type timestamps. */
     private final Map<FailureType, Long> threadDumpPerFailureTypeTs;
 
@@ -124,9 +137,27 @@ public class FailureProcessor extends GridProcessorAdapter 
{
 
         this.hnd = hnd;
 
+        if (hnd instanceof AbstractFailureHandler failureHnd) {
+            for (FailureType t : failureHnd.getIgnoredFailureTypes()) {
+                ignoredFailuresCntMetrics.put(t, new 
AtomicLongMetric(metricName(IGNORED_FAILURES_REG, t.name()),
+                    String.format(IGNORED_FAILURES_DESC, t)));
+            }
+        }
+
         U.quietAndInfo(log, "Configured failure handler: [hnd=" + hnd + ']');
     }
 
+    /** {@inheritDoc} */
+    @Override public void onKernalStart(boolean active) throws 
IgniteCheckedException {
+        if (ignoredFailuresCntMetrics.isEmpty())
+            return;
+
+        MetricRegistryImpl mreg = ctx.metric().registry(IGNORED_FAILURES_REG);
+
+        for (AtomicLongMetric m : ignoredFailuresCntMetrics.values())
+            mreg.register(m);
+    }
+
     /**
      * @return @{code True} if a node will be stopped by current handler in 
near time.
      */
@@ -176,6 +207,8 @@ public class FailureProcessor extends GridProcessorAdapter {
             return false;
 
         if (failureTypeIgnored(failureCtx, hnd)) {
+            ignoredFailuresCntMetrics.get(failureCtx.type()).increment();
+
             U.quietAndWarn(ignite.log(), IGNORED_FAILURE_LOG_MSG +
                 "[hnd=" + hnd + ", failureCtx=" + failureCtx + ']', 
failureCtx.error());
         }
diff --git 
a/modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java
 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java
new file mode 100644
index 00000000000..a1f7060d6cb
--- /dev/null
+++ 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.failure;
+
+import java.util.Collections;
+import java.util.Set;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.failure.TestFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.spi.metric.LongMetric;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static org.apache.ignite.failure.FailureType.SEGMENTATION;
+import static 
org.apache.ignite.failure.FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_BLOCKED;
+import static 
org.apache.ignite.internal.processors.failure.FailureProcessor.IGNORED_FAILURES_REG;
+
+/**
+ * Tests that the failure processor counts suppressed (ignored) failures per 
failure type via dedicated metrics
+ * of the form {@code failure.ignored.<FailureType>}.
+ */
+@WithSystemProperty(key = 
IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE, value = "false")
+public class FailureProcessorMetricsTest extends GridCommonAbstractTest {
+    /** */
+    private static final String ERR_MSG = "Failure context error";
+
+    /** */
+    private Set<FailureType> ignoredTypes;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String 
igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TestFailureHandler hnd = new TestFailureHandler(false);
+
+        hnd.setIgnoredFailureTypes(ignoredTypes);
+
+        cfg.setFailureHandler(hnd);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /** */
+    @Test
+    public void testPerTypeIgnoredFailuresCountMetrics() throws Exception {
+        ignoredTypes = Set.of(SYSTEM_CRITICAL_OPERATION_TIMEOUT, 
SYSTEM_WORKER_BLOCKED);
+
+        IgniteEx ignite = startGrid(0);
+
+        LongMetric criticalTimeoutCnt = ignoredFailuresMetric(ignite, 
SYSTEM_CRITICAL_OPERATION_TIMEOUT);
+        LongMetric workerBlockedCnt = ignoredFailuresMetric(ignite, 
SYSTEM_WORKER_BLOCKED);
+
+        assertEquals(0, criticalTimeoutCnt.value());
+        assertEquals(0, workerBlockedCnt.value());
+
+        ignite.context().failure().process(new 
FailureContext(SYSTEM_CRITICAL_OPERATION_TIMEOUT, new Throwable(ERR_MSG)));
+
+        assertEquals(1, criticalTimeoutCnt.value());
+        assertEquals(0, workerBlockedCnt.value());
+
+        ignite.context().failure().process(new FailureContext(SEGMENTATION, 
new Throwable(ERR_MSG)));
+
+        assertEquals(1, criticalTimeoutCnt.value());
+        assertEquals(0, workerBlockedCnt.value());
+
+        ignite.context().failure().process(new 
FailureContext(SYSTEM_WORKER_BLOCKED, new Throwable(ERR_MSG)));
+
+        assertEquals(1, criticalTimeoutCnt.value());
+        assertEquals(1, workerBlockedCnt.value());
+
+        ignite.context().failure().process(new 
FailureContext(SYSTEM_CRITICAL_OPERATION_TIMEOUT, new Throwable(ERR_MSG)));
+
+        assertEquals(2, criticalTimeoutCnt.value());
+        assertEquals(1, workerBlockedCnt.value());
+    }
+
+    /** */
+    @Test
+    public void testNoMetricsRegisteredWhenNothingIgnored() throws Exception {
+        ignoredTypes = Collections.emptySet();
+
+        IgniteEx ignite = startGrid(0);
+
+        for (FailureType t : FailureType.values())
+            assertNull(ignoredFailuresMetric(ignite, t));
+    }
+
+    /** */
+    private LongMetric ignoredFailuresMetric(IgniteEx ignite, FailureType 
type) {
+        return 
ignite.context().metric().registry(IGNORED_FAILURES_REG).findMetric(type.name());
+    }
+}
diff --git 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
index 9b5ca62ce85..4a97a041dc2 100644
--- 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
@@ -76,6 +76,7 @@ import 
org.apache.ignite.internal.processors.database.FreeListCutTailDifferentGc
 import org.apache.ignite.internal.processors.database.IndexStorageSelfTest;
 import 
org.apache.ignite.internal.processors.database.SwapPathConstructionSelfTest;
 import 
org.apache.ignite.internal.processors.failure.FailureProcessorLoggingTest;
+import 
org.apache.ignite.internal.processors.failure.FailureProcessorMetricsTest;
 import 
org.apache.ignite.internal.processors.failure.FailureProcessorThreadDumpThrottlingTest;
 import 
org.apache.ignite.internal.processors.metastorage.DistributedMetaStorageClassloadingTest;
 import 
org.apache.ignite.internal.processors.metastorage.DistributedMetaStorageTest;
@@ -164,6 +165,7 @@ import org.junit.runners.Suite;
     OomFailureHandlerTest.class,
     TransactionIntegrityWithSystemWorkerDeathTest.class,
     FailureProcessorLoggingTest.class,
+    FailureProcessorMetricsTest.class,
     FailureProcessorThreadDumpThrottlingTest.class,
     ExchangeTaskHandlerWaitingForTasksTest.class,
 

Reply via email to