This is an automated email from the ASF dual-hosted git repository.
wernerdv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new d625c06e00f IGNITE-29047 Add ignored failure count metric to
FailureProcessor (#13574)
d625c06e00f is described below
commit d625c06e00f2e2aa65beda24b95cdbbe571bc181
Author: Dmitry Werner <[email protected]>
AuthorDate: Sat Oct 3 00:43:29 2026 +0500
IGNITE-29047 Add ignored failure count metric to FailureProcessor (#13574)
---
docs/_docs/monitoring-metrics/new-metrics.adoc | 12 +++
.../processors/failure/FailureProcessor.java | 33 ++++++
.../failure/FailureProcessorMetricsTest.java | 119 +++++++++++++++++++++
.../ignite/testsuites/IgniteBasicTestSuite2.java | 2 +
4 files changed, 166 insertions(+)
diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc
b/docs/_docs/monitoring-metrics/new-metrics.adoc
index 7482933ed43..bba01c74dd8 100644
--- a/docs/_docs/monitoring-metrics/new-metrics.adoc
+++ b/docs/_docs/monitoring-metrics/new-metrics.adoc
@@ -773,6 +773,18 @@ Register name: `Services.{service_name}`
|{method_name} | histogram | Duration in nanoseconds of '{method_name}()'.
|===
+== Failure Handling
+
+Counters of the failures suppressed by the configured failure handler. One
counter is registered per failure type the handler ignores, see
link:perf-and-troubleshooting/handling-exceptions#critical-workers-health-check[Critical
Workers Health Check].
+
+Register name: `failure.ignored`
+
+[cols="2,1,3",opts="header"]
+|===
+|Name | Type | Description
+|{failure_type} | long | The number of failures of the given type suppressed
in accordance with the configured failure handler.
+|===
+
== Custom Metrics
Registers created by the application through `Ignite.metrics()`. Their names
start with `custom.`, see link:monitoring-metrics/custom-metrics[Custom
Metrics].
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
index b2fabbe4109..57a0b65d53d 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -34,12 +34,15 @@ import
org.apache.ignite.internal.processors.GridProcessorAdapter;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import
org.apache.ignite.internal.processors.cache.persistence.CorruptedDataStructureException;
import org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor;
+import org.apache.ignite.internal.processors.metric.MetricRegistryImpl;
+import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import static
org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE;
import static
org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT;
import static
org.apache.ignite.IgniteSystemProperties.IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE;
+import static
org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName;
import static org.apache.ignite.internal.util.IgniteUtils.validateRamUsage;
/**
@@ -67,6 +70,16 @@ public class FailureProcessor extends GridProcessorAdapter {
static final String FAILURE_LOG_MSG = "Critical system error detected. " +
"Will be handled accordingly to configured handler ";
+ /** Ignored failures metrics registry name. */
+ static final String IGNORED_FAILURES_REG = metricName("failure",
"ignored");
+
+ /** Description template for per-type ignored failures count metrics. */
+ private static final String IGNORED_FAILURES_DESC =
+ "The number of %s failures suppressed in accordance with a configured
failure handler.";
+
+ /** Per-type ignored failures count metrics. */
+ private final Map<FailureType, AtomicLongMetric> ignoredFailuresCntMetrics
= new EnumMap<>(FailureType.class);
+
/** Thread dump per failure type timestamps. */
private final Map<FailureType, Long> threadDumpPerFailureTypeTs;
@@ -124,9 +137,27 @@ public class FailureProcessor extends GridProcessorAdapter
{
this.hnd = hnd;
+ if (hnd instanceof AbstractFailureHandler failureHnd) {
+ for (FailureType t : failureHnd.getIgnoredFailureTypes()) {
+ ignoredFailuresCntMetrics.put(t, new
AtomicLongMetric(metricName(IGNORED_FAILURES_REG, t.name()),
+ String.format(IGNORED_FAILURES_DESC, t)));
+ }
+ }
+
U.quietAndInfo(log, "Configured failure handler: [hnd=" + hnd + ']');
}
+ /** {@inheritDoc} */
+ @Override public void onKernalStart(boolean active) throws
IgniteCheckedException {
+ if (ignoredFailuresCntMetrics.isEmpty())
+ return;
+
+ MetricRegistryImpl mreg = ctx.metric().registry(IGNORED_FAILURES_REG);
+
+ for (AtomicLongMetric m : ignoredFailuresCntMetrics.values())
+ mreg.register(m);
+ }
+
/**
* @return @{code True} if a node will be stopped by current handler in
near time.
*/
@@ -176,6 +207,8 @@ public class FailureProcessor extends GridProcessorAdapter {
return false;
if (failureTypeIgnored(failureCtx, hnd)) {
+ ignoredFailuresCntMetrics.get(failureCtx.type()).increment();
+
U.quietAndWarn(ignite.log(), IGNORED_FAILURE_LOG_MSG +
"[hnd=" + hnd + ", failureCtx=" + failureCtx + ']',
failureCtx.error());
}
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java
new file mode 100644
index 00000000000..a1f7060d6cb
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.failure;
+
+import java.util.Collections;
+import java.util.Set;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.failure.TestFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.spi.metric.LongMetric;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static org.apache.ignite.failure.FailureType.SEGMENTATION;
+import static
org.apache.ignite.failure.FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_BLOCKED;
+import static
org.apache.ignite.internal.processors.failure.FailureProcessor.IGNORED_FAILURES_REG;
+
+/**
+ * Tests that the failure processor counts suppressed (ignored) failures per
failure type via dedicated metrics
+ * of the form {@code failure.ignored.<FailureType>}.
+ */
+@WithSystemProperty(key =
IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE, value = "false")
+public class FailureProcessorMetricsTest extends GridCommonAbstractTest {
+ /** */
+ private static final String ERR_MSG = "Failure context error";
+
+ /** */
+ private Set<FailureType> ignoredTypes;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ TestFailureHandler hnd = new TestFailureHandler(false);
+
+ hnd.setIgnoredFailureTypes(ignoredTypes);
+
+ cfg.setFailureHandler(hnd);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /** */
+ @Test
+ public void testPerTypeIgnoredFailuresCountMetrics() throws Exception {
+ ignoredTypes = Set.of(SYSTEM_CRITICAL_OPERATION_TIMEOUT,
SYSTEM_WORKER_BLOCKED);
+
+ IgniteEx ignite = startGrid(0);
+
+ LongMetric criticalTimeoutCnt = ignoredFailuresMetric(ignite,
SYSTEM_CRITICAL_OPERATION_TIMEOUT);
+ LongMetric workerBlockedCnt = ignoredFailuresMetric(ignite,
SYSTEM_WORKER_BLOCKED);
+
+ assertEquals(0, criticalTimeoutCnt.value());
+ assertEquals(0, workerBlockedCnt.value());
+
+ ignite.context().failure().process(new
FailureContext(SYSTEM_CRITICAL_OPERATION_TIMEOUT, new Throwable(ERR_MSG)));
+
+ assertEquals(1, criticalTimeoutCnt.value());
+ assertEquals(0, workerBlockedCnt.value());
+
+ ignite.context().failure().process(new FailureContext(SEGMENTATION,
new Throwable(ERR_MSG)));
+
+ assertEquals(1, criticalTimeoutCnt.value());
+ assertEquals(0, workerBlockedCnt.value());
+
+ ignite.context().failure().process(new
FailureContext(SYSTEM_WORKER_BLOCKED, new Throwable(ERR_MSG)));
+
+ assertEquals(1, criticalTimeoutCnt.value());
+ assertEquals(1, workerBlockedCnt.value());
+
+ ignite.context().failure().process(new
FailureContext(SYSTEM_CRITICAL_OPERATION_TIMEOUT, new Throwable(ERR_MSG)));
+
+ assertEquals(2, criticalTimeoutCnt.value());
+ assertEquals(1, workerBlockedCnt.value());
+ }
+
+ /** */
+ @Test
+ public void testNoMetricsRegisteredWhenNothingIgnored() throws Exception {
+ ignoredTypes = Collections.emptySet();
+
+ IgniteEx ignite = startGrid(0);
+
+ for (FailureType t : FailureType.values())
+ assertNull(ignoredFailuresMetric(ignite, t));
+ }
+
+ /** */
+ private LongMetric ignoredFailuresMetric(IgniteEx ignite, FailureType
type) {
+ return
ignite.context().metric().registry(IGNORED_FAILURES_REG).findMetric(type.name());
+ }
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
index 9b5ca62ce85..4a97a041dc2 100644
---
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
+++
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
@@ -76,6 +76,7 @@ import
org.apache.ignite.internal.processors.database.FreeListCutTailDifferentGc
import org.apache.ignite.internal.processors.database.IndexStorageSelfTest;
import
org.apache.ignite.internal.processors.database.SwapPathConstructionSelfTest;
import
org.apache.ignite.internal.processors.failure.FailureProcessorLoggingTest;
+import
org.apache.ignite.internal.processors.failure.FailureProcessorMetricsTest;
import
org.apache.ignite.internal.processors.failure.FailureProcessorThreadDumpThrottlingTest;
import
org.apache.ignite.internal.processors.metastorage.DistributedMetaStorageClassloadingTest;
import
org.apache.ignite.internal.processors.metastorage.DistributedMetaStorageTest;
@@ -164,6 +165,7 @@ import org.junit.runners.Suite;
OomFailureHandlerTest.class,
TransactionIntegrityWithSystemWorkerDeathTest.class,
FailureProcessorLoggingTest.class,
+ FailureProcessorMetricsTest.class,
FailureProcessorThreadDumpThrottlingTest.class,
ExchangeTaskHandlerWaitingForTasksTest.class,