This is an automated email from the ASF dual-hosted git repository.
lizhimins pushed a commit to branch rocketmq-studio
in repository https://gitbox.apache.org/repos/asf/rocketmq-dashboard.git
The following commit(s) were added to refs/heads/rocketmq-studio by this push:
new 9c180ae64 fix(alert): require metric equality when reconciling native
alert presence (#3145)
9c180ae64 is described below
commit 9c180ae641976873e16aec7cd41544af49bb627b
Author: 烤化の初雪 <[email protected]>
AuthorDate: Mon Sep 7 17:46:56 2026 +0800
fix(alert): require metric equality when reconciling native alert presence
(#3145)
Registry rows were not the only identity confusion: native alert
reconciliation treated any sample matching a rule's instance and
resource labels as proof that the rule's own metric is still reported.
Because fingerprints are derived from ruleId + instanceId + labels, a
different metric emitted with the same labels (e.g. consumer.lag.total
for a group whose consumer.delay.seconds sample disappeared) kept the
stale FIRING/ACKED state active forever and suppressed the RESOLVED
event and recovery notification.
Require the sample metric to equal the rule metric when building the
reconciliation presentKeys, so a missing metric in a successful
collection scope resolves its prior active state even when another
same-label metric remains.
Regression test:
resolvesMissingMetricEvenWhenAnotherMetricSharesTheSameLabelsTest
(fails before the fix: state was never saved as RESOLVED).
Fixes #3104
Co-authored-by: unbridled-41
<[email protected]>
---
.../studio/ops/alert/NativeAlertProcessor.java | 4 +-
.../studio/ops/alert/NativeAlertProcessorTest.java | 43 ++++++++++++++++++++++
2 files changed, 46 insertions(+), 1 deletion(-)
diff --git
a/server/src/main/java/org/apache/rocketmq/studio/ops/alert/NativeAlertProcessor.java
b/server/src/main/java/org/apache/rocketmq/studio/ops/alert/NativeAlertProcessor.java
index fad40db82..2ae7b22d1 100644
---
a/server/src/main/java/org/apache/rocketmq/studio/ops/alert/NativeAlertProcessor.java
+++
b/server/src/main/java/org/apache/rocketmq/studio/ops/alert/NativeAlertProcessor.java
@@ -24,6 +24,7 @@ import
org.apache.rocketmq.studio.cluster.metrics.MetricSample;
import org.apache.rocketmq.studio.common.domain.enums.AlertLevel;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.StringUtils;
import java.time.Instant;
import java.time.LocalDateTime;
@@ -97,7 +98,7 @@ public class NativeAlertProcessor {
List<AlertRuleVO> rules =
alertService.listRules(scope.domain()).stream()
.filter(rule -> rule.getId() != null)
.filter(AlertRuleVO::isEnabled)
- .filter(rule -> scope.metricKeys().contains(rule.getMetric()))
+ .filter(rule ->
scope.metricKeys().contains(StringUtils.trimWhitespace(rule.getMetric())))
.filter(rule -> rule.getInstanceId() == null ||
scope.instanceId().equals(rule.getInstanceId()))
.toList();
if (rules.isEmpty()) {
@@ -106,6 +107,7 @@ public class NativeAlertProcessor {
Set<AlertStateKey> presentKeys = samples.stream()
.filter(scope::contains)
.flatMap(sample -> rules.stream()
+ .filter(rule ->
sample.metricKey().equals(StringUtils.trimWhitespace(rule.getMetric())))
.filter(rule ->
NativeAlertRuleScopeMatcher.matches(rule, sample))
.map(rule -> new AlertStateKey(rule.getId(),
AlertFingerprint.of(rule.getId(),
sample.instanceId(), sample.labels()))))
diff --git
a/server/src/test/java/org/apache/rocketmq/studio/ops/alert/NativeAlertProcessorTest.java
b/server/src/test/java/org/apache/rocketmq/studio/ops/alert/NativeAlertProcessorTest.java
index e7f530711..54c462407 100644
---
a/server/src/test/java/org/apache/rocketmq/studio/ops/alert/NativeAlertProcessorTest.java
+++
b/server/src/test/java/org/apache/rocketmq/studio/ops/alert/NativeAlertProcessorTest.java
@@ -477,6 +477,49 @@ class NativeAlertProcessorTest {
verify(alerts, never()).saveAlert(any(SystemAlertVO.class));
}
+ @Test
+ void resolvesMissingMetricEvenWhenAnotherMetricSharesTheSameLabelsTest() {
+ AlertService service = mock(AlertService.class);
+ AlertRuleVO rule =
AlertRuleVO.builder().id(1L).domain(AlertDomain.BUSINESS).name("Orders delay")
+
.metric("consumer.delay.seconds").operator(">").threshold(10).enabled(true)
+
.instanceId("local").consumerGroup("orders").consecutiveSamples(1).build();
+
when(service.listRules(AlertDomain.BUSINESS)).thenReturn(List.of(rule));
+ MetricSample previousDelay = new
MetricSample("consumer.delay.seconds", AlertDomain.BUSINESS, "local",
+ null, Map.of("consumerGroup", "orders"), 120D,
MetricAvailability.AVAILABLE, Instant.now());
+ AlertStateKey key = new AlertStateKey(rule.getId(),
+ AlertFingerprint.of(rule.getId(), previousDelay.instanceId(),
previousDelay.labels()));
+ ActiveAlertState active = new ActiveAlertState(key,
+ new AlertRuleState(AlertStateStatus.FIRING, 1, 120D,
previousDelay.collectedAt().minusSeconds(60),
+ previousDelay.collectedAt().minusSeconds(60),
previousDelay.collectedAt().minusSeconds(60),
+ null),
+ previousDelay.instanceId(), previousDelay.labels());
+ AlertStateRepository states = mock(AlertStateRepository.class);
+ when(states.findActive(any(MetricCollectionScope.class),
eq(List.of(rule)))).thenReturn(List.of(active));
+ when(states.save(eq(key), any(AlertRuleState.class))).thenReturn(true);
+ AlertRepository alerts = mock(AlertRepository.class);
+ when(alerts.saveAlert(any(SystemAlertVO.class))).thenAnswer(invocation
-> invocation.getArgument(0));
+ NotificationOutboxService outbox =
mock(NotificationOutboxService.class);
+
+ MetricSample lagSample = new MetricSample("consumer.lag.total",
AlertDomain.BUSINESS, "local",
+ null, Map.of("consumerGroup", "orders"), 5D,
MetricAvailability.AVAILABLE, Instant.now());
+
+ new NativeAlertProcessor(service,
+ new NativeAlertEvaluationService(new AlertRuleEvaluator(), new
AlertStateMachine(), states,
+ mock(MetricSnapshotRepository.class), alerts, outbox,
suppression()),
+ new AlertStateMachine(), states, alerts, outbox, suppression())
+ .processSuccessfulCollection(new
MetricCollectionScope(AlertDomain.BUSINESS, "local",
+ java.util.Set.of("consumer.delay.seconds",
"consumer.lag.total")), List.of(lagSample));
+
+ org.mockito.ArgumentCaptor<AlertRuleState> state =
org.mockito.ArgumentCaptor.forClass(AlertRuleState.class);
+ org.mockito.ArgumentCaptor<SystemAlertVO> event =
org.mockito.ArgumentCaptor.forClass(SystemAlertVO.class);
+ verify(states).save(eq(key), state.capture());
+
assertThat(state.getValue().status()).isEqualTo(AlertStateStatus.RESOLVED);
+ verify(alerts).saveAlert(event.capture());
+
assertThat(event.getValue().getTransition()).isEqualTo(AlertStateTransition.RESOLVED.name());
+
assertThat(event.getValue().getLabels()).isEqualTo(previousDelay.labels());
+ verify(outbox).enqueue(any(SystemAlertVO.class), eq(rule),
eq(previousDelay.labels()));
+ }
+
@Test
void
doesNotResolveMissingActiveStateWhenCollectionReportsWholeScopeUnavailableTest()
{
AlertService service = mock(AlertService.class);
