This is an automated email from the ASF dual-hosted git repository.
lizhimins pushed a commit to branch rocketmq-studio
in repository https://gitbox.apache.org/repos/asf/rocketmq-dashboard.git
The following commit(s) were added to refs/heads/rocketmq-studio by this push:
new cd204a87c fix(alert): correct two built-in Prometheus rule expressions
(#5419)
cd204a87c is described below
commit cd204a87c99b94410caf340dc1101ac6be7edf70
Author: 烤化の初雪 <[email protected]>
AuthorDate: Fri Oct 9 16:25:21 2026 +0800
fix(alert): correct two built-in Prometheus rule expressions (#5419)
Two built-in alert rules did not behave as documented.
- `RocketMQJVMCpuHigh` multiplied a label-less `avg()` by a `rate()` that
carries `cluster`/`broker`,
so the vector match came back empty and the rule could never fire. It now
aggregates with the
grouping preserved - `sum by (cluster, broker)
(rate(jvm_gc_pause_seconds_sum[5m])) > 0.3` - which
keeps the intended "this broker spends 30% of its time in GC" meaning
instead of summing ratios
across the fleet. `cluster`/`broker` are the labels the exporter puts on
JVM metrics, as
`grafana/rocketmq-jvm.json` already assumes. (#5419)
- `RocketMQDLQResendHigh` used `rate(...[5m]) > 10`, i.e. ten resends per
second (~3000 per window),
while its annotation and the sibling rules describe a five-minute count.
It now uses
`increase(...[5m]) > 10`, applied to all three copies: the rule yaml, the
generator and the
frontend mock. (#5420)
Assets regenerated with `server/scripts/gen_alert_rule_yaml.py`; the
generator output matches the
committed yaml and mock byte for byte. `AlertRuleAssetServiceTest` 15 tests
green, 0 checkstyle
violations. The test added for the DLQ rule was renamed to end with `Test`
per the repository
convention.
Folded in #5420 (same author, same generator and asset files).
---
server/scripts/gen_alert_rule_yaml.py | 4 +-
.../resources/alerts/rocketmq-dlq-resend-high.yaml | 2 +-
.../resources/alerts/rocketmq-jvm-gc-cpu-high.yaml | 2 +-
.../ops/alert/AlertRuleAssetServiceTest.java | 51 ++++++++++++++++++++++
web/src/mock/alertRuleAssets.ts | 4 +-
5 files changed, 57 insertions(+), 6 deletions(-)
diff --git a/server/scripts/gen_alert_rule_yaml.py
b/server/scripts/gen_alert_rule_yaml.py
index 17073f0a8..16bf207f3 100644
--- a/server/scripts/gen_alert_rule_yaml.py
+++ b/server/scripts/gen_alert_rule_yaml.py
@@ -73,13 +73,13 @@ RULES = [
'rate(rocketmq_broker_exception_count[5m]) > 0', "5m", "critical",
"broker",
"Broker exceptions", "Broker runtime exceptions have been observed."),
("rocketmq-dlq-resend-high", "RocketMQDLQResendHigh",
"rocketmq-errors.rules",
- 'rate(rocketmq_dlq_resend_count[5m]) > 10', "5m", "warning", "consumer",
+ 'increase(rocketmq_dlq_resend_count[5m]) > 10', "5m", "warning",
"consumer",
"DLQ resends high", "More than 10 dead-letter queue resends occurred in 5
minutes."),
("rocketmq-threadpool-reject", "RocketMQThreadPoolReject",
"rocketmq-broker.rules",
'increase(rocketmq_threadpool_reject_count[5m]) > 0', "5m", "critical",
"broker",
"Thread pool rejections", "The broker thread pool rejected tasks,
indicating saturation."),
("rocketmq-jvm-gc-cpu-high", "RocketMQJVMCpuHigh", "rocketmq-broker.rules",
- 'rate(jvm_gc_pause_seconds_count[5m]) *
avg(rate(jvm_gc_pause_seconds_sum[5m])) > 0.3', "5m", "warning", "broker",
+ 'sum by (cluster, broker) (rate(jvm_gc_pause_seconds_sum[5m])) > 0.3',
"5m", "warning", "broker",
"JVM GC CPU high", "The broker spends more than 30% of CPU time in GC
pauses."),
]
diff --git a/server/src/main/resources/alerts/rocketmq-dlq-resend-high.yaml
b/server/src/main/resources/alerts/rocketmq-dlq-resend-high.yaml
index ab0fa3f35..3644bab8f 100644
--- a/server/src/main/resources/alerts/rocketmq-dlq-resend-high.yaml
+++ b/server/src/main/resources/alerts/rocketmq-dlq-resend-high.yaml
@@ -6,7 +6,7 @@ groups:
- name: rocketmq-errors.rules
rules:
- alert: RocketMQDLQResendHigh
- expr: rate(rocketmq_dlq_resend_count[5m]) > 10
+ expr: increase(rocketmq_dlq_resend_count[5m]) > 10
for: 5m
labels:
severity: warning
diff --git a/server/src/main/resources/alerts/rocketmq-jvm-gc-cpu-high.yaml
b/server/src/main/resources/alerts/rocketmq-jvm-gc-cpu-high.yaml
index cd9e4a995..524fd4a5c 100644
--- a/server/src/main/resources/alerts/rocketmq-jvm-gc-cpu-high.yaml
+++ b/server/src/main/resources/alerts/rocketmq-jvm-gc-cpu-high.yaml
@@ -6,7 +6,7 @@ groups:
- name: rocketmq-broker.rules
rules:
- alert: RocketMQJVMCpuHigh
- expr: rate(jvm_gc_pause_seconds_count[5m]) *
avg(rate(jvm_gc_pause_seconds_sum[5m])) > 0.3
+ expr: sum by (cluster, broker) (rate(jvm_gc_pause_seconds_sum[5m])) >
0.3
for: 5m
labels:
severity: warning
diff --git
a/server/src/test/java/org/apache/rocketmq/studio/ops/alert/AlertRuleAssetServiceTest.java
b/server/src/test/java/org/apache/rocketmq/studio/ops/alert/AlertRuleAssetServiceTest.java
index 74bb73430..5c071b1d2 100644
---
a/server/src/test/java/org/apache/rocketmq/studio/ops/alert/AlertRuleAssetServiceTest.java
+++
b/server/src/test/java/org/apache/rocketmq/studio/ops/alert/AlertRuleAssetServiceTest.java
@@ -64,6 +64,31 @@ class AlertRuleAssetServiceTest {
}
}
+ @Test
+ void gcCpuAlertShouldCompareTheGcTimeRatioTest() {
+ PrometheusAlertRule rule = service.loadDefaultRules().stream()
+ .filter(candidate ->
"RocketMQJVMCpuHigh".equals(candidate.alert()))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("bundled
RocketMQJVMCpuHigh rule missing"));
+
+ // rate(jvm_gc_pause_seconds_sum[5m]) is the share of wall clock spent
in GC pauses, so it
+ // needs no companion factor; grouping keeps the threshold per broker
instead of summing
+ // the whole fleet into one label-less series, and keeps
cluster/broker on the alert.
+ assertEquals("sum by (cluster, broker)
(rate(jvm_gc_pause_seconds_sum[5m])) > 0.3", rule.expr());
+ }
+
+ @Test
+ void generatorShouldUseSameGroupedGcPauseRatioExpressionTest() throws
IOException {
+ PrometheusAlertRule rule = service.loadDefaultRules().stream()
+ .filter(candidate ->
"RocketMQJVMCpuHigh".equals(candidate.alert()))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("bundled
RocketMQJVMCpuHigh rule missing"));
+ String generator = Files.readString(Path.of("scripts",
"gen_alert_rule_yaml.py"));
+
+ assertTrue(generator.contains("'" + rule.expr() + "'"),
+ "generator must keep emitting the shipped expression");
+ }
+
@Test
void getAssetYamlShouldReturnRawContent() {
List<AlertRuleAssetInfo> assets = service.listAssets();
@@ -172,6 +197,32 @@ class AlertRuleAssetServiceTest {
assertTrue(service.getAssetYaml("duplicate").contains("FirstRule"));
}
+ @Test
+ void dlqResendAlertShouldCountResendsOverTheDocumentedWindowTest() {
+ PrometheusAlertRule rule = service.loadDefaultRules().stream()
+ .filter(candidate ->
"RocketMQDLQResendHigh".equals(candidate.alert()))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("bundled
RocketMQDLQResendHigh rule missing"));
+
+ // The annotation promises "More than 10 dead-letter queue resends
occurred in 5 minutes",
+ // and the sibling rebalance rule expresses the same "more than N in 5
minutes" contract
+ // with increase(...[5m]) > N. A rate(...) > 10 threshold instead
means 10 resends per
+ // *second* (about 3000 per window), so a queue quietly re-driving
resends stays silent.
+ assertEquals("increase(rocketmq_dlq_resend_count[5m]) > 10",
rule.expr());
+ }
+
+ @Test
+ void generatorShouldUseSameDlqResendWindowExpressionTest() throws
IOException {
+ PrometheusAlertRule rule = service.loadDefaultRules().stream()
+ .filter(candidate ->
"RocketMQDLQResendHigh".equals(candidate.alert()))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("bundled
RocketMQDLQResendHigh rule missing"));
+ String generator = Files.readString(Path.of("scripts",
"gen_alert_rule_yaml.py"));
+
+ assertTrue(generator.contains("'" + rule.expr() + "'"),
+ "generator must keep emitting the shipped expression");
+ }
+
private static AlertRuleAssetService serviceWithResources(Resource...
resources) {
return new AlertRuleAssetService() {
@Override
diff --git a/web/src/mock/alertRuleAssets.ts b/web/src/mock/alertRuleAssets.ts
index abd6cbbc5..ba3a32900 100644
--- a/web/src/mock/alertRuleAssets.ts
+++ b/web/src/mock/alertRuleAssets.ts
@@ -177,7 +177,7 @@ export const mockAlertRuleAssets: MockAlertRuleAsset[] = [
'rocketmq-dlq-resend-high',
'rocketmq-errors.rules',
'RocketMQDLQResendHigh',
- 'rate(rocketmq_dlq_resend_count[5m]) > 10',
+ 'increase(rocketmq_dlq_resend_count[5m]) > 10',
'warning',
),
},
@@ -203,7 +203,7 @@ export const mockAlertRuleAssets: MockAlertRuleAsset[] = [
'rocketmq-jvm-gc-cpu-high',
'rocketmq-broker.rules',
'RocketMQJVMCpuHigh',
- 'rate(jvm_gc_pause_seconds_count[5m]) *
avg(rate(jvm_gc_pause_seconds_sum[5m])) > 0.3',
+ 'sum by (cluster, broker) (rate(jvm_gc_pause_seconds_sum[5m])) > 0.3',
'warning',
),
},
