Ottomata has submitted this change and it was merged.
Change subject: Adding alerts for Kafka Broker replication metrics
......................................................................
Adding alerts for Kafka Broker replication metrics
Change-Id: I3116f74b7fcf6a66f865f4c5acba98593a513ffe
---
M manifests/role/analytics/kafka.pp
1 file changed, 26 insertions(+), 1 deletion(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/analytics/kafka.pp
b/manifests/role/analytics/kafka.pp
index 6108416..b7dcbeb 100644
--- a/manifests/role/analytics/kafka.pp
+++ b/manifests/role/analytics/kafka.pp
@@ -171,5 +171,30 @@
critical => ':1000.0',
require => Class['::kafka::server::jmxtrans'],
}
-}
+ # Alert if any Kafka has under replicated partitions.
+ # If it does, this means a broker replica is falling behind
+ # and will be removed from the ISR.
+ monitor_ganglia { 'kafka-broker-UnderReplicatedPartitions':
+ description => 'Kafka Broker Under Replicated Partitions',
+ metric =>
'kafka.server.ReplicaManager.UnderReplicatedPartitions.Value',
+ # Any under replicated partitions are bad.
+ # Over 10 means (probably) that at least an entire topic
+ # is under replicated.
+ warning => '1',
+ critical => '10',
+ require => Class['::kafka::server::jmxtrans'],
+ }
+
+ # Alert if any Kafka Broker replica lag is too high
+ monitor_ganglia { 'kafka-broker-Replica-MaxLag':
+ description => 'Kafka Broker Replica Lag',
+ metric =>
'kafka.server.ReplicaFetcherManager.Replica-MaxLag.Value',
+ # As of 2014-02 replag could catch up at more than 1000 msgs / sec,
+ # (probably more like 2 or 3 K / second). At that rate, 1M messages
+ # behind should catch back up in at least 30 minutes.
+ warning => '1000000',
+ critical => '5000000',
+ require => Class['::kafka::server::jmxtrans'],
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/112065
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3116f74b7fcf6a66f865f4c5acba98593a513ffe
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
