Ottomata has submitted this change and it was merged.
Change subject: Adding icinga check to make sure kafka brokers aren't getting
too many produce requests
......................................................................
Adding icinga check to make sure kafka brokers aren't getting too many produce
requests
Change-Id: If8bf706692331d1e2963e5dda289b5be819d309a
---
M manifests/misc/analytics.pp
M templates/icinga/checkcommands.cfg.erb
2 files changed, 18 insertions(+), 6 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/manifests/misc/analytics.pp b/manifests/misc/analytics.pp
index cf598fa..c190eea 100644
--- a/manifests/misc/analytics.pp
+++ b/manifests/misc/analytics.pp
@@ -29,10 +29,16 @@
class misc::analytics::monitoring::kafka::server {
# Set up icinga monitoring of Kafka broker server produce requests per
second.
# If this drops too low, trigger an alert
- # for this udp2log instance.
- monitor_service { "kakfa-broker-ProduceRequestsPerSecond":
- description =>
"kafka_network_SocketServerStats.ProduceRequestsPerSecond",
- check_command =>
"check_kafka_broker_produce_requests!2!1",
- contact_group => "analytics",
+ monitor_service { 'kafka_broker-ProduceRequestsPerSecond':
+ description =>
'kafka_network_SocketServerStats.ProduceRequestsPerSecond',
+ check_command => 'check_kafka_broker_produce_requests_min!2!1',
+ contact_group => 'analytics',
+ }
+ # Set up icinga monitoring of Kafka broker server produce requests per
second.
+ # If this drops too high, trigger an alert
+ monitor_service { 'kafka_broker-ProduceRequestsPerSecond':
+ description =>
'kafka_network_SocketServerStats.ProduceRequestsPerSecond',
+ check_command =>
'check_kafka_broker_produce_requests_max!15!20',
+ contact_group => 'analytics',
}
}
diff --git a/templates/icinga/checkcommands.cfg.erb
b/templates/icinga/checkcommands.cfg.erb
index b156f7e..b40ea35 100644
--- a/templates/icinga/checkcommands.cfg.erb
+++ b/templates/icinga/checkcommands.cfg.erb
@@ -617,10 +617,16 @@
# Analytics Kraken checks
# Check that Kafka Brokers are getting ProduceRequests.
define command{
- command_name check_kafka_broker_produce_requests
+ command_name check_kafka_broker_produce_requests_min
command_line $USER1$/check_ganglios_generic_value -H $HOSTADDRESS$
-m kafka_network_SocketServerStats.ProduceRequestsPerSecond -w $ARG1$ -c $ARG2$
-o lt
}
+# Check that Kafka Brokers aren't getting TOO many ProduceRequests
+define command{
+ command_name check_kafka_producer_produce_requests_max
+ command_line $USER1$/check_ganglios_generic_value -H $HOSTADDRESS$
-m kafka_network_SocketServerStats.ProduceRequestsPerSecond -w $ARG1$ -c $ARG2$
-o gt
+}
+
# Alerts for data loss in Kraken HDFS.
define command{
command_name check_kraken_webrequest_loss_positive
--
To view, visit https://gerrit.wikimedia.org/r/72004
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: If8bf706692331d1e2963e5dda289b5be819d309a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
