Ottomata has submitted this change and it was merged.
Change subject: Abstracting out udp2log monitoring into its own define
......................................................................
Abstracting out udp2log monitoring into its own define
Configuring monitoring for analytics udp2log instances
Change-Id: I0d6c49b8acafd0fcdc3e0a379bdb2f6b0129c4bc
---
M manifests/misc/analytics.pp
M manifests/misc/icinga.pp
M manifests/misc/udp2log.pp
M manifests/site.pp
D templates/icinga/analytics.cfg.erb
5 files changed, 69 insertions(+), 62 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/misc/analytics.pp b/manifests/misc/analytics.pp
index 9b66a11..baaf8a8 100644
--- a/manifests/misc/analytics.pp
+++ b/manifests/misc/analytics.pp
@@ -23,4 +23,4 @@
minute => 15,
require => File[$tmp_dir],
}
-}
\ No newline at end of file
+}
diff --git a/manifests/misc/icinga.pp b/manifests/misc/icinga.pp
index 8239661..9bd63f8 100644
--- a/manifests/misc/icinga.pp
+++ b/manifests/misc/icinga.pp
@@ -164,13 +164,10 @@
mode => '0644';
# TEMP: analytics eqiad cluster manual entries.
- # These will be removed from this manually managed file
- # once analytics puppetization goes to production.
+ # This has been removed since analytics cluster
+ # udp2log instances are now puppetized.
'/etc/icinga/analytics.cfg':
- content => template('icinga/analytics.cfg.erb'),
- owner => 'root',
- group => 'root',
- mode => '0644';
+ ensure => 'absent';
'/etc/icinga/checkcommands.cfg':
content => template('icinga/checkcommands.cfg.erb'),
diff --git a/manifests/misc/udp2log.pp b/manifests/misc/udp2log.pp
index 8b46c65..7f297f0 100644
--- a/manifests/misc/udp2log.pp
+++ b/manifests/misc/udp2log.pp
@@ -169,7 +169,37 @@
require => [Package["udplog"], File["/etc/udp2log/${name}"],
File["/etc/init.d/udp2log-${name}"]],
}
+ # include monitoring for this udp2log instance.
+ misc::udp2log::instance::monitoring { $name:
+ log_directory => $log_directory,
+ ensure => $ensure,
+ packet_loss_log => $packet_loss_log,
+ monitor_packet_loss => $monitor_packet_loss,
+ monitor_processes => $monitor_processes,
+ monitor_log_age => $monitor_log_age,
+ require => Service["udp2log-${name}"],
+ }
+}
+
+# == Define misc::udp2log::instance::monitoring
+# Monitoring configs for a udp2log instance.
+# This is abstracted out of the udp2log::instance
+# define so it is possible to monitor non-puppetized
+# udp2log instances.
+#
+# == Parameters:
+# See documentation for misc::udp2log::instance.
+#
+define misc::udp2log::instance::monitoring(
+ $log_directory = "/var/log/udp2log",
+ $ensure = "running",
+ $packet_loss_log = undef,
+ $monitor_packet_loss = true,
+ $monitor_processes = true,
+ $monitor_log_age = true,
+) {
+ require misc::udp2log::monitoring
# Monitoring configs.
# There are 3 ways udp2log instances are currently defined:
@@ -186,13 +216,10 @@
description => "udp2log log age for ${name}",
nrpe_command =>
"/usr/lib/nagios/plugins/check_udp2log_log_age ${name}",
contact_group => "admins,analytics",
- require => Class["misc::udp2log::monitoring"],
ensure => "present";
}
}
# TODO else ensure absent, can't do this right now due to missing
dependencies
-
-
# Monitor that each filter process defined in
# /etc/udp2log/$name is running
@@ -202,14 +229,10 @@
nrpe_command =>
"/usr/lib/nagios/plugins/check_udp2log_procs ${name}",
contact_group => "admins,analytics",
retries => 10,
- require => Class["misc::udp2log::monitoring"],
ensure => "present";
}
}
# TODO else ensure absent, can't do this right now due to missing
dependencies
-
-
-
# Monitor packet loss using the $packet_loss_log.
# This requires that filters.$name.erb has a
@@ -237,7 +260,6 @@
command => "/usr/sbin/ganglia-logtailer --classname
PacketLossLogtailer --log_file ${packet_loss_log_file} --mode cron",
user => 'root',
minute => '*/5',
- require => Class["misc::udp2log::monitoring"],
ensure => $ensure_monitor_packet_loss,
}
@@ -254,7 +276,6 @@
normal_check_interval => 2,
retry_check_interval => 2,
retries => 4,
- require =>
Class["misc::udp2log::monitoring"],
ensure => $ensure_monitor_packet_loss,
}
}
@@ -332,6 +353,7 @@
}
}
+
class misc::udp2log::iptables_purges {
require "iptables::tables"
# The deny rule must always be purged, otherwise ACCEPTs can be placed
below it
diff --git a/manifests/site.pp b/manifests/site.pp
index 22bb58f..4e1caa4 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -134,6 +134,36 @@
include misc::udp2log::iptables
}
+node "analytics1002.eqiad.wmnet" {
+ include role::analytics
+}
+
+# analytics1003 - analytics1006 are udp2log instances.
+node /analytics100[3456].eqiad.wmnet/ inherits analytics_basenode {
+ # ganglia aggregator for the Analytics cluster.
+ $ganglia_aggregator = "true"
+
+ # monitor the (currently unpuppetized)
+ # udp2log instances.
+ misc::udp2log::instance::monitoring { 'webrequest':
+ log_directory => '/var/log/udp2log/webrequest',
+ monitor_packet_loss => true,
+ monitor_processes => true,
+ monitor_log_age => false,
+ }
+}
+
+
+# analytics1007 - analytics1026
+node /analytics10(0[7-9]|1[0-9]|2[0-6])\.eqiad\.wmnet/ {
+ # ganglia aggregator for the Analytics cluster.
+ if ($hostname == "analytics1011") {
+ $ganglia_aggregator = "true"
+ }
+
+ include role::analytics
+}
+
# analytics1027 hosts the frontend
# interfaces to Kraken and Hadoop.
node "analytics1027.eqiad.wmnet" {
@@ -149,15 +179,11 @@
}
}
-# analytics1002 - analytics1026
-node /analytics10(0[2-9]|1[0-9]|2[0-6])\.eqiad\.wmnet/ {
- # ganglia aggregator for the Analytics cluster.
- if ($hostname == "analytics1003" or $hostname == "analytics1011") {
- $ganglia_aggregator = "true"
- }
- include role::analytics
-}
+
+
+
+
node /(arsenic|niobium|strontium|palladium)\.(wikimedia\.org|eqiad\.wmnet)/ {
if $hostname =~ /^(arsenic|niobium)$/ {
diff --git a/templates/icinga/analytics.cfg.erb
b/templates/icinga/analytics.cfg.erb
deleted file mode 100644
index b1467c9..0000000
--- a/templates/icinga/analytics.cfg.erb
+++ /dev/null
@@ -1,38 +0,0 @@
-# Note: This file is managed by Puppet.
-#
-<%
-# This file currently contains manually maintained analytics
-# cluster nagios checks. This file is intended to be temporary
-# and will managed via puppet monitor_service defines rather
-# than as this manual ERb template once Analytics puppetization
-# is in full production.
-
-analytics_webrequest_udp2log_hosts = [
- "analytics1003",
- "analytics1004",
- "analytics1005",
- "analytics1006",
-]
-
-analytics_webrequest_udp2log_hosts.each do |hostname| -%>
-
-# Check <%= hostname %> for webrequest udp2log packet loss
-define service {
- is_volatile 0
- passive_checks_enabled 1
- check_command check_packet_loss_ave!4!8
- check_period 24x7
- retry_check_interval 2
- notification_interval 0
- notification_options c,r,f
- normal_check_interval 2
- service_description Packetloss_Average
- ## --PUPPET_NAME-- (called '_naginator_name' in the manifest)
<%= hostname %> udp2log-<%= hostname %>-packetloss
- contact_groups analytics
- host_name <%= hostname %>
- max_check_attempts 4
- notification_period 24x7
- check_freshness 0
- active_checks_enabled 1
-}
-<% end -%>
--
To view, visit https://gerrit.wikimedia.org/r/56537
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0d6c49b8acafd0fcdc3e0a379bdb2f6b0129c4bc
Gerrit-PatchSet: 3
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Faidon <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
