Volans has submitted this change and it was merged.
Change subject: Monitoring: add event handler for RAID checks
......................................................................
Monitoring: add event handler for RAID checks
Add an event handler to Icinga RAID checks to automatically handle their
alarms. If a CRITICAL state is detected a Phabricator task with the
appropriate DC-Ops tag will be created and the Icinga alarm
acknowledged.
Bug: T142085
Change-Id: I3f2b4899089b8eed2f70f1b7b95182b9a3fb67bb
Depends-On: I585f1cc8d4eaff408c8ebc43252e09770d10f3cd
---
M manifests/role/icinga.pp
A modules/icinga/files/raid_handler.py
A modules/icinga/manifests/event_handlers/raid.pp
A modules/icinga/templates/event_handlers/raid_handler.cfg.erb
M modules/monitoring/manifests/service.pp
M modules/nrpe/manifests/monitor_service.pp
M modules/raid/manifests/init.pp
7 files changed, 301 insertions(+), 6 deletions(-)
Approvals:
Volans: Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/icinga.pp b/manifests/role/icinga.pp
index f109823..29423b7 100644
--- a/manifests/role/icinga.pp
+++ b/manifests/role/icinga.pp
@@ -27,6 +27,7 @@
include icinga::monitor::commons
include icinga::monitor::elasticsearch
include icinga::monitor::wdqs
+ include icinga::event_handlers::raid
include role::authdns::monitoring
include netops::monitoring
diff --git a/modules/icinga/files/raid_handler.py
b/modules/icinga/files/raid_handler.py
new file mode 100644
index 0000000..00d2609
--- /dev/null
+++ b/modules/icinga/files/raid_handler.py
@@ -0,0 +1,237 @@
+#!/usr/bin/env python
+"""Nagios/Icinga Event Handler for RAID checks"""
+
+import argparse
+import ConfigParser
+import logging
+import subprocess
+import time
+import zlib
+
+from logging.handlers import RotatingFileHandler
+
+from phabricator import Phabricator
+
+SERVICE_STATES = ('OK', 'UNKNOWN', 'WARNING', 'CRITICAL')
+SERVICE_STATE_TYPES = ('SOFT', 'HARD')
+
+RAID_TYPES = ('megacli', 'hpssacli', 'mpt', 'md')
+COMPRESSED_RAID_TYPES = ('megacli', 'hpssacli')
+
+LOG_PATH = '/var/log/icinga/raid_handler.log'
+COMMAND_FILE = '/var/lib/nagios/rw/nagios.cmd'
+CHECK_NRPE_PATH = '/usr/lib/nagios/plugins/check_nrpe'
+
+NRPE_REMOTE_COMMAND = 'get_raid_status_{}'
+ACK_MESSAGE = 'RAID handler auto-ack: {}'
+ICINGA_URL = ('https://icinga.wikimedia.org/cgi-bin/icinga/extinfo.cgi?type=2&;'
+ 'host={host}&service={service}')
+
+PHABRICATOR_CONFIG_FILE = '/etc/phabricator_ops-monitoring-bot.conf'
+PHABRICATOR_TAG_PREFIX = 'ops-'
+PHABRICATOR_TASK_TITLE = "Degraded RAID on {host}"
+PHABRICATOR_TASK_DESCRIPTION_PREFIX = (
+ "TASK AUTO-GENERATED by Nagios/Icinga RAID event handler\n\n"
+ "A degraded RAID [[ {url} | was detected ]] on host `{host}`. An "
+ "automatic snapshot of the current RAID status is attached below.\n\n"
+ "Please **sync with the service owner** to find the appropriate time "
+ "window before actually replacing any failed hardware."
+)
+
+logger = logging.getLogger('raid_handler')
+
+
+def parse_args():
+ """Parse command line arguments"""
+
+ parser = argparse.ArgumentParser(
+ description='Nagios/Icinga event handler for RAID checks')
+ parser.add_argument(
+ '-s', dest='service_state', action='store', required=True,
+ choices=SERVICE_STATES, help='Nagios/Icinga service state')
+ parser.add_argument(
+ '-t', dest='service_state_type', action='store', required=True,
+ choices=SERVICE_STATE_TYPES, help='Nagios/Icinga service state type')
+ parser.add_argument(
+ '-a', dest='service_attempts', action='store', required=True, type=int,
+ help='Nagios/Icinga service retry attemp counter')
+ parser.add_argument(
+ '-H', dest='host_address', action='store', required=True,
+ help='Hostname/address of the monitored host')
+ parser.add_argument(
+ '-r', dest='raid_type', action='store', required=True,
+ choices=RAID_TYPES, help='The RAID type')
+ parser.add_argument(
+ '-D', dest='service_description', action='store', required=True,
+ help='The Nagios/Icinga service description')
+ parser.add_argument(
+ '-c', dest='datacenter', action='store', required=True,
+ help='The name of the datacenter the host is located in')
+ parser.add_argument(
+ '-d', dest='debug', action='store_true', help='Debug level logging')
+
+ return parser.parse_args()
+
+
+def get_raid_status(host, raid_type):
+ """ Get and return the RAID status of a remote host
+
+ Arguments:
+ host -- hostname to be passed to the NRPE check
+ raid_type -- the RAID type to check, see RAID_TYPES for accepted values
+ """
+
+ try:
+ nrpe_command = [CHECK_NRPE_PATH, '-H', host,
+ '-c', NRPE_REMOTE_COMMAND.format(raid_type)]
+ proc = subprocess.Popen(nrpe_command, stdout=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
+ except Exception as e:
+ logger.error("Unable to execute '{}': {}".format(nrpe_command, e))
+ raise
+
+ if raid_type in COMPRESSED_RAID_TYPES:
+ # NRPE doesn't handle NULL bytes, decoding them.
+ # Given the specific domain there was no need of a full yEnc encoding.
+ status = zlib.decompress(stdout.replace('###NULL###', '\x00'))
+ else:
+ status = stdout
+
+ logger.debug(status)
+ return status
+
+
+def get_phabricator_client():
+ """Return a Phabricator client instance"""
+
+ parser = ConfigParser.SafeConfigParser()
+ parser_mode = 'phabricator_bot'
+ parser.read(PHABRICATOR_CONFIG_FILE)
+
+ client = Phabricator(
+ username=parser.get(parser_mode, 'username'),
+ token=parser.get(parser_mode, 'token'),
+ host=parser.get(parser_mode, 'host'))
+
+ return client
+
+
+def get_phabricator_project_ids(phab_client, datacenter):
+ """ Return a list of Phabricator's projectPHID
+
+ Find the project IDs of the datacenter's tag and add the one of
+ Operations group.
+
+ Arguments:
+ phab_client -- a Phabricator client instance
+ datacenter -- the name of the datacenter the host is located in
+ """
+
+ project_name = '{}{}'.format(PHABRICATOR_TAG_PREFIX, datacenter)
+ projects = phab_client.project.query(names=[project_name, 'Operations'])
+
+ if len(projects.data.keys()) != 2:
+ logger.error("Unable to find PHID for project '{}', found: {}".format(
+ project_name, projects))
+ raise RuntimeError("Unable to find PHID")
+
+ logger.debug("Found PHIDs '{}' for project '{}' and Operations".format(
+ projects.data.keys(), project_name))
+
+ return projects.data.keys()
+
+
+def open_phabricator_task(
+ phab_client, project_ids, host, raid_status, icinga_url):
+ """ Open a task on Phabricator and return it
+
+ Arguments:
+ phab_client -- a Phabricator client instance
+ project_ids -- the PHIDs to tag the task with
+ host -- the hostname of the affected host
+ raid_status -- the RAID status message to include in the task
+ icinga_url -- the URL of the Icinga alarm that triggered this handler
+ """
+
+ description_prefix = PHABRICATOR_TASK_DESCRIPTION_PREFIX.format(
+ host=host, url=icinga_url)
+ description = '{description_prefix}\n```\n{raid_status}\n```'.format(
+ description_prefix=description_prefix, raid_status=raid_status)
+
+ task = phab_client.maniphest.createtask(
+ title=PHABRICATOR_TASK_TITLE.format(host=host),
+ projectPHIDs=project_ids, description=description)
+
+ logger.debug('Opened Phabricator task: {}'.format(task))
+ return task
+
+
+def acknowledge_nagios_alert(host, service_description, task_uri):
+ """ Acknowledge the Nagios/Icinga alert
+
+ Arguments:
+ host -- the hostname of the affected host
+ service_description -- the Nagios/Icinga service description
+ task_uri -- the URI of the related Phabricator task
+ """
+
+ message = (
+ '[{time}] ACKNOWLEDGE_SVC_PROBLEM;{host};{service};2;1;1;'
+ 'nagiosadmin;{message}\n'
+ ).format(time=int(time.time()), host=host, service=service_description,
+ message=ACK_MESSAGE.format(task_uri))
+
+ with open(COMMAND_FILE, 'w') as f:
+ f.write(message)
+
+ logger.debug('Acknowledged Nagios/Icinga alert: {}'.format(message))
+
+
+def main():
+ """Run the Nagios/Icinga Event Handler for RAID checks"""
+
+ log_formatter = logging.Formatter(
+ fmt='%(asctime)s [%(levelname)s] %(name)s::%(funcName)s: %(message)s',
+ datefmt='%F %T')
+ log_handler = RotatingFileHandler(
+ LOG_PATH, maxBytes=5*(1024**2), backupCount=10)
+ log_handler.setFormatter(log_formatter)
+ logger.addHandler(log_handler)
+ logger.raiseExceptions = False
+ logger.setLevel(logging.INFO)
+
+ args = parse_args()
+ if args.debug:
+ logger.setLevel(logging.DEBUG)
+
+ logger.debug('RAID Handler called with args: {}'.format(args))
+
+ if args.service_state != 'CRITICAL' or args.service_state_type != 'HARD':
+ logger.debug('Nothing to do, exiting')
+ return
+
+ raid_status = get_raid_status(args.host_address, args.raid_type)
+ phab_client = get_phabricator_client()
+ project_ids = get_phabricator_project_ids(phab_client, args.datacenter)
+
+ icinga_url = ICINGA_URL.format(
+ host=args.host_address, service=args.service_description)
+ task = open_phabricator_task(
+ phab_client, project_ids, args.host_address, raid_status, icinga_url)
+
+ acknowledge_nagios_alert(
+ args.host_address, args.service_description, task['uri'])
+
+ logger.info(
+ ("RAID Handler executed for host '{}' and RAID type '{}'. "
+ "Created task ID '{}'").format(
+ args.host_address, args.raid_type, task['id']))
+
+ logger.debug('RAID Handler completed')
+
+
+if __name__ == '__main__':
+ try:
+ main()
+ except Exception:
+ logger.exception("Unable to handle RAID check alert")
diff --git a/modules/icinga/manifests/event_handlers/raid.pp
b/modules/icinga/manifests/event_handlers/raid.pp
new file mode 100644
index 0000000..ea27fb8
--- /dev/null
+++ b/modules/icinga/manifests/event_handlers/raid.pp
@@ -0,0 +1,39 @@
+# = Class: icinga::event_handlers::raid
+#
+# Sets up icinga RAID event handler
+class icinga::event_handlers::raid {
+ include passwords::phabricator
+
+ class { '::phabricator::bot':
+ username => 'ops-monitoring-bot',
+ token => $passwords::phabricator::ops_monitoring_bot_token,
+ owner => 'icinga',
+ group => 'icinga',
+ }
+
+ package { 'python-phabricator':
+ ensure => 'present',
+ }
+
+ file { '/usr/lib/nagios/plugins/eventhandlers/raid_handler':
+ source => 'puppet:///modules/icinga/raid_handler.py',
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ require => [
+ File['/etc/phabricator_ops-monitoring-bot.conf'],
+ File['/var/lib/nagios/rw/nagios.cmd'],
+ File['/usr/lib/nagios/plugins/check_nrpe'],
+ Package['icinga'],
+ ],
+ }
+
+ nagios_common::check_command::config { 'raid_handler':
+ ensure => present,
+ content => template('icinga/event_handlers/raid_handler.cfg.erb'),
+ config_dir => '/etc/icinga',
+ owner => 'icinga',
+ group => 'icinga',
+ require =>
File['/usr/lib/nagios/plugins/eventhandlers/raid_handler'],
+ }
+}
diff --git a/modules/icinga/templates/event_handlers/raid_handler.cfg.erb
b/modules/icinga/templates/event_handlers/raid_handler.cfg.erb
new file mode 100644
index 0000000..a19167b
--- /dev/null
+++ b/modules/icinga/templates/event_handlers/raid_handler.cfg.erb
@@ -0,0 +1,4 @@
+define command{
+ command_name raid_handler
+ command_line $USER1$/eventhandlers/raid_handler -s $SERVICESTATE$ -t
$SERVICESTATETYPE$ -a $SERVICEATTEMPT$ -H $HOSTADDRESS$ -r $ARG1$ -D "$ARG2$"
-c $ARG3$
+ }
diff --git a/modules/monitoring/manifests/service.pp
b/modules/monitoring/manifests/service.pp
index b85e85b..2fbba7e 100644
--- a/modules/monitoring/manifests/service.pp
+++ b/modules/monitoring/manifests/service.pp
@@ -12,6 +12,7 @@
$retry_check_interval = 1,
$contact_group = hiera('contactgroups', 'admins'),
$config_dir = '/etc/nagios',
+ $event_handler = undef,
)
{
# the list of characters is the default for illegal_object_name_chars
@@ -91,6 +92,7 @@
is_volatile => $check_volatile,
check_freshness => $check_fresh,
freshness_threshold => $is_fresh,
+ event_handler => $event_handler,
}
}
# This is a hack. We detect if we are running on the scope of an icinga
diff --git a/modules/nrpe/manifests/monitor_service.pp
b/modules/nrpe/manifests/monitor_service.pp
index 43ee28e..3fef03a 100644
--- a/modules/nrpe/manifests/monitor_service.pp
+++ b/modules/nrpe/manifests/monitor_service.pp
@@ -22,6 +22,9 @@
# $critical
# Defaults to false. It will passed directly to monitoring::service which
# will use nagios_service, so extra care, it is not a boolean, it is a
string
+# $event_handler
+# Default to false. If present execute this registered command on the
+# Nagios server.
# $ensure
# Defaults to present
#
@@ -31,10 +34,10 @@
$retries = 3,
$timeout = 10,
$critical = false,
+ $event_handler = undef,
$normal_check_interval = 1,
$retry_check_interval = 1,
$ensure = 'present') {
-
nrpe::check { "check_${title}":
command => $nrpe_command,
before => Monitoring::Service[$title],
@@ -47,6 +50,7 @@
contact_group => $contact_group,
retries => $retries,
critical => $critical,
+ event_handler => $event_handler,
normal_check_interval => $normal_check_interval,
retry_check_interval => $retry_check_interval,
}
diff --git a/modules/raid/manifests/init.pp b/modules/raid/manifests/init.pp
index a9a4f1f..e49cada 100644
--- a/modules/raid/manifests/init.pp
+++ b/modules/raid/manifests/init.pp
@@ -44,11 +44,13 @@
command => "/usr/bin/sudo ${get_raid_status_megacli} -c",
}
+ $service_description = 'MegaRAID'
nrpe::monitor_service { 'raid_megaraid':
- description => 'MegaRAID',
+ description => $service_description,
nrpe_command => "${check_raid} megacli",
normal_check_interval => $normal_check_interval,
retry_check_interval => $retry_check_interval,
+ event_handler =>
"raid_handler!megacli!${service_description}!${::site}",
}
}
@@ -86,12 +88,14 @@
],
}
+ $service_description = 'HP RAID'
nrpe::monitor_service { 'raid_hpssacli':
- description => 'HP RAID',
+ description => $service_description,
nrpe_command =>
'/usr/local/lib/nagios/plugins/check_hpssacli',
timeout => 50, # can take > 10s on servers with lots
of disks
normal_check_interval => $normal_check_interval,
retry_check_interval => $retry_check_interval,
+ event_handler =>
"raid_handler!hpssacli!${service_description}!${::site}",
}
$get_raid_status_hpssacli =
'/usr/local/lib/nagios/plugins/get-raid-status-hpssacli'
@@ -123,11 +127,13 @@
before => Package['mpt-status'],
}
+ $service_description = 'MPT RAID'
nrpe::monitor_service { 'raid_mpt':
- description => 'MPT RAID',
+ description => $service_description,
nrpe_command => "${check_raid} mpt",
normal_check_interval => $normal_check_interval,
retry_check_interval => $retry_check_interval,
+ event_handler =>
"raid_handler!mpt!${service_description}!${::site}",
}
nrpe::check { 'get_raid_status_mpt':
@@ -138,9 +144,11 @@
if 'md' in $raid {
# if there is an "md" RAID configured, mdadm is already installed
+ $service_description = 'MD RAID'
nrpe::monitor_service { 'raid_md':
- description => 'MD RAID',
- nrpe_command => "${check_raid} md",
+ description => $service_description,
+ nrpe_command => "${check_raid} md",
+ event_handler =>
"raid_handler!md!${service_description}!${::site}",
}
nrpe::check { 'get_raid_status_md':
--
To view, visit https://gerrit.wikimedia.org/r/304026
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3f2b4899089b8eed2f70f1b7b95182b9a3fb67bb
Gerrit-PatchSet: 8
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Volans <[email protected]>
Gerrit-Reviewer: Faidon Liambotis <[email protected]>
Gerrit-Reviewer: Filippo Giunchedi <[email protected]>
Gerrit-Reviewer: Jcrespo <[email protected]>
Gerrit-Reviewer: Volans <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
