coren has uploaded a new change for review.
https://gerrit.wikimedia.org/r/227887
Change subject: nagios_common: add new checks for systemd unit health
......................................................................
nagios_common: add new checks for systemd unit health
nrpe_check_systemd_unit_lastrun UNIT WARN CRIT
Checks the time since UNIT last ran, and emits a warning
or critical if it has been over (respectively) WARN or CRIT
hours.
nrpe_check_systemd_unit_result UNIT
Checks the last result of running UNIT and emits a critical
if it was not 'successful'
Change-Id: I9cba4f63e7fc60d2dcd9da3cb09b0a35fbb9f4cd
---
A modules/nagios_common/files/check_commands/check_systemd_unit_lastrun
A modules/nagios_common/files/check_commands/check_systemd_unit_result
M modules/nagios_common/files/checkcommands.cfg
M modules/nagios_common/manifests/commands.pp
4 files changed, 122 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/87/227887/1
diff --git
a/modules/nagios_common/files/check_commands/check_systemd_unit_lastrun
b/modules/nagios_common/files/check_commands/check_systemd_unit_lastrun
new file mode 100755
index 0000000..a922d40
--- /dev/null
+++ b/modules/nagios_common/files/check_commands/check_systemd_unit_lastrun
@@ -0,0 +1,96 @@
+#! /usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015 Marc-André Pelletier <[email protected]>
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+#
+# THIS FILE IS MANAGED BY PUPPET
+#
+# Source: modules/labstore/storage-replicate
+# From: modules/labstore/manifests/fileserve.rpp
+#
+
+"""
+check_systemd_unit_lastrun
+
+usage: check_systemd_unit_lastrun <unit> <warn> <crit>
+
+Checks that the systemd unit has been run recently
+enough. Warns if the last start/stop activity is older
+than warn hours, and criticals if it is older than
+crit hours.
+"""
+
+import argparse
+import time
+import datetime
+import subprocess
+import logging
+import json
+import sys
+
+def main():
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('unit', help='Systemd unit to check')
+ parser.add_argument('warn', help='Number of hours past which a warning
should be emitted')
+ parser.add_argument('crit', help='Number of hours past which a critical
should be emitted')
+ args = parser.parse_args()
+
+ logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+ log = []
+
+ try:
+ raw = subprocess.check_output(
+ ['/bin/journalctl', '--output=json', '--reverse', '--unit',
args.unit],
+ stderr=subprocess.STDOUT).decode()
+ for entry in raw.splitlines():
+ log.append(json.loads(entry))
+ except subprocess.CalledProcessError:
+ print('UNKNOWN - Unable to get systemd journal for unit "%s"' %
args.unit)
+ sys.exit(3)
+ except ValueError:
+ print('UNKNOWN - Unable to parse systemd journal for unit "%s"' %
args.unit)
+ sys.exit(3)
+
+ lastrun = None
+ for entry in log:
+ try:
+ if entry['CODE_FUNCTION'] ==
'unit_status_log_starting_stopping_reloading':
+ lastrun = int(entry['__REALTIME_TIMESTAMP'][:-6]) # because
microseconds
+ except (KeyError):
+ pass
+
+ if not lastrun:
+ print('UNKNOWN - No start/stop information for unit "%s"' % args.unit)
+ sys.exit(3)
+
+ age = datetime.timedelta(seconds=int(time.time()) - lastrun)
+
+ if age > datetime.timedelta(hours=int(args.crit)):
+ print('CRITICAL - Last run %s ago' % age)
+ sys.exit(2)
+
+ if age > datetime.timedelta(hours=int(args.warn)):
+ print('WARNING - Last run %s ago' % age)
+ sys.exit(1)
+
+ print('OK - Last run %s ago' % age)
+ sys.exit(0)
+
+if __name__ == "__main__":
+ main()
+
diff --git
a/modules/nagios_common/files/check_commands/check_systemd_unit_result
b/modules/nagios_common/files/check_commands/check_systemd_unit_result
new file mode 100755
index 0000000..71e6f50
--- /dev/null
+++ b/modules/nagios_common/files/check_commands/check_systemd_unit_result
@@ -0,0 +1,13 @@
+#! /bin/bash
+
+eval $(/bin/systemctl show --property=Result,LoadState "$1")
+if [ "x$LoadState" != "xloaded" ]; then
+ echo "UNKNOWN - Unit not loaded: $LoadState"
+ exit 3
+fi
+if [ "x$Result" != "xsuccess" ]; then
+ echo "CRITICAL - Last result is $Result"
+ exit 2
+fi
+echo "OK - Last result is success"
+exit 0
diff --git a/modules/nagios_common/files/checkcommands.cfg
b/modules/nagios_common/files/checkcommands.cfg
index 866f949..d0e8a81 100644
--- a/modules/nagios_common/files/checkcommands.cfg
+++ b/modules/nagios_common/files/checkcommands.cfg
@@ -458,6 +458,17 @@
command_line $USER1$/check_nrpe -H $HOSTADDRESS$ -c check_ocg_health
}
+# check for systemd unit health (lastrun, and last result)
+define command {
+ command_name nrpe_check_systemd_unit_lastrun
+ command_line $USER1$/check_nrpe -H $HOSTADDRESS$ -c
check_systemd_unit_lastrun $ARG1$ $ARG2$ $ARG3$
+}
+
+define command {
+ command_name nrpe_check_systemd_unit_result
+ command_line $USER1$/check_nrpe -H $HOSTADDRESS$ -c
check_systemd_unit_result $ARG1$
+}
+
# custom command to check phabricator behind misc-web
# using the -S should also give us cert expiry monitoring
# we are also checking for string 'Wikimedia' to be in there
diff --git a/modules/nagios_common/manifests/commands.pp
b/modules/nagios_common/manifests/commands.pp
index 639f2f7..fdeb9c0 100644
--- a/modules/nagios_common/manifests/commands.pp
+++ b/modules/nagios_common/manifests/commands.pp
@@ -46,6 +46,8 @@
'check_to_check_nagios_paging',
'check_ifstatus_nomon',
'check_bgp',
+ 'check_systemd_unit_lastrun',
+ 'check_systemd_unit_result',
] :
require => File["${config_dir}/commands"],
config_dir => $config_dir,
--
To view, visit https://gerrit.wikimedia.org/r/227887
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9cba4f63e7fc60d2dcd9da3cb09b0a35fbb9f4cd
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: coren <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
