Volans has uploaded a new change for review.
https://gerrit.wikimedia.org/r/316827
Change subject: Icinga: improve raid_handler false alarm detection
......................................................................
Icinga: improve raid_handler false alarm detection
Bug: T142085
Change-Id: Ie82d615e2e3f8c9a12eba9b7d43a4cd2901154fb
---
M modules/icinga/files/raid_handler.py
1 file changed, 11 insertions(+), 6 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/27/316827/1
diff --git a/modules/icinga/files/raid_handler.py
b/modules/icinga/files/raid_handler.py
index ed91a7f..b81302b 100644
--- a/modules/icinga/files/raid_handler.py
+++ b/modules/icinga/files/raid_handler.py
@@ -18,6 +18,8 @@
RAID_TYPES = ('megacli', 'hpssacli', 'mpt', 'md')
COMPRESSED_RAID_TYPES = ('megacli', 'hpssacli')
+SKIP_STRINGS = ('timeout', 'timed out', 'connection refused')
+
LOG_PATH = '/var/log/icinga/raid_handler.log'
COMMAND_FILE = '/var/lib/nagios/rw/nagios.cmd'
CHECK_NRPE_PATH = '/usr/lib/nagios/plugins/check_nrpe'
@@ -213,12 +215,15 @@
logger.debug('Nothing to do, exiting')
return
- if 'timeout' in args.message.lower():
- logger.info(
- ("Skipping RAID Handler execution for host '{}' and "
- "RAID type '{}', timeout detected: {}").format(
- args.host_address, args.raid_type, args.message))
- return
+ message_lower = args.message.lower()
+ for skip_string in SKIP_STRINGS:
+ if skip_string in message_lower:
+ logger.info(
+ ("Skipping RAID Handler execution for host '{}' and "
+ "RAID type '{}', skip string '{}' detected in '{}'").format(
+ args.host_address, args.raid_type, skip_string,
+ args.message))
+ return
raid_status = get_raid_status(args.host_address, args.raid_type)
phab_client = get_phabricator_client()
--
To view, visit https://gerrit.wikimedia.org/r/316827
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie82d615e2e3f8c9a12eba9b7d43a4cd2901154fb
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Volans <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
