Volans has uploaded a new change for review.
https://gerrit.wikimedia.org/r/318128
Change subject: Icinga: raid_handler improve failure detection
......................................................................
Icinga: raid_handler improve failure detection
Bug: T142085
Change-Id: Ie89d0ea765d271681a08d9f39b0aa93bc482d27a
---
M modules/icinga/files/raid_handler.py
1 file changed, 8 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/28/318128/1
diff --git a/modules/icinga/files/raid_handler.py
b/modules/icinga/files/raid_handler.py
index b81302b..9d2a3f2 100644
--- a/modules/icinga/files/raid_handler.py
+++ b/modules/icinga/files/raid_handler.py
@@ -18,7 +18,7 @@
RAID_TYPES = ('megacli', 'hpssacli', 'mpt', 'md')
COMPRESSED_RAID_TYPES = ('megacli', 'hpssacli')
-SKIP_STRINGS = ('timeout', 'timed out', 'connection refused')
+SKIP_STRINGS = ('timeout', 'timed out', 'connection refused', 'out of bounds')
LOG_PATH = '/var/log/icinga/raid_handler.log'
COMMAND_FILE = '/var/lib/nagios/rw/nagios.cmd'
@@ -91,9 +91,14 @@
'-c', NRPE_REMOTE_COMMAND.format(raid_type)]
proc = subprocess.Popen(nrpe_command, stdout=subprocess.PIPE)
stdout, stderr = proc.communicate()
+ if proc.returncode != 0 or len(stdout) == 0:
+ raise RuntimeError(('RETCODE: {retcode}\nSTDOUT:\n{stdout}\n'
+ 'STDERR:\n{stderr}').format(
+ retcode=proc.returncode, stdout=stdout, stderr=stderr))
except Exception as e:
- logger.error("Unable to execute '{}': {}".format(nrpe_command, e))
- raise
+ status = "Failed to execute '{}': {}".format(nrpe_command, e)
+ logger.error(status)
+ return status
if raid_type in COMPRESSED_RAID_TYPES:
# NRPE doesn't handle NULL bytes, decoding them.
--
To view, visit https://gerrit.wikimedia.org/r/318128
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie89d0ea765d271681a08d9f39b0aa93bc482d27a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Volans <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
