Yuvipanda has submitted this change and it was merged.
Change subject: Extend Exim diamond collector for Tool Labs
......................................................................
Extend Exim diamond collector for Tool Labs
- forked and extended existing exim collector
- extended queue information
- added paniclog information
- moved sudo configuration to collector manifest
- updated tool labs mail relay for new collector
Bug: T96898
Change-Id: I1d0517e41d61201e7f6c9b85c116f952658f73a3
---
A modules/diamond/files/collector/extendedexim.py
A modules/diamond/manifests/collector/extendedexim.pp
M modules/toollabs/manifests/mailrelay.pp
3 files changed, 211 insertions(+), 11 deletions(-)
Approvals:
Yuvipanda: Verified; Looks good to me, approved
diff --git a/modules/diamond/files/collector/extendedexim.py
b/modules/diamond/files/collector/extendedexim.py
new file mode 100644
index 0000000..87f63e8
--- /dev/null
+++ b/modules/diamond/files/collector/extendedexim.py
@@ -0,0 +1,173 @@
+# coding=utf-8
+
+"""
+Exim collector. Collects queue properties and paniclog size.
+
+Queue properties:
+ - queue.oldest: age of oldest e-mail in queue (seconds)
+ - queue.youngest: age of youngest e-mail in queue (seconds)
+ - queue.size: total size of the queue (bytes)
+ - queue.length: total number of e-mails in the queue
+ - queue.num_frozen: number of frozen e-mails in the queue
+
+Paniclog properties:
+ - paniclog.length: number of lines in /var/log/exim4/paniclog
+
+Queue length is retrieved from exim; the paniclog is read directly.
+Both support the use of sudo, so diamond can run as unprivileged user.
+
+#### History
+Based on EximCollector bundled with Diamond (collectors/exim/exim.py)
+Extended by Merlijn van Deen <valhall...@arctus.nl>
+
+#### Dependencies
+ * /usr/sbin/exim
+
+"""
+
+import diamond.collector
+import subprocess
+import os
+from datetime import timedelta
+from collections import namedtuple
+from diamond.collector import str_to_bool
+
+
+class EximCollectorException(Exception):
+ pass
+
+
+EximQueueLine = namedtuple('EximQueueLine',
+ ['age', 'size', 'mail_id', 'frozen'])
+
+
+class ExtendedEximCollector(diamond.collector.Collector):
+ def get_default_config_help(self):
+ config_help = super(ExtendedEximCollector,
self).get_default_config_help() # noqa
+ config_help.update({
+ 'bin': 'The path to the exim binary',
+ 'use_sudo': 'Use sudo?',
+ 'sudo_cmd': 'Path to sudo',
+ 'sudo_user': 'User to sudo as',
+ })
+ return config_help
+
+ def get_default_config(self):
+ """
+ Returns the default collector settings
+ """
+ config = super(ExtendedEximCollector, self).get_default_config()
+ config.update({
+ 'path': 'exim',
+ 'bin': '/usr/sbin/exim',
+ 'use_sudo': False,
+ 'sudo_cmd': '/usr/bin/sudo',
+ 'sudo_user': 'root',
+ })
+ return config
+
+ def _get_file(self, file):
+ if not str_to_bool(self.config['use_sudo']):
+ return open(file).read()
+ else:
+ command = [self.config['sudo_cmd'], "-u", self.config['sudo_user'],
+ "cat", file]
+ self.log.debug('Running %s' % (' '.join(command)))
+ try:
+ return subprocess.check_output(
+ command,
+ stderr=subprocess.STDOUT
+ )
+ except subprocess.CalledProcessError as e:
+ raise IOError(e)
+
+ def _get_queue(self):
+ if not os.access(self.config['bin'], os.X_OK):
+ raise EximCollectorException('exim not found')
+
+ command = [self.config['bin'], '-bpr']
+
+ if str_to_bool(self.config['use_sudo']):
+ command = [
+ self.config['sudo_cmd'],
+ '-u',
+ self.config['sudo_user']
+ ] + command
+ self.log.debug('Running %s' % (' '.join(command)))
+ queue = subprocess.check_output(command)
+
+ # remove empty lines
+ queue = [l.strip() for l in queue.split("\n")]
+ queue = [l for l in queue if l]
+
+ # remove indented lines
+ queue = [l for l in queue if l[0] != " "]
+
+ return queue
+
+ def _parse_age(self, age):
+ postfix = age[-1]
+ if postfix == "m":
+ return timedelta(minutes=int(age[:-1]))
+ elif postfix == "h":
+ return timedelta(hours=int(age[:-1]))
+ elif postfix == "d":
+ return timedelta(days=int(age[:-1]))
+ elif postfix == "w":
+ return timedelta(weeks=int(age[:-1]))
+
+ def _parse_size(self, size):
+ postfix = size[-1]
+ if postfix == "K":
+ return float(size[:-1]) * 1024
+ elif postfix == "M":
+ return float(size[:-1]) * 1024
+ else:
+ return float(size)
+
+ def _parse_line(self, line):
+ parts = [x for x in line.split(" ") if x]
+ age = self._parse_age(parts[0])
+ size = self._parse_size(parts[1])
+ mail_id = parts[2]
+ frozen = "*** frozen ***" in line
+
+ return EximQueueLine(age=age, size=size,
+ mail_id=mail_id, frozen=frozen)
+
+ def collect_queue(self):
+ try:
+ queue = self._get_queue()
+ except EximCollectorException:
+ return
+
+ parsed_queue = [self._parse_line(line) for line in queue]
+
+ if parsed_queue:
+ oldest = max(l.age for l in parsed_queue)
+ youngest = min(l.age for l in parsed_queue)
+ size = sum(l.size for l in parsed_queue)
+ length = len(parsed_queue)
+ num_frozen = sum(l.frozen for l in parsed_queue)
+ else:
+ oldest = youngest = size = length = num_frozen = 0
+
+ self.publish('queue.oldest', oldest)
+ self.publish('queue.youngest', youngest)
+ self.publish('queue.size', size)
+ self.publish('queue.length', length)
+ self.publish('queue.num_frozen', num_frozen)
+
+ def collect_paniclog(self):
+ try:
+ contents = self._get_file('/var/log/exim4/paniclog')
+ except IOError:
+ self.log.warning('Cannot open Exim paniclog!')
+ return
+
+ num_lines = contents.count("\n")
+ self.publish('paniclog.length', num_lines)
+
+ def collect(self):
+ self.collect_queue()
+ self.collect_paniclog()
diff --git a/modules/diamond/manifests/collector/extendedexim.pp
b/modules/diamond/manifests/collector/extendedexim.pp
new file mode 100644
index 0000000..aff8d58
--- /dev/null
+++ b/modules/diamond/manifests/collector/extendedexim.pp
@@ -0,0 +1,37 @@
+# == Define: diamond::collector::extendedexim
+#
+# Exim collector. Collects queue properties and paniclog size.
+#
+# Queue properties:
+# - queue.oldest: age of oldest e-mail in queue (seconds)
+# - queue.youngest: age of youngest e-mail in queue (seconds)
+# - queue.size: total size of the queue (bytes)
+# - queue.length: total number of e-mails in the queue
+# - queue.num_frozen: number of frozen e-mails in the queue
+#
+# Paniclog properties:
+# - paniclog.length: number of lines in /var/log/exim4/paniclog
+
+include stdlib
+
+define diamond::collector::extendedexim(
+ $settings = {},
+ $ensure = present,
+) {
+ $default_settings = {'use_sudo' => 'true'}
+ $merged_settings = merge($default_settings, $settings)
+
+ diamond::collector { 'ExtendedExim':
+ settings => $merged_settings,
+ source => 'puppet:///modules/diamond/collector/extendedexim.py',
+ ensure => $ensure,
+ }
+
+ if str2bool($merged_settings[use_sudo]) {
+ sudo::user { 'diamond_sudo_for_exim':
+ user => 'diamond',
+ privileges => ['ALL=(root) NOPASSWD: /usr/sbin/exim, /bin/cat
/var/log/exim4/paniclog'],
+ ensure => $ensure,
+ }
+ }
+}
diff --git a/modules/toollabs/manifests/mailrelay.pp
b/modules/toollabs/manifests/mailrelay.pp
index e68f01a..46e1b9c 100644
--- a/modules/toollabs/manifests/mailrelay.pp
+++ b/modules/toollabs/manifests/mailrelay.pp
@@ -62,15 +62,5 @@
notify => Service['exim4'],
}
- # Diamond user needs sudo to access exim
- sudo::user { 'diamond_sudo_for_exim':
- user => 'diamond',
- privileges => ['ALL=(root) NOPASSWD: /usr/sbin/exim']
- }
-
- diamond::collector { 'Exim':
- settings => {
- use_sudo => 'true', # used in a template, not a puppet bool
- }
- }
+ diamond::collector::extendedexim { 'extended_exim_collector': }
}
--
To view, visit https://gerrit.wikimedia.org/r/206118
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I1d0517e41d61201e7f6c9b85c116f952658f73a3
Gerrit-PatchSet: 10
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Merlijn van Deen <valhall...@arctus.nl>
Gerrit-Reviewer: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Merlijn van Deen <valhall...@arctus.nl>
Gerrit-Reviewer: Yuvipanda <yuvipa...@gmail.com>
Gerrit-Reviewer: coren <mpellet...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
