Ori.livneh has submitted this change and it was merged.
Change subject: Report MediaWiki wfDebug() log counts to Ganglia via Gmetric
......................................................................
Report MediaWiki wfDebug() log counts to Ganglia via Gmetric
- udp2log on fluorine: forward everything to localhost:8324
- configure 'wfdebug-ganglia' upstart service
- service runs wfdebug-ganglia.py, which keeps count and flushes to ganglia
using gmetric.
- gets rid of centralauth_log_processor
Change-Id: Ibf3bb07c968673b9d5b291da34a18fd19754cd97
---
A files/udp2log/wfdebug-ganglia.py
M manifests/role/logging.pp
M templates/udp2log/filters.mw.erb
A templates/udp2log/wfdebug-ganglia.conf.erb
4 files changed, 111 insertions(+), 4 deletions(-)
Approvals:
Ori.livneh: Looks good to me, approved
jenkins-bot: Verified
diff --git a/files/udp2log/wfdebug-ganglia.py b/files/udp2log/wfdebug-ganglia.py
new file mode 100644
index 0000000..4cee2e5
--- /dev/null
+++ b/files/udp2log/wfdebug-ganglia.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+Listen for wfDebug()s on UDP; forward to Ganglia using Gmetric.
+Usage: wfdebug-ganglia.py UDP_LISTEN_PORT
+
+"""
+import sys
+reload(sys)
+sys.setdefaultencoding('utf-8')
+
+import threading
+import socket
+import subprocess
+import time
+
+
+REPORTING_INTERVAL = 5 # In seconds.
+UDP_BUFSIZE = 65536 # Udp2LogConfig::BLOCK_SIZE
+METRIC_FORMAT = 'mediaWiki.wfDebug.%s' # Format string for metric name
+
+try:
+ port = int(sys.argv[1])
+except (IndexError, ValueError):
+ print __doc__.strip()
+ sys.exit(1)
+
+sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+sock.bind(('0.0.0.0', port))
+
+counts = {}
+lock = threading.Lock()
+defaults = {
+ 'group': 'wfDebug',
+ 'slope': 'positive',
+ 'spoof': 'mediaWiki:mediaWiki',
+ 'tmax': REPORTING_INTERVAL,
+ 'type': 'uint32',
+ 'units': 'messages',
+}
+
+
+def listen(port):
+ while 1:
+ dgram = sock.recv(UDP_BUFSIZE)
+ seq_id, log_name, rest = dgram.split(' ', 2)
+ with lock:
+ counts[log_name] = counts.get(log_name, 0) + 1
+
+
+def send_with_gmetric(metric):
+ command = ['gmetric']
+ args = sorted('--%s=%s' % (k, v) for k, v in metric.items())
+ command.extend(args)
+ subprocess.call(command)
+
+
+def dispatch_stats():
+ """Send metrics to Ganglia by shelling out to gmetric."""
+ with lock:
+ stats = counts.copy()
+ for log_name, count in stats.items():
+ metric = dict(defaults, name=METRIC_FORMAT % log_name, value=count)
+ send_with_gmetric(metric)
+
+
+# Start listener
+listener = threading.Thread(target=listen, args=(port,))
+listener.daemon = True
+listener.start()
+
+# Report stats
+while 1:
+ start = time.time()
+ dispatch_stats()
+ elapsed = time.time() - start
+ time.sleep(REPORTING_INTERVAL - elapsed)
diff --git a/manifests/role/logging.pp b/manifests/role/logging.pp
index f96eeac..c4213b4 100644
--- a/manifests/role/logging.pp
+++ b/manifests/role/logging.pp
@@ -50,6 +50,9 @@
labs => 'deployment-fluoride.pmtpa.wmflabs',
}
+ $ganglia_reporter_host = 'localhost'
+ $ganglia_reporter_port = 8324
+
misc::udp2log::instance { "mw":
log_directory => $log_directory,
monitor_log_age => false,
@@ -58,9 +61,10 @@
template_variables => {
error_processor_host => $error_processor_host,
error_processor_port => 8423,
- # Quick hack to get authentication metrics in Ganglia (ori-l,
20-Aug-2013):
- centralauth_log_processor_host => $error_processor_host,
- centralauth_log_processor_port => 8324,
+
+ # forwarding to wfdebug-ganglia.py (see below)
+ ganglia_reporter_host => $ganglia_reporter_host,
+ ganglia_reporter_port => $ganglia_reporter_port,
},
}
@@ -75,6 +79,21 @@
source => "puppet:///files/misc/scripts/mw-log-cleanup",
mode => '0555'
}
+
+ file { '/usr/local/bin/wfdebug-ganglia.py':
+ source => 'puppet:///files/udp2log/wfdebug-ganglia.py',
+ mode => '0555',
+ }
+
+ file { '/etc/init/wfdebug-ganglia.conf':
+ content => template('udp2log/wfdebug-ganglia.conf.erb'),
+ }
+
+ service { 'wfdebug-ganglia':
+ ensure => running,
+ provider => 'upstart',
+ subscribe => File['/usr/local/bin/wfdebug-ganglia.py',
'/etc/init/wfdebug-ganglia.conf'],
+ }
}
class role::beta::logging::mediawiki {
diff --git a/templates/udp2log/filters.mw.erb b/templates/udp2log/filters.mw.erb
index 6fdf01b..16c80d5 100644
--- a/templates/udp2log/filters.mw.erb
+++ b/templates/udp2log/filters.mw.erb
@@ -2,4 +2,6 @@
# Relay MediaWiki exceptions and fatals to vanadium for generating reports.
pipe 1 egrep '^(fatal|exception)' | /usr/bin/log2udp -h <%=
@template_variables['error_processor_host'] %> -p <%=
@template_variables['error_processor_port'] %>
-pipe 1 egrep '^centralauth' | /usr/bin/log2udp -h <%=
@template_variables['centralauth_log_processor_host'] %> -p <%=
@template_variables['centralauth_log_processor_port'] %>
+
+# Forward all messages to Ganglia reporter
+pipe 1 /usr/bin/log2udp -h <%= @template_variables['ganglia_reporter_host'] %>
-p <%= @template_variables['ganglia_reporter_port'] %>
diff --git a/templates/udp2log/wfdebug-ganglia.conf.erb
b/templates/udp2log/wfdebug-ganglia.conf.erb
new file mode 100755
index 0000000..78f4a58
--- /dev/null
+++ b/templates/udp2log/wfdebug-ganglia.conf.erb
@@ -0,0 +1,9 @@
+# vim: set ft=upstart:
+
+# Upstart job configuration for wfdebug-ganglia.py
+# This file is managed by Puppet
+description "Report MediaWiki log counts to Ganglia"
+start on (local-filesystems and net-device-up IFACE!=lo)
+setuid nobody
+exec python /usr/local/bin/wfdebug-ganglia.py <%= @ganglia_reporter_port %>
+respawn
--
To view, visit https://gerrit.wikimedia.org/r/91804
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ibf3bb07c968673b9d5b291da34a18fd19754cd97
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ori.livneh <[email protected]>
Gerrit-Reviewer: Ori.livneh <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
