Ori.livneh has submitted this change and it was merged.
Change subject: Forward UDP logs to logstash1001.eqiad.wmnet
......................................................................
Forward UDP logs to logstash1001.eqiad.wmnet
Route wfDebugLog and related log output collected by udp2log hosts using
the role::logging::mediawiki puppet class to the logstash cluster.
Also deprecates wfdebug-ganglia.py.
Change-Id: Id8c40e1c052977d942cbc7447d1e3da59a77cec0
---
D files/udp2log/wfdebug-ganglia.py
M manifests/role/logging.pp
M templates/udp2log/filters.mw.erb
D templates/udp2log/wfdebug-ganglia.conf.erb
4 files changed, 10 insertions(+), 114 deletions(-)
Approvals:
Ori.livneh: Looks good to me, approved
jenkins-bot: Verified
diff --git a/files/udp2log/wfdebug-ganglia.py b/files/udp2log/wfdebug-ganglia.py
deleted file mode 100644
index 98e1d52..0000000
--- a/files/udp2log/wfdebug-ganglia.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Listen for wfDebug()s on UDP; forward to Ganglia using Gmetric.
-Usage: wfdebug-ganglia.py UDP_LISTEN_PORT
-
-"""
-import sys
-reload(sys)
-sys.setdefaultencoding('utf-8')
-
-import threading
-import socket
-import subprocess
-import time
-
-
-REPORTING_INTERVAL = 60 # In seconds.
-UDP_BUFSIZE = 65536 # Udp2LogConfig::BLOCK_SIZE
-METRIC_FORMAT = 'mediaWiki.wfDebug.%s' # Format string for metric name
-
-try:
- port = int(sys.argv[1])
-except (IndexError, ValueError):
- print __doc__.strip()
- sys.exit(1)
-
-sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-sock.bind(('0.0.0.0', port))
-
-counts = {}
-lock = threading.Lock()
-defaults = {
- 'group': 'wfDebug',
- 'slope': 'positive',
- 'spoof': 'mediaWiki:mediaWiki',
- 'tmax': REPORTING_INTERVAL,
- 'type': 'uint32',
- 'units': 'messages',
-}
-
-
-def listen(port):
- while 1:
- dgram = sock.recv(UDP_BUFSIZE)
- seq_id, log_name, rest = dgram.split(' ', 2)
- with lock:
- counts[log_name] = counts.get(log_name, 0) + 1
-
-
-def send_with_gmetric(metric):
- command = ['gmetric']
- args = sorted('--%s=%s' % (k, v) for k, v in metric.items())
- command.extend(args)
- subprocess.call(command)
-
-
-def dispatch_stats():
- """Send metrics to Ganglia by shelling out to gmetric."""
- with lock:
- stats = counts.copy()
- for log_name, count in stats.items():
- metric = dict(defaults, name=METRIC_FORMAT % log_name, value=count)
- send_with_gmetric(metric)
-
-
-# Start listener
-listener = threading.Thread(target=listen, args=(port,))
-listener.daemon = True
-listener.start()
-
-# Report stats
-while 1:
- start = time.time()
- dispatch_stats()
- elapsed = time.time() - start
- time.sleep(REPORTING_INTERVAL - elapsed)
diff --git a/manifests/role/logging.pp b/manifests/role/logging.pp
index 15f62d3..52873f7 100644
--- a/manifests/role/logging.pp
+++ b/manifests/role/logging.pp
@@ -50,12 +50,14 @@
labs => 'deployment-fluoride.pmtpa.wmflabs',
}
- $ganglia_reporter_host = $::realm ? {
- production => 'localhost',
+ $logstash_host = $::realm ? {
+ # TODO: Find a way to use multicast that doesn't cause duplicate
+ # messages to be stored in logstash. This is a SPOF.
+ production => 'logstash1001.eqiad.wmnet',
labs => 'logstash.pmtpa.wmflabs',
}
- $ganglia_reporter_port = 8324
+ $logstash_port = 8324
misc::udp2log::instance { "mw":
log_directory => $log_directory,
@@ -66,9 +68,9 @@
error_processor_host => $error_processor_host,
error_processor_port => 8423,
- # forwarding to wfdebug-ganglia.py (see below)
- ganglia_reporter_host => $ganglia_reporter_host,
- ganglia_reporter_port => $ganglia_reporter_port,
+ # forwarding to logstash
+ logstash_host => $logstash_host,
+ logstash_port => $logstash_port,
},
}
@@ -84,26 +86,6 @@
group => 'root',
mode => '0555',
source => "puppet:///files/misc/scripts/mw-log-cleanup",
- }
-
- file { '/usr/local/bin/wfdebug-ganglia.py':
- owner => 'root',
- group => 'root',
- mode => '0555',
- source => 'puppet:///files/udp2log/wfdebug-ganglia.py',
- }
-
- file { '/etc/init/wfdebug-ganglia.conf':
- owner => 'root',
- group => 'root',
- mode => '0444',
- content => template('udp2log/wfdebug-ganglia.conf.erb'),
- }
-
- service { 'wfdebug-ganglia':
- ensure => running,
- provider => 'upstart',
- subscribe => File['/usr/local/bin/wfdebug-ganglia.py',
'/etc/init/wfdebug-ganglia.conf'],
}
}
diff --git a/templates/udp2log/filters.mw.erb b/templates/udp2log/filters.mw.erb
index 3c218ce..b6c943d 100644
--- a/templates/udp2log/filters.mw.erb
+++ b/templates/udp2log/filters.mw.erb
@@ -6,5 +6,5 @@
# exception-json
pipe 1 egrep '^(fatal|exception) ' | /usr/bin/log2udp -h <%=
@template_variables['error_processor_host'] %> -p <%=
@template_variables['error_processor_port'] %>
-# Forward all messages to Ganglia reporter
-pipe 1 /usr/bin/log2udp -h <%= @template_variables['ganglia_reporter_host'] %>
-p <%= @template_variables['ganglia_reporter_port'] %>
+# Forward all messages to Logstash
+pipe 1 /usr/bin/log2udp -h <%= @template_variables['logstash_host'] %> -p <%=
@template_variables['logstash_port'] %>
diff --git a/templates/udp2log/wfdebug-ganglia.conf.erb
b/templates/udp2log/wfdebug-ganglia.conf.erb
deleted file mode 100755
index 78f4a58..0000000
--- a/templates/udp2log/wfdebug-ganglia.conf.erb
+++ /dev/null
@@ -1,9 +0,0 @@
-# vim: set ft=upstart:
-
-# Upstart job configuration for wfdebug-ganglia.py
-# This file is managed by Puppet
-description "Report MediaWiki log counts to Ganglia"
-start on (local-filesystems and net-device-up IFACE!=lo)
-setuid nobody
-exec python /usr/local/bin/wfdebug-ganglia.py <%= @ganglia_reporter_port %>
-respawn
--
To view, visit https://gerrit.wikimedia.org/r/108153
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id8c40e1c052977d942cbc7447d1e3da59a77cec0
Gerrit-PatchSet: 3
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <bda...@wikimedia.org>
Gerrit-Reviewer: Ori.livneh <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
