Ottomata has submitted this change and it was merged.
Change subject: Added per dc / server role breakdown of udp2log packetloss
monitoring.
......................................................................
Added per dc / server role breakdown of udp2log packetloss monitoring.
In addition of having a single packetloss metric per udp2log host,
this patchset introduces additional functionality to map a hostname
to it's role and dc and to calculate packetloss per dc / role.
The rolematcher.py determines the role and dc of a hostname; this information is
obtained every 24 hours from noc.wikimedia.org/pybal. Roles include apache,
api, bits, https, mobile, rendering, text, upload.
The interfaces of PacketLossLogtailer.py have not been changed, this changeset
might require some changes in Ganglia as well. A local test of ganglia-logtailer
did work.
Code is pep8 compliant with the exception of some line lengths > 79.
Change-Id: I084c210d996068f213977f2fb942d4a80021be8f
---
M files/misc/PacketLossLogtailer.py
A files/misc/rolematcher.py
M manifests/misc/udp2log.pp
3 files changed, 326 insertions(+), 38 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/files/misc/PacketLossLogtailer.py
b/files/misc/PacketLossLogtailer.py
index cdc5078..4c2c2f9 100644
--- a/files/misc/PacketLossLogtailer.py
+++ b/files/misc/PacketLossLogtailer.py
@@ -1,14 +1,14 @@
# -*- coding: utf-8 -*-
-###
-### This plugin for logtailer will crunch WMF packet loss logs and return:
-### * average percent loss
-### * ninetieth percentile loss
-### It will throw out
-### * packet loss numbers greater than 98%
-### * large margins of error
-### * margins of error are greater than percent packet loss
-### Note that this plugin depends on a certain apache log format, documented
in
-## __init__.
+#
+# This plugin for logtailer will crunch WMF packet loss logs and return:
+# * average percent loss per server role
+# * ninetieth percentile loss per server role
+# It will throw out
+# * packet loss numbers greater than 98%
+# * large margins of error
+# * margins of error are greater than percent packet loss
+# Note that this plugin depends on a certain apache log format, documented in
+# __init__.
import time
import threading
@@ -17,22 +17,30 @@
# local dependencies
from ganglia_logtailer_helper import GangliaMetricObject
from ganglia_logtailer_helper import LogtailerParsingException,
LogtailerStateException
+import rolematcher
+
class PacketLossLogtailer(object):
# only used in daemon mode
period = 30
+
def __init__(self):
'''This function should initialize any data structures or variables
needed for the internal state of the line parser.'''
self.reset_state()
self.lock = threading.RLock()
+ # a list of rolematchers which are simple object to determine the role
of a particular server
+ # this list is obtained from crawling noc.wikimedia.org/pybal and
parse the available configurations
+ self.day_in_seconds = 24 * 60 * 60
+ self.last_update_time = time.time()
+ self.matchers = rolematcher.init()
# this is what will match the packet loss lines
# packet loss format :
# %[%Y-%m-%dT%H:%M:%S]t %server lost: (%percentloss +/- %margin)
# [2011-10-26T21:20:25] sq86.wikimedia.org lost: (3.61446 +/-
19.67462)%
# match keys: date, server, percentloss, margin
- self.reg = re.compile ('^\[(?P<date>[^]]+)\] (?P<server>[^ ]+) lost:
\((?P<percentloss>[^ ]+) \+\/- (?P<margin>[^)]+)\)%')
+ self.reg = re.compile('^\[(?P<date>[^]]+)\] (?P<server>[^ ]+) lost:
\((?P<percentloss>[^ ]+) \+\/- (?P<margin>[^)]+)\)%')
# assume we're in daemon mode unless set_check_duration gets called
self.dur_override = False
@@ -47,15 +55,24 @@
regMatch = self.reg.match(line)
if regMatch:
linebits = regMatch.groupdict()
- self.num_hits+=1
+ self.num_hits += 1
# capture data
percentloss = float(linebits['percentloss'])
margin = float(linebits['margin'])
+ role = self.determine_role(linebits['server'])
# store for 90th % and average calculations
# on ssl servers, sequence numbers are out of order.
# http://rt.wikimedia.org/Ticket/Display.html?id=1616
- if( ( margin <= 20 ) and ( percentloss <= 98 ) ):
- self.percentloss_list.append(percentloss)
+ if((margin <= 20) and (percentloss <= 98)):
+ self.percentloss_dict.setdefault(role, [])
+ self.percentloss_dict[role].append(percentloss)
+ # keep behavior of PacketLossLogtailer consistent with
+ # previous version and have one overall metric that
+ # can be used for sending notifications, for now keep
+ # filtering out HTTPS/IP6 traffic as the packetloss
numbers
+ # are inaccurate.
+ if role != 'https':
+ self.percentloss_dict['all_roles'].append(percentloss)
else:
raise LogtailerParsingException, "regmatch failed to match"
@@ -63,6 +80,26 @@
self.lock.release()
raise LogtailerParsingException, "regmatch or contents failed with
%s" % e
self.lock.release()
+
+ def update_rolematchers(self):
+ if (time.time() - self.last_update_time) > self.day_in_seconds:
+ self.matchers = rolematcher.init()
+ self.last_update_time = time.time()
+ if self.matchers == []:
+ # downloading the roles did not go well apparently
+ # fall back to hardcoded roles.
+ rolematcher.manual_init()
+
+ def determine_role(self, hostname):
+ if hostname == 'total':
+ return 'total'
+ role = 'misc' # default group for when we were not able to determine
the role
+ for matcher in self.matchers:
+ if matcher == hostname:
+ role = matcher.get_role()
+ break
+ return role
+
# example function for deep copy
# takes no arguments
# returns one object
@@ -72,9 +109,9 @@
currently being modified so that the other thread can deal with it
without fear of it changing out from under it. The format of this
object is internal to the plugin.'''
- myret = dict(percentloss_list=self.percentloss_list
- )
+ myret = dict(percentloss_dict=self.percentloss_dict)
return myret
+
# example function for reset_state
# takes no arguments
# returns nothing
@@ -87,8 +124,10 @@
reset_state should store now() each time it's called, and get_state
will use the time since that now() to do its calculations'''
self.num_hits = 0
- self.percentloss_list = list()
+ self.percentloss_dict = dict()
self.last_reset_time = time.time()
+ self.update_rolematchers()
+
# example for keeping track of runtimes
# takes no arguments
# returns float number of seconds for this run
@@ -98,11 +137,12 @@
it.'''
self.duration = dur
self.dur_override = True
+
def get_check_duration(self):
'''This function should return the time since the last check. If
called
from cron mode, this must be set using set_check_duration(). If in
daemon mode, it should be calculated internally.'''
- if( self.dur_override ):
+ if(self.dur_override):
duration = self.duration
else:
cur_time = time.time()
@@ -111,8 +151,10 @@
acceptable_duration_min = self.period - (self.period / 10.0)
acceptable_duration_max = self.period + (self.period / 10.0)
if (duration < acceptable_duration_min or duration >
acceptable_duration_max):
- raise LogtailerStateException, "time calculation problem -
duration (%s) > 10%% away from period (%s)" % (duration, self.period)
+ raise LogtailerStateException, "time calculation problem -
duration (%s) > 10%% away from period (%s)" % (
+ duration, self.period)
return duration
+
# example function for get_state
# takes no arguments
# returns a dictionary of (metric => metric_object) pairs
@@ -121,7 +163,7 @@
current time if necessary, call reset_state, then do its
calculations. It should return a list of metric objects.'''
# if number of log lines is 0, then return no data
- if ( self.num_hits == 0):
+ if (self.num_hits == 0):
return list()
self.lock.acquire()
@@ -137,20 +179,29 @@
raise e
# calculate 90th % and average request times
- percentloss_list = mydata['percentloss_list']
- percentloss_list.sort()
- num_entries = len(percentloss_list)
- if (num_entries != 0 ):
- packetloss_90th = percentloss_list[int(num_entries * 0.9)]
- packetloss_ave = sum(percentloss_list) / len(percentloss_list)
- else:
- # in this event, all data was thrown out in parse_line
- packetloss_90th = 99
- packetloss_ave = 99
- # package up the data you want to submit
- # setting tmax to 960 seconds as data may take as long as 15 minutes
to be processed
- packetloss_ave_metric = GangliaMetricObject( 'packet_loss_average',
packetloss_ave, units='%', tmax=960 )
- packetloss_90th_metric = GangliaMetricObject( 'packet_loss_90th',
packetloss_90th, units='%', tmax=960 )
+ percentloss_dict = mydata['percentloss_dict']
+ metrics = list()
+ for role, percentloss_list in percentloss_dict.iteritems():
+
+ percentloss_list.sort()
+ num_entries = len(percentloss_list)
+ if (num_entries != 0):
+ packetloss_90th = percentloss_list[int(num_entries * 0.9)]
+ packetloss_ave = sum(percentloss_list) / len(percentloss_list)
+ else:
+ # in this event, all data was thrown out in parse_line
+ packetloss_90th = 99
+ packetloss_ave = 99
+ # package up the data you want to submit
+ # setting tmax to 960 seconds as data may take as long as 15
minutes to be processed
+ if (role == 'all_roles'):
+ packetloss_ave_metric =
GangliaMetricObject('packet_loss_average', packetloss_ave, units='%', tmax=960)
+ packetloss_90th_metric =
GangliaMetricObject('packet_loss_90th', packetloss_90th, units='%', tmax=960)
+ else:
+ packetloss_ave_metric =
GangliaMetricObject('packet_loss_average:%s' % (role), packetloss_ave,
units='%', tmax=960)
+ packetloss_90th_metric =
GangliaMetricObject('packet_loss_90th:%s' % (role), packetloss_90th, units='%',
tmax=960)
+ metrics.append(packetloss_ave_metric)
+ metrics.append(packetloss_90th_metric)
# return a list of metric objects
- return [ packetloss_ave_metric, packetloss_90th_metric, ]
\ No newline at end of file
+ return metrics
diff --git a/files/misc/rolematcher.py b/files/misc/rolematcher.py
new file mode 100644
index 0000000..1a4dd6d
--- /dev/null
+++ b/files/misc/rolematcher.py
@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env python
+
+'''
+This script parses the packet-loss.log file and for each server entry
+it determines what it's role is. Server role data has been obtained
+from noc.wikimeda.org/pybal and can either be downloaded or the
+hard-coded version can be used.
+
+Purpose of this script is to use it in conjunction with
+PacketLossLogTailer.py and send packetloss metrics per dc/role to
+Ganglia instead of having one overall packetloss metric.
+
+When this script is called directly it runs in testmode,
+PacketLossLogTailer.py is the regular point of entry.
+'''
+
+import re
+import urllib2
+import json
+import sys
+
+
+numbers = re.compile('([0-9]+)')
+base_url = 'http://noc.wikimedia.org/pybal'
+dcs = {
+ 'eqiad': ['apaches', 'api', 'bits', 'https', 'mobile', 'rendering',
'text', 'upload'],
+ 'pmtpa': ['apaches', 'api', 'bits', 'https', 'mobile', 'rendering',
'text', 'upload'],
+ 'esams': ['bits', 'https', 'text', 'upload'],
+}
+
+
+class RoleMatcher(object):
+ def __init__(self, role, regex, start=None, end=None):
+ self.role = role
+ self.regex = re.compile(regex)
+ self.start = start
+ self.end = end
+
+ def __str__(self):
+ if self.start is not None:
+ return '%s:%s-%s' % (self.role, self.start, self.end)
+ else:
+ return self.role
+
+ def __eq__(self, hostname):
+ match = self.regex.match(hostname)
+ if match:
+ if self.start and self.end:
+ try:
+ number = int(match.group(1))
+ except IndexError:
+ number = -1
+ return self.start <= number and number <= self.end
+ else:
+ return True
+ else:
+ return False
+
+ def get_role(self):
+ return self.__str__()
+
+
+def manual_init():
+ '''
+ This is the hard-coded version of the rolematchers, it's useful for
+ testing purposes.
+ '''
+ matchers = [
+ RoleMatcher('pmtpa_apache_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 226, 249),
+ RoleMatcher('pmtpa_apache_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 258, 289),
+ RoleMatcher('pmtpa_apache_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 17, 59),
+ RoleMatcher('pmtpa_apache_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 81, 111),
+ RoleMatcher('pmtpa_rendering_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 75, 80),
+ RoleMatcher('pmtpa_api_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 250, 257),
+ RoleMatcher('pmtpa_api_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 290, 301),
+ RoleMatcher('pmtpa_api_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 62, 74),
+ RoleMatcher('pmtpa_api_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 112, 125),
+ RoleMatcher('pmtpa_bits_sq', 'sq([0-9]+)\.pmtpa\.wmnet', 67, 70),
+ RoleMatcher('pmtpa_text_sq', 'sq([0-9]+)\.pmtpa\.wmnet', 37, 37),
+ RoleMatcher('pmtpa_text_sq', 'sq([0-9]+)\.pmtpa\.wmnet', 59, 66),
+ RoleMatcher('pmtpa_text_sq', 'sq([0-9]+)\.pmtpa\.wmnet', 71, 78),
+ RoleMatcher('pmtpa_ssl-ip6_ssl', 'ssl([0-9]{1})', 1, 4),
+ RoleMatcher('pmtpa_mobile_mobile', 'mobile([0-9]{1})', 1, 4),
+ RoleMatcher('pmtpa_upload_sq', 'sq([0-9]+)', 41, 58),
+ RoleMatcher('pmtpa_upload_sq', 'sq([0-9]+)', 79, 86),
+
+ RoleMatcher('eqiad_apache_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1017, 1113),
+ RoleMatcher('eqiad_apache_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1161, 1220),
+ RoleMatcher('eqiad_api_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1189, 1208),
+ RoleMatcher('eqiad_api_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1114, 1148),
+ RoleMatcher('eqiad_text_cp', 'cp(10[0-9]+)', 1001, 1020),
+ RoleMatcher('eqiad_upload_cp', 'cp(10[0-9]+)', 1021, 1036),
+ RoleMatcher('eqiad_mobile_cp', 'cp(104[0-9])+', 1041, 1044),
+ RoleMatcher('eqiad_bits',
'(arsenic|strontium|niobum|palladium|dysprosium)'),
+ RoleMatcher('eqiad_ssl-ip6_ssl', 'ssl(10[0-9])+', 1001, 1004),
+
+ RoleMatcher('esams_bits_cp', 'cp(30[0-9]+)\.esams', 3019, 3022),
+ RoleMatcher('esams_upload_cp', 'cp(30[0-9]+)\.esams', 3003, 3010),
+ RoleMatcher('esams_ssl-ip6_ssl', 'ssl(300[0-9]+)\.esams', 3001, 3004),
+ RoleMatcher('esams_text_kns', 'knsq([0-9]+)\.esams', 23, 30),
+ RoleMatcher('esams_text_amssq', 'amssq([0-9]+)\.esams', 31, 46),
+ RoleMatcher('esams_upload_knsq', 'knsq([0-9]+)\.esams', 16, 22),
+ RoleMatcher('esams_upload_amssq', 'amssq([0-9]+)\.esams', 47, 62),
+ ]
+ return matchers
+
+
+def parse(data):
+ sections = []
+ section = []
+ for row in data:
+ # pybal outputs python dictionaries but we are not going to use eval(),
+ # hence make the dictionary JSON compatible.
+ row = row.strip().replace('"', '').replace("'", '"').replace('True',
'true').replace('False', 'false')
+ if row == '':
+ sections.append(section)
+ section = []
+ elif not row.endswith('}'): # have to strip out some random comments
+ pos = row.find('}')
+ row = row[0:pos]
+ else:
+ if row.startswith('{'):
+ try:
+ section.append(json.loads(row))
+ except ValueError:
+ pass
+ sections.append(section)
+ return sections
+
+
+def determine_server_id(hostname):
+ match = re.findall(numbers, hostname)
+ try:
+ return int(match[0])
+ except IndexError:
+ return None
+
+
+def determine_start_end_range(section, hostname):
+ if len(section) == 1:
+ start = determine_server_id(hostname)
+ end = start
+ else:
+ start = determine_server_id(hostname)
+ end = determine_server_id(section[-1]['host'])
+ return start, end
+
+
+def determine_hostname_suffix(hostname):
+ suffix = '.'.join(hostname.split('.')[1:])
+ suffix = suffix.replace('.', '\\.')
+ return suffix
+
+
+def determine_hostname_prefix(hostname):
+ prefix = hostname.split('.')[0]
+ prefix = ''.join([i for i in prefix if not i.isdigit()])
+ return prefix
+
+def fetch_url(url):
+ req = urllib2.Request(url)
+ data = []
+ try:
+ response = urllib2.urlopen(req)
+ data = response.readlines()
+ except urllib2.HTTPError, e:
+ sys.stderr.write('HTTPError = %s' % e.code)
+ except urllib2.URLError, e:
+ sys.stderr.write('URLError = %s' % e.reason)
+ except httplib.HTTPException, e:
+ sys.stderr.write('HTTPException')
+ except Exception:
+ '''
+ just to be sure that nothing falls through the cracks
+ '''
+ import traceback
+ sys.stderr.write('Generic exception: %s' % traceback.format_exc())
+ return data
+
+def init():
+ matchers = []
+ elements = {}
+ for dc, roles in dcs.iteritems():
+ for role in roles:
+ url = '/'.join([base_url, dc, role])
+ data = fetch_url(url)
+ sections = parse(data)
+ for section in sections:
+ if len(section) > 0:
+ hostname = section[0]['host']
+ start, end = determine_start_end_range(section, hostname)
+ prefix = determine_hostname_prefix(hostname)
+ suffix = determine_hostname_suffix(hostname)
+ if prefix in ['arsenic', 'strontium', 'niobium',
'palladium', 'dysprosium', 'holmium']:
+ if '%s-%s' % (dc, role) not in elements:
+ matcher = RoleMatcher('%s_%s_elements' % (dc,
role), '(arsenic|strontium|niobum|palladium|dysprosium|holmium)')
+ elements['%s-%s' % (dc, role)] = True
+ matchers.append(matcher)
+ else:
+ matcher = RoleMatcher('%s_%s_%s' % (dc, role, prefix),
'%s([0-9]+)\.%s' % (prefix, suffix), start, end)
+ matchers.append(matcher)
+ return matchers
+
+
+if __name__ == '__main__':
+ if len(sys.argv) != 1:
+ print 'Please specify path to packetloss log file, call this file only
for testing purposes.'
+ sys.exit(-1)
+ else:
+ path = sys.argv[0]
+
+ matchers = init()
+ line_matcher = re.compile('^\[(?P<date>[^]]+)\] (?P<server>[^ ]+) lost:
\((?P<percentloss>[^ ]+) \+\/- (?P<margin>[^)]+)\)%')
+ fh = open(path, 'r')
+ for line in fh:
+ regMatch = line_matcher.match(line)
+ if regMatch:
+ fields = regMatch.groupdict()
+ hostname = fields['server']
+ role = 'misc' # default group for when we were not able to
determine the role
+ for matcher in matchers:
+ if matcher == hostname:
+ role = matcher.get_role()
+ break
+ if hostname == 'total':
+ role = 'total'
+ print hostname, role
+ fh.close()
diff --git a/manifests/misc/udp2log.pp b/manifests/misc/udp2log.pp
index 74ecd10..c0f552c 100644
--- a/manifests/misc/udp2log.pp
+++ b/manifests/misc/udp2log.pp
@@ -332,12 +332,20 @@
source => "puppet:///files/icinga/check_udp2log_procs",
}
- file { "PacketLossLogtailer.py":
- path => "/usr/share/ganglia-logtailer/PacketLossLogtailer.py",
+ file { 'rolematcher.py':
+ path => '/usr/share/ganglia-logtailer/rolematcher.py',
mode => 0444,
owner => root,
group => root,
- source => "puppet:///files/misc/PacketLossLogtailer.py"
+ source => 'puppet:///files/misc/rolematcher.py'
+ }
+
+ file { 'PacketLossLogtailer.py':
+ path => '/usr/share/ganglia-logtailer/PacketLossLogtailer.py',
+ mode => 0444,
+ owner => root,
+ group => root,
+ source => 'puppet:///files/misc/PacketLossLogtailer.py'
}
# send udp2log socket stats to ganglia.
--
To view, visit https://gerrit.wikimedia.org/r/63220
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I084c210d996068f213977f2fb942d4a80021be8f
Gerrit-PatchSet: 9
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Diederik <[email protected]>
Gerrit-Reviewer: Asher <[email protected]>
Gerrit-Reviewer: Diederik <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
