Diederik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/63220
Change subject: Added per dc / server role breakdown of udp2log packetloss
monitoring.
......................................................................
Added per dc / server role breakdown of udp2log packetloss monitoring.
Instead of having one overall packetloss metric in Ganglia per udp2log host,
this patchset introduces functionality to map a hostname to it's role and dc and
to calculate packetloss per dc / role.
The rolematcher.py determines the role and dc of a hostname; this information is
now hardcoded and was obtained from noc.wikimedia.org/pybal; Not sure if it's
worth the effort to dynamically fetching this information. Roles include apache,
api, mobile, text, upload, bits, ssl/ip6.
The interfaces of PacketLossLogtailer.py have not been changed, this changeset
might require some changes in Ganglia as well. A local test of ganglia-logtailer
did work.
Change-Id: I084c210d996068f213977f2fb942d4a80021be8f
---
M files/misc/PacketLossLogtailer.py
A files/misc/rolematcher.py
2 files changed, 160 insertions(+), 20 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/20/63220/1
diff --git a/files/misc/PacketLossLogtailer.py
b/files/misc/PacketLossLogtailer.py
index cdc5078..3ec2f15 100644
--- a/files/misc/PacketLossLogtailer.py
+++ b/files/misc/PacketLossLogtailer.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
###
### This plugin for logtailer will crunch WMF packet loss logs and return:
-### * average percent loss
-### * ninetieth percentile loss
+### * average percent loss per server role
+### * ninetieth percentile loss per server role
### It will throw out
### * packet loss numbers greater than 98%
### * large margins of error
@@ -17,6 +17,7 @@
# local dependencies
from ganglia_logtailer_helper import GangliaMetricObject
from ganglia_logtailer_helper import LogtailerParsingException,
LogtailerStateException
+import rolematcher
class PacketLossLogtailer(object):
# only used in daemon mode
@@ -26,6 +27,8 @@
needed for the internal state of the line parser.'''
self.reset_state()
self.lock = threading.RLock()
+ # a list of rolematchers which are simple object to determine the role
of a particular server
+ self.matchers = rolematcher.init()
# this is what will match the packet loss lines
# packet loss format :
# %[%Y-%m-%dT%H:%M:%S]t %server lost: (%percentloss +/- %margin)
@@ -51,11 +54,13 @@
# capture data
percentloss = float(linebits['percentloss'])
margin = float(linebits['margin'])
+ role = self.determine_role(linebits['server'])
# store for 90th % and average calculations
# on ssl servers, sequence numbers are out of order.
# http://rt.wikimedia.org/Ticket/Display.html?id=1616
if( ( margin <= 20 ) and ( percentloss <= 98 ) ):
- self.percentloss_list.append(percentloss)
+ self.percentloss_dict.setdefault(role, [])
+ self.percentloss_dict[role].append(percentloss)
else:
raise LogtailerParsingException, "regmatch failed to match"
@@ -63,6 +68,17 @@
self.lock.release()
raise LogtailerParsingException, "regmatch or contents failed with
%s" % e
self.lock.release()
+
+ def determine_role(self, hostname):
+ if hostname == 'total':
+ return 'total'
+ role = 'misc' # default group for when we were not able to determine
the role
+ for matcher in self.matchers:
+ if matcher == hostname:
+ role = matcher.get_role()
+ break;
+ return role
+
# example function for deep copy
# takes no arguments
# returns one object
@@ -72,9 +88,10 @@
currently being modified so that the other thread can deal with it
without fear of it changing out from under it. The format of this
object is internal to the plugin.'''
- myret = dict(percentloss_list=self.percentloss_list
+ myret = dict(percentloss_dict=self.percentloss_dict
)
return myret
+
# example function for reset_state
# takes no arguments
# returns nothing
@@ -87,8 +104,9 @@
reset_state should store now() each time it's called, and get_state
will use the time since that now() to do its calculations'''
self.num_hits = 0
- self.percentloss_list = list()
+ self.percentloss_dict = dict()
self.last_reset_time = time.time()
+
# example for keeping track of runtimes
# takes no arguments
# returns float number of seconds for this run
@@ -98,6 +116,7 @@
it.'''
self.duration = dur
self.dur_override = True
+
def get_check_duration(self):
'''This function should return the time since the last check. If
called
from cron mode, this must be set using set_check_duration(). If in
@@ -113,6 +132,7 @@
if (duration < acceptable_duration_min or duration >
acceptable_duration_max):
raise LogtailerStateException, "time calculation problem -
duration (%s) > 10%% away from period (%s)" % (duration, self.period)
return duration
+
# example function for get_state
# takes no arguments
# returns a dictionary of (metric => metric_object) pairs
@@ -137,20 +157,25 @@
raise e
# calculate 90th % and average request times
- percentloss_list = mydata['percentloss_list']
- percentloss_list.sort()
- num_entries = len(percentloss_list)
- if (num_entries != 0 ):
- packetloss_90th = percentloss_list[int(num_entries * 0.9)]
- packetloss_ave = sum(percentloss_list) / len(percentloss_list)
- else:
- # in this event, all data was thrown out in parse_line
- packetloss_90th = 99
- packetloss_ave = 99
- # package up the data you want to submit
- # setting tmax to 960 seconds as data may take as long as 15 minutes
to be processed
- packetloss_ave_metric = GangliaMetricObject( 'packet_loss_average',
packetloss_ave, units='%', tmax=960 )
- packetloss_90th_metric = GangliaMetricObject( 'packet_loss_90th',
packetloss_90th, units='%', tmax=960 )
+ percentloss_dict = mydata['percentloss_dict']
+ metrics = list()
+ for role, percentloss_list in percentloss_dict.iteritems():
+
+ percentloss_list.sort()
+ num_entries = len(percentloss_list)
+ if (num_entries != 0 ):
+ packetloss_90th = percentloss_list[int(num_entries * 0.9)]
+ packetloss_ave = sum(percentloss_list) / len(percentloss_list)
+ else:
+ # in this event, all data was thrown out in parse_line
+ packetloss_90th = 99
+ packetloss_ave = 99
+ # package up the data you want to submit
+ # setting tmax to 960 seconds as data may take as long as 15
minutes to be processed
+ packetloss_ave_metric = GangliaMetricObject(
'%s:packet_loss_average' % role, packetloss_ave, units='%', tmax=960 )
+ packetloss_90th_metric = GangliaMetricObject(
'%s:packet_loss_90th' % role, packetloss_90th, units='%', tmax=960 )
+ metrics.append(packetloss_ave_metric)
+ metrics.append(packetloss_90th_metric)
# return a list of metric objects
- return [ packetloss_ave_metric, packetloss_90th_metric, ]
\ No newline at end of file
+ return metrics
\ No newline at end of file
diff --git a/files/misc/rolematcher.py b/files/misc/rolematcher.py
new file mode 100644
index 0000000..c70dcc4
--- /dev/null
+++ b/files/misc/rolematcher.py
@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env python
+
+'''
+This script parses the packet-loss.log file and for each server entry
+it determines what it's role is. Server role data has been obtained
+from noc.wikimeda.org/pybal and is currently hardcoded.
+
+Purpose of this script is to use it in conjunction with
+PacketLossLogTailer.py and send packetloss metrics per dc/role to
+Ganglia instead of having one overall packetloss metric.
+
+When this script is called directly it runs in testmode,
+PacketLossLogTailer.py is the regular point of entry.
+'''
+
+import re
+
+numbers = re.compile('[0-9]{1-4}')
+
+class RoleMatcher(object):
+ def __init__(self, role, regex, start=None, end=None):
+ self.role = role
+ self.regex= re.compile(regex)
+ self.start = start
+ self.end = end
+
+ def __str__(self):
+ if self.start != None:
+ return '%s:%s-%s' % (self.role, self.start, self.end)
+ else:
+ return self.role
+
+ def __eq__(self, hostname):
+ match = self.regex.match(hostname)
+ if match:
+ if self.start and self.end:
+ try:
+ number = int(match.group(1))
+ except IndexError:
+ number = -1
+ return self.start <= number and number <= self.end
+ else:
+ return True
+ else:
+ return False
+
+ def get_role(self):
+ return self.__str__()
+
+def init():
+ matchers = [
+ RoleMatcher('pmtpa_apache_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 226, 249),
+ RoleMatcher('pmtpa_apache_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 258, 289),
+ RoleMatcher('pmtpa_apache_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 17, 59),
+ RoleMatcher('pmtpa_apache_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 81, 111),
+ RoleMatcher('pmtpa_rendering_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 75, 80),
+ RoleMatcher('pmtpa_api_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 250, 257),
+ RoleMatcher('pmtpa_api_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 290, 301),
+ RoleMatcher('pmtpa_api_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 62, 74),
+ RoleMatcher('pmtpa_api_mw', 'mw([0-9]+)\.pmtpa\.wmnet', 112, 125),
+ RoleMatcher('pmtpa_bits_sq', 'sq([0-9]+)\.pmtpa\.wmnet', 67, 70),
+ RoleMatcher('pmtpa_text_sq', 'sq([0-9]+)\.pmtpa\.wmnet', 37, 37),
+ RoleMatcher('pmtpa_text_sq', 'sq([0-9]+)\.pmtpa\.wmnet', 59, 66),
+ RoleMatcher('pmtpa_text_sq', 'sq([0-9]+)\.pmtpa\.wmnet', 71, 78),
+ RoleMatcher('pmtpa_ssl-ip6_ssl', 'ssl([0-9]{1})', 1, 4),
+ RoleMatcher('pmtpa_mobile_mobile', 'mobile([0-9]{1})', 1, 4),
+ RoleMatcher('pmtpa_upload_sq', 'sq([0-9]+)', 41, 58),
+ RoleMatcher('pmtpa_upload_sq', 'sq([0-9]+)', 79, 86),
+
+ RoleMatcher('eqiad_apache_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1017, 1113),
+ RoleMatcher('eqiad_apache_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1161, 1220),
+ RoleMatcher('eqiad_api_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1189, 1208),
+ RoleMatcher('eqiad_api_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1114, 1148),
+ RoleMatcher('eqiad_text_cp', 'cp(10[0-9]+)', 1001, 1020),
+ RoleMatcher('eqiad_upload_cp', 'cp(10[0-9]+)', 1021, 1036),
+ RoleMatcher('eqiad_mobile_cp', 'cp(104[0-9])+', 1041, 1044),
+ RoleMatcher('eqiad_bits',
'(arsenic|strontium|niobum|palladium|dysprosium)'),
+ RoleMatcher('eqiad_ssl-ip6_ssl', 'ssl(10[0-9])+', 1001, 1004),
+
+ RoleMatcher('esams_bits_cp', 'cp(30[0-9]+)\.esams', 3019, 3022),
+ RoleMatcher('esams_upload_cp', 'cp(30[0-9]+)\.esams', 3003, 3010),
+ RoleMatcher('esams_ssl-ip6_ssl', 'ssl(300[0-9]+)\.esams', 3001, 3004),
+ RoleMatcher('esams_text_kns', 'knsq([0-9]+)\.esams', 23, 30),
+ RoleMatcher('esams_text_amssq', 'amssq([0-9]+)\.esams', 31, 46),
+ RoleMatcher('esams_upload_knsq', 'knsq([0-9]+)\.esams', 16, 22),
+ RoleMatcher('esams_upload_amssq', 'amssq([0-9]+)\.esams', 47, 62),
+ ]
+ return matchers
+
+if __name__ == '__main__':
+ line_matcher = re.compile ('^\[(?P<date>[^]]+)\] (?P<server>[^ ]+) lost:
\((?P<percentloss>[^ ]+) \+\/- (?P<margin>[^)]+)\)%')
+ fh = open('/Users/diederik/Downloads/packet-loss.log-20130510', 'r')
+ matchers = init()
+
+ for line in fh:
+ regMatch = line_matcher.match(line)
+ if regMatch:
+ fields = regMatch.groupdict()
+ hostname = fields['server']
+ role = 'misc' # default group for when we were not able to
determine the role
+ for matcher in matchers:
+ if matcher == hostname:
+ role = matcher.get_role()
+ break;
+ if hostname == 'total':
+ role = 'total'
+ print hostname, role
+ fh.close()
+
+
+
+
+
+
--
To view, visit https://gerrit.wikimedia.org/r/63220
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I084c210d996068f213977f2fb942d4a80021be8f
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Diederik <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
