Volans has uploaded a new change for review.
https://gerrit.wikimedia.org/r/306646
Change subject: udpprofile: fix Flake 8
......................................................................
udpprofile: fix Flake 8
Fix Flake 8 errors except module level import not at top of file because
it might change the behaviour, silenced those with noqa.
Bug: T143559
Change-Id: Ibca061e2426fa25fc2f53a675b863a0501faa3d7
---
M udpprofile/profile-stats-logger.py
M udpprofile/web/admin.py
M udpprofile/web/config.py
M udpprofile/web/extractprofile.py
M udpprofile/web/pcache-hit-rate.py
M udpprofile/web/pcache-request-rate.py
M udpprofile/web/report.py
7 files changed, 199 insertions(+), 164 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/software
refs/changes/46/306646/1
diff --git a/udpprofile/profile-stats-logger.py
b/udpprofile/profile-stats-logger.py
index 15fea90..a4ff802 100755
--- a/udpprofile/profile-stats-logger.py
+++ b/udpprofile/profile-stats-logger.py
@@ -7,9 +7,9 @@
import time
sys.path.append('/usr/lib/cgi-bin/ng')
-import config
-from extractprofile import SocketProfile
-import rrdtool
+import config # noqa
+from extractprofile import SocketProfile # noqa
+import rrdtool # noqa
profId = 'stats/all'
rrdFileName = '/var/lib/profile-stats-logger/stats.rrd'
@@ -36,7 +36,7 @@
nextTime += 10
try:
- fullProfile = SocketProfile(config.host,config.port).extract()
+ fullProfile = SocketProfile(config.host, config.port).extract()
except:
continue
@@ -44,13 +44,13 @@
continue
profile = fullProfile[profId]['-']
if 'pcache_hit' not in profile:
- profile['pcache_hit'] = {'count':0}
+ profile['pcache_hit'] = {'count': 0}
if 'pcache_miss_absent' not in profile:
- profile['pcache_miss_absent'] = {'count':0}
+ profile['pcache_miss_absent'] = {'count': 0}
if 'pcache_miss_expired' not in profile:
- profile['pcache_miss_expired'] = {'count':0}
+ profile['pcache_miss_expired'] = {'count': 0}
if 'pcache_miss_invalid' not in profile:
- profile['pcache_miss_invalid'] = {'count':0}
+ profile['pcache_miss_invalid'] = {'count': 0}
rrdtool.update(rrdFileName,
'N:' + str(profile['pcache_hit']['count']) +
diff --git a/udpprofile/web/admin.py b/udpprofile/web/admin.py
index 694a21a..83ef088 100644
--- a/udpprofile/web/admin.py
+++ b/udpprofile/web/admin.py
@@ -9,29 +9,36 @@
cgitb.enable()
-password=config.password
+password = config.password
print "Content-type: text/html\n"
-form=cgi.SvFormContentDict()
+form = cgi.SvFormContentDict()
if 'password' in form:
if form['password'] != "" and form['password'] != password:
print "access denied!!!!!!!!!1111oneoneeleven"
sys.exit()
else:
- authed=True
+ authed = True
else:
- authed=False
+ authed = False
print """
- <script>function deletesample(sample)
{form=document.forms['actions'];form['sample'].value=sample;form.submit();}</script>
+ <script>
+ function deletesample(sample) {
+ form=document.forms['actions'];
+ form['sample'].value=sample;
+ form.submit();
+ }
+ </script>
<form name='actions' method='POST' action='admin.py'>
<input type='submit' name='action' value='take'>
<input type='submit' name='action' value='clear'>
<input type='hidden' name='sample' value=''>
<input type='%s' name='password' value='%s'
- </form>""" % ((authed and 'hidden' or 'password'),(authed and password or
''))
+ </form>""" % (
+ (authed and 'hidden' or 'password'), (authed and password or ''))
store = shelve.open('baselines')
@@ -40,15 +47,18 @@
del store[form['sample']]
elif form['action'] == 'clear' and authed:
import socket
- sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
- sock.sendto('-truncate',(config.host,config.port))
+ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ sock.sendto('-truncate', (config.host, config.port))
elif form['action'] == 'take' and authed:
from extractprofile import SocketProfile
- store[str(datetime.datetime.now()).replace(" ","-")] =
SocketProfile(config.host,config.port).extract()
+ store[str(datetime.datetime.now()).replace(" ", "-")] = SocketProfile(
+ config.host, config.port).extract()
elif 'sample' in form and authed:
del store[form['sample']]
for entry in store.keys():
- print """<div><a href='report.py?sample=%s'>%s</a> <a
href='javascript:deletesample("%s")'>delete</a></div>""" % (entry,entry,entry)
+ print (
+ """<div><a href='report.py?sample={0}'>{0}</a> <a href='javascript:"""
+ """deletesample("{0}")'>delete</a></div>""").format(entry)
print "<div><a href='report.py'>current</a></div>"
diff --git a/udpprofile/web/config.py b/udpprofile/web/config.py
index 7f1be01..f03bb0d 100644
--- a/udpprofile/web/config.py
+++ b/udpprofile/web/config.py
@@ -1,6 +1,6 @@
#!/usr/bin/python
-password="thepassword"
-host="127.0.0.1"
-port=3811
-db="phase3"
+password = "thepassword"
+host = "127.0.0.1"
+port = 3811
+db = "phase3"
diff --git a/udpprofile/web/extractprofile.py b/udpprofile/web/extractprofile.py
index 5696bb5..43acabf 100755
--- a/udpprofile/web/extractprofile.py
+++ b/udpprofile/web/extractprofile.py
@@ -11,44 +11,48 @@
from xml.sax import make_parser
from xml.sax.handler import feature_namespaces
+
# XML SAX parser class, puts stuff into some array!
class ExtractProfile(xml.sax.handler.ContentHandler):
def __init__(self):
- self.parser=make_parser()
- self.parser.setFeature(feature_namespaces,0)
+ self.parser = make_parser()
+ self.parser.setFeature(feature_namespaces, 0)
self.parser.setContentHandler(self)
- def startElement(self,name,attrs):
- if name=="db":
- self.db=attrs.get("name")
- self.profile[self.db]={}
- if name=="host":
- self.host=attrs.get("name")
- self.profile[self.db][self.host]={}
- if name=="eventname":
- self.inContent=1
- self.contentData=[]
- if name=="stats":
- self.event["count"]=int(attrs.get("count"))
- if name=="cputime":
- self.event["cpu"]=float(attrs.get("total"))
- self.event["cpusq"]=float(attrs.get("totalsq"))
- if name=="realtime":
- self.event["real"]=float(attrs.get("total"))
- self.event["realsq"]=float(attrs.get("totalsq"))
- if name=="samples":
- self.event["samples"]=attrs.get("real").split(" ")
- def endElement(self,name):
- if name=="eventname":
- self.inContent=0
+ def startElement(self, name, attrs):
+ if name == "db":
+ self.db = attrs.get("name")
+ self.profile[self.db] = {}
+ if name == "host":
+ self.host = attrs.get("name")
+ self.profile[self.db][self.host] = {}
+ if name == "eventname":
+ self.inContent = 1
+ self.contentData = []
+ if name == "stats":
+ self.event["count"] = int(attrs.get("count"))
+ if name == "cputime":
+ self.event["cpu"] = float(attrs.get("total"))
+ self.event["cpusq"] = float(attrs.get("totalsq"))
+ if name == "realtime":
+ self.event["real"] = float(attrs.get("total"))
+ self.event["realsq"] = float(attrs.get("totalsq"))
+ if name == "samples":
+ self.event["samples"] = attrs.get("real").split(" ")
+
+ def endElement(self, name):
+ if name == "eventname":
+ self.inContent = 0
self.eventname = "".join(self.contentData)
- self.profile[self.db][self.host][self.eventname]={}
- self.event=self.profile[self.db][self.host][self.eventname]
- if name=="stats":
+ self.profile[self.db][self.host][self.eventname] = {}
+ self.event = self.profile[self.db][self.host][self.eventname]
+ if name == "stats":
if self.event["count"] > 0:
- self.event["onereal"] = self.event["real"] /
self.event["count"]
+ self.event["onereal"] = (self.event["real"] /
+ self.event["count"])
self.event["onecpu"] = self.event["cpu"] / self.event["count"]
- def characters(self,chars):
+
+ def characters(self, chars):
if self.inContent:
self.contentData.append(chars)
@@ -58,24 +62,29 @@
def extract(self, file=False):
if (file is False):
file = open("profile.xml")
- self.profile={}
- self.inContent=0
+ self.profile = {}
+ self.inContent = 0
self.parser.parse(file)
return self.profile
+
class SocketProfile:
- def __init__(self,host='localhost',port=3811):
- self.sock=SocketSource()
- self.sock.connect((host,port))
+ def __init__(self, host='localhost', port=3811):
+ self.sock = SocketSource()
+ self.sock.connect((host, port))
def extract(self):
return ExtractProfile().extract(self.sock)
+
class SocketSource (socket.socket):
- """Stub class for extending socket object to support file source
mechanics"""
- def read(self,what):
+ """ Stub class for extending socket object
+
+ Add support for file source mechanics
+ """
+ def read(self, what):
"""Alias recv to read, missing in socket.socket"""
- return self.recv(what,0)
+ return self.recv(what, 0)
if __name__ == '__main__':
print "\nNot a valid entry point"
diff --git a/udpprofile/web/pcache-hit-rate.py
b/udpprofile/web/pcache-hit-rate.py
index 8b28741..2fe1319 100755
--- a/udpprofile/web/pcache-hit-rate.py
+++ b/udpprofile/web/pcache-hit-rate.py
@@ -21,30 +21,33 @@
print
rrdFileName = '/var/lib/profile-stats-logger/stats.rrd'
-graph = ['-',
- '--start', '-' + str(period) + 'm',
- '--width', '800',
- '--height', '500',
- '--upper-limit', '100',
- '--lower-limit', '0',
- '--rigid',
- '--title', 'Parser cache ratios (all wikis)',
- '--vertical-label', 'Ratio (%)',
- 'DEF:pcache_hit=' + rrdFileName + ':pcache_hit:AVERAGE',
- 'DEF:pcache_miss_absent=' + rrdFileName +
':pcache_miss_absent:AVERAGE',
- 'DEF:pcache_miss_expired=' + rrdFileName +
':pcache_miss_expired:AVERAGE',
- 'DEF:pcache_miss_invalid=' + rrdFileName +
':pcache_miss_invalid:AVERAGE',
-
'CDEF:total=pcache_hit,pcache_miss_absent,pcache_miss_expired,pcache_miss_invalid,+,+,+',
- 'CDEF:pcache_hit_percent=100,pcache_hit,total,/,*',
- 'CDEF:pcache_miss_absent_percent=100,pcache_miss_absent,total,/,*',
- 'CDEF:pcache_miss_expired_percent=100,pcache_miss_expired,total,/,*',
- 'CDEF:pcache_miss_invalid_percent=100,pcache_miss_invalid,total,/,*',
- 'CDEF:pcache_hit_avg=pcache_hit_percent,120,TREND',
- 'AREA:pcache_hit_percent#00ff00:Hit %:STACK',
- 'AREA:pcache_miss_absent_percent#ffff00:Miss (absent) %:STACK',
- 'AREA:pcache_miss_expired_percent#ff8888:Miss (expired) %:STACK',
- 'AREA:pcache_miss_invalid_percent#ff0000:Miss (invalid) %:STACK',
- ]
+graph = [
+ '-',
+ '--start', '-' + str(period) + 'm',
+ '--width', '800',
+ '--height', '500',
+ '--upper-limit', '100',
+ '--lower-limit', '0',
+ '--rigid',
+ '--title', 'Parser cache ratios (all wikis)',
+ '--vertical-label', 'Ratio (%)',
+ 'DEF:pcache_hit=' + rrdFileName + ':pcache_hit:AVERAGE',
+ 'DEF:pcache_miss_absent=' + rrdFileName + ':pcache_miss_absent:AVERAGE',
+ 'DEF:pcache_miss_expired=' + rrdFileName + ':pcache_miss_expired:AVERAGE',
+ 'DEF:pcache_miss_invalid=' + rrdFileName + ':pcache_miss_invalid:AVERAGE',
+ ('CDEF:total=pcache_hit,pcache_miss_absent,pcache_miss_expired,'
+ 'pcache_miss_invalid,+,+,+'),
+ 'CDEF:pcache_hit_percent=100,pcache_hit,total,/,*',
+ 'CDEF:pcache_miss_absent_percent=100,pcache_miss_absent,total,/,*',
+ 'CDEF:pcache_miss_expired_percent=100,pcache_miss_expired,total,/,*',
+ 'CDEF:pcache_miss_invalid_percent=100,pcache_miss_invalid,total,/,*',
+ 'CDEF:pcache_hit_avg=pcache_hit_percent,120,TREND',
+ 'AREA:pcache_hit_percent#00ff00:Hit %:STACK',
+ 'AREA:pcache_miss_absent_percent#ffff00:Miss (absent) %:STACK',
+ 'AREA:pcache_miss_expired_percent#ff8888:Miss (expired) %:STACK',
+ 'AREA:pcache_miss_invalid_percent#ff0000:Miss (invalid) %:STACK',
+]
+
if period < 1440:
graph.append('LINE:pcache_hit_avg#000080:Hit % (2 min. avg)')
diff --git a/udpprofile/web/pcache-request-rate.py
b/udpprofile/web/pcache-request-rate.py
index 84a4812..93579c6 100755
--- a/udpprofile/web/pcache-request-rate.py
+++ b/udpprofile/web/pcache-request-rate.py
@@ -21,20 +21,24 @@
print
rrdFileName = '/var/lib/profile-stats-logger/stats.rrd'
-graph = ['-',
- '--start', '-' + str(period) + 'm',
- '--width', '800',
- '--height', '500',
- '--vertical-label', 'Request rate (req/s)',
- '--title', 'Parser cache request rate (all wikis)',
- '--lower-limit', '0',
- 'DEF:pcache_hit=' + rrdFileName + ':pcache_hit:AVERAGE',
- 'DEF:pcache_miss_absent=' + rrdFileName +
':pcache_miss_absent:AVERAGE',
- 'DEF:pcache_miss_expired=' + rrdFileName +
':pcache_miss_expired:AVERAGE',
- 'DEF:pcache_miss_invalid=' + rrdFileName +
':pcache_miss_invalid:AVERAGE',
-
'CDEF:total=pcache_hit,pcache_miss_absent,pcache_miss_expired,pcache_miss_invalid,+,+,+',
- 'CDEF:total_avg_300=total,300,TREND',
- 'LINE:total#000080']
+graph = [
+ '-',
+ '--start', '-' + str(period) + 'm',
+ '--width', '800',
+ '--height', '500',
+ '--vertical-label', 'Request rate (req/s)',
+ '--title', 'Parser cache request rate (all wikis)',
+ '--lower-limit', '0',
+ 'DEF:pcache_hit=' + rrdFileName + ':pcache_hit:AVERAGE',
+ 'DEF:pcache_miss_absent=' + rrdFileName + ':pcache_miss_absent:AVERAGE',
+ 'DEF:pcache_miss_expired=' + rrdFileName + ':pcache_miss_expired:AVERAGE',
+ 'DEF:pcache_miss_invalid=' + rrdFileName + ':pcache_miss_invalid:AVERAGE',
+ ('CDEF:total=pcache_hit,pcache_miss_absent,pcache_miss_expired,'
+ 'pcache_miss_invalid,+,+,+'),
+ 'CDEF:total_avg_300=total,300,TREND',
+ 'LINE:total#000080'
+]
+
if period < 1440:
graph.append('LINE:total_avg_300#ff0000')
diff --git a/udpprofile/web/report.py b/udpprofile/web/report.py
index ed3d1b8..82fc998 100755
--- a/udpprofile/web/report.py
+++ b/udpprofile/web/report.py
@@ -4,71 +4,74 @@
import config
-db=config.db
+db = config.db
-sort="real"
-limit=50
+sort = "real"
+limit = 50
-from extractprofile import SocketProfile
+from extractprofile import SocketProfile # noqa
-import cgi
-import cgitb
+import cgi # noqa
+import cgitb # noqa
cgitb.enable()
-import shelve
+import shelve # noqa
print "Content-type: text/html"
print "\n"
-form=cgi.SvFormContentDict()
+form = cgi.SvFormContentDict()
store = shelve.open('baselines')
if "db" in form:
- db=form["db"]
+ db = form["db"]
if "sort" in form:
- sort=form["sort"]
+ sort = form["sort"]
if "limit" in form:
- limit=int(form["limit"])
+ limit = int(form["limit"])
if "compare" in form:
- compare=form["compare"]
- compared=store[compare]
+ compare = form["compare"]
+ compared = store[compare]
else:
- compare=""
- compared=None
+ compare = ""
+ compared = None
if 'sample' not in form:
- fullprofile=SocketProfile(config.host,config.port).extract()
- sample=""
+ fullprofile = SocketProfile(config.host, config.port).extract()
+ sample = ""
else:
- fullprofile=store[form['sample']]
- sample=form['sample']
+ fullprofile = store[form['sample']]
+ sample = form['sample']
-events=fullprofile[db]["-"].items()
-dbs=fullprofile.keys()
-total=fullprofile[db]["-"]["-total"]
+events = fullprofile[db]["-"].items()
+dbs = fullprofile.keys()
+total = fullprofile[db]["-"]["-total"]
# Limit the scope
if compare:
- compared=compared[db]["-"]
- oldtotal=compared["-total"]
+ compared = compared[db]["-"]
+ oldtotal = compared["-total"]
-#cache.close()
-if sort=="name":
- events.sort(lambda x,y: cmp(x[0],y[0]))
+# cache.close()
+if sort == "name":
+ events.sort(lambda x, y: cmp(x[0], y[0]))
else:
- events.sort(lambda y,x: cmp(x[1][sort],y[1][sort]))
+ events.sort(lambda y, x: cmp(x[1][sort], y[1][sort]))
-def surl(stype,stext=None,limit=50):
+
+def surl(stype, stext=None, limit=50):
""" Simple URL formatter for headers """
if (stext is None):
- stext=stype
+ stext = stype
if (stype == sort):
return """<td><b>%s</b></td>""" % stext
- return """<td><a
href='report.py?db=%s&sort=%s&limit=%d&sample=%s&compare=%s'>%s</a></td>""" %
(db, stype, limit, sample, compare, stext)
+ return ("""<td><a href='report.py?db=%s&sort=%s&limit=%d&sample=%s&"""
+ """compare=%s'>%s</a></td>""") % (db, stype, limit, sample,
+ compare, stext)
print """
<style>
@@ -86,7 +89,8 @@
</style>"""
if sample:
- print "<div>Using old sample: %s, <a href='report.py?db=%s'>reset to
current</a></div>" % (sample,db)
+ print ("<div>Using old sample: %s, <a href='report.py?db=%s'>"
+ "reset to current</a></div>") % (sample, db)
# Top list of databases
for dbname in dbs:
@@ -95,10 +99,14 @@
else:
print " [<a href='report.py?db=%s'>%s</a>] " % (dbname, dbname)
-if limit==50:
- print " [ showing %d events, <a
href='report.py?db=%s&sort=%s&sample=%s&compare=%s&limit=5000'>show more</a> ]
" % (limit, db, sort, sample, compare)
+if limit == 50:
+ print (" [ showing %d events, <a href='report.py?db=%s&sort=%s&sample=%s&"
+ "compare=%s&limit=5000'>show more</a> ] ") % (limit, db, sort,
+ sample, compare)
else:
- print " [ showing %d events, <a
href='report.py?db=%s&sort=%s&sample=%s&compare=%s&limit=50'>show less</a> ] "
% (limit,db,sort,sample,compare)
+ print (" [ showing %d events, <a href='report.py?db=%s&sort=%s&sample=%s&"
+ "compare=%s&limit=50'>show less</a> ] ") % (limit, db, sort,
+ sample, compare)
print " [ <a href='admin.py'>admin</a> ]</div>"
@@ -107,12 +115,13 @@
<input type='hidden' name='sort' value='%s'>
<input type='hidden' name='limit' value='%d'>
<input type='hidden' name='sample' value='%s'>
-<select name='compare'><option></option>""" %(db,sort,limit,sample)
+<select name='compare'><option></option>""" % (db, sort, limit, sample)
-samples=store.keys()
+samples = store.keys()
samples.sort()
for baseline in samples:
- print "<option%s>%s</option>" % ((compare==baseline and " SELECTED" or
""),baseline)
+ print "<option%s>%s</option>" % (
+ (compare == baseline and " SELECTED" or ""), baseline)
print "</select><input type='submit' value='compare'></form>"
@@ -124,17 +133,17 @@
print surl("count")
print "<td>count%</td>"
print "<td>change</td>"
-print surl("cpu","cpu%")
+print surl("cpu", "cpu%")
print "<td>change</td>"
-print surl("onecpu","cpu/c")
+print surl("onecpu", "cpu/c")
print "<td>change</td>"
-print surl("real","real%")
+print surl("real", "real%")
print "<td>change</td>"
-print surl("onereal","real/c")
+print surl("onereal", "real/c")
print "<td>change</td>"
print "</tr>"
-rowformat="""
+rowformat = """
<tr class="data"><td class="name">%s</td><td>%d</td>
<td>%.2f</td><td>%.1f</td><td>%.2f</td><td>%.1f</td></tr>"""
@@ -144,7 +153,7 @@
# cputime: total compared percall compared
# realtime: total compared percall compared
-comparedformat="""
+comparedformat = """
<tr class="data"><td class="name">%s</td>
<td>%d</td><td>%.2f</td><td>%.1f</td>
<td>%.2f</td><td>%.1f</td><td>%.1f</td><td>%.1f</td>
@@ -157,26 +166,26 @@
# And this is output of results.
for event in events:
(name, event) = event
- if name=="close":
+ if name == "close":
continue
if compared and name in compared:
- old=compared[name]
+ old = compared[name]
else:
- old=None
+ old = None
limit -= 1
if limit < 0:
break
- callcount=float(event["count"]) / total["count"]
- cpupct=event["cpu"] / total["cpu"]
- onecpu=event["onecpu"]
- realpct=event["real"] / total["real"]
- onereal=event["onereal"]
+ callcount = float(event["count"]) / total["count"]
+ cpupct = event["cpu"] / total["cpu"]
+ onecpu = event["onecpu"]
+ realpct = event["real"] / total["real"]
+ onereal = event["onereal"]
if old:
try:
- oldcount=float(old["count"]) / oldtotal["count"]
+ oldcount = float(old["count"]) / oldtotal["count"]
countdiff = (callcount - oldcount) / oldcount
oldcpupct = old["cpu"] / oldtotal["cpu"]
@@ -189,27 +198,27 @@
onerealdiff = (onereal - old["onereal"]) / old["onereal"]
except ZeroDivisionError:
- countdiff=0
- cpupctdiff=0
- onecpudiff=0
- realpctdiff=0
- onerealdiff=0
+ countdiff = 0
+ cpupctdiff = 0
+ onecpudiff = 0
+ realpctdiff = 0
+ onerealdiff = 0
else:
- countdiff=0
- cpupctdiff=0
- onecpudiff=0
- realpctdiff=0
- onerealdiff=0
+ countdiff = 0
+ cpupctdiff = 0
+ onecpudiff = 0
+ realpctdiff = 0
+ onerealdiff = 0
- dbg=0
+ dbg = 0
- if dbg and name=="wfMsgReal":
+ if dbg and name == "wfMsgReal":
print old
print oldtotal
print event
print total
if not dbg:
- print comparedformat % (name.replace(",",", "),
+ print comparedformat % (name.replace(",", ", "),
event["count"], callcount, countdiff * 100,
cpupct * 100, cpupctdiff * 100,
onecpu * 1000, onecpudiff * 100,
--
To view, visit https://gerrit.wikimedia.org/r/306646
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibca061e2426fa25fc2f53a675b863a0501faa3d7
Gerrit-PatchSet: 1
Gerrit-Project: operations/software
Gerrit-Branch: master
Gerrit-Owner: Volans <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
