Andrew Bogott has submitted this change and it was merged.
Change subject: pep8 cleanup for udpprofile
......................................................................
pep8 cleanup for udpprofile
Change-Id: I3b149ab467e5aae2daf26b14829ec5b6b7d50a1e
---
M udpprofile/profile-stats-logger.py
M udpprofile/profiler-to-carbon
M udpprofile/web/admin.py
M udpprofile/web/config.py
M udpprofile/web/extractprofile.py
M udpprofile/web/pcache-hit-rate.py
M udpprofile/web/pcache-request-rate.py
M udpprofile/web/report.py
8 files changed, 400 insertions(+), 388 deletions(-)
Approvals:
Andrew Bogott: Verified; Looks good to me, approved
Hashar: Looks good to me, but someone else must approve
jenkins-bot: Verified
diff --git a/udpprofile/profile-stats-logger.py
b/udpprofile/profile-stats-logger.py
index d4c893a..e3a68b3 100755
--- a/udpprofile/profile-stats-logger.py
+++ b/udpprofile/profile-stats-logger.py
@@ -1,8 +1,13 @@
#!/usr/bin/python
-import sys, time, os, os.path, math, pprint
+import math
+import os
+import os.path
+import pprint
+import sys
+import time
-sys.path.append( '/usr/lib/cgi-bin/ng' )
+sys.path.append('/usr/lib/cgi-bin/ng')
import config
from extractprofile import SocketProfile
import rrdtool
@@ -10,50 +15,46 @@
profId = 'stats/all'
rrdFileName = '/var/lib/profile-stats-logger/stats.rrd'
-if not os.path.exists( rrdFileName ):
- rrdtool.create(
- rrdFileName,
- '--step', '10',
- 'DS:pcache_hit:DERIVE:60:0:U',
- 'DS:pcache_miss_absent:DERIVE:60:0:U',
- 'DS:pcache_miss_expired:DERIVE:60:0:U',
- 'DS:pcache_miss_invalid:DERIVE:60:0:U',
- 'RRA:AVERAGE:0.1:1:8640', # 10s intervals for 24 hours
- 'RRA:AVERAGE:0.1:30:8640', # 5 min intervals for 1 month
- 'RRA:AVERAGE:0.1:360:8640' # 1 hour intervals for 1 year
- )
+if not os.path.exists(rrdFileName):
+ rrdtool.create(rrdFileName,
+ '--step', '10',
+ 'DS:pcache_hit:DERIVE:60:0:U',
+ 'DS:pcache_miss_absent:DERIVE:60:0:U',
+ 'DS:pcache_miss_expired:DERIVE:60:0:U',
+ 'DS:pcache_miss_invalid:DERIVE:60:0:U',
+ 'RRA:AVERAGE:0.1:1:8640', # 10s intervals for 24 hours
+ 'RRA:AVERAGE:0.1:30:8640', # 5 min intervals for 1 month
+ 'RRA:AVERAGE:0.1:360:8640' # 1 hour intervals for 1 year
+ )
nextTime = math.floor(time.time() / 10) * 10 + 10
while True:
- t = time.time()
- while t < nextTime:
- time.sleep(nextTime - t)
- t = time.time()
- nextTime += 10
+ t = time.time()
+ while t < nextTime:
+ time.sleep(nextTime - t)
+ t = time.time()
+ nextTime += 10
- try:
- fullProfile = SocketProfile(config.host,config.port).extract()
- except:
- continue
+ try:
+ fullProfile = SocketProfile(config.host,config.port).extract()
+ except:
+ continue
- if not ( profId in fullProfile ):
- continue
- profile = fullProfile[profId]['-']
- if 'pcache_hit' not in profile:
- profile['pcache_hit'] = {'count':0}
- if 'pcache_miss_absent' not in profile:
- profile['pcache_miss_absent'] = {'count':0}
- if 'pcache_miss_expired' not in profile:
- profile['pcache_miss_expired'] = {'count':0}
- if 'pcache_miss_invalid' not in profile:
- profile['pcache_miss_invalid'] = {'count':0}
+ if not (profId in fullProfile):
+ continue
+ profile = fullProfile[profId]['-']
+ if 'pcache_hit' not in profile:
+ profile['pcache_hit'] = {'count':0}
+ if 'pcache_miss_absent' not in profile:
+ profile['pcache_miss_absent'] = {'count':0}
+ if 'pcache_miss_expired' not in profile:
+ profile['pcache_miss_expired'] = {'count':0}
+ if 'pcache_miss_invalid' not in profile:
+ profile['pcache_miss_invalid'] = {'count':0}
-
- rrdtool.update(
- rrdFileName,
- 'N:' + str(profile['pcache_hit']['count']) +
- ':' + str(profile['pcache_miss_absent']['count']) +
- ':' + str(profile['pcache_miss_expired']['count']) +
- ':' + str(profile['pcache_miss_invalid']['count'] ) )
-
+ rrdtool.update(rrdFileName,
+ 'N:' + str(profile['pcache_hit']['count']) +
+ ':' + str(profile['pcache_miss_absent']['count']) +
+ ':' + str(profile['pcache_miss_expired']['count']) +
+ ':' + str(profile['pcache_miss_invalid']['count']))
diff --git a/udpprofile/profiler-to-carbon b/udpprofile/profiler-to-carbon
index 23d4623..93c6f03 100755
--- a/udpprofile/profiler-to-carbon
+++ b/udpprofile/profiler-to-carbon
@@ -29,130 +29,130 @@
prior={}
class SocketSource(socket.socket):
- def read(self,what):
- enc = self.recv(what,0)
- return enc.decode('latin-1').encode('utf-8')
+ def read(self,what):
+ enc = self.recv(what,0)
+ return enc.decode('latin-1').encode('utf-8')
def BuildStats(db, fullprofile):
- stats = {}
- events=fullprofile[db]["-"].items()
- bad = 0
+ stats = {}
+ events=fullprofile[db]["-"].items()
+ bad = 0
- for event in events:
- if "close" in event[0]:
- continue
- if "Profiling error" in event[0]:
- continue
- # . is the graphite path separator
- # stats are sent as stats.$stat, regular functions with
- # up to two path levels(i.e. API::Foo::Bar = API.Foo.Bar)
- if(db.startswith('stats')):
- name = 'stats.' + invalid.sub('_',
str(event[0])).rstrip('_')
- else:
- try:
- name = invalid.sub('_',
str(event[0])).rstrip('_').replace('_', '.', 2)
- except:
- logging.debug("skipping on failed unicode
conversion: %s", event[0])
- continue
- for skip in skips:
- if skip.match(name):
- bad = 1
- break
- if bad == 1:
- bad = 0
- continue
+ for event in events:
+ if "close" in event[0]:
+ continue
+ if "Profiling error" in event[0]:
+ continue
+ # . is the graphite path separator
+ # stats are sent as stats.$stat, regular functions with
+ # up to two path levels(i.e. API::Foo::Bar = API.Foo.Bar)
+ if(db.startswith('stats')):
+ name = 'stats.' + invalid.sub('_', str(event[0])).rstrip('_')
+ else:
+ try:
+ name = invalid.sub('_',
str(event[0])).rstrip('_').replace('_', '.', 2)
+ except:
+ logging.debug("skipping on failed unicode conversion: %s",
event[0])
+ continue
+ for skip in skips:
+ if skip.match(name):
+ bad = 1
+ break
+ if bad == 1:
+ bad = 0
+ continue
- stats[name] = {}
- stats[name]['count'] = event[1]['count']
- # real = time in ms
- stats[name]['real'] = event[1]['real'] * 1000
- stats[name]['samples'] = event[1]['samples']
+ stats[name] = {}
+ stats[name]['count'] = event[1]['count']
+ # real = time in ms
+ stats[name]['real'] = event[1]['real'] * 1000
+ stats[name]['samples'] = event[1]['samples']
- return stats
+ return stats
def SendStats(db, current, graph):
- now = int(time.time())
- message = ""
- for key in current.keys():
- if key not in prior[db]:
- continue
- count = current[key]['count'] - prior[db][key]['count']
+ now = int(time.time())
+ message = ""
+ for key in current.keys():
+ if key not in prior[db]:
+ continue
+ count = current[key]['count'] - prior[db][key]['count']
- if(count <= 0):
- continue
+ if(count <= 0):
+ continue
- message = "%s.count %d %d\n" %(key, count, now)
- logging.debug("sending: %s", message)
- graph.send(message)
+ message = "%s.count %d %d\n" %(key, count, now)
+ logging.debug("sending: %s", message)
+ graph.send(message)
- if(db.startswith('stats')):
- continue
+ if(db.startswith('stats')):
+ continue
- real = current[key]['real'] - prior[db][key]['real']
- message = "%s.tavg %.3f %d\n" % (key, real / count, now)
- logging.debug("sending: %s", message)
- graph.send(message)
+ real = current[key]['real'] - prior[db][key]['real']
+ message = "%s.tavg %.3f %d\n" % (key, real / count, now)
+ logging.debug("sending: %s", message)
+ graph.send(message)
- # cast all elements of [key]['samples'] from str to float
- samples = sorted(map(lambda i: float(i),
current[key]['samples']))
- tp50 = int(round(len(samples) * 0.5)) - 1
- tp90 = int(round(len(samples) * 0.9)) - 1
- tp99 = int(round(len(samples) * 0.99)) - 1
+ # cast all elements of [key]['samples'] from str to float
+ samples = sorted(map(lambda i: float(i), current[key]['samples']))
+ tp50 = int(round(len(samples) * 0.5)) - 1
+ tp90 = int(round(len(samples) * 0.9)) - 1
+ tp99 = int(round(len(samples) * 0.99)) - 1
- message = "%s.tp50 %.3f %d\n" % (key, float(samples[tp50]) *
1000, now)
- logging.debug("sending: %s", message)
- graph.send(message)
+ message = "%s.tp50 %.3f %d\n" % (key, float(samples[tp50]) * 1000, now)
+ logging.debug("sending: %s", message)
+ graph.send(message)
- message = "%s.tp90 %.3f %d\n" % (key, float(samples[tp90]) *
1000, now)
- logging.debug("sending: %s", message)
- graph.send(message)
+ message = "%s.tp90 %.3f %d\n" % (key, float(samples[tp90]) * 1000, now)
+ logging.debug("sending: %s", message)
+ graph.send(message)
- message = "%s.tp99 %.3f %d\n" % (key, float(samples[tp99]) *
1000, now)
- logging.debug("sending: %s", message)
- graph.send(message)
+ message = "%s.tp99 %.3f %d\n" % (key, float(samples[tp99]) * 1000, now)
+ logging.debug("sending: %s", message)
+ graph.send(message)
while True:
- profsock=SocketSource()
- try:
- profsock.connect((profile_host,profile_port))
- except:
- logging.debug("Couldn't connect to %s on port %d, is collector
running?",
- profile_host, profile_port)
- time.sleep(delay)
- continue
+ profsock=SocketSource()
+ try:
+ profsock.connect((profile_host,profile_port))
+ except:
+ logging.debug("Couldn't connect to %s on port %d, is collector
running?",
+ profile_host, profile_port)
+ time.sleep(delay)
+ continue
- graph = socket.socket()
- try:
- graph.connect((carbon_host,carbon_port))
- except:
- logging.debug("Couldn't connect to %s on port %d, is
carbon-agent.py running?",
- carbon_host, profile_port)
- time.sleep(delay)
- continue
+ graph = socket.socket()
+ try:
+ graph.connect((carbon_host,carbon_port))
+ except:
+ logging.debug("Couldn't connect to %s on port %d, is carbon-agent.py
running?",
+ carbon_host, profile_port)
+ time.sleep(delay)
+ continue
- try:
- fullprofile=ExtractProfile().extract(profsock)
- except Exception as detail:
- logging.debug("failed extracting data from collector: %s",
detail)
- time.sleep(delay)
- continue
+ try:
+ fullprofile=ExtractProfile().extract(profsock)
+ except Exception as detail:
+ logging.debug("failed extracting data from collector: %s", detail)
+ time.sleep(delay)
+ continue
- profsock.shutdown(socket.SHUT_RDWR)
- profsock.close()
+ profsock.shutdown(socket.SHUT_RDWR)
+ profsock.close()
- current={}
+ current={}
- for db in dbs:
- if db not in prior:
- prior[db] = []
- current[db] = BuildStats(db, fullprofile)
- if(len(current[db]) > 1):
- try:
- SendStats(db, current[db], graph)
- except:
- logging.debug("error sending stats")
- continue
- else:
- logging.debug("%s is empty", db)
- prior = current
- time.sleep(delay)
+ for db in dbs:
+ if db not in prior:
+ prior[db] = []
+ current[db] = BuildStats(db, fullprofile)
+ if(len(current[db]) > 1):
+ try:
+ SendStats(db, current[db], graph)
+ except:
+ logging.debug("error sending stats")
+ continue
+ else:
+ logging.debug("%s is empty", db)
+ prior = current
+ time.sleep(delay)
diff --git a/udpprofile/web/admin.py b/udpprofile/web/admin.py
index 0dd9e0f..33fefe0 100644
--- a/udpprofile/web/admin.py
+++ b/udpprofile/web/admin.py
@@ -1,12 +1,13 @@
#!/usr/bin/python
-import shelve
import cgi
-import cgitb; cgitb.enable()
-
-import datetime
-import sys
+import cgitb
import config
+import datetime
+import shelve
+import sys
+
+cgitb.enable()
password=config.password
@@ -15,39 +16,39 @@
form=cgi.SvFormContentDict()
if 'password' in form:
- if form['password'] != "" and form['password'] != password:
- print "access denied!!!!!!!!!1111oneoneeleven"
- sys.exit()
- else:
- authed=True
+ if form['password'] != "" and form['password'] != password:
+ print "access denied!!!!!!!!!1111oneoneeleven"
+ sys.exit()
+ else:
+ authed=True
else:
- authed=False
+ authed=False
print """
- <script>function deletesample(sample)
{form=document.forms['actions'];form['sample'].value=sample;form.submit();}</script>
- <form name='actions' method='POST' action='admin.py'>
- <input type='submit' name='action' value='take'>
- <input type='submit' name='action' value='clear'>
- <input type='hidden' name='sample' value=''>
- <input type='%s' name='password' value='%s'
- </form>""" % ((authed and 'hidden' or 'password'),(authed and password
or ''))
+ <script>function deletesample(sample)
{form=document.forms['actions'];form['sample'].value=sample;form.submit();}</script>
+ <form name='actions' method='POST' action='admin.py'>
+ <input type='submit' name='action' value='take'>
+ <input type='submit' name='action' value='clear'>
+ <input type='hidden' name='sample' value=''>
+ <input type='%s' name='password' value='%s'
+ </form>""" % ((authed and 'hidden' or 'password'),(authed and password or
''))
store = shelve.open('baselines')
if 'action' not in form:
- if 'sample' in form and authed:
- del store[form['sample']]
+ if 'sample' in form and authed:
+ del store[form['sample']]
elif form['action'] == 'clear' and authed:
- from socket import *
- sock = socket(AF_INET,SOCK_DGRAM)
- sock.sendto('-truncate',(config.host,config.port))
+ from socket import *
+ sock = socket(AF_INET,SOCK_DGRAM)
+ sock.sendto('-truncate',(config.host,config.port))
elif form['action'] == 'take' and authed:
- from extractprofile import SocketProfile
- store[str(datetime.datetime.now()).replace(" ","-")] =
SocketProfile(config.host,config.port).extract()
+ from extractprofile import SocketProfile
+ store[str(datetime.datetime.now()).replace(" ","-")] =
SocketProfile(config.host,config.port).extract()
elif 'sample' in form and authed:
- del store[form['sample']]
+ del store[form['sample']]
for entry in store.keys():
- print """<div><a href='report.py?sample=%s'>%s</a> <a
href='javascript:deletesample("%s")'>delete</a></div>""" % (entry,entry,entry)
-
-print "<div><a href='report.py'>current</a></div>"
\ No newline at end of file
+ print """<div><a href='report.py?sample=%s'>%s</a> <a
href='javascript:deletesample("%s")'>delete</a></div>""" % (entry,entry,entry)
+
+print "<div><a href='report.py'>current</a></div>"
diff --git a/udpprofile/web/config.py b/udpprofile/web/config.py
index 87faba3..7f1be01 100644
--- a/udpprofile/web/config.py
+++ b/udpprofile/web/config.py
@@ -4,4 +4,3 @@
host="127.0.0.1"
port=3811
db="phase3"
-
diff --git a/udpprofile/web/extractprofile.py b/udpprofile/web/extractprofile.py
index e154623..888f1dc 100755
--- a/udpprofile/web/extractprofile.py
+++ b/udpprofile/web/extractprofile.py
@@ -16,69 +16,69 @@
# XML SAX parser class, puts stuff into some array!
class ExtractProfile(xml.sax.handler.ContentHandler):
- def __init__(self):
- self.parser=make_parser()
- self.parser.setFeature(feature_namespaces,0)
- self.parser.setContentHandler(self)
- def startElement(self,name,attrs):
- if name=="db":
- self.db=attrs.get("name")
- self.profile[self.db]={}
- if name=="host":
- self.host=attrs.get("name")
- self.profile[self.db][self.host]={}
- if name=="eventname":
- self.inContent=1
- self.contentData=[]
- if name=="stats":
- self.event["count"]=int(attrs.get("count"))
- if name=="cputime":
- self.event["cpu"]=float(attrs.get("total"))
- self.event["cpusq"]=float(attrs.get("totalsq"))
- if name=="realtime":
- self.event["real"]=float(attrs.get("total"))
- self.event["realsq"]=float(attrs.get("totalsq"))
- if name=="samples":
- self.event["samples"]=attrs.get("real").split(" ")
-
- def endElement(self,name):
- if name=="eventname":
- self.inContent=0
- self.eventname = "".join(self.contentData)
- self.profile[self.db][self.host][self.eventname]={}
-
self.event=self.profile[self.db][self.host][self.eventname]
- if name=="stats":
- if self.event["count"] > 0:
-
self.event["onereal"]=self.event["real"]/self.event["count"]
-
self.event["onecpu"]=self.event["cpu"]/self.event["count"]
- def characters(self,chars):
- if self.inContent:
- self.contentData.append(chars)
+ def __init__(self):
+ self.parser=make_parser()
+ self.parser.setFeature(feature_namespaces,0)
+ self.parser.setContentHandler(self)
+ def startElement(self,name,attrs):
+ if name=="db":
+ self.db=attrs.get("name")
+ self.profile[self.db]={}
+ if name=="host":
+ self.host=attrs.get("name")
+ self.profile[self.db][self.host]={}
+ if name=="eventname":
+ self.inContent=1
+ self.contentData=[]
+ if name=="stats":
+ self.event["count"]=int(attrs.get("count"))
+ if name=="cputime":
+ self.event["cpu"]=float(attrs.get("total"))
+ self.event["cpusq"]=float(attrs.get("totalsq"))
+ if name=="realtime":
+ self.event["real"]=float(attrs.get("total"))
+ self.event["realsq"]=float(attrs.get("totalsq"))
+ if name=="samples":
+ self.event["samples"]=attrs.get("real").split(" ")
- def getProfile(self):
- return self.profile
+ def endElement(self,name):
+ if name=="eventname":
+ self.inContent=0
+ self.eventname = "".join(self.contentData)
+ self.profile[self.db][self.host][self.eventname]={}
+ self.event=self.profile[self.db][self.host][self.eventname]
+ if name=="stats":
+ if self.event["count"] > 0:
+ self.event["onereal"] = self.event["real"] /
self.event["count"]
+ self.event["onecpu"] = self.event["cpu"] / self.event["count"]
+ def characters(self,chars):
+ if self.inContent:
+ self.contentData.append(chars)
- def extract(self,file=False):
- if (file==False):
- file=open("profile.xml")
- self.profile={}
- self.inContent=0
- self.parser.parse(file)
- return self.profile
-
+ def getProfile(self):
+ return self.profile
+
+ def extract(self, file=False):
+ if (file is False):
+ file = open("profile.xml")
+ self.profile={}
+ self.inContent=0
+ self.parser.parse(file)
+ return self.profile
+
class SocketProfile:
- def __init__(self,host='localhost',port=3811):
- self.sock=SocketSource()
- self.sock.connect((host,port))
-
- def extract(self):
- return ExtractProfile().extract(self.sock)
-
+ def __init__(self,host='localhost',port=3811):
+ self.sock=SocketSource()
+ self.sock.connect((host,port))
+
+ def extract(self):
+ return ExtractProfile().extract(self.sock)
+
class SocketSource (socket.socket):
- """Stub class for extending socket object to support file source
mechanics"""
- def read(self,what):
- """Alias recv to read, missing in socket.socket"""
- return self.recv(what,0)
+ """Stub class for extending socket object to support file source
mechanics"""
+ def read(self,what):
+ """Alias recv to read, missing in socket.socket"""
+ return self.recv(what,0)
if __name__ == '__main__':
print "\nNot a valid entry point"
diff --git a/udpprofile/web/pcache-hit-rate.py
b/udpprofile/web/pcache-hit-rate.py
index 92bb46e..dc6d9e7 100755
--- a/udpprofile/web/pcache-hit-rate.py
+++ b/udpprofile/web/pcache-hit-rate.py
@@ -1,50 +1,53 @@
#!/usr/bin/python
-import sys, os, rrdtool, cgi, cgitb
+import cgi
+import cgitb
+import os
+import rrdtool
+import sys
cgitb.enable()
form = cgi.SvFormContentDict()
if 'period' in form:
- if form['period'].endswith( 'h' ):
- period = int( form['period'][0:-1] ) * 60
- elif form['period'].endswith( 'd' ):
- period = int( form['period'][0:-1] ) * 60 * 24
- else:
- period = int( form['period'] )
+ if form['period'].endswith('h'):
+ period = int(form['period'][0:-1]) * 60
+ elif form['period'].endswith('d'):
+ period = int(form['period'][0:-1]) * 60 * 24
+ else:
+ period = int(form['period'])
else:
- period = 30
+ period = 30
print 'Content-Type: image/png'
print 'Cache-Control: no-cache'
print
rrdFileName = '/var/lib/profile-stats-logger/stats.rrd'
-graph = [
- '-',
- '--start', '-' + str( period ) + 'm',
- '--width', '800',
- '--height', '500',
- '--upper-limit', '100',
- '--lower-limit', '0',
- '--rigid',
- '--title', 'Parser cache ratios (all wikis)',
- '--vertical-label', 'Ratio (%)',
- 'DEF:pcache_hit=' + rrdFileName + ':pcache_hit:AVERAGE',
- 'DEF:pcache_miss_absent=' + rrdFileName + ':pcache_miss_absent:AVERAGE',
- 'DEF:pcache_miss_expired=' + rrdFileName +
':pcache_miss_expired:AVERAGE',
- 'DEF:pcache_miss_invalid=' + rrdFileName +
':pcache_miss_invalid:AVERAGE',
-
'CDEF:total=pcache_hit,pcache_miss_absent,pcache_miss_expired,pcache_miss_invalid,+,+,+',
- 'CDEF:pcache_hit_percent=100,pcache_hit,total,/,*',
- 'CDEF:pcache_miss_absent_percent=100,pcache_miss_absent,total,/,*',
- 'CDEF:pcache_miss_expired_percent=100,pcache_miss_expired,total,/,*',
- 'CDEF:pcache_miss_invalid_percent=100,pcache_miss_invalid,total,/,*',
- 'CDEF:pcache_hit_avg=pcache_hit_percent,120,TREND',
- 'AREA:pcache_hit_percent#00ff00:Hit %:STACK',
- 'AREA:pcache_miss_absent_percent#ffff00:Miss (absent) %:STACK',
- 'AREA:pcache_miss_expired_percent#ff8888:Miss (expired) %:STACK',
- 'AREA:pcache_miss_invalid_percent#ff0000:Miss (invalid) %:STACK',
-]
+graph = ['-',
+ '--start', '-' + str(period) + 'm',
+ '--width', '800',
+ '--height', '500',
+ '--upper-limit', '100',
+ '--lower-limit', '0',
+ '--rigid',
+ '--title', 'Parser cache ratios (all wikis)',
+ '--vertical-label', 'Ratio (%)',
+ 'DEF:pcache_hit=' + rrdFileName + ':pcache_hit:AVERAGE',
+ 'DEF:pcache_miss_absent=' + rrdFileName +
':pcache_miss_absent:AVERAGE',
+ 'DEF:pcache_miss_expired=' + rrdFileName +
':pcache_miss_expired:AVERAGE',
+ 'DEF:pcache_miss_invalid=' + rrdFileName +
':pcache_miss_invalid:AVERAGE',
+
'CDEF:total=pcache_hit,pcache_miss_absent,pcache_miss_expired,pcache_miss_invalid,+,+,+',
+ 'CDEF:pcache_hit_percent=100,pcache_hit,total,/,*',
+ 'CDEF:pcache_miss_absent_percent=100,pcache_miss_absent,total,/,*',
+ 'CDEF:pcache_miss_expired_percent=100,pcache_miss_expired,total,/,*',
+ 'CDEF:pcache_miss_invalid_percent=100,pcache_miss_invalid,total,/,*',
+ 'CDEF:pcache_hit_avg=pcache_hit_percent,120,TREND',
+ 'AREA:pcache_hit_percent#00ff00:Hit %:STACK',
+ 'AREA:pcache_miss_absent_percent#ffff00:Miss (absent) %:STACK',
+ 'AREA:pcache_miss_expired_percent#ff8888:Miss (expired) %:STACK',
+ 'AREA:pcache_miss_invalid_percent#ff0000:Miss (invalid) %:STACK',
+ ]
if period < 1440:
- graph.append('LINE:pcache_hit_avg#000080:Hit % (2 min. avg)')
+ graph.append('LINE:pcache_hit_avg#000080:Hit % (2 min. avg)')
rrdtool.graph(*graph)
diff --git a/udpprofile/web/pcache-request-rate.py
b/udpprofile/web/pcache-request-rate.py
index 5173e00..6c69fd4 100755
--- a/udpprofile/web/pcache-request-rate.py
+++ b/udpprofile/web/pcache-request-rate.py
@@ -1,42 +1,43 @@
#!/usr/bin/python
-import sys, os, rrdtool, cgi, cgitb
+import cgi
+import cgitb
+import os
+import rrdtool
+import sys
cgitb.enable()
form = cgi.SvFormContentDict()
if 'period' in form:
- if form['period'].endswith( 'h' ):
- period = int( form['period'][0:-1] ) * 60
- elif form['period'].endswith( 'd' ):
- period = int( form['period'][0:-1] ) * 60 * 24
- else:
- period = int( form['period'] )
+ if form['period'].endswith('h'):
+ period = int(form['period'][0:-1]) * 60
+ elif form['period'].endswith('d'):
+ period = int(form['period'][0:-1]) * 60 * 24
+ else:
+ period = int(form['period'])
else:
- period = 30
+ period = 30
print 'Content-Type: image/png'
print 'Cache-Control: no-cache'
print
rrdFileName = '/var/lib/profile-stats-logger/stats.rrd'
-graph = [
- '-',
- '--start', '-' + str( period ) + 'm',
- '--width', '800',
- '--height', '500',
- '--vertical-label', 'Request rate (req/s)',
- '--title', 'Parser cache request rate (all wikis)',
- '--lower-limit', '0',
- 'DEF:pcache_hit=' + rrdFileName + ':pcache_hit:AVERAGE',
- 'DEF:pcache_miss_absent=' + rrdFileName + ':pcache_miss_absent:AVERAGE',
- 'DEF:pcache_miss_expired=' + rrdFileName +
':pcache_miss_expired:AVERAGE',
- 'DEF:pcache_miss_invalid=' + rrdFileName +
':pcache_miss_invalid:AVERAGE',
-
'CDEF:total=pcache_hit,pcache_miss_absent,pcache_miss_expired,pcache_miss_invalid,+,+,+',
- 'CDEF:total_avg_300=total,300,TREND',
- 'LINE:total#000080',
-]
+graph = ['-',
+ '--start', '-' + str(period) + 'm',
+ '--width', '800',
+ '--height', '500',
+ '--vertical-label', 'Request rate (req/s)',
+ '--title', 'Parser cache request rate (all wikis)',
+ '--lower-limit', '0',
+ 'DEF:pcache_hit=' + rrdFileName + ':pcache_hit:AVERAGE',
+ 'DEF:pcache_miss_absent=' + rrdFileName +
':pcache_miss_absent:AVERAGE',
+ 'DEF:pcache_miss_expired=' + rrdFileName +
':pcache_miss_expired:AVERAGE',
+ 'DEF:pcache_miss_invalid=' + rrdFileName +
':pcache_miss_invalid:AVERAGE',
+
'CDEF:total=pcache_hit,pcache_miss_absent,pcache_miss_expired,pcache_miss_invalid,+,+,+',
+ 'CDEF:total_avg_300=total,300,TREND',
+ 'LINE:total#000080']
if period < 1440:
- graph.append('LINE:total_avg_300#ff0000')
+ graph.append('LINE:total_avg_300#ff0000')
rrdtool.graph(*graph)
-
diff --git a/udpprofile/web/report.py b/udpprofile/web/report.py
index b4c69e6..f1436df 100755
--- a/udpprofile/web/report.py
+++ b/udpprofile/web/report.py
@@ -13,40 +13,40 @@
from extractprofile import SocketProfile
import cgi
-import cgitb; cgitb.enable()
+import cgitb
+cgitb.enable()
import socket
import shelve
-print "Content-type: text/html";
+print "Content-type: text/html"
print "\n"
form=cgi.SvFormContentDict()
store = shelve.open('baselines')
if "db" in form:
- db=form["db"]
+ db=form["db"]
if "sort" in form:
- sort=form["sort"]
+ sort=form["sort"]
-if "limit" in form: limit=int(form["limit"])
+if "limit" in form:
+ limit=int(form["limit"])
if "compare" in form:
- compare=form["compare"]
- compared=store[compare]
-else:
- compare=""
- compared=None
+ compare=form["compare"]
+ compared=store[compare]
+else:
+ compare=""
+ compared=None
if 'sample' not in form:
- fullprofile=SocketProfile(config.host,config.port).extract()
- sample=""
+ fullprofile=SocketProfile(config.host,config.port).extract()
+ sample=""
else:
- fullprofile=store[form['sample']]
- sample=form['sample']
-
-
+ fullprofile=store[form['sample']]
+ sample=form['sample']
events=fullprofile[db]["-"].items()
dbs=fullprofile.keys()
@@ -54,8 +54,8 @@
# Limit the scope
if compare:
- compared=compared[db]["-"]
- oldtotal=compared["-total"]
+ compared=compared[db]["-"]
+ oldtotal=compared["-total"]
#cache.close()
if sort=="name":
@@ -64,38 +64,42 @@
events.sort(lambda y,x: cmp(x[1][sort],y[1][sort]))
def surl(stype,stext=None,limit=50):
- """ Simple URL formatter for headers """
- if(stext==None): stext=stype
- if stype==sort: return """<td><b>%s</b></td>"""%stext
- return """<td><a
href='report.py?db=%s&sort=%s&limit=%d&sample=%s&compare=%s'>%s</a></td>"""%(db,stype,limit,sample,compare,stext)
+ """ Simple URL formatter for headers """
+ if (stext is None):
+ stext=stype
+ if (stype == sort):
+ return """<td><b>%s</b></td>""" % stext
+ return """<td><a
href='report.py?db=%s&sort=%s&limit=%d&sample=%s&compare=%s'>%s</a></td>""" %
(db, stype, limit, sample, compare, stext)
print """
<style>
table { width: 100%; font-size: 9pt; }
-td { cell-padding: 1px;
- text-align: right;
- vertical-align: top;
- argin: 2px;
- border: 1px silver dotted;
- white-space: nowrap;
- background-color: #eeeeee;
+td { cell-padding: 1px;
+ text-align: right;
+ vertical-align: top;
+ argin: 2px;
+ border: 1px silver dotted;
+ white-space: nowrap;
+ background-color: #eeeeee;
}
td.name { text-align: left; width: 100%; white-space: normal;}
tr.head td { text-align: center; }
</style>"""
if sample:
- print "<div>Using old sample: %s, <a href='report.py?db=%s'>reset to
current</a></div>" % (sample,db)
+ print "<div>Using old sample: %s, <a href='report.py?db=%s'>reset to
current</a></div>" % (sample,db)
# Top list of databases
for dbname in dbs:
- if db == dbname: print " [%s] "%dbname
- else: print " [<a href='report.py?db=%s'>%s</a>] "%(dbname,dbname)
+ if db == dbname:
+ print " [%s] " % dbname
+ else:
+ print " [<a href='report.py?db=%s'>%s</a>] " % (dbname, dbname)
if limit==50:
- print " [ showing %d events, <a
href='report.py?db=%s&sort=%s&sample=%s&compare=%s&limit=5000'>show more</a> ]
" % (limit,db,sort,sample,compare)
+ print " [ showing %d events, <a
href='report.py?db=%s&sort=%s&sample=%s&compare=%s&limit=5000'>show more</a> ]
" % (limit, db, sort, sample, compare)
else:
- print " [ showing %d events, <a
href='report.py?db=%s&sort=%s&sample=%s&compare=%s&limit=50'>show less</a> ] "
% (limit,db,sort,sample,compare)
+ print " [ showing %d events, <a
href='report.py?db=%s&sort=%s&sample=%s&compare=%s&limit=50'>show less</a> ] "
% (limit,db,sort,sample,compare)
print " [ <a href='admin.py'>admin</a> ]</div>"
@@ -109,7 +113,7 @@
samples=store.keys()
samples.sort()
for baseline in samples:
- print "<option%s>%s</option>" % ((compare==baseline and " SELECTED" or
""),baseline)
+ print "<option%s>%s</option>" % ((compare==baseline and " SELECTED" or
""),baseline)
print "</select><input type='submit' value='compare'></form>"
@@ -151,63 +155,66 @@
# This is really really hacky way of reporting percentages
-# And this is output of results.
+# And this is output of results.
for event in events:
- (name,event)=event
- if name=="close": continue
- if compared and name in compared: old=compared[name]
- else: old=None
-
- limit-=1
- if limit<0: break
-
- callcount=float(event["count"])/total["count"]
- cpupct=event["cpu"]/total["cpu"]
- onecpu=event["onecpu"]
- realpct=event["real"]/total["real"]
- onereal=event["onereal"]
-
- if old:
- try:
- oldcount=float(old["count"])/oldtotal["count"]
- countdiff = (callcount-oldcount)/oldcount
-
- oldcpupct = old["cpu"]/oldtotal["cpu"]
- cpupctdiff = (cpupct-oldcpupct)/oldcpupct
-
- onecpudiff = ( onecpu - old["onecpu"] ) / old["onecpu"]
-
- oldrealpct = old["real"]/oldtotal["real"]
- realpctdiff = (realpct-oldrealpct)/oldrealpct
-
- onerealdiff = ( onereal - old["onereal"] ) /
old["onereal"]
- except ZeroDivisionError:
- countdiff=0
- cpupctdiff=0
- onecpudiff=0
- realpctdiff=0
- onerealdiff=0
- else:
- countdiff=0
- cpupctdiff=0
- onecpudiff=0
- realpctdiff=0
- onerealdiff=0
-
- dbg=0
-
- if dbg and name=="wfMsgReal":
- print old
- print oldtotal
- print event
- print total
- if not dbg: print comparedformat % (
- name.replace(",",", "),
- event["count"], callcount,countdiff*100,
- cpupct*100,cpupctdiff*100,
- onecpu*1000,onecpudiff*100,
- realpct*100,realpctdiff*100,
- onereal*1000,onerealdiff*100
- )
+ (name, event) = event
+ if name=="close":
+ continue
+ if compared and name in compared:
+ old=compared[name]
+ else:
+ old=None
+
+ limit -= 1
+ if limit < 0:
+ break
+
+ callcount=float(event["count"]) / total["count"]
+ cpupct=event["cpu"] / total["cpu"]
+ onecpu=event["onecpu"]
+ realpct=event["real"] / total["real"]
+ onereal=event["onereal"]
+
+ if old:
+ try:
+ oldcount=float(old["count"]) / oldtotal["count"]
+ countdiff = (callcount - oldcount) / oldcount
+
+ oldcpupct = old["cpu"] / oldtotal["cpu"]
+ cpupctdiff = (cpupct - oldcpupct) / oldcpupct
+
+ onecpudiff = (onecpu - old["onecpu"]) / old["onecpu"]
+
+ oldrealpct = old["real"] / oldtotal["real"]
+ realpctdiff = (realpct - oldrealpct) / oldrealpct
+
+ onerealdiff = (onereal - old["onereal"]) / old["onereal"]
+ except ZeroDivisionError:
+ countdiff=0
+ cpupctdiff=0
+ onecpudiff=0
+ realpctdiff=0
+ onerealdiff=0
+ else:
+ countdiff=0
+ cpupctdiff=0
+ onecpudiff=0
+ realpctdiff=0
+ onerealdiff=0
+
+ dbg=0
+
+ if dbg and name=="wfMsgReal":
+ print old
+ print oldtotal
+ print event
+ print total
+ if not dbg:
+ print comparedformat % (name.replace(",",", "),
+ event["count"], callcount, countdiff * 100,
+ cpupct * 100, cpupctdiff * 100,
+ onecpu * 1000, onecpudiff * 100,
+ realpct * 100, realpctdiff * 100,
+ onereal * 1000, onerealdiff * 100)
print "</table>"
--
To view, visit https://gerrit.wikimedia.org/r/67893
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3b149ab467e5aae2daf26b14829ec5b6b7d50a1e
Gerrit-PatchSet: 3
Gerrit-Project: operations/software
Gerrit-Branch: master
Gerrit-Owner: Andrew Bogott <[email protected]>
Gerrit-Reviewer: Andrew Bogott <[email protected]>
Gerrit-Reviewer: Faidon <[email protected]>
Gerrit-Reviewer: Hashar <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
