[Zope-Checkins] SVN: Zope/trunk/utilities/tracelog.py Added url summary statistics.

2006-07-18 Thread Jim Fulton
Log message for revision 41497:
  Added url summary statistics.
  
  Added HTML output that makes it easy to jump from a URL shown in a
  minute detail to the statistics for that URL.
  

Changed:
  U   Zope/trunk/utilities/tracelog.py

-=-
Modified: Zope/trunk/utilities/tracelog.py
===
--- Zope/trunk/utilities/tracelog.py2006-01-30 16:18:33 UTC (rev 41496)
+++ Zope/trunk/utilities/tracelog.py2006-01-30 16:31:04 UTC (rev 41497)
@@ -11,15 +11,15 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##
-Yet another trace log analysis tool
+Yet another lag analysis tool
 
+
+
 $Id$
 
 
 import datetime, optparse, sys
 
-
-
 class Request:
 
 output_bytes = '-'
@@ -52,6 +52,60 @@
 def total_seconds(self):
 return (self.end - self.start).seconds
 
+class Times:
+
+tid = 1l
+
+def __init__(self):
+self.times = []
+self.hangs = 0
+Times.tid += 1
+self.tid = Times.tid # generate a unique id
+
+def finished(self, request):
+self.times.append(request.app_seconds)
+
+def hung(self):
+self.hangs += 1
+
+def impact(self):
+times = self.times
+if not times:
+self.median = self.mean = self.impact = 0
+return 0
+self.times.sort()
+n = len(times) 
+if n % 2:
+m = times[(n+1)/2-1]
+else:
+m = .5 * (times[n/2]+times[n/2-1])
+self.median = m
+self.mean = sum(times)/n
+self.impact = self.mean * (n+self.hangs)
+return self.impact
+
+def __str__(self):
+times = self.times
+if not times:
+return   0 %5d % (
+self.hangs)
+
+n = len(times)
+m = self.median
+return %9.1f %5d %6.0f %6.2f %6.2f %6.0f %5d % (
+self.impact, n, times[0], m, self.mean, times[-1], self.hangs)
+
+def html(self):
+times = self.times
+if not times:
+print td('', 0, '', '', '', '', self.hangs)
+else:
+n = len(times)
+m = self.median
+impact = 'a name=u%s%s' % (self.tid, self.impact)
+print td(impact, n, times[0], m, self.mean, times[-1],
+ self.hangs)
+
 def parsedt(s):
 date, time = s.split('T')
 return datetime.datetime(*(
@@ -69,15 +123,33 @@
 restarts = find_restarts(options.event_log)
 else:
 restarts = []
-
+
 restarts.append(datetime.datetime.utcnow()+datetime.timedelta(1000))
 
+if options.html:
+print_app_requests = print_app_requests_html
+output_minute = output_minute_html
+output_stats = output_stats_html
+minutes_header = minutes_header_html
+minutes_footer = minutes_footer_html
+print 'html title=trace log statisticsbody'
+else:
+print_app_requests = print_app_requests_text
+output_minute = output_minute_text
+output_stats = output_stats_text
+minutes_header = minutes_header_text
+minutes_footer = minutes_footer_text
+
+
+urls = {}
 [file] = args
 lmin = ldt = None
 requests = {}
 input = apps = output = n = 0
 spr = spa = 0.0
 restart = restarts.pop(0)
+minutes_header()
+remove_prefix = options.remove_prefix
 for record in open(file):
 record = record.split()
 typ, rid, dt = record[:3]
@@ -89,7 +161,9 @@
 print_app_requests(requests, ldt,
options.old_requests,
options.app_requests,
+   urls,
\nLeft over:)
+record_hung(urls, requests)
 requests = {}
 input = apps = output = n = 0
 spr = spa = 0.0
@@ -98,18 +172,12 @@
 
 if min != lmin:
 if lmin is not None:
-
-print lmin.replace('T', ' '), %4d I=%3d A=%3d O=%3d  % (
-len(requests), input, apps, output),
-if n:
-print N=%4d %10.2f %10.2f % (n, spr/n, spa/n)
-else:
-print
-
+output_minute(lmin, requests, input, apps, output, n, spr, spa)
 if apps  options.apps:
 print_app_requests(requests, dt,
options.old_requests,
options.app_requests,
+   urls,
)
 lmin = min
 spr = 0.0
@@ -117,8 +185,23 @@
 n = 0
 
 if typ == 'B':
+if rid in requests:
+request = requests[rid]
+if request.state == 

[Zope-Checkins] SVN: Zope/trunk/utilities/tracelog.py Added url summary statistics.

2006-01-30 Thread Jim Fulton
Log message for revision 41497:
  Added url summary statistics.
  
  Added HTML output that makes it easy to jump from a URL shown in a
  minute detail to the statistics for that URL.
  

Changed:
  U   Zope/trunk/utilities/tracelog.py

-=-
Modified: Zope/trunk/utilities/tracelog.py
===
--- Zope/trunk/utilities/tracelog.py2006-01-30 16:18:33 UTC (rev 41496)
+++ Zope/trunk/utilities/tracelog.py2006-01-30 16:31:04 UTC (rev 41497)
@@ -11,15 +11,15 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##
-Yet another trace log analysis tool
+Yet another lag analysis tool
 
+
+
 $Id$
 
 
 import datetime, optparse, sys
 
-
-
 class Request:
 
 output_bytes = '-'
@@ -52,6 +52,60 @@
 def total_seconds(self):
 return (self.end - self.start).seconds
 
+class Times:
+
+tid = 1l
+
+def __init__(self):
+self.times = []
+self.hangs = 0
+Times.tid += 1
+self.tid = Times.tid # generate a unique id
+
+def finished(self, request):
+self.times.append(request.app_seconds)
+
+def hung(self):
+self.hangs += 1
+
+def impact(self):
+times = self.times
+if not times:
+self.median = self.mean = self.impact = 0
+return 0
+self.times.sort()
+n = len(times) 
+if n % 2:
+m = times[(n+1)/2-1]
+else:
+m = .5 * (times[n/2]+times[n/2-1])
+self.median = m
+self.mean = sum(times)/n
+self.impact = self.mean * (n+self.hangs)
+return self.impact
+
+def __str__(self):
+times = self.times
+if not times:
+return   0 %5d % (
+self.hangs)
+
+n = len(times)
+m = self.median
+return %9.1f %5d %6.0f %6.2f %6.2f %6.0f %5d % (
+self.impact, n, times[0], m, self.mean, times[-1], self.hangs)
+
+def html(self):
+times = self.times
+if not times:
+print td('', 0, '', '', '', '', self.hangs)
+else:
+n = len(times)
+m = self.median
+impact = 'a name=u%s%s' % (self.tid, self.impact)
+print td(impact, n, times[0], m, self.mean, times[-1],
+ self.hangs)
+
 def parsedt(s):
 date, time = s.split('T')
 return datetime.datetime(*(
@@ -69,15 +123,33 @@
 restarts = find_restarts(options.event_log)
 else:
 restarts = []
-
+
 restarts.append(datetime.datetime.utcnow()+datetime.timedelta(1000))
 
+if options.html:
+print_app_requests = print_app_requests_html
+output_minute = output_minute_html
+output_stats = output_stats_html
+minutes_header = minutes_header_html
+minutes_footer = minutes_footer_html
+print 'html title=trace log statisticsbody'
+else:
+print_app_requests = print_app_requests_text
+output_minute = output_minute_text
+output_stats = output_stats_text
+minutes_header = minutes_header_text
+minutes_footer = minutes_footer_text
+
+
+urls = {}
 [file] = args
 lmin = ldt = None
 requests = {}
 input = apps = output = n = 0
 spr = spa = 0.0
 restart = restarts.pop(0)
+minutes_header()
+remove_prefix = options.remove_prefix
 for record in open(file):
 record = record.split()
 typ, rid, dt = record[:3]
@@ -89,7 +161,9 @@
 print_app_requests(requests, ldt,
options.old_requests,
options.app_requests,
+   urls,
\nLeft over:)
+record_hung(urls, requests)
 requests = {}
 input = apps = output = n = 0
 spr = spa = 0.0
@@ -98,18 +172,12 @@
 
 if min != lmin:
 if lmin is not None:
-
-print lmin.replace('T', ' '), %4d I=%3d A=%3d O=%3d  % (
-len(requests), input, apps, output),
-if n:
-print N=%4d %10.2f %10.2f % (n, spr/n, spa/n)
-else:
-print
-
+output_minute(lmin, requests, input, apps, output, n, spr, spa)
 if apps  options.apps:
 print_app_requests(requests, dt,
options.old_requests,
options.app_requests,
+   urls,
)
 lmin = min
 spr = 0.0
@@ -117,8 +185,23 @@
 n = 0
 
 if typ == 'B':
+if rid in requests:
+request = requests[rid]
+if request.state ==