Here are some more complete per-stage times:
Average run time for stage 0 (over 12 iterations):
7.52713871002
Minimum run time for stage 0 : 6.72078919411
Maximum run time for stage 0 : 11.6402471066
Average run time for stage 1 (over 12 iterations): 8.2796501716
Minimum run time for stage 1 : 7.9734389782
Maximum run time for stage 1 : 9.64906215668
Average run time for stage 2 (over 12 iterations):
4.51219139496
Minimum run time for stage 2 : 4.09449696541
Maximum run time for stage 2 : 6.57607793808
Average run time for stage 3 (over 11 iterations):
69.0876322876
Minimum run time for stage 3 : 68.3255240917
Maximum run time for stage 3 : 70.5667920113
Average run time for stage 4 (over 11 iterations):
60.2609413104
Minimum run time for stage 4 : 59.3143589497
Maximum run time for stage 4 : 62.1066789627
Average run time for stage 5 (over 11 iterations):
3.23239835826
Minimum run time for stage 5 : 2.98085188866
Maximum run time for stage 5 : 3.33371901512
#!/usr/bin/env python
"""This program is adapted from getIOTime to (hopefully) read
a log file for main.system.start events and main.system.end events
in the stageslices and get per-stage, per-slice start and end times,
then average the start and end times until we get the actual average
time per stage.
"""
from gov.lbl.dsd.netlogger import nllite
import os
import sys
inputfile = sys.argv[1]
class StageIterationInfo:
def __init__(self):
self.startTime = None
self.endTime = None
startEvent = "stageleader.main.MPI_Barrier2.start"
endEvent = "stageleader.main.MPI_Barrier2.end"
def setStartTime(info, value):
if info.startTime != None:
print "Warning: overwriting data..."
info.startTime = value
def setEndTime(info, value):
if info.endTime != None:
print "Warning: overwriting data..."
info.endTime = value
stringToFieldSetterMap = {startEvent:setStartTime,
endEvent:setEndTime }
InfoDict = { }
# InfoDict [ stage ] [iteration ] --> StageIterationInfo.
# (after we've populated it, anyway).
if __name__ == "__main__":
myLIS = nllite.LogInputStream(inputfile)
Done = False
rec = myLIS.read()
while not Done:
if rec['EVNT'] in [startEvent, endEvent]:
curStage = rec['STAGENUM']
curIteration = rec['ITERATION']
p = InfoDict
if curStage not in p.keys():
p[curStage] = { }
p = p[curStage]
if curIteration not in p.keys():
p[curIteration] = StageIterationInfo()
p = p[curIteration]
# slightly obfuscated (but highly reusable)
# way of setting the start or end time
# of p (which is now an Info object with start and end times)
# to start or end, depending
# on whether this is a startEvent or endEvent.
stringToFieldSetterMap[rec['EVNT']](p, float(rec['DATE']))
try:
rec = myLIS.read()
except:
Done = True
if rec == None:
Done = True
for k in InfoDict.keys():
minimumTime = 999999999999999.9 #hack, but good enough for us!
maximumTime = 0.0
totalTime = 0
totalIterations = 0
for j in InfoDict[k].keys():
#average runtimes across all slices for this file
if None not in [InfoDict[k][j].endTime, InfoDict[k][j].startTime]:
thisIterationTime = InfoDict[k][j].endTime - InfoDict[k][j].startTime
totalTime += thisIterationTime
if thisIterationTime < minimumTime:
minimumTime = thisIterationTime
if thisIterationTime > maximumTime:
maximumTime = thisIterationTime
totalIterations += 1
averageTime = totalTime / totalIterations
print "Average run time for stage\t", k, " (over ", totalIterations, "iterations):\t", averageTime
print "\tMinimum run time for stage\t", k, ":\t", minimumTime
print "\tMaximum run time for stage\t", k, ":\t", maximumTime
_______________________________________________
LSST-data mailing list
[email protected]
http://www.lsstmail.org/mailman/listinfo/lsst-data
