Author: eudoxos Date: 2009-02-28 20:06:57 +0100 (Sat, 28 Feb 2009) New Revision: 1700
Added: trunk/examples/collider-perf/ trunk/examples/collider-perf/README trunk/examples/collider-perf/perf.py trunk/examples/collider-perf/perf.table Modified: trunk/gui/py/utils.py trunk/gui/py/yade-multi trunk/gui/py/yadeControl.cpp Log: 1. Make the yade-multi scheduler multithread-aware. 2. Add the collider performance test as example 3. added utils.replaceCollider(anotherCollider) for convenience 4. added Omega().isChildClassOf(classNameChild,classNameParent) Added: trunk/examples/collider-perf/README =================================================================== --- trunk/examples/collider-perf/README 2009-02-28 10:26:47 UTC (rev 1699) +++ trunk/examples/collider-perf/README 2009-02-28 19:06:57 UTC (rev 1700) @@ -0,0 +1,21 @@ +This example tests performance of PersistentSAPCollider and SpatialQuickSortCollider +and was basis for http://yade.wikia.com/wiki/Colliders_performace . + +To run the test, say: + + yade-trunk-multi perf.table perf.py + +and wait. If you feel brave enough, uncomment lines with many spheres in perf.table. +They take very long time (128k spheres with SAP collider over 3 hours) + + +1. File with nSpheres spheres (loose packing) is generated first, it if doesn't exist yet. + This can take a few minutes, but is done only the first time for the particular sphere + number. +2. First iteration on the scene (TriaxialTest and the selected collider) with timings is + done and timing.stats() printed (appears in the log file). +3. Another 100 iterations are measured with siming.stats(), after which the test exits. + +To get the results, grep the resulting log files, named like perf.64k.log (64k spheres with +PersistentSAPCollider), perf.32k.q.log (32k spheres, SpatialQuickSortCollider) etc. + Added: trunk/examples/collider-perf/perf.py =================================================================== --- trunk/examples/collider-perf/perf.py 2009-02-28 10:26:47 UTC (rev 1699) +++ trunk/examples/collider-perf/perf.py 2009-02-28 19:06:57 UTC (rev 1700) @@ -0,0 +1,27 @@ + +utils.readParamsFromTable(nSpheres=1000,collider='PersistentSAPCollider',noTableOk=True) +# name of file containing sphere packing with given number of spheres +spheresFile="packing-%dk.spheres"%(nSpheres/1000) + +import os +if not os.path.exists(spheresFile): + print "Generating packing" + p=Preprocessor('TriaxialTest',{'numberOfGrains':nSpheres,'radiusMean':1e-3,'lowerCorner':[0,0,0],'upperCorner':[1,1,1]}) + p.load() + utils.spheresToFile(spheresFile) + O.reset() + print "Packing %s done"%spheresFile +else: print "Packing found (%s), using it."%spheresFile + +from yade import timing +O.timingEnabled=True + +p=Preprocessor('TriaxialTest',{'importFilename':spheresFile}).load() +utils.replaceCollider(StandAloneEngine(collider)) + +O.step() +timing.stats() +timing.reset() +O.run(100,True) +timing.stats() +quit() Added: trunk/examples/collider-perf/perf.table =================================================================== --- trunk/examples/collider-perf/perf.table 2009-02-28 10:26:47 UTC (rev 1699) +++ trunk/examples/collider-perf/perf.table 2009-02-28 19:06:57 UTC (rev 1700) @@ -0,0 +1,51 @@ +description nSpheres collider +#128k 128000 'PersistentSAPCollider' +#96k 96000 'PersistentSAPCollider' +#64k 64000 'PersistentSAPCollider' +#56k 56000 'PersistentSAPCollider' +#48k 48000 'PersistentSAPCollider' +#40k 40000 'PersistentSAPCollider' +36k 36000 'PersistentSAPCollider' +32k 32000 'PersistentSAPCollider' +28k 28000 'PersistentSAPCollider' +24k 24000 'PersistentSAPCollider' +20k 20000 'PersistentSAPCollider' +18k 18000 'PersistentSAPCollider' +16k 16000 'PersistentSAPCollider' +14k 14000 'PersistentSAPCollider' +12k 12000 'PersistentSAPCollider' +10k 10000 'PersistentSAPCollider' +9k 9000 'PersistentSAPCollider' +8k 8000 'PersistentSAPCollider' +7k 7000 'PersistentSAPCollider' +6k 6000 'PersistentSAPCollider' +5k 5000 'PersistentSAPCollider' +4k 4000 'PersistentSAPCollider' +3k 3000 'PersistentSAPCollider' +2k 2000 'PersistentSAPCollider' +1k 1000 'PersistentSAPCollider' +128k.q 128000 'SpatialQuickSortCollider' +96k.q 96000 'SpatialQuickSortCollider' +64k.q 64000 'SpatialQuickSortCollider' +56k.q 56000 'SpatialQuickSortCollider' +48k.q 48000 'SpatialQuickSortCollider' +40k.q 40000 'SpatialQuickSortCollider' +36k.q 36000 'SpatialQuickSortCollider' +32k.q 32000 'SpatialQuickSortCollider' +28k.q 28000 'SpatialQuickSortCollider' +24k.q 24000 'SpatialQuickSortCollider' +20k.q 20000 'SpatialQuickSortCollider' +18k.q 18000 'SpatialQuickSortCollider' +16k.q 16000 'SpatialQuickSortCollider' +14k.q 14000 'SpatialQuickSortCollider' +12k.q 12000 'SpatialQuickSortCollider' +10k.q 10000 'SpatialQuickSortCollider' +9k.q 9000 'SpatialQuickSortCollider' +8k.q 8000 'SpatialQuickSortCollider' +7k.q 7000 'SpatialQuickSortCollider' +6k.q 6000 'SpatialQuickSortCollider' +5k.q 5000 'SpatialQuickSortCollider' +4k.q 4000 'SpatialQuickSortCollider' +3k.q 3000 'SpatialQuickSortCollider' +2k.q 2000 'SpatialQuickSortCollider' +1k.q 1000 'SpatialQuickSortCollider' Modified: trunk/gui/py/utils.py =================================================================== --- trunk/gui/py/utils.py 2009-02-28 10:26:47 UTC (rev 1699) +++ trunk/gui/py/utils.py 2009-02-28 19:06:57 UTC (rev 1700) @@ -354,10 +354,12 @@ for i in range(len(names)): if names[i]=='description': o.tags['description']=values[i] else: - if names[i] not in kw.keys() and (not unknownOk or names[i][0]=='!'): raise NameError("Parameter `%s' has no default value assigned"%names[i]) - if names[i] in kw.keys(): kw.pop(names[i]) - eq="%s=%s"%(names[i],repr(values[i])) - exec('__builtin__.%s=%s'%(names[i],values[i])); tagsParams+=['%s=%s'%(names[i],values[i])]; dictParams[names[i]]=values[i] + print 'Parameter name:',names[i],names[i][0] + if names[i] not in kw.keys(): + if (not unknownOk) and names[i][0]!='!': raise NameError("Parameter `%s' has no default value assigned"%names[i]) + else: kw.pop(names[i]) + if names[i][0]!='!': + exec('__builtin__.%s=%s'%(names[i],values[i])); tagsParams+=['%s=%s'%(names[i],values[i])]; dictParams[names[i]]=values[i] defaults=[] for k in kw.keys(): exec("__builtin__.%s=%s"%(k,repr(kw[k]))) @@ -420,3 +422,13 @@ f = DeusExMachina('PythonRunnerFilter',{'command':command,'isFilterActivated':isFilterActivated}) O.engines+=[f] return f + +def replaceCollider(colliderEngine): + """Replaces collider (BroadInteractor) engine with the engine supplied. Raises error if no collider is in engines.""" + colliderIdx=-1 + for i,e in enumerate(O.engines): + if O.isChildClassOf(e.name,"BroadInteractor"): + colliderIdx=i + break + if colliderIdx<0: raise RuntimeError("No BroadInteractor found within O.engines.") + O.engines=O.engines[:colliderIdx]+[colliderEngine]+O.engines[colliderIdx+1:] Modified: trunk/gui/py/yade-multi =================================================================== --- trunk/gui/py/yade-multi 2009-02-28 10:26:47 UTC (rev 1699) +++ trunk/gui/py/yade-multi 2009-02-28 19:06:57 UTC (rev 1700) @@ -6,9 +6,11 @@ import os, sys, thread, time class JobInfo(): - def __init__(self,num,id,command,log): + def __init__(self,num,id,command,log,nSlots): self.started,self.finished,self.duration,self.exitStatus=None,None,None,None - self.command=command; self.num=num; self.log=log; self.id=id + self.command=command; self.num=num; self.log=log; self.id=id; self.nSlots=nSlots + self.status='PENDING' + self.threadNum=None def saveInfo(self): log=file(self.log,'a') log.write(""" @@ -22,85 +24,40 @@ """%(self.id,self.exitStatus,'OK' if self.exitStatus==0 else 'FAILED',self.duration,self.command,time.asctime(time.localtime(self.started)),time.asctime(time.localtime(self.finished)))); log.close() -# -# _MANY_ thanks to Mark Pettit for concurrent jobs handling! -# http://code.activestate.com/recipes/534160/ -# -def __concurrent_batch(cmd,thread_num,completion_status_dict,exit_code_dict): - """Helper routine for 'concurrent_batches.""" - job=jobs[thread_num] +def runJob(job): + job.status='RUNNING' job.started=time.time(); - print '#%d started on %s'%(thread_num,time.asctime()) - exit_code_dict[thread_num] = os.system(cmd) - completion_status_dict[thread_num] = 1 # for sum() routine - job.finished=time.time(); dt=job.finished-job.started; - job.exitStatus=exit_code_dict[thread_num] + print '#%d (%s%s) started on %s'%(job.num,job.id,'' if job.nSlots==1 else '/%d'%job.nSlots,time.asctime()) + job.exitStatus=os.system(job.command) + job.finished=time.time() + dt=job.finished-job.started; job.duration='%02d:%02d:%02d'%(dt//3600,(dt%3600)//60,(dt%60)) strStatus='done ' if job.exitStatus==0 else 'FAILED ' - print "#%d %s (exit status %d), duration %s, log %s"%(thread_num,strStatus,exit_code_dict[thread_num],job.duration,job.log) + job.status='DONE' + print "#%d (%s%s) %s (exit status %d), duration %s, log %s"%(job.num,job.id,'' if job.nSlots==1 else '/%d'%job.nSlots,strStatus,job.exitStatus,job.duration,job.log) job.saveInfo() + +def runJobs(jobs,numSlots): + running,pending=0,len(jobs) + inf=1000000 + while (running>0) or (pending>0): + pending,running,done=sum([j.nSlots for j in jobs if j.status=='PENDING']),sum([j.nSlots for j in jobs if j.status=='RUNNING']),sum([j.nSlots for j in jobs if j.status=='DONE']) + #print [j.status for j in jobs] + freeSlots=numSlots-running + minRequire=min([inf]+[j.nSlots for j in jobs if j.status=='PENDING']) + if minRequire==inf: minRequire=0 + #print pending,'pending;',running,'running;',done,'done;',freeSlots,'free;',minRequire,'min' + if minRequire>freeSlots and running==0: + freeSlots=minRequire + for j in [j for j in jobs if j.status=='PENDING']: + if j.nSlots<=freeSlots: + thread.start_new_thread(runJob,(j,)) + break + time.sleep(.5) -def concurrent_batches(batchlist,maxjobs=0,maxtime=0): - """Run a list of batch commands simultaneously. - 'batchlist' is a list of strings suitable for submitting under os.system(). - Each job will run in a separate thread, with the thread ending when the - subprocess ends. - 'maxjobs' will, if greater then zero, be the maximum number of simultaneous - jobs which can concurrently run. This would be used to limit the number of - processes where too many could flood a system, causing performance issues. - 'maxtime', when greater than zero, be the maximum amount of time (seconds) - that we will wait for processes to complete. After that, we will return, - but no jobs will be killed. In other words, the jobs still running will - continue to run, and hopefully finish in the course of time. - - example: concurrent_batches(("gzip abc","gzip def","gzip xyz")) - - returns: a dictionary of exit status codes, but only when ALL jobs are - complete, or the maximum time has been exceeded. - Note that if returning due to exceeding time, the dictionary will - continue to be updated by the threads as they complete. - The key of the dictionary is the thread number, which matches the - index of the list of batch commands. The value is the result of - the os.system call. - - gotcha: If both the maxjobs and maxtime is set, there is a possibility that - not all jobs will be submitted. The only way to detect this will be - by checking for the absence of the KEY in the returned dictionary. - """ - - if not batchlist: return {} - completion_status_dict, exit_code_dict = {}, {} - num_jobs = len(batchlist) - start_time = time.time() - for thread_num, cmd in enumerate(batchlist): - exit_code_dict[thread_num] = None - completion_status_dict[thread_num] = 0 # for sum() routine - thread.start_new_thread(__concurrent_batch, - (cmd,thread_num,completion_status_dict,exit_code_dict)) - while True: - completed = sum(completion_status_dict.values()) - if num_jobs == completed: - return exit_code_dict # all done - running = thread_num - completed + 1 - if maxtime > 0: - if time.time() - start_time > maxtime: - return exit_code_dict - if not maxjobs: - if thread_num < num_jobs-1: # have we submitted all jobs ? - break # no, so break to for cmd loop - else: - time.sleep(.2) # yes, so wait until jobs are complete - continue - if running < maxjobs and thread_num < num_jobs-1: - break # for next for loop - time.sleep(.2) -# -# now begins the yade code -# - import sys,re,optparse def getNumCores(): return len([l for l in open('/proc/cpuinfo','r') if l.find('processor')==0]) @@ -151,16 +108,16 @@ elif l==1: print "WARNING: skipping line 1 that should contain variable labels" else: useLines+=[l] else: useLines=availableLines -print "Will use lines ",', '.join([str(i) for i in useLines])+'.' -# find column where description is try: idColumn=headings.index('description') idStrings={} for i in useLines: idStrings[i]=values[i][idColumn] # textual descripion of respective lines - print idStrings + print "Will use lines ",', '.join([str(i)+' (%s)'%idStrings[i] for i in useLines])+'.' + #print idStrings except ValueError: idColumn=None idStrings=None + print "Will use lines ",', '.join([str(i) for i in useLines])+'.' jobs=[] @@ -169,14 +126,18 @@ if idStrings: logFile=logFile.replace('@',idStrings[l]) else: logFile=logFile.replace('@',str(l)) envVars=[] + nSlots=1 for col,head in enumerate(headings): if head=='!EXEC': executable=values[l][col] + if head=='!OMP_NUM_THREADS': nSlots=int(values[l][col]) if head[0]=='!': envVars+=['%s=%s'%(head[1:],values[l][col])] - jobs.append(JobInfo(i,idStrings[l] if idStrings else '#'+str(i),'PARAM_TABLE=%s:%d %s nice -n %d %s -N PythonUI -- -n -x %s > %s 2>&1'%(table,l,' '.join(envVars),nice,executable,simul,logFile),logFile)) + if nSlots>maxJobs: print 'WARNING: job #%d wants %d slots but only %d are available'%(i,nSlots,maxJobs) + jobs.append(JobInfo(i,idStrings[l] if idStrings else '#'+str(i),'PARAM_TABLE=%s:%d %s nice -n %d %s -N PythonUI -- -n -x %s > %s 2>&1'%(table,l,' '.join(envVars),nice,executable,simul,logFile),logFile,nSlots)) print "Job summary:" for job in jobs: - print ' #%d (%s):'%(job.num,job.id),job.command + print ' #%d (%s%s):'%(job.num,job.id,'' if job.nSlots==1 else '/%d'%job.nSlots),job.command + # OK, go now -concurrent_batches([job.command for job in jobs],maxjobs=maxJobs) +runJobs(jobs,maxJobs) print 'All jobs finished, bye.' Modified: trunk/gui/py/yadeControl.cpp =================================================================== --- trunk/gui/py/yadeControl.cpp 2009-02-28 10:26:47 UTC (rev 1699) +++ trunk/gui/py/yadeControl.cpp 2009-02-28 19:06:57 UTC (rev 1700) @@ -584,6 +584,10 @@ return ret; } + bool isChildClassOf(const string& child, const string& base){ + return (Omega::instance().isInheritingFrom(child,base)); + } + pyTags tags_get(void){assertRootBody(); return pyTags(OMEGA.getRootBody());} void interactionContainer_set(string clss){ @@ -663,6 +667,7 @@ .add_property("actions",&pyOmega::actions_get) .add_property("tags",&pyOmega::tags_get) .def("childClasses",&pyOmega::listChildClasses) + .def("isChildClassOf",&pyOmega::isChildClassOf) .add_property("bodyContainer",&pyOmega::bodyContainer_get,&pyOmega::bodyContainer_set) .add_property("interactionContainer",&pyOmega::interactionContainer_get,&pyOmega::interactionContainer_set) .add_property("actionContainer",&pyOmega::physicalActionContainer_get,&pyOmega::physicalActionContainer_set) _______________________________________________ Mailing list: https://launchpad.net/~yade-dev Post to : [email protected] Unsubscribe : https://launchpad.net/~yade-dev More help : https://help.launchpad.net/ListHelp
