Many thanks. Find it attached.
It's a tiny http server. People use a web page to send an HTTP request to
it to kick off builds.
On Wed, Aug 10, 2016 at 1:55 PM, Pierre Tardy <[email protected]> wrote:
> I'll take a look.
>
> Le mer. 10 août 2016 12:45, Francesco Di Mizio <[email protected]>
> a écrit :
>
>> I am starting to think it may be due to my own custom scheduler. If I
>> uploaded the code somewhere could anybody take a look? it's a 150 lines py
>> script.
>>
>> On Wed, Aug 10, 2016 at 12:10 PM, Francesco Di Mizio <
>> [email protected]> wrote:
>>
>>> Thanks for that Pierre. I have it on on production and the performances
>>> do not degrade indeed.
>>>
>>> buildbot@e2123a50d1fb:~$ cat profile.txt
>>> No samples recorded.
>>>
>>> I need to study statprof a bit now ;)
>>>
>>> On Wed, Aug 10, 2016 at 11:20 AM, Vasily <[email protected]> wrote:
>>>
>>>> One can also use Intel® VTune™, you know... :-)
>>>>
>>>> Thanks,
>>>> Vasily
>>>> 09 авг. 2016 г. 23:56 пользователь "Pierre Tardy" <[email protected]>
>>>> написал:
>>>>
>>>>> You can add following snippet to your master.cfg. It looks statprof
>>>>> does not work on osx, so I cannot test it to the end, but I know similar
>>>>> did work on my prod
>>>>>
>>>>>
>>>>> import statprof
>>>>> from twisted.application.internet import TimerService
>>>>> from buildbot.util.service import BuildbotService
>>>>> class BuildbotTimerService(TimerService, BuildbotService):
>>>>> name = "timer"
>>>>> def __init__(self):
>>>>> BuildbotService.__init__(self)
>>>>> TimerService.__init__(self, 10, self.dump_stats)
>>>>> def dump_stats(self):
>>>>> statprof.stop()
>>>>> with open("profile.txt", "w") as f:
>>>>> statprof.display(f)
>>>>> statprof.start()
>>>>>
>>>>> c['services'] = [ BuildbotTimerService() ]
>>>>>
>>>>>
>>>>> Le mar. 9 août 2016 à 21:44, Francesco Di Mizio <
>>>>> [email protected]> a écrit :
>>>>>
>>>>>> Pierre,
>>>>>>
>>>>>> if you can enlighten on how you intend to use statprof, I will be
>>>>>> happy to give it a try.
>>>>>> This problem here is on top of my list right now as it's making my
>>>>>> prod env unusable.
>>>>>>
>>>>>> On Tue, Aug 9, 2016 at 8:25 PM, Pierre Tardy <[email protected]>
>>>>>> wrote:
>>>>>>
>>>>>>> Please note that cprofile is very intrusive and will lead to about
>>>>>>> 3x slower code.
>>>>>>>
>>>>>>> So I would not run that in prod. This is why I prefer statprof which
>>>>>>> is using a non intrusive method to get the stats
>>>>>>>
>>>>>>> Le mar. 9 août 2016 19:15, Francesco Di Mizio <
>>>>>>> [email protected]> a écrit :
>>>>>>>
>>>>>>>> On a side note I have managed to get twistd to dump a blob I can
>>>>>>>> then load and analyze. Got to see if I can run this in production.
>>>>>>>> If anybody is curious the following worked for me:
>>>>>>>> twistd --savestats -n --profiler=cprofile --profile=/crcdata/
>>>>>>>> profile.stats -y ./buildbot.tac
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> On Tue, Aug 9, 2016 at 5:21 PM, Francesco Di Mizio <
>>>>>>>> [email protected]> wrote:
>>>>>>>>
>>>>>>>>> Well that's expected to an extent - p4 poller is running 'p4
>>>>>>>>> changes' every pollinterval seconds.
>>>>>>>>>
>>>>>>>>> Anyway just as a test I've tried to disable the poller and still
>>>>>>>>> at times I am seeing the CPU spiking up to above 100%. I believe I
>>>>>>>>> really
>>>>>>>>> need to profile this somehow.
>>>>>>>>>
>>>>>>>>> On Tue, Aug 9, 2016 at 5:08 PM, Dan Kegel <[email protected]> wrote:
>>>>>>>>>
>>>>>>>>>> Also watch 'top' and see if poller processes are hogging
>>>>>>>>>> resources.
>>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>
>>>>>>
>>>>> _______________________________________________
>>>>> users mailing list
>>>>> [email protected]
>>>>> https://lists.buildbot.net/mailman/listinfo/users
>>>>>
>>>>
>>>
>>
from buildbot.plugins import schedulers, util, steps
from buildbot.schedulers.base import BaseScheduler
from buildbot.schedulers.trysched import TryBase
from buildbot.process.properties import Properties
from buildbot.util import ascii2unicode
from twisted.web import http
from twisted.web import server
from twisted.internet import defer
from twisted.application import strports, service
from twisted.internet.protocol import Protocol
from twisted.python import log
from copy import deepcopy
import json
from . import config
class TryJobHTTPRequest(http.Request):
def __init__(self, channel, queued):
http.Request.__init__(self, channel, queued)
@defer.inlineCallbacks
def process(self):
ret = None
log.msg("Received URI: %s" % self.uri)
try:
# Support only one URI for now.
if self.uri.find('send_try_patch') == -1:
log.msg("Received invalid URI: %s" % self.uri)
self.code = http.NOT_FOUND
return
print 'method %s' %self.method
options = dict((k, v) for k, v in self.args.iteritems() if v)
bsid, brids = yield
self.channel.factory.parent.messageReceived(options)
self.code = 200
except Exception as e:
log.msg(str(e))
finally:
self.code_message = http.RESPONSES[self.code]
dict_to_ret = deepcopy(brids)
dict_to_ret['bsid'] = bsid
self.write(json.dumps(dict_to_ret))
log.msg(json.dumps(dict_to_ret))
self.finish()
class TryJobHTTP(TryBase, service.MultiService):
"""Opens a HTTP port to accept patch files and to execute these on the try
server."""
_PROPERTY_SOURCE = 'Try job'
def __init__(self, port, name, builderNames, properties):
TryBase.__init__(self, name, builderNames, properties)
service.MultiService.__init__(self)
if type(port) is int:
port = "tcp:%d" % port
self.port = port
f = http.HTTPFactory()
f.protocol.requestFactory = TryJobHTTPRequest
f.parent = self
s = strports.service(port, f)
s.setServiceParent(self)
log.msg('TryJobHTTP listening on port %s' % self.port)
# We stop listening to incoming connections
# When running reconfig this is not quick enough. Consider using a delay
(either here or on start up)
# Right now it needs configure twice
def stopService(self):
service.MultiService.stopService(self)
self.services[0].disownServiceParent()
# Here we set all the properties not related to the sourcestamp
def get_props(self, options, reason):
properties = {
'branch' : options['branch'][0],
#'git_branch' : 'master',
'shelf' : options['p4_cl'][0],
'owner' : options['email'][0],
'email' : options['email'][0],
'reason' : reason,
'stream' : 'starcitizen' #maybe work this out in the future,
should we change stream name
}
try:
if (options['sync_mode'][0]):
# sync_mode defaults to 'incremental' in the p4 step
# so we only set it to 'full' if it's been passed by the
try_page with its checkbox
properties['sync_mode'] = 'full'
except:
pass
props = Properties()
props.updateFromProperties(self.properties)
props.update(properties, self._PROPERTY_SOURCE)
return props
@defer.inlineCallbacks
def messageReceived(self, options):
who = options['who'][0]
p4_cl = options['p4_cl'][0]
branch = options['branch'][0]
# we dont need to add a Change. This would be nice coz would allow us
to see the changed files in the build page.
# However it would require a scheduler (probably anybranch scedulers)
for each builder.
# what we have now is more simple: this http scheduler adding buildsets
only with requested build requests
# change_id = yield self.master.db.changes.addChange(
# author=options['email'][0],
# comments='trybuild for %s for CL number
%s' %(who, str(p4_cl)),
# branch = branch
# )
# log.msg('pushed change_id %s' % change_id)
# We set Revision to None because we want to compile against latest
revision = None
# changes = change_id
patch_body = options['patch'][0]
job_id = options['job_id'][0]
email = options['email'][0]
starcitizen_sourcestamp = dict(branch=branch,
codebase='starcitizen',
# revision=revision,
# patch_body=patch_body,
# patch_level=1,
# patch_author=email, #We do this so
Mailnotifier will just use this (see Domain class in master.cfg)
# patch_comment='patch for shelf %s by %s'
%(p4_cl, who),
# patch_subdir='',
project='',
repository=''
)
github_sourcestamp = dict(branch=config.git_branch,
codebase='github',
project='' ,
repository=''
)
reason = u"try job"
if who:
reason += u" by user %s for CL %s" % (ascii2unicode(who), p4_cl)
requested_builder_names = options['builder_names'][0].split(',')
builder_names = TryBase.filterBuilderList(self, requested_builder_names)
if not builder_names:
log.msg("incoming Try job did specify at least one builder that
does not exist")
return
bsid, brids = yield self.addBuildsetForSourceStamps(
sourcestamps=[
starcitizen_sourcestamp,
github_sourcestamp
],
reason=reason,
external_idstring=ascii2unicode(job_id),
builderNames=builder_names,
properties=self.get_props(options, reason))
defer.returnValue((bsid, brids))_______________________________________________
users mailing list
[email protected]
https://lists.buildbot.net/mailman/listinfo/users
