fair enough. -b
On Fri, Feb 19, 2016 at 9:07 AM, Smith, Elliot <[email protected]> wrote: > On 19 February 2016 at 16:54, Brian Avery <[email protected]> wrote: >> >> I wouldn't remove the ParseStarted event from the event list. Earlier >> (before you or I worked on it) toaster would show transitions in the >> progress bar before the build started so that users got feedback. >> Things like >> 1) checking out layers >> 2) parsing >> 3) then the build % > > > I agree, but I'd argue for putting that event back into the mask when we > actually need it. > > Also, that event doesn't consistently fire for a build (if I remember my > testing correctly), so I wouldn't want to rely on it for anything. > > We could actually use TargetsAcquired, as that marks the start of the build > accurately and consistently. > > Elliot > >> >> >> It would be nice at some point to restore this and we'd need >> Parsestarted for that. >> >> -b >> >> On Fri, Feb 19, 2016 at 4:17 AM, Elliot Smith <[email protected]> >> wrote: >> > If a build fails because of a bitbake error occurring before the >> > BuildStarted event fires, we do not generate a Build object >> > for command-line builds. This means that failed command-line builds >> > don't appear in Toaster at all. >> > >> > To resolve, split build creation into two steps: >> > >> > 1. Just before buildTargets() is invoked on the XMLRPC server: create >> > the base Build object. Note that as soon as a Toaster-triggered >> > build starts, targets are added to it; but this event is the earliest >> > point when task and targets are available for command-line builds. >> > (This requires a new TargetsAcquired event to be fired by the XMLRPC >> > server when the buildTargets() command is called.) >> > >> > 2. BuildStarted event: add any layer information to either type of build >> > (command-line or Toaster-triggered). >> > >> > Note that the build_name property cannot be set until BuildStarted, >> > as it is not available until then, which could cause problems >> > for creating Build objects earlier; however, this property is >> > redundant, as it's never used anywhere in Toaster, so it has been >> > removed (along with any functions which refer to it). >> > >> > [YOCTO #8440] >> > >> > Signed-off-by: Elliot Smith <[email protected]> >> > --- >> > bitbake/lib/bb/event.py | 7 +++ >> > bitbake/lib/bb/server/xmlrpc.py | 3 + >> > bitbake/lib/bb/ui/buildinfohelper.py | 73 >> > ++++++++++------------ >> > bitbake/lib/bb/ui/toasterui.py | 21 ++++--- >> > .../orm/migrations/0005_remove_build_build_name.py | 18 ++++++ >> > bitbake/lib/toaster/orm/models.py | 1 - >> > .../fixtures/toastergui-unittest-data.xml | 4 -- >> > 7 files changed, 73 insertions(+), 54 deletions(-) >> > create mode 100644 >> > bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py >> > >> > diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py >> > index 5ffe89e..e94cdb8 100644 >> > --- a/bitbake/lib/bb/event.py >> > +++ b/bitbake/lib/bb/event.py >> > @@ -444,6 +444,13 @@ class MultipleProviders(Event): >> > """ >> > return self._candidates >> > >> > +class TargetsAcquired(Event): >> > + """Target we are aiming at known""" >> > + def __init__(self, task, targetsList): >> > + Event.__init__(self) >> > + self.task = task >> > + self.targetsList = targetsList >> > + >> > class ParseStarted(OperationStarted): >> > """Recipe parsing for the runqueue has begun""" >> > def __init__(self, total): >> > diff --git a/bitbake/lib/bb/server/xmlrpc.py >> > b/bitbake/lib/bb/server/xmlrpc.py >> > index 1ceca51..5f735ab 100644 >> > --- a/bitbake/lib/bb/server/xmlrpc.py >> > +++ b/bitbake/lib/bb/server/xmlrpc.py >> > @@ -112,6 +112,9 @@ class BitBakeServerCommands(): >> > """ >> > Run a cooker command on the server >> > """ >> > + if 'buildTargets' in command[0]: >> > + bb.event.fire(bb.event.TargetsAcquired(command[2], >> > command[1]), self.server.readonly) >> > + >> > return self.cooker.command.runCommand(command, >> > self.server.readonly) >> > >> > def getEventHandle(self): >> > diff --git a/bitbake/lib/bb/ui/buildinfohelper.py >> > b/bitbake/lib/bb/ui/buildinfohelper.py >> > index 81abede..46380fa 100644 >> > --- a/bitbake/lib/bb/ui/buildinfohelper.py >> > +++ b/bitbake/lib/bb/ui/buildinfohelper.py >> > @@ -127,13 +127,12 @@ class ORMWrapper(object): >> > # pylint: disable=bad-continuation >> > # we do not follow the python conventions for continuation >> > indentation due to long lines here >> > >> > - def create_build_object(self, build_info, brbe, project_id): >> > + def create_build_object(self, build_info, brbe, project_id = None): >> > assert 'machine' in build_info >> > assert 'distro' in build_info >> > assert 'distro_version' in build_info >> > assert 'started_on' in build_info >> > assert 'cooker_log_path' in build_info >> > - assert 'build_name' in build_info >> > assert 'bitbake_version' in build_info >> > >> > prj = None >> > @@ -161,7 +160,6 @@ class ORMWrapper(object): >> > build.distro=build_info['distro'] >> > build.distro_version=build_info['distro_version'] >> > build.cooker_log_path=build_info['cooker_log_path'] >> > - build.build_name=build_info['build_name'] >> > build.bitbake_version=build_info['bitbake_version'] >> > build.save() >> > >> > @@ -174,7 +172,6 @@ class ORMWrapper(object): >> > >> > started_on=build_info['started_on'], >> > >> > completed_on=build_info['started_on'], >> > >> > cooker_log_path=build_info['cooker_log_path'], >> > - >> > build_name=build_info['build_name'], >> > >> > bitbake_version=build_info['bitbake_version']) >> > >> > logger.debug(1, "buildinfohelper: build is created %s" % build) >> > @@ -854,7 +851,6 @@ class BuildInfoHelper(object): >> > build_info['started_on'] = timezone.now() >> > build_info['completed_on'] = timezone.now() >> > build_info['cooker_log_path'] = build_log_path >> > - build_info['build_name'] = >> > self.server.runCommand(["getVariable", "BUILDNAME"])[0] >> > build_info['bitbake_version'] = >> > self.server.runCommand(["getVariable", "BB_VERSION"])[0] >> > build_info['project'] = self.project = >> > self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0] >> > return build_info >> > @@ -919,26 +915,6 @@ class BuildInfoHelper(object): >> > >> > return recipe_info >> > >> > - def _get_path_information(self, task_object): >> > - assert isinstance(task_object, Task) >> > - build_stats_format = >> > "{tmpdir}/buildstats/{buildname}/{package}/" >> > - build_stats_path = [] >> > - >> > - for t in self.internal_state['targets']: >> > - buildname = self.internal_state['build'].build_name >> > - pe, pv = task_object.recipe.version.split(":",1) >> > - if len(pe) > 0: >> > - package = task_object.recipe.name + "-" + pe + "_" + pv >> > - else: >> > - package = task_object.recipe.name + "-" + pv >> > - >> > - >> > build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, >> > - >> > buildname=buildname, >> > - package=package)) >> > - >> > - return build_stats_path >> > - >> > - >> > ################################ >> > ## external available methods to store information >> > @staticmethod >> > @@ -962,17 +938,43 @@ class BuildInfoHelper(object): >> > except NotExisting as nee: >> > logger.warn("buildinfohelper: cannot identify layer >> > exception:%s ", nee) >> > >> > - >> > - def store_started_build(self, event, build_log_path): >> > - assert '_pkgs' in vars(event) >> > + def store_new_build(self, build_log_path): >> > + """ >> > + create a skeletal build object (or retrieve an existing one) as >> > soon as >> > + bitbake starts trying to do the build; we use the >> > buildTargets() >> > + command on the XMLRPC server as the indicator of a build start >> > + """ >> > build_information = self._get_build_information(build_log_path) >> > + self.internal_state['build'] = >> > self.orm_wrapper.create_build_object(build_information, self.brbe) >> > >> > - # Update brbe and project as they can be changed for every >> > build >> > - self.project = build_information['project'] >> > + def store_targets(self, event): >> > + """ >> > + store targets for the current build, if that build was started >> > from >> > + the command line; targets for non-cli builds are irrelevant, as >> > we >> > + create them from the BuildRequest anyway >> > + >> > + event: a TargetsAcquired event with a task property (e.g. >> > "build") >> > + and a targetsList property (e.g. ["zlib", "dropbear"]) >> > + """ >> > + if self.internal_state['build'].project.is_default: >> > + targets = map(lambda target: target + ':' + event.task, >> > event.targetsList) >> > + >> > + target_information = { >> > + 'targets': targets, >> > + 'build': self.internal_state['build'] >> > + } >> > + >> > + self.internal_state['targets'] = >> > self.orm_wrapper.get_or_create_targets(target_information) >> > >> > - build_obj = >> > self.orm_wrapper.create_build_object(build_information, self.brbe, >> > self.project) >> > + def update_build(self, event): >> > + """ >> > + update the current build with layer and config data once it >> > + actually starts >> > + >> > + event: a BuildStarted event >> > + """ >> > >> > - self.internal_state['build'] = build_obj >> > + build_obj = self.internal_state['build'] >> > >> > # save layer version information for this build >> > if not 'lvs' in self.internal_state: >> > @@ -983,13 +985,6 @@ class BuildInfoHelper(object): >> > >> > del self.internal_state['lvs'] >> > >> > - # create target information >> > - target_information = {} >> > - target_information['targets'] = event._pkgs >> > - target_information['build'] = build_obj >> > - >> > - self.internal_state['targets'] = >> > self.orm_wrapper.get_or_create_targets(target_information) >> > - >> > # Save build configuration >> > data = self.server.runCommand(["getAllKeysWithFlags", ["doc", >> > "func"]])[0] >> > >> > diff --git a/bitbake/lib/bb/ui/toasterui.py >> > b/bitbake/lib/bb/ui/toasterui.py >> > index 32b1889..1584872 100644 >> > --- a/bitbake/lib/bb/ui/toasterui.py >> > +++ b/bitbake/lib/bb/ui/toasterui.py >> > @@ -93,14 +93,14 @@ def _close_build_log(build_log): >> > logger.removeHandler(build_log) >> > >> > _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", >> > "logging.LogRecord", >> > - "bb.build.TaskFailed", "bb.build.TaskBase", >> > "bb.event.ParseStarted", >> > + "bb.build.TaskFailed", "bb.build.TaskBase", >> > "bb.event.ParseProgress", "bb.event.ParseCompleted", >> > "bb.event.CacheLoadStarted", >> > "bb.event.CacheLoadProgress", >> > "bb.event.CacheLoadCompleted", "bb.command.CommandFailed", >> > "bb.command.CommandExit", "bb.command.CommandCompleted", >> > "bb.cooker.CookerExit", >> > "bb.event.MultipleProviders", "bb.event.NoProvider", >> > "bb.runqueue.sceneQueueTaskStarted", >> > "bb.runqueue.runQueueTaskStarted", >> > "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed", >> > "bb.event.BuildBase", "bb.build.TaskStarted", >> > "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent", >> > - "bb.event.MetadataEvent"] >> > + "bb.event.MetadataEvent", "bb.event.TargetsAcquired"] >> > >> > def main(server, eventHandler, params): >> > # set to a logging.FileHandler instance when a build starts; >> > @@ -186,19 +186,20 @@ def main(server, eventHandler, params): >> > # pylint: disable=protected-access >> > # the code will look into the protected variables of the >> > event; no easy way around this >> > >> > - # we treat ParseStarted as the first event of >> > toaster-triggered >> > - # builds; that way we get the Build Configuration included >> > in the log >> > - # and any errors that occur before BuildStarted is fired >> > - if isinstance(event, bb.event.ParseStarted): >> > + # start of build: this event is fired just before the >> > buildTargets() >> > + # command is invoked on the XMLRPC server >> > + if isinstance(event, bb.event.TargetsAcquired): >> > if not (build_log and build_log_file_path): >> > build_log, build_log_file_path = >> > _open_build_log(log_dir) >> > + buildinfohelper.store_new_build(build_log_file_path) >> > + buildinfohelper.store_targets(event) >> > continue >> > >> > + # when the build proper starts, we extract information >> > about >> > + # any layers and config data >> > if isinstance(event, bb.event.BuildStarted): >> > - if not (build_log and build_log_file_path): >> > - build_log, build_log_file_path = >> > _open_build_log(log_dir) >> > - >> > - buildinfohelper.store_started_build(event, >> > build_log_file_path) >> > + buildinfohelper.update_build(event) >> > + continue >> > >> > if isinstance(event, (bb.build.TaskStarted, >> > bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): >> > buildinfohelper.update_and_store_task(event) >> > diff --git >> > a/bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py >> > b/bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py >> > new file mode 100644 >> > index 0000000..7159793 >> > --- /dev/null >> > +++ b/bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py >> > @@ -0,0 +1,18 @@ >> > +# -*- coding: utf-8 -*- >> > +from __future__ import unicode_literals >> > + >> > +from django.db import migrations, models >> > + >> > + >> > +class Migration(migrations.Migration): >> > + >> > + dependencies = [ >> > + ('orm', '0004_provides'), >> > + ] >> > + >> > + operations = [ >> > + migrations.RemoveField( >> > + model_name='build', >> > + name='build_name', >> > + ), >> > + ] >> > diff --git a/bitbake/lib/toaster/orm/models.py >> > b/bitbake/lib/toaster/orm/models.py >> > index ab6940f..39eb2b1 100644 >> > --- a/bitbake/lib/toaster/orm/models.py >> > +++ b/bitbake/lib/toaster/orm/models.py >> > @@ -370,7 +370,6 @@ class Build(models.Model): >> > completed_on = models.DateTimeField() >> > outcome = models.IntegerField(choices=BUILD_OUTCOME, >> > default=IN_PROGRESS) >> > cooker_log_path = models.CharField(max_length=500) >> > - build_name = models.CharField(max_length=100) >> > bitbake_version = models.CharField(max_length=50) >> > >> > @staticmethod >> > diff --git >> > a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml >> > b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml >> > index 2d83ff8..a554e62 100644 >> > --- >> > a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml >> > +++ >> > b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml >> > @@ -39,7 +39,6 @@ >> > <field type="DateTimeField" >> > name="completed_on">2016-02-14T18:46:20.114530+00:00</field> >> > <field type="IntegerField" name="outcome">0</field> >> > <field type="CharField" name="cooker_log_path"></field> >> > - <field type="CharField" name="build_name">a</field> >> > <field type="CharField" name="bitbake_version"></field> >> > </object> >> > <object pk="2" model="orm.build"> >> > @@ -51,7 +50,6 @@ >> > <field type="DateTimeField" >> > name="completed_on">2016-02-13T18:46:20.114530+00:00</field> >> > <field type="IntegerField" name="outcome">0</field> >> > <field type="CharField" name="cooker_log_path"></field> >> > - <field type="CharField" name="build_name">b</field> >> > <field type="CharField" name="bitbake_version"></field> >> > </object> >> > <object pk="3" model="orm.build"> >> > @@ -63,7 +61,6 @@ >> > <field type="DateTimeField" >> > name="completed_on">2016-02-12T18:46:20.114530+00:00</field> >> > <field type="IntegerField" name="outcome">1</field> >> > <field type="CharField" name="cooker_log_path"></field> >> > - <field type="CharField" name="build_name">c</field> >> > <field type="CharField" name="bitbake_version"></field> >> > </object> >> > <object pk="4" model="orm.build"> >> > @@ -75,7 +72,6 @@ >> > <field type="DateTimeField" >> > name="completed_on">2016-02-11T18:46:20.114530+00:00</field> >> > <field type="IntegerField" name="outcome">0</field> >> > <field type="CharField" name="cooker_log_path"></field> >> > - <field type="CharField" name="build_name">d</field> >> > <field type="CharField" name="bitbake_version"></field> >> > </object> >> > <object pk="1" model="orm.target"> >> > -- >> > Elliot Smith >> > Software Engineer >> > Intel OTC >> > >> > --------------------------------------------------------------------- >> > Intel Corporation (UK) Limited >> > Registered No. 1134945 (England) >> > Registered Office: Pipers Way, Swindon SN3 1RJ >> > VAT No: 860 2173 47 >> > >> > This e-mail and any attachments may contain confidential material for >> > the sole use of the intended recipient(s). Any review or distribution >> > by others is strictly prohibited. If you are not the intended >> > recipient, please contact the sender and delete all copies. >> > >> > -- >> > _______________________________________________ >> > toaster mailing list >> > [email protected] >> > https://lists.yoctoproject.org/listinfo/toaster > > > > > -- > Elliot Smith > Software Engineer > Intel Open Source Technology Centre -- _______________________________________________ toaster mailing list [email protected] https://lists.yoctoproject.org/listinfo/toaster
