On 19 February 2016 at 16:54, Brian Avery <[email protected]> wrote:
> I wouldn't remove the ParseStarted event from the event list. Earlier > (before you or I worked on it) toaster would show transitions in the > progress bar before the build started so that users got feedback. > Things like > 1) checking out layers > 2) parsing > 3) then the build % > I agree, but I'd argue for putting that event back into the mask when we actually need it. Also, that event doesn't consistently fire for a build (if I remember my testing correctly), so I wouldn't want to rely on it for anything. We could actually use TargetsAcquired, as that marks the start of the build accurately and consistently. Elliot > > It would be nice at some point to restore this and we'd need > Parsestarted for that. > > -b > > On Fri, Feb 19, 2016 at 4:17 AM, Elliot Smith <[email protected]> > wrote: > > If a build fails because of a bitbake error occurring before the > > BuildStarted event fires, we do not generate a Build object > > for command-line builds. This means that failed command-line builds > > don't appear in Toaster at all. > > > > To resolve, split build creation into two steps: > > > > 1. Just before buildTargets() is invoked on the XMLRPC server: create > > the base Build object. Note that as soon as a Toaster-triggered > > build starts, targets are added to it; but this event is the earliest > > point when task and targets are available for command-line builds. > > (This requires a new TargetsAcquired event to be fired by the XMLRPC > > server when the buildTargets() command is called.) > > > > 2. BuildStarted event: add any layer information to either type of build > > (command-line or Toaster-triggered). > > > > Note that the build_name property cannot be set until BuildStarted, > > as it is not available until then, which could cause problems > > for creating Build objects earlier; however, this property is > > redundant, as it's never used anywhere in Toaster, so it has been > > removed (along with any functions which refer to it). > > > > [YOCTO #8440] > > > > Signed-off-by: Elliot Smith <[email protected]> > > --- > > bitbake/lib/bb/event.py | 7 +++ > > bitbake/lib/bb/server/xmlrpc.py | 3 + > > bitbake/lib/bb/ui/buildinfohelper.py | 73 > ++++++++++------------ > > bitbake/lib/bb/ui/toasterui.py | 21 ++++--- > > .../orm/migrations/0005_remove_build_build_name.py | 18 ++++++ > > bitbake/lib/toaster/orm/models.py | 1 - > > .../fixtures/toastergui-unittest-data.xml | 4 -- > > 7 files changed, 73 insertions(+), 54 deletions(-) > > create mode 100644 > bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py > > > > diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py > > index 5ffe89e..e94cdb8 100644 > > --- a/bitbake/lib/bb/event.py > > +++ b/bitbake/lib/bb/event.py > > @@ -444,6 +444,13 @@ class MultipleProviders(Event): > > """ > > return self._candidates > > > > +class TargetsAcquired(Event): > > + """Target we are aiming at known""" > > + def __init__(self, task, targetsList): > > + Event.__init__(self) > > + self.task = task > > + self.targetsList = targetsList > > + > > class ParseStarted(OperationStarted): > > """Recipe parsing for the runqueue has begun""" > > def __init__(self, total): > > diff --git a/bitbake/lib/bb/server/xmlrpc.py > b/bitbake/lib/bb/server/xmlrpc.py > > index 1ceca51..5f735ab 100644 > > --- a/bitbake/lib/bb/server/xmlrpc.py > > +++ b/bitbake/lib/bb/server/xmlrpc.py > > @@ -112,6 +112,9 @@ class BitBakeServerCommands(): > > """ > > Run a cooker command on the server > > """ > > + if 'buildTargets' in command[0]: > > + bb.event.fire(bb.event.TargetsAcquired(command[2], > command[1]), self.server.readonly) > > + > > return self.cooker.command.runCommand(command, > self.server.readonly) > > > > def getEventHandle(self): > > diff --git a/bitbake/lib/bb/ui/buildinfohelper.py > b/bitbake/lib/bb/ui/buildinfohelper.py > > index 81abede..46380fa 100644 > > --- a/bitbake/lib/bb/ui/buildinfohelper.py > > +++ b/bitbake/lib/bb/ui/buildinfohelper.py > > @@ -127,13 +127,12 @@ class ORMWrapper(object): > > # pylint: disable=bad-continuation > > # we do not follow the python conventions for continuation > indentation due to long lines here > > > > - def create_build_object(self, build_info, brbe, project_id): > > + def create_build_object(self, build_info, brbe, project_id = None): > > assert 'machine' in build_info > > assert 'distro' in build_info > > assert 'distro_version' in build_info > > assert 'started_on' in build_info > > assert 'cooker_log_path' in build_info > > - assert 'build_name' in build_info > > assert 'bitbake_version' in build_info > > > > prj = None > > @@ -161,7 +160,6 @@ class ORMWrapper(object): > > build.distro=build_info['distro'] > > build.distro_version=build_info['distro_version'] > > build.cooker_log_path=build_info['cooker_log_path'] > > - build.build_name=build_info['build_name'] > > build.bitbake_version=build_info['bitbake_version'] > > build.save() > > > > @@ -174,7 +172,6 @@ class ORMWrapper(object): > > started_on=build_info['started_on'], > > > completed_on=build_info['started_on'], > > > cooker_log_path=build_info['cooker_log_path'], > > - build_name=build_info['build_name'], > > > bitbake_version=build_info['bitbake_version']) > > > > logger.debug(1, "buildinfohelper: build is created %s" % build) > > @@ -854,7 +851,6 @@ class BuildInfoHelper(object): > > build_info['started_on'] = timezone.now() > > build_info['completed_on'] = timezone.now() > > build_info['cooker_log_path'] = build_log_path > > - build_info['build_name'] = > self.server.runCommand(["getVariable", "BUILDNAME"])[0] > > build_info['bitbake_version'] = > self.server.runCommand(["getVariable", "BB_VERSION"])[0] > > build_info['project'] = self.project = > self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0] > > return build_info > > @@ -919,26 +915,6 @@ class BuildInfoHelper(object): > > > > return recipe_info > > > > - def _get_path_information(self, task_object): > > - assert isinstance(task_object, Task) > > - build_stats_format = > "{tmpdir}/buildstats/{buildname}/{package}/" > > - build_stats_path = [] > > - > > - for t in self.internal_state['targets']: > > - buildname = self.internal_state['build'].build_name > > - pe, pv = task_object.recipe.version.split(":",1) > > - if len(pe) > 0: > > - package = task_object.recipe.name + "-" + pe + "_" + pv > > - else: > > - package = task_object.recipe.name + "-" + pv > > - > > - > build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, > > - > buildname=buildname, > > - package=package)) > > - > > - return build_stats_path > > - > > - > > ################################ > > ## external available methods to store information > > @staticmethod > > @@ -962,17 +938,43 @@ class BuildInfoHelper(object): > > except NotExisting as nee: > > logger.warn("buildinfohelper: cannot identify layer > exception:%s ", nee) > > > > - > > - def store_started_build(self, event, build_log_path): > > - assert '_pkgs' in vars(event) > > + def store_new_build(self, build_log_path): > > + """ > > + create a skeletal build object (or retrieve an existing one) as > soon as > > + bitbake starts trying to do the build; we use the buildTargets() > > + command on the XMLRPC server as the indicator of a build start > > + """ > > build_information = self._get_build_information(build_log_path) > > + self.internal_state['build'] = > self.orm_wrapper.create_build_object(build_information, self.brbe) > > > > - # Update brbe and project as they can be changed for every build > > - self.project = build_information['project'] > > + def store_targets(self, event): > > + """ > > + store targets for the current build, if that build was started > from > > + the command line; targets for non-cli builds are irrelevant, as > we > > + create them from the BuildRequest anyway > > + > > + event: a TargetsAcquired event with a task property (e.g. > "build") > > + and a targetsList property (e.g. ["zlib", "dropbear"]) > > + """ > > + if self.internal_state['build'].project.is_default: > > + targets = map(lambda target: target + ':' + event.task, > event.targetsList) > > + > > + target_information = { > > + 'targets': targets, > > + 'build': self.internal_state['build'] > > + } > > + > > + self.internal_state['targets'] = > self.orm_wrapper.get_or_create_targets(target_information) > > > > - build_obj = > self.orm_wrapper.create_build_object(build_information, self.brbe, > self.project) > > + def update_build(self, event): > > + """ > > + update the current build with layer and config data once it > > + actually starts > > + > > + event: a BuildStarted event > > + """ > > > > - self.internal_state['build'] = build_obj > > + build_obj = self.internal_state['build'] > > > > # save layer version information for this build > > if not 'lvs' in self.internal_state: > > @@ -983,13 +985,6 @@ class BuildInfoHelper(object): > > > > del self.internal_state['lvs'] > > > > - # create target information > > - target_information = {} > > - target_information['targets'] = event._pkgs > > - target_information['build'] = build_obj > > - > > - self.internal_state['targets'] = > self.orm_wrapper.get_or_create_targets(target_information) > > - > > # Save build configuration > > data = self.server.runCommand(["getAllKeysWithFlags", ["doc", > "func"]])[0] > > > > diff --git a/bitbake/lib/bb/ui/toasterui.py > b/bitbake/lib/bb/ui/toasterui.py > > index 32b1889..1584872 100644 > > --- a/bitbake/lib/bb/ui/toasterui.py > > +++ b/bitbake/lib/bb/ui/toasterui.py > > @@ -93,14 +93,14 @@ def _close_build_log(build_log): > > logger.removeHandler(build_log) > > > > _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", > "logging.LogRecord", > > - "bb.build.TaskFailed", "bb.build.TaskBase", > "bb.event.ParseStarted", > > + "bb.build.TaskFailed", "bb.build.TaskBase", > > "bb.event.ParseProgress", "bb.event.ParseCompleted", > "bb.event.CacheLoadStarted", > > "bb.event.CacheLoadProgress", > "bb.event.CacheLoadCompleted", "bb.command.CommandFailed", > > "bb.command.CommandExit", "bb.command.CommandCompleted", > "bb.cooker.CookerExit", > > "bb.event.MultipleProviders", "bb.event.NoProvider", > "bb.runqueue.sceneQueueTaskStarted", > > "bb.runqueue.runQueueTaskStarted", > "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed", > > "bb.event.BuildBase", "bb.build.TaskStarted", > "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent", > > - "bb.event.MetadataEvent"] > > + "bb.event.MetadataEvent", "bb.event.TargetsAcquired"] > > > > def main(server, eventHandler, params): > > # set to a logging.FileHandler instance when a build starts; > > @@ -186,19 +186,20 @@ def main(server, eventHandler, params): > > # pylint: disable=protected-access > > # the code will look into the protected variables of the > event; no easy way around this > > > > - # we treat ParseStarted as the first event of > toaster-triggered > > - # builds; that way we get the Build Configuration included > in the log > > - # and any errors that occur before BuildStarted is fired > > - if isinstance(event, bb.event.ParseStarted): > > + # start of build: this event is fired just before the > buildTargets() > > + # command is invoked on the XMLRPC server > > + if isinstance(event, bb.event.TargetsAcquired): > > if not (build_log and build_log_file_path): > > build_log, build_log_file_path = > _open_build_log(log_dir) > > + buildinfohelper.store_new_build(build_log_file_path) > > + buildinfohelper.store_targets(event) > > continue > > > > + # when the build proper starts, we extract information about > > + # any layers and config data > > if isinstance(event, bb.event.BuildStarted): > > - if not (build_log and build_log_file_path): > > - build_log, build_log_file_path = > _open_build_log(log_dir) > > - > > - buildinfohelper.store_started_build(event, > build_log_file_path) > > + buildinfohelper.update_build(event) > > + continue > > > > if isinstance(event, (bb.build.TaskStarted, > bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): > > buildinfohelper.update_and_store_task(event) > > diff --git > a/bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py > b/bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py > > new file mode 100644 > > index 0000000..7159793 > > --- /dev/null > > +++ b/bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py > > @@ -0,0 +1,18 @@ > > +# -*- coding: utf-8 -*- > > +from __future__ import unicode_literals > > + > > +from django.db import migrations, models > > + > > + > > +class Migration(migrations.Migration): > > + > > + dependencies = [ > > + ('orm', '0004_provides'), > > + ] > > + > > + operations = [ > > + migrations.RemoveField( > > + model_name='build', > > + name='build_name', > > + ), > > + ] > > diff --git a/bitbake/lib/toaster/orm/models.py > b/bitbake/lib/toaster/orm/models.py > > index ab6940f..39eb2b1 100644 > > --- a/bitbake/lib/toaster/orm/models.py > > +++ b/bitbake/lib/toaster/orm/models.py > > @@ -370,7 +370,6 @@ class Build(models.Model): > > completed_on = models.DateTimeField() > > outcome = models.IntegerField(choices=BUILD_OUTCOME, > default=IN_PROGRESS) > > cooker_log_path = models.CharField(max_length=500) > > - build_name = models.CharField(max_length=100) > > bitbake_version = models.CharField(max_length=50) > > > > @staticmethod > > diff --git > a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml > b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml > > index 2d83ff8..a554e62 100644 > > --- > a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml > > +++ > b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml > > @@ -39,7 +39,6 @@ > > <field type="DateTimeField" > name="completed_on">2016-02-14T18:46:20.114530+00:00</field> > > <field type="IntegerField" name="outcome">0</field> > > <field type="CharField" name="cooker_log_path"></field> > > - <field type="CharField" name="build_name">a</field> > > <field type="CharField" name="bitbake_version"></field> > > </object> > > <object pk="2" model="orm.build"> > > @@ -51,7 +50,6 @@ > > <field type="DateTimeField" > name="completed_on">2016-02-13T18:46:20.114530+00:00</field> > > <field type="IntegerField" name="outcome">0</field> > > <field type="CharField" name="cooker_log_path"></field> > > - <field type="CharField" name="build_name">b</field> > > <field type="CharField" name="bitbake_version"></field> > > </object> > > <object pk="3" model="orm.build"> > > @@ -63,7 +61,6 @@ > > <field type="DateTimeField" > name="completed_on">2016-02-12T18:46:20.114530+00:00</field> > > <field type="IntegerField" name="outcome">1</field> > > <field type="CharField" name="cooker_log_path"></field> > > - <field type="CharField" name="build_name">c</field> > > <field type="CharField" name="bitbake_version"></field> > > </object> > > <object pk="4" model="orm.build"> > > @@ -75,7 +72,6 @@ > > <field type="DateTimeField" > name="completed_on">2016-02-11T18:46:20.114530+00:00</field> > > <field type="IntegerField" name="outcome">0</field> > > <field type="CharField" name="cooker_log_path"></field> > > - <field type="CharField" name="build_name">d</field> > > <field type="CharField" name="bitbake_version"></field> > > </object> > > <object pk="1" model="orm.target"> > > -- > > Elliot Smith > > Software Engineer > > Intel OTC > > > > --------------------------------------------------------------------- > > Intel Corporation (UK) Limited > > Registered No. 1134945 (England) > > Registered Office: Pipers Way, Swindon SN3 1RJ > > VAT No: 860 2173 47 > > > > This e-mail and any attachments may contain confidential material for > > the sole use of the intended recipient(s). Any review or distribution > > by others is strictly prohibited. If you are not the intended > > recipient, please contact the sender and delete all copies. > > > > -- > > _______________________________________________ > > toaster mailing list > > [email protected] > > https://lists.yoctoproject.org/listinfo/toaster > -- Elliot Smith Software Engineer Intel Open Source Technology Centre
-- _______________________________________________ toaster mailing list [email protected] https://lists.yoctoproject.org/listinfo/toaster
