The poky-contrib branch available to verify the patch set is : http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=sujith/Import-external-layers-toaster-and-error-fix
On Mon, Oct 5, 2015 at 8:23 PM, Sujith H <[email protected]> wrote: > This patch helps to import external layers from toasterconf.json > which are already checked out. The branch of the checkedout layers > is HEAD. Hence no checkout happens when toaster is using the configuration. > This essentially speeds up toaster, while using locally checked out layers. > > Signed-off-by: Sujith H <[email protected]> > Signed-off-by: Sujith Haridasan <[email protected]> > --- > .../toaster/bldcontrol/localhostbecontroller.py | 108 > +++++++++++++-------- > .../bldcontrol/management/commands/loadconf.py | 14 ++- > 2 files changed, 76 insertions(+), 46 deletions(-) > > diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py > b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py > index a8d8398..2b3c452 100644 > --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py > +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py > @@ -192,6 +192,21 @@ class > LocalhostBEController(BuildEnvironmentController): > #logger.debug("localhostbecontroller: using HEAD checkout in %s" > % local_checkout_path) > return local_checkout_path > > + def getHeadLayers(self, gitrepos, layers): > + """ HEAD layers are special cases. They don't need to be cloned. > """ > + > + layerlist = [] > + headlayers = [commit for giturl, commit in gitrepos.keys() if > commit == "HEAD"] > + if set(headlayers) == set(["HEAD"]): > + # Populate layerlist if all of them are HEAD > + for layer_version in > Layer_Version.objects.all().filter(local_path__startswith = "/"): > + if layer_version.commit == "HEAD": > + if layer_version.local_path not in layerlist: > + layerlist.append(layer_version.local_path) > + if > os.path.exists(os.path.join(os.path.dirname(layer_version.local_path), > "oe-init-build-env")): > + self.pokydirname = > os.path.dirname(layer_version.local_path) > + return layerlist > + > > def setLayers(self, bitbakes, layers, targets): > """ a word of attention: by convention, the first layer for any > build will be poky! """ > @@ -239,51 +254,58 @@ class > LocalhostBEController(BuildEnvironmentController): > layerlist = [] > > > - # 3. checkout the repositories > - for giturl, commit in gitrepos.keys(): > - localdirname = os.path.join(self.be.sourcedir, > self.getGitCloneDirectory(giturl, commit)) > - logger.debug("localhostbecontroller: giturl %s:%s checking > out in current directory %s" % (giturl, commit, localdirname)) > + # Verify HEAD layers > > - # make sure our directory is a git repository > - if os.path.exists(localdirname): > - localremotes = self._shellcmd("git remote -v", > localdirname) > - if not giturl in localremotes: > - raise BuildSetupException("Existing git repository at > %s, but with different remotes ('%s', expected '%s'). Toaster will not > continue out of fear of damaging something." % (localdirname, ", > ".join(localremotes.split("\n")), giturl)) > - else: > - if giturl in cached_layers: > - logger.debug("localhostbecontroller git-copying %s to > %s" % (cached_layers[giturl], localdirname)) > - self._shellcmd("git clone \"%s\" \"%s\"" % > (cached_layers[giturl], localdirname)) > - self._shellcmd("git remote remove origin", > localdirname) > - self._shellcmd("git remote add origin \"%s\"" % > giturl, localdirname) > + layerlist = self.getHeadLayers(gitrepos,layers) > + > + # If the layerlist doesn't have already cloned layers then > checkout repositories > + if len(layerlist) == 0: > + > + # 3. checkout the repositories > + for giturl, commit in gitrepos.keys(): > + localdirname = os.path.join(self.be.sourcedir, > self.getGitCloneDirectory(giturl, commit)) > + logger.debug("localhostbecontroller: giturl %s:%s > checking out in current directory %s" % (giturl, commit, localdirname)) > + > + # make sure our directory is a git repository > + if os.path.exists(localdirname): > + localremotes = self._shellcmd("git remote -v", > localdirname) > + if not giturl in localremotes: > + raise BuildSetupException("Existing git > repository at %s, but with different remotes ('%s', expected '%s'). Toaster > will not continue out of fear of damaging something." % (localdirname, ", > ".join(localremotes.split("\n")), giturl)) > else: > - logger.debug("localhostbecontroller: cloning %s in > %s" % (giturl, localdirname)) > - self._shellcmd('git clone "%s" "%s"' % (giturl, > localdirname)) > - > - # branch magic name "HEAD" will inhibit checkout > - if commit != "HEAD": > - logger.debug("localhostbecontroller: checking out commit > %s to %s " % (commit, localdirname)) > - ref = commit if re.match('^[a-fA-F0-9]+$', commit) else > 'origin/%s' % commit > - self._shellcmd('git fetch --all && git reset --hard "%s"' > % ref, localdirname) > - > - # take the localdirname as poky dir if we can find the > oe-init-build-env > - if self.pokydirname is None and > os.path.exists(os.path.join(localdirname, "oe-init-build-env")): > - logger.debug("localhostbecontroller: selected poky dir > name %s" % localdirname) > - self.pokydirname = localdirname > - > - # make sure we have a working bitbake > - if not os.path.exists(os.path.join(self.pokydirname, > 'bitbake')): > - logger.debug("localhostbecontroller: checking bitbake > into the poky dirname %s " % self.pokydirname) > - self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % > (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, > 'bitbake'))) > - > - # verify our repositories > - for name, dirpath in gitrepos[(giturl, commit)]: > - localdirpath = os.path.join(localdirname, dirpath) > - logger.debug("localhostbecontroller: localdirpath > expected '%s'" % localdirpath) > - if not os.path.exists(localdirpath): > - raise BuildSetupException("Cannot find layer git path > '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, > commit)) > - > - if name != "bitbake": > - layerlist.append(localdirpath.rstrip("/")) > + if giturl in cached_layers: > + logger.debug("localhostbecontroller git-copying > %s to %s" % (cached_layers[giturl], localdirname)) > + self._shellcmd("git clone \"%s\" \"%s\"" % > (cached_layers[giturl], localdirname)) > + self._shellcmd("git remote remove origin", > localdirname) > + self._shellcmd("git remote add origin \"%s\"" % > giturl, localdirname) > + else: > + logger.debug("localhostbecontroller: cloning %s > in %s" % (giturl, localdirname)) > + self._shellcmd('git clone "%s" "%s"' % (giturl, > localdirname)) > + > + # branch magic name "HEAD" will inhibit checkout > + if commit != "HEAD": > + logger.debug("localhostbecontroller: checking out > commit %s to %s " % (commit, localdirname)) > + ref = commit if re.match('^[a-fA-F0-9]+$', commit) > else 'origin/%s' % commit > + self._shellcmd('git fetch --all && git reset --hard > "%s"' % ref, localdirname) > + > + # take the localdirname as poky dir if we can find the > oe-init-build-env > + if self.pokydirname is None and > os.path.exists(os.path.join(localdirname, "oe-init-build-env")): > + logger.debug("localhostbecontroller: selected poky > dir name %s" % localdirname) > + self.pokydirname = localdirname > + > + # make sure we have a working bitbake > + if not os.path.exists(os.path.join(self.pokydirname, > 'bitbake')): > + logger.debug("localhostbecontroller: checking > bitbake into the poky dirname %s " % self.pokydirname) > + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" > " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, > 'bitbake'))) > + > + # verify our repositories > + for name, dirpath in gitrepos[(giturl, commit)]: > + localdirpath = os.path.join(localdirname, dirpath) > + logger.debug("localhostbecontroller: localdirpath > expected '%s'" % localdirpath) > + if not os.path.exists(localdirpath): > + raise BuildSetupException("Cannot find layer git > path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, > giturl, commit)) > + > + if name != "bitbake": > + layerlist.append(localdirpath.rstrip("/")) > > logger.debug("localhostbecontroller: current layer list %s " % > pformat(layerlist)) > > diff --git > a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py > b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py > index 5022b59..21bc380 100644 > --- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py > +++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py > @@ -45,12 +45,14 @@ class Command(BaseCommand): > for i in ['bitbake', 'releases', 'defaultrelease', 'config', > 'layersources']: > assert i in data > > - def _read_git_url_from_local_repository(address): > + def _read_git_url_from_local_repository(address, path = None): > url = None > + if not path: > + path = os.path.dirname(filepath) > # we detect the remote name at runtime > import subprocess > (remote, remote_name) = address.split(":", 1) > - cmd = subprocess.Popen("git remote -v", shell=True, cwd = > os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE) > + cmd = subprocess.Popen("git remote -v", shell=True, cwd = > path, stdout=subprocess.PIPE, stderr = subprocess.PIPE) > (out,err) = cmd.communicate() > if cmd.returncode != 0: > logging.warning("Error while importing layer vcs_url: git > error: %s" % err) > @@ -121,7 +123,11 @@ class Command(BaseCommand): > logger.error("Local layer path %s must exists. > Are you trying to import a layer that does not exist ? Check your local > toasterconf.json" % lo.local_path) > > if layerinfo['vcs_url'].startswith("remote:"): > - lo.vcs_url = > _read_git_url_from_local_repository(layerinfo['vcs_url']) > + if not layerinfo['local_path'].startswith("/"): > + path = None > + else: > + path = layerinfo['local_path'] > + lo.vcs_url = > _read_git_url_from_local_repository(layerinfo['vcs_url'], path) > if lo.vcs_url is None: > logger.error("The toaster config file > references the local git repo, but Toaster cannot detect it.\nYour local > configuration for layer %s is invalid. Make sure that the toasterconf.json > file is correct." % layerinfo['name']) > > @@ -138,6 +144,8 @@ class Command(BaseCommand): > commit = branch.name, > layer = lo) > lvo.dirpath = layerinfo['dirpath'] > + if len(layerinfo['local_path']) > 1 and > layerinfo['local_path'].startswith("/") and branch.name == "HEAD": > + lvo.local_path = layerinfo['local_path'] > lvo.save() > # set releases > for ri in data['releases']: > -- > 1.9.1 > > -- സുജിത് ഹരിദാസന് Bangalore <Project>Contributor to KDE project http://fci.wikia.com/wiki/Anti-DRM-Campaign <Blog> http://sujithh.info
-- _______________________________________________ toaster mailing list [email protected] https://lists.yoctoproject.org/listinfo/toaster
