yes. this is the patch I am on. I'll try you approach later today. Thanks for the clarifications :) -b
On Mon, Aug 31, 2015 at 4:17 AM, sujith h <[email protected]> wrote: > Hi Brian, > > I hope you are using this patch for testing. > > Thanks, > Sujith H > > On Tue, Aug 25, 2015 at 3:23 PM, Sujith H <[email protected]> wrote: >> >> This patch helps to import external layers from toasterconf.json >> which are already checked out. The branch of the checkedout layers >> is HEAD. Hence no checkout happens when toaster is using the >> configuration. >> This essentially speeds up toaster, while using locally checked out >> layers. >> >> Signed-off-by: Sujith H <[email protected]> >> Signed-off-by: Sujith Haridasan <[email protected]> >> --- >> .../toaster/bldcontrol/localhostbecontroller.py | 112 >> +++++++++++++-------- >> .../bldcontrol/management/commands/loadconf.py | 14 ++- >> 2 files changed, 79 insertions(+), 47 deletions(-) >> >> diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py >> b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py >> index 231a7d3..837145b 100644 >> --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py >> +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py >> @@ -32,6 +32,8 @@ from toastermain import settings >> >> from bbcontroller import BuildEnvironmentController, ShellCmdException, >> BuildSetupException >> >> +from orm.models import Layer_Version >> + >> import logging >> logger = logging.getLogger("toaster") >> >> @@ -196,6 +198,21 @@ class >> LocalhostBEController(BuildEnvironmentController): >> #logger.debug("localhostbecontroller: using HEAD checkout in %s" >> % local_checkout_path) >> return local_checkout_path >> >> + def getHeadLayers(self, gitrepos, layers): >> + """ HEAD layers are special cases. They don't need to be cloned. >> """ >> + >> + layerlist = [] >> + headlayers = [commit for giturl, commit in gitrepos.keys() if >> commit == "HEAD"] >> + if set(headlayers) == set(["HEAD"]): >> + # Populate layerlist if all of them are HEAD >> + for layer_version in >> Layer_Version.objects.all().filter(local_path__startswith = "/"): >> + if layer_version.commit == "HEAD": >> + if layer_version.local_path not in layerlist: >> + layerlist.append(layer_version.local_path) >> + if "poky" in layer_version.local_path and >> os.path.exists(os.path.join(os.path.dirname(layer_version.local_path), >> "oe-init-build-env")): >> + self.pokydirname = >> os.path.dirname(layer_version.local_path) >> + return layerlist >> + >> >> def setLayers(self, bitbakes, layers): >> """ a word of attention: by convention, the first layer for any >> build will be poky! """ >> @@ -242,51 +259,58 @@ class >> LocalhostBEController(BuildEnvironmentController): >> >> layerlist = [] >> >> + # Verify HEAD layers >> + >> + layerlist = self.getHeadLayers(gitrepos,layers) >> + >> + # If the layerlist doesn't have already cloned layers then >> checkout repositories >> + if len(layerlist) == 0: >> + >> + # 3. checkout the repositories >> + for giturl, commit in gitrepos.keys(): >> + localdirname = os.path.join(self.be.sourcedir, >> self.getGitCloneDirectory(giturl, commit)) >> + logger.debug("localhostbecontroller: giturl %s:%s checking >> out in current directory %s" % (giturl, commit, localdirname)) >> + >> + # make sure our directory is a git repository >> + if os.path.exists(localdirname): >> + localremotes = self._shellcmd("git remote -v", >> localdirname) >> + if not giturl in localremotes: >> + raise BuildSetupException("Existing git repository >> at %s, but with different remotes ('%s', expected '%s'). Toaster will not >> continue out of fear of damaging something." % (localdirname, ", >> ".join(localremotes.split("\n")), giturl)) >> + else: >> + if giturl in cached_layers: >> + logger.debug("localhostbecontroller git-copying %s >> to %s" % (cached_layers[giturl], localdirname)) >> + self._shellcmd("git clone \"%s\" \"%s\"" % >> (cached_layers[giturl], localdirname)) >> + self._shellcmd("git remote remove origin", >> localdirname) >> + self._shellcmd("git remote add origin \"%s\"" % >> giturl, localdirname) >> + else: >> + logger.debug("localhostbecontroller: cloning %s:%s >> in %s" % (giturl, commit, localdirname)) >> + self._shellcmd("git clone \"%s\" --single-branch >> --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) >> + >> + # branch magic name "HEAD" will inhibit checkout >> + if commit != "HEAD": >> + logger.debug("localhostbecontroller: checking out >> commit %s to %s " % (commit, localdirname)) >> + self._shellcmd("git fetch --all && git checkout \"%s\" >> && git rebase \"origin/%s\"" % (commit, commit) , localdirname) >> + >> + # take the localdirname as poky dir if we can find the >> oe-init-build-env >> + if self.pokydirname is None and >> os.path.exists(os.path.join(localdirname, "oe-init-build-env")): >> + logger.debug("localhostbecontroller: selected poky dir >> name %s" % localdirname) >> + self.pokydirname = localdirname >> + >> + # make sure we have a working bitbake >> + if not os.path.exists(os.path.join(self.pokydirname, >> 'bitbake')): >> + logger.debug("localhostbecontroller: checking >> bitbake into the poky dirname %s " % self.pokydirname) >> + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" >> " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, >> 'bitbake'))) >> + >> + # verify our repositories >> + for name, dirpath in gitrepos[(giturl, commit)]: >> + localdirpath = os.path.join(localdirname, dirpath) >> + logger.debug("localhostbecontroller: localdirpath >> expected '%s'" % localdirpath) >> + if not os.path.exists(localdirpath): >> + raise BuildSetupException("Cannot find layer git >> path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, >> giturl, commit)) >> + >> + if name != "bitbake": >> + layerlist.append(localdirpath.rstrip("/")) >> >> - # 3. checkout the repositories >> - for giturl, commit in gitrepos.keys(): >> - localdirname = os.path.join(self.be.sourcedir, >> self.getGitCloneDirectory(giturl, commit)) >> - logger.debug("localhostbecontroller: giturl %s:%s checking >> out in current directory %s" % (giturl, commit, localdirname)) >> - >> - # make sure our directory is a git repository >> - if os.path.exists(localdirname): >> - localremotes = self._shellcmd("git remote -v", >> localdirname) >> - if not giturl in localremotes: >> - raise BuildSetupException("Existing git repository at >> %s, but with different remotes ('%s', expected '%s'). Toaster will not >> continue out of fear of damaging something." % (localdirname, ", >> ".join(localremotes.split("\n")), giturl)) >> - else: >> - if giturl in cached_layers: >> - logger.debug("localhostbecontroller git-copying %s to >> %s" % (cached_layers[giturl], localdirname)) >> - self._shellcmd("git clone \"%s\" \"%s\"" % >> (cached_layers[giturl], localdirname)) >> - self._shellcmd("git remote remove origin", >> localdirname) >> - self._shellcmd("git remote add origin \"%s\"" % >> giturl, localdirname) >> - else: >> - logger.debug("localhostbecontroller: cloning %s:%s in >> %s" % (giturl, commit, localdirname)) >> - self._shellcmd("git clone \"%s\" --single-branch >> --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) >> - >> - # branch magic name "HEAD" will inhibit checkout >> - if commit != "HEAD": >> - logger.debug("localhostbecontroller: checking out commit >> %s to %s " % (commit, localdirname)) >> - self._shellcmd("git fetch --all && git checkout \"%s\" && >> git rebase \"origin/%s\"" % (commit, commit) , localdirname) >> - >> - # take the localdirname as poky dir if we can find the >> oe-init-build-env >> - if self.pokydirname is None and >> os.path.exists(os.path.join(localdirname, "oe-init-build-env")): >> - logger.debug("localhostbecontroller: selected poky dir >> name %s" % localdirname) >> - self.pokydirname = localdirname >> - >> - # make sure we have a working bitbake >> - if not os.path.exists(os.path.join(self.pokydirname, >> 'bitbake')): >> - logger.debug("localhostbecontroller: checking bitbake >> into the poky dirname %s " % self.pokydirname) >> - self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % >> (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, >> 'bitbake'))) >> - >> - # verify our repositories >> - for name, dirpath in gitrepos[(giturl, commit)]: >> - localdirpath = os.path.join(localdirname, dirpath) >> - logger.debug("localhostbecontroller: localdirpath >> expected '%s'" % localdirpath) >> - if not os.path.exists(localdirpath): >> - raise BuildSetupException("Cannot find layer git path >> '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, >> commit)) >> - >> - if name != "bitbake": >> - layerlist.append(localdirpath.rstrip("/")) >> >> logger.debug("localhostbecontroller: current layer list %s " % >> pformat(layerlist)) >> >> diff --git >> a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py >> b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py >> index 5022b59..21bc380 100644 >> --- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py >> +++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py >> @@ -45,12 +45,14 @@ class Command(BaseCommand): >> for i in ['bitbake', 'releases', 'defaultrelease', 'config', >> 'layersources']: >> assert i in data >> >> - def _read_git_url_from_local_repository(address): >> + def _read_git_url_from_local_repository(address, path = None): >> url = None >> + if not path: >> + path = os.path.dirname(filepath) >> # we detect the remote name at runtime >> import subprocess >> (remote, remote_name) = address.split(":", 1) >> - cmd = subprocess.Popen("git remote -v", shell=True, cwd = >> os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE) >> + cmd = subprocess.Popen("git remote -v", shell=True, cwd = >> path, stdout=subprocess.PIPE, stderr = subprocess.PIPE) >> (out,err) = cmd.communicate() >> if cmd.returncode != 0: >> logging.warning("Error while importing layer vcs_url: git >> error: %s" % err) >> @@ -121,7 +123,11 @@ class Command(BaseCommand): >> logger.error("Local layer path %s must exists. >> Are you trying to import a layer that does not exist ? Check your local >> toasterconf.json" % lo.local_path) >> >> if layerinfo['vcs_url'].startswith("remote:"): >> - lo.vcs_url = >> _read_git_url_from_local_repository(layerinfo['vcs_url']) >> + if not layerinfo['local_path'].startswith("/"): >> + path = None >> + else: >> + path = layerinfo['local_path'] >> + lo.vcs_url = >> _read_git_url_from_local_repository(layerinfo['vcs_url'], path) >> if lo.vcs_url is None: >> logger.error("The toaster config file >> references the local git repo, but Toaster cannot detect it.\nYour local >> configuration for layer %s is invalid. Make sure that the toasterconf.json >> file is correct." % layerinfo['name']) >> >> @@ -138,6 +144,8 @@ class Command(BaseCommand): >> commit = branch.name, >> layer = lo) >> lvo.dirpath = layerinfo['dirpath'] >> + if len(layerinfo['local_path']) > 1 and >> layerinfo['local_path'].startswith("/") and branch.name == "HEAD": >> + lvo.local_path = layerinfo['local_path'] >> lvo.save() >> # set releases >> for ri in data['releases']: >> -- >> 1.9.1 >> > > > > -- > സുജിത് ഹരിദാസന് > Bangalore > <Project>Contributor to KDE project > http://fci.wikia.com/wiki/Anti-DRM-Campaign > <Blog> http://sujithh.info -- _______________________________________________ toaster mailing list [email protected] https://lists.yoctoproject.org/listinfo/toaster
