commit: 7c12da2ec9470fcf82b37943e2a1cc523351aefe Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Thu Jan 28 01:09:38 2021 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Thu Jan 28 01:09:38 2021 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7c12da2e
Add SetReposConf and UpdateRepos Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org> buildbot_gentoo_ci/config/buildfactorys.py | 13 ++-- buildbot_gentoo_ci/db/model.py | 3 +- buildbot_gentoo_ci/db/projects.py | 29 ++++++++- buildbot_gentoo_ci/steps/builders.py | 98 ++++++++++++++++++++++++++---- 4 files changed, 123 insertions(+), 20 deletions(-) diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index a6bafb5..3fad219 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -88,8 +88,8 @@ def build_request_check(): def run_build_request(): f = util.BuildFactory() # FIXME: 5 - # update repo for the profile - f.addStep(builders.UpdateProfileRepo()) + # set needed Propertys + f.addStep(builders.SetupPropertys()) # Clean and add new /etc/portage f.addStep(buildbot_steps.RemoveDirectory(dir="portage", workdir='/etc/')) @@ -99,9 +99,10 @@ def run_build_request(): f.addStep(buildbot_steps.MakeDirectory(dir="make.profile", workdir='/etc/portage/')) f.addStep(builders.SetMakeProfile()) - # setup repo.conf dir - #f.addStep(buildbot_steps.MakeDirectory(dir="repo.conf", + # setup repos.conf dir + f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf", workdir='/etc/portage/')) - # check if we have all repository's in repos.conf listed in project_repository's - # update all repos listed in project_repository's + f.addStep(builders.SetReposConf()) + # update the repositorys listed in project_repository + f.addStep(builders.UpdateRepos()) return f diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py index 0defb0c..596d04e 100644 --- a/buildbot_gentoo_ci/db/model.py +++ b/buildbot_gentoo_ci/db/model.py @@ -138,7 +138,8 @@ class Model(base.DBConnectorComponent): sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), - sa.Column('directorys', sa.Enum('make.profile'), nullable=False), + # FIXME: directorys should be moved to own table + sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'), nullable=False), sa.Column('value', sa.String(255), nullable=False), ) diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py index 5c3406a..00e1569 100644 --- a/buildbot_gentoo_ci/db/projects.py +++ b/buildbot_gentoo_ci/db/projects.py @@ -80,7 +80,19 @@ class ProjectsConnectorComponent(base.DBConnectorComponent): return res @defer.inlineCallbacks - def getProjectPortageByUuidAndDirectory(self, uuid, directory): + def getRepositorysByProjectUuid(self, uuid, auto=True): + def thd(conn): + tbl = self.db.model.projects_repositorys + q = tbl.select() + q = q.where(tbl.c.project_uuid == uuid) + q = q.where(tbl.c.auto == auto) + return [self._row2dict_projects_repositorys(conn, row) + for row in conn.execute(q).fetchall()] + res = yield self.db.pool.do(thd) + return res + + @defer.inlineCallbacks + def getAllProjectPortageByUuidAndDirectory(self, uuid, directory): def thd(conn): tbl = self.db.model.projects_portage q = tbl.select() @@ -91,6 +103,21 @@ class ProjectsConnectorComponent(base.DBConnectorComponent): res = yield self.db.pool.do(thd) return res + @defer.inlineCallbacks + def getProjectPortageByUuidAndDirectory(self, uuid, directory): + def thd(conn): + tbl = self.db.model.projects_portage + q = tbl.select() + q = q.where(tbl.c.project_uuid == uuid) + q = q.where(tbl.c.directorys == directory) + res = conn.execute(q) + row = res.fetchone() + if not row: + return None + return self._row2dict_projects_portage(conn, row) + res = yield self.db.pool.do(thd) + return res + def _row2dict(self, conn, row): return dict( uuid=row.uuid, diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index 1c8cbb0..d3b3607 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -80,9 +80,9 @@ class GetProjectRepositoryData(BuildStep): yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()]) return SUCCESS -class UpdateProfileRepo(BuildStep): +class SetupPropertys(BuildStep): - name = 'UpdateProfileRepo' + name = 'SetupPropertys' description = 'Running' descriptionDone = 'Ran' descriptionSuffix = None @@ -101,17 +101,8 @@ class UpdateProfileRepo(BuildStep): self.setProperty('portage_repos_path', self.portage_repos_path, 'portage_repos_path') projectrepository_data = self.getProperty('projectrepository_data') print(projectrepository_data) - repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid']) project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid']) self.setProperty('project_data', project_data, 'project_data') - self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid']) - profile_repository_path = yield os.path.join(self.portage_repos_path, self.profile_repository_data['name']) - yield self.build.addStepsAfterCurrentStep([ - steps.Git(repourl=self.profile_repository_data['mirror_url'], - mode='incremental', - submodules=True, - workdir=os.path.join(profile_repository_path, '')) - ]) return SUCCESS class SetMakeProfile(BuildStep): @@ -133,7 +124,7 @@ class SetMakeProfile(BuildStep): project_data = self.getProperty('project_data') profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid']) makeprofiles_paths = [] - makeprofiles_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile') + makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile') for makeprofile in makeprofiles_data: makeprofile_path = yield os.path.join(portage_repos_path, profile_repository_data['name'], 'profiles', makeprofile['value'], '') makeprofiles_paths.append('../../..' + makeprofile_path) @@ -145,3 +136,86 @@ class SetMakeProfile(BuildStep): workdir='/etc/portage/') ]) return SUCCESS + +class SetReposConf(BuildStep): + + name = 'SetReposConf' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + portage_repos_path = self.getProperty('portage_repos_path') + project_data = self.getProperty('project_data') + # setup the default.conf + repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'repos.conf') + if repos_conf_data is None: + print('Default repo is not set in repos.conf') + return FAILURE + # check if repos_conf_data['value'] is vaild repo name + separator = '\n' + default_conf = [] + default_conf.append('[DEFAULT]') + default_conf.append('main-repo = ' + repos_conf_data['value']) + default_conf.append('auto-sync = no') + default_conf_string = separator.join(default_conf) + yield self.build.addStepsAfterCurrentStep([ + steps.StringDownload(default_conf_string + separator, + workerdest="repos.conf/default.conf", + workdir='/etc/portage/') + ]) + # add all repos that project have in projects_repositorys to repos.conf/reponame.conf + projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid']) + for project_repository_data in projects_repositorys_data: + repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid']) + repository_path = yield os.path.join(portage_repos_path, repository_data['name']) + repository_conf = [] + repository_conf.append('[' + repository_data['name'] + ']') + repository_conf.append('location = ' + repository_path) + repository_conf.append('sync-uri = ' + repository_data['mirror_url']) + repository_conf.append('sync-type = git') + repository_conf.append('auto-sync = no') + repository_conf_string = separator.join(repository_conf) + yield self.build.addStepsAfterCurrentStep([ + steps.StringDownload(repository_conf_string + separator, + workerdest='repos.conf/' + repository_data['name'] + '.conf', + workdir='/etc/portage/') + ]) + return SUCCESS + +class UpdateRepos(BuildStep): + + name = 'UpdateRepos' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + portage_repos_path = self.getProperty('portage_repos_path') + project_data = self.getProperty('project_data') + # update/add all repos that in project_repository for the project + projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid']) + for project_repository_data in projects_repositorys_data: + repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid']) + repository_path = yield os.path.join(portage_repos_path, repository_data['name']) + yield self.build.addStepsAfterCurrentStep([ + steps.Git(repourl=repository_data['mirror_url'], + mode='incremental', + submodules=True, + workdir=os.path.join(repository_path, '')) + ]) + return SUCCESS