[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: aba0962b8f901301302a2fefacac14f8cd34a51b Author: Magnus Granberg gentoo org> AuthorDate: Wed Jul 27 19:32:09 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Wed Jul 27 19:32:09 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aba0962b Move UpdateRepos to repos.py and support gitlab stage Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/buildfactorys.py | 4 +- buildbot_gentoo_ci/db/model.py | 12 - buildbot_gentoo_ci/db/repositorys.py | 19 +++- buildbot_gentoo_ci/steps/builders.py | 40 buildbot_gentoo_ci/steps/repos.py | 73 +- 5 files changed, 102 insertions(+), 46 deletions(-) diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index 67447aa..7f2dff9 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -46,7 +46,7 @@ def update_db_cpv(): # set needed Propertys f.addStep(package.SetupPropertys()) # update the repositorys listed in project_repository -f.addStep(builders.UpdateRepos()) +f.addStep(repos.UpdateRepos()) # add repo.conf #f.addStep(portage.SetReposConf()) # if categorys in db @@ -99,7 +99,7 @@ def run_build_request(): # set needed Propertys f.addStep(builders.SetupPropertys()) # update the repositorys listed in project_repository -f.addStep(builders.UpdateRepos()) +f.addStep(repos.UpdateRepos()) # Clean and add new /etc/portage #NOTE: remove the symlink befor the dir #f.addStep(buildbot_steps.ShellCommand( diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py index 908ebb5..d6e9860 100644 --- a/buildbot_gentoo_ci/db/model.py +++ b/buildbot_gentoo_ci/db/model.py @@ -71,10 +71,18 @@ class Model(base.DBConnectorComponent): # description of the repository sa.Column('description', sa.Text, nullable=True), sa.Column('url', sa.String(255), nullable=True), -sa.Column('type', sa.Enum('gitpuller'), nullable=False, default='gitpuller'), +sa.Column('branch', sa.String(255), nullable=True), +sa.Column('type', sa.Enum('git', 'gitlab'), nullable=False, default='git'), +sa.Column('mode', sa.Enum('full', 'incremental'), nullable=False, default='incremental'), +sa.Column('method', sa.Enum('clobber', 'fresh', 'clean', 'copy'), nullable=False, default='fresh'), +sa.Column('alwaysuselatest', sa.Boolean, default=False), sa.Column('auto', sa.Boolean, default=False), sa.Column('enabled', sa.Boolean, default=False), -sa.Column('ebuild', sa.Boolean, default=False), +sa.Column('ebuild', sa.Boolean, default=True), +sa.Column('merge', sa.Boolean, default=False), +# filenames for SecretInAFile +sa.Column('sshprivatekey', sa.String(50), nullable=True), +sa.Column('sshhostkey', sa.String(50), nullable=True), ) # Use by GitPoller diff --git a/buildbot_gentoo_ci/db/repositorys.py b/buildbot_gentoo_ci/db/repositorys.py index 8bac1a1..4d5d048 100644 --- a/buildbot_gentoo_ci/db/repositorys.py +++ b/buildbot_gentoo_ci/db/repositorys.py @@ -91,15 +91,32 @@ class RepositorysConnectorComponent(base.DBConnectorComponent): yield self.db.pool.do(thd) def _row2dict(self, conn, row): +if row.branch == 'none': +branch = False +else: +branch = row.branch +if row.sshprivatekey == 'none': +sshprivatekey = False +sshhostkey = False +else: +sshprivatekey = row.sshprivatekey +sshhostkey = row.sshhostkey return dict( uuid=row.uuid, name=row.name, description=row.description, url=row.url, +type=row.type, +branch=branch, +mode=row.mode, +method=row.method, +alwaysuselatest=row.alwaysuselatest, auto=row.auto, enabled=row.enabled, ebuild=row.ebuild, -type=row.type +merge=row.merge, +sshprivatekey=sshprivatekey, +sshhostkey=sshhostkey ) def _row2dict_gitpuller(self, conn, row): diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index 383ae60..a54c68c 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -343,46 +343,6 @@ class SetupPropertys(BuildStep): self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for project', self.getProperty('project_data')['name']]) return SUCCESS -# Should be moved to repos.py -class UpdateRepos(BuildStep): - -name = 'UpdateRepos' -description = 'Running' -descriptionDone = 'Ran' -descriptionSuffix = None -haltOn
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: 5176761ad56557e45359729d7b2989dadeace7b5 Author: Magnus Granberg gentoo org> AuthorDate: Thu Mar 11 01:14:02 2021 + Commit: Magnus Granberg gentoo org> CommitDate: Thu Mar 11 01:14:02 2021 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5176761a Add GitPuller to change_source and change cpv_changes to git_changes Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/change_source.py | 18 ++ buildbot_gentoo_ci/config/schedulers.py| 40 + buildbot_gentoo_ci/db/model.py | 10 +++- buildbot_gentoo_ci/db/repositorys.py | 48 ++- buildbot_gentoo_ci/steps/update_db.py | 95 +++--- 5 files changed, 134 insertions(+), 77 deletions(-) diff --git a/buildbot_gentoo_ci/config/change_source.py b/buildbot_gentoo_ci/config/change_source.py new file mode 100644 index 000..454ed4a --- /dev/null +++ b/buildbot_gentoo_ci/config/change_source.py @@ -0,0 +1,18 @@ +# Copyright 2021 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from buildbot.plugins import changes, util + +#FIXME: +# Get the repositorys info from the gentoo-ci db +def gentoo_change_source(cs=[]): +cs.append(changes.GitPoller( +repourl='https://github.com/gentoo/gentoo.git', +branches=True, +workdir= 'repositorys' + '/gentoo.git/', +pollInterval=240, +pollRandomDelayMin=20, +pollRandomDelayMax=60, +project='gentoo' +)) +return cs diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py index 7cfb122..0cbec96 100644 --- a/buildbot_gentoo_ci/config/schedulers.py +++ b/buildbot_gentoo_ci/config/schedulers.py @@ -4,7 +4,7 @@ from buildbot.plugins import schedulers, util @util.renderer -def builderUpdateDbNames(self, props): +def builderUpdateDbNames(props): builders = set() for f in props.files: if f.endswith('.ebuild'): @@ -12,22 +12,38 @@ def builderUpdateDbNames(self, props): return list(builders) @util.renderer -def cpvUpdateDb(props): -cpv_changes = [] -for f in props.files: -if f.endswith('.ebuild'): -cppv = f.split('.eb', 0) -cpv = cppv.split('/', 0) + '/' + cppv.split('/', 2) -if not cpv in cpv_changes: -cpv_changes.append(cpv) -return cpv_changes +def gitUpdateDb(props): +git_changes = [] +print(props.changes) +for k in props.changes: +change_data = {} +print(k) +change_data['cpvs'] = [] +for v in k['files']: +if v.endswith('.ebuild'): +c = v.split('/')[0] +pv = v.split('/')[2][:-7] +cpv = c + '/' + pv +print(cpv) +change_data['cpvs'].append(cpv) +if k['repository'].endswith('.git'): +for v in k['repository'].split('/'): +if v.endswith('.git'): +change_data['repository'] = v[:-4] +change_data['author'] = k['author'] +change_data['committer'] = k['committer'] +change_data['comments'] = k['comments'] +change_data['revision'] = k['revision'] +git_changes.append(change_data) +print(git_changes) +return git_changes def gentoo_schedulers(): scheduler_update_db = schedulers.SingleBranchScheduler( name='scheduler_update_db', treeStableTimer=60, properties = { -'cpv_changes' : cpvUpdateDb, +'git_changes' : gitUpdateDb, }, builderNames = builderUpdateDbNames, change_filter=util.ChangeFilter(branch='master'), @@ -60,7 +76,7 @@ def gentoo_schedulers(): builderNames=["run_build_request"]) s = [] s.append(test_updatedb) -#s.append(scheduler_update_db) +s.append(scheduler_update_db) s.append(update_cpv_data) s.append(update_v_data) s.append(build_request_data) diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py index 2221ba4..939059c 100644 --- a/buildbot_gentoo_ci/db/model.py +++ b/buildbot_gentoo_ci/db/model.py @@ -71,14 +71,15 @@ class Model(base.DBConnectorComponent): # description of the repository sa.Column('description', sa.Text, nullable=True), sa.Column('mirror_url', sa.String(255), nullable=True), +sa.Column('type', sa.Enum('gitpuller'), nullable=False, default='gitpuller'), sa.Column('auto', sa.Boolean, default=False), sa.Column('enabled', sa.Boolean, default=False), sa.Column('ebuild', sa.Boolean, default=False), ) # Use by GitPoller -repository_gitpuller = sautils.Table( -"repository_gitpuller", metadata, +repositorys_gitpullers = sautils.Table( +"repositorys_gitpullers", metadat
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: 22ad0178cca549026ab51f9f26e3fb9ad28f6374 Author: Magnus Granberg gentoo org> AuthorDate: Sat Feb 27 22:45:35 2021 + Commit: Magnus Granberg gentoo org> CommitDate: Sat Feb 27 22:45:35 2021 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=22ad0178 Add SetPackageDefault and SetEnvDefault to steps Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/buildfactorys.py | 6 +- buildbot_gentoo_ci/db/model.py | 27 - buildbot_gentoo_ci/db/portages.py | 14 + buildbot_gentoo_ci/db/projects.py | 44 ++ buildbot_gentoo_ci/steps/portage.py| 97 ++ 5 files changed, 185 insertions(+), 3 deletions(-) diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index ac8bac1..acafed8 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -9,6 +9,7 @@ from buildbot_gentoo_ci.steps import category from buildbot_gentoo_ci.steps import package from buildbot_gentoo_ci.steps import version from buildbot_gentoo_ci.steps import builders +from buildbot_gentoo_ci.steps import portage def update_db_check(): f = util.BuildFactory() @@ -114,9 +115,10 @@ def run_build_request(): f.addStep(builders.UpdateRepos()) # setup make.conf f.addStep(builders.SetMakeConf()) -# setup package.* -#f.addStep(portages.SetPackageUse()) # setup env +f.addStep(portage.SetEnvDefault()) +# setup package.* +#f.addStep(portage.SetPackageDefault()) # setup files in /etc if needed # run --regen if needed on repo # update packages before any tests diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py index 2153b79..b91f158 100644 --- a/buildbot_gentoo_ci/db/model.py +++ b/buildbot_gentoo_ci/db/model.py @@ -139,7 +139,6 @@ class Model(base.DBConnectorComponent): sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), -# FIXME: directorys should be moved to own table sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'), nullable=False), sa.Column('value', sa.String(255), nullable=False), ) @@ -162,6 +161,32 @@ class Model(base.DBConnectorComponent): sa.Column('value', sa.String(255), nullable=False), ) +# projects etc/portage/env settings +projects_portages_env = sautils.Table( +"projects_portages_env", metadata, +sa.Column('id', sa.Integer, primary_key=True), +sa.Column('project_uuid', sa.String(36), + sa.ForeignKey('projects.uuid', ondelete='CASCADE'), + nullable=False), +sa.Column('makeconf_id', sa.String(255), + sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'), + nullable=False), +sa.Column('name', sa.String(255), nullable=False), +sa.Column('value', sa.String(255), nullable=False), +) + +# projects etc/portage/package.* settings +projects_portage_package = sautils.Table( +"projects_portage_package", metadata, +sa.Column('id', sa.Integer, primary_key=True), +sa.Column('project_uuid', sa.String(36), + sa.ForeignKey('projects.uuid', ondelete='CASCADE'), + nullable=False), +sa.Column('directorys', sa.Enum('use', 'accept_keywords', 'env'), nullable=False), +sa.Column('value1', sa.String(255), nullable=False), +sa.Column('value2', sa.String(255), nullable=True), +) + projects_emerge_options = sautils.Table( "projects_emerge_options", metadata, sa.Column('id', sa.Integer, primary_key=True), diff --git a/buildbot_gentoo_ci/db/portages.py b/buildbot_gentoo_ci/db/portages.py index 428fb84..86321e3 100644 --- a/buildbot_gentoo_ci/db/portages.py +++ b/buildbot_gentoo_ci/db/portages.py @@ -36,6 +36,20 @@ class PortagesConnectorComponent(base.DBConnectorComponent): res = yield self.db.pool.do(thd) return res +@defer.inlineCallbacks +def getVariableById(self, id): +def thd(conn): +tbl = self.db.model.portages_makeconf +q = tbl.select() +q = q.where(tbl.c.id == id) +res = conn.execute(q) +row = res.fetchone() +if not row: +return None +return self._row2dict(conn, row) +res = yield self.db.pool.do(thd) +return res + def _row2dict(self, conn, row): return dict( id=row.id, diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py index 37d401c..37ae2f1 100644 --- a/buildbot_gentoo_ci/db/projects.py +++ b/buildbot_gentoo_ci/db/projects.py @@ -130,6 +130,28 @@ class ProjectsConnectorComponent(base.DBConnectorComponen
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, /, buildbot_gentoo_ci/config/
commit: d9dcf85c92b58ca427123ef2c38b8374ecdae293 Author: Magnus Granberg gentoo org> AuthorDate: Sun Jan 10 15:24:41 2021 + Commit: Magnus Granberg gentoo org> CommitDate: Sun Jan 10 15:24:41 2021 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d9dcf85c Add BuildFactory update_db_check Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/builders.py | 18 +++- buildbot_gentoo_ci/config/buildfactorys.py | 66 buildbot_gentoo_ci/config/config.py| 2 +- buildbot_gentoo_ci/config/schedulers.py| 9 +- buildbot_gentoo_ci/config/service.py | 6 +- buildbot_gentoo_ci/db/connector.py | 7 +- buildbot_gentoo_ci/db/model.py | 6 +- buildbot_gentoo_ci/db/projects.py | 69 buildbot_gentoo_ci/db/repositorys.py | 73 + buildbot_gentoo_ci/steps/__init__.py | 0 buildbot_gentoo_ci/steps/update_db.py | 162 + gentooci.cfg | 2 +- 12 files changed, 383 insertions(+), 37 deletions(-) diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py index 114bb83..c069841 100644 --- a/buildbot_gentoo_ci/config/builders.py +++ b/buildbot_gentoo_ci/config/builders.py @@ -1,4 +1,4 @@ -# Copyright 2020 Gentoo Authors +# Copyright 2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from buildbot.plugins import util @@ -7,9 +7,19 @@ from buildbot_gentoo_ci.config import buildfactorys def gentoo_builders(b=[]): # FIXME: get workers from db b.append(util.BuilderConfig( -name='update_db_packages', -workername=[ 'updatedb_1', ], -factory=buildfactorys.f_update_db_packages() +name='update_db_check', +workername='updatedb_1', +factory=buildfactorys.update_db_check() +) +) +# FIXME: get workers from db +# Use multiplay workers depend on Property(cpv) +# if cp do not match next one, use diffrent worker then +# or first cp have done its buildsteps. +b.append(util.BuilderConfig( +name='update_cpv_data', +workername='updatedb_1', +factory=buildfactorys.update_db_cp() ) ) return b diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index ef479a2..3fae376 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -1,28 +1,50 @@ -# Copyright 2020 Gentoo Authors +# Copyright 2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from buildbot.plugins import util, steps - -def f_update_db_packages(): +from buildbot.plugins import steps as buildbot_steps +from buildbot.plugins import util + +from buildbot_gentoo_ci.steps import update_db +#from buildbot_gentoo_ci.steps import category +#from buildbot_gentoo_ci.steps import package + +def update_db_check(): f = util.BuildFactory() -# FIXME: -# Get base project from db -# Check if base project repo for is cloned -# Check if gentoo repo is cloned +# FIXME: 1 +# Get base project data from db +# return project_repository, profile_repository, +# project +f.addStep(update_db.GetDataGentooCiProject()) +# Check if base project repo is cloned +# Check if profile repo is cloned # check if the profile link is there +# check if the repository is cloned +f.addStep(update_db.CheckPathGentooCiProject()) # check if etc/portage has no error -# Get the repository some way -repository=util.Property('repository') -# For loop -# for cpv in util.Property('cpv_changes') -# check if not categorys in db -# run update categorys db step -# check if not package in db -# run update package db step -# and get foo{cpv] with metadata back -# if new or updated package -# else -# run update package db step in a pool of workers -# and get foo{cpv] with metadata back -# if new or updated package +# return config_root +f.addStep(update_db.CheckProjectGentooCiProject()) +# Make a for loop and trigger new builders with cpv from cpv_changes +# return cpv, repository, project_data, config_root +f.addStep(update_db.CheckCPVGentooCiProject()) +return f + +def update_db_cp(): +f = util.BuildFactory() +# FIXME: 2 +# if categorys in db +# return category_data +# add check category path step at end +# else +# add category to db step +# return category_data +#f.addStep(category.CheckCGentooCiProject()) +# if package in db +# return package_data +# add check package path step at end +# else +# add package to db step +# return package_data +#f.addStep(package.CheckPGentooCiProj